Emscripten: Big size discrepancy between fastcomp/upstream for WASM=0

[JCash]

First of all, thanks to everyone involved for this great tool!
And it's super nice getting the LLVM backend support 👍

When upgrading from 1.38.12 to 1.39.16, I noticed a big size increase on the JS target.
As suggested by @kripken, the difference is noticable between upstream/fastcomp, and upon his request, I've attached two builds using the --profiling-funcs

These are using 1.39.16-fastcomp and 1.39.16 respectively:

5854333 dmengine_fastcomp.js
7741568 dmengine_upstream.js

which is a ~ 32% increase. (Wasm was ~ -12% decrease, which is very nice).

They are both compiled with these settings:

        opt_level = '3'
        wasm_enabled = 0
        legacy_vm_support = 1

        emflags = ['WASM=%d' % wasm_enabled, 'LEGACY_VM_SUPPORT=%d' % legacy_vm_support, 'DISABLE_EXCEPTION_CATCHING=1', 'AGGRESSIVE_VARIABLE_ELIMINATION=1', 'PRECISE_F32=2',
                   'EXTRA_EXPORTED_RUNTIME_METHODS=["stringToUTF8","ccall","stackTrace","UTF8ToString","callMain"]', 'EXPORTED_FUNCTIONS=["_main"]',
                   'ERROR_ON_UNDEFINED_SYMBOLS=1', 'TOTAL_MEMORY=268435456']
        emflags = zip(['-s'] * len(emflags), emflags)
        emflags =[j for i in emflags for j in i]

        extra = ['--profiling-funcs']  # as requested by @kripken

        for f in ['CCFLAGS', 'CXXFLAGS']:
            self.env.append_value(f, ['-O%s' % opt_level, '-Wall', '-fPIC', '-fno-exceptions', '-fno-rtti',
                                        '-DGL_ES_VERSION_2_0', '-DGOOGLE_PROTOBUF_NO_RTTI', '-D__STDC_LIMIT_MACROS', '-DDDF_EXPOSE_DESCRIPTORS'])
            self.env.append_value(f, emflags)
            self.env.append_value(f, extra)

        self.env.append_value('LINKFLAGS', ['-O%s' % opt_level, '-Wno-warn-absolute-paths', '--emit-symbol-map', '--memory-init-file', '0', '-lidbfs.js'])
        self.env.append_value('LINKFLAGS', emflags)
        self.env.append_value('LINKFLAGS', extra)

upstream-size-issue.tar.gz

I'm not sure if it qualifies as an issue, but I thought it'd be best to upload it here, as opposed to some file server. And it serves better as a discussion forum, as opposed to twitter.

[kripken]

Thanks @JCash ! Turns out we didn't minify fully with --emit-symbol-map, fix is in #11279

@juj if you use that flag, the fix might help you too.

[JCash]

Thank you for fixing this!

[JCash]

HI again @kripken !

Now that I tried the 1.39.17 version I get this error with --emit-symbol-map.
1.39.16 didn't produce this error, nor does it when I omit the emit flag.

[23/23] cxx_link: build/default/src/__exported_symbols_6_6.o build/default/src/main_6.o -> build/default/src/dmengine_release.js
wasm2js: warning: global scope may be colliding with other scope: dmScript__GetURL_28lua_State__2c_20dmMessage__URL__29
Stack: Error
    at assertTrue (eval at globalEval (/Users/mawe/work/defold/tmp/dynamo_home/ext/SDKs/emsdk-1.39.17/upstream/emscripten/tools/js-optimizer.js:114:8), <anonymous>:65:26)
    at /Users/mawe/work/defold/tmp/dynamo_home/ext/SDKs/emsdk-1.39.17/upstream/emscripten/tools/js-optimizer.js:4677:11
    at traverse (/Users/mawe/work/defold/tmp/dynamo_home/ext/SDKs/emsdk-1.39.17/upstream/emscripten/tools/js-optimizer.js:247:18)
    at traverseChildren (/Users/mawe/work/defold/tmp/dynamo_home/ext/SDKs/emsdk-1.39.17/upstream/emscripten/tools/js-optimizer.js:224:25)
    at traverse (/Users/mawe/work/defold/tmp/dynamo_home/ext/SDKs/emsdk-1.39.17/upstream/emscripten/tools/js-optimizer.js:252:9)
    at traverseChildren (/Users/mawe/work/defold/tmp/dynamo_home/ext/SDKs/emsdk-1.39.17/upstream/emscripten/tools/js-optimizer.js:224:25)
    at traverse (/Users/mawe/work/defold/tmp/dynamo_home/ext/SDKs/emsdk-1.39.17/upstream/emscripten/tools/js-optimizer.js:252:9)
    at traverseChildren (/Users/mawe/work/defold/tmp/dynamo_home/ext/SDKs/emsdk-1.39.17/upstream/emscripten/tools/js-optimizer.js:224:25)
    at traverse (/Users/mawe/work/defold/tmp/dynamo_home/ext/SDKs/emsdk-1.39.17/upstream/emscripten/tools/js-optimizer.js:252:9)
    at traverseChildren (/Users/mawe/work/defold/tmp/dynamo_home/ext/SDKs/emsdk-1.39.17/upstream/emscripten/tools/js-optimizer.js:224:25)

undefined:66
    throw msg;
    ^
Assertion failed: cannot call a local
Stack: Error
    at assertTrue (eval at globalEval (/Users/mawe/work/defold/tmp/dynamo_home/ext/SDKs/emsdk-1.39.17/upstream/emscripten/tools/js-optimizer.js:114:8), <anonymous>:65:26)
    at /Users/mawe/work/defold/tmp/dynamo_home/ext/SDKs/emsdk-1.39.17/upstream/emscripten/tools/js-optimizer.js:4677:11
    at traverse (/Users/mawe/work/defold/tmp/dynamo_home/ext/SDKs/emsdk-1.39.17/upstream/emscripten/tools/js-optimizer.js:247:18)
    at traverseChildren (/Users/mawe/work/defold/tmp/dynamo_home/ext/SDKs/emsdk-1.39.17/upstream/emscripten/tools/js-optimizer.js:224:25)
    at traverse (/Users/mawe/work/defold/tmp/dynamo_home/ext/SDKs/emsdk-1.39.17/upstream/emscripten/tools/js-optimizer.js:252:9)
    at traverseChildren (/Users/mawe/work/defold/tmp/dynamo_home/ext/SDKs/emsdk-1.39.17/upstream/emscripten/tools/js-optimizer.js:224:25)
    at traverse (/Users/mawe/work/defold/tmp/dynamo_home/ext/SDKs/emsdk-1.39.17/upstream/emscripten/tools/js-optimizer.js:252:9)
    at traverseChildren (/Users/mawe/work/defold/tmp/dynamo_home/ext/SDKs/emsdk-1.39.17/upstream/emscripten/tools/js-optimizer.js:224:25)
    at traverse (/Users/mawe/work/defold/tmp/dynamo_home/ext/SDKs/emsdk-1.39.17/upstream/emscripten/tools/js-optimizer.js:252:9)
    at traverseChildren (/Users/mawe/work/defold/tmp/dynamo_home/ext/SDKs/emsdk-1.39.17/upstream/emscripten/tools/js-optimizer.js:224:25)

undefined:66
    throw msg;
    ^
Assertion failed: cannot call a local
em++: error: '/Users/mawe/work/defold/tmp/dynamo_home/ext/SDKs/emsdk-1.39.17/node/12.9.1_64bit/bin/node /Users/mawe/work/defold/tmp/dynamo_home/ext/SDKs/emsdk-1.39.17/upstream/emscripten/tools/js-optimizer.js /var/folders/j9/fxdfn0n96jl8tyn857pcdm8r0000gn/T/tmpio4g9y1j.jsfunc_5.js noPrintMetadata minifyLocals minifyWhitespace last' failed (1)

Is there anything else I need to do?

[kripken]

That could be an unknown bug if it hits that assertion, there are no reports that sound similar.

Can you provide a full testcase here? The files for the link step + link command, reduced as much as you can, would be great.

[JCash]

Hi again!

It took a while to condense to a very minimal example, but here is a quite small example that triggers it:

#include <stdarg.h>
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>

typedef struct lua_State {
    int top;
} lua_State;

typedef struct Token {
    int token;
} Token;

typedef struct LexState {
  Token t;
  struct lua_State* L;
} LexState;

typedef struct expdesc {
    int foo;
} expdesc;

const char *luaO_pushvfstring (lua_State *L, const char *fmt, va_list argp) {
    L->top -= strlen(fmt);
    return fmt;
}

static void check (LexState *ls, int c) {
  if (ls->t.token != c)
    luaO_pushvfstring(ls->L, "char(%d)", (void *)(uintptr_t)c);
}

static void checknext (LexState *ls, int c) {
  check(ls, c);
}

static void constructor (LexState *ls, expdesc *t) {
  checknext(ls, '{');
  do {
    switch(ls->t.token) {
      case '[': {
        break;
      }
      default: {
        break;
      }
    }
  } while (1);
}

static void primaryexp (LexState *ls, expdesc *v) {
  constructor(ls, v);
}

static void* luaY_parser_x (lua_State *L, const char *name) {
    struct LexState lexstate;
    lexstate.L = L;

    expdesc v;
    primaryexp(&lexstate, &v);
    return 0;
}

int main(int argc, char* argv[])
{
    lua_State L;
    luaY_parser_x(&L, argv[0]);
    return 0;
}

Compile with:

GCC=.../emsdk-1.39.17/upstream/emscripten/emcc

#OPT=-O0 # O0 and O1 works
OPT=-O3  # O2,O3,Os doesn't work

echo "Compiling issue_minify.o"
$GCC -O3 -Wall -fPIC -fno-exceptions -fno-rtti -s WASM=0 ./issue_minify.c -c -o issue_minify.o

echo "Linking issue_minify.js"
$GCC issue_minify.o -o issue_minify.js -O3 --emit-symbol-map -s WASM=0

[kripken]

Thanks @JCash ! Fix in #11417