Uglifyjs: fuzz script

[kzc]

@alexlamsl @qfox Put this script in the root directory of UglifyJS2, run it without arguments and watch the sparks fly.

// ufuzz.js
// derived from https://github.com/qfox/uglyfuzzer by Peter van der Zee

var vm = require("vm");
var uglify = require(".");

function run_code(code) {
    var stdout = "";
    var original_write = process.stdout.write;
    process.stdout.write = function(chunk) {
        stdout += chunk;
    };
    try {
        new vm.Script(code).runInNewContext({ console: console }, { timeout: 5000 });
        return stdout;
    } catch (ex) {
        return ex;
    } finally {
        process.stdout.write = original_write;
    }
}

function rng(max) {
  return Math.floor(max * Math.random());
}

function createFunctionDecls(n, recurmax) {
  if (--recurmax < 0) { return ';'; }
  var s = '';
  while (--n > 0) {
    s += createFunctionDecl(recurmax) + '\n';
  }
  return s;
}

var funcs = 0;
function createFunctionDecl(recurmax) {
  if (--recurmax < 0) { return ';'; }
  var func = funcs++;
  return 'function f' + func + '(){' + createStatements(3, recurmax) + '}\nf' + func + '();';
}

function createStatements(n, recurmax) {
  if (--recurmax < 0) { return ';'; }
  var s = '';
  while (--n > 0) {
    s += createStatement(recurmax);
  }
  return s;
}

var loops = 0;
function createStatement(recurmax) {
  var loop = ++loops;
  if (--recurmax < 0) { return ';'; }
  switch (rng(7)) {
    case 0:
      return '{' + createStatement(recurmax) + '}';
    case 1:
      return 'if (' + createExpression(recurmax) + ')' + createStatement(recurmax);
    case 2:
      return '{var brake' + loop + ' = 5; do {' + createStatement(recurmax) + '} while ((' + createExpression(recurmax) + ') && --brake' + loop + ' > 0);}';
    case 3:
      return '{var brake' + loop + ' = 5; while ((' + createExpression(recurmax) + ') && --brake' + loop + ' > 0)' + createStatement(recurmax) + '}';
    case 4:
      return 'for (var brake' + loop + ' = 5; (' + createExpression(recurmax) + ') && brake' + loop + ' > 0; --brake' + loop + ')' + createStatement(recurmax);
    case 5:
      return ';';
    case 6:
      return createExpression() + ';';
  }
}

function createExpression(recurmax) {
  if (--recurmax < 0) { return '0'; }
  switch (rng(8)) {
    case 0:
      return '(' + createUnaryOp() + 'a)';
    case 1:
      return '(a' + (Math.random() > 0.5 ? '++' : '--') + ')';
    case 2:
      return '(b ' + createAssignment() + ' a)';
    case 3:
      return '(' + Math.random() + ' > 0.5 ? a : b)';
    case 4:
      return createExpression(recurmax) + createBinaryOp() + createExpression(recurmax);
    case 5:
      return createValue();
    case 6:
      return '(' + createExpression(recurmax) + ')';
    case 7:
      return createExpression(recurmax) + '?(' + createExpression(recurmax) + '):(' + createExpression(recurmax) + ')';
  }
}

function createValue() {
  var values = [
    'true',
    'false',
    '22',
    '0',
    '(-1)',
    'NaN',
    'undefined',
    'null',
    '"foo"',
    '"bar"' ];
  return values[rng(values.length)];
}

function createBinaryOp() {
  switch (rng(6)) {
    case 0:
      return '+';
    case 1:
      return '-';
    case 2:
      return ',';
    case 3:
      return '&&';
    case 4:
      return '||';
    case 5:
      return '^';
  }
}

function createAssignment() {
  switch (rng(4)) {
    case 0:
      return '=';
    case 1:
      return '-=';
    case 2:
      return '^=';
    case 3:
      return '+=';
  }
}

function createUnaryOp() {
  switch (rng(4)) {
    case 0:
      return '--';
    case 1:
      return '++';
    case 2:
      return '~';
    case 3:
      return '!';
  }
}

for (;;) {
    var original_code = "var a = 100, b = 10;\n" +
        (createFunctionDecls(rng(3) + 1, 10)) + "\nconsole.log(a, b);\n";

    console.log("//=============================================================");
    console.log("// original code");
    console.log("//");
    console.log(original_code + '\n');

    var code = uglify.minify(original_code, {
        fromString: true,
        mangle: false,
        compress: false,
        output: {
            beautify: true,
            bracketize: true,
        },
    }).code;
    console.log(code + '\n');

    console.log("//-------------------------------------------------------------");
    console.log("// original code (beautify'd)");
    console.log("//");
    console.log(code + '\n');

    console.log("//-------------------------------------------------------------");
    console.log("// uglified code");
    console.log("//");
    var uglify_code = uglify.minify(code, {
        fromString: true,
        mangle: false,
        compress: {
            passes: 3,
        },
        output: {
            beautify: true,
            bracketize: true,
        },
    }).code;
    console.log(uglify_code + '\n');

    var original_result = run_code(code);
    console.log("original result:");
    console.log(original_result);

    var uglify_result = run_code(uglify_code);
    console.log("uglified result:");
    console.log(uglify_result);

    if (original_result != uglify_result) { break; }
}

[kzc]

Script above was updated to output the unmodified original code.

[kzc]

@qfox Thanks again - fuzzing will keep us busy for quite a while.

[pvdz]

You're welcome. I think this is only the tip of the iceberg. But don't let that get you down! You're probably aware of many severe security bugs firefox found when they first started fuzzing. It only get's better :)

One problem, as you probably discovered by now, is that once you hit a certain bug, you'll probably keep hitting that bug over and over again. So reducing test cases gets annoying because you end up with the same problem, just slightly different variation (at least you'll get multiple test cases out of it).

The way the fuzzer evolves is by adding probabilistic weights to certain structures. Like the statement header only getting one expression, and only sometimes two or more. And by adding more complex structures like anonymous functions that are either immediately, or later, called. Clearly those are more involved, but as the fuzzer stabilizes in terms of finding bugs (or lack thereof) more complex paths can be traversed.

[kzc]

@alexlamsl The fuzz script will have to be adapted - with your latest fixes in master I've been unable to hit a breaking case in 3 minutes. Used to take no more than 10 seconds.

Definitely time for a new release.

[kzc]

The script could be adapted to vary the minify() options.

mangle, -c toplevel, -m toplevel, -c pure_getters, and the various unsafe optimizations would be good candidates.

[kzc]

I've run the fuzz script above against latest master for 30 minutes without a breaking case. That's promising.

[alexlamsl]

I've run the fuzz script above against latest master for 30 minutes without a breaking case.

Glad to know - and thanks for taking the trouble.

I tried it earlier on today and was struggling to decipher the problematic code snippet it gave out, so I really appreciate your effort in sorting them out for me 😉

[kzc]

Had a few consoles running the script collectively over an hour. That well has run dry. The script will need to be upgraded to catch new bugs.

Time for a new release!

[kzc]

@alexlamsl After the next npm publish could you please npm deprecate all uglify-js versions from 2.8.0 to 2.8.15? A few of those serious bugs can't be that uncommon.

webpack locked down [email protected] in their yarn.lock file last time I looked. Would be nice if webpack yarn users got a warning upon install.

[alexlamsl]

Sounds good - a message to urge them to upgrade to 2.8.16+ would be nice. (We haven't introduced any features that warrant a minor version bump yet, IIRC)

[kzc]

If you want to wrestle with their issue template - be my guest. :-)

[alexlamsl]

If you want to wrestle with their issue template - be my guest. :-)

Oh? 😮

Edit: are you referring to my "message" in the comment above? I meant the deprecation message in npm.

[kzc]

I just meant webpack's Issue template is as complicated to fill out as our Issue template. (Actually I borrowed a few ideas from them in that regard.)

[alexlamsl]

I see - no I wasn't planning to dabble into other projects' issue trackers, especially ones which I don't personally use.

One can leave a message with npm deprecate and I thought it'd be helpful to mention which version we prefer our users to fetch instead.

[kzc]

I meant the deprecation message in npm.

I see. That's fine.

Side note: npm install webpack will not have an issue since they don't appear to have an npm shrinkwrap file for the uglify-js version. It's only their yarn.lock file where it's an issue.

npm deprecate is sufficient for [email protected] to [email protected] users.

[kzc]

One can leave a message with npm deprecate and I thought it'd be helpful to mention which version we prefer our users to fetch instead.

No need to make a message. They can assume the latest version is good. It's just that fuzzing revealed some significant regressions in the 2.8.x series.

[pvdz]

yay

[alexlamsl]

@qfox looking forward to your beefing up of the fuzzer 😉