Improve fuzzer. :) (#1665)

@qfox	Put value constants in a global constant			74c0fb9
 @qfox	And the other string based values as well			a5033c5
 @qfox	Be more strict about parameters, allow max to be optional			9c7ce70
 @qfox	Support a `V` (capital) flag to only log out at intervals			2d822c7
 @qfox	Fewer magic variables			a6a9a7c
 @qfox	Fix decrement such that a function is created when n=1			7e4b017
 @qfox	Add more values			64e596e
 @qfox	Make `b` appear more often			d33191a
 @qfox	Add functions that contain (only..) functions			29a86e3
 @qfox	Allow the block statement to contain multiple statements			7570484
 @qfox	Make the interval count a constant			d587ad8
 @qfox	Enable mangling, disable post-processing  …			4dc8d35
 @qfox	Add more simple value that may trigger syntactic errors			8496d58
 @qfox	Add `else` to some `if` statements			a4aed65
 @qfox	Move iife to expr generator, fix missing recursion arg			e453159
 @qfox	Improve output on error where it wasnt printing the last code properly			4565a1a
 @qfox	Add switch statement to generator			ceafa76
 @qfox	Add var statement, support optional comma for expr generator			b83921b
 @qfox	Expression generator should use a simple value instead of `0` as recu…  …			9d1a5c7
 @qfox	const -> var to keep things es5...			0143099
 @qfox	Add more simple values that may trigger edge cases			5e124f1
 @qfox	Add central name generator, take special care for global functions			aeb7682
 @qfox	Add some `return` and function declaration cases to statement generator			6c9c3cc
 @qfox	Exclude switches from generator for now			91124b2

Put value constants in a global constant

And the other string based values as well

Be more strict about parameters, allow max to be optional

Support a `V` (capital) flag to only log out at intervals

Fewer magic variables

Fix decrement such that a function is created when n=1

Add more values

Make `b` appear more often

Add functions that contain (only..) functions

Allow the block statement to contain multiple statements

Make the interval count a constant

Enable mangling, disable post-processing

Mangling is kind of the whole point...

Similarly, to beautify the minified code afterwards may supress bugs so it's probably best not to beautify the code prematurely. And there's no point anyways since you won't see it most of the time and only care about the main input anyways.

Add more simple value that may trigger syntactic errors

Add `else` to some `if` statements

Move iife to expr generator, fix missing recursion arg

Improve output on error where it wasnt printing the last code properly

Add switch statement to generator

Add var statement, support optional comma for expr generator

Expression generator should use a simple value instead of `0` as recursion default

const -> var to keep things es5...

Add more simple values that may trigger edge cases

Add central name generator, take special care for global functions

Add some `return` and function declaration cases to statement generator

Exclude switches from generator for now

Enable switch generation because #1667 was merged

Add typeof generator

Add some elision tests

Add a new edge case that returns an object explicitly

Add all binary ops to try and cover more paths

Forgot four binops and added `Math` to var name pool

Harden the incremental pre/postfix tests

Improve switch generator, allow `default` to appear at any clause index

Add try/catch/finally generation

Prevent function statements being generated

Add edge case with decremental op and a group

Disable switch generation until #1679 and #1680 are solved

Only allow `default` clause as last clause for now

Tentatively enable `throw`, `break` and `continue` statements when in valid contexts

1 files changed, 324 insertions, 94 deletions

diff --git a/test/ufuzz.js b/test/ufuzz.js
index ac2ded7c..c56c6224 100644
--- a/test/ufuzz.js
+++ b/test/ufuzz.js
@@ -11,6 +11,116 @@
 var vm = require("vm");
 var minify = require("..").minify;
 
+var MAX_GENERATED_FUNCTIONS_PER_RUN = 1;
+var MAX_GENERATION_RECURSION_DEPTH = 15;
+var INTERVAL_COUNT = 100;
+
+var VALUES = [
+  'true',
+  'false',
+  '22',
+  '0',
+  '-0', // 0/-0 !== 0
+  '23..toString()',
+  '24 .toString()',
+  '25. ',
+  '0x26.toString()',
+  '(-1)',
+  'NaN',
+  'undefined',
+  'Infinity',
+  'null',
+  '[]',
+  '[,0][1]', // an array with elisions... but this is always false
+  '([,0].length === 2)', // an array with elisions... this is always true
+  '({})', // wrapped the object causes too many syntax errors in statements
+  '"foo"',
+  '"bar"' ];
+
+var BINARY_OPS_NO_COMMA = [
+  ' + ', // spaces needed to disambiguate with ++ cases (could otherwise cause syntax errors)
+  ' - ',
+  '/',
+  '*',
+  '&',
+  '|',
+  '^',
+  '<<',
+  '>>',
+  '>>>',
+  '%',
+  '&&',
+  '||',
+  '^' ];
+
+var BINARY_OPS = [','].concat(BINARY_OPS_NO_COMMA);
+
+var ASSIGNMENTS = [
+  '=',
+  '=',
+  '=',
+  '=',
+  '=',
+  '=',
+
+  '==',
+  '!=',
+  '===',
+  '!==',
+  '+=',
+  '-=',
+  '*=',
+  '/=',
+  '&=',
+  '|=',
+  '^=',
+  '<<=',
+  '>>=',
+  '>>>=',
+  '%=' ];
+
+var UNARY_OPS = [
+  '--',
+  '++',
+  '~',
+  '!',
+  'void ',
+  'delete ', // should be safe, even `delete foo` and `delete f()` shouldn't crash
+  ' - ',
+  ' + ' ];
+
+var NO_COMMA = true;
+var MAYBE = true;
+var NESTED = true;
+var CAN_THROW = true;
+var CANNOT_THROW = false;
+var CAN_BREAK = true;
+var CAN_CONTINUE = true;
+
+var VAR_NAMES = [
+  'foo',
+  'bar',
+  'a',
+  'b',
+  'undefined', // fun!
+  'eval', // mmmm, ok, also fun!
+  'NaN', // mmmm, ok, also fun!
+  'Infinity', // the fun never ends!
+  'arguments', // this one is just creepy
+  'Math', // since Math is assumed to be a non-constructor/function it may trip certain cases
+  'let' ]; // maybe omit this, it's more a parser problem than minifier
+
+var TYPEOF_OUTCOMES = [
+  'undefined',
+  'string',
+  'number',
+  'object',
+  'boolean',
+  'special',
+  'unknown',
+  'symbol',
+  'crap' ];
+
 function run_code(code) {
     var stdout = "";
     var original_write = process.stdout.write;
@@ -31,135 +141,241 @@ function rng(max) {
   return Math.floor(max * Math.random());
 }
 
-function createFunctionDecls(n, recurmax) {
+function createFunctionDecls(n, recurmax, nested) {
   if (--recurmax < 0) { return ';'; }
   var s = '';
-  while (--n > 0) {
-    s += createFunctionDecl(recurmax) + '\n';
+  while (n-- > 0) {
+    s += createFunctionDecl(recurmax, nested) + '\n';
   }
   return s;
 }
 
 var funcs = 0;
-function createFunctionDecl(recurmax) {
+function createFunctionDecl(recurmax, nested) {
   if (--recurmax < 0) { return ';'; }
   var func = funcs++;
-  return 'function f' + func + '(){' + createStatements(3, recurmax) + '}\nf' + func + '();';
+  var name = rng(5) > 0 ? 'f' + func : createVarName();
+  if (name === 'a' || name === 'b') name = 'f' + func; // quick hack to prevent assignment to func names of being called
+  if (!nested && name === 'undefined' || name === 'NaN' || name === 'Infinity') name = 'f' + func; // cant redefine these in global space
+  var s = '';
+  if (rng(5) === 1) {
+    // functions with functions. lower the recursion to prevent a mess.
+    s = 'function ' + name + '(){' + createFunctionDecls(rng(5) + 1, Math.ceil(recurmax / 2), NESTED) + '}\n';
+  } else {
+    // functions with statements
+    s = 'function ' + name + '(){' + createStatements(3, recurmax) + '}\n';
+  }
+
+  if (nested) s = '!' + nested; // avoid "function statements" (decl inside statements)
+  else s += name + '();'
+
+  return s;
 }
 
-function createStatements(n, recurmax) {
+function createStatements(n, recurmax, canThrow, canBreak, canContinue) {
   if (--recurmax < 0) { return ';'; }
   var s = '';
   while (--n > 0) {
-    s += createStatement(recurmax);
+    s += createStatement(recurmax, canThrow, canBreak, canContinue);
   }
   return s;
 }
 
 var loops = 0;
-function createStatement(recurmax) {
+function createStatement(recurmax, canThrow, canBreak, canContinue) {
   var loop = ++loops;
   if (--recurmax < 0) { return ';'; }
-  switch (rng(7)) {
+  switch (rng(16)) {
     case 0:
-      return '{' + createStatement(recurmax) + '}';
+      return '{' + createStatements(rng(5) + 1, recurmax, canThrow, canBreak, canContinue) + '}';
     case 1:
-      return 'if (' + createExpression(recurmax) + ')' + createStatement(recurmax);
+      return 'if (' + createExpression(recurmax) + ')' + createStatement(recurmax, canThrow, canBreak, canContinue) + (rng(2) === 1 ? ' else ' + createStatement(recurmax, canThrow, canBreak, canContinue) : '');
     case 2:
-      return '{var brake' + loop + ' = 5; do {' + createStatement(recurmax) + '} while ((' + createExpression(recurmax) + ') && --brake' + loop + ' > 0);}';
+      return '{var brake' + loop + ' = 5; do {' + createStatement(recurmax, canThrow, CAN_BREAK, CAN_CONTINUE) + '} while ((' + createExpression(recurmax) + ') && --brake' + loop + ' > 0);}';
     case 3:
-      return '{var brake' + loop + ' = 5; while ((' + createExpression(recurmax) + ') && --brake' + loop + ' > 0)' + createStatement(recurmax) + '}';
+      return '{var brake' + loop + ' = 5; while ((' + createExpression(recurmax) + ') && --brake' + loop + ' > 0)' + createStatement(recurmax, canThrow, CAN_BREAK, CAN_CONTINUE) + '}';
     case 4:
-      return 'for (var brake' + loop + ' = 5; (' + createExpression(recurmax) + ') && brake' + loop + ' > 0; --brake' + loop + ')' + createStatement(recurmax);
+      return 'for (var brake' + loop + ' = 5; (' + createExpression(recurmax) + ') && brake' + loop + ' > 0; --brake' + loop + ')' + createStatement(recurmax, canThrow, CAN_BREAK, CAN_CONTINUE);
     case 5:
       return ';';
     case 6:
-      return createExpression() + ';';
+      return createExpression(recurmax) + ';';
+    case 7:
+      return ';'; // TODO: disabled until some switch issues are resolved
+      // note: case args are actual expressions
+      // note: default does not _need_ to be last
+      return 'switch (' + createExpression(recurmax) + ') { ' + createSwitchParts(recurmax, 4) + '}';
+    case 8:
+      return 'var ' + createVarName() + ';';
+    case 9:
+      // initializer can only have one expression
+      return 'var ' + createVarName() + ' = ' + createExpression(recurmax, NO_COMMA) + ';';
+    case 10:
+      // initializer can only have one expression
+      return 'var ' + createVarName() + ' = ' + createExpression(recurmax, NO_COMMA) + ', ' + createVarName() + ' = ' + createExpression(recurmax, NO_COMMA) + ';';
+    case 11:
+      if (canBreak && rng(5) === 0) return 'break;';
+      if (canContinue && rng(5) === 0) return 'continue;';
+      return 'return;';
+    case 12:
+      // must wrap in curlies to prevent orphaned `else` statement
+      if (canThrow && rng(5) === 0) return '{ throw ' + createExpression(recurmax) + '}';
+      return '{ return ' + createExpression(recurmax) + '}';
+    case 13:
+      // this is actually more like a parser test, but perhaps it hits some dead code elimination traps
+      // must wrap in curlies to prevent orphaned `else` statement
+      if (canThrow && rng(5) === 0) return '{ throw\n' + createExpression(recurmax) + '}';
+      return '{ return\n' + createExpression(recurmax) + '}';
+    case 14:
+      // "In non-strict mode code, functions can only be declared at top level, inside a block, or ..."
+      // (dont both with func decls in `if`; it's only a parser thing because you cant call them without a block)
+      return '{' + createFunctionDecl(recurmax, NESTED) + '}';
+    case 15:
+      return ';';
+      // catch var could cause some problems
+      // note: the "blocks" are syntactically mandatory for try/catch/finally
+      var s = 'try {' + createStatement(recurmax, CAN_THROW, canBreak, canContinue) + ' }';
+      var n = rng(3); // 0=only catch, 1=only finally, 2=catch+finally
+      if (n !== 1) s += ' catch (' + createVarName() + ') { ' + createStatements(3, recurmax, canBreak, canContinue) + ' }';
+      if (n !== 0) s += ' finally { ' + createStatements(3, recurmax, canBreak, canContinue) + ' }';
+      return s;
   }
 }
 
-function createExpression(recurmax) {
-  if (--recurmax < 0) { return '0'; }
-  switch (rng(8)) {
+function createSwitchParts(recurmax, n) {
+  var hadDefault = false;
+  var s = '';
+  while (n-- > 0) {
+    hadDefault = n > 0;
+    if (hadDefault || rng(4) > 0) {
+      s += '' +
+        'case ' + createExpression(recurmax) + ':\n' +
+            createStatements(rng(3) + 1, recurmax, CANNOT_THROW, CAN_BREAK) +
+            '\n' +
+            (rng(10) > 0 ? ' break;' : '/* fall-through */') +
+        '\n';
+    } else {
+      hadDefault = true;
+      s += '' +
+        'default:\n' +
+            createStatements(rng(3) + 1, recurmax, CANNOT_THROW, CAN_BREAK) +
+        '\n';
+    }
+  }
+  return s;
+}
+
+function createExpression(recurmax, noComma) {
+  if (--recurmax < 0) {
+    return createValue(); // note: should return a simple non-recursing expression value!
+  }
+  switch (rng(12)) {
     case 0:
-      return '(' + createUnaryOp() + 'a)';
+      return '(' + createUnaryOp() + (rng(2) === 1 ? 'a' : 'b') + ')';
     case 1:
-      return '(a' + (Math.random() > 0.5 ? '++' : '--') + ')';
+      return '(a' + (rng(2) == 1 ? '++' : '--') + ')';
     case 2:
       return '(b ' + createAssignment() + ' a)';
     case 3:
-      return '(' + Math.random() + ' > 0.5 ? a : b)';
+      return '(' + rng(2) + ' === 1 ? a : b)';
     case 4:
-      return createExpression(recurmax) + createBinaryOp() + createExpression(recurmax);
+      return createExpression(recurmax, noComma) + createBinaryOp(noComma) + createExpression(recurmax, noComma);
     case 5:
       return createValue();
     case 6:
       return '(' + createExpression(recurmax) + ')';
     case 7:
-      return createExpression(recurmax) + '?(' + createExpression(recurmax) + '):(' + createExpression(recurmax) + ')';
+      return createExpression(recurmax, noComma) + '?(' + createExpression(recurmax) + '):(' + createExpression(recurmax) + ')';
+    case 8:
+      switch(rng(4)) {
+        case 0:
+          return '(function ' + createVarName(MAYBE) + '(){' + createStatements(rng(5) + 1, recurmax) + '})()';
+        case 1:
+          return '+function ' + createVarName(MAYBE) + '(){' + createStatements(rng(5) + 1, recurmax) + '}';
+        case 2:
+          return '!function ' + createVarName(MAYBE) + '(){' + createStatements(rng(5) + 1, recurmax) + '}';
+        case 3:
+          return 'void function ' + createVarName(MAYBE) + '(){' + createStatements(rng(5) + 1, recurmax) + '}';
+        default:
+          return 'void function ' + createVarName(MAYBE) + '(){' + createStatements(rng(5) + 1, recurmax) + '}';
+      }
+    case 9:
+      return createTypeofExpr(recurmax);
+    case 10:
+      // you could statically infer that this is just `Math`, regardless of the other expression
+      // I don't think Uglify does this at this time...
+      return ''+
+        'new function(){ \n' +
+        (rng(2) === 1 ? createExpression(recurmax) + '\n' : '') +
+        'return Math;\n' +
+      '}';
+    case 11:
+      // more like a parser test but perhaps comment nodes mess up the analysis?
+      switch (rng(5)) {
+        case 0:
+          return '(a/* ignore */++)';
+        case 1:
+          return '(b/* ignore */--)';
+        case 2:
+          return '(++/* ignore */a)';
+        case 3:
+          return '(--/* ignore */b)';
+        case 4:
+          // only groups that wrap a single variable return a "Reference", so this is still valid.
+          // may just be a parser edge case that is invisible to uglify...
+          return '(--(b))';
+        default:
+          return '(--/* ignore */b)';
+      }
   }
 }
 
-function createValue() {
-  var values = [
-    'true',
-    'false',
-    '22',
-    '0',
-    '(-1)',
-    'NaN',
-    'undefined',
-    'null',
-    '"foo"',
-    '"bar"' ];
-  return values[rng(values.length)];
-}
-
-function createBinaryOp() {
-  switch (rng(6)) {
+function createTypeofExpr(recurmax) {
+  if (--recurmax < 0) {
+    return 'typeof undefined === "undefined"';
+  }
+
+  switch (rng(5)) {
     case 0:
-      return '+';
+      return '(typeof ' + createVarName() + ' === "' + TYPEOF_OUTCOMES[rng(TYPEOF_OUTCOMES.length)] + '")';
     case 1:
-      return '-';
+      return '(typeof ' + createVarName() + ' !== "' + TYPEOF_OUTCOMES[rng(TYPEOF_OUTCOMES.length)] + '")';
     case 2:
-      return ',';
+      return '(typeof ' + createVarName() + ' == "' + TYPEOF_OUTCOMES[rng(TYPEOF_OUTCOMES.length)] + '")';
     case 3:
-      return '&&';
+      return '(typeof ' + createVarName() + ' != "' + TYPEOF_OUTCOMES[rng(TYPEOF_OUTCOMES.length)] + '")';
     case 4:
-      return '||';
-    case 5:
-      return '^';
+      return '(typeof ' + createVarName() + ')';
   }
 }
 
+function createValue() {
+  return VALUES[rng(VALUES.length)];
+}
+
+function createBinaryOp(noComma) {
+  if (noComma) return BINARY_OPS_NO_COMMA[rng(BINARY_OPS_NO_COMMA.length)];
+  return BINARY_OPS[rng(BINARY_OPS.length)];
+}
+
 function createAssignment() {
-  switch (rng(4)) {
-    case 0:
-      return '=';
-    case 1:
-      return '-=';
-    case 2:
-      return '^=';
-    case 3:
-      return '+=';
-  }
+  return ASSIGNMENTS[rng(ASSIGNMENTS.length)];
 }
 
 function createUnaryOp() {
-  switch (rng(4)) {
-    case 0:
-      return '--';
-    case 1:
-      return '++';
-    case 2:
-      return '~';
-    case 3:
-      return '!';
+  return UNARY_OPS[rng(UNARY_OPS.length)];
+}
+
+function createVarName(maybe) {
+  if (!maybe || rng(2) === 1) {
+    return VAR_NAMES[rng(VAR_NAMES.length)] + (rng(5) > 0 ? ++loops : '');
   }
+  return '';
 }
 
-function log() {
+function log(ok) {
     console.log("//=============================================================");
+    if (!ok) console.log("// !!!!!! Failed...");
     console.log("// original code");
     console.log("//");
     console.log(original_code);
@@ -183,43 +399,57 @@ function log() {
     console.log(beautify_result);
     console.log("uglified result:");
     console.log(uglify_result);
+    if (!ok) console.log("!!!!!! Failed...");
 }
 
 var num_iterations = +process.argv[2] || 1/0;
-var verbose = !!process.argv[3];
+var verbose = process.argv[3] === 'v' || process.argv[2] === 'v';
+var verbose_interval = process.argv[3] === 'V' || process.argv[2] === 'V';
 for (var round = 0; round < num_iterations; round++) {
+    var parse_error = false;
     process.stdout.write(round + " of " + num_iterations + "\r");
     var original_code = [
         "var a = 100, b = 10;",
-        createFunctionDecls(rng(3) + 1, 10),
+        createFunctionDecls(rng(MAX_GENERATED_FUNCTIONS_PER_RUN) + 1, MAX_GENERATION_RECURSION_DEPTH),
         "console.log(a, b);"
     ].join("\n");
-    var beautify_code = minify(original_code, {
-        fromString: true,
-        mangle: false,
-        compress: false,
-        output: {
-            beautify: true,
-            bracketize: true,
-        },
-    }).code;
-
-    var uglify_code = minify(beautify_code, {
-        fromString: true,
-        mangle: false,
-        compress: {
-            passes: 3,
-        },
-        output: {
-            beautify: true,
-            bracketize: true,
-        },
-    }).code;
-
     var original_result = run_code(original_code);
+
+    try {
+        var beautify_code = minify(original_code, {
+            fromString: true,
+            mangle: false,
+            compress: false,
+            output: {
+                beautify: true,
+                bracketize: true,
+            },
+        }).code;
+    } catch(e) {
+        parse_error = 1;
+    }
     var beautify_result = run_code(beautify_code);
+
+    try {
+      var uglify_code = minify(beautify_code, {
+          fromString: true,
+          mangle: true,
+          compress: {
+              passes: 3,
+          },
+          output: {
+              //beautify: true,
+              //bracketize: true,
+          },
+      }).code;
+    } catch(e) {
+        parse_error = 2;
+    }
     var uglify_result = run_code(uglify_code);
-    var ok = original_result == beautify_result && original_result == uglify_result;
-    if (verbose || !ok) log();
-    if (!ok) process.exit(1);
+
+    var ok = !parse_error && original_result == beautify_result && original_result == uglify_result;
+    if (verbose || (verbose_interval && !(round % INTERVAL_COUNT)) || !ok) log(ok);
+    if (parse_error === 1) console.log('Parse error while beautifying');
+    if (parse_error === 2) console.log('Parse error while uglifying');
+    if (!ok) break;
 }