introduce `templates` (#4603)

9 files changed, 281 insertions, 74 deletions

diff --git a/README.md b/README.md
index 30fc1238..9a0be306 100644
--- a/README.md
+++ b/README.md
@@ -1260,3 +1260,10 @@ To allow for better optimizations, the compiler makes various assumptions:
   // TypeError: can't convert BigInt to number
   ```
   UglifyJS may modify the input which in turn may suppress those errors.
+- Some versions of JavaScript will throw `SyntaxError` with the
+  following:
+  ```javascript
+  console.log(String.raw`\uFo`);
+  // SyntaxError: Invalid Unicode escape sequence
+  ```
+  UglifyJS may modify the input which in turn may suppress those errors.
diff --git a/lib/ast.js b/lib/ast.js
index 5bfe7be1..ead2c7d5 100644
--- a/lib/ast.js
+++ b/lib/ast.js
@@ -1422,7 +1422,7 @@ var AST_Template = DEFNODE("Template", "expressions strings tag", {
     $documentation: "A template literal, i.e. tag`str1${expr1}...strN${exprN}strN+1`",
     $propdoc: {
         expressions: "[AST_Node*] the placeholder expressions",
-        strings: "[string*] the interpolating text segments",
+        strings: "[string*] the raw text segments",
         tag: "[AST_Node] tag function, or null if absent",
     },
     walk: function(visitor) {
diff --git a/lib/compress.js b/lib/compress.js
index 700f3748..f894a6cb 100644
--- a/lib/compress.js
+++ b/lib/compress.js
@@ -91,6 +91,7 @@ function Compressor(options, false_by_default) {
         spreads         : !false_by_default,
         strings         : !false_by_default,
         switches        : !false_by_default,
+        templates       : !false_by_default,
         top_retain      : null,
         toplevel        : !!(options && options["top_retain"]),
         typeofs         : !false_by_default,
@@ -3689,6 +3690,9 @@ merge(Compressor.prototype, {
             delete this.is_string;
             return result;
         });
+        def(AST_Template, function(compressor) {
+            return !this.tag || is_raw_tag(compressor, this.tag);
+        });
         def(AST_UnaryPrefix, function() {
             return this.operator == "typeof";
         });
@@ -4298,6 +4302,16 @@ merge(Compressor.prototype, {
             }
             return this;
         });
+        function eval_all(nodes, compressor, ignore_side_effects, cached, depth) {
+            var values = [];
+            for (var i = 0; i < nodes.length; i++) {
+                var node = nodes[i];
+                var value = node._eval(compressor, ignore_side_effects, cached, depth);
+                if (node === value) return;
+                values.push(value);
+            }
+            return values;
+        }
         def(AST_Call, function(compressor, ignore_side_effects, cached, depth) {
             var exp = this.expression;
             var fn = exp instanceof AST_SymbolRef ? exp.fixed_value() : exp;
@@ -4305,7 +4319,7 @@ merge(Compressor.prototype, {
                 if (fn.evaluating) return this;
                 if (fn.name && fn.name.definition().recursive_refs > 0) return this;
                 if (this.is_expr_pure(compressor)) return this;
-                var args = eval_args(this.args);
+                var args = eval_all(this.args, compressor, ignore_side_effects, cached, depth);
                 if (!all(fn.argnames, function(sym, index) {
                     if (sym instanceof AST_DefaultValue) {
                         if (!args) return false;
@@ -4380,7 +4394,7 @@ merge(Compressor.prototype, {
                     if (!native_fn || !native_fn[key]) return this;
                     if (val instanceof RegExp && val.global && !(e instanceof AST_RegExp)) return this;
                 }
-                var args = eval_args(this.args);
+                var args = eval_all(this.args, compressor, ignore_side_effects, cached, depth);
                 if (!args) return this;
                 if (key == "replace" && typeof args[1] == "function") return this;
                 try {
@@ -4397,19 +4411,35 @@ merge(Compressor.prototype, {
                 }
             }
             return this;
+        });
+        def(AST_New, return_this);
+        def(AST_Template, function(compressor, ignore_side_effects, cached, depth) {
+            if (!compressor.option("templates")) return this;
+            if (this.tag) {
+                if (!is_raw_tag(compressor, this.tag)) return this;
+                decode = function(str) {
+                    return str;
+                };
+            }
+            var exprs = eval_all(this.expressions, compressor, ignore_side_effects, cached, depth);
+            if (!exprs) return this;
+            var ret = decode(this.strings[0]);
+            var malformed = false;
+            for (var i = 0; i < exprs.length; i++) {
+                ret += exprs[i] + decode(this.strings[i + 1]);
+            }
+            if (!malformed) return ret;
+            this._eval = return_this;
+            return this;
 
-            function eval_args(args) {
-                var values = [];
-                for (var i = 0; i < args.length; i++) {
-                    var arg = args[i];
-                    var value = arg._eval(compressor, ignore_side_effects, cached, depth);
-                    if (arg === value) return;
-                    values.push(value);
-                }
-                return values;
+            function decode(str) {
+                return str.replace(/\\(u[0-9a-fA-F]{4}|u\{[0-9a-fA-F]+\}|x[0-9a-fA-F]{2}|[0-9]+|[\s\S])/g, function(match, seq) {
+                    var s = decode_escape_sequence(seq);
+                    if (typeof s != "string") malformed = true;
+                    return s;
+                });
             }
         });
-        def(AST_New, return_this);
     })(function(node, func) {
         node.DEFMETHOD("_eval", func);
     });
@@ -4664,7 +4694,7 @@ merge(Compressor.prototype, {
             return !this.is_declared(compressor) || !can_drop_symbol(this);
         });
         def(AST_Template, function(compressor) {
-            return any(this.expressions, compressor);
+            return this.tag && !is_raw_tag(compressor, this.tag) || any(this.expressions, compressor);
         });
         def(AST_This, return_false);
         def(AST_Try, function(compressor) {
@@ -7019,6 +7049,7 @@ merge(Compressor.prototype, {
             return this.is_declared(compressor) && can_drop_symbol(this) ? null : this;
         });
         def(AST_Template, function(compressor, first_in_statement) {
+            if (this.tag && !is_raw_tag(compressor, this.tag)) return this;
             var expressions = this.expressions;
             if (expressions.length == 0) return null;
             return make_sequence(this, expressions).drop_side_effect_free(compressor, first_in_statement);
@@ -9907,6 +9938,32 @@ merge(Compressor.prototype, {
         }
     });
 
+    function is_raw_tag(compressor, tag) {
+        return compressor.option("unsafe")
+            && tag instanceof AST_Dot
+            && tag.property == "raw"
+            && is_undeclared_ref(tag.expression)
+            && tag.expression.name == "String";
+    }
+
+    OPT(AST_Template, function(self, compressor) {
+        if (!compressor.option("templates")) return self;
+        if (!self.tag || is_raw_tag(compressor, self.tag)) {
+            var exprs = self.expressions;
+            var strs = self.strings;
+            for (var i = exprs.length; --i >= 0;) {
+                var node = exprs[i];
+                var ev = node.evaluate(compressor);
+                if (ev === node) continue;
+                ev = "" + ev;
+                if (ev.length > node.print_to_string().length + 3) continue;
+                exprs.splice(i, 1);
+                strs.splice(i, 2, strs[i] + ev + strs[i + 1]);
+            }
+        }
+        return try_evaluate(compressor, self);
+    });
+
     function is_atomic(lhs, self) {
         return lhs instanceof AST_SymbolRef || lhs.TYPE === self.TYPE;
     }
diff --git a/lib/output.js b/lib/output.js
index 73a63c7d..a171c7e7 100644
--- a/lib/output.js
+++ b/lib/output.js
@@ -780,7 +780,9 @@ function OutputStream(options) {
         // (new foo)(bar)
         if (p instanceof AST_Call) return p.expression === this;
         // (new Date).getTime(), (new Date)["getTime"]()
-        return p instanceof AST_PropAccess;
+        if (p instanceof AST_PropAccess) return true;
+        // (new foo)`bar`
+        if (p instanceof AST_Template) return p.tag === this;
     });
 
     PARENS(AST_Number, function(output) {
@@ -802,6 +804,8 @@ function OutputStream(options) {
         if (p instanceof AST_Conditional) return p.condition === self;
         // (a = foo)["prop"] —or— (a = foo).prop
         if (p instanceof AST_PropAccess) return p.expression === self;
+        // (a = foo)`bar`
+        if (p instanceof AST_Template) return p.tag === self;
         // !(a = false) → true
         if (p instanceof AST_Unary) return true;
     }
diff --git a/lib/parse.js b/lib/parse.js
index 5b77d675..bc72550f 100644
--- a/lib/parse.js
+++ b/lib/parse.js
@@ -145,6 +145,43 @@ function is_identifier_string(str) {
     return /^[a-z_$][a-z0-9_$]*$/i.test(str);
 }
 
+function decode_escape_sequence(seq) {
+    switch (seq[0]) {
+      case "b": return "\b";
+      case "f": return "\f";
+      case "n": return "\n";
+      case "r": return "\r";
+      case "t": return "\t";
+      case "u":
+        var code;
+        if (seq.length == 5) {
+            code = seq.slice(1);
+        } else if (seq[1] == "{" && seq.slice(-1) == "}") {
+            code = seq.slice(2, -1);
+        } else {
+            return;
+        }
+        var num = parseInt(code, 16);
+        if (num < 0 || isNaN(num)) return;
+        if (num < 0x10000) return String.fromCharCode(num);
+        if (num > 0x10ffff) return;
+        return String.fromCharCode((num >> 10) + 0xd7c0) + String.fromCharCode((num & 0x03ff) + 0xdc00);
+      case "v": return "\u000b";
+      case "x":
+        if (seq.length != 3) return;
+        var num = parseInt(seq.slice(1), 16);
+        if (num < 0 || isNaN(num)) return;
+        return String.fromCharCode(num);
+      case "\r":
+      case "\n":
+        return "";
+      default:
+        if (seq == "0") return "\0";
+        if (seq[0] >= "0" && seq[0] <= "9") return;
+        return seq;
+    }
+}
+
 function parse_js_number(num) {
     var match;
     if (match = RE_BIN_NUMBER.exec(num)) return parseInt(match[1], 2);
@@ -340,36 +377,23 @@ function tokenizer($TEXT, filename, html5_comments, shebang) {
     }
 
     function read_escaped_char(in_string) {
-        var ch = next(true, in_string);
-        switch (ch.charCodeAt(0)) {
-          case 110: return "\n";
-          case 114: return "\r";
-          case 116: return "\t";
-          case 98:  return "\b";
-          case 118: return "\u000b";                            // \v
-          case 102: return "\f";
-          case 120: return String.fromCharCode(hex_bytes(2));   // \x
-          case 117:                                             // \u
-            if (peek() != "{") return String.fromCharCode(hex_bytes(4));
-            next();
-            var num = 0;
-            do {
-                var digit = parseInt(next(true), 16);
-                if (isNaN(digit)) parse_error("Invalid hex-character pattern in string");
-                num = num * 16 + digit;
-            } while (peek() != "}");
-            next();
-            if (num < 0x10000) return String.fromCharCode(num);
-            if (num > 0x10ffff) parse_error("Invalid character code: " + num);
-            return String.fromCharCode((num >> 10) + 0xd7c0) + String.fromCharCode((num & 0x03ff) + 0xdc00);
-          case 13:                                              // \r
-            // DOS newline
-            if (peek() == "\n") next(true, in_string);
-          case 10:  return "";                                  // \n
-        }
-        if (ch >= "0" && ch <= "7")
-            return read_octal_escape_sequence(ch);
-        return ch;
+        var seq = next(true, in_string);
+        if (seq >= "0" && seq <= "7") return read_octal_escape_sequence(seq);
+        if (seq == "u") {
+            var ch = next(true, in_string);
+            seq += ch;
+            if (ch != "{") {
+                seq += next(true, in_string) + next(true, in_string) + next(true, in_string);
+            } else do {
+                ch = next(true, in_string);
+                seq += ch;
+            } while (ch != "}");
+        } else if (seq == "x") {
+            seq += next(true, in_string) + next(true, in_string);
+        }
+        var str = decode_escape_sequence(seq);
+        if (typeof str != "string") parse_error("Invalid escape sequence: \\" + seq);
+        return str;
     }
 
     function read_octal_escape_sequence(ch) {
@@ -388,17 +412,6 @@ function tokenizer($TEXT, filename, html5_comments, shebang) {
         return String.fromCharCode(parseInt(ch, 8));
     }
 
-    function hex_bytes(n) {
-        var num = 0;
-        for (; n > 0; --n) {
-            var digit = parseInt(next(true), 16);
-            if (isNaN(digit))
-                parse_error("Invalid hex-character pattern in string");
-            num = (num << 4) | digit;
-        }
-        return num;
-    }
-
     var read_string = with_eof_error("Unterminated string constant", function(quote_char) {
         var quote = next(), ret = "";
         for (;;) {
diff --git a/test/compress/templates.js b/test/compress/templates.js
index 07aff0c5..ef2a534b 100644
--- a/test/compress/templates.js
+++ b/test/compress/templates.js
@@ -53,6 +53,26 @@ tagged_chain: {
     node_version: ">=4"
 }
 
+tag_parenthesis_arrow: {
+    input: {
+        console.log((s => s.raw[0])`\tPASS`.slice(2));
+    }
+    expect_exact: "console.log((s=>s.raw[0])`\\tPASS`.slice(2));"
+    expect_stdout: "PASS"
+    node_version: ">=4"
+}
+
+tag_parenthesis_new: {
+    input: {
+        (new function() {
+            return console.log;
+        })`foo`;
+    }
+    expect_exact: "(new function(){return console.log})`foo`;"
+    expect_stdout: true
+    node_version: ">=4"
+}
+
 malformed_escape: {
     input: {
         (function(s) {
@@ -68,6 +88,7 @@ malformed_escape: {
 evaluate: {
     options = {
         evaluate: true,
+        templates: false,
     }
     input: {
         console.log(`foo ${ function(a, b) {
@@ -80,3 +101,100 @@ evaluate: {
     expect_stdout: "foo 42"
     node_version: ">=4"
 }
+
+evaluate_templates: {
+    options = {
+        evaluate: true,
+        templates: true,
+    }
+    input: {
+        console.log(`foo ${ function(a, b) {
+            return a * b;
+        }(6, 7) }`);
+    }
+    expect: {
+        console.log("foo 42");
+    }
+    expect_stdout: "foo 42"
+    node_version: ">=4"
+}
+
+partial_evaluate: {
+    options = {
+        evaluate: true,
+        templates: true,
+    }
+    input: {
+        console.log(`${6 * 7} foo ${console ? `PA` + "SS" : `FA` + `IL`}`);
+    }
+    expect: {
+        console.log(`42 foo ${console ? "PASS" : "FAIL"}`);
+    }
+    expect_stdout: "42 foo PASS"
+    node_version: ">=4"
+}
+
+malformed_evaluate: {
+    options = {
+        evaluate: true,
+        templates: true,
+    }
+    input: {
+        console.log(`\67 ${6 * 7}`);
+    }
+    expect: {
+        console.log(`\67 42`);
+    }
+    expect_stdout: true
+    node_version: ">=4"
+}
+
+unsafe_evaluate: {
+    options = {
+        evaluate: true,
+        templates: true,
+        unsafe: true,
+    }
+    input: {
+        console.log(String.raw`\uFo`);
+    }
+    expect: {
+        console.log("\\uFo");
+    }
+    expect_stdout: "\\uFo"
+    node_version: ">=8"
+}
+
+side_effects: {
+    options = {
+        side_effects: true,
+    }
+    input: {
+        `42`;
+        `${console.log("foo")}`;
+        console.log`\nbar`;
+    }
+    expect: {
+        console.log("foo");
+        console.log`\nbar`;
+    }
+    expect_stdout: true
+    node_version: ">=4"
+}
+
+unsafe_side_effects: {
+    options = {
+        side_effects: true,
+        unsafe: true,
+    }
+    input: {
+        `42`;
+        `${console.log("foo")}`;
+        String.raw`\nbar`;
+    }
+    expect: {
+        console.log("foo");
+    }
+    expect_stdout: "foo"
+    node_version: ">=4"
+}
diff --git a/test/mocha/string-literal.js b/test/mocha/string-literal.js
index 7bb777bd..02747232 100644
--- a/test/mocha/string-literal.js
+++ b/test/mocha/string-literal.js
@@ -115,8 +115,8 @@ describe("String literals", function() {
                 UglifyJS.parse(test);
             }, function(e) {
                 return e instanceof UglifyJS.JS_Parse_Error
-                    && e.message === "Invalid hex-character pattern in string";
-            });
+                    && /^Invalid escape sequence: \\u/.test(e.message);
+            }, test);
         });
     });
     it("Should reject invalid code points in Unicode escape sequence", function() {
@@ -130,8 +130,8 @@ describe("String literals", function() {
                 UglifyJS.parse(test);
             }, function(e) {
                 return e instanceof UglifyJS.JS_Parse_Error
-                    && /^Invalid character code: /.test(e.message);
-            });
+                    && /^Invalid escape sequence: \\u{1/.test(e.message);
+            }, test);
         });
     });
 });
diff --git a/test/mocha/templates.js b/test/mocha/templates.js
index 7036be3d..169e791e 100644
--- a/test/mocha/templates.js
+++ b/test/mocha/templates.js
@@ -53,7 +53,10 @@ describe("Template literals", function() {
             [ "`foo\\\\r\nbar`", "`foo\\\\r\nbar`" ],
         ].forEach(function(test) {
             var input = "console.log(" + test[0] + ");";
-            var result = UglifyJS.minify(input);
+            var result = UglifyJS.minify(input, {
+                compress: false,
+                mangle: false,
+            });
             if (result.error) throw result.error;
             var expected = "console.log(" + test[1] + ");";
             assert.strictEqual(result.code, expected, test[0]);
diff --git a/test/ufuzz/index.js b/test/ufuzz/index.js
index 86badf3a..5256ec63 100644
--- a/test/ufuzz/index.js
+++ b/test/ufuzz/index.js
@@ -410,8 +410,9 @@ function createParams(was_async, noDuplicate) {
     return addTrailingComma(params.join(", "));
 }
 
-function createArgs(recurmax, stmtDepth, canThrow) {
+function createArgs(recurmax, stmtDepth, canThrow, noTemplate) {
     recurmax--;
+    if (SUPPORT.template && !noTemplate && rng(20) == 0) return createTemplateLiteral(recurmax, stmtDepth, canThrow);
     var args = [];
     for (var n = rng(4); --n >= 0;) switch (SUPPORT.spread ? rng(50) : 3) {
       case 0:
@@ -430,7 +431,7 @@ function createArgs(recurmax, stmtDepth, canThrow) {
         args.push(rng(2) ? createValue() : createExpression(recurmax, NO_COMMA, stmtDepth, canThrow));
         break;
     }
-    return addTrailingComma(args.join(", "));
+    return "(" + addTrailingComma(args.join(", ")) + ")";
 }
 
 function createAssignmentPairs(recurmax, stmtDepth, canThrow, nameLenBefore, was_async) {
@@ -731,7 +732,7 @@ function createFunction(recurmax, allowDefun, canThrow, stmtDepth) {
             var pairs = createAssignmentPairs(recurmax, stmtDepth, canThrow, nameLenBefore, save_async);
             params = pairs.names.join(", ");
             if (!pairs.has_rest) params = addTrailingComma(params);
-            args = addTrailingComma(pairs.values.join(", "));
+            args = "(" + addTrailingComma(pairs.values.join(", ")) + ")";
         } else {
             params = createParams(save_async);
         }
@@ -753,10 +754,10 @@ function createFunction(recurmax, allowDefun, canThrow, stmtDepth) {
     if (!allowDefun) {
         // avoid "function statements" (decl inside statements)
         s = "var " + createVarName(MANDATORY) + " = " + s;
-        s += "(" + (args || createArgs(recurmax, stmtDepth, canThrow)) + ")";
+        s += args || createArgs(recurmax, stmtDepth, canThrow);
     } else if (!(name in called) || args || rng(3)) {
         s += "var " + createVarName(MANDATORY) + " = " + name;
-        s += "(" + (args || createArgs(recurmax, stmtDepth, canThrow)) + ")";
+        s += args || createArgs(recurmax, stmtDepth, canThrow);
     }
 
     return s + ";";
@@ -1039,7 +1040,11 @@ function _createExpression(recurmax, noComma, stmtDepth, canThrow) {
       case p++:
         return rng(2) + " === 1 ? a : b";
       case p++:
-        if (SUPPORT.template && rng(20) == 0) return createTemplateLiteral(recurmax, stmtDepth, canThrow);
+        if (SUPPORT.template && rng(20) == 0) {
+            var tmpl = createTemplateLiteral(recurmax, stmtDepth, canThrow);
+            if (rng(10) == 0) tmpl = "String.raw" + tmpl;
+            return tmpl;
+        }
       case p++:
         return createValue();
       case p++:
@@ -1093,7 +1098,7 @@ function _createExpression(recurmax, noComma, stmtDepth, canThrow) {
                         var pairs = createAssignmentPairs(recurmax, stmtDepth, canThrow, nameLenBefore, save_async);
                         params = pairs.names.join(", ");
                         if (!pairs.has_rest) params = addTrailingComma(params);
-                        args = addTrailingComma(pairs.values.join(", "));
+                        args = "(" + addTrailingComma(pairs.values.join(", ")) + ")";
                     } else {
                         params = createParams(save_async, NO_DUPLICATE);
                     }
@@ -1125,7 +1130,7 @@ function _createExpression(recurmax, noComma, stmtDepth, canThrow) {
                 async = save_async;
                 VAR_NAMES.length = nameLenBefore;
                 if (!args && rng(2)) args = createArgs(recurmax, stmtDepth, canThrow);
-                if (args) suffix += "(" + args + ")";
+                if (args) suffix += args;
                 s.push(suffix);
             } else {
                 s.push(
@@ -1162,8 +1167,8 @@ function _createExpression(recurmax, noComma, stmtDepth, canThrow) {
             break;
           default:
             async = false;
+            var instantiate = rng(4) ? "new " : "";
             createBlockVariables(recurmax, stmtDepth, canThrow, function(defns) {
-                var instantiate = rng(4) ? "new " : "";
                 s.push(
                     instantiate + "function " + name + "(" + createParams(save_async) + "){",
                     strictMode(),
@@ -1177,7 +1182,7 @@ function _createExpression(recurmax, noComma, stmtDepth, canThrow) {
             });
             async = save_async;
             VAR_NAMES.length = nameLenBefore;
-            s.push(rng(2) ? "}" : "}(" + createArgs(recurmax, stmtDepth, canThrow) + ")");
+            s.push(rng(2) ? "}" : "}" + createArgs(recurmax, stmtDepth, canThrow, instantiate));
             break;
         }
         async = save_async;
@@ -1255,7 +1260,7 @@ function _createExpression(recurmax, noComma, stmtDepth, canThrow) {
       case p++:
         var name = getVarName();
         var s = name + "." + getDotKey();
-        s = "typeof " + s + ' == "function" && --_calls_ >= 0 && ' + s + "(" + createArgs(recurmax, stmtDepth, canThrow) + ")";
+        s = "typeof " + s + ' == "function" && --_calls_ >= 0 && ' + s + createArgs(recurmax, stmtDepth, canThrow);
         return canThrow && rng(8) == 0 ? s : name + " && " + s;
       case p++:
       case p++:
@@ -1266,7 +1271,7 @@ function _createExpression(recurmax, noComma, stmtDepth, canThrow) {
             name = rng(3) == 0 ? getVarName() : "f" + rng(funcs + 2);
         } while (name in called && !called[name]);
         called[name] = true;
-        return "typeof " + name + ' == "function" && --_calls_ >= 0 && ' + name + "(" + createArgs(recurmax, stmtDepth, canThrow) + ")";
+        return "typeof " + name + ' == "function" && --_calls_ >= 0 && ' + name + createArgs(recurmax, stmtDepth, canThrow);
     }
     _createExpression.N = p;
     return _createExpression(recurmax, noComma, stmtDepth, canThrow);
@@ -1308,7 +1313,7 @@ function createTemplateLiteral(recurmax, stmtDepth, canThrow) {
         s.push("${", createExpression(recurmax, COMMA_OK, stmtDepth, canThrow), "}");
         addText();
     }
-    return (rng(10) ? "`" : "String.raw`") + s.join(rng(5) ? "" : "\n") + "`";
+    return "`" + s.join(rng(5) ? "" : "\n") + "`";
 
     function addText() {
         while (rng(5) == 0) s.push([