8134941: Implement ES6 template literal support

Reviewed-by: attila, hannesw

nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/ir/RuntimeNode.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -83,7 +83,9 @@ public class RuntimeNode extends Expression {
         /** is undefined */
         IS_UNDEFINED(TokenType.EQ_STRICT, Type.BOOLEAN, 2),
         /** is not undefined */
-        IS_NOT_UNDEFINED(TokenType.NE_STRICT, Type.BOOLEAN, 2);
+        IS_NOT_UNDEFINED(TokenType.NE_STRICT, Type.BOOLEAN, 2),
+        /** Get template object from raw and cooked string arrays. */
+        GET_TEMPLATE_OBJECT(TokenType.TEMPLATE, Type.SCRIPT_OBJECT, 2);
 
         /** token type */
         private final TokenType tokenType;

nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Lexer.java

@@ -46,6 +46,10 @@ import static jdk.nashorn.internal.parser.TokenType.RBRACE;
 import static jdk.nashorn.internal.parser.TokenType.REGEX;
 import static jdk.nashorn.internal.parser.TokenType.RPAREN;
 import static jdk.nashorn.internal.parser.TokenType.STRING;
+import static jdk.nashorn.internal.parser.TokenType.TEMPLATE;
+import static jdk.nashorn.internal.parser.TokenType.TEMPLATE_HEAD;
+import static jdk.nashorn.internal.parser.TokenType.TEMPLATE_MIDDLE;
+import static jdk.nashorn.internal.parser.TokenType.TEMPLATE_TAIL;
 import static jdk.nashorn.internal.parser.TokenType.XML;
 
 import java.io.Serializable;
@@ -96,6 +100,8 @@ public class Lexer extends Scanner {
     private final boolean pauseOnFunctionBody;
     private boolean pauseOnNextLeftBrace;
 
+    private int templateExpressionOpenBraces;
+
     private static final String SPACETAB = " \t";  // ASCII space and tab
     private static final String LFCR     = "\n\r"; // line feed and carriage return (ctrl-m)
 
@@ -392,13 +398,19 @@ public class Lexer extends Scanner {
     }
 
     /**
-     * Test if char is a string delimiter, e.g. '\' or '"'.  Also scans exec
+     * Test if char is a string delimiter, e.g. '\' or '"'.
-     * strings ('`') in scripting mode.
      * @param ch a char
      * @return true if string delimiter
      */
     protected boolean isStringDelimiter(final char ch) {
-        return ch == '\'' || ch == '"' || (scripting && ch == '`');
+        return ch == '\'' || ch == '"';
+    }
+
+    /**
+     * Test if char is a template literal delimiter ('`').
+     */
+    private static boolean isTemplateDelimiter(char ch) {
+        return ch == '`';
     }
 
     /**
@@ -943,6 +955,10 @@ public class Lexer extends Scanner {
                     sb.append(next);
                     break;
                 }
+            } else if (ch0 == '\r') {
+                // Convert CR-LF or CR to LF line terminator.
+                sb.append('\n');
+                skip(ch1 == '\n' ? 2 : 1);
             } else {
                 // Add regular character.
                 sb.append(ch0);
@@ -958,7 +974,7 @@ public class Lexer extends Scanner {
 
     /**
      * Scan over a string literal.
-     * @param add true if we nare not just scanning but should actually modify the token stream
+     * @param add true if we are not just scanning but should actually modify the token stream
      */
     protected void scanString(final boolean add) {
         // Type of string.
@@ -1033,6 +1049,70 @@ public class Lexer extends Scanner {
         }
     }
 
+    /**
+     * Scan over a template string literal.
+     */
+    private void scanTemplate() {
+        assert ch0 == '`';
+        TokenType type = TEMPLATE;
+
+        // Skip over quote and record beginning of string content.
+        skip(1);
+        State stringState = saveState();
+
+        // Scan until close quote
+        while (!atEOF()) {
+            // Skip over escaped character.
+            if (ch0 == '`') {
+                skip(1);
+                // Record end of string.
+                stringState.setLimit(position - 1);
+                add(type == TEMPLATE ? type : TEMPLATE_TAIL, stringState.position, stringState.limit);
+                return;
+            } else if (ch0 == '$' && ch1 == '{') {
+                skip(2);
+                stringState.setLimit(position - 2);
+                add(type == TEMPLATE ? TEMPLATE_HEAD : type, stringState.position, stringState.limit);
+
+                // scan to RBRACE
+                Lexer expressionLexer = new Lexer(this, saveState());
+                expressionLexer.templateExpressionOpenBraces = 1;
+                expressionLexer.lexify();
+                restoreState(expressionLexer.saveState());
+
+                // scan next middle or tail of the template literal
+                assert ch0 == '}';
+                type = TEMPLATE_MIDDLE;
+
+                // Skip over rbrace and record beginning of string content.
+                skip(1);
+                stringState = saveState();
+
+                continue;
+            } else if (ch0 == '\\') {
+                skip(1);
+                // EscapeSequence
+                if (!isEscapeCharacter(ch0)) {
+                    error(Lexer.message("invalid.escape.char"), TEMPLATE, position, limit);
+                }
+                if (isEOL(ch0)) {
+                    // LineContinuation
+                    skipEOL(false);
+                    continue;
+                }
+            }  else if (isEOL(ch0)) {
+                // LineTerminatorSequence
+                skipEOL(false);
+                continue;
+            }
+
+            // Skip literal character.
+            skip(1);
+        }
+
+        error(Lexer.message("missing.close.quote"), TEMPLATE, position, limit);
+    }
+
     /**
      * Is the given character a valid escape char after "\" ?
      *
@@ -1621,6 +1701,16 @@ public class Lexer extends Scanner {
                 // Scan and add a number.
                 scanNumber();
             } else if ((type = TokenLookup.lookupOperator(ch0, ch1, ch2, ch3)) != null) {
+                if (templateExpressionOpenBraces > 0) {
+                    if (type == LBRACE) {
+                        templateExpressionOpenBraces++;
+                    } else if (type == RBRACE) {
+                        if (--templateExpressionOpenBraces == 0) {
+                            break;
+                        }
+                    }
+                }
+
                 // Get the number of characters in the token.
                 final int typeLength = type.getLength();
                 // Skip that many characters.
@@ -1644,6 +1734,12 @@ public class Lexer extends Scanner {
             } else if (Character.isDigit(ch0)) {
                 // Scan and add a number.
                 scanNumber();
+            } else if (isTemplateDelimiter(ch0) && es6) {
+                // Scan and add template in ES6 mode.
+                scanTemplate();
+            } else if (isTemplateDelimiter(ch0) && scripting) {
+                // Scan and add an exec string ('`') in scripting mode.
+                scanString(true);
             } else {
                 // Don't recognize this character.
                 skip(1);
@@ -1699,6 +1795,11 @@ public class Lexer extends Scanner {
             return valueOfIdent(start, len); // String
         case REGEX:
             return valueOfPattern(start, len); // RegexToken::LexerToken
+        case TEMPLATE:
+        case TEMPLATE_HEAD:
+        case TEMPLATE_MIDDLE:
+        case TEMPLATE_TAIL:
+            return valueOfString(start, len, true); // String
         case XML:
             return valueOfXML(start, len); // XMLToken::LexerToken
         case DIRECTIVE_COMMENT:
@@ -1710,6 +1811,45 @@ public class Lexer extends Scanner {
         return null;
     }
 
+    /**
+     * Get the raw string value of a template literal string part.
+     *
+     * @param token template string token
+     * @return raw string
+     */
+    public String valueOfRawString(final long token) {
+        final int start  = Token.descPosition(token);
+        final int length = Token.descLength(token);
+
+        // Save the current position.
+        final int savePosition = position;
+        // Calculate the end position.
+        final int end = start + length;
+        // Reset to beginning of string.
+        reset(start);
+
+        // Buffer for recording characters.
+        final StringBuilder sb = new StringBuilder(length);
+
+        // Scan until end of string.
+        while (position < end) {
+            if (ch0 == '\r') {
+                // Convert CR-LF or CR to LF line terminator.
+                sb.append('\n');
+                skip(ch1 == '\n' ? 2 : 1);
+            } else {
+                // Add regular character.
+                sb.append(ch0);
+                skip(1);
+            }
+        }
+
+        // Restore position.
+        reset(savePosition);
+
+        return sb.toString();
+    }
+
     /**
      * Get the correctly localized error message for a given message id format arguments
      * @param msgId message id

nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Parser.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -51,6 +51,10 @@ import static jdk.nashorn.internal.parser.TokenType.RBRACE;
 import static jdk.nashorn.internal.parser.TokenType.RBRACKET;
 import static jdk.nashorn.internal.parser.TokenType.RPAREN;
 import static jdk.nashorn.internal.parser.TokenType.SEMICOLON;
+import static jdk.nashorn.internal.parser.TokenType.TEMPLATE;
+import static jdk.nashorn.internal.parser.TokenType.TEMPLATE_HEAD;
+import static jdk.nashorn.internal.parser.TokenType.TEMPLATE_MIDDLE;
+import static jdk.nashorn.internal.parser.TokenType.TEMPLATE_TAIL;
 import static jdk.nashorn.internal.parser.TokenType.TERNARY;
 import static jdk.nashorn.internal.parser.TokenType.WHILE;
 
@@ -64,6 +68,7 @@ import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+
 import jdk.internal.dynalink.support.NameCodec;
 import jdk.nashorn.internal.codegen.CompilerConstants;
 import jdk.nashorn.internal.codegen.Namespace;
@@ -1926,10 +1931,10 @@ loop:
      *      Literal
      *      ArrayLiteral
      *      ObjectLiteral
+     *      RegularExpressionLiteral
+     *      TemplateLiteral
      *      ( Expression )
      *
-     *  See 11.1
-     *
      * Parse primary expression.
      * @return Expression node.
      */
@@ -1989,6 +1994,9 @@ loop:
             expect(RPAREN);
 
             return expression;
+        case TEMPLATE:
+        case TEMPLATE_HEAD:
+            return templateLiteral();
 
         default:
             // In this context some operator tokens mark the start of a literal.
@@ -2387,6 +2395,8 @@ loop:
     }
 
     /**
+     * Parse left hand side expression.
+     *
      * LeftHandSideExpression :
      *      NewExpression
      *      CallExpression
@@ -2396,10 +2406,8 @@ loop:
      *      CallExpression Arguments
      *      CallExpression [ Expression ]
      *      CallExpression . IdentifierName
+     *      CallExpression TemplateLiteral
      *
-     * See 11.2
-     *
-     * Parse left hand side expression.
      * @return Expression node.
      */
     private Expression leftHandSideExpression() {
@@ -2426,7 +2434,7 @@ loop:
             callToken = token;
 
             switch (type) {
-            case LPAREN:
+            case LPAREN: {
                 // Get NEW or FUNCTION arguments.
                 final List<Expression> arguments = optimizeList(argumentList());
 
@@ -2434,8 +2442,8 @@ loop:
                 lhs = new CallNode(callLine, callToken, finish, lhs, arguments, false);
 
                 break;
-
+            }
-            case LBRACKET:
+            case LBRACKET: {
                 next();
 
                 // Get array index.
@@ -2447,8 +2455,8 @@ loop:
                 lhs = new IndexNode(callToken, finish, lhs, rhs);
 
                 break;
-
+            }
-            case PERIOD:
+            case PERIOD: {
                 next();
 
                 final IdentNode property = getIdentifierName();
@@ -2457,7 +2465,16 @@ loop:
                 lhs = new AccessNode(callToken, finish, lhs, property.getName());
 
                 break;
+            }
+            case TEMPLATE:
+            case TEMPLATE_HEAD: {
+                // tagged template literal
+                final List<Expression> arguments = templateLiteralArgumentList();
 
+                lhs = new CallNode(callLine, callToken, finish, lhs, arguments, false);
+
+                break;
+            }
             default:
                 break loop;
             }
@@ -2516,16 +2533,16 @@ loop:
     }
 
     /**
+     * Parse member expression.
+     *
      * MemberExpression :
      *      PrimaryExpression
      *      FunctionExpression
      *      MemberExpression [ Expression ]
      *      MemberExpression . IdentifierName
+     *      MemberExpression TemplateLiteral
      *      new MemberExpression Arguments
      *
-     * See 11.2
-     *
-     * Parse member expression.
      * @return Expression node.
      */
     private Expression memberExpression() {
@@ -2582,6 +2599,16 @@ loop:
 
                 break;
             }
+            case TEMPLATE:
+            case TEMPLATE_HEAD: {
+                // tagged template literal
+                final int callLine = line;
+                final List<Expression> arguments = templateLiteralArgumentList();
+
+                lhs = new CallNode(callLine, callToken, finish, lhs, arguments, false);
+
+                break;
+            }
             default:
                 break loop;
             }
@@ -3035,6 +3062,20 @@ loop:
         final ParserState parserState = (ParserState)data.getEndParserState();
         assert parserState != null;
 
+        if (k < stream.last() && start < parserState.position && parserState.position <= Token.descPosition(stream.get(stream.last()))) {
+            // RBRACE is already in the token stream, so fast forward to it
+            for (; k < stream.last(); k++) {
+                long nextToken = stream.get(k + 1);
+                if (Token.descPosition(nextToken) == parserState.position && Token.descType(nextToken) == RBRACE) {
+                    token = stream.get(k);
+                    type = Token.descType(token);
+                    next();
+                    assert type == RBRACE && start == parserState.position;
+                    return true;
+                }
+            }
+        }
+
         stream.reset();
         lexer = parserState.createLexer(source, lexer, stream, scripting && !env._no_syntax_extensions, env._es6);
         line = parserState.line;
@@ -3425,6 +3466,79 @@ loop:
         }
     }
 
+    /**
+     * Parse untagged template literal as string concatenation.
+     */
+    private Expression templateLiteral() {
+        assert type == TEMPLATE || type == TEMPLATE_HEAD;
+        final boolean noSubstitutionTemplate = type == TEMPLATE;
+        long lastLiteralToken = token;
+        LiteralNode<?> literal = getLiteral();
+        if (noSubstitutionTemplate) {
+            return literal;
+        }
+
+        Expression concat = literal;
+        TokenType lastLiteralType;
+        do {
+            Expression expression = expression();
+            if (type != TEMPLATE_MIDDLE && type != TEMPLATE_TAIL) {
+                throw error(AbstractParser.message("unterminated.template.expression"), token);
+            }
+            concat = new BinaryNode(Token.recast(lastLiteralToken, TokenType.ADD), concat, expression);
+            lastLiteralType = type;
+            lastLiteralToken = token;
+            literal = getLiteral();
+            concat = new BinaryNode(Token.recast(lastLiteralToken, TokenType.ADD), concat, literal);
+        } while (lastLiteralType == TEMPLATE_MIDDLE);
+        return concat;
+    }
+
+    /**
+     * Parse tagged template literal as argument list.
+     * @return argument list for a tag function call (template object, ...substitutions)
+     */
+    private List<Expression> templateLiteralArgumentList() {
+        assert type == TEMPLATE || type == TEMPLATE_HEAD;
+        final ArrayList<Expression> argumentList = new ArrayList<>();
+        final ArrayList<Expression> rawStrings = new ArrayList<>();
+        final ArrayList<Expression> cookedStrings = new ArrayList<>();
+        argumentList.add(null); // filled at the end
+
+        final long templateToken = token;
+        final boolean hasSubstitutions = type == TEMPLATE_HEAD;
+        addTemplateLiteralString(rawStrings, cookedStrings);
+
+        if (hasSubstitutions) {
+            TokenType lastLiteralType;
+            do {
+                Expression expression = expression();
+                if (type != TEMPLATE_MIDDLE && type != TEMPLATE_TAIL) {
+                    throw error(AbstractParser.message("unterminated.template.expression"), token);
+                }
+                argumentList.add(expression);
+
+                lastLiteralType = type;
+                addTemplateLiteralString(rawStrings, cookedStrings);
+            } while (lastLiteralType == TEMPLATE_MIDDLE);
+        }
+
+        final LiteralNode<Expression[]> rawStringArray = LiteralNode.newInstance(templateToken, finish, rawStrings);
+        final LiteralNode<Expression[]> cookedStringArray = LiteralNode.newInstance(templateToken, finish, cookedStrings);
+        final RuntimeNode templateObject = new RuntimeNode(templateToken, finish, RuntimeNode.Request.GET_TEMPLATE_OBJECT, rawStringArray, cookedStringArray);
+        argumentList.set(0, templateObject);
+        return optimizeList(argumentList);
+    }
+
+    private void addTemplateLiteralString(final ArrayList<Expression> rawStrings, final ArrayList<Expression> cookedStrings) {
+        final long stringToken = token;
+        final String rawString = lexer.valueOfRawString(stringToken);
+        final String cookedString = (String) getValue();
+        next();
+        rawStrings.add(LiteralNode.newInstance(stringToken, finish, rawString));
+        cookedStrings.add(LiteralNode.newInstance(stringToken, finish, cookedString));
+    }
+
     @Override
     public String toString() {
         return "'JavaScript Parsing'";

nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Token.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -71,11 +71,21 @@ public class Token {
     public static long withDelimiter(final long token) {
         final TokenType tokenType = Token.descType(token);
         switch(tokenType) {
-            case STRING: case ESCSTRING: case EXECSTRING: {
+            case STRING:
+            case ESCSTRING:
+            case EXECSTRING:
+            case TEMPLATE:
+            case TEMPLATE_TAIL: {
                 final int start = Token.descPosition(token) - 1;
                 final int len = Token.descLength(token) + 2;
                 return toDesc(tokenType, start, len);
             }
+            case TEMPLATE_HEAD:
+            case TEMPLATE_MIDDLE: {
+                final int start = Token.descPosition(token) - 1;
+                final int len = Token.descLength(token) + 3;
+                return toDesc(tokenType, start, len);
+            }
             default: {
                 return token;
             }

nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/TokenType.java

@@ -183,6 +183,10 @@ public enum TokenType {
     XML            (LITERAL,  null),
     OBJECT         (LITERAL,  null),
     ARRAY          (LITERAL,  null),
+    TEMPLATE       (LITERAL,  null),
+    TEMPLATE_HEAD  (LITERAL,  null),
+    TEMPLATE_MIDDLE(LITERAL,  null),
+    TEMPLATE_TAIL  (LITERAL,  null),
 
     COMMALEFT      (IR,       null),
     DECPOSTFIX     (IR,       null),

nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/runtime/ScriptRuntime.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -45,6 +45,7 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.Objects;
+
 import jdk.internal.dynalink.beans.StaticClass;
 import jdk.nashorn.api.scripting.JSObject;
 import jdk.nashorn.api.scripting.ScriptObjectMirror;
@@ -1050,4 +1051,20 @@ public final class ScriptRuntime {
         context.getLogger(ApplySpecialization.class).info("Overwrote special name '" + name +"' - invalidating switchpoint");
         SwitchPoint.invalidateAll(new SwitchPoint[] { sp });
     }
+
+    /**
+     * ES6 12.2.9.3 Runtime Semantics: GetTemplateObject(templateLiteral).
+     *
+     * @param rawStrings array of template raw values
+     * @param cookedStrings array of template values
+     * @return template object
+     */
+    public static ScriptObject GET_TEMPLATE_OBJECT(final Object rawStrings, final Object cookedStrings) {
+        final ScriptObject template = (ScriptObject)cookedStrings;
+        final ScriptObject rawObj = (ScriptObject)rawStrings;
+        assert rawObj.getArray().length() == template.getArray().length();
+        template.addOwnProperty("raw", Property.NOT_WRITABLE | Property.NOT_ENUMERABLE | Property.NOT_CONFIGURABLE, rawObj.freeze());
+        template.freeze();
+        return template;
+    }
 }

nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/runtime/resources/Messages.properties

@@ -60,6 +60,7 @@ parser.error.regex.repeated.flag=Repeated RegExp flag: {0}
 parser.error.regex.syntax={0}
 parser.error.trailing.comma.in.json=Trailing comma is not allowed in JSON
 parser.error.missing.const.assignment=Missing assignment to constant "{0}"
+parser.error.unterminated.template.expression=Expected } after expression in template literal
 
 # strict mode error messages
 parser.error.strict.no.with="with" statement cannot be used in strict mode