Provide actual support for template literals.

- Implement the lookahead and properly check the newlines.
- However, that whole escape sequence check might have to be done
  _inside_ the token formation as this is still going to fail valid
  input templates such as `foo \ bar` when it will reduce to effectively
  `foo  bar`, rather than a syntax error.  Likewise for string literals.

Author: metatoaster

src/calmjs/parse/lexers/es2015.py

@@ -14,16 +14,22 @@ class Lexer(ES5Lexer):
     """
 
     # Punctuators (ES6)
-    t_ARROW        = r'=>'
-    t_SPREAD       = r'\.\.\.'
-    # this is now a right brace operator...
+    # t_DOLLAR_LBRACE  = r'${'
+    # this is also a right brace punctuator...
     # t_RBRACE        = r'}'
+    t_ARROW          = r'=>'
+    t_SPREAD         = r'\.\.\.'
+
+    tokens = ES5Lexer.tokens + (
+        # ES2015 punctuators
+        'ARROW', 'SPREAD',    # => ...
+
+        # ES2015 terminal types
+        'TEMPLATE',
+    )
 
-    # TODO verify that the standard string rule will work.
-    # TODO complete the actual implementation to make this actually
-    # usable.
     template = r"""
-    (?:`                               # opening backquote
+    (?:(?:`|})                         # opening ` or }
         (?: [^`\\]                     # no \, or `
             | \\[a-zA-Z!-\/:-@\[-`{-~] # or escaped characters
             | \\(\n|\r(?!\n)|\u2028|\u2029|\r\n)  # or line continuation
@@ -32,17 +38,9 @@ class Lexer(ES5Lexer):
             | \\(?:[1-7][0-7]{0,2}|[0-7]{2,3}) # or octal_escape_sequence
             | \\0                      # or <NUL> (15.10.2.11)
         )*?                            # zero or many times
-    `)                                 # closing backquote
+    (?:`|\${))                         # closing ` or ${
     """  # `
 
-    tokens = ES5Lexer.tokens + (
-        # ES2015 punctuators
-        'ARROW', 'SPREAD',    # => ...
-
-        # ES2015 terminal types
-        'TEMPLATE',
-    )
-
     @ply.lex.TOKEN(template)
     def t_TEMPLATE(self, token):
         # remove escape + new line sequence used for strings

src/calmjs/parse/tests/lexer.py

@@ -477,6 +477,35 @@
         'template_literal',
         ('`foo`',
          ['TEMPLATE `foo`']),
+    ), (
+        'template_multiline',
+        ('`foo\nbar\r\nfoo`',
+         ['TEMPLATE `foo\nbar\r\nfoo`']),
+    ), (
+        'template_other_newlines',
+        ('`foo\u2028\u2029foo`',
+         ['TEMPLATE `foo\u2028\u2029foo`']),
+    ), (
+        'template_literal_with_dollar',
+        ('`foo$`',
+         ['TEMPLATE `foo$`']),
+    ), (
+        'template_head_tail',
+        (r'`hello ${name} while this`',
+         ['TEMPLATE `hello ${', 'ID name', 'TEMPLATE } while this`']),
+    ), (
+        'template_empty_head_tail',
+        (r'`${name}`',
+         ['TEMPLATE `${', 'ID name', 'TEMPLATE }`']),
+    ), (
+        'template_nested',
+        (r'`${`${a * 2}`} ${b}`',
+         ['TEMPLATE `${', 'TEMPLATE `${', 'ID a', 'MULT *', 'NUMBER 2',
+          'TEMPLATE }`', 'TEMPLATE } ${', 'ID b', 'TEMPLATE }`']),
+    ), (
+        'template_some_keywords',
+        (r'`this -> ${this}.`',
+         ['TEMPLATE `this -> ${', 'THIS this', 'TEMPLATE }.`']),
     ), (
         'template_literal_escape',
         (r'`f\`o`',

src/calmjs/parse/tests/test_es2015_lexer.py

@@ -73,5 +73,5 @@ def test_initial_template_character(self):
         run_lexer_pos, lexer_cls=Lexer), es5_pos_cases)
 
 LexerES2015TestCase = build_equality_testcase(
-    'LexerTestCase', partial(run_lexer, lexer_cls=Lexer), (
+    'LexerES2015TestCase', partial(run_lexer, lexer_cls=Lexer), (
         (label, data[0], data[1],) for label, data in es2015_cases))