Skip to content

Commit 912db9d

Browse files
metatoastermetatoaster
committed
Correct tracking of nested template states
- This must be done as templates can nest templates - Not to mention the fact that objects can be provided as values. Given they share the RBRACE symbol, there needs to be a way to disambiguate that symbol for objects and the opening of the template middle/tail fragments.
1 parent f73f5e3 commit 912db9d

File tree

4 files changed

+182
-8
lines changed

4 files changed

+182
-8
lines changed

‎src/calmjs/parse/lexers/es2015.py

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
ES2015 (ECMAScript 6th Edition/ES6) lexer.
44
"""
55

6+
from __future__ import unicode_literals
7+
68
import re
79
import ply
810
from itertools import chain
@@ -75,9 +77,9 @@ def broken_template_token_handler(lexer, token):
7577
if lexer.current_template_tokens:
7678
# join all tokens together
7779
tmpl = '...'.join(
78-
t.value for t in chain(lexer.current_template_tokens, [token]))
79-
lineno = lexer.current_template_tokens[0].lineno
80-
colno = lexer.current_template_tokens[0].colno
80+
t.value for t in chain(lexer.current_template_tokens[-1], [token]))
81+
lineno = lexer.current_template_tokens[-1][0].lineno
82+
colno = lexer.current_template_tokens[-1][0].colno
8183
else:
8284
tmpl = token.value
8385
lineno = token.lineno
@@ -96,6 +98,7 @@ def __init__(self, with_comments=False, yield_comments=False):
9698
with_comments=with_comments, yield_comments=yield_comments)
9799
self.error_token_handlers.append(broken_template_token_handler)
98100
self.current_template_tokens = []
101+
self.current_template_tokens_braces = []
99102

100103
# Punctuators (ES6)
101104
# t_DOLLAR_LBRACE = r'${'
@@ -149,6 +152,7 @@ def __init__(self, with_comments=False, yield_comments=False):
149152
(?:`|\${)) # closing ` or ${
150153
"""
151154

155+
LBRACE = r'{'
152156
RBRACE = r'}'
153157

154158
@ply.lex.TOKEN(template)
@@ -157,13 +161,38 @@ def t_TEMPLATE_RAW(self, token):
157161
if patt.match(token.value):
158162
token.type = token_type
159163
break
160-
if token.type == 'TEMPLATE_HEAD':
161-
self.current_template_tokens = [token]
162-
elif token.type == 'TEMPLATE_MIDDLE':
163-
self.current_template_tokens.append(token)
164164
else:
165-
self.current_template_tokens = []
165+
raise ValueError("invalid token %r" % token)
166+
167+
if token.type == 'TEMPLATE_HEAD':
168+
self.current_template_tokens.append([token])
169+
self.current_template_tokens_braces.append(0)
170+
return token
171+
elif token.type == 'TEMPLATE_NOSUB':
172+
return token
173+
174+
if not self.current_template_tokens_braces:
175+
raise ECMASyntaxError('Unexpected %s at %s:%s' % (
176+
repr_compat('}'), token.lineno, self._get_colno(token)))
177+
if self.current_template_tokens_braces[-1] > 0:
178+
# produce a LBRACE token instead
179+
self.current_template_tokens_braces[-1] -= 1
180+
self.lexer.lexpos = self.lexer.lexpos - len(token.value) + 1
181+
token.value = token.value[0]
182+
token.type = 'RBRACE'
183+
return token
184+
185+
if token.type == 'TEMPLATE_MIDDLE':
186+
self.current_template_tokens[-1].append(token)
187+
elif token.type == 'TEMPLATE_TAIL':
188+
self.current_template_tokens_braces.pop()
189+
self.current_template_tokens.pop()
190+
return token
166191

192+
@ply.lex.TOKEN(LBRACE)
193+
def t_LBRACE(self, token):
194+
if self.current_template_tokens_braces:
195+
self.current_template_tokens_braces[-1] += 1
167196
return token
168197

169198
@ply.lex.TOKEN(RBRACE)

‎src/calmjs/parse/tests/lexer.py

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -629,6 +629,18 @@
629629
'template_literal_escape',
630630
(r'`f\`o`',
631631
[r'TEMPLATE_NOSUB `f\`o`']),
632+
), (
633+
'template_middle_with_object',
634+
('`object${{1:1}} ${foo}`',
635+
['TEMPLATE_HEAD `object${',
636+
'LBRACE {', 'NUMBER 1', 'COLON :', 'NUMBER 1', 'RBRACE }',
637+
'TEMPLATE_MIDDLE } ${', 'ID foo', 'TEMPLATE_TAIL }`']),
638+
), (
639+
'template_tail_with_object',
640+
('`object${{1:1}}`',
641+
['TEMPLATE_HEAD `object${',
642+
'LBRACE {', 'NUMBER 1', 'COLON :', 'NUMBER 1', 'RBRACE }',
643+
'TEMPLATE_TAIL }`']),
632644
), (
633645
'template_literal_assignment',
634646
('s = `hello world`',
@@ -684,6 +696,15 @@
684696
"var foo = `${foo}bar${baz}fail",
685697
# the specific identifiers are not tracked, thus ...
686698
"Unterminated template literal '`${...}bar${...}...' at 1:11",
699+
), (
700+
'unterminated_template_nested',
701+
"var foo = `${`${foo}bar${baz}fail`}",
702+
# the specific identifiers are not tracked, thus ...
703+
"Unterminated template literal '`${...}' at 1:11",
704+
), (
705+
'unexpected_template_tail',
706+
"var foo = `${value}`}`",
707+
"Unexpected '}' at 1:21",
687708
), (
688709
'invalid_hex_sequence',
689710
"var foo = `fail\\x1`",

‎src/calmjs/parse/tests/parser.py

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2871,6 +2871,24 @@ def parse_to_repr(value):
28712871
>>
28722872
]>
28732873
""",
2874+
), (
2875+
'template_with_many_rbrace',
2876+
"""
2877+
value = `string is ${wat}}}}`
2878+
""",
2879+
"""
2880+
<Program @1:1 ?children=[
2881+
<ExprStatement @1:1 expr=<Assign @1:7 left=<
2882+
Identifier @1:1 value='value'>,
2883+
op='=',
2884+
right=<TemplateLiteral @1:9 ?children=[
2885+
<TemplateHead @1:9 value='`string is ${'>,
2886+
<Identifier @1:22 value='wat'>,
2887+
<TemplateTail @1:25 value='}}}}`'>
2888+
]>
2889+
>>
2890+
]>
2891+
""",
28742892
), (
28752893
'template_in_template',
28762894
"""
@@ -2895,6 +2913,88 @@ def parse_to_repr(value):
28952913
>>
28962914
]>
28972915
""",
2916+
), (
2917+
'template_tail_with_object',
2918+
"""
2919+
value = `object${{1:1}}}`
2920+
""",
2921+
"""
2922+
<Program @1:1 ?children=[
2923+
<ExprStatement @1:1 expr=<Assign @1:7 left=<
2924+
Identifier @1:1 value='value'>, op='=',
2925+
right=<TemplateLiteral @1:9 ?children=[
2926+
<TemplateHead @1:9 value='`object${'>,
2927+
<Object @1:18 properties=[
2928+
<Assign @1:20 left=<Number @1:19 value='1'>,
2929+
op=':',
2930+
right=<Number @1:21 value='1'>>
2931+
]>,
2932+
<TemplateTail @1:23 value='}}`'>
2933+
]>
2934+
>>
2935+
]>
2936+
""",
2937+
), (
2938+
'template_middle_with_object',
2939+
"""
2940+
value = `object${{1:1}}middle${tail}`
2941+
""",
2942+
"""
2943+
<ES2015Program @1:1 ?children=[
2944+
<ExprStatement @1:1 expr=<Assign @1:7 left=<
2945+
Identifier @1:1 value='value'>,
2946+
op='=', right=<TemplateLiteral @1:9 ?children=[
2947+
<TemplateHead @1:9 value='`object${'>,
2948+
<Object @1:18 properties=[
2949+
<Assign @1:20 left=<Number @1:19 value='1'>,
2950+
op=':', right=<Number @1:21 value='1'>>
2951+
]>,
2952+
<TemplateMiddle @1:23 value='}middle${'>,
2953+
<Identifier @1:32 value='tail'>,
2954+
<TemplateTail @1:36 value='}`'>
2955+
]>
2956+
>>
2957+
]>
2958+
""",
2959+
), (
2960+
'template_with_object_with_template_with_object',
2961+
"""
2962+
value = `object
2963+
${{1:`${{
2964+
2:`${{3:3}}`
2965+
}}`}}
2966+
`
2967+
""",
2968+
r"""
2969+
<Program @1:1 ?children=[
2970+
<ExprStatement @1:1 expr=<Assign @1:7 left=<
2971+
Identifier @1:1 value='value'>,
2972+
op='=', right=<TemplateLiteral @1:9 ?children=[
2973+
<TemplateHead @1:9 value='`object\n${'>,
2974+
<Object @2:3 properties=[
2975+
<Assign @2:5 left=<Number @2:4 value='1'>, op=':',
2976+
right=<TemplateLiteral @2:6 ?children=[
2977+
<TemplateHead @2:6 value='`${'>,
2978+
<Object @2:9 properties=[
2979+
<Assign @3:4 left=<Number @3:3 value='2'>, op=':',
2980+
right=<TemplateLiteral @3:5 ?children=[
2981+
<TemplateHead @3:5 value='`${'>,
2982+
<Object @3:8 properties=[
2983+
<Assign @3:10 left=<Number @3:9 value='3'>,
2984+
op=':', right=<Number @3:11 value='3'>>
2985+
]>,
2986+
<TemplateTail @3:13 value='}`'>
2987+
]>>
2988+
]>,
2989+
<TemplateTail @4:2 value='}`'>
2990+
]>
2991+
>
2992+
]>,
2993+
<TemplateTail @4:5 value='}\n`'>
2994+
]>
2995+
>>
2996+
]>
2997+
""",
28982998
)]))
28992999

29003000

@@ -2983,6 +3083,22 @@ def build_es2015_syntax_error_test_cases(clsname, parse):
29833083
'empty_expression_in_template',
29843084
'`head${}tail`',
29853085
"Unexpected '}tail`' at 1:8 after '`head${' at 1:1",
3086+
), (
3087+
'mismatched_template_termination_eof',
3088+
"var foo = `${`${foo}bar${baz}fail`",
3089+
"Unexpected end of input after '}fail`' at 1:29",
3090+
), (
3091+
'mismatched_template_termination',
3092+
"var foo = `head${`${foo}bar${baz}fail`}",
3093+
"Unterminated template literal '`head${...}' at 1:11",
3094+
), (
3095+
'unexpected_block',
3096+
"var foo = `${{11}}`",
3097+
"Unexpected '}' at 1:17 after '11' at 1:15",
3098+
), (
3099+
'object_no_template_keys',
3100+
"var foo = {`foo`: `foo`}",
3101+
"Unexpected '`foo`' at 1:12 between '{' at 1:11 and ':' at 1:17",
29863102
)]), ECMASyntaxError)
29873103

29883104

‎src/calmjs/parse/tests/test_es2015_lexer.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,14 @@ def test_initial_template_character(self):
2929
self.assertEqual(
3030
str(e.exception), "Unterminated template literal '`' at 1:1")
3131

32+
def test_invalid_template_token(self):
33+
lexer = Lexer()
34+
lexer.input('""')
35+
token = lexer.next()
36+
# force an invalid token into the statement
37+
with self.assertRaises(ValueError):
38+
lexer.t_TEMPLATE_RAW(token)
39+
3240

3341
LexerKeywordTestCase = build_equality_testcase(
3442
'LexerTestCase', partial(run_lexer, lexer_cls=Lexer), (

0 commit comments

Comments
 (0)