Skip to content

Commit 1faef6e

Browse files
metatoastermetatoaster
committed
Correct tracking of nested template states
- This must be done as templates can nest templates - Not to mention the fact that objects can be provided as values. Given they share the RBRACE symbol, there needs to be a way to disambiguate that symbol for objects and the opening of the template middle/tail fragments.
1 parent 116fb1a commit 1faef6e

File tree

4 files changed

+182
-8
lines changed

4 files changed

+182
-8
lines changed

‎src/calmjs/parse/lexers/es2015.py

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
ES2015 (ECMAScript 6th Edition/ES6) lexer.
44
"""
55

6+
from __future__ import unicode_literals
7+
68
import re
79
import ply
810
from itertools import chain
@@ -75,9 +77,9 @@ def broken_template_token_handler(lexer, token):
7577
if lexer.current_template_tokens:
7678
# join all tokens together
7779
tmpl = '...'.join(
78-
t.value for t in chain(lexer.current_template_tokens, [token]))
79-
lineno = lexer.current_template_tokens[0].lineno
80-
colno = lexer.current_template_tokens[0].colno
80+
t.value for t in chain(lexer.current_template_tokens[-1], [token]))
81+
lineno = lexer.current_template_tokens[-1][0].lineno
82+
colno = lexer.current_template_tokens[-1][0].colno
8183
else:
8284
tmpl = token.value
8385
lineno = token.lineno
@@ -96,6 +98,7 @@ def __init__(self, with_comments=False, yield_comments=False):
9698
with_comments=with_comments, yield_comments=yield_comments)
9799
self.error_token_handlers.append(broken_template_token_handler)
98100
self.current_template_tokens = []
101+
self.current_template_tokens_braces = []
99102

100103
# Punctuators (ES6)
101104
# t_DOLLAR_LBRACE = r'${'
@@ -149,6 +152,7 @@ def __init__(self, with_comments=False, yield_comments=False):
149152
(?:`|\${)) # closing ` or ${
150153
"""
151154

155+
LBRACE = r'{'
152156
RBRACE = r'}'
153157

154158
@ply.lex.TOKEN(template)
@@ -157,13 +161,38 @@ def t_TEMPLATE_RAW(self, token):
157161
if patt.match(token.value):
158162
token.type = token_type
159163
break
160-
if token.type == 'TEMPLATE_HEAD':
161-
self.current_template_tokens = [token]
162-
elif token.type == 'TEMPLATE_MIDDLE':
163-
self.current_template_tokens.append(token)
164164
else:
165-
self.current_template_tokens = []
165+
raise ValueError("invalid token %r" % token)
166+
167+
if token.type == 'TEMPLATE_HEAD':
168+
self.current_template_tokens.append([token])
169+
self.current_template_tokens_braces.append(0)
170+
return token
171+
elif token.type == 'TEMPLATE_NOSUB':
172+
return token
173+
174+
if not self.current_template_tokens_braces:
175+
raise ECMASyntaxError('Unexpected %s at %s:%s' % (
176+
repr_compat('}'), token.lineno, self._get_colno(token)))
177+
if self.current_template_tokens_braces[-1] > 0:
178+
# produce a LBRACE token instead
179+
self.current_template_tokens_braces[-1] -= 1
180+
self.lexer.lexpos = self.lexer.lexpos - len(token.value) + 1
181+
token.value = token.value[0]
182+
token.type = 'RBRACE'
183+
return token
184+
185+
if token.type == 'TEMPLATE_MIDDLE':
186+
self.current_template_tokens[-1].append(token)
187+
elif token.type == 'TEMPLATE_TAIL':
188+
self.current_template_tokens_braces.pop()
189+
self.current_template_tokens.pop()
190+
return token
166191

192+
@ply.lex.TOKEN(LBRACE)
193+
def t_LBRACE(self, token):
194+
if self.current_template_tokens_braces:
195+
self.current_template_tokens_braces[-1] += 1
167196
return token
168197

169198
@ply.lex.TOKEN(RBRACE)

‎src/calmjs/parse/tests/lexer.py

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -629,6 +629,18 @@
629629
'template_literal_escape',
630630
(r'`f\`o`',
631631
[r'TEMPLATE_NOSUB `f\`o`']),
632+
), (
633+
'template_middle_with_object',
634+
('`object${{1:1}} ${foo}`',
635+
['TEMPLATE_HEAD `object${',
636+
'LBRACE {', 'NUMBER 1', 'COLON :', 'NUMBER 1', 'RBRACE }',
637+
'TEMPLATE_MIDDLE } ${', 'ID foo', 'TEMPLATE_TAIL }`']),
638+
), (
639+
'template_tail_with_object',
640+
('`object${{1:1}}`',
641+
['TEMPLATE_HEAD `object${',
642+
'LBRACE {', 'NUMBER 1', 'COLON :', 'NUMBER 1', 'RBRACE }',
643+
'TEMPLATE_TAIL }`']),
632644
), (
633645
'template_literal_assignment',
634646
('s = `hello world`',
@@ -684,6 +696,15 @@
684696
"var foo = `${foo}bar${baz}fail",
685697
# the specific identifiers are not tracked, thus ...
686698
"Unterminated template literal '`${...}bar${...}...' at 1:11",
699+
), (
700+
'unterminated_template_nested',
701+
"var foo = `${`${foo}bar${baz}fail`}",
702+
# the specific identifiers are not tracked, thus ...
703+
"Unterminated template literal '`${...}' at 1:11",
704+
), (
705+
'unexpected_template_tail',
706+
"var foo = `${value}`}`",
707+
"Unexpected '}' at 1:21",
687708
), (
688709
'invalid_hex_sequence',
689710
"var foo = `fail\\x1`",

‎src/calmjs/parse/tests/parser.py

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2861,6 +2861,24 @@ def parse_to_repr(value):
28612861
>>
28622862
]>
28632863
""",
2864+
), (
2865+
'template_with_many_rbrace',
2866+
"""
2867+
value = `string is ${wat}}}}`
2868+
""",
2869+
"""
2870+
<Program @1:1 ?children=[
2871+
<ExprStatement @1:1 expr=<Assign @1:7 left=<
2872+
Identifier @1:1 value='value'>,
2873+
op='=',
2874+
right=<TemplateLiteral @1:9 ?children=[
2875+
<TemplateHead @1:9 value='`string is ${'>,
2876+
<Identifier @1:22 value='wat'>,
2877+
<TemplateTail @1:25 value='}}}}`'>
2878+
]>
2879+
>>
2880+
]>
2881+
""",
28642882
), (
28652883
'template_in_template',
28662884
"""
@@ -2885,6 +2903,88 @@ def parse_to_repr(value):
28852903
>>
28862904
]>
28872905
""",
2906+
), (
2907+
'template_tail_with_object',
2908+
"""
2909+
value = `object${{1:1}}}`
2910+
""",
2911+
"""
2912+
<Program @1:1 ?children=[
2913+
<ExprStatement @1:1 expr=<Assign @1:7 left=<
2914+
Identifier @1:1 value='value'>, op='=',
2915+
right=<TemplateLiteral @1:9 ?children=[
2916+
<TemplateHead @1:9 value='`object${'>,
2917+
<Object @1:18 properties=[
2918+
<Assign @1:20 left=<Number @1:19 value='1'>,
2919+
op=':',
2920+
right=<Number @1:21 value='1'>>
2921+
]>,
2922+
<TemplateTail @1:23 value='}}`'>
2923+
]>
2924+
>>
2925+
]>
2926+
""",
2927+
), (
2928+
'template_middle_with_object',
2929+
"""
2930+
value = `object${{1:1}}middle${tail}`
2931+
""",
2932+
"""
2933+
<ES2015Program @1:1 ?children=[
2934+
<ExprStatement @1:1 expr=<Assign @1:7 left=<
2935+
Identifier @1:1 value='value'>,
2936+
op='=', right=<TemplateLiteral @1:9 ?children=[
2937+
<TemplateHead @1:9 value='`object${'>,
2938+
<Object @1:18 properties=[
2939+
<Assign @1:20 left=<Number @1:19 value='1'>,
2940+
op=':', right=<Number @1:21 value='1'>>
2941+
]>,
2942+
<TemplateMiddle @1:23 value='}middle${'>,
2943+
<Identifier @1:32 value='tail'>,
2944+
<TemplateTail @1:36 value='}`'>
2945+
]>
2946+
>>
2947+
]>
2948+
""",
2949+
), (
2950+
'template_with_object_with_template_with_object',
2951+
"""
2952+
value = `object
2953+
${{1:`${{
2954+
2:`${{3:3}}`
2955+
}}`}}
2956+
`
2957+
""",
2958+
r"""
2959+
<Program @1:1 ?children=[
2960+
<ExprStatement @1:1 expr=<Assign @1:7 left=<
2961+
Identifier @1:1 value='value'>,
2962+
op='=', right=<TemplateLiteral @1:9 ?children=[
2963+
<TemplateHead @1:9 value='`object\n${'>,
2964+
<Object @2:3 properties=[
2965+
<Assign @2:5 left=<Number @2:4 value='1'>, op=':',
2966+
right=<TemplateLiteral @2:6 ?children=[
2967+
<TemplateHead @2:6 value='`${'>,
2968+
<Object @2:9 properties=[
2969+
<Assign @3:4 left=<Number @3:3 value='2'>, op=':',
2970+
right=<TemplateLiteral @3:5 ?children=[
2971+
<TemplateHead @3:5 value='`${'>,
2972+
<Object @3:8 properties=[
2973+
<Assign @3:10 left=<Number @3:9 value='3'>,
2974+
op=':', right=<Number @3:11 value='3'>>
2975+
]>,
2976+
<TemplateTail @3:13 value='}`'>
2977+
]>>
2978+
]>,
2979+
<TemplateTail @4:2 value='}`'>
2980+
]>
2981+
>
2982+
]>,
2983+
<TemplateTail @4:5 value='}\n`'>
2984+
]>
2985+
>>
2986+
]>
2987+
""",
28882988
)]))
28892989

28902990

@@ -2973,6 +3073,22 @@ def build_es2015_syntax_error_test_cases(clsname, parse):
29733073
'empty_expression_in_template',
29743074
'`head${}tail`',
29753075
"Unexpected '}tail`' at 1:8 after '`head${' at 1:1",
3076+
), (
3077+
'mismatched_template_termination_eof',
3078+
"var foo = `${`${foo}bar${baz}fail`",
3079+
"Unexpected end of input after '}fail`' at 1:29",
3080+
), (
3081+
'mismatched_template_termination',
3082+
"var foo = `head${`${foo}bar${baz}fail`}",
3083+
"Unterminated template literal '`head${...}' at 1:11",
3084+
), (
3085+
'unexpected_block',
3086+
"var foo = `${{11}}`",
3087+
"Unexpected '}' at 1:17 after '11' at 1:15",
3088+
), (
3089+
'object_no_template_keys',
3090+
"var foo = {`foo`: `foo`}",
3091+
"Unexpected '`foo`' at 1:12 between '{' at 1:11 and ':' at 1:17",
29763092
)]), ECMASyntaxError)
29773093

29783094

‎src/calmjs/parse/tests/test_es2015_lexer.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,14 @@ def test_initial_template_character(self):
2929
self.assertEqual(
3030
str(e.exception), "Unterminated template literal '`' at 1:1")
3131

32+
def test_invalid_template_token(self):
33+
lexer = Lexer()
34+
lexer.input('""')
35+
token = lexer.next()
36+
# force an invalid token into the statement
37+
with self.assertRaises(ValueError):
38+
lexer.t_TEMPLATE_RAW(token)
39+
3240

3341
LexerKeywordTestCase = build_equality_testcase(
3442
'LexerTestCase', partial(run_lexer, lexer_cls=Lexer), (

0 commit comments

Comments
 (0)