Skip to content

Commit f7d7afd

Browse files
metatoastermetatoaster
committed
Initial implementation of TemplateLiteral rules
- The parser can now parse template literals from tokens produced by the lexer and produce the ast with the new asttypes for templates. - Initial set of test cases included, however there are certain limitations which have been found that needed correction in the lexer itself.
1 parent 58f26eb commit f7d7afd

File tree

4 files changed

+287
-1
lines changed

4 files changed

+287
-1
lines changed

‎src/calmjs/parse/asttypes.py

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,43 @@ def __init__(self, value):
153153
self.value = value
154154

155155

156+
class Template(Node):
157+
"""
158+
All template subclasses.
159+
"""
160+
161+
162+
class TemplateLiteral(Template):
163+
"""
164+
The top level template literal object
165+
"""
166+
167+
168+
class TemplateFragment(Template):
169+
"""
170+
All template fragments
171+
"""
172+
173+
def __init__(self, value):
174+
self.value = value
175+
176+
177+
class TemplateNoSub(TemplateFragment):
178+
pass
179+
180+
181+
class TemplateHead(TemplateFragment):
182+
pass
183+
184+
185+
class TemplateMiddle(TemplateFragment):
186+
pass
187+
188+
189+
class TemplateTail(TemplateFragment):
190+
pass
191+
192+
156193
class Regex(Node):
157194
def __init__(self, value):
158195
self.value = value

‎src/calmjs/parse/parsers/es2015.py

Lines changed: 57 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,6 @@ def p_literal(self, p):
167167
| boolean_literal
168168
| numeric_literal
169169
| string_literal
170-
| regex_literal
171170
"""
172171
p[0] = p[1]
173172

@@ -198,6 +197,61 @@ def p_regex_literal(self, p):
198197
p[0] = self.asttypes.Regex(p[1])
199198
p[0].setpos(p)
200199

200+
# 11.8.6 Template
201+
def p_template_nosub(self, p):
202+
"""template_nosub : TEMPLATE_NOSUB"""
203+
# no_sub_template is called as such here for consistency
204+
p[0] = self.asttypes.TemplateNoSub(p[1])
205+
p[0].setpos(p)
206+
207+
def p_template_head(self, p):
208+
"""template_head : TEMPLATE_HEAD"""
209+
p[0] = self.asttypes.TemplateHead(p[1])
210+
p[0].setpos(p)
211+
212+
def p_template_middle(self, p):
213+
"""template_middle : TEMPLATE_MIDDLE"""
214+
p[0] = self.asttypes.TemplateMiddle(p[1])
215+
p[0].setpos(p)
216+
217+
def p_template_tail(self, p):
218+
"""template_tail : TEMPLATE_TAIL"""
219+
p[0] = self.asttypes.TemplateTail(p[1])
220+
p[0].setpos(p)
221+
222+
def p_template_literal(self, p):
223+
"""template_literal : template_nosub
224+
| template_head expr template_spans
225+
"""
226+
literals = [p[1]]
227+
if len(p) > 2:
228+
# append the expression and extend with template spans
229+
literals.append(p[2])
230+
literals.extend(p[3])
231+
p[0] = self.asttypes.TemplateLiteral(literals)
232+
p[0].setpos(p)
233+
234+
def p_template_spans(self, p):
235+
"""template_spans : template_tail
236+
| template_middle_list template_tail
237+
"""
238+
if len(p) == 2:
239+
p[0] = [p[1]]
240+
else:
241+
p[1].append(p[2])
242+
p[0] = p[1]
243+
244+
def p_template_middle_list(self, p):
245+
"""template_middle_list : template_middle expr
246+
| template_middle_list template_middle \
247+
expr
248+
"""
249+
if len(p) == 3:
250+
p[0] = [p[1], p[2]]
251+
else:
252+
p[1].extend([p[2], p[3]])
253+
p[0] = p[1]
254+
201255
def p_identifier(self, p):
202256
"""identifier : ID"""
203257
p[0] = self.asttypes.Identifier(p[1])
@@ -276,6 +330,8 @@ def p_primary_expr_no_brace_2(self, p):
276330
def p_primary_expr_no_brace_3(self, p):
277331
"""primary_expr_no_brace : literal
278332
| array_literal
333+
| regex_literal
334+
| template_literal
279335
"""
280336
p[0] = p[1]
281337

‎src/calmjs/parse/tests/parser.py

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2589,6 +2589,175 @@ def regenerate(value):
25892589
)]))
25902590

25912591

2592+
def build_es2015_node_repr_test_cases(clsname, parse, program_type):
2593+
2594+
def parse_to_repr(value):
2595+
return repr_walker.walk(parse(value), pos=True)
2596+
2597+
return build_equality_testcase(clsname, parse_to_repr, ((
2598+
label,
2599+
textwrap.dedent(argument).strip(),
2600+
singleline(result).replace('<Program', '<' + program_type),
2601+
) for label, argument, result in [(
2602+
'template_literal_nosub',
2603+
"""
2604+
var t = `some_template`
2605+
""",
2606+
"""
2607+
<Program @1:1 ?children=[
2608+
<VarStatement @1:1 ?children=[
2609+
<VarDecl @1:5 identifier=<Identifier @1:5 value='t'>,
2610+
initializer=<TemplateLiteral @1:9 ?children=[
2611+
<TemplateNoSub @1:9 value='`some_template`'>
2612+
]>
2613+
>
2614+
]>
2615+
]>
2616+
""",
2617+
), (
2618+
'template_literal_sub',
2619+
"""
2620+
var t = `some_template${value}tail`
2621+
""",
2622+
"""
2623+
<Program @1:1 ?children=[
2624+
<VarStatement @1:1 ?children=[
2625+
<VarDecl @1:5 identifier=<Identifier @1:5 value='t'>,
2626+
initializer=<TemplateLiteral @1:9 ?children=[
2627+
<TemplateHead @1:9 value='`some_template${'>,
2628+
<Identifier @1:25 value='value'>,
2629+
<TemplateTail @1:30 value='}tail`'>
2630+
]>
2631+
>
2632+
]>
2633+
]>
2634+
""",
2635+
), (
2636+
'template_literal_sub_once',
2637+
"""
2638+
var t = `some_template${value}middle${value}tail`
2639+
""",
2640+
"""
2641+
<Program @1:1 ?children=[
2642+
<VarStatement @1:1 ?children=[
2643+
<VarDecl @1:5 identifier=<Identifier @1:5 value='t'>,
2644+
initializer=<TemplateLiteral @1:9 ?children=[
2645+
<TemplateHead @1:9 value='`some_template${'>,
2646+
<Identifier @1:25 value='value'>,
2647+
<TemplateMiddle @1:30 value='}middle${'>,
2648+
<Identifier @1:39 value='value'>,
2649+
<TemplateTail @1:44 value='}tail`'>
2650+
]>
2651+
>
2652+
]>
2653+
]>
2654+
""",
2655+
), (
2656+
'template_literal_sub_multiple',
2657+
"""
2658+
var t = `some_template${value}middle${value}another${tail}tail`
2659+
""",
2660+
"""
2661+
<Program @1:1 ?children=[
2662+
<VarStatement @1:1 ?children=[
2663+
<VarDecl @1:5 identifier=<Identifier @1:5 value='t'>,
2664+
initializer=<TemplateLiteral @1:9 ?children=[
2665+
<TemplateHead @1:9 value='`some_template${'>,
2666+
<Identifier @1:25 value='value'>,
2667+
<TemplateMiddle @1:30 value='}middle${'>,
2668+
<Identifier @1:39 value='value'>,
2669+
<TemplateMiddle @1:44 value='}another${'>,
2670+
<Identifier @1:54 value='tail'>,
2671+
<TemplateTail @1:58 value='}tail`'>
2672+
]>
2673+
>
2674+
]>
2675+
]>
2676+
""",
2677+
), (
2678+
'template_multiline_between_expressions',
2679+
"""
2680+
t = `tt${
2681+
s + s
2682+
}ttttt`
2683+
""",
2684+
"""
2685+
<Program @1:1 ?children=[
2686+
<ExprStatement @1:1 expr=<Assign @1:3 left=<
2687+
Identifier @1:1 value='t'>,
2688+
op='=',
2689+
right=<TemplateLiteral @1:5 ?children=[
2690+
<TemplateHead @1:5 value='`tt${'>,
2691+
<BinOp @2:3 left=<Identifier @2:1 value='s'>,
2692+
op='+', right=<Identifier @2:5 value='s'>>,
2693+
<TemplateTail @3:1 value='}ttttt`'>
2694+
]>
2695+
>>
2696+
]>
2697+
""",
2698+
), (
2699+
'template_with_regex',
2700+
"""
2701+
value = `regex is ${/wat/}`
2702+
""",
2703+
"""
2704+
<Program @1:1 ?children=[
2705+
<ExprStatement @1:1 expr=<Assign @1:7 left=<
2706+
Identifier @1:1 value='value'>,
2707+
op='=',
2708+
right=<TemplateLiteral @1:9 ?children=[
2709+
<TemplateHead @1:9 value='`regex is ${'>,
2710+
<Regex @1:21 value='/wat/'>,
2711+
<TemplateTail @1:26 value='}`'>
2712+
]>
2713+
>>
2714+
]>
2715+
""",
2716+
), (
2717+
'template_with_string',
2718+
"""
2719+
value = `string is ${'wat'}`
2720+
""",
2721+
"""
2722+
<Program @1:1 ?children=[
2723+
<ExprStatement @1:1 expr=<Assign @1:7 left=<
2724+
Identifier @1:1 value='value'>,
2725+
op='=',
2726+
right=<TemplateLiteral @1:9 ?children=[
2727+
<TemplateHead @1:9 value='`string is ${'>,
2728+
<String @1:22 value="'wat'">,
2729+
<TemplateTail @1:27 value='}`'>
2730+
]>
2731+
>>
2732+
]>
2733+
""",
2734+
), (
2735+
'template_in_template',
2736+
"""
2737+
value = `template embed ${`another${`template`}`} inside`
2738+
""",
2739+
"""
2740+
<Program @1:1 ?children=[
2741+
<ExprStatement @1:1 expr=<Assign @1:7 left=<
2742+
Identifier @1:1 value='value'>,
2743+
op='=',
2744+
right=<TemplateLiteral @1:9 ?children=[
2745+
<TemplateHead @1:9 value='`template embed ${'>,
2746+
<TemplateLiteral @1:27 ?children=[
2747+
<TemplateHead @1:27 value='`another${'>,
2748+
<TemplateLiteral @1:37 ?children=[
2749+
<TemplateNoSub @1:37 value='`template`'>
2750+
]>,
2751+
<TemplateTail @1:47 value='}`'>
2752+
]>,
2753+
<TemplateTail @1:49 value='} inside`'>
2754+
]>
2755+
>>
2756+
]>
2757+
""",
2758+
)]))
2759+
2760+
25922761
def build_syntax_error_test_cases(clsname, parse):
25932762
return build_exception_testcase(clsname, parse, ((
25942763
label,
@@ -2639,6 +2808,22 @@ def build_syntax_error_test_cases(clsname, parse):
26392808
)]), ECMASyntaxError)
26402809

26412810

2811+
def build_es2015_syntax_error_test_cases(clsname, parse):
2812+
return build_exception_testcase(clsname, parse, ((
2813+
label,
2814+
textwrap.dedent(argument).strip(),
2815+
msg,
2816+
) for label, argument, msg in [(
2817+
'unexpected_if_in_template',
2818+
'`${if (wat)}`',
2819+
"Unexpected 'if' at 1:4 between '`${' at 1:1 and '(' at 1:7",
2820+
), (
2821+
'empty_expression_in_template',
2822+
'`head${}tail`',
2823+
"Unexpected '}tail`' at 1:8 after '`head${' at 1:1",
2824+
)]), ECMASyntaxError)
2825+
2826+
26422827
def build_regex_syntax_error_test_cases(clsname, parse):
26432828
return build_exception_testcase(clsname, parse, ((
26442829
label,

‎src/calmjs/parse/tests/test_es2015_parser.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,10 @@
1515
from calmjs.parse.tests.parser import (
1616
ParserCaseMixin,
1717
build_node_repr_test_cases,
18+
build_es2015_node_repr_test_cases,
1819
# build_asi_test_cases,
1920
build_syntax_error_test_cases,
21+
build_es2015_syntax_error_test_cases,
2022
build_regex_syntax_error_test_cases,
2123
)
2224

@@ -60,12 +62,18 @@ def test_read(self):
6062
ParsedNodeTypeTestCase = build_node_repr_test_cases(
6163
'ParsedNodeTypeTestCase', parse, 'ES2015Program')
6264

65+
ParsedES2015NodeTypeTestCase = build_es2015_node_repr_test_cases(
66+
'ParsedES2015NodeTypeTestCase', parse, 'ES2015Program')
67+
6368
# ASI - Automatic Semicolon Insertion
6469
# ParserToECMAASITestCase = build_asi_test_cases(
6570
# 'ParserToECMAASITestCase', parse, pretty_print)
6671

6772
ECMASyntaxErrorsTestCase = build_syntax_error_test_cases(
6873
'ECMASyntaxErrorsTestCase', parse)
6974

75+
ECMA2015SyntaxErrorsTestCase = build_es2015_syntax_error_test_cases(
76+
'ECMA2015SyntaxErrorsTestCase', parse)
77+
7078
ECMARegexSyntaxErrorsTestCase = build_regex_syntax_error_test_cases(
7179
'ECMARegexSyntaxErrorsTestCase', parse)

0 commit comments

Comments
 (0)