Skip to content

Commit df749a7

Browse files
ariyaariya
committed
Make the pure tokenizer a bit aggressive in recognizing regex literals.
At the same time, ensure that it can fall back if the regex literal proved to be invalid. Fix jquery#1772 Fix jquery#1873 Close jquerygh-1889
1 parent 512ce40 commit df749a7

12 files changed

+434
-19
lines changed

‎src/tokenizer.ts

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { ErrorHandler } from './error-handler';
2-
import { Comment, Scanner, SourceLocation } from './scanner';
2+
import { Comment, RawToken, Scanner, SourceLocation } from './scanner';
33
import { Token, TokenName } from './token';
44

55
type ReaderEntry = string | null;
@@ -58,7 +58,7 @@ class Reader {
5858
case '}':
5959
// Dividing a function by anything makes little sense,
6060
// but we have to check for that.
61-
regex = false;
61+
regex = true;
6262
if (this.values[this.curly - 3] === 'function') {
6363
// Anonymous function, e.g. function(){} /42
6464
const check = this.values[this.curly - 4];
@@ -160,10 +160,21 @@ export class Tokenizer {
160160
};
161161
}
162162

163-
const startRegex = (this.scanner.source[this.scanner.index] === '/') && this.reader.isRegexStart();
164-
const token = startRegex ? this.scanner.scanRegExp() : this.scanner.lex();
165-
this.reader.push(token);
163+
const maybeRegex = (this.scanner.source[this.scanner.index] === '/') && this.reader.isRegexStart();
164+
let token: RawToken;
165+
if (maybeRegex) {
166+
const state = this.scanner.saveState();
167+
try {
168+
token = this.scanner.scanRegExp();
169+
} catch (e) {
170+
this.scanner.restoreState(state);
171+
token = this.scanner.lex();
172+
}
173+
} else {
174+
token = this.scanner.lex();
175+
}
166176

177+
this.reader.push(token);
167178
const entry: BufferEntry = {
168179
type: TokenName[token.type],
169180
value: this.scanner.source.slice(token.start, token.end)

‎test/fixtures/tokenize/after_if.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
if(x){} /y/.test(z)

‎test/fixtures/tokenize/after_if.tokens.json

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
[
2+
{
3+
"type": "Keyword",
4+
"value": "if",
5+
"range": [
6+
0,
7+
2
8+
],
9+
"loc": {
10+
"start": {
11+
"line": 1,
12+
"column": 0
13+
},
14+
"end": {
15+
"line": 1,
16+
"column": 2
17+
}
18+
}
19+
},
20+
{
21+
"type": "Punctuator",
22+
"value": "(",
23+
"range": [
24+
2,
25+
3
26+
],
27+
"loc": {
28+
"start": {
29+
"line": 1,
30+
"column": 2
31+
},
32+
"end": {
33+
"line": 1,
34+
"column": 3
35+
}
36+
}
37+
},
38+
{
39+
"type": "Identifier",
40+
"value": "x",
41+
"range": [
42+
3,
43+
4
44+
],
45+
"loc": {
46+
"start": {
47+
"line": 1,
48+
"column": 3
49+
},
50+
"end": {
51+
"line": 1,
52+
"column": 4
53+
}
54+
}
55+
},
56+
{
57+
"type": "Punctuator",
58+
"value": ")",
59+
"range": [
60+
4,
61+
5
62+
],
63+
"loc": {
64+
"start": {
65+
"line": 1,
66+
"column": 4
67+
},
68+
"end": {
69+
"line": 1,
70+
"column": 5
71+
}
72+
}
73+
},
74+
{
75+
"type": "Punctuator",
76+
"value": "{",
77+
"range": [
78+
5,
79+
6
80+
],
81+
"loc": {
82+
"start": {
83+
"line": 1,
84+
"column": 5
85+
},
86+
"end": {
87+
"line": 1,
88+
"column": 6
89+
}
90+
}
91+
},
92+
{
93+
"type": "Punctuator",
94+
"value": "}",
95+
"range": [
96+
6,
97+
7
98+
],
99+
"loc": {
100+
"start": {
101+
"line": 1,
102+
"column": 6
103+
},
104+
"end": {
105+
"line": 1,
106+
"column": 7
107+
}
108+
}
109+
},
110+
{
111+
"type": "RegularExpression",
112+
"value": "/y/",
113+
"range": [
114+
8,
115+
11
116+
],
117+
"loc": {
118+
"start": {
119+
"line": 1,
120+
"column": 8
121+
},
122+
"end": {
123+
"line": 1,
124+
"column": 11
125+
}
126+
},
127+
"regex": {
128+
"pattern": "y",
129+
"flags": ""
130+
}
131+
},
132+
{
133+
"type": "Punctuator",
134+
"value": ".",
135+
"range": [
136+
11,
137+
12
138+
],
139+
"loc": {
140+
"start": {
141+
"line": 1,
142+
"column": 11
143+
},
144+
"end": {
145+
"line": 1,
146+
"column": 12
147+
}
148+
}
149+
},
150+
{
151+
"type": "Identifier",
152+
"value": "test",
153+
"range": [
154+
12,
155+
16
156+
],
157+
"loc": {
158+
"start": {
159+
"line": 1,
160+
"column": 12
161+
},
162+
"end": {
163+
"line": 1,
164+
"column": 16
165+
}
166+
}
167+
},
168+
{
169+
"type": "Punctuator",
170+
"value": "(",
171+
"range": [
172+
16,
173+
17
174+
],
175+
"loc": {
176+
"start": {
177+
"line": 1,
178+
"column": 16
179+
},
180+
"end": {
181+
"line": 1,
182+
"column": 17
183+
}
184+
}
185+
},
186+
{
187+
"type": "Identifier",
188+
"value": "z",
189+
"range": [
190+
17,
191+
18
192+
],
193+
"loc": {
194+
"start": {
195+
"line": 1,
196+
"column": 17
197+
},
198+
"end": {
199+
"line": 1,
200+
"column": 18
201+
}
202+
}
203+
},
204+
{
205+
"type": "Punctuator",
206+
"value": ")",
207+
"range": [
208+
18,
209+
19
210+
],
211+
"loc": {
212+
"start": {
213+
"line": 1,
214+
"column": 18
215+
},
216+
"end": {
217+
"line": 1,
218+
"column": 19
219+
}
220+
}
221+
}
222+
]

‎test/fixtures/tokenize/invalid_number.failure.json

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"tokenize": true,
3+
"index": 2,
4+
"lineNumber": 1,
5+
"column": 3,
6+
"message": "Error: Line 1: Unexpected token ILLEGAL"
7+
}

‎test/fixtures/tokenize/invalid_number.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
1e

‎test/fixtures/tokenize/migrated_0011.tokens.json

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,5 +34,59 @@
3434
"column": 4
3535
}
3636
}
37+
},
38+
{
39+
"type": "Punctuator",
40+
"value": "/",
41+
"range": [
42+
4,
43+
5
44+
],
45+
"loc": {
46+
"start": {
47+
"line": 1,
48+
"column": 4
49+
},
50+
"end": {
51+
"line": 1,
52+
"column": 5
53+
}
54+
}
55+
},
56+
{
57+
"type": "Numeric",
58+
"value": "42",
59+
"range": [
60+
5,
61+
7
62+
],
63+
"loc": {
64+
"start": {
65+
"line": 1,
66+
"column": 5
67+
},
68+
"end": {
69+
"line": 1,
70+
"column": 7
71+
}
72+
}
73+
},
74+
{
75+
"type": "Punctuator",
76+
"value": "]",
77+
"range": [
78+
7,
79+
8
80+
],
81+
"loc": {
82+
"start": {
83+
"line": 1,
84+
"column": 7
85+
},
86+
"end": {
87+
"line": 1,
88+
"column": 8
89+
}
90+
}
3791
}
3892
]

‎test/fixtures/tokenize/migrated_0014.failure.json

Lines changed: 0 additions & 7 deletions
This file was deleted.

‎test/fixtures/tokenize/migrated_0014.tokens.json

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
[
2+
{
3+
"type": "Punctuator",
4+
"value": "/",
5+
"range": [
6+
0,
7+
1
8+
],
9+
"loc": {
10+
"start": {
11+
"line": 1,
12+
"column": 0
13+
},
14+
"end": {
15+
"line": 1,
16+
"column": 1
17+
}
18+
}
19+
},
20+
{
21+
"type": "Numeric",
22+
"value": "42",
23+
"range": [
24+
1,
25+
3
26+
],
27+
"loc": {
28+
"start": {
29+
"line": 1,
30+
"column": 1
31+
},
32+
"end": {
33+
"line": 1,
34+
"column": 3
35+
}
36+
}
37+
}
38+
]

‎test/fixtures/tokenize/migrated_0015.failure.json

Lines changed: 0 additions & 7 deletions
This file was deleted.

0 commit comments

Comments
 (0)