Skip to content

Commit b6f9605

Browse files
ahejlsbergahejlsberg
authored
Merge pull request #19976 from Microsoft/optimizeParser
Optimize parsing
2 parents f352e46 + ee85178 commit b6f9605

11 files changed

+308
-408
lines changed

‎src/compiler/binder.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1835,7 +1835,7 @@ namespace ts {
18351835
}
18361836

18371837
function checkStrictModeNumericLiteral(node: NumericLiteral) {
1838-
if (inStrictMode && node.numericLiteralFlags & NumericLiteralFlags.Octal) {
1838+
if (inStrictMode && node.numericLiteralFlags & TokenFlags.Octal) {
18391839
file.bindDiagnostics.push(createDiagnosticForNode(node, Diagnostics.Octal_literals_are_not_allowed_in_strict_mode));
18401840
}
18411841
}
@@ -3319,7 +3319,7 @@ namespace ts {
33193319
break;
33203320

33213321
case SyntaxKind.NumericLiteral:
3322-
if ((<NumericLiteral>node).numericLiteralFlags & NumericLiteralFlags.BinaryOrOctalSpecifier) {
3322+
if ((<NumericLiteral>node).numericLiteralFlags & TokenFlags.BinaryOrOctalSpecifier) {
33233323
transformFlags |= TransformFlags.AssertES2015;
33243324
}
33253325
break;

‎src/compiler/checker.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26274,7 +26274,7 @@ namespace ts {
2627426274

2627526275
function checkGrammarNumericLiteral(node: NumericLiteral): boolean {
2627626276
// Grammar checking
26277-
if (node.numericLiteralFlags & NumericLiteralFlags.Octal) {
26277+
if (node.numericLiteralFlags & TokenFlags.Octal) {
2627826278
let diagnosticMessage: DiagnosticMessage | undefined;
2627926279
if (languageVersion >= ScriptTarget.ES5) {
2628026280
diagnosticMessage = Diagnostics.Octal_literals_are_not_available_when_targeting_ECMAScript_5_and_higher_Use_the_syntax_0;

‎src/compiler/parser.ts

Lines changed: 238 additions & 301 deletions
Large diffs are not rendered by default.

‎src/compiler/scanner.ts

Lines changed: 32 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ namespace ts {
2727
isReservedWord(): boolean;
2828
isUnterminated(): boolean;
2929
/* @internal */
30-
getNumericLiteralFlags(): NumericLiteralFlags;
30+
getTokenFlags(): TokenFlags;
3131
reScanGreaterToken(): SyntaxKind;
3232
reScanSlashToken(): SyntaxKind;
3333
reScanTemplateToken(): SyntaxKind;
@@ -815,10 +815,7 @@ namespace ts {
815815

816816
let token: SyntaxKind;
817817
let tokenValue: string;
818-
let precedingLineBreak: boolean;
819-
let hasExtendedUnicodeEscape: boolean;
820-
let tokenIsUnterminated: boolean;
821-
let numericLiteralFlags: NumericLiteralFlags;
818+
let tokenFlags: TokenFlags;
822819

823820
setText(text, start, length);
824821

@@ -829,12 +826,12 @@ namespace ts {
829826
getTokenPos: () => tokenPos,
830827
getTokenText: () => text.substring(tokenPos, pos),
831828
getTokenValue: () => tokenValue,
832-
hasExtendedUnicodeEscape: () => hasExtendedUnicodeEscape,
833-
hasPrecedingLineBreak: () => precedingLineBreak,
829+
hasExtendedUnicodeEscape: () => (tokenFlags & TokenFlags.ExtendedUnicodeEscape) !== 0,
830+
hasPrecedingLineBreak: () => (tokenFlags & TokenFlags.PrecedingLineBreak) !== 0,
834831
isIdentifier: () => token === SyntaxKind.Identifier || token > SyntaxKind.LastReservedWord,
835832
isReservedWord: () => token >= SyntaxKind.FirstReservedWord && token <= SyntaxKind.LastReservedWord,
836-
isUnterminated: () => tokenIsUnterminated,
837-
getNumericLiteralFlags: () => numericLiteralFlags,
833+
isUnterminated: () => (tokenFlags & TokenFlags.Unterminated) !== 0,
834+
getTokenFlags: () => tokenFlags,
838835
reScanGreaterToken,
839836
reScanSlashToken,
840837
reScanTemplateToken,
@@ -871,7 +868,7 @@ namespace ts {
871868
let end = pos;
872869
if (text.charCodeAt(pos) === CharacterCodes.E || text.charCodeAt(pos) === CharacterCodes.e) {
873870
pos++;
874-
numericLiteralFlags = NumericLiteralFlags.Scientific;
871+
tokenFlags |= TokenFlags.Scientific;
875872
if (text.charCodeAt(pos) === CharacterCodes.plus || text.charCodeAt(pos) === CharacterCodes.minus) pos++;
876873
if (isDigit(text.charCodeAt(pos))) {
877874
pos++;
@@ -943,7 +940,7 @@ namespace ts {
943940
while (true) {
944941
if (pos >= end) {
945942
result += text.substring(start, pos);
946-
tokenIsUnterminated = true;
943+
tokenFlags |= TokenFlags.Unterminated;
947944
error(Diagnostics.Unterminated_string_literal);
948945
break;
949946
}
@@ -961,7 +958,7 @@ namespace ts {
961958
}
962959
if (isLineBreak(ch)) {
963960
result += text.substring(start, pos);
964-
tokenIsUnterminated = true;
961+
tokenFlags |= TokenFlags.Unterminated;
965962
error(Diagnostics.Unterminated_string_literal);
966963
break;
967964
}
@@ -985,7 +982,7 @@ namespace ts {
985982
while (true) {
986983
if (pos >= end) {
987984
contents += text.substring(start, pos);
988-
tokenIsUnterminated = true;
985+
tokenFlags |= TokenFlags.Unterminated;
989986
error(Diagnostics.Unterminated_template_literal);
990987
resultingToken = startedWithBacktick ? SyntaxKind.NoSubstitutionTemplateLiteral : SyntaxKind.TemplateTail;
991988
break;
@@ -1071,7 +1068,7 @@ namespace ts {
10711068
case CharacterCodes.u:
10721069
// '\u{DDDDDDDD}'
10731070
if (pos < end && text.charCodeAt(pos) === CharacterCodes.openBrace) {
1074-
hasExtendedUnicodeEscape = true;
1071+
tokenFlags |= TokenFlags.ExtendedUnicodeEscape;
10751072
pos++;
10761073
return scanExtendedUnicodeEscape();
10771074
}
@@ -1240,10 +1237,7 @@ namespace ts {
12401237

12411238
function scan(): SyntaxKind {
12421239
startPos = pos;
1243-
hasExtendedUnicodeEscape = false;
1244-
precedingLineBreak = false;
1245-
tokenIsUnterminated = false;
1246-
numericLiteralFlags = 0;
1240+
tokenFlags = 0;
12471241
while (true) {
12481242
tokenPos = pos;
12491243
if (pos >= end) {
@@ -1265,7 +1259,7 @@ namespace ts {
12651259
switch (ch) {
12661260
case CharacterCodes.lineFeed:
12671261
case CharacterCodes.carriageReturn:
1268-
precedingLineBreak = true;
1262+
tokenFlags |= TokenFlags.PrecedingLineBreak;
12691263
if (skipTrivia) {
12701264
pos++;
12711265
continue;
@@ -1396,6 +1390,9 @@ namespace ts {
13961390
// Multi-line comment
13971391
if (text.charCodeAt(pos + 1) === CharacterCodes.asterisk) {
13981392
pos += 2;
1393+
if (text.charCodeAt(pos) === CharacterCodes.asterisk && text.charCodeAt(pos + 1) !== CharacterCodes.slash) {
1394+
tokenFlags |= TokenFlags.PrecedingJSDocComment;
1395+
}
13991396

14001397
let commentClosed = false;
14011398
while (pos < end) {
@@ -1408,7 +1405,7 @@ namespace ts {
14081405
}
14091406

14101407
if (isLineBreak(ch)) {
1411-
precedingLineBreak = true;
1408+
tokenFlags |= TokenFlags.PrecedingLineBreak;
14121409
}
14131410
pos++;
14141411
}
@@ -1421,7 +1418,9 @@ namespace ts {
14211418
continue;
14221419
}
14231420
else {
1424-
tokenIsUnterminated = !commentClosed;
1421+
if (!commentClosed) {
1422+
tokenFlags |= TokenFlags.Unterminated;
1423+
}
14251424
return token = SyntaxKind.MultiLineCommentTrivia;
14261425
}
14271426
}
@@ -1442,7 +1441,7 @@ namespace ts {
14421441
value = 0;
14431442
}
14441443
tokenValue = "" + value;
1445-
numericLiteralFlags = NumericLiteralFlags.HexSpecifier;
1444+
tokenFlags |= TokenFlags.HexSpecifier;
14461445
return token = SyntaxKind.NumericLiteral;
14471446
}
14481447
else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.B || text.charCodeAt(pos + 1) === CharacterCodes.b)) {
@@ -1453,7 +1452,7 @@ namespace ts {
14531452
value = 0;
14541453
}
14551454
tokenValue = "" + value;
1456-
numericLiteralFlags = NumericLiteralFlags.BinarySpecifier;
1455+
tokenFlags |= TokenFlags.BinarySpecifier;
14571456
return token = SyntaxKind.NumericLiteral;
14581457
}
14591458
else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.O || text.charCodeAt(pos + 1) === CharacterCodes.o)) {
@@ -1464,13 +1463,13 @@ namespace ts {
14641463
value = 0;
14651464
}
14661465
tokenValue = "" + value;
1467-
numericLiteralFlags = NumericLiteralFlags.OctalSpecifier;
1466+
tokenFlags |= TokenFlags.OctalSpecifier;
14681467
return token = SyntaxKind.NumericLiteral;
14691468
}
14701469
// Try to parse as an octal
14711470
if (pos + 1 < end && isOctalDigit(text.charCodeAt(pos + 1))) {
14721471
tokenValue = "" + scanOctalDigits();
1473-
numericLiteralFlags = NumericLiteralFlags.Octal;
1472+
tokenFlags |= TokenFlags.Octal;
14741473
return token = SyntaxKind.NumericLiteral;
14751474
}
14761475
// This fall-through is a deviation from the EcmaScript grammar. The grammar says that a leading zero
@@ -1627,7 +1626,7 @@ namespace ts {
16271626
continue;
16281627
}
16291628
else if (isLineBreak(ch)) {
1630-
precedingLineBreak = true;
1629+
tokenFlags |= TokenFlags.PrecedingLineBreak;
16311630
pos++;
16321631
continue;
16331632
}
@@ -1670,14 +1669,14 @@ namespace ts {
16701669
// If we reach the end of a file, or hit a newline, then this is an unterminated
16711670
// regex. Report error and return what we have so far.
16721671
if (p >= end) {
1673-
tokenIsUnterminated = true;
1672+
tokenFlags |= TokenFlags.Unterminated;
16741673
error(Diagnostics.Unterminated_regular_expression_literal);
16751674
break;
16761675
}
16771676

16781677
const ch = text.charCodeAt(p);
16791678
if (isLineBreak(ch)) {
1680-
tokenIsUnterminated = true;
1679+
tokenFlags |= TokenFlags.Unterminated;
16811680
error(Diagnostics.Unterminated_regular_expression_literal);
16821681
break;
16831682
}
@@ -1895,7 +1894,7 @@ namespace ts {
18951894
const saveTokenPos = tokenPos;
18961895
const saveToken = token;
18971896
const saveTokenValue = tokenValue;
1898-
const savePrecedingLineBreak = precedingLineBreak;
1897+
const saveTokenFlags = tokenFlags;
18991898
const result = callback();
19001899

19011900
// If our callback returned something 'falsy' or we're just looking ahead,
@@ -1906,7 +1905,7 @@ namespace ts {
19061905
tokenPos = saveTokenPos;
19071906
token = saveToken;
19081907
tokenValue = saveTokenValue;
1909-
precedingLineBreak = savePrecedingLineBreak;
1908+
tokenFlags = saveTokenFlags;
19101909
}
19111910
return result;
19121911
}
@@ -1917,10 +1916,8 @@ namespace ts {
19171916
const saveStartPos = startPos;
19181917
const saveTokenPos = tokenPos;
19191918
const saveToken = token;
1920-
const savePrecedingLineBreak = precedingLineBreak;
19211919
const saveTokenValue = tokenValue;
1922-
const saveHasExtendedUnicodeEscape = hasExtendedUnicodeEscape;
1923-
const saveTokenIsUnterminated = tokenIsUnterminated;
1920+
const saveTokenFlags = tokenFlags;
19241921

19251922
setText(text, start, length);
19261923
const result = callback();
@@ -1930,10 +1927,8 @@ namespace ts {
19301927
startPos = saveStartPos;
19311928
tokenPos = saveTokenPos;
19321929
token = saveToken;
1933-
precedingLineBreak = savePrecedingLineBreak;
19341930
tokenValue = saveTokenValue;
1935-
hasExtendedUnicodeEscape = saveHasExtendedUnicodeEscape;
1936-
tokenIsUnterminated = saveTokenIsUnterminated;
1931+
tokenFlags = saveTokenFlags;
19371932

19381933
return result;
19391934
}
@@ -1974,11 +1969,8 @@ namespace ts {
19741969
startPos = textPos;
19751970
tokenPos = textPos;
19761971
token = SyntaxKind.Unknown;
1977-
precedingLineBreak = false;
1978-
19791972
tokenValue = undefined;
1980-
hasExtendedUnicodeEscape = false;
1981-
tokenIsUnterminated = false;
1973+
tokenFlags = 0;
19821974
}
19831975
}
19841976
}

‎src/compiler/transformers/es2015.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3633,7 +3633,7 @@ namespace ts {
36333633
* @param node A string literal.
36343634
*/
36353635
function visitNumericLiteral(node: NumericLiteral) {
3636-
if (node.numericLiteralFlags & NumericLiteralFlags.BinaryOrOctalSpecifier) {
3636+
if (node.numericLiteralFlags & TokenFlags.BinaryOrOctalSpecifier) {
36373637
return setTextRange(createNumericLiteral(node.text), node);
36383638
}
36393639
return node;

‎src/compiler/types.ts

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1474,20 +1474,25 @@ namespace ts {
14741474
}
14751475

14761476
/* @internal */
1477-
export const enum NumericLiteralFlags {
1477+
export const enum TokenFlags {
14781478
None = 0,
1479-
Scientific = 1 << 1, // e.g. `10e2`
1480-
Octal = 1 << 2, // e.g. `0777`
1481-
HexSpecifier = 1 << 3, // e.g. `0x00000000`
1482-
BinarySpecifier = 1 << 4, // e.g. `0b0110010000000000`
1483-
OctalSpecifier = 1 << 5, // e.g. `0o777`
1479+
PrecedingLineBreak = 1 << 0,
1480+
PrecedingJSDocComment = 1 << 1,
1481+
Unterminated = 1 << 2,
1482+
ExtendedUnicodeEscape = 1 << 3,
1483+
Scientific = 1 << 4, // e.g. `10e2`
1484+
Octal = 1 << 5, // e.g. `0777`
1485+
HexSpecifier = 1 << 6, // e.g. `0x00000000`
1486+
BinarySpecifier = 1 << 7, // e.g. `0b0110010000000000`
1487+
OctalSpecifier = 1 << 8, // e.g. `0o777`
14841488
BinaryOrOctalSpecifier = BinarySpecifier | OctalSpecifier,
1489+
NumericLiteralFlags = Scientific | Octal | HexSpecifier | BinarySpecifier | OctalSpecifier
14851490
}
14861491

14871492
export interface NumericLiteral extends LiteralExpression {
14881493
kind: SyntaxKind.NumericLiteral;
14891494
/* @internal */
1490-
numericLiteralFlags?: NumericLiteralFlags;
1495+
numericLiteralFlags?: TokenFlags;
14911496
}
14921497

14931498
export interface TemplateHead extends LiteralLikeNode {

‎tests/baselines/reference/duplicateIdentifierDifferentSpelling.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ var X = { 0b11: '', 3: '' };
1010
//// [duplicateIdentifierDifferentSpelling.js]
1111
var A = /** @class */ (function () {
1212
function A() {
13-
this[0b11] = '';
13+
this[3] = '';
1414
this[3] = '';
1515
}
1616
return A;

‎tests/baselines/reference/parserArrowFunctionExpression6.errors.txt

Lines changed: 0 additions & 37 deletions
This file was deleted.

‎tests/baselines/reference/parserArrowFunctionExpression6.js

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ function foo(q: string, b: number) {
66

77
//// [parserArrowFunctionExpression6.js]
88
function foo(q, b) {
9-
return true ? function (q, , ) { } : ;
10-
;
9+
return true ? (q ? true : false) : (b = q.length, function () { });
1110
}
1211
;

‎tests/baselines/reference/parserArrowFunctionExpression6.symbols

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,11 @@ function foo(q: string, b: number) {
55
>b : Symbol(b, Decl(parserArrowFunctionExpression6.ts, 0, 23))
66

77
return true ? (q ? true : false) : (b = q.length, function() { });
8-
>q : Symbol(q, Decl(parserArrowFunctionExpression6.ts, 1, 19))
9-
> : Symbol((Missing), Decl(parserArrowFunctionExpression6.ts, 1, 22))
10-
> : Symbol((Missing), Decl(parserArrowFunctionExpression6.ts, 1, 29))
11-
>b : Symbol(b, Decl(parserArrowFunctionExpression6.ts, 1, 40))
12-
>q : Symbol(q, Decl(parserArrowFunctionExpression6.ts, 1, 19))
8+
>q : Symbol(q, Decl(parserArrowFunctionExpression6.ts, 0, 13))
9+
>b : Symbol(b, Decl(parserArrowFunctionExpression6.ts, 0, 23))
10+
>q.length : Symbol(String.length, Decl(lib.d.ts, --, --))
11+
>q : Symbol(q, Decl(parserArrowFunctionExpression6.ts, 0, 13))
12+
>length : Symbol(String.length, Decl(lib.d.ts, --, --))
1313

1414
};
1515

0 commit comments

Comments
 (0)