Skip to content
This repository was archived by the owner on Dec 15, 2022. It is now read-only.

Commit a6c28e8

Browse files
MaximSokolovMaximSokolov
committed
Tokenize illegal identifiers
1 parent 58c6a2f commit a6c28e8

File tree

2 files changed

+69
-29
lines changed

2 files changed

+69
-29
lines changed

‎grammars/javascript.cson‎

Lines changed: 33 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -858,15 +858,19 @@
858858
{
859859
'include': '#function_call'
860860
}
861-
{
862-
'include': '#properties'
863-
}
864861
{
865862
'include': '#object_variable'
866863
}
867864
{
868865
'include': '#numbers'
869866
}
867+
{
868+
'include': '#properties'
869+
}
870+
{
871+
'match': '(?<!\\$)\\b[0-9]+[\\w$]*'
872+
'name': 'invalid.illegal.identifier.js'
873+
}
870874
{
871875
'match': '\\;'
872876
'name': 'punctuation.terminator.statement.js'
@@ -918,35 +922,42 @@
918922
'numbers':
919923
'patterns': [
920924
{
921-
'match': '\\b(?<!\\$|_)0(x|X)[0-9a-fA-F]+'
925+
'match': '\\b(?<!\\$)0(x|X)[0-9a-fA-F]+\\b(?!\\$)'
922926
'name': 'constant.numeric.hex.js'
923927
}
924928
{
925-
'match': '\\b(?<!\\$|_)0(b|B)[01]+'
929+
'match': '\\b(?<!\\$)0(b|B)[01]+\\b(?!\\$)'
926930
'name': 'constant.numeric.binary.js'
927931
}
928932
{
929-
'match': '\\b(?<!\\$|_)0(o|O)?[0-7]+'
933+
'match': '\\b(?<!\\$)0(o|O)?[0-7]+\\b(?!\\$)'
930934
'name': 'constant.numeric.octal.js'
931935
}
932936
{
933937
'match': '''(?x)
934-
(?<!\\$|_) (?:
935-
(?:
936-
(?:\\b[0-9]+(\\.)(?:[0-9]+\\b)?)|
937-
(?:(\\.)[0-9]+)|
938-
(?:\\b[0-9]+)
939-
)
940-
([eE][+-]?[0-9]+)?
941-
)
938+
(?<!\\$)(?:
939+
(?:\\b[0-9]+(\\.)[0-9]+[eE][+-]?[0-9]+\\b)| # 1.1E+3
940+
(?:\\B(\\.)[0-9]+[eE][+-]?[0-9]+\\b)| # .1E+3
941+
(?:\\b[0-9]+[eE][+-]?[0-9]+\\b)| # 1E+3
942+
(?:\\b[0-9]+(\\.)[0-9]+\\b)| # 1.1
943+
(?:\\b[0-9]+(\\.)\\B)| # 1.
944+
(?:\\B(\\.)[0-9]+\\b)| # .1
945+
(?:\\b[0-9]+\\b) # 1
946+
)(?!\\$)
942947
'''
943948
'captures':
944949
'0':
945950
'name': 'constant.numeric.decimal.js'
946951
'1':
947-
'name': 'meta.delimiter.decimal-mark.period.js'
952+
'name': 'meta.delimiter.decimal.period.js'
948953
'2':
949-
'name': 'meta.delimiter.decimal-mark.period.js'
954+
'name': 'meta.delimiter.decimal.period.js'
955+
'3':
956+
'name': 'meta.delimiter.decimal.period.js'
957+
'4':
958+
'name': 'meta.delimiter.decimal.period.js'
959+
'5':
960+
'name': 'meta.delimiter.decimal.period.js'
950961
}
951962
]
952963
'operators':
@@ -1205,7 +1216,7 @@
12051216
'begin': '(?:(\\d\\w*)|([a-zA-Z_$][\\w$]*))\\s*(\\()'
12061217
'beginCaptures':
12071218
'1':
1208-
'name': 'invalid.illegal.js'
1219+
'name': 'invalid.illegal.identifier.js'
12091220
'2':
12101221
'name': 'entity.name.function.js'
12111222
'3':
@@ -1231,20 +1242,22 @@
12311242
'1':
12321243
'name': 'variable.other.object.js'
12331244
'2':
1234-
'name': 'invalid.illegal.js'
1245+
'name': 'invalid.illegal.identifier.js'
12351246
}
12361247
]
12371248
'properties':
12381249
'patterns': [
12391250
{
12401251
'comment': '`prop` in `obj.prop`, `func().prop`'
1241-
'match': '(\\.)\\s*(?:([A-Z][A-Z0-9_$]*\\b\\$*)|(\\$*[a-zA-Z_$][\\w$]*))'
1252+
'match': '(\\.)\\s*(?:([0-9]+[\\w$]*)|([A-Z][A-Z0-9_$]*\\b\\$*)|(\\$*[a-zA-Z_$][\\w$]*))'
12421253
'captures':
12431254
'1':
12441255
'name': 'meta.delimiter.property.period.js'
12451256
'2':
1246-
'name': 'constant.other.property.js'
1257+
'name': 'invalid.illegal.identifier.js'
12471258
'3':
1259+
'name': 'constant.other.property.js'
1260+
'4':
12481261
'name': 'variable.other.property.js'
12491262
}
12501263
]

‎spec/javascript-spec.coffee‎

Lines changed: 36 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -213,19 +213,19 @@ describe "Javascript grammar", ->
213213

214214
{tokens} = grammar.tokenizeLine('9.')
215215
expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']
216-
expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal-mark.period.js']
216+
expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js']
217217

218218
{tokens} = grammar.tokenizeLine('.9')
219-
expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal-mark.period.js']
219+
expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js']
220220
expect(tokens[1]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']
221221

222222
{tokens} = grammar.tokenizeLine('9.9')
223223
expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']
224-
expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal-mark.period.js']
224+
expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js']
225225
expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']
226226

227227
{tokens} = grammar.tokenizeLine('.1e-23')
228-
expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal-mark.period.js']
228+
expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js']
229229
expect(tokens[1]).toEqual value: '1e-23', scopes: ['source.js', 'constant.numeric.decimal.js']
230230

231231
it "does not tokenize numbers that are part of a variable", ->
@@ -1050,6 +1050,16 @@ describe "Javascript grammar", ->
10501050
{tokens} = grammar.tokenizeLine('super')
10511051
expect(tokens[0]).toEqual value: 'super', scopes: ['source.js', 'variable.language.js']
10521052

1053+
it "tokenizes illegal identifiers", ->
1054+
{tokens} = grammar.tokenizeLine('0illegal')
1055+
expect(tokens[0]).toEqual value: '0illegal', scopes: ['source.js', 'invalid.illegal.identifier.js']
1056+
1057+
{tokens} = grammar.tokenizeLine('123illegal')
1058+
expect(tokens[0]).toEqual value: '123illegal', scopes: ['source.js', 'invalid.illegal.identifier.js']
1059+
1060+
{tokens} = grammar.tokenizeLine('123$illegal')
1061+
expect(tokens[0]).toEqual value: '123$illegal', scopes: ['source.js', 'invalid.illegal.identifier.js']
1062+
10531063
describe "objects", ->
10541064
it "tokenizes them", ->
10551065
{tokens} = grammar.tokenizeLine('obj.prop')
@@ -1063,13 +1073,13 @@ describe "Javascript grammar", ->
10631073

10641074
it "tokenizes illegal objects", ->
10651075
{tokens} = grammar.tokenizeLine('1.prop')
1066-
expect(tokens[0]).toEqual value: '1', scopes: ['source.js', 'invalid.illegal.js']
1076+
expect(tokens[0]).toEqual value: '1', scopes: ['source.js', 'invalid.illegal.identifier.js']
10671077

10681078
{tokens} = grammar.tokenizeLine('123.prop')
1069-
expect(tokens[0]).toEqual value: '123', scopes: ['source.js', 'invalid.illegal.js']
1079+
expect(tokens[0]).toEqual value: '123', scopes: ['source.js', 'invalid.illegal.identifier.js']
10701080

10711081
{tokens} = grammar.tokenizeLine('123a.prop')
1072-
expect(tokens[0]).toEqual value: '123a', scopes: ['source.js', 'invalid.illegal.js']
1082+
expect(tokens[0]).toEqual value: '123a', scopes: ['source.js', 'invalid.illegal.identifier.js']
10731083

10741084
describe "function calls", ->
10751085
it "tokenizes function calls", ->
@@ -1119,15 +1129,27 @@ describe "Javascript grammar", ->
11191129

11201130
it "tokenizes illegal function calls", ->
11211131
{tokens} = grammar.tokenizeLine('0illegal()')
1122-
expect(tokens[0]).toEqual value: '0illegal', scopes: ['source.js', 'meta.function-call.js', 'invalid.illegal.js']
1132+
expect(tokens[0]).toEqual value: '0illegal', scopes: ['source.js', 'meta.function-call.js', 'invalid.illegal.identifier.js']
11231133
expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.begin.js']
11241134
expect(tokens[2]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js']
11251135

11261136
it "tokenizes illegal arguments", ->
1137+
{tokens} = grammar.tokenizeLine('a(1a)')
1138+
expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js']
1139+
expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.begin.js']
1140+
expect(tokens[2]).toEqual value: '1a', scopes: ['source.js', 'meta.function-call.js', 'invalid.illegal.identifier.js']
1141+
expect(tokens[3]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js']
1142+
1143+
{tokens} = grammar.tokenizeLine('a(123a)')
1144+
expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js']
1145+
expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.begin.js']
1146+
expect(tokens[2]).toEqual value: '123a', scopes: ['source.js', 'meta.function-call.js', 'invalid.illegal.identifier.js']
1147+
expect(tokens[3]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js']
1148+
11271149
{tokens} = grammar.tokenizeLine('a(1.prop)')
11281150
expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js']
11291151
expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.begin.js']
1130-
expect(tokens[2]).toEqual value: '1', scopes: ['source.js', 'meta.function-call.js', 'invalid.illegal.js']
1152+
expect(tokens[2]).toEqual value: '1', scopes: ['source.js', 'meta.function-call.js', 'invalid.illegal.identifier.js']
11311153
expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'meta.function-call.js', 'meta.delimiter.property.period.js']
11321154
expect(tokens[4]).toEqual value: 'prop', scopes: ['source.js', 'meta.function-call.js', 'variable.other.property.js']
11331155
expect(tokens[5]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js']
@@ -1224,6 +1246,11 @@ describe "Javascript grammar", ->
12241246
expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js']
12251247
expect(tokens[4]).toEqual value: 'b', scopes: ['source.js', 'variable.other.property.js']
12261248

1249+
{tokens} = grammar.tokenizeLine('a.123illegal')
1250+
expect(tokens[0]).toEqual value: 'a', scopes: ['source.js']
1251+
expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js']
1252+
expect(tokens[2]).toEqual value: '123illegal', scopes: ['source.js', 'invalid.illegal.identifier.js']
1253+
12271254
it "tokenizes constant properties", ->
12281255
{tokens} = grammar.tokenizeLine('obj.MY_CONSTANT')
12291256
expect(tokens[0]).toEqual value: 'obj', scopes: ['source.js', 'variable.other.object.js']

0 commit comments

Comments
 (0)