Skip to content
This repository was archived by the owner on Dec 15, 2022. It is now read-only.

Commit 58c6a2f

Browse files
MaximSokolovMaximSokolov
committed
Tokenize decimal marks
1 parent da8fa4a commit 58c6a2f

File tree

2 files changed

+69
-34
lines changed

2 files changed

+69
-34
lines changed

‎grammars/javascript.cson‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -918,8 +918,35 @@
918918
'numbers':
919919
'patterns': [
920920
{
921-
'match': '\\b(?<!\\$|_)((0(x|X)[0-9a-fA-F]+)|(0(b|B)[01]+)|(0(o|O)[0-7]+)|([0-9]+(\\.[0-9]+)?)((e|E)[+-]?[0-9]+)?)\\b'
922-
'name': 'constant.numeric.js'
921+
'match': '\\b(?<!\\$|_)0(x|X)[0-9a-fA-F]+'
922+
'name': 'constant.numeric.hex.js'
923+
}
924+
{
925+
'match': '\\b(?<!\\$|_)0(b|B)[01]+'
926+
'name': 'constant.numeric.binary.js'
927+
}
928+
{
929+
'match': '\\b(?<!\\$|_)0(o|O)?[0-7]+'
930+
'name': 'constant.numeric.octal.js'
931+
}
932+
{
933+
'match': '''(?x)
934+
(?<!\\$|_) (?:
935+
(?:
936+
(?:\\b[0-9]+(\\.)(?:[0-9]+\\b)?)|
937+
(?:(\\.)[0-9]+)|
938+
(?:\\b[0-9]+)
939+
)
940+
([eE][+-]?[0-9]+)?
941+
)
942+
'''
943+
'captures':
944+
'0':
945+
'name': 'constant.numeric.decimal.js'
946+
'1':
947+
'name': 'meta.delimiter.decimal-mark.period.js'
948+
'2':
949+
'name': 'meta.delimiter.decimal-mark.period.js'
923950
}
924951
]
925952
'operators':

‎spec/javascript-spec.coffee‎

Lines changed: 40 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ describe "Javascript grammar", ->
146146

147147
{tokens} = grammar.tokenizeLine('[1, /test/]')
148148
expect(tokens[0]).toEqual value: '[', scopes: ['source.js', 'meta.brace.square.js']
149-
expect(tokens[1]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.js']
149+
expect(tokens[1]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js']
150150
expect(tokens[2]).toEqual value: ',', scopes: ['source.js', 'meta.delimiter.object.comma.js']
151151
expect(tokens[3]).toEqual value: ' ', scopes: ['source.js', 'string.regexp.js']
152152
expect(tokens[4]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js']
@@ -179,34 +179,54 @@ describe "Javascript grammar", ->
179179
describe "numbers", ->
180180
it "tokenizes hexadecimals", ->
181181
{tokens} = grammar.tokenizeLine('0x1D306')
182-
expect(tokens[0]).toEqual value: '0x1D306', scopes: ['source.js', 'constant.numeric.js']
182+
expect(tokens[0]).toEqual value: '0x1D306', scopes: ['source.js', 'constant.numeric.hex.js']
183183

184184
{tokens} = grammar.tokenizeLine('0X1D306')
185-
expect(tokens[0]).toEqual value: '0X1D306', scopes: ['source.js', 'constant.numeric.js']
185+
expect(tokens[0]).toEqual value: '0X1D306', scopes: ['source.js', 'constant.numeric.hex.js']
186186

187187
it "tokenizes binary literals", ->
188188
{tokens} = grammar.tokenizeLine('0b011101110111010001100110')
189-
expect(tokens[0]).toEqual value: '0b011101110111010001100110', scopes: ['source.js', 'constant.numeric.js']
189+
expect(tokens[0]).toEqual value: '0b011101110111010001100110', scopes: ['source.js', 'constant.numeric.binary.js']
190190

191191
{tokens} = grammar.tokenizeLine('0B011101110111010001100110')
192-
expect(tokens[0]).toEqual value: '0B011101110111010001100110', scopes: ['source.js', 'constant.numeric.js']
192+
expect(tokens[0]).toEqual value: '0B011101110111010001100110', scopes: ['source.js', 'constant.numeric.binary.js']
193193

194194
it "tokenizes octal literals", ->
195195
{tokens} = grammar.tokenizeLine('0o1411')
196-
expect(tokens[0]).toEqual value: '0o1411', scopes: ['source.js', 'constant.numeric.js']
196+
expect(tokens[0]).toEqual value: '0o1411', scopes: ['source.js', 'constant.numeric.octal.js']
197197

198198
{tokens} = grammar.tokenizeLine('0O1411')
199-
expect(tokens[0]).toEqual value: '0O1411', scopes: ['source.js', 'constant.numeric.js']
199+
expect(tokens[0]).toEqual value: '0O1411', scopes: ['source.js', 'constant.numeric.octal.js']
200+
201+
{tokens} = grammar.tokenizeLine('0010')
202+
expect(tokens[0]).toEqual value: '0010', scopes: ['source.js', 'constant.numeric.octal.js']
200203

201204
it "tokenizes decimals", ->
202205
{tokens} = grammar.tokenizeLine('1234')
203-
expect(tokens[0]).toEqual value: '1234', scopes: ['source.js', 'constant.numeric.js']
206+
expect(tokens[0]).toEqual value: '1234', scopes: ['source.js', 'constant.numeric.decimal.js']
204207

205208
{tokens} = grammar.tokenizeLine('5e-10')
206-
expect(tokens[0]).toEqual value: '5e-10', scopes: ['source.js', 'constant.numeric.js']
209+
expect(tokens[0]).toEqual value: '5e-10', scopes: ['source.js', 'constant.numeric.decimal.js']
207210

208211
{tokens} = grammar.tokenizeLine('5E+5')
209-
expect(tokens[0]).toEqual value: '5E+5', scopes: ['source.js', 'constant.numeric.js']
212+
expect(tokens[0]).toEqual value: '5E+5', scopes: ['source.js', 'constant.numeric.decimal.js']
213+
214+
{tokens} = grammar.tokenizeLine('9.')
215+
expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']
216+
expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal-mark.period.js']
217+
218+
{tokens} = grammar.tokenizeLine('.9')
219+
expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal-mark.period.js']
220+
expect(tokens[1]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']
221+
222+
{tokens} = grammar.tokenizeLine('9.9')
223+
expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']
224+
expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal-mark.period.js']
225+
expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']
226+
227+
{tokens} = grammar.tokenizeLine('.1e-23')
228+
expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal-mark.period.js']
229+
expect(tokens[1]).toEqual value: '1e-23', scopes: ['source.js', 'constant.numeric.decimal.js']
210230

211231
it "does not tokenize numbers that are part of a variable", ->
212232
{tokens} = grammar.tokenizeLine('hi$1')
@@ -300,9 +320,9 @@ describe "Javascript grammar", ->
300320
it "tokenizes the arithmetic operators when separated by newlines", ->
301321
for operator in operators
302322
lines = grammar.tokenizeLines '1\n' + operator + ' 2'
303-
expect(lines[0][0]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.js']
323+
expect(lines[0][0]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js']
304324
expect(lines[1][0]).toEqual value: operator, scopes: ['source.js', 'keyword.operator.js']
305-
expect(lines[1][2]).toEqual value: '2', scopes: ['source.js', 'constant.numeric.js']
325+
expect(lines[1][2]).toEqual value: '2', scopes: ['source.js', 'constant.numeric.decimal.js']
306326

307327
describe "assignment", ->
308328
it "tokenizes '=' operator", ->
@@ -338,7 +358,7 @@ describe "Javascript grammar", ->
338358
expect(tokens[3]).toEqual value: ' ', scopes: ['source.js']
339359
expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js']
340360
expect(tokens[5]).toEqual value: ' ', scopes: ['source.js']
341-
expect(tokens[6]).toEqual value: '42', scopes: ['source.js', 'constant.numeric.js']
361+
expect(tokens[6]).toEqual value: '42', scopes: ['source.js', 'constant.numeric.decimal.js']
342362
expect(tokens[7]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js']
343363

344364
{tokens} = grammar.tokenizeLine('something = MY_COOL_VAR * 1;')
@@ -349,7 +369,7 @@ describe "Javascript grammar", ->
349369
expect(tokens[4]).toEqual value: ' ', scopes: ['source.js']
350370
expect(tokens[5]).toEqual value: '*', scopes: ['source.js', 'keyword.operator.js']
351371
expect(tokens[6]).toEqual value: ' ', scopes: ['source.js']
352-
expect(tokens[7]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.js']
372+
expect(tokens[7]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js']
353373
expect(tokens[8]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js']
354374

355375
it "tokenizes variables declared using `const` as constants", ->
@@ -360,7 +380,7 @@ describe "Javascript grammar", ->
360380
expect(tokens[3]).toEqual value: ' ', scopes: ['source.js']
361381
expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js']
362382
expect(tokens[5]).toEqual value: ' ', scopes: ['source.js']
363-
expect(tokens[6]).toEqual value: '42', scopes: ['source.js', 'constant.numeric.js']
383+
expect(tokens[6]).toEqual value: '42', scopes: ['source.js', 'constant.numeric.decimal.js']
364384
expect(tokens[7]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js']
365385

366386
lines = grammar.tokenizeLines """
@@ -763,7 +783,7 @@ describe "Javascript grammar", ->
763783
{tokens} = grammar.tokenizeLine('export default 123;')
764784
expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js']
765785
expect(tokens[2]).toEqual value: 'default', scopes: ['source.js', 'meta.export.js', 'variable.language.default.js']
766-
expect(tokens[4]).toEqual value: '123', scopes: ['source.js', 'constant.numeric.js']
786+
expect(tokens[4]).toEqual value: '123', scopes: ['source.js', 'constant.numeric.decimal.js']
767787

768788
{tokens} = grammar.tokenizeLine('export default name;')
769789
expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js']
@@ -1070,14 +1090,14 @@ describe "Javascript grammar", ->
10701090
expect(tokens[10]).toEqual value: '{', scopes: ['source.js', 'meta.function-call.js', 'meta.brace.curly.js']
10711091
expect(tokens[11]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js']
10721092
expect(tokens[12]).toEqual value: ':', scopes: ['source.js', 'meta.function-call.js', 'keyword.operator.js']
1073-
expect(tokens[14]).toEqual value: '123', scopes: ['source.js', 'meta.function-call.js', 'constant.numeric.js']
1093+
expect(tokens[14]).toEqual value: '123', scopes: ['source.js', 'meta.function-call.js', 'constant.numeric.decimal.js']
10741094
expect(tokens[15]).toEqual value: '}', scopes: ['source.js', 'meta.function-call.js', 'meta.brace.curly.js']
10751095
expect(tokens[16]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js']
10761096

10771097
{tokens} = grammar.tokenizeLine('functionCall((123).toString())')
10781098
expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.begin.js']
10791099
expect(tokens[2]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'meta.brace.round.js']
1080-
expect(tokens[3]).toEqual value: '123', scopes: ['source.js', 'meta.function-call.js', 'constant.numeric.js']
1100+
expect(tokens[3]).toEqual value: '123', scopes: ['source.js', 'meta.function-call.js', 'constant.numeric.decimal.js']
10811101
expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.brace.round.js']
10821102
expect(tokens[9]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js']
10831103

@@ -1104,18 +1124,6 @@ describe "Javascript grammar", ->
11041124
expect(tokens[2]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js']
11051125

11061126
it "tokenizes illegal arguments", ->
1107-
{tokens} = grammar.tokenizeLine('a(1a)')
1108-
expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js']
1109-
expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.begin.js']
1110-
expect(tokens[2]).toEqual value: '1a', scopes: ['source.js', 'meta.function-call.js']
1111-
expect(tokens[3]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js']
1112-
1113-
{tokens} = grammar.tokenizeLine('a(123a)')
1114-
expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js']
1115-
expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.begin.js']
1116-
expect(tokens[2]).toEqual value: '123a', scopes: ['source.js', 'meta.function-call.js']
1117-
expect(tokens[3]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js']
1118-
11191127
{tokens} = grammar.tokenizeLine('a(1.prop)')
11201128
expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js']
11211129
expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.begin.js']
@@ -1147,9 +1155,9 @@ describe "Javascript grammar", ->
11471155
expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js']
11481156
expect(tokens[2]).toEqual value: 'b', scopes: ['source.js', 'meta.method-call.js', 'entity.name.function.js']
11491157
expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'punctuation.definition.arguments.begin.js']
1150-
expect(tokens[4]).toEqual value: '1', scopes: ['source.js', 'meta.method-call.js', 'constant.numeric.js']
1158+
expect(tokens[4]).toEqual value: '1', scopes: ['source.js', 'meta.method-call.js', 'constant.numeric.decimal.js']
11511159
expect(tokens[5]).toEqual value: '+', scopes: ['source.js', 'meta.method-call.js', 'keyword.operator.js']
1152-
expect(tokens[6]).toEqual value: '1', scopes: ['source.js', 'meta.method-call.js', 'constant.numeric.js']
1160+
expect(tokens[6]).toEqual value: '1', scopes: ['source.js', 'meta.method-call.js', 'constant.numeric.decimal.js']
11531161
expect(tokens[7]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'punctuation.definition.arguments.end.js']
11541162

11551163
{tokens} = grammar.tokenizeLine('a.$abc$()')

0 commit comments

Comments
 (0)