1
1
/* eslint max-depth: "off" */
2
2
'use strict' ;
3
3
4
- const LexerError = require ( './error/Lexer' ) ;
5
- const {
4
+ import LexerError from './error/Lexer' ;
5
+ import {
6
6
TOKEN_TYPE_VALUE ,
7
7
TOKEN_TYPE_STRUCTURE ,
8
8
TOKEN_TYPE_STATEMENT ,
@@ -41,8 +41,8 @@ const {
41
41
OPERATOR_GREATER_THAN ,
42
42
OPERATOR_LESS_THAN ,
43
43
OPERATOR_GREATER_EQUAL_THAN ,
44
- OPERATOR_LESS_EQUAL_THAN
45
- } = require ( './constants' ) ;
44
+ OPERATOR_LESS_EQUAL_THAN ,
45
+ } from './constants' ;
46
46
47
47
const boundaryTypeLookup = {
48
48
'{{' : TOKEN_BOUNDARY_TAG_START ,
@@ -97,20 +97,39 @@ Array.prototype.concat(
97
97
delimiters [ delimiter . length ] . push ( delimiter ) ;
98
98
} ) ;
99
99
100
- class Lexer {
100
+ export default class Lexer {
101
101
constructor ( input ) {
102
102
this . state = STATE_TEXT_LITERAL ;
103
103
this . pointer = 0 ;
104
+ this . columnNumber = 0 ;
105
+ this . rowNumber = 0 ;
104
106
this . input = input ;
105
107
// length used for cases of checking ahead by one character from pointer
106
108
this . lookAheadLength = this . input . length - 1 ;
107
109
this . stringDelimiter = null ;
108
110
}
109
111
112
+ movePointer ( ) {
113
+ if ( this . input [ this . pointer ] === '\n' ) {
114
+ this . rowNumber ++ ;
115
+ this . columnNumber = 0 ;
116
+ }
117
+ else {
118
+ this . columnNumber ++ ;
119
+ }
120
+ this . pointer ++ ;
121
+ }
122
+
110
123
// skip all whitespace or until EOF reached
111
124
skipWhitespace ( ) {
112
125
while ( this . pointer < this . input . length && this . input [ this . pointer ] . trim ( ) . length === 0 ) {
113
- this . pointer ++ ;
126
+ this . movePointer ( ) ;
127
+ }
128
+ }
129
+
130
+ skipNewline ( ) {
131
+ if ( this . input [ this . pointer ] === '\n' ) {
132
+ this . movePointer ( ) ;
114
133
}
115
134
}
116
135
@@ -123,13 +142,13 @@ class Lexer {
123
142
&& this . input [ this . pointer + 1 ] === '{'
124
143
)
125
144
) {
126
- this . pointer ++ ;
145
+ this . movePointer ( ) ;
127
146
}
128
147
129
148
// because we look two characters ahead, we need to increment the
130
149
// pointer by one if end of input is reached
131
150
if ( this . lookAheadLength >= 0 && this . pointer >= this . lookAheadLength ) {
132
- this . pointer ++ ;
151
+ this . movePointer ( ) ;
133
152
}
134
153
}
135
154
@@ -144,9 +163,9 @@ class Lexer {
144
163
if ( this . input [ this . pointer + 1 ] === this . stringDelimiter && this . input [ this . pointer ] !== '\\' ) {
145
164
break ;
146
165
}
147
- this . pointer ++ ;
166
+ this . movePointer ( ) ;
148
167
}
149
- this . pointer ++ ;
168
+ this . movePointer ( ) ;
150
169
}
151
170
152
171
isAtDelimiter ( ) {
@@ -176,26 +195,31 @@ class Lexer {
176
195
if ( this . isAtDelimiter ( ) ) {
177
196
return ;
178
197
}
179
- this . pointer ++ ;
198
+ this . movePointer ( ) ;
180
199
181
200
while ( this . pointer < this . input . length ) {
182
201
if ( this . isAtDelimiter ( ) ) {
183
202
return ;
184
203
}
185
- this . pointer ++ ;
204
+ this . movePointer ( ) ;
186
205
}
187
206
}
188
207
189
208
// scan pass the next delimiter
190
209
scanNextDelimiter ( ) {
191
210
const delimiter = this . isAtDelimiter ( ) || '' ;
192
-
193
- this . pointer = this . pointer + delimiter . length ;
211
+ for ( let i = 0 ; i < delimiter . length ; i ++ ) {
212
+ this . movePointer ( ) ;
213
+ }
194
214
}
195
215
196
216
* tokens ( ) {
217
+ let column = - 1 ;
218
+ let row = - 1 ;
197
219
while ( this . pointer < this . input . length ) {
198
220
if ( this . state === STATE_TEXT_LITERAL ) {
221
+ column = this . columnNumber ;
222
+ row = this . rowNumber ;
199
223
const startIndex = this . pointer ;
200
224
201
225
this . scanTextLiteral ( ) ;
@@ -207,6 +231,8 @@ class Lexer {
207
231
continue ;
208
232
}
209
233
yield {
234
+ column,
235
+ row,
210
236
type : TOKEN_TYPE_STRUCTURE ,
211
237
subType : TOKEN_STRUCTURE_TEXT_LITERAL ,
212
238
value,
@@ -219,6 +245,8 @@ class Lexer {
219
245
220
246
this . skipWhitespace ( ) ;
221
247
248
+ column = this . columnNumber ;
249
+ row = this . rowNumber ;
222
250
let startIndex = this . pointer ;
223
251
224
252
// scan until next delimiter
@@ -229,6 +257,8 @@ class Lexer {
229
257
type = statementTypeLookup [ value ] ;
230
258
if ( type ) {
231
259
yield {
260
+ column,
261
+ row,
232
262
type : TOKEN_TYPE_STATEMENT ,
233
263
subType : type ,
234
264
value,
@@ -253,6 +283,8 @@ class Lexer {
253
283
}
254
284
255
285
yield {
286
+ column,
287
+ row,
256
288
type : TOKEN_TYPE_VALUE ,
257
289
subType : type ,
258
290
value,
@@ -262,6 +294,8 @@ class Lexer {
262
294
continue ;
263
295
}
264
296
297
+ column = this . columnNumber ;
298
+ row = this . rowNumber ;
265
299
startIndex = this . pointer ;
266
300
this . scanNextDelimiter ( ) ;
267
301
value = this . input . substring ( startIndex , this . pointer ) ;
@@ -273,9 +307,12 @@ class Lexer {
273
307
this . state = this . state === STATE_END_STRING ? STATE_TAG : STATE_STRING ;
274
308
}
275
309
else if ( type === TOKEN_BOUNDARY_TAG_END ) {
310
+ this . skipNewline ( ) ;
276
311
this . state = STATE_TEXT_LITERAL ;
277
312
}
278
313
yield {
314
+ column,
315
+ row,
279
316
type : TOKEN_TYPE_BOUNDARY ,
280
317
subType : type ,
281
318
value,
@@ -287,6 +324,8 @@ class Lexer {
287
324
type = binaryOperatorTypeLookup [ value ] ;
288
325
if ( type ) {
289
326
yield {
327
+ column,
328
+ row,
290
329
type : TOKEN_TYPE_BINARY_OPERATOR ,
291
330
subType : type ,
292
331
value,
@@ -297,6 +336,8 @@ class Lexer {
297
336
}
298
337
type = unaryOperatorTypeLookup [ value ] ;
299
338
yield {
339
+ column,
340
+ row,
300
341
type : TOKEN_TYPE_UNARY_OPERATOR ,
301
342
subType : type ,
302
343
value,
@@ -305,11 +346,15 @@ class Lexer {
305
346
} ;
306
347
}
307
348
else if ( this . state === STATE_STRING ) {
349
+ column = this . columnNumber ;
350
+ row = this . rowNumber ;
308
351
const startIndex = this . pointer ;
309
352
310
353
this . scanString ( ) ;
311
354
this . state = STATE_END_STRING ;
312
355
yield {
356
+ column,
357
+ row,
313
358
type : TOKEN_TYPE_VALUE ,
314
359
subType : TOKEN_VALUE_STRING ,
315
360
value : this . input . substring ( startIndex , this . pointer ) ,
@@ -323,6 +368,8 @@ class Lexer {
323
368
}
324
369
325
370
yield {
371
+ column : this . columnNumber ,
372
+ row : this . rowNumber ,
326
373
type : TOKEN_TYPE_STRUCTURE ,
327
374
subType : TOKEN_STRUCTURE_EOF ,
328
375
value : null ,
@@ -331,5 +378,3 @@ class Lexer {
331
378
} ;
332
379
}
333
380
}
334
-
335
- module . exports = Lexer ;
0 commit comments