@@ -54,19 +54,64 @@ if (isBrowserSupported()) {
54
54
* Asian languages are not supported by our current tokenizer strategy.
55
55
*/
56
56
it ( 'Tokenized results matches words and symbols in TEST_STRINGS' , ( ) => {
57
- const expectedParagraphMixed = [ 'This' , "isn't" , 'a' , 'sentence' , "it's" , "RUM's" , 'test' , '💥' , '=' , '+' , '+' , 'and' , 'more' ]
57
+ const expectedParagraphMixed = [
58
+ 'This' ,
59
+ "isn't" ,
60
+ 'a' ,
61
+ 'sentence' ,
62
+ "it's" ,
63
+ "RUM's" ,
64
+ 'test' ,
65
+ '💥' ,
66
+ '=' ,
67
+ '+' ,
68
+ '+' ,
69
+ 'and' ,
70
+ 'more' ,
71
+ ]
58
72
expect ( tokenize ( TEST_STRINGS . PARAGRAPH_MIXED ) . sort ( ) ) . toEqual ( expectedParagraphMixed . sort ( ) )
59
73
60
- const expectedFrench = [ "C'est" , 'pas' , 'un' , 'test' , "c'est" , "RUM's" , 'test' , '💥' , '=' , '+' , '+' , 'et' , 'plus' ]
74
+ const expectedFrench = [ "C'est" , 'pas' , 'un' , 'test' , "c'est" , "RUM's" , 'test' , '💥' , '=' , '+' , '+' , 'et' , 'plus' ]
61
75
expect ( tokenize ( LANGUAGES_TEST_STRINGS . FRENCH_MIXED_SENTENCE ) . sort ( ) ) . toEqual ( expectedFrench . sort ( ) )
62
76
63
77
const expectedSpanish = [ 'Este' , 'no' , 'es' , 'un' , 'test' , 'es' , "RUM's" , 'test' , '💥' , '=' , '+' , '+' , 'y' , 'más' ]
64
78
expect ( tokenize ( LANGUAGES_TEST_STRINGS . SPANISH_MIXED_SENTENCE ) . sort ( ) ) . toEqual ( expectedSpanish . sort ( ) )
65
79
66
- const expectedGerman = [ 'Das' , 'ist' , 'kein' , 'Test' , 'das' , 'ist' , "RUM's" , 'Test' , '💥' , '=' , '+' , '+' , 'und' , 'mehr' ]
80
+ const expectedGerman = [
81
+ 'Das' ,
82
+ 'ist' ,
83
+ 'kein' ,
84
+ 'Test' ,
85
+ 'das' ,
86
+ 'ist' ,
87
+ "RUM's" ,
88
+ 'Test' ,
89
+ '💥' ,
90
+ '=' ,
91
+ '+' ,
92
+ '+' ,
93
+ 'und' ,
94
+ 'mehr' ,
95
+ ]
67
96
expect ( tokenize ( LANGUAGES_TEST_STRINGS . GERMAN_MIXED_SENTENCE ) . sort ( ) ) . toEqual ( expectedGerman . sort ( ) )
68
97
69
- const expectedPortuguese = [ 'Este' , 'não' , 'é' , 'um' , 'teste' , 'este' , 'é' , "RUM's" , 'test' , '💥' , '=' , '+' , '+' , 'e' , 'mais' ]
98
+ const expectedPortuguese = [
99
+ 'Este' ,
100
+ 'não' ,
101
+ 'é' ,
102
+ 'um' ,
103
+ 'teste' ,
104
+ 'este' ,
105
+ 'é' ,
106
+ "RUM's" ,
107
+ 'test' ,
108
+ '💥' ,
109
+ '=' ,
110
+ '+' ,
111
+ '+' ,
112
+ 'e' ,
113
+ 'mais' ,
114
+ ]
70
115
expect ( tokenize ( LANGUAGES_TEST_STRINGS . PORTUGUESE_MIXED_SENTENCE ) . sort ( ) ) . toEqual ( expectedPortuguese . sort ( ) )
71
116
} )
72
117
} )
0 commit comments