Merge pull request #1014 from Thirumalai-Shaktivel/indent

Author: Thirumalai-Shaktivel

src/lpython/parser/tokenizer.re

@@ -323,7 +323,7 @@ int Tokenizer::lex(Allocator &al, YYSTYPE &yylval, Location &loc, diag::Diagnost
                     RET(TK_INDENT);
                 } else {
                     if(last_token == yytokentype::TK_NEWLINE && cur[0] != ' '
-                            && last_indent_length > cur-tok) {
+                            && cur[0] != '\t' && last_indent_length > cur-tok) {
                         last_indent_length = cur-tok;
                         dedent = 2;
                         if (!indent_length.empty()) {
@@ -376,8 +376,8 @@ int Tokenizer::lex(Allocator &al, YYSTYPE &yylval, Location &loc, diag::Diagnost
                         || colon_actual_last_token) {
                     colon_actual_last_token = false;
                     indent = true;
-                } else if (cur[0] != ' ' && cur[0] != '\n'
-                        && last_indent_length > cur-tok) {
+                } else if (cur[0] != ' ' && cur[0] != '\t' && cur[0] != '\n'
+                        && last_indent_length >= cur-tok) {
                     last_indent_length = 0;
                     dedent = 1;
                 }

tests/reference/tokens-indent2-e702789.json

@@ -2,11 +2,11 @@
     "basename": "tokens-indent2-e702789",
     "cmd": "lpython --no-color --show-tokens {infile} -o {outfile}",
     "infile": "tests/tokens/indent2.py",
-    "infile_hash": "0bf5680f2c0bd9241c0622181be765f900672a0d54ca6366fd5842ff",
+    "infile_hash": "6ee8bbeb2e48631f07b13012f569904545ce5c14e714003d030fc385",
     "outfile": null,
     "outfile_hash": null,
     "stdout": "tokens-indent2-e702789.stdout",
-    "stdout_hash": "d21868282c9ca047305e18e15d2961166902b5eecba2f8d905d7d219",
+    "stdout_hash": "e259b5fcc4c1d83a04924c51a6a6f6d9be631970d3663e5be065ed7d",
     "stderr": null,
     "stderr_hash": null,
     "returncode": 0

tests/reference/tokens-indent2-e702789.stdout

@@ -25,6 +25,64 @@
 (KEYWORD "return") 84:89
 (TOKEN "string" "bar") 91:95
 (NEWLINE) 96:96
-(TOKEN "dedent") 96:96
-(TOKEN "dedent") 96:96
-(EOF) 97:97
+(NEWLINE) 97:97
+(TOKEN "dedent") 97:97
+(TOKEN "dedent") 97:97
+(KEYWORD "def") 98:100
+(TOKEN "identifier" test) 102:105
+(TOKEN "(") 106:106
+(TOKEN ")") 107:107
+(TOKEN ":") 108:108
+(NEWLINE) 109:109
+(TOKEN "indent") 110:110
+(KEYWORD "if") 111:112
+(TOKEN "True") 114:117
+(TOKEN ":") 118:118
+(NEWLINE) 119:119
+(TOKEN "indent") 120:121
+(KEYWORD "pass") 122:125
+(NEWLINE) 126:126
+(TOKEN "dedent") 127:127
+(KEYWORD "else") 128:131
+(TOKEN ":") 132:132
+(NEWLINE) 133:133
+(TOKEN "indent") 134:135
+(KEYWORD "pass") 136:139
+(NEWLINE) 140:140
+(NEWLINE) 141:141
+(TOKEN "dedent") 141:141
+(TOKEN "dedent") 141:141
+(KEYWORD "def") 142:144
+(TOKEN "identifier" test) 146:149
+(TOKEN "(") 150:150
+(TOKEN "identifier" x) 151:151
+(TOKEN ":") 152:152
+(TOKEN "identifier" i32) 154:156
+(TOKEN ")") 157:157
+(TOKEN ":") 158:158
+(NEWLINE) 159:159
+(TOKEN "indent") 160:160
+(KEYWORD "if") 161:162
+(TOKEN "identifier" x) 164:164
+(TOKEN "<") 166:166
+(TOKEN "integer" 0) 168:168
+(TOKEN ":") 169:169
+(NEWLINE) 170:170
+(TOKEN "indent") 171:172
+(KEYWORD "pass") 173:176
+(NEWLINE) 177:177
+(TOKEN "dedent") 178:178
+(KEYWORD "else") 179:182
+(TOKEN ":") 183:183
+(NEWLINE) 184:184
+(TOKEN "indent") 185:186
+(TOKEN "identifier" Test) 187:190
+(TOKEN ".") 191:191
+(TOKEN "identifier" foo) 192:194
+(TOKEN "(") 195:195
+(TOKEN "identifier" x) 196:196
+(TOKEN ")") 197:197
+(NEWLINE) 198:198
+(TOKEN "dedent") 198:198
+(TOKEN "dedent") 198:198
+(EOF) 199:199

tests/tokens/indent2.py

@@ -3,3 +3,15 @@ def foo(a):
         if a == 'foo':
             return 'foo'
         return 'bar'
+
+def test():
+ if True:
+  pass
+ else:
+  pass
+
+def test(x: i32):
+	if x < 0:
+		pass
+	else:
+		Test.foo(x)