Add support for \r inside strings and chars, fix col counting in CHAR

yangdanny97 · Feb 3, 2019 · b547887 · b547887
1 parent 5ec94dc
commit b547887
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 7 deletions.
diff --git a/src/main/java/lexer/XiToken.java b/src/main/java/lexer/XiToken.java
@@ -72,10 +72,12 @@ private String format(Object o) {
         if (o instanceof String) {
             String s = (String) o;
             s = s.replace("\n", "\\n");
+            s = s.replace("\r", "\\r");
             s = s.replace("\t", "\\t");
             return s;
         } else if (o instanceof Character) {
             if (o.equals('\n')) return "\\n";
+            if (o.equals('\r')) return "\\r";
             if (o.equals('\t')) return "\\t";
             return o.toString();
         } else {

diff --git a/src/test.flex b/src/test.flex
@@ -101,12 +101,13 @@ INTEGER = 0 | [1-9][0-9]*
     \"              { yybegin(YYINITIAL);
                       return new XiToken(TokenType.STRING_LIT, yyline,
                       stringLiteralStartCol, stringLiteral.toString()); }
-    [^\n\"\\]+ { stringLiteral.append( yytext() ); }//TODO exclude HEX?
+    [^\n\r\"\\]+ { stringLiteral.append( yytext() ); }
     {HEX}           { stringLiteral.append( (char) Integer.parseInt(
                         yytext().substring(2, yylength()), 16
                     )); }
     \\t             { stringLiteral.append( '\t' ); }
     \\n             { stringLiteral.append( '\n' ); }
+    \\r             { stringLiteral.append( '\r' ); }
     \\\"            { stringLiteral.append( '\"' ); }
     \\              { stringLiteral.append( '\\' ); }
 }
@@ -115,22 +116,24 @@ INTEGER = 0 | [1-9][0-9]*
     \'        { yybegin(YYINITIAL);
                 return new XiToken(TokenType.ERROR, yyline,
                 charLiteralStartCol, "empty character literal"); }
-     [^\n\'\\]\'    { yybegin(YYINITIAL); //TODO exclude HEX?
+     [^\n\r\'\\]\'    { yybegin(YYINITIAL);
                     return new XiToken(TokenType.CHAR_LIT, yyline,
                     charLiteralStartCol, yytext().charAt(0)); }
      {HEX}\' {yybegin(YYINITIAL); return new XiToken(TokenType.CHAR_LIT, yyline,
-     yycolumn,
+     charLiteralStartCol,
        Character.forDigit(
                Integer.parseInt(yytext().substring(2,yylength()) ,16), 10
                ));}
      \\t\'          { yybegin(YYINITIAL); return new XiToken(TokenType
-     .CHAR_LIT, yyline, yycolumn, '\t'); }
+     .CHAR_LIT, yyline, charLiteralStartCol, '\t'); }
      \\n\'          { yybegin(YYINITIAL); return new XiToken(TokenType
-     .CHAR_LIT, yyline, yycolumn, '\n'); }
+     .CHAR_LIT, yyline, charLiteralStartCol, '\n'); }
+     \\r\'          { yybegin(YYINITIAL); return new XiToken(TokenType
+     .CHAR_LIT, yyline, charLiteralStartCol, '\r'); }
      \\\'           { yybegin(YYINITIAL); return new XiToken(TokenType
-     .CHAR_LIT, yyline, yycolumn, '\\'); }
+     .CHAR_LIT, yyline, charLiteralStartCol, '\\'); }
      \\\'\'         { yybegin(YYINITIAL); return new XiToken(TokenType
-     .CHAR_LIT, yyline, yycolumn, '\''); }
+     .CHAR_LIT, yyline, charLiteralStartCol, '\''); }
 }
 
 /* error fallback */