Merge pull request #1013 from Thirumalai-Shaktivel/string

Author: Thirumalai-Shaktivel

src/lpython/parser/tokenizer.re

@@ -275,10 +275,18 @@ int Tokenizer::lex(Allocator &al, YYSTYPE &yylval, Location &loc, diag::Diagnost
             integer = int_dec | int_oct | int_bin | int_hex;
             real = (significand exp?) | (digit+ exp);
             imag_number = (real | digit+)[jJ];
-            string1 = '"' ('\\"'|[^"\x00])* '"';
-            string2 = "'" ("\\'"|[^'\x00])* "'";
-            string3 = '"""' ( '\\"' | '"' [^"\x00] | '""' [^"\x00] | [^"\x00] )* '"""';
-            string4 = "'''" ( "\\'" | "'" [^'\x00] | "''" [^'\x00] | [^'\x00] )* "'''";
+            string1 = '"' ('\\'[^\x00] | [^"\x00\n\\])* '"';
+            string2 = "'" ("\\"[^\x00] | [^'\x00\n\\])* "'";
+            string3 = '"""' ( '\\'[^\x00]
+                            | ('"' | '"' '\\'+) [^"\x00\\]
+                            | ('""' | '""' '\\'+) [^"\x00\\]
+                            | [^"\x00\\] )*
+                      '"""';
+            string4 = "'''" ( "\\"[^\x00]
+                            | ("'" | "'" "\\"+) [^'\x00\\]
+                            | ("''" | "''" "\\"+) [^'\x00\\]
+                            | [^'\x00\\] )*
+                      "'''";
             type_ignore = "#" whitespace? "type:" whitespace? "ignore" [^\n\x00]*;
             type_comment = "#" whitespace? "type:" whitespace? [^\n\x00]*;
             comment = "#" [^\n\x00]*;

tests/parser/string2.py

@@ -0,0 +1,17 @@
+"\"\\"
+"\n\\"
+"\\"
+"   \\"
+"\\ "
+"\
+"
+"\\" "|"
+
+r"""Text\\"""
+"""\
+"""
+x("""\
+class a:
+    def test():
+        return r"\\LaTeX"
+""")

tests/reference/ast_new-string2-44323ea.json

@@ -0,0 +1,13 @@
+{
+    "basename": "ast_new-string2-44323ea",
+    "cmd": "lpython --show-ast --new-parser --no-color {infile} -o {outfile}",
+    "infile": "tests/parser/string2.py",
+    "infile_hash": "fc6c3d133d5b93c5b5f052b9d86c2b1f270149aa48dba8ae8ced3e86",
+    "outfile": null,
+    "outfile_hash": null,
+    "stdout": "ast_new-string2-44323ea.stdout",
+    "stdout_hash": "cf514f21d4a57de66c5fbfc107a6ab58818a09359ef9ba651896d71f",
+    "stderr": null,
+    "stderr_hash": null,
+    "returncode": 0
+}

tests/reference/ast_new-string2-44323ea.stdout

@@ -0,0 +1,8 @@
+(Module [(Expr (ConstantStr "\"\\" ())) (Expr (ConstantStr "
+\\" ())) (Expr (ConstantStr "\\" ())) (Expr (ConstantStr "   \\" ())) (Expr (ConstantStr "\\ " ())) (Expr (ConstantStr "\
+" ())) (Expr (ConstantStr "\\|" ())) (Expr (ConstantStr "Text\\" ())) (Expr (ConstantStr "\
+" ())) (Expr (Call (Name x Load) [(ConstantStr "\
+class a:
+    def test():
+        return r"\\LaTeX"
+" ())] []))] [])

tests/tests.toml

@@ -506,6 +506,10 @@ ast_new = true
 filename = "parser/string1.py"
 ast_new = true
 
+[[test]]
+filename = "parser/string2.py"
+ast_new = true
+
 [[test]]
 filename = "parser/global1.py"
 ast = true