Disallow unescaped newlines in string literals

Author: tim-dlang

docs/syntax.md

@@ -464,7 +464,7 @@ token StringLiteral
     = <b>&quot;\\&quot;&quot;</b> <a href="#string-literal">StringPart</a>* <b>&quot;\\&quot;&quot;</b>
     ;
 fragment StringPart
-    = <b>[^\\&quot;\\\\]</b>
+    = <b>[^\\&quot;\\\\\\r\\n]</b>
     | <a href="#escape-sequence">EscapeSequence</a>
     ;
 </pre>

generator/dparsergen/generator/grammarebnf.ebnf

@@ -367,7 +367,7 @@ token StringLiteral
 
 /// ditto
 fragment StringPart
-    = [^\"\\]
+    = [^\"\\\r\n]
     | EscapeSequence
     ;
 

generator/dparsergen/generator/grammarebnf_lexer.d

@@ -968,7 +968,7 @@ void generateStateMachine(ref CodeWriter code, const EBNFGrammar lexerGrammar,
                                         goto lexerend;
                                 $$}
                                 else
-                                    throw lexerException(text("Error unexpected \'", $(ascii?"currentChar":"currentDchar"), "\'"), "$(codepointSetToStr(allAllowedSet).escapeD)", inputCopy.ptr - input.ptr);
+                                    throw lexerException(text("Error unexpected \'", $(ascii?"currentChar":"currentDchar").escapeChar(false), "\'"), "$(codepointSetToStr(allAllowedSet).escapeD)", inputCopy.ptr - input.ptr);
                             $$}
                         $$} else {
                             $$needsElse = true;
@@ -1010,15 +1010,15 @@ void generateStateMachine(ref CodeWriter code, const EBNFGrammar lexerGrammar,
                                         storedStart = size_t.max;
                                     $$}
                                     $$if (e.next3.id == size_t.max) {
-                                        throw lexerException(text("Error unexpected \'", $(ascii?"currentChar":"currentDchar"), "\'"), "$(codepointSetToStr(allAllowedSet).escapeD)", inputCopy.ptr - input.ptr);
+                                        throw lexerException(text("Error unexpected \'", $(ascii?"currentChar":"currentDchar").escapeChar(false), "\'"), "$(codepointSetToStr(allAllowedSet).escapeD)", inputCopy.ptr - input.ptr);
                                     $$} else {
                                         goto state$(e.next3.id);
                                     $$}
                                 }
                                 else if (compareString.length < storedString.length && currentCharCorrect)
                                 {
                                     $$if (e.next2.id == size_t.max) {
-                                        throw lexerException(text("Error unexpected \'", $(ascii?"currentChar":"currentDchar"), "\'"), "$(codepointSetToStr(allAllowedSet).escapeD)", inputCopy.ptr - input.ptr);
+                                        throw lexerException(text("Error unexpected \'", $(ascii?"currentChar":"currentDchar").escapeChar(false), "\'"), "$(codepointSetToStr(allAllowedSet).escapeD)", inputCopy.ptr - input.ptr);
                                     $$} else {
                                         goto state$(e.next2.id);
                                     $$}
@@ -1027,7 +1027,7 @@ void generateStateMachine(ref CodeWriter code, const EBNFGrammar lexerGrammar,
                                 {
                                     storedStart = size_t.max;
                                     $$if (e.next.id == size_t.max) {
-                                        throw lexerException(text("Error unexpected \'", $(ascii?"currentChar":"currentDchar"), "\'"), "$(codepointSetToStr(allAllowedSet).escapeD)", inputCopy.ptr - input.ptr);
+                                        throw lexerException(text("Error unexpected \'", $(ascii?"currentChar":"currentDchar").escapeChar(false), "\'"), "$(codepointSetToStr(allAllowedSet).escapeD)", inputCopy.ptr - input.ptr);
                                     $$} else {
                                         goto state$(e.next.id);
                                     $$}
@@ -1541,6 +1541,7 @@ const(char)[] createLexerCode(EBNFGrammar lexerGrammar, string modulename, strin
         module $(modulename);
         import dparsergen.core.grammarinfo;
         import dparsergen.core.parseexception;
+        import dparsergen.core.utils;
         import std.conv;
         import std.string;
         import std.typecons;

generator/dparsergen/generator/lexergenerator.d

@@ -0,0 +1 @@
+tokennewline1.ebnf text eol=lf

tests/fail_grammars/.gitattributes

@@ -0,0 +1,8 @@
+/*
+GENERATOR_OUTPUT(optdescent, glr, normal, lexer):
+---
+tests/fail_grammars/tokennewline1.ebnf:7:4: Error unexpected '\n', expected [^\n\r]
+---
+*/
+S = "x
+";