Backport(v1.19): config: accept empty lines in quoted strings (#5448)

lib/fluent/config/basic_parser.rb

@@ -28,7 +28,6 @@ def initialize(strscan)
       SPACING = /(?:[ \t\r\n]|\z|\#.*?(?:\z|[\r\n]))+/
       ZERO_OR_MORE_SPACING = /(?:[ \t\r\n]|\z|\#.*?(?:\z|[\r\n]))*/
       SPACING_WITHOUT_COMMENT = /(?:[ \t\r\n]|\z)+/
-      LINE_END_WITHOUT_SPACING_AND_COMMENT = /(?:\z|[\r\n])/
 
       module ClassMethods
         def symbol(string)

lib/fluent/config/literal_parser.rb

@@ -27,6 +27,15 @@
 module Fluent
   module Config
     class LiteralParser < BasicParser
+      # A physical line break (LF, CR, or CRLF) inside a quoted string.
+      # CRLF is normalized to LF so that the same config text does not produce a
+      # different value depending on whether the file was saved with LF or CRLF.
+      # A lone CR is kept as-is, and an escaped "\r\n" still produces CRLF.
+      LINE_BREAK = /\r\n|[\r\n]/
+      # A backslash immediately followed by a physical line break.
+      # It works as a line continuation, so both are stripped from the value.
+      LINE_CONTINUATION = /\\#{LINE_BREAK}/o
+
       def self.unescape_char(c)
         case c
         when '"'
@@ -99,11 +108,10 @@ def scan_double_quoted_string
             else
               return string.join
             end
-          elsif check(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/o)
-            if s = check(/[^\\]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/o)
-              string << s
-            end
-            skip(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/o)
+          elsif skip(LINE_CONTINUATION)
+            next
+          elsif s = scan(LINE_BREAK)
+            string << (s == "\r\n" ? "\n" : s)
           elsif s = scan(/\\./)
             string << eval_escape_char(s[1,1])
           elsif skip(/\#\{/)
@@ -126,6 +134,8 @@ def scan_single_quoted_string
             string << "'"
           elsif s = scan(/\\\\/)
             string << "\\"
+          elsif s = scan(LINE_BREAK)
+            string << (s == "\r\n" ? "\n" : s)
           elsif s = scan(/./)
             string << s
           else

test/config/test_config_parser.rb

@@ -149,6 +149,7 @@ def parse_text(text)
         end
 
         test "support multiline string" do
+          assert_text_parsed_as(e('ROOT', '', {"k1" => "world\n\n"}), "k1 \"world\n\n\"")
           assert_text_parsed_as(e('ROOT', '',
             {"k1" => %[line1
                        line2]

test/config/test_literal_parser.rb

@@ -111,6 +111,14 @@ def test_falseX
       test('"t') { assert_parse_error('"t') }  # non-terminated quoted character
       test("\"t\nt\"") { assert_text_parsed_as("t\nt", "\"t\nt\"" ) } # multiline string
       test("\"t\\\nt\"") { assert_text_parsed_as("tt", "\"t\\\nt\"" ) } # multiline string
+      test("\"t\n\nt\"") { assert_text_parsed_as("t\n\nt", "\"t\n\nt\"") }
+      test("\"\nt\"") { assert_text_parsed_as("\nt", "\"\nt\"") }
+      test("\"t\\\\\nt\"") { assert_text_parsed_as("t\\\nt", "\"t\\\\\nt\"") }
+      test("\"t\r\nt\"") { assert_text_parsed_as("t\nt", "\"t\r\nt\"") }
+      test("\"t\rt\"") { assert_text_parsed_as("t\rt", "\"t\rt\"") }
+      test("\"t\\\r\nt\"") { assert_text_parsed_as("tt", "\"t\\\r\nt\"") }
+      test("\"t\n") { assert_parse_error("\"t\n") }
+      test("\"t\\\n") { assert_parse_error("\"t\\\n") }
       test('t"') { assert_text_parsed_as('t"', 't"') }
       test('"."') { assert_text_parsed_as('.', '"."') }
       test('"*"') { assert_text_parsed_as('*', '"*"') }
@@ -138,6 +146,13 @@ def test_falseX
       test("'\\0'") { assert_text_parsed_as('\0', "'\\0'") }
       test("'\\1'") { assert_text_parsed_as('\1', "'\\1'") }
       test("'t") { assert_parse_error("'t") }  # non-terminated quoted character
+      test("'t\nt'") { assert_text_parsed_as("t\nt", "'t\nt'") }
+      test("'t\n\nt'") { assert_text_parsed_as("t\n\nt", "'t\n\nt'") }
+      test("'\nt'") { assert_text_parsed_as("\nt", "'\nt'") }
+      test("'t\r\nt'") { assert_text_parsed_as("t\nt", "'t\r\nt'") }
+      test("'t\rt'") { assert_text_parsed_as("t\rt", "'t\rt'") }
+      test("'t\\\nt'") { assert_text_parsed_as("t\\\nt", "'t\\\nt'") }
+      test("'t\n") { assert_parse_error("'t\n") }
       test("t'") { assert_text_parsed_as("t'", "t'") }
       test("'.'") { assert_text_parsed_as('.', "'.'") }
       test("'*'") { assert_text_parsed_as('*', "'*'") }