Merge pull request #134 from goccy/feature/fix-quoted-string-with-new…

…-line-char Fix quoted string with new line character
goccy · Jun 15, 2020 · a9c4340 · a9c4340
2 parents 1c5a6a5 + 2a15b3d
commit a9c4340
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 2 deletions.
diff --git a/decode_test.go b/decode_test.go
@@ -353,6 +353,14 @@ func TestDecoder(t *testing.T) {
 			`'1': '\\"2\\"'`,
 			map[interface{}]interface{}{"1": `\\"2\\"`},
 		},
+		{
+			"'1': '   1\n    2\n    3'",
+			map[interface{}]interface{}{"1": "   1 2 3"},
+		},
+		{
+			"'1': '\n    2\n    3'",
+			map[interface{}]interface{}{"1": " 2 3"},
+		},
 
 		// Double Quoted values.
 		{
@@ -383,6 +391,14 @@ func TestDecoder(t *testing.T) {
 			`"1": "\\\"2\\\""`,
 			map[interface{}]interface{}{"1": `\"2\"`},
 		},
+		{
+			"'1': \"   1\n    2\n    3\"",
+			map[interface{}]interface{}{"1": "   1 2 3"},
+		},
+		{
+			"'1': \"\n    2\n    3\"",
+			map[interface{}]interface{}{"1": " 2 3"},
+		},
 
 		/*
 			// TODO: Escape string

diff --git a/scanner/scanner.go b/scanner/scanner.go
@@ -200,12 +200,20 @@ func (s *Scanner) scanSingleQuote(ctx *Context) (tk *token.Token, pos int) {
 	src := ctx.src
 	size := len(src)
 	value := []rune{}
+	isFirstLineChar := false
 	for idx := startIndex; idx < size; idx++ {
 		c := src[idx]
 		pos = idx + 1
 		ctx.addOriginBuf(c)
-		if c != '\'' {
+		if s.isNewLineChar(c) {
+			value = append(value, ' ')
+			isFirstLineChar = true
+			continue
+		} else if c == ' ' && isFirstLineChar {
+			continue
+		} else if c != '\'' {
 			value = append(value, c)
+			isFirstLineChar = false
 			continue
 		}
 		if idx+1 < len(ctx.src) && ctx.src[idx+1] == '\'' {
@@ -229,11 +237,18 @@ func (s *Scanner) scanDoubleQuote(ctx *Context) (tk *token.Token, pos int) {
 	src := ctx.src
 	size := len(src)
 	value := []rune{}
+	isFirstLineChar := false
 	for idx := startIndex; idx < size; idx++ {
 		c := src[idx]
 		pos = idx + 1
 		ctx.addOriginBuf(c)
-		if c == '\\' {
+		if s.isNewLineChar(c) {
+			value = append(value, ' ')
+			isFirstLineChar = true
+			continue
+		} else if c == ' ' && isFirstLineChar {
+			continue
+		} else if c == '\\' {
 			if idx+1 < size {
 				nextChar := src[idx+1]
 				switch nextChar {
@@ -253,9 +268,11 @@ func (s *Scanner) scanDoubleQuote(ctx *Context) (tk *token.Token, pos int) {
 				}
 			}
 			value = append(value, c)
+			isFirstLineChar = false
 			continue
 		} else if c != '"' {
 			value = append(value, c)
+			isFirstLineChar = false
 			continue
 		}
 		tk = token.DoubleQuote(string(value), string(ctx.obuf), s.pos())