From 2a15b3d6d1b31710670a4a0c4a812863ca53438e Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Mon, 15 Jun 2020 17:30:27 +0900 Subject: [PATCH] Fix quoted string with new line character --- decode_test.go | 16 ++++++++++++++++ scanner/scanner.go | 21 +++++++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/decode_test.go b/decode_test.go index deec7b59..e402a38b 100644 --- a/decode_test.go +++ b/decode_test.go @@ -353,6 +353,14 @@ func TestDecoder(t *testing.T) { `'1': '\\"2\\"'`, map[interface{}]interface{}{"1": `\\"2\\"`}, }, + { + "'1': ' 1\n 2\n 3'", + map[interface{}]interface{}{"1": " 1 2 3"}, + }, + { + "'1': '\n 2\n 3'", + map[interface{}]interface{}{"1": " 2 3"}, + }, // Double Quoted values. { @@ -383,6 +391,14 @@ func TestDecoder(t *testing.T) { `"1": "\\\"2\\\""`, map[interface{}]interface{}{"1": `\"2\"`}, }, + { + "'1': \" 1\n 2\n 3\"", + map[interface{}]interface{}{"1": " 1 2 3"}, + }, + { + "'1': \"\n 2\n 3\"", + map[interface{}]interface{}{"1": " 2 3"}, + }, /* // TODO: Escape string diff --git a/scanner/scanner.go b/scanner/scanner.go index 5b91b317..addc7769 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -200,12 +200,20 @@ func (s *Scanner) scanSingleQuote(ctx *Context) (tk *token.Token, pos int) { src := ctx.src size := len(src) value := []rune{} + isFirstLineChar := false for idx := startIndex; idx < size; idx++ { c := src[idx] pos = idx + 1 ctx.addOriginBuf(c) - if c != '\'' { + if s.isNewLineChar(c) { + value = append(value, ' ') + isFirstLineChar = true + continue + } else if c == ' ' && isFirstLineChar { + continue + } else if c != '\'' { value = append(value, c) + isFirstLineChar = false continue } if idx+1 < len(ctx.src) && ctx.src[idx+1] == '\'' { @@ -229,11 +237,18 @@ func (s *Scanner) scanDoubleQuote(ctx *Context) (tk *token.Token, pos int) { src := ctx.src size := len(src) value := []rune{} + isFirstLineChar := false for idx := startIndex; idx < size; idx++ { c := src[idx] pos = idx + 1 ctx.addOriginBuf(c) - if c == '\\' { + if s.isNewLineChar(c) { + value = append(value, ' ') + isFirstLineChar = true + continue + } else if c == ' ' && isFirstLineChar { + continue + } else if c == '\\' { if idx+1 < size { nextChar := src[idx+1] switch nextChar { @@ -253,9 +268,11 @@ func (s *Scanner) scanDoubleQuote(ctx *Context) (tk *token.Token, pos int) { } } value = append(value, c) + isFirstLineChar = false continue } else if c != '"' { value = append(value, c) + isFirstLineChar = false continue } tk = token.DoubleQuote(string(value), string(ctx.obuf), s.pos())