positioner: simplify, fix and add more tests (#342)

* positioner: simplify fix and add more tests

Signed-off-by: Denys Smirnov <[email protected]>

Author: dennwc

uast/transformer/positioner/positions.go

@@ -76,21 +76,20 @@ type positionIndex struct {
 }
 
 func newPositionIndex(data []byte) *positionIndex {
-	idx := &positionIndex{}
-	idx.size = len(data)
+	idx := &positionIndex{
+		size: len(data),
+	}
 	idx.addLineOffset(0)
 	for offset, b := range data {
 		if b == '\n' {
 			idx.addLineOffset(offset + 1)
 		}
 	}
-
 	return idx
 }
 
 func (idx *positionIndex) addLineOffset(offset int) {
 	idx.offsetByLine = append(idx.offsetByLine, offset)
-
 }
 
 // LineCol returns a one-based line and col given a zero-based byte offset.
@@ -146,7 +145,7 @@ func (idx *positionIndex) Offset(line, col int) (int, error) {
 		maxCol = 1
 	}
 
-	if col < minCol || (maxCol > 0 && col - 1 > maxCol) {
+	if col < minCol || (maxCol > 0 && col-1 > maxCol) {
 		return 0, fmt.Errorf("column out of bounds: %d [%d, %d]", col, minCol, maxCol)
 	}
 

uast/transformer/positioner/positions_test.go

@@ -8,26 +8,42 @@ import (
 	"gopkg.in/bblfsh/sdk.v2/uast/nodes"
 )
 
-func TestFillLineColFromOffset(t *testing.T) {
+func offset(v int) nodes.Object {
+	return uast.Position{Offset: uint32(v)}.ToObject()
+}
+
+func lineCol(line, col int) nodes.Object {
+	return uast.Position{Line: uint32(line), Col: uint32(col)}.ToObject()
+}
+
+func fullPos(off, line, col int) nodes.Object {
+	return uast.Position{Offset: uint32(off), Line: uint32(line), Col: uint32(col)}.ToObject()
+}
+
+func TestFillLineColNested(t *testing.T) {
 	require := require.New(t)
 
 	data := "hello\n\nworld"
 
 	input := nodes.Object{
-		uast.KeyStart: uast.Position{Offset: 0}.ToObject(),
-		uast.KeyEnd:   uast.Position{Offset: 4}.ToObject(),
-		"a": nodes.Array{nodes.Object{
-			uast.KeyStart: uast.Position{Offset: 7}.ToObject(),
-			uast.KeyEnd:   uast.Position{Offset: 12}.ToObject(),
+		"a": nodes.Object{
+			uast.KeyStart: offset(0),
+			uast.KeyEnd:   offset(4),
+		},
+		"b": nodes.Array{nodes.Object{
+			uast.KeyStart: offset(7),
+			uast.KeyEnd:   offset(12),
 		}},
 	}
 
 	expected := nodes.Object{
-		uast.KeyStart: uast.Position{Offset: 0, Line: 1, Col: 1}.ToObject(),
-		uast.KeyEnd:   uast.Position{Offset: 4, Line: 1, Col: 5}.ToObject(),
-		"a": nodes.Array{nodes.Object{
-			uast.KeyStart: uast.Position{Offset: 7, Line: 3, Col: 1}.ToObject(),
-			uast.KeyEnd:   uast.Position{Offset: 12, Line: 3, Col: 6}.ToObject(),
+		"a": nodes.Object{
+			uast.KeyStart: fullPos(0, 1, 1),
+			uast.KeyEnd:   fullPos(4, 1, 5),
+		},
+		"b": nodes.Array{nodes.Object{
+			uast.KeyStart: fullPos(7, 3, 1),
+			uast.KeyEnd:   fullPos(12, 3, 6),
 		}},
 	}
 
@@ -37,25 +53,30 @@ func TestFillLineColFromOffset(t *testing.T) {
 	require.Equal(expected, out)
 }
 
-func TestFillOffsetFromLineCol(t *testing.T) {
+func TestFillOffsetNested(t *testing.T) {
 	require := require.New(t)
 
 	data := "hello\n\nworld"
+
 	input := nodes.Object{
-		uast.KeyStart: uast.Position{Line: 1, Col: 1}.ToObject(),
-		uast.KeyEnd:   uast.Position{Line: 1, Col: 5}.ToObject(),
-		"a": nodes.Array{nodes.Object{
-			uast.KeyStart: uast.Position{Line: 3, Col: 1}.ToObject(),
-			uast.KeyEnd:   uast.Position{Line: 3, Col: 5}.ToObject(),
+		"a": nodes.Object{
+			uast.KeyStart: lineCol(1, 1),
+			uast.KeyEnd:   lineCol(1, 5),
+		},
+		"b": nodes.Array{nodes.Object{
+			uast.KeyStart: lineCol(3, 1),
+			uast.KeyEnd:   lineCol(3, 6),
 		}},
 	}
 
 	expected := nodes.Object{
-		uast.KeyStart: uast.Position{Offset: 0, Line: 1, Col: 1}.ToObject(),
-		uast.KeyEnd:   uast.Position{Offset: 4, Line: 1, Col: 5}.ToObject(),
-		"a": nodes.Array{nodes.Object{
-			uast.KeyStart: uast.Position{Offset: 7, Line: 3, Col: 1}.ToObject(),
-			uast.KeyEnd:   uast.Position{Offset: 11, Line: 3, Col: 5}.ToObject(),
+		"a": nodes.Object{
+			uast.KeyStart: fullPos(0, 1, 1),
+			uast.KeyEnd:   fullPos(4, 1, 5),
+		},
+		"b": nodes.Array{nodes.Object{
+			uast.KeyStart: fullPos(7, 3, 1),
+			uast.KeyEnd:   fullPos(12, 3, 6),
 		}},
 	}
 
@@ -65,23 +86,58 @@ func TestFillOffsetFromLineCol(t *testing.T) {
 	require.Equal(expected, out)
 }
 
-func TestEmptyFile(t *testing.T) {
+func TestFillOffsetEmptyFile(t *testing.T) {
 	require := require.New(t)
 
 	data := ""
 
 	input := nodes.Object{
-		uast.KeyStart: uast.Position{Line: 1, Col: 1}.ToObject(),
-		uast.KeyEnd:   uast.Position{Line: 1, Col: 1}.ToObject(),
+		uast.KeyStart: lineCol(1, 1),
+		uast.KeyEnd:   lineCol(1, 1),
 	}
 
 	expected := nodes.Object{
-		uast.KeyStart: uast.Position{Offset: 0, Line: 1, Col: 1}.ToObject(),
-		uast.KeyEnd:   uast.Position{Offset: 0, Line: 1, Col: 1}.ToObject(),
+		uast.KeyStart: fullPos(0, 1, 1),
+		uast.KeyEnd:   fullPos(0, 1, 1),
 	}
 
 	p := NewFillOffsetFromLineCol()
 	out, err := p.OnCode(data).Do(input)
 	require.NoError(err)
 	require.Equal(expected, out)
 }
+
+func TestPosIndex(t *testing.T) {
+	// Verify that a multi-byte Unicode rune does not displace offsets after
+	// its occurrence in the input. Test few other simple cases as well.
+	const source = `line1
+ё2
+a3`
+	var cases = []uast.Position{
+		{Offset: 0, Line: 1, Col: 1},
+		{Offset: 4, Line: 1, Col: 5},
+
+		// multi-byte unicode rune
+		{Offset: 6, Line: 2, Col: 1},
+		{Offset: 8, Line: 2, Col: 3}, // col is a byte offset+1, not a rune index
+
+		{Offset: 10, Line: 3, Col: 1},
+		{Offset: 11, Line: 3, Col: 2},
+
+		{Offset: 12, Line: 3, Col: 3}, // special case — EOF position
+	}
+
+	ind := newPositionIndex([]byte(source))
+	for _, c := range cases {
+		t.Run("", func(t *testing.T) {
+			line, col, err := ind.LineCol(int(c.Offset))
+			require.NoError(t, err)
+			require.Equal(t, c.Line, uint32(line))
+			require.Equal(t, c.Col, uint32(col))
+
+			off, err := ind.Offset(int(c.Line), int(c.Col))
+			require.NoError(t, err)
+			require.Equal(t, c.Offset, uint32(off))
+		})
+	}
+}

uast/uast.go

@@ -82,7 +82,7 @@ type Position struct {
 	Offset uint32 `json:"offset"`
 	// Line is the line number. It is a 1-based index.
 	Line uint32 `json:"line"`
-	// Col is the column number (the byte offset of the position relative to
+	// Col is the column number — the byte offset of the position relative to
 	// a line. It is a 1-based index.
 	Col uint32 `json:"col"`
 }