Gitbase multi version testing

.gitignore

@@ -27,3 +27,4 @@ main
 # CI
 .ci/
 build/
+regression-testing-cache/

Makefile

@@ -17,22 +17,46 @@ $(MAKEFILE):
 
 -include $(MAKEFILE)
 
-GOTEST_INTEGRATION = $(GOTEST) -timeout 20m -parallel 1 -count 1 -tags=integration -ldflags "$(LD_FLAGS)"
+GOTEST_BASE = $(GOTEST) -timeout 20m -parallel 1 -count 1 -ldflags "$(LD_FLAGS)"
+GOTEST_INTEGRATION = $(GOTEST_BASE) -tags=integration
+GOTEST_REGRESSION = $(GOTEST_BASE) -tags=regression
 
 OS := $(shell uname)
 
 ifeq ($(OS),Darwin)
 test-integration-clean:
-	$(eval TMPDIR_TEST := $(PWD)/integration-test-tmp)
-	$(eval GOTEST_INTEGRATION := TMPDIR=$(TMPDIR_TEST) $(GOTEST_INTEGRATION))
-	rm -rf $(TMPDIR_TEST)
-	mkdir $(TMPDIR_TEST)
+	$(eval TMPDIR_INTEGRATION_TEST := $(PWD)/integration-test-tmp)
+	$(eval GOTEST_INTEGRATION := TMPDIR=$(TMPDIR_INTEGRATION_TEST) $(GOTEST_INTEGRATION))
+	rm -rf $(TMPDIR_INTEGRATION_TEST)
+	mkdir $(TMPDIR_INTEGRATION_TEST)
 else
 test-integration-clean:
 endif
 
+ifeq ($(OS),Darwin)
+test-regression-clean:
+	$(eval TMPDIR_REGRESSION_TEST := $(PWD)/regression-test-tmp)
+	$(eval GOTEST_REGRESSION := TMPDIR=$(TMPDIR_REGRESSION_TEST) $(GOTEST_REGRESSION))
+	rm -rf $(TMPDIR_REGRESSION_TEST)
+	mkdir $(TMPDIR_REGRESSION_TEST)
+else
+test-regression-clean:
+endif
+
 test-integration-no-build: test-integration-clean
 	TEST_PRUNE_WITH_IMAGE=false $(GOTEST_INTEGRATION) github.com/src-d/engine/cmdtests/
 	$(GOTEST_INTEGRATION) github.com/src-d/engine/cmdtests/ -run TestPruneTestSuite/TestRunningContainersWithImages
 
 test-integration: clean build docker-build test-integration-no-build
+
+test-regression-usage:
+	@echo
+	@echo "Usage: \`PREV_ENGINE_VERSION=<first engine version to compare (default: 'latest')> CURR_ENGINE_VERSION=<second engine version to compare (default: 'local:HEAD')> make test-regression\`"
+	@echo "Examples:"
+	@echo "- \`make test-regression\`                                                          # tests that latest version is forward-compatible with current (HEAD) version"
+	@echo "- \`PREV_ENGINE_VERSION=v0.10.0 make test-regression\`                              # tests that v0.10.0 version is forward-compatible with current (HEAD) version"
+	@echo "- \`PREV_ENGINE_VERSION=v0.10.0 CURR_ENGINE_VERSION=v0.11.0 make test-regression\`  # tests that v0.10.0 version is forward-compatible with v0.11.0 version"
+	@echo
+
+test-regression: test-regression-usage test-regression-clean
+	$(GOTEST_REGRESSION) github.com/src-d/engine/cmdtests/

cmdtests/commander.go

@@ -0,0 +1,53 @@
+// +build integration regression
+
+package cmdtests
+
+import (
+	"path"
+	"runtime"
+	"time"
+
+	"gotest.tools/icmd"
+)
+
+type Commander struct {
+	bin string
+}
+
+func NewCommander(bin string) *Commander {
+	return &Commander{bin: bin}
+}
+
+func (s *Commander) Bin() string {
+	return s.bin
+}
+
+func (s *Commander) RunCmd(cmd string, args []string, cmdOperators ...icmd.CmdOp) *icmd.Result {
+	args = append([]string{cmd}, args...)
+	return icmd.RunCmd(icmd.Command(s.bin, args...), cmdOperators...)
+}
+
+func (s *Commander) RunCommand(cmd string, args ...string) *icmd.Result {
+	return s.RunCmd(cmd, args)
+}
+
+func (s *Commander) StartCommand(cmd string, args []string, cmdOperators ...icmd.CmdOp) *icmd.Result {
+	args = append([]string{cmd}, args...)
+	return icmd.StartCmd(icmd.Command(s.bin, args...))
+}
+
+func (s *Commander) Wait(timeout time.Duration, r *icmd.Result) *icmd.Result {
+	return icmd.WaitOnCmd(timeout, r)
+}
+
+// RunInit runs srcd init with workdir and custom config for integration tests
+func (s *Commander) RunInit(workdir string) *icmd.Result {
+	return s.RunInitWithTimeout(workdir, 0)
+}
+
+// RunInitWithTimeout runs srcd init with workdir and custom config for integration tests with timeout
+func (s *Commander) RunInitWithTimeout(workdir string, timeout time.Duration) *icmd.Result {
+	_, filename, _, _ := runtime.Caller(0)
+	configFile := path.Join(path.Dir(filename), "..", "integration-testing-config.yaml")
+	return s.RunCmd("init", []string{workdir, "--config", configFile}, icmd.WithTimeout(timeout))
+}

cmdtests/common.go

@@ -1,4 +1,4 @@
-// +build integration
+// +build integration regression
 
 package cmdtests
 
@@ -7,19 +7,19 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"reflect"
 	"regexp"
 	"runtime"
+	"sort"
 	"strings"
 	"time"
 
 	"github.com/src-d/engine/docker"
 	"github.com/stretchr/testify/suite"
-	"gotest.tools/icmd"
 )
 
 // TODO (carlosms) this could be build/bin, workaround for https://github.com/src-d/ci/issues/97
 var srcdBin = fmt.Sprintf("../build/engine_%s_%s/srcd", runtime.GOOS, runtime.GOARCH)
-var configFile = "../integration-testing-config.yaml"
 
 func init() {
 	if os.Getenv("SRCD_BIN") != "" {
@@ -29,6 +29,11 @@ func init() {
 
 type IntegrationSuite struct {
 	suite.Suite
+	*Commander
+}
+
+func NewIntegrationSuite() IntegrationSuite {
+	return IntegrationSuite{Commander: &Commander{bin: srcdBin}}
 }
 
 func (s *IntegrationSuite) SetupTest() {
@@ -40,33 +45,6 @@ func (s *IntegrationSuite) SetupTest() {
 	s.Require().NoError(r.Error, r.Combined())
 }
 
-func (s *IntegrationSuite) Bin() string {
-	return srcdBin
-}
-
-func (s *IntegrationSuite) RunCmd(cmd string, args []string, cmdOperators ...icmd.CmdOp) *icmd.Result {
-	args = append([]string{cmd}, args...)
-	return icmd.RunCmd(icmd.Command(srcdBin, args...), cmdOperators...)
-}
-
-func (s *IntegrationSuite) RunCommand(cmd string, args ...string) *icmd.Result {
-	return s.RunCmd(cmd, args)
-}
-
-func (s *IntegrationSuite) StartCommand(cmd string, args []string, cmdOperators ...icmd.CmdOp) *icmd.Result {
-	args = append([]string{cmd}, args...)
-	return icmd.StartCmd(icmd.Command(srcdBin, args...))
-}
-
-func (s *IntegrationSuite) Wait(timeout time.Duration, r *icmd.Result) *icmd.Result {
-	return icmd.WaitOnCmd(timeout, r)
-}
-
-// RunInit runs srcd init with workdir and custom config for integration tests
-func (s *IntegrationSuite) RunInit(workdir string) *icmd.Result {
-	return s.RunCommand("init", workdir, "--config", configFile)
-}
-
 var logMsgRegex = regexp.MustCompile(`.*msg="(.+?[^\\])"`)
 
 func (s *IntegrationSuite) ParseLogMessages(memLog string) []string {
@@ -111,6 +89,10 @@ type IntegrationTmpDirSuite struct {
 	TestDir string
 }
 
+func NewIntegrationTmpDirSuite() IntegrationTmpDirSuite {
+	return IntegrationTmpDirSuite{IntegrationSuite: NewIntegrationSuite()}
+}
+
 func (s *IntegrationTmpDirSuite) SetupTest() {
 	s.IntegrationSuite.SetupTest()
 
@@ -138,7 +120,89 @@ func (cr *ChannelWriter) Write(b []byte) (int, error) {
 	return len(b), nil
 }
 
+type RegressionSuite struct {
+	suite.Suite
+	PrevCmd *Commander
+	CurrCmd *Commander
+}
+
+func NewRegressionSuite(prevBin, currentBin string) RegressionSuite {
+	return RegressionSuite{
+		PrevCmd: &Commander{bin: prevBin},
+		CurrCmd: &Commander{bin: currentBin},
+	}
+}
+
+type SQLOutputTable struct {
+	Data  map[string][]string
+	cols  []string
+	rowsN int
+}
+
+func (s *SQLOutputTable) RequireEqual(o *SQLOutputTable) error {
+	return s.requireEqual(o, false)
+}
+
+func (s *SQLOutputTable) RequireStrictlyEqual(o *SQLOutputTable) error {
+	return s.requireEqual(o, true)
+}
+
+func (s *SQLOutputTable) requireEqual(o *SQLOutputTable, strictEmpty bool) error {
+	if !strictEmpty && s.rowsN == 0 && o.rowsN == 0 {
+		return nil
+	}
+
+	if s.rowsN != o.rowsN {
+		return s.diffErr("rows number", s.rowsN, o.rowsN)
+	}
+
+	if !reflect.DeepEqual(s.cols, o.cols) {
+		return s.diffErr("columns", s.cols, o.cols)
+	}
+
+	var thisRows []string
+	var otherRows []string
+
+	for i := 0; i < s.rowsN; i++ {
+		var thisRow []string
+		var otherRow []string
+
+		for _, c := range s.cols {
+			thisRow = append(thisRow, s.Data[c][i])
+			otherRow = append(otherRow, o.Data[c][i])
+		}
+
+		thisRows = append(thisRows, strings.Join(thisRow, "|"))
+		otherRows = append(otherRows, strings.Join(otherRow, "|"))
+	}
+
+	sort.Strings(thisRows)
+	sort.Strings(otherRows)
+
+	eq := reflect.DeepEqual(thisRows, otherRows)
+	if eq {
+		return nil
+	}
+
+	return s.diffErr("rows", thisRows, otherRows)
+}
+
+func (s *SQLOutputTable) diffErr(what string, this, other interface{}) error {
+	return fmt.Errorf("Different %s:\n- actual:   %v\n- expected: %v",
+		what, this, other)
+}
+
+func AreSQLOutputStrictlyEqual(s1 string, s2 string) error {
+	return ParseSQLOutput(s1).RequireStrictlyEqual(ParseSQLOutput(s2))
+}
+
+func AreSQLOutputEqual(s1 string, s2 string) error {
+	return ParseSQLOutput(s1).RequireEqual(ParseSQLOutput(s2))
+}
+
 var newLineFormatter = regexp.MustCompile(`(\r\n|\r|\n)`)
+var lineSepReg = regexp.MustCompile(`^\+[-+]+\+$`)
+var lineReg = regexp.MustCompile(`(?:\s+((?:[\w-\s.]+)?)\s+)`)
 
 func normalizeNewLine(s string) string {
 	return newLineFormatter.ReplaceAllString(s, "\n")
@@ -215,3 +279,72 @@ func (sl *StreamLinifier) Linify(in chan string) chan string {
 
 	return out
 }
+
+func normalizeColName(s string) string {
+	normCol := strings.ToUpper(strings.TrimSpace(s))
+	return strings.Replace(
+		strings.Replace(normCol, " ", "_", -1),
+		"-", "_", -1)
+}
+
+// ParseSQLOutput parses a string into a `SQLOutputTable` in order to facilitate
+// comparisons between different results.
+// This has been introduced mainly for two different reasons:
+//   1. handling ordering of rows (assuming that in equality checks the order is
+//      not important). This could be solved also by forcing the queries to have an
+//      'ORDER BY' clause.
+//   2. during the replacement of the SQL cli, there was a version using our custom
+//      SQL cli and another using the MySQL one. In one case the headers were all
+//      capitalized and in the other were not. Additionally, if the header is
+//      composed by multiple words, the separator in one case is the space and in
+//      the other is the dash. Moreover, in our SQL cli the newline was '\n' while on
+//      MySQL is '\r\n'. This latter thing is also a minimal problem as we can simply
+//      replace all the newlines of the buffer.
+func ParseSQLOutput(raw string) *SQLOutputTable {
+	splitted := strings.Split(normalizeNewLine(raw), "\n")
+	header := false
+	body := false
+	var cols []string
+	fields := make(map[string][]string)
+	nRows := 0
+	for _, s := range splitted {
+		if !header && !body {
+			if lineSepReg.MatchString(s) {
+				header = true
+			}
+
+			continue
+		}
+
+		if header {
+			if lineSepReg.MatchString(s) {
+				header = false
+				body = true
+				continue
+			}
+
+			for _, match := range lineReg.FindAllStringSubmatch(s, -1) {
+				cols = append(cols, normalizeColName(match[1]))
+			}
+		}
+
+		if body {
+			if lineSepReg.MatchString(s) {
+				break
+			}
+
+			nRows++
+			for i, match := range lineReg.FindAllStringSubmatch(s, -1) {
+				key := cols[i]
+				fields[key] = append(fields[key], match[1])
+			}
+		}
+	}
+
+	sort.Strings(cols)
+	return &SQLOutputTable{
+		Data:  fields,
+		cols:  cols,
+		rowsN: nRows,
+	}
+}

cmdtests/components_test.go

@@ -19,7 +19,7 @@ type ComponentsTestSuite struct {
 }
 
 func TestComponentsTestSuite(t *testing.T) {
-	s := ComponentsTestSuite{}
+	s := ComponentsTestSuite{IntegrationTmpDirSuite: cmdtests.NewIntegrationTmpDirSuite()}
 	suite.Run(t, &s)
 }