Merge pull request #403 from arangodb-helper/GT-592-deprecate-arangosync

GT-592 Deprecate ArangoSync

Author: jwierzbo

.circleci/config.yml

@@ -3,21 +3,21 @@ version: 2.1
 parameters:
   goImage:
     type: string
-    default: "gcr.io/gcr-for-testing/golang:1.21.5"
+    default: "gcr.io/gcr-for-testing/golang:1.21.6"
   arangodbImage:
     type: string
-    default: "gcr.io/gcr-for-testing/arangodb/enterprise:3.9.2"
+    default: "gcr.io/gcr-for-testing/arangodb/enterprise-preview:devel-nightly"
   alpineImage:
     type: string
-    default: "gcr.io/gcr-for-testing/alpine:3.17"
+    default: "gcr.io/gcr-for-testing/alpine:3.18"
   starterModes:
     type: string
-    default: "single,cluster,activefailover"
+    default: "single,cluster"
 
 executors:
   golang-executor:
     docker:
-      - image: gcr.io/gcr-for-testing/golang:1.21.5
+      - image: gcr.io/gcr-for-testing/golang:1.21.6
   machine-executor:
     machine:
       image: ubuntu-2204:current
@@ -52,7 +52,6 @@ jobs:
       GOIMAGE: << pipeline.parameters.goImage >>
       ALPINE_IMAGE: << pipeline.parameters.alpineImage >>
       STARTER_MODES: << pipeline.parameters.starterModes >>
-      MULTIARCH: 0
       VERBOSE: 1
 
 workflows:

.gitignore

@@ -10,4 +10,4 @@ vendor/
 .idea
 
 resource.syso
-
+CONTAINER

CHANGELOG.md

@@ -1,6 +1,10 @@
 # ArangoDB Starter Changelog
 
 ## [master](https://github.com/arangodb-helper/arangodb/tree/master) (N/A)
+- Mark ArangoSync component as deprecated
+- Do not allow starting ArangoDB v3.12+ with ArangoSync component
+- Do not allow upgrade if the cluster is using ArangoSync component (if the new version is v3.12 or newer)
+- Bump Go version (v1.21.6) and dependencies for CVE fixes
 
 ## [v0.18.2](https://github.com/arangodb-helper/arangodb/tree/0.18.2) (2024-02-13)
 - Fix waiting for local slaves to terminate

Dockerfile

@@ -1,17 +1,18 @@
-ARG IMAGE=alpine:3.16
+ARG IMAGE
 FROM ${IMAGE}
 MAINTAINER Max Neunhoeffer <[email protected]>
 
-COPY bin/linux/amd64/arangodb /app/
+ARG TARGETARCH
+COPY bin/linux/${TARGETARCH}/arangodb /app/
 
-EXPOSE 8528 
+EXPOSE 8528
 
 VOLUME /data
 
-# Data directory 
-ENV DATA_DIR=/data 
+# Data directory
+ENV DATA_DIR=/data
 
-# Signal running in docker 
+# Signal running in docker
 ENV RUNNING_IN_DOCKER=true
 
 # Docker image containing arangod.

Dockerfile.ma

@@ -21,14 +21,8 @@ ifeq (, $(findstring -preview,$(VERSION)))
 		-t $(IMAGE_NAME):latest
 endif
 
-ALPINE_IMAGE ?= alpine:3.16
-
-DOCKERCLI ?= $(shell which docker)
 GOBUILDLINKTARGET := ../../../..
 
-DOCKERMACLI := $(DOCKERCLI) buildx build -f "$(ROOTDIR)/Dockerfile.ma" --build-arg "IMAGE=$(ALPINE_IMAGE)" \
-               			--platform linux/amd64,linux/arm64
-
 BUILDDIR ?= $(ROOTDIR)
 
 GOBUILDDIR := $(BUILDDIR)/.gobuild
@@ -43,9 +37,11 @@ REPONAME := $(PROJECT)
 REPODIR := $(ORGDIR)/$(REPONAME)
 REPOPATH := $(ORGPATH)/$(REPONAME)
 
+ALPINE_IMAGE ?= alpine:3.18
+
 GOPATH := $(GOBUILDDIR)
-GOVERSION := 1.21.5
-GOIMAGE ?= golang:$(GOVERSION)-alpine3.17
+GOVERSION := 1.21.6
+GOIMAGE ?= golang:$(GOVERSION)-alpine3.18
 
 GOOS ?= linux
 GOARCH ?= amd64
@@ -54,6 +50,9 @@ ifeq ("$(GOOS)", "windows")
 	GOEXE := .exe
 endif
 
+DOCKERCLI ?= $(shell which docker)
+DOCKER_BUILD_CLI := $(DOCKERCLI) buildx build --build-arg "IMAGE=$(ALPINE_IMAGE)" --platform linux/$(GOARCH)
+
 ARANGODB ?= arangodb/arangodb:latest
 
 IP ?= $(shell hostname -I | cut -d ' ' -f 1)
@@ -140,15 +139,28 @@ $(RELEASEBIN): vendor $(BIN)
 	@cp "$(BIN)" "$(RELEASEBIN)"
 
 build: vendor $(BIN)
+	@echo ">> Build Bin $(BIN) done"
 
 build-test: vendor $(TESTBIN)
+	@echo ">> Build Tests Bin $(TESTBIN) done"
 
-binaries:
+binary-linux:
 	@${MAKE} -f $(MAKEFILE) -B GOOS=linux GOARCH=amd64 build
 	@${MAKE} -f $(MAKEFILE) -B GOOS=linux GOARCH=arm64 build
+	@${MAKE} -f $(MAKEFILE) -B GOOS=linux GOARCH=amd64 build-test
+	@${MAKE} -f $(MAKEFILE) -B GOOS=linux GOARCH=arm64 build-test
+
+binary-darwin:
 	@${MAKE} -f $(MAKEFILE) -B GOOS=darwin GOARCH=amd64 build
 	@${MAKE} -f $(MAKEFILE) -B GOOS=darwin GOARCH=arm64 build
+	@${MAKE} -f $(MAKEFILE) -B GOOS=darwin GOARCH=amd64 build-test
+	@${MAKE} -f $(MAKEFILE) -B GOOS=darwin GOARCH=arm64 build-test
+
+binary-windows:
 	@${MAKE} -f $(MAKEFILE) -B GOOS=windows GOARCH=amd64 build
+	@${MAKE} -f $(MAKEFILE) -B GOOS=windows GOARCH=amd64 build-test
+
+binaries: binary-linux binary-darwin binary-windows
 
 releases:
 	@${MAKE} -f $(MAKEFILE) -B GOOS=linux GOARCH=amd64 release
@@ -158,20 +170,14 @@ releases:
 	@${MAKE} -f $(MAKEFILE) -B GOOS=windows GOARCH=amd64 release
 	@(cd "$(RELEASEDIR)"; sha256sum arangodb-* > SHA256SUMS; cat SHA256SUMS | sha256sum -c)
 
-binaries-test:
-	@${MAKE} -f $(MAKEFILE) -B GOOS=linux GOARCH=amd64 build-test
-	@${MAKE} -f $(MAKEFILE) -B GOOS=linux GOARCH=arm64 build-test
-	@${MAKE} -f $(MAKEFILE) -B GOOS=darwin GOARCH=amd64 build-test
-	@${MAKE} -f $(MAKEFILE) -B GOOS=darwin GOARCH=arm64 build-test
-	@${MAKE} -f $(MAKEFILE) -B GOOS=windows GOARCH=amd64 build-test
-
 $(BIN): $(GOBUILDDIR) $(GO_SOURCES)
 	@mkdir -p $(BINDIR)
-	@-rm resource.syso
+	@-rm -f resource.syso
 ifeq ($(GOOS),windows)
 	@echo ">> Generating versioninfo syso file ..."
 	@$(GOPATH)/bin/goversioninfo -64 -file-version=$(VERSION) -product-version=$(VERSION)
 	$(DOCKER_CMD) go build -ldflags "-X main.projectVersion=$(VERSION) -X main.projectBuild=$(COMMIT)" -o "$(BUILD_BIN)" .
+	@-rm -f resource.syso
 else
 	$(DOCKER_CMD) go build -installsuffix netgo -tags netgo -ldflags "-X main.projectVersion=$(VERSION) -X main.projectBuild=$(COMMIT)" -o "$(BUILD_BIN)" .
 endif
@@ -180,17 +186,12 @@ $(TESTBIN): $(GOBUILDDIR) $(TEST_SOURCES) $(BIN)
 	@mkdir -p $(BINDIR)
 	$(DOCKER_CMD) go test -c -o "$(TEST_BIN)" ./test
 
-ifneq ($(MULTIARCH),1)
 docker: build
-	$(DOCKERCLI) build -t arangodb/arangodb-starter --build-arg "IMAGE=$(ALPINE_IMAGE)" .
-else
-docker: binaries
-	$(DOCKERMACLI) -t arangodb/arangodb-starter .
-endif
+	@echo ">> Building Docker Image with buildx"
+	$(DOCKER_BUILD_CLI) -t arangodb/arangodb-starter .
 
 docker-push-version: docker
-	$(DOCKERMACLI) --push $(STARTER_TAGS) .
-
+	$(DOCKER_BUILD_CLI) --push $(STARTER_TAGS) .
 
 $(RELEASE): $(GOBUILDDIR) $(GO_SOURCES)
 	$(DOCKER_CMD) go build -o "$(RELEASE_BIN)" $(REPOPATH)/tools/release
@@ -217,7 +218,7 @@ TESTCONTAINER := arangodb-starter-test
 
 # Run all unit tests
 run-unit-tests: $(GO_SOURCES)
-	go test --count=1 \
+	go test --count=1 -v \
 		$(REPOPATH) \
 		$(REPOPATH)/pkg/... \
 		$(REPOPATH)/service/... \
@@ -226,13 +227,13 @@ run-unit-tests: $(GO_SOURCES)
 # Run all integration tests
 run-tests: run-tests-local-process run-tests-docker
 
-run-tests-local-process: build-test build run-tests-local-process-run
+run-tests-local-process: binary-linux run-tests-local-process-run
 run-tests-local-process-run: export TEST_MODES=localprocess
 run-tests-local-process-run: export DOCKER_IMAGE=$(ARANGODB)
-run-tests-local-process-run: export DOCKER_PARAMS:=-e "TEST_MODES=$(TEST_MODES)" -e "STARTER_MODES=$(STARTER_MODES)" -e "STARTER=/usr/code/bin/linux/amd64/arangodb"
+run-tests-local-process-run: export DOCKER_PARAMS:=-e "TEST_MODES=$(TEST_MODES)" -e "STARTER_MODES=$(STARTER_MODES)" -e "STARTER=/usr/code/bin/linux/$(GOARCH)/arangodb"
 run-tests-local-process-run:
 	@-$(DOCKERCLI) rm -f -v $(TESTCONTAINER) &> /dev/null
-	$(DOCKER_CMD) /usr/code/bin/linux/amd64/test -test.timeout $(TEST_TIMEOUT) -test.v $(TESTOPTIONS)
+	$(DOCKER_CMD) /usr/code/bin/linux/$(GOARCH)/test -test.timeout $(TEST_TIMEOUT) -test.v $(TESTOPTIONS)
 
 _run-tests: build-test build
 	@TEST_MODES=$(TEST_MODES) STARTER_MODES=$(STARTER_MODES) STARTER=$(BIN) IP=$(IP) ARANGODB=$(ARANGODB) $(TESTBIN) -test.timeout $(TEST_TIMEOUT) -test.failfast -test.v $(TESTOPTIONS)

Makefile

@@ -150,6 +150,7 @@ func init() {
 	f.BoolVar(&opts.starter.debugCluster, "starter.debug-cluster", getEnvVar("DEBUG_CLUSTER", "") != "", "If set, log more information to debug a cluster")
 	f.BoolVar(&opts.starter.disableIPv6, "starter.disable-ipv6", !net.IsIPv6Supported(), "If set, no IPv6 notation will be used. Use this only when IPv6 address family is disabled")
 	f.BoolVar(&opts.starter.enableSync, "starter.sync", false, "If set, the starter will also start arangosync instances")
+	f.MarkDeprecated("starter.sync", "ArangoSync component is deprecated and will be removed in coming releases")
 	f.DurationVar(&opts.starter.instanceUpTimeout, "starter.instance-up-timeout", defaultInstanceUpTimeout, "Timeout to wait for an instance start")
 	if err := features.JWTRotation().RegisterDeprecated(f); err != nil {
 		panic(err)
@@ -169,6 +170,7 @@ func init() {
 	f.BoolVar(&opts.server.useLocalBin, "server.use-local-bin", false, "If true, starter will try searching for binaries in local directory first")
 	f.StringVar(&opts.server.arangodPath, "server.arangod", defaultArangodPath, "Path of arangod")
 	f.StringVar(&opts.server.arangoSyncPath, "server.arangosync", defaultArangoSyncPath, "Path of arangosync")
+	f.MarkDeprecated("server.arangosync", "ArangoSync component is deprecated and will be removed in coming releases")
 	f.StringVar(&opts.server.arangodJSPath, "server.js-dir", "/usr/share/arangodb3/js", "Path of arango JS folder")
 	f.StringVar(&opts.server.rrPath, "server.rr", "", "Path of rr")
 	f.IntVar(&opts.server.threads, "server.threads", 0, "Adjust server.threads of each server")
@@ -180,6 +182,7 @@ func init() {
 	f.StringVar(&opts.docker.Endpoint, "docker.endpoint", "unix:///var/run/docker.sock", "Endpoint used to reach the docker daemon")
 	f.StringVar(&opts.docker.ImageArangoD, "docker.image", getEnvVar("DOCKER_IMAGE", ""), "name of the Docker image to use to launch arangod instances (leave empty to avoid using docker)")
 	f.StringVar(&opts.docker.ImageArangoSync, "docker.sync-image", getEnvVar("DOCKER_ARANGOSYNC_IMAGE", ""), "name of the Docker image to use to launch arangosync instances")
+	f.MarkDeprecated("docker.sync-image", "ArangoSync component is deprecated and will be removed in coming releases")
 	f.StringVar(&opts.docker.imagePullPolicyRaw, "docker.imagePullPolicy", "", "pull docker image from docker hub (Always|IfNotPresent|Never)")
 	f.StringVar(&opts.docker.User, "docker.user", "", "use the given name as user to run the Docker container")
 	f.StringVar(&opts.docker.HostContainerName, "docker.container", "", "name of the docker container that is running this process")
@@ -199,12 +202,19 @@ func init() {
 	f.StringVar(&opts.ssl.autoOrganization, "ssl.auto-organization", "ArangoDB", "Organization name put into self-signed certificate. See --ssl.auto-key")
 
 	f.BoolSliceVar(&opts.sync.startSyncMaster, "sync.start-master", nil, "should an ArangoSync master instance be started (only relevant when starter.sync is enabled)")
+	f.MarkDeprecated("sync.start-master", "ArangoSync component is deprecated and will be removed in coming releases")
 	f.BoolSliceVar(&opts.sync.startSyncWorker, "sync.start-worker", nil, "should an ArangoSync worker instance be started (only relevant when starter.sync is enabled)")
+	f.MarkDeprecated("sync.start-worker", "ArangoSync component is deprecated and will be removed in coming releases")
 	f.StringVar(&opts.sync.monitoring.token, "sync.monitoring.token", "", "Bearer token used to access ArangoSync monitoring endpoints")
+	f.MarkDeprecated("sync.monitoring.token", "ArangoSync component is deprecated and will be removed in coming releases")
 	f.StringVar(&opts.sync.master.jwtSecretFile, "sync.master.jwt-secret", "", "File containing JWT secret used to access the Sync Master (from Sync Worker)")
+	f.MarkDeprecated("sync.master.jwt-secret", "ArangoSync component is deprecated and will be removed in coming releases")
 	f.StringVar(&opts.sync.mq.Type, "sync.mq.type", "direct", "Type of message queue used by the Sync Master")
+	f.MarkDeprecated("sync.mq.type", "ArangoSync component is deprecated and will be removed in coming releases")
 	f.StringVar(&opts.sync.server.keyFile, "sync.server.keyfile", "", "TLS keyfile of local sync master")
+	f.MarkDeprecated("sync.server.keyfile", "ArangoSync component is deprecated and will be removed in coming releases")
 	f.StringVar(&opts.sync.server.clientCAFile, "sync.server.client-cafile", "", "CA Certificate used for client certificate verification")
+	f.MarkDeprecated("sync.server.client-cafile", "ArangoSync component is deprecated and will be removed in coming releases")
 
 	cmdMain.Flags().SetNormalizeFunc(normalizeOptionNames)
 
@@ -617,6 +627,7 @@ func mustPrepareService(generateAutoKeyFile bool) (*service.Service, service.Boo
 		if opts.sync.monitoring.token == "" {
 			opts.sync.monitoring.token = uniuri.New()
 		}
+		log.Warn().Msgf("ArangoSync component is deprecated and will be removed in coming releases")
 	} else {
 		opts.sync.startSyncMaster = []bool{false}
 		opts.sync.startSyncWorker = []bool{false}

main.go

@@ -211,23 +211,23 @@ func preparePassthroughPrefixes() options.ConfigurationPrefixes {
 		},
 		"args.sync": {
 			Usage: func(key string) string {
-				return fmt.Sprintf("Passed through to all sync instances as --%s", key)
+				return fmt.Sprintf("Deprecated: Passed through to all sync instances as --%s", key)
 			},
 			FieldSelector: func(p *options.Configuration, key string) *[]string {
 				return p.ArgByProcessTypeAndName(definitions.ServerTypeSyncMaster, key)
 			},
 		},
 		"args.syncmasters": {
 			Usage: func(key string) string {
-				return fmt.Sprintf("Passed through to all sync master instances as --%s", key)
+				return fmt.Sprintf("Deprecated: Passed through to all sync master instances as --%s", key)
 			},
 			FieldSelector: func(p *options.Configuration, key string) *[]string {
 				return p.ArgByServerTypeAndName(definitions.ServerTypeSyncMaster, key)
 			},
 		},
 		"args.syncworkers": {
 			Usage: func(key string) string {
-				return fmt.Sprintf("Passed through to all sync workers instances as --%s", key)
+				return fmt.Sprintf("Deprecated: Passed through to all sync workers instances as --%s", key)
 			},
 			FieldSelector: func(p *options.Configuration, key string) *[]string {
 				return p.ArgByServerTypeAndName(definitions.ServerTypeSyncWorker, key)
@@ -268,23 +268,23 @@ func preparePassthroughPrefixes() options.ConfigurationPrefixes {
 		},
 		"envs.sync": {
 			Usage: func(key string) string {
-				return fmt.Sprintf("Env passed to all sync instances as %s", key)
+				return fmt.Sprintf("Deprecated: Env passed to all sync instances as %s", key)
 			},
 			FieldSelector: func(p *options.Configuration, key string) *[]string {
 				return p.EnvByProcessTypeAndName(definitions.ServerTypeSyncMaster, key)
 			},
 		},
 		"envs.syncmasters": {
 			Usage: func(key string) string {
-				return fmt.Sprintf("Env passed to all sync master instances as %s", key)
+				return fmt.Sprintf("Deprecated: Env passed to all sync master instances as %s", key)
 			},
 			FieldSelector: func(p *options.Configuration, key string) *[]string {
 				return p.EnvByServerTypeAndName(definitions.ServerTypeSyncMaster, key)
 			},
 		},
 		"envs.syncworkers": {
 			Usage: func(key string) string {
-				return fmt.Sprintf("Env passed to all sync master instances as %s", key)
+				return fmt.Sprintf("Deprecated: Env passed to all sync master instances as %s", key)
 			},
 			FieldSelector: func(p *options.Configuration, key string) *[]string {
 				return p.EnvByServerTypeAndName(definitions.ServerTypeSyncWorker, key)

options.go

@@ -98,3 +98,8 @@ func (v DatabaseFeatures) GetJWTFolderOption() bool {
 func (v DatabaseFeatures) SupportsActiveFailover() bool {
 	return v.Version.CompareTo(v312) < 0
 }
+
+// SupportsArangoSync returns true if ArangoSync is supported
+func (v DatabaseFeatures) SupportsArangoSync() bool {
+	return v.Version.CompareTo(v312) < 0
+}

service/database_features.go

@@ -1262,7 +1262,13 @@ func (s *Service) Run(rootCtx context.Context, bsCfg BootstrapConfig, clusterCon
 
 	if !s.DatabaseFeatures().SupportsActiveFailover() {
 		if bsCfg.Mode.IsActiveFailoverMode() || boolFromRef(bsCfg.StartResilientSingle, false) {
-			return fmt.Errorf("This ArangoDB version does not support running in Active-Failover (resilient-single) mode")
+			return fmt.Errorf("this ArangoDB version does not support running in Active-Failover (resilient-single) mode")
+		}
+	}
+
+	if !s.DatabaseFeatures().SupportsArangoSync() {
+		if boolFromRef(bsCfg.StartSyncMaster, false) || boolFromRef(bsCfg.StartSyncWorker, false) {
+			return fmt.Errorf("this ArangoDB version does not support running with ArangoSync component")
 		}
 	}