refactor: redefine configuration structure for all Database Lab products (#278)

To standardize configuration file names and store metadata:
* redefine configuration structure for all Database Lab products
* adjust Docker images of Database Lab Engine and CI Checker
* remove old Docker images

Author: agneum

.gitignore

@@ -8,5 +8,7 @@
 /deploy/
 
 /configs/config.yml
+/configs/server.yml
 /configs/run_ci.yaml
+/configs/ci_checker.yml
 /packer/example.com.key

Dockerfile.ci-checker

@@ -6,6 +6,5 @@ RUN apk update && apk add --no-cache bash
 WORKDIR /home/dblab
 
 COPY ./bin/run-ci ./bin/run-ci
-COPY ./configs ./configs
 
 CMD ./bin/run-ci

Dockerfile.dblab-server

@@ -1,33 +1,4 @@
-# Run with server.yml mount:
-# sudo docker run \
-#   --name dblab_server \
-#   --label dblab_control \
-#   --privileged \
-#   --publish 2345:2345 \
-#   --restart on-failure \
-#   --volume /var/run/docker.sock:/var/run/docker.sock \
-#   --volume /var/lib/dblab:/var/lib/dblab:rshared \
-#   --volume ~/.dblab/server.yml:/home/dblab/configs/config.yml \
-#   --env VERIFICATION_TOKEN=secret_token \
-#   --detach \
-#   postgresai/dblab-server:latest
-#
-# or run with envs options:
-# sudo docker run \
-#   --name dblab_server \
-#   --label dblab_control \
-#   --privileged \
-#   --publish 2345:2345 \
-#   --restart on-failure \
-#   --volume /var/run/docker.sock:/var/run/docker.sock \
-#   --volume /var/lib/dblab:/var/lib/dblab:rshared \
-#   --volume ~/.dblab/server.yml:/home/dblab/configs/config.yml \
-#   --env VERIFICATION_TOKEN=token \
-#   --env MOUNT_DIR=/var/lib/dblab/clones \
-#   --env UNIX_SOCKET_DIR=/var/lib/dblab/sockets \
-#   --env DOCKER_IMAGE=postgres:13-alpine \
-#   --detach \
-#   postgresai/dblab-server:latest
+# See Guides to learn how to start a container: https://postgres.ai/docs/how-to-guides/administration/engine-manage
 
 FROM docker:19.03.14
 
@@ -46,7 +17,7 @@ WORKDIR /home/dblab
 COPY ./bin/dblab-server ./bin/dblab-server
 COPY ./api ./api
 COPY ./web ./web
-COPY ./configs ./configs
+COPY ./configs/standard ./standard
 COPY ./scripts ./scripts
 
 CMD ./bin/dblab-server

Dockerfile.runci

@@ -13,7 +13,8 @@ import (
 )
 
 const (
-	configPath     = ".dblab"
+	dblabDir       = ".dblab"
+	configPath     = "cli"
 	configFilename = "cli.yml"
 )
 
@@ -24,7 +25,7 @@ func GetDirname() (string, error) {
 		return "", err
 	}
 
-	dirname := path.Join(currentUser.HomeDir, configPath)
+	dirname := path.Join(currentUser.HomeDir, dblabDir, configPath)
 
 	return dirname, nil
 }

Dockerfile.sync-instance

@@ -49,7 +49,7 @@ func main() {
 
 	log.Msg("Database Lab Instance ID:", instanceID)
 
-	cfg, err := loadConfiguration(instanceID)
+	cfg, err := config.LoadConfiguration(instanceID)
 	if err != nil {
 		log.Fatal(errors.WithMessage(err, "failed to parse config"))
 	}
@@ -161,23 +161,9 @@ func main() {
 	shutdownDatabaseLabEngine(shutdownCtx, dockerCLI, cfg.Global, pm.Active().Pool())
 }
 
-func loadConfiguration(instanceID string) (*config.Config, error) {
-	cfg, err := config.LoadConfig("config.yml")
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to parse config")
-	}
-
-	log.SetDebug(cfg.Global.Debug)
-	log.Dbg("Config loaded", cfg)
-
-	cfg.Global.InstanceID = instanceID
-
-	return cfg, nil
-}
-
 func reloadConfig(ctx context.Context, instanceID string, provisionSvc *provision.Provisioner, retrievalSvc *retrieval.Retrieval,
 	pm *pool.Manager, cloningSvc cloning.Cloning, platformSvc *platform.Service, est *estimator.Estimator, server *srv.Server) error {
-	cfg, err := loadConfiguration(instanceID)
+	cfg, err := config.LoadConfiguration(instanceID)
 	if err != nil {
 		return err
 	}

cmd/cli/commands/config/file.go

@@ -10,15 +10,13 @@ import (
 	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/client"
 	"github.com/pkg/errors"
-	"gopkg.in/yaml.v2"
 
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/client/dblabapi"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/log"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/retrieval/engine/postgres/tools"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/runci"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/runci/source"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/services/platform"
-	"gitlab.com/postgres-ai/database-lab/v2/pkg/util"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/util/networks"
 )
 
@@ -31,12 +29,15 @@ func main() {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	cfg, err := loadConfiguration()
+	cfg, err := runci.LoadConfiguration()
 	if err != nil {
 		log.Errf("Failed to load config: %v", err)
 		return
 	}
 
+	log.SetDebug(cfg.App.Debug)
+	log.Dbg("Config loaded: ", cfg)
+
 	networkID := discoverNetwork(ctx, cfg, dockerCLI)
 	if networkID != "" {
 		hostname := os.Getenv("HOSTNAME")
@@ -129,25 +130,3 @@ func discoverNetwork(ctx context.Context, cfg *runci.Config, dockerCLI *client.C
 
 	return networkID
 }
-
-func loadConfiguration() (*runci.Config, error) {
-	configPath, err := util.GetConfigPath("run_ci.yaml")
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to get config path")
-	}
-
-	b, err := os.ReadFile(configPath)
-	if err != nil {
-		return nil, errors.Errorf("error loading %s config file", configPath)
-	}
-
-	cfg := &runci.Config{}
-	if err := yaml.Unmarshal(b, cfg); err != nil {
-		return nil, errors.WithMessagef(err, "error parsing %s config", configPath)
-	}
-
-	log.SetDebug(cfg.App.Debug)
-	log.Dbg("Config loaded: ", cfg)
-
-	return cfg, nil
-}

cmd/database-lab/main.go

@@ -1,4 +1,4 @@
-# Copy the following to: ~/.dblab/server.yml
+# Copy the following to: ~/.dblab/engine/configs/server.yml
 
 # Database Lab API server. This API is used to work with clones
 # (list them, create, delete, see how to connect to a clone).