refactor: redefine configuration structure for all Database Lab products (#278)

To standardize configuration file names and store metadata:
* redefine configuration structure for all Database Lab products
* adjust Docker images of Database Lab Engine and CI Checker
* remove old Docker images

Author: agneum

Diff for: .gitignore

@@ -8,5 +8,7 @@
 /deploy/
 
 /configs/config.yml
+/configs/server.yml
 /configs/run_ci.yaml
+/configs/ci_checker.yml
 /packer/example.com.key

Diff for: Dockerfile.ci-checker

@@ -6,6 +6,5 @@ RUN apk update && apk add --no-cache bash
 WORKDIR /home/dblab
 
 COPY ./bin/run-ci ./bin/run-ci
-COPY ./configs ./configs
 
 CMD ./bin/run-ci

Diff for: Dockerfile.dblab-server

@@ -1,33 +1,4 @@
-# Run with server.yml mount:
-# sudo docker run \
-#   --name dblab_server \
-#   --label dblab_control \
-#   --privileged \
-#   --publish 2345:2345 \
-#   --restart on-failure \
-#   --volume /var/run/docker.sock:/var/run/docker.sock \
-#   --volume /var/lib/dblab:/var/lib/dblab:rshared \
-#   --volume ~/.dblab/server.yml:/home/dblab/configs/config.yml \
-#   --env VERIFICATION_TOKEN=secret_token \
-#   --detach \
-#   postgresai/dblab-server:latest
-#
-# or run with envs options:
-# sudo docker run \
-#   --name dblab_server \
-#   --label dblab_control \
-#   --privileged \
-#   --publish 2345:2345 \
-#   --restart on-failure \
-#   --volume /var/run/docker.sock:/var/run/docker.sock \
-#   --volume /var/lib/dblab:/var/lib/dblab:rshared \
-#   --volume ~/.dblab/server.yml:/home/dblab/configs/config.yml \
-#   --env VERIFICATION_TOKEN=token \
-#   --env MOUNT_DIR=/var/lib/dblab/clones \
-#   --env UNIX_SOCKET_DIR=/var/lib/dblab/sockets \
-#   --env DOCKER_IMAGE=postgres:13-alpine \
-#   --detach \
-#   postgresai/dblab-server:latest
+# See Guides to learn how to start a container: https://postgres.ai/docs/how-to-guides/administration/engine-manage
 
 FROM docker:19.03.14
 
@@ -46,7 +17,7 @@ WORKDIR /home/dblab
 COPY ./bin/dblab-server ./bin/dblab-server
 COPY ./api ./api
 COPY ./web ./web
-COPY ./configs ./configs
+COPY ./configs/standard ./standard
 COPY ./scripts ./scripts
 
 CMD ./bin/dblab-server

Diff for: Dockerfile.runci

@@ -13,7 +13,8 @@ import (
 )
 
 const (
-	configPath     = ".dblab"
+	dblabDir       = ".dblab"
+	configPath     = "cli"
 	configFilename = "cli.yml"
 )
 
@@ -24,7 +25,7 @@ func GetDirname() (string, error) {
 		return "", err
 	}
 
-	dirname := path.Join(currentUser.HomeDir, configPath)
+	dirname := path.Join(currentUser.HomeDir, dblabDir, configPath)
 
 	return dirname, nil
 }

Diff for: Dockerfile.sync-instance

@@ -49,7 +49,7 @@ func main() {
 
 	log.Msg("Database Lab Instance ID:", instanceID)
 
-	cfg, err := loadConfiguration(instanceID)
+	cfg, err := config.LoadConfiguration(instanceID)
 	if err != nil {
 		log.Fatal(errors.WithMessage(err, "failed to parse config"))
 	}
@@ -161,23 +161,9 @@ func main() {
 	shutdownDatabaseLabEngine(shutdownCtx, dockerCLI, cfg.Global, pm.Active().Pool())
 }
 
-func loadConfiguration(instanceID string) (*config.Config, error) {
-	cfg, err := config.LoadConfig("config.yml")
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to parse config")
-	}
-
-	log.SetDebug(cfg.Global.Debug)
-	log.Dbg("Config loaded", cfg)
-
-	cfg.Global.InstanceID = instanceID
-
-	return cfg, nil
-}
-
 func reloadConfig(ctx context.Context, instanceID string, provisionSvc *provision.Provisioner, retrievalSvc *retrieval.Retrieval,
 	pm *pool.Manager, cloningSvc cloning.Cloning, platformSvc *platform.Service, est *estimator.Estimator, server *srv.Server) error {
-	cfg, err := loadConfiguration(instanceID)
+	cfg, err := config.LoadConfiguration(instanceID)
 	if err != nil {
 		return err
 	}

Diff for: cmd/cli/commands/config/file.go

@@ -10,15 +10,13 @@ import (
 	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/client"
 	"github.com/pkg/errors"
-	"gopkg.in/yaml.v2"
 
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/client/dblabapi"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/log"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/retrieval/engine/postgres/tools"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/runci"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/runci/source"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/services/platform"
-	"gitlab.com/postgres-ai/database-lab/v2/pkg/util"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/util/networks"
 )
 
@@ -31,12 +29,15 @@ func main() {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	cfg, err := loadConfiguration()
+	cfg, err := runci.LoadConfiguration()
 	if err != nil {
 		log.Errf("Failed to load config: %v", err)
 		return
 	}
 
+	log.SetDebug(cfg.App.Debug)
+	log.Dbg("Config loaded: ", cfg)
+
 	networkID := discoverNetwork(ctx, cfg, dockerCLI)
 	if networkID != "" {
 		hostname := os.Getenv("HOSTNAME")
@@ -129,25 +130,3 @@ func discoverNetwork(ctx context.Context, cfg *runci.Config, dockerCLI *client.C
 
 	return networkID
 }
-
-func loadConfiguration() (*runci.Config, error) {
-	configPath, err := util.GetConfigPath("run_ci.yaml")
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to get config path")
-	}
-
-	b, err := os.ReadFile(configPath)
-	if err != nil {
-		return nil, errors.Errorf("error loading %s config file", configPath)
-	}
-
-	cfg := &runci.Config{}
-	if err := yaml.Unmarshal(b, cfg); err != nil {
-		return nil, errors.WithMessagef(err, "error parsing %s config", configPath)
-	}
-
-	log.SetDebug(cfg.App.Debug)
-	log.Dbg("Config loaded: ", cfg)
-
-	return cfg, nil
-}

Diff for: cmd/database-lab/main.go

@@ -1,4 +1,4 @@
-# Copy the following to: ~/.dblab/server.yml
+# Copy the following to: ~/.dblab/engine/configs/server.yml
 
 # Database Lab API server. This API is used to work with clones
 # (list them, create, delete, see how to connect to a clone).

Diff for: cmd/runci/main.go

@@ -1,4 +1,4 @@
-# Copy the following to: ~/.dblab/server.yml
+# Copy the following to: ~/.dblab/engine/configs/server.yml
 
 # Database Lab API server. This API is used to work with clones
 # (list them, create, delete, see how to connect to a clone).

Diff for: configs/config.example.run_ci.yaml renamed to configs/config.example.ci_checker.yml

@@ -1,4 +1,4 @@
-# Copy the following to: ~/.dblab/server.yml
+# Copy the following to: ~/.dblab/engine/configs/server.yml
 
 # Database Lab API server. This API is used to work with clones
 # (list them, create, delete, see how to connect to a clone).

Diff for: configs/config.example.logical_generic.yml

@@ -1,4 +1,4 @@
-# Copy the following to: ~/.dblab/server.yml
+# Copy the following to: ~/.dblab/engine/configs/server.yml
 
 # Database Lab API server. This API is used to work with clones
 # (list them, create, delete, see how to connect to a clone).

Diff for: configs/config.example.logical_rds_iam.yml

@@ -5,7 +5,7 @@ app:
   verificationToken: "secret_token"
 dle:
   url: "https://dblab_server"
-  verificationToken: "dle_cerification_token"
+  verificationToken: "dle_certification_token"
 platform:
   url: "https://postgres.ai/api/general"
   accessToken: "platform_access_token"

Diff for: configs/config.example.physical_generic.yml

@@ -47,12 +47,12 @@ build {
 
   provisioner "file"{
     source = "joe.yml"
-    destination = "/home/ubuntu/joe.yml"
+    destination = "/home/ubuntu/.dblab/configs/joe.yml"
   }
 
   provisioner "file"{
-    source = "run_ci.yaml"
-    destination = "/home/ubuntu/run_ci.yaml"
+    source = "ci_checker.yml"
+    destination = "/home/ubuntu/.dblab/configs/ci_checker.yml"
   }
 
   provisioner "shell" {

Diff for: configs/config.example.physical_walg.yml

@@ -13,6 +13,7 @@ import (
 
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/config/global"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/estimator"
+	"gitlab.com/postgres-ai/database-lab/v2/pkg/log"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/observer"
 	retConfig "gitlab.com/postgres-ai/database-lab/v2/pkg/retrieval/config"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/services/cloning"
@@ -23,6 +24,10 @@ import (
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/util"
 )
 
+const (
+	configName = "server.yml"
+)
+
 // Config contains a common database-lab configuration.
 type Config struct {
 	Server      srv.Config       `yaml:"server"`
@@ -36,21 +41,36 @@ type Config struct {
 	PoolManager pool.Config      `yaml:"poolManager"`
 }
 
-// LoadConfig instances a new Config by configuration filename.
-func LoadConfig(name string) (*Config, error) {
-	configPath, err := util.GetConfigPath(name)
+// LoadConfiguration instances a new application configuration.
+func LoadConfiguration(instanceID string) (*Config, error) {
+	cfg, err := readConfig()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to parse config")
+	}
+
+	log.SetDebug(cfg.Global.Debug)
+	log.Dbg("Config loaded", cfg)
+
+	cfg.Global.InstanceID = instanceID
+
+	return cfg, nil
+}
+
+// readConfig reads application configuration.
+func readConfig() (*Config, error) {
+	configPath, err := util.GetConfigPath(configName)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get config path")
 	}
 
 	b, err := os.ReadFile(configPath)
 	if err != nil {
-		return nil, errors.Errorf("error loading %s config file", name)
+		return nil, errors.Errorf("error loading %s config file", configPath)
 	}
 
 	cfg := &Config{}
 	if err := yaml.Unmarshal(b, cfg); err != nil {
-		return nil, errors.WithMessagef(err, "error parsing %s config", name)
+		return nil, errors.WithMessagef(err, "error parsing %s config", configPath)
 	}
 
 	return cfg, nil

Diff for: configs/postgres/pg_hba.conf renamed to configs/standard/postgres/control/pg_hba.conf

@@ -2,12 +2,22 @@
 2021 © Postgres.ai
 */
 
-// Package runci provides a tools to run and check migrations in CI.
+// Package runci provides tools to run and check migrations in CI.
 package runci
 
 import (
+	"io/ioutil"
+
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
+
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/runci/source"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/services/platform"
+	"gitlab.com/postgres-ai/database-lab/v2/pkg/util"
+)
+
+const (
+	configFilename = "ci_checker.yml"
 )
 
 // Config contains a runner configuration.
@@ -39,3 +49,23 @@ type DLE struct {
 type Runner struct {
 	Image string `yaml:"image"`
 }
+
+// LoadConfiguration loads configuration of DB Migration Checker.
+func LoadConfiguration() (*Config, error) {
+	configPath, err := util.GetConfigPath(configFilename)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get config path")
+	}
+
+	b, err := ioutil.ReadFile(configPath)
+	if err != nil {
+		return nil, errors.Errorf("error loading %s config file", configPath)
+	}
+
+	cfg := &Config{}
+	if err := yaml.Unmarshal(b, cfg); err != nil {
+		return nil, errors.WithMessagef(err, "error parsing %s config", configPath)
+	}
+
+	return cfg, nil
+}