Skip to content

Commit 5211d5c

Browse files
committed
Merge branch '310-recreate-clones' into 'master'
feat: keep clone containers after VM reboot (#310) Closes #310 See merge request postgres-ai/database-lab!389
2 parents cc0f09e + 1c0fdd6 commit 5211d5c

File tree

8 files changed

+78
-7
lines changed

8 files changed

+78
-7
lines changed

cmd/database-lab/main.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ func main() {
127127
defer retrievalSvc.Stop()
128128

129129
// Create a cloning service to provision new clones.
130-
provisionSvc, err := provision.New(ctx, &cfg.Provision, dbCfg, dockerCLI, pm, internalNetworkID)
130+
provisionSvc, err := provision.New(ctx, &cfg.Provision, dbCfg, dockerCLI, pm, engProps.InstanceID, internalNetworkID)
131131
if err != nil {
132132
log.Errf(errors.WithMessage(err, `error in the "provision" section of the config`).Error())
133133
}

internal/cloning/base.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,8 @@ func (c *Base) Run(ctx context.Context) error {
105105
log.Err("Failed to load stored sessions:", err)
106106
}
107107

108+
c.restartCloneContainers(ctx)
109+
108110
c.filterRunningClones(ctx)
109111

110112
if err := c.cleanupInvalidClones(); err != nil {

internal/cloning/storage.go

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,31 @@ func (c *Base) loadSessionState(sessionsPath string) error {
4646

4747
return json.Unmarshal(data, &c.clones)
4848
}
49+
func (c *Base) restartCloneContainers(ctx context.Context) {
50+
c.cloneMutex.Lock()
51+
defer c.cloneMutex.Unlock()
52+
53+
for _, wrapper := range c.clones {
54+
if wrapper.Clone == nil || wrapper.Session == nil {
55+
continue
56+
}
57+
58+
cloneName := util.GetCloneName(wrapper.Session.Port)
59+
if c.provision.IsCloneRunning(ctx, cloneName) {
60+
continue
61+
}
62+
63+
if err := c.provision.ReconnectClone(ctx, cloneName); err != nil {
64+
log.Err(fmt.Sprintf("Clone container %s cannot be reconnected to the internal network: %s", cloneName, err))
65+
}
66+
67+
if err := c.provision.StartCloneContainer(ctx, cloneName); err != nil {
68+
log.Err(fmt.Sprintf("Clone container %s cannot start: %s", cloneName, err))
69+
}
70+
71+
log.Dbg(fmt.Sprintf("Clone container %s is running", cloneName))
72+
}
73+
}
4974

5075
func (c *Base) filterRunningClones(ctx context.Context) {
5176
c.cloneMutex.Lock()

internal/cloning/storage_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ func newProvisioner() (*provision.Provisioner, error) {
8383
From: 1,
8484
To: 5,
8585
},
86-
}, nil, nil, nil, "nwID")
86+
}, nil, nil, nil, "instID", "nwID")
8787
}
8888

8989
func TestLoadingSessionState(t *testing.T) {

internal/provision/docker/docker.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -261,8 +261,8 @@ func PullImage(r runners.Runner, dockerImage string) error {
261261
return nil
262262
}
263263

264-
// IsContainerExist checks the existence of Docker container.
265-
func IsContainerExist(ctx context.Context, docker *client.Client, containerName string) (bool, error) {
264+
// IsContainerRunning checks if specified container is running.
265+
func IsContainerRunning(ctx context.Context, docker *client.Client, containerName string) (bool, error) {
266266
inspection, err := docker.ContainerInspect(ctx, containerName)
267267
if err != nil {
268268
return false, fmt.Errorf("failed to inpect container: %w", err)

internal/provision/mode_local.go

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"sync/atomic"
1919
"time"
2020

21+
"github.com/docker/docker/api/types"
2122
"github.com/docker/docker/client"
2223
"github.com/pkg/errors"
2324

@@ -31,6 +32,7 @@ import (
3132
"gitlab.com/postgres-ai/database-lab/v3/pkg/log"
3233
"gitlab.com/postgres-ai/database-lab/v3/pkg/models"
3334
"gitlab.com/postgres-ai/database-lab/v3/pkg/util"
35+
"gitlab.com/postgres-ai/database-lab/v3/pkg/util/networks"
3436
"gitlab.com/postgres-ai/database-lab/v3/pkg/util/pglog"
3537
)
3638

@@ -67,11 +69,12 @@ type Provisioner struct {
6769
portChecker portChecker
6870
pm *pool.Manager
6971
networkID string
72+
instanceID string
7073
}
7174

7275
// New creates a new Provisioner instance.
7376
func New(ctx context.Context, cfg *Config, dbCfg *resources.DB, docker *client.Client, pm *pool.Manager,
74-
networkID string) (*Provisioner, error) {
77+
instanceID, networkID string) (*Provisioner, error) {
7578
if err := IsValidConfig(*cfg); err != nil {
7679
return nil, errors.Wrap(err, "configuration is not valid")
7780
}
@@ -86,6 +89,7 @@ func New(ctx context.Context, cfg *Config, dbCfg *resources.DB, docker *client.C
8689
portChecker: &localPortChecker{},
8790
pm: pm,
8891
networkID: networkID,
92+
instanceID: instanceID,
8993
ports: make([]bool, cfg.PortPool.To-cfg.PortPool.From),
9094
}
9195

@@ -699,14 +703,24 @@ func (p *Provisioner) prepareDB(pgConf *resources.AppConfig, user resources.Ephe
699703

700704
// IsCloneRunning checks if clone is running.
701705
func (p *Provisioner) IsCloneRunning(ctx context.Context, cloneName string) bool {
702-
isRunning, err := docker.IsContainerExist(ctx, p.dockerClient, cloneName)
706+
isRunning, err := docker.IsContainerRunning(ctx, p.dockerClient, cloneName)
703707
if err != nil {
704708
log.Err(err)
705709
}
706710

707711
return isRunning
708712
}
709713

714+
// ReconnectClone disconnects clone from the old instance network and connect to the actual one.
715+
func (p *Provisioner) ReconnectClone(ctx context.Context, cloneName string) error {
716+
return networks.Reconnect(ctx, p.dockerClient, p.instanceID, cloneName)
717+
}
718+
719+
// StartCloneContainer starts clone container.
720+
func (p *Provisioner) StartCloneContainer(ctx context.Context, containerName string) error {
721+
return p.dockerClient.ContainerStart(ctx, containerName, types.ContainerStartOptions{})
722+
}
723+
710724
// DetectDBVersion detects version of the database.
711725
func (p *Provisioner) DetectDBVersion() string {
712726
pgVersion, err := tools.DetectPGVersion(p.pm.First().Pool().DataDir())

internal/provision/mode_local_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ func TestPortAllocation(t *testing.T) {
2424
},
2525
}
2626

27-
p, err := New(context.Background(), cfg, &resources.DB{}, &client.Client{}, &pool.Manager{}, "networkID")
27+
p, err := New(context.Background(), cfg, &resources.DB{}, &client.Client{}, &pool.Manager{}, "instanceID", "networkID")
2828
require.NoError(t, err)
2929

3030
// Allocate a new port.

pkg/util/networks/networks.go

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,36 @@ func Connect(ctx context.Context, dockerCLI *client.Client, instanceID, containe
117117
return nil
118118
}
119119

120+
// Reconnect connects a container to internal Docker network.
121+
func Reconnect(ctx context.Context, dockerCLI *client.Client, instanceID, containerID string) error {
122+
log.Dbg(fmt.Sprintf("Reconnect container %s to internal network", containerID))
123+
124+
networkName := getNetworkName(instanceID)
125+
126+
log.Dbg("Discovering internal network:", networkName)
127+
128+
networkResource, err := dockerCLI.NetworkInspect(ctx, networkName, types.NetworkInspectOptions{})
129+
if err != nil {
130+
return fmt.Errorf("internal network not found: %w", err)
131+
}
132+
133+
log.Dbg(fmt.Sprintf("Disconnecting container %s from internal network", containerID))
134+
135+
if err := dockerCLI.NetworkDisconnect(context.Background(), networkName, containerID, true); err != nil {
136+
return err
137+
}
138+
139+
log.Dbg(fmt.Sprintf("Container %s has been disconnected from internal network", containerID))
140+
141+
if err := dockerCLI.NetworkConnect(ctx, networkResource.ID, containerID, &network.EndpointSettings{}); err != nil {
142+
return err
143+
}
144+
145+
log.Dbg(fmt.Sprintf("Container %s has been connected to %s", instanceID, networkName))
146+
147+
return nil
148+
}
149+
120150
func getNetworkName(instanceID string) string {
121151
return networkPrefix + instanceID
122152
}

0 commit comments

Comments
 (0)