Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement preparing the deploy source in pipedv1 planner / scheduler #5410

Merged
merged 11 commits into from
Dec 13, 2024
4 changes: 4 additions & 0 deletions pkg/app/pipedv1/cmd/piped/grpcapi/tool_registry.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@
}

func newToolRegistry(toolsDir string) (*toolRegistry, error) {
if err := os.MkdirAll(toolsDir, 0o755); err != nil {
return nil, fmt.Errorf("failed to create the tools directory: %w", err)
}

Check warning on line 51 in pkg/app/pipedv1/cmd/piped/grpcapi/tool_registry.go

View check run for this annotation

Codecov / codecov/patch

pkg/app/pipedv1/cmd/piped/grpcapi/tool_registry.go#L50-L51

Added lines #L50 - L51 were not covered by tests

tmpDir, err := os.MkdirTemp("", "tool-registry")
if err != nil {
return nil, fmt.Errorf("failed to create a temporary directory: %w", err)
Expand Down
52 changes: 33 additions & 19 deletions pkg/app/pipedv1/controller/planner.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
"context"
"encoding/json"
"fmt"
"io"
"path/filepath"
"sort"
"time"

Expand All @@ -27,6 +29,7 @@
"go.uber.org/zap"

"github.com/pipe-cd/pipecd/pkg/app/pipedv1/controller/controllermetrics"
"github.com/pipe-cd/pipecd/pkg/app/pipedv1/deploysource"
"github.com/pipe-cd/pipecd/pkg/app/pipedv1/metadatastore"
"github.com/pipe-cd/pipecd/pkg/app/server/service/pipedservice"
config "github.com/pipe-cd/pipecd/pkg/configv1"
Expand Down Expand Up @@ -181,27 +184,38 @@
controllermetrics.UpdateDeploymentStatus(p.deployment, p.doneDeploymentStatus)
}()

// TODO: Prepare running deploy source and target deploy source.
// Prepare running deploy source and target deploy source.
var runningDS, targetDS *deployment.DeploymentSource

// repoCfg := config.PipedRepository{
// RepoID: p.deployment.GitPath.Repo.Id,
// Remote: p.deployment.GitPath.Repo.Remote,
// Branch: p.deployment.GitPath.Repo.Branch,
// }

// Prepare target deploy source.
// targetDSP := deploysource.NewProvider(
// filepath.Join(p.workingDir, "deploysource"),
// deploysource.NewGitSourceCloner(p.gitClient, repoCfg, "target", p.deployment.Trigger.Commit.Hash),
// *p.deployment.GitPath,
// nil, // TODO: Revise this secret decryter, is this need?
// )

// targetDS, err := targetDSP.Get(ctx, io.Discard)
// if err != nil {
// return fmt.Errorf("error while preparing deploy source data (%v)", err)
// }
repoCfg := config.PipedRepository{
RepoID: p.deployment.GitPath.Repo.Id,
Remote: p.deployment.GitPath.Repo.Remote,
Branch: p.deployment.GitPath.Repo.Branch,
}

runningDSP := deploysource.NewProvider(
filepath.Join(p.workingDir, "running-deploysource"),
deploysource.NewGitSourceCloner(p.gitClient, repoCfg, "running", p.lastSuccessfulCommitHash),
p.deployment.GetGitPath(), nil, // TODO: pass secret decrypter?
)
rds, err := runningDSP.Get(ctx, io.Discard) // TODO: pass not io.Discard
if err != nil {
// TODO: log error
return fmt.Errorf("error while preparing deploy source data (%v)", err)
}
runningDS = rds.ToPluginDeploySource()

targetDSP := deploysource.NewProvider(
filepath.Join(p.workingDir, "target-deploysource"),
deploysource.NewGitSourceCloner(p.gitClient, repoCfg, "target", p.deployment.Trigger.Commit.Hash),
p.deployment.GetGitPath(), nil, // TODO: pass secret decrypter?
)
tds, err := targetDSP.Get(ctx, io.Discard) // TODO: pass not io.Discard
if err != nil {
// TODO: log error
return fmt.Errorf("error while preparing deploy source data (%v)", err)
}
targetDS = tds.ToPluginDeploySource()

Check warning on line 218 in pkg/app/pipedv1/controller/planner.go

View check run for this annotation

Codecov / codecov/patch

pkg/app/pipedv1/controller/planner.go#L190-L218

Added lines #L190 - L218 were not covered by tests

// TODO: Pass running DS as well if need?
out, err := p.buildPlan(ctx, runningDS, targetDS)
Expand Down
53 changes: 45 additions & 8 deletions pkg/app/pipedv1/controller/scheduler.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
"context"
"encoding/json"
"fmt"
"io"
"path/filepath"
"time"

"go.opentelemetry.io/otel/attribute"
Expand All @@ -27,6 +29,7 @@
"go.uber.org/zap"

"github.com/pipe-cd/pipecd/pkg/app/pipedv1/controller/controllermetrics"
"github.com/pipe-cd/pipecd/pkg/app/pipedv1/deploysource"
"github.com/pipe-cd/pipecd/pkg/app/pipedv1/metadatastore"
"github.com/pipe-cd/pipecd/pkg/app/server/service/pipedservice"
config "github.com/pipe-cd/pipecd/pkg/configv1"
Expand All @@ -47,8 +50,8 @@
metadataStore metadatastore.MetadataStore
notifier notifier

targetDS *deployment.DeploymentSource
runningDS *deployment.DeploymentSource
targetDSP deploysource.Provider
runningDSP deploysource.Provider

// Current status of each stages.
// We stores their current statuses into this field
Expand Down Expand Up @@ -214,9 +217,32 @@
)
deploymentStatus = model.DeploymentStatus_DEPLOYMENT_SUCCESS

/// TODO: prepare the targetDS and runningDS
var targetDS *deployment.DeploymentSource
cfg, err := config.DecodeYAML[*config.GenericApplicationSpec](targetDS.GetApplicationConfig())
repoCfg := config.PipedRepository{
RepoID: s.deployment.GitPath.Repo.Id,
Remote: s.deployment.GitPath.Repo.Remote,
Branch: s.deployment.GitPath.Repo.Branch,
}

s.runningDSP = deploysource.NewProvider(
filepath.Join(s.workingDir, "running-deploysource"),
deploysource.NewGitSourceCloner(s.gitClient, repoCfg, "running", s.deployment.RunningCommitHash),
s.deployment.GetGitPath(), nil, // TODO: pass secret decrypter?
)

s.targetDSP = deploysource.NewProvider(
filepath.Join(s.workingDir, "target-deploysource"),
deploysource.NewGitSourceCloner(s.gitClient, repoCfg, "target", s.deployment.Trigger.Commit.Hash),
s.deployment.GetGitPath(), nil, // TODO: pass secret decrypter?
)

ds, err := s.targetDSP.Get(ctx, io.Discard)
if err != nil {
deploymentStatus = model.DeploymentStatus_DEPLOYMENT_FAILURE
statusReason = fmt.Sprintf("Failed to get deploy source at target commit (%v)", err)
s.reportDeploymentCompleted(ctx, deploymentStatus, statusReason, "")
return err
}
cfg, err := config.DecodeYAML[*config.GenericApplicationSpec](ds.ApplicationConfig)

Check warning on line 245 in pkg/app/pipedv1/controller/scheduler.go

View check run for this annotation

Codecov / codecov/patch

pkg/app/pipedv1/controller/scheduler.go#L220-L245

Added lines #L220 - L245 were not covered by tests
if err != nil {
deploymentStatus = model.DeploymentStatus_DEPLOYMENT_FAILURE
statusReason = fmt.Sprintf("Failed to decode application configuration at target commit (%v)", err)
Expand Down Expand Up @@ -441,6 +467,18 @@
originalStatus = ps.Status
)

rds, err := s.runningDSP.Get(ctx, io.Discard)
if err != nil {
s.logger.Error("failed to get running deployment source", zap.Error(err))
return model.StageStatus_STAGE_FAILURE
}

Check warning on line 474 in pkg/app/pipedv1/controller/scheduler.go

View check run for this annotation

Codecov / codecov/patch

pkg/app/pipedv1/controller/scheduler.go#L472-L474

Added lines #L472 - L474 were not covered by tests

tds, err := s.targetDSP.Get(ctx, io.Discard)
if err != nil {
s.logger.Error("failed to get target deployment source", zap.Error(err))
return model.StageStatus_STAGE_FAILURE
}

Check warning on line 480 in pkg/app/pipedv1/controller/scheduler.go

View check run for this annotation

Codecov / codecov/patch

pkg/app/pipedv1/controller/scheduler.go#L478-L480

Added lines #L478 - L480 were not covered by tests

// Check whether to execute the script rollback stage or not.
// If the base stage is executed, the script rollback stage will be executed.
if ps.Rollback {
Expand Down Expand Up @@ -477,7 +515,6 @@
}

// Load the stage configuration.
// TODO: Check this works with pre-defined stages. (stages added to the pipeline without user-defined configuration)
stageConfig, stageConfigFound := s.genericApplicationConfig.GetStageByte(ps.Index)
if !stageConfigFound {
s.logger.Error("Unable to find the stage configuration")
Expand All @@ -493,8 +530,8 @@
Deployment: s.deployment,
Stage: ps,
StageConfig: stageConfig,
RunningDeploymentSource: s.runningDS, // TODO: prepare this
TargetDeploymentSource: s.targetDS, // TODO: prepare this
RunningDeploymentSource: rds.ToPluginDeploySource(),
TargetDeploymentSource: tds.ToPluginDeploySource(),
},
})
if err != nil {
Expand Down
26 changes: 21 additions & 5 deletions pkg/app/pipedv1/controller/scheduler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,15 @@ package controller

import (
"context"
"io"
"testing"
"time"

"github.com/stretchr/testify/assert"
"go.uber.org/zap/zaptest"
"google.golang.org/grpc"

"github.com/pipe-cd/pipecd/pkg/app/pipedv1/deploysource"
"github.com/pipe-cd/pipecd/pkg/app/server/service/pipedservice"
config "github.com/pipe-cd/pipecd/pkg/configv1"
"github.com/pipe-cd/pipecd/pkg/model"
Expand Down Expand Up @@ -181,7 +183,7 @@ func TestExecuteStage(t *testing.T) {
expected: model.StageStatus_STAGE_FAILURE,
},
{
name: "stage without config, should be set as failed",
name: "stage without config, should be success",
deployment: &model.Deployment{
Stages: []*model.PipelineStage{
{
Expand All @@ -200,7 +202,7 @@ func TestExecuteStage(t *testing.T) {
Stages: []config.PipelineStage{},
},
},
expected: model.StageStatus_STAGE_FAILURE,
expected: model.StageStatus_STAGE_SUCCESS,
},
}

Expand All @@ -210,7 +212,9 @@ func TestExecuteStage(t *testing.T) {
for _, tc := range testcases {
t.Run(tc.name, func(t *testing.T) {
s := &scheduler{
apiClient: &fakeApiClient{},
apiClient: &fakeApiClient{},
targetDSP: &fakeDeploySourceProvider{},
runningDSP: &fakeDeploySourceProvider{},
stageBasedPluginsMap: map[string]pluginapi.PluginClient{
"stage-name": &fakeExecutorPluginClient{},
},
Expand All @@ -237,12 +241,22 @@ func TestExecuteStage(t *testing.T) {
}
}

type fakeDeploySourceProvider struct {
deploysource.Provider
}

func (f *fakeDeploySourceProvider) Get(ctx context.Context, logWriter io.Writer) (*deploysource.DeploySource, error) {
return &deploysource.DeploySource{}, nil
}

func TestExecuteStage_SignalTerminated(t *testing.T) {
logger := zaptest.NewLogger(t)
sig, handler := NewStopSignal()

s := &scheduler{
apiClient: &fakeApiClient{},
apiClient: &fakeApiClient{},
targetDSP: &fakeDeploySourceProvider{},
runningDSP: &fakeDeploySourceProvider{},
stageBasedPluginsMap: map[string]pluginapi.PluginClient{
"stage-name": &fakeExecutorPluginClient{},
},
Expand Down Expand Up @@ -278,7 +292,9 @@ func TestExecuteStage_SignalCancelled(t *testing.T) {
sig, handler := NewStopSignal()

s := &scheduler{
apiClient: &fakeApiClient{},
apiClient: &fakeApiClient{},
targetDSP: &fakeDeploySourceProvider{},
runningDSP: &fakeDeploySourceProvider{},
stageBasedPluginsMap: map[string]pluginapi.PluginClient{
"stage-name": &fakeExecutorPluginClient{},
},
Expand Down
60 changes: 36 additions & 24 deletions pkg/app/pipedv1/deploysource/deploysource.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,26 @@
"sync"

"github.com/pipe-cd/pipecd/pkg/app/pipedv1/sourceprocesser"
"github.com/pipe-cd/pipecd/pkg/config"
config "github.com/pipe-cd/pipecd/pkg/configv1"
"github.com/pipe-cd/pipecd/pkg/model"
"github.com/pipe-cd/pipecd/pkg/plugin/api/v1alpha1/deployment"
)

type DeploySource struct {
RepoDir string
AppDir string
Revision string
ApplicationConfig *config.Config
GenericApplicationConfig *config.GenericApplicationSpec
RepoDir string
AppDir string
Revision string
ApplicationConfig []byte
ApplicationConfigFilename string
}

func (d *DeploySource) ToPluginDeploySource() *deployment.DeploymentSource {
return &deployment.DeploymentSource{
ApplicationDirectory: d.AppDir,
Revision: d.Revision,
ApplicationConfig: d.ApplicationConfig,
ApplicationConfigFilename: d.ApplicationConfigFilename,
}
}

type Provider interface {
Expand All @@ -50,7 +60,7 @@
cloner SourceCloner
revisionName string
revision string
appGitPath model.ApplicationGitPath
appGitPath *model.ApplicationGitPath
secretDecrypter secretDecrypter

done bool
Expand All @@ -63,7 +73,7 @@
func NewProvider(
workingDir string,
cloner SourceCloner,
appGitPath model.ApplicationGitPath,
appGitPath *model.ApplicationGitPath,
sd secretDecrypter,
) Provider {

Expand Down Expand Up @@ -134,7 +144,14 @@
cfgFileRelPath = p.appGitPath.GetApplicationConfigFilePath()
cfgFileAbsPath = filepath.Join(repoDir, cfgFileRelPath)
)
cfg, err := config.LoadFromYAML(cfgFileAbsPath)

cfgFileContent, err := os.ReadFile(cfgFileAbsPath)
if err != nil {
fmt.Fprintf(lw, "Unable to load the application configuration file at %s (%v)\n", cfgFileRelPath, err)
return nil, err
}
cfg, err := config.DecodeYAML[*config.GenericApplicationSpec](cfgFileContent)

Check warning on line 154 in pkg/app/pipedv1/deploysource/deploysource.go

View check run for this annotation

Codecov / codecov/patch

pkg/app/pipedv1/deploysource/deploysource.go#L147-L154

Added lines #L147 - L154 were not covered by tests
if err != nil {
fmt.Fprintf(lw, "Unable to load the application configuration file at %s (%v)\n", cfgFileRelPath, err)

Expand All @@ -144,11 +161,8 @@
return nil, err
}

gac, ok := cfg.GetGenericApplication()
if !ok {
fmt.Fprintf(lw, "Invalid application kind %s\n", cfg.Kind)
return nil, fmt.Errorf("unsupport application kind %s", cfg.Kind)
}
gac := cfg.Spec

Check warning on line 165 in pkg/app/pipedv1/deploysource/deploysource.go

View check run for this annotation

Codecov / codecov/patch

pkg/app/pipedv1/deploysource/deploysource.go#L164-L165

Added lines #L164 - L165 were not covered by tests
fmt.Fprintln(lw, "Successfully loaded the application configuration file")

var templProcessors []sourceprocesser.SourceTemplateProcessor
Expand All @@ -172,11 +186,10 @@
}

return &DeploySource{
RepoDir: repoDir,
AppDir: appDir,
Revision: p.revision,
ApplicationConfig: cfg,
GenericApplicationConfig: &gac,
RepoDir: repoDir,
AppDir: appDir,
Revision: p.revision,
ApplicationConfig: cfgFileContent,

Check warning on line 192 in pkg/app/pipedv1/deploysource/deploysource.go

View check run for this annotation

Codecov / codecov/patch

pkg/app/pipedv1/deploysource/deploysource.go#L189-L192

Added lines #L189 - L192 were not covered by tests
}, nil
}

Expand All @@ -201,10 +214,9 @@
}

return &DeploySource{
RepoDir: dest,
AppDir: filepath.Join(dest, p.appGitPath.Path),
Revision: p.revision,
ApplicationConfig: p.source.ApplicationConfig,
GenericApplicationConfig: p.source.GenericApplicationConfig,
RepoDir: dest,
AppDir: filepath.Join(dest, p.appGitPath.Path),
Revision: p.revision,
ApplicationConfig: p.source.ApplicationConfig,

Check warning on line 220 in pkg/app/pipedv1/deploysource/deploysource.go

View check run for this annotation

Codecov / codecov/patch

pkg/app/pipedv1/deploysource/deploysource.go#L217-L220

Added lines #L217 - L220 were not covered by tests
}, nil
}
2 changes: 1 addition & 1 deletion pkg/app/pipedv1/deploysource/sourcecloner.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ package deploysource
import (
"context"

"github.com/pipe-cd/pipecd/pkg/config"
config "github.com/pipe-cd/pipecd/pkg/configv1"
"github.com/pipe-cd/pipecd/pkg/git"
)

Expand Down
Loading
Loading