From bcac6a89ae422ee93dc67ef418c38df6a514f6f9 Mon Sep 17 00:00:00 2001 From: Patryk Dobrowolski Date: Wed, 24 Jul 2024 09:48:03 +0200 Subject: [PATCH] Rewrite AutoBumper to test-infra with switchable host (#11307) * init * init * init * init * xx * xx * xx * xx * xx * xx * xx * xx * review corrections * add test * linter * fix * magic values * tests * add comment * add comment * go mod tidy --- .ko.yaml | 1 + .koapps.yaml | 1 + cmd/image-autobumper/bumper/bumper.go | 461 +++++++++++++ cmd/image-autobumper/bumper/bumper_test.go | 227 +++++++ .../imagebumper/imagebumper.go | 322 +++++++++ .../imagebumper/imagebumper_test.go | 376 +++++++++++ cmd/image-autobumper/main.go | 630 ++++++++++++++++++ cmd/image-autobumper/updater/updater.go | 162 +++++ cmd/image-autobumper/updater/updater_test.go | 169 +++++ go.mod | 2 +- go.sum | 4 +- 11 files changed, 2352 insertions(+), 3 deletions(-) create mode 100644 cmd/image-autobumper/bumper/bumper.go create mode 100644 cmd/image-autobumper/bumper/bumper_test.go create mode 100644 cmd/image-autobumper/imagebumper/imagebumper.go create mode 100644 cmd/image-autobumper/imagebumper/imagebumper_test.go create mode 100644 cmd/image-autobumper/main.go create mode 100644 cmd/image-autobumper/updater/updater.go create mode 100644 cmd/image-autobumper/updater/updater_test.go diff --git a/.ko.yaml b/.ko.yaml index dfbae878a727..dde4ec95ac43 100644 --- a/.ko.yaml +++ b/.ko.yaml @@ -2,6 +2,7 @@ baseImageOverrides: github.com/kyma-project/test-infra/cmd/tools/pjtester: europe-docker.pkg.dev/kyma-project/prod/testimages/alpine-git:v20240723-b6e143ec github.com/kyma-project/test-infra/cmd/markdown-index: europe-docker.pkg.dev/kyma-project/prod/testimages/alpine-git:v20240723-b6e143ec github.com/kyma-project/test-infra/cmd/image-detector: europe-docker.pkg.dev/kyma-project/prod/testimages/alpine-git:v20240723-b6e143ec + github.com/kyma-project/test-infra/cmd/image-autobumper: europe-docker.pkg.dev/kyma-project/prod/testimages/alpine-git:v20240723-b6e143ec github.com/kyma-project/test-infra/cmd/external-plugins/automated-approver: europe-docker.pkg.dev/kyma-project/prod/testimages/alpine-git:v20240723-b6e143ec defaultPlatforms: - linux/arm64 diff --git a/.koapps.yaml b/.koapps.yaml index 1e2143207583..e2c1c7a9e81a 100644 --- a/.koapps.yaml +++ b/.koapps.yaml @@ -2,6 +2,7 @@ apps: - ko://github.com/kyma-project/test-infra/cmd/tools/pjtester - ko://github.com/kyma-project/test-infra/cmd/image-syncer - ko://github.com/kyma-project/test-infra/cmd/image-detector + - ko://github.com/kyma-project/test-infra/cmd/image-autobumper - ko://github.com/kyma-project/test-infra/cmd/image-url-helper - ko://github.com/kyma-project/test-infra/cmd/markdown-index - ko://github.com/kyma-project/test-infra/cmd/tools/usersmapchecker diff --git a/cmd/image-autobumper/bumper/bumper.go b/cmd/image-autobumper/bumper/bumper.go new file mode 100644 index 000000000000..24a321e73bc3 --- /dev/null +++ b/cmd/image-autobumper/bumper/bumper.go @@ -0,0 +1,461 @@ +package bumper + +import ( + "bytes" + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "os/exec" + "strings" + + "github.com/kyma-project/test-infra/cmd/image-autobumper/updater" + "github.com/sirupsen/logrus" + "k8s.io/test-infra/prow/config/secret" + "k8s.io/test-infra/prow/github" +) + +const ( + forkRemoteName = "bumper-fork-remote" + + defaultHeadBranchName = "autobump" + + gitCmd = "git" +) + +// Options is the options for autobumper operations. +type Options struct { + // The target GitHub org name where the autobump PR will be created. Only required when SkipPullRequest is false. + GitHubOrg string `json:"gitHubOrg" yaml:"gitHubOrg"` + // The target GitHub repo name where the autobump PR will be created. Only required when SkipPullRequest is false. + GitHubRepo string `json:"gitHubRepo" yaml:"gitHubRepo"` + // The name of the branch in the target GitHub repo on which the autobump PR will be based. If not specified, will be autodetected via GitHub API. + GitHubBaseBranch string `json:"gitHubBaseBranch" yaml:"gitHubBaseBranch"` + // The GitHub username to use. If not specified, uses values from the user associated with the access token. + GitHubLogin string `json:"gitHubLogin" yaml:"gitHubLogin"` + // The path to the GitHub token file. Only required when SkipPullRequest is false. + GitHubToken string `json:"gitHubToken" yaml:"gitHubToken"` + // The name to use on the git commit. Only required when GitEmail is specified and SkipPullRequest is false. If not specified, uses values from the user associated with the access token + GitName string `json:"gitName" yaml:"gitName"` + // The email to use on the git commit. Only required when GitName is specified and SkipPullRequest is false. If not specified, uses values from the user associated with the access token. + GitEmail string `json:"gitEmail" yaml:"gitEmail"` + // AssignTo specifies who to assign the created PR to. Takes precedence over onCallAddress and onCallGroup if set. + AssignTo string `json:"assign_to" yaml:"assign_to"` + // Whether to skip creating the pull request for this bump. + SkipPullRequest bool `json:"skipPullRequest" yaml:"skipPullRequest"` + // Whether to signoff the commits. + Signoff bool `json:"signoff" yaml:"signoff"` + // The name used in the address when creating remote. This should be the same name as the fork. If fork does not exist this will be the name of the fork that is created. + // If it is not the same as the fork, the robot will change the name of the fork to this. Format will be git@github.com:{GitLogin}/{RemoteName}.git + RemoteName string `json:"remoteName" yaml:"remoteName"` + // The name of the branch that will be used when creating the pull request. If unset, defaults to "autobump". + HeadBranchName string `json:"headBranchName" yaml:"headBranchName"` + // Optional list of labels to add to the bump PR + Labels []string `json:"labels" yaml:"labels"` + // The GitHub host to use, defaulting to github.com + GitHubHost string `json:"gitHubHost" yaml:"gitHubHost"` +} + +// PRHandler is the interface implemented by consumer of prcreator, for +// manipulating the repo, and provides commit messages, PR title and body. +type PRHandler interface { + // Changes returns a slice of functions, each one does some stuff, and + // returns commit message for the changes + Changes() []func(context.Context) (string, error) + // PRTitleBody returns the body of the PR, this function runs after all + // changes have been executed + PRTitleBody() (string, string) +} + +// GitAuthorOptions is specifically to read the author info for a commit +type GitAuthorOptions struct { + GitName string + GitEmail string +} + +// AddFlags will read the author info from the command line parameters +func (o *GitAuthorOptions) AddFlags(fs *flag.FlagSet) { + fs.StringVar(&o.GitName, "git-name", "", "The name to use on the git commit.") + fs.StringVar(&o.GitEmail, "git-email", "", "The email to use on the git commit.") +} + +// Validate will validate the input GitAuthorOptions +func (o *GitAuthorOptions) Validate() error { + if (o.GitEmail == "") != (o.GitName == "") { + return fmt.Errorf("--git-name and --git-email must be specified together") + } + return nil +} + +// GitCommand is used to pass the various components of the git command which needs to be executed +type GitCommand struct { + baseCommand string + args []string + workingDir string +} + +// Call will execute the Git command and switch the working directory if specified +func (gc GitCommand) Call(stdout, stderr io.Writer, opts ...CallOption) error { + return Call(stdout, stderr, gc.baseCommand, gc.buildCommand(), opts...) +} + +func (gc GitCommand) buildCommand() []string { + args := []string{} + if gc.workingDir != "" { + args = append(args, "-C", gc.workingDir) + } + args = append(args, gc.args...) + return args +} + +func (gc GitCommand) getCommand() string { + return fmt.Sprintf("%s %s", gc.baseCommand, strings.Join(gc.buildCommand(), " ")) +} + +func validateOptions(o *Options) error { + if !o.SkipPullRequest { + if o.GitHubToken == "" { + return fmt.Errorf("gitHubToken is mandatory when skipPullRequest is false or unspecified") + } + if (o.GitEmail == "") != (o.GitName == "") { + return fmt.Errorf("gitName and gitEmail must be specified together") + } + if o.GitHubOrg == "" || o.GitHubRepo == "" { + return fmt.Errorf("gitHubOrg and gitHubRepo are mandatory when skipPullRequest is false or unspecified") + } + if o.RemoteName == "" { + return fmt.Errorf("remoteName is mandatory when skipPullRequest is false or unspecified") + } + } + if !o.SkipPullRequest { + if o.HeadBranchName == "" { + o.HeadBranchName = defaultHeadBranchName + } + } + if o.GitHubHost == "" { + o.GitHubHost = "github.com" + } + + return nil +} + +// Run is the entrypoint which will update Prow config files based on the +// provided options. +// +// updateFunc: a function that returns commit message and error +func Run(ctx context.Context, o *Options, prh PRHandler) error { + if err := validateOptions(o); err != nil { + return fmt.Errorf("validating options: %w", err) + } + + if o.SkipPullRequest { + logrus.Debugf("--skip-pull-request is set to true, won't create a pull request.") + } + + return processGitHub(ctx, o, prh) +} + +func processGitHub(ctx context.Context, o *Options, prh PRHandler) error { + stdout := HideSecretsWriter{Delegate: os.Stdout, Censor: secret.Censor} + stderr := HideSecretsWriter{Delegate: os.Stderr, Censor: secret.Censor} + if err := secret.Add(o.GitHubToken); err != nil { + return fmt.Errorf("start secrets agent: %w", err) + } + + gitHubHost := "https://api.github.com" + if o.GitHubHost != "" { + gitHubHost = fmt.Sprintf("https://%s/api/v3", o.GitHubHost) + } + + gc, err := github.NewClient(secret.GetTokenGenerator(o.GitHubToken), secret.Censor, gitHubHost, gitHubHost) + if err != nil { + return fmt.Errorf("failed to construct GitHub client: %v", err) + } + + if o.GitHubLogin == "" || o.GitName == "" || o.GitEmail == "" { + user, err := gc.BotUser() + if err != nil { + return fmt.Errorf("get the user data for the provided GH token: %w", err) + } + if o.GitHubLogin == "" { + o.GitHubLogin = user.Login + } + if o.GitName == "" { + o.GitName = user.Name + } + if o.GitEmail == "" { + o.GitEmail = user.Email + } + } + + // Make change, commit and push + var anyChange bool + for i, changeFunc := range prh.Changes() { + msg, err := changeFunc(ctx) + if err != nil { + return fmt.Errorf("process function %d: %w", i, err) + } + + changed, err := HasChanges(o) + if err != nil { + return fmt.Errorf("checking changes: %w", err) + } + + if !changed { + logrus.WithField("function", i).Info("Nothing changed, skip commit ...") + continue + } + + anyChange = true + if err := gitCommit(o.GitName, o.GitEmail, msg, stdout, stderr, o.Signoff); err != nil { + return fmt.Errorf("git commit: %w", err) + } + } + if !anyChange { + logrus.Info("Nothing changed from all functions, skip PR ...") + return nil + } + + if err := MinimalGitPush(fmt.Sprintf("https://%s:%s@%s/%s/%s.git", o.GitHubLogin, string(secret.GetTokenGenerator(o.GitHubToken)()), o.GitHubHost, o.GitHubLogin, o.RemoteName), o.HeadBranchName, stdout, stderr, o.SkipPullRequest); err != nil { + return fmt.Errorf("push changes to the remote branch: %w", err) + } + + summary, body := prh.PRTitleBody() + if o.GitHubBaseBranch == "" { + repo, err := gc.GetRepo(o.GitHubOrg, o.GitHubRepo) + if err != nil { + return fmt.Errorf("detect default remote branch for %s/%s: %w", o.GitHubOrg, o.GitHubRepo, err) + } + o.GitHubBaseBranch = repo.DefaultBranch + } + if err := updatePRWithLabels(gc, o.GitHubOrg, o.GitHubRepo, getAssignment(o.AssignTo), o.GitHubLogin, o.GitHubBaseBranch, o.HeadBranchName, updater.PreventMods, summary, body, o.Labels, o.SkipPullRequest); err != nil { + return fmt.Errorf("to create the PR: %w", err) + } + return nil +} + +type callOptions struct { + ctx context.Context + dir string +} + +type CallOption func(*callOptions) + +func Call(stdout, stderr io.Writer, cmd string, args []string, opts ...CallOption) error { + var options callOptions + for _, opt := range opts { + opt(&options) + } + logger := (&logrus.Logger{ + Out: stderr, + Formatter: logrus.StandardLogger().Formatter, + Hooks: logrus.StandardLogger().Hooks, + Level: logrus.StandardLogger().Level, + }).WithField("cmd", cmd). + // The default formatting uses a space as separator, which is hard to read if an arg contains a space + WithField("args", fmt.Sprintf("['%s']", strings.Join(args, "', '"))) + + if options.dir != "" { + logger = logger.WithField("dir", options.dir) + } + logger.Info("running command") + + var c *exec.Cmd + if options.ctx != nil { + c = exec.CommandContext(options.ctx, cmd, args...) + } else { + c = exec.Command(cmd, args...) + } + c.Stdout = stdout + c.Stderr = stderr + if options.dir != "" { + c.Dir = options.dir + } + return c.Run() +} + +type HideSecretsWriter struct { + Delegate io.Writer + Censor func(content []byte) []byte +} + +func (w HideSecretsWriter) Write(content []byte) (int, error) { + _, err := w.Delegate.Write(w.Censor(content)) + if err != nil { + return 0, err + } + return len(content), nil +} + +func updatePRWithLabels(gc github.Client, org, repo string, extraLineInPRBody, login, baseBranch, headBranch string, allowMods bool, summary, body string, labels []string, dryrun bool) error { + return UpdatePullRequestWithLabels(gc, org, repo, summary, generatePRBody(body, extraLineInPRBody), login+":"+headBranch, baseBranch, headBranch, allowMods, labels, dryrun) +} + +// UpdatePullRequestWithLabels updates with GitHub client "gc" the PR of GitHub repo org/repo +// with "title" and "body" of PR matching author and headBranch from "source" to "baseBranch" with labels +func UpdatePullRequestWithLabels(gc github.Client, org, repo, title, body, source, baseBranch, + headBranch string, allowMods bool, labels []string, dryrun bool) error { + logrus.Info("Creating or updating PR...") + if dryrun { + logrus.Info("[Dryrun] ensure PR with:") + logrus.Info(org, repo, title, body, source, baseBranch, headBranch, allowMods, gc, labels, dryrun) + return nil + } + n, err := updater.EnsurePRWithLabels(org, repo, title, body, source, baseBranch, headBranch, allowMods, gc, labels) + if err != nil { + return fmt.Errorf("ensure PR exists: %w", err) + } + logrus.Infof("PR %s/%s#%d will merge %s into %s: %s", org, repo, *n, source, baseBranch, title) + return nil +} + +// HasChanges checks if the current git repo contains any changes +func HasChanges(o *Options) (bool, error) { + // Configure Git to recognize the /workspace directory as safe + additionalArgs := []string{"config", "--global", "user.email", o.GitEmail} + logrus.WithField("cmd", gitCmd).WithField("args", additionalArgs).Info("running command ...") + additionalOutput, configErr := exec.Command(gitCmd, additionalArgs...).CombinedOutput() + if configErr != nil { + logrus.WithField("cmd", gitCmd).Debugf("output is '%s'", string(additionalOutput)) + return false, fmt.Errorf("running command %s %s: %w", gitCmd, additionalArgs, configErr) + } + + additionalArgs2 := []string{"config", "--global", "user.name", o.GitName} + logrus.WithField("cmd", gitCmd).WithField("args", additionalArgs2).Info("running command ...") + additionalOutput2, configErr := exec.Command(gitCmd, additionalArgs2...).CombinedOutput() + if configErr != nil { + logrus.WithField("cmd", gitCmd).Debugf("output is '%s'", string(additionalOutput2)) + return false, fmt.Errorf("running command %s %s: %w", gitCmd, additionalArgs2, configErr) + } + + // Configure Git to recognize the /workspace directory as safe + configArgs := []string{"config", "--global", "--add", "safe.directory", "/workspace"} + logrus.WithField("cmd", gitCmd).WithField("args", configArgs).Info("running command ...") + configOutput, configErr := exec.Command(gitCmd, configArgs...).CombinedOutput() + if configErr != nil { + logrus.WithField("cmd", gitCmd).Debugf("output is '%s'", string(configOutput)) + return false, fmt.Errorf("running command %s %s: %w", gitCmd, configArgs, configErr) + } + + // Check for changes using git status + statusArgs := []string{"status", "--porcelain"} + logrus.WithField("cmd", gitCmd).WithField("args", statusArgs).Info("running command ...") + combinedOutput, err := exec.Command(gitCmd, statusArgs...).CombinedOutput() + if err != nil { + logrus.WithField("cmd", gitCmd).Debugf("output is '%s'", string(combinedOutput)) + return false, fmt.Errorf("running command %s %s: %w", gitCmd, statusArgs, err) + } + hasChanges := len(strings.TrimSuffix(string(combinedOutput), "\n")) > 0 + + // If there are changes, get the diff + if hasChanges { + diffArgs := []string{"diff"} + logrus.WithField("cmd", gitCmd).WithField("args", diffArgs).Info("running command ...") + diffOutput, diffErr := exec.Command(gitCmd, diffArgs...).CombinedOutput() + if diffErr != nil { + logrus.WithField("cmd", gitCmd).Debugf("output is '%s'", string(diffOutput)) + return true, fmt.Errorf("running command %s %s: %w", gitCmd, diffArgs, diffErr) + } + logrus.WithField("cmd", gitCmd).Debugf("diff output is '%s'", string(diffOutput)) + } + + return hasChanges, nil +} + +func gitCommit(name, email, message string, stdout, stderr io.Writer, signoff bool) error { + if err := Call(stdout, stderr, gitCmd, []string{"add", "-A"}); err != nil { + return fmt.Errorf("git add: %w", err) + } + commitArgs := []string{"commit", "-m", message} + if name != "" && email != "" { + commitArgs = append(commitArgs, "--author", fmt.Sprintf("%s <%s>", name, email)) + } + if signoff { + commitArgs = append(commitArgs, "--signoff") + } + if err := Call(stdout, stderr, gitCmd, commitArgs); err != nil { + return fmt.Errorf("git commit: %w", err) + } + return nil +} + +// MinimalGitPush pushes the content of the local repository to the remote, checking to make +// sure that there are real changes that need updating by diffing the tree refs, ensuring that +// no metadata-only pushes occur, as those re-trigger tests, remove LGTM, and cause churn without +// changing the content being proposed in the PR. +func MinimalGitPush(remote, remoteBranch string, stdout, stderr io.Writer, dryrun bool, opts ...CallOption) error { + if err := Call(stdout, stderr, gitCmd, []string{"remote", "add", forkRemoteName, remote}, opts...); err != nil { + return fmt.Errorf("add remote: %w", err) + } + fetchStderr := &bytes.Buffer{} + var remoteTreeRef string + if err := Call(stdout, fetchStderr, gitCmd, []string{"fetch", forkRemoteName, remoteBranch}, opts...); err != nil { + logrus.Info("fetchStderr is : ", fetchStderr.String()) + if !strings.Contains(strings.ToLower(fetchStderr.String()), fmt.Sprintf("couldn't find remote ref %s", remoteBranch)) { + return fmt.Errorf("fetch from fork: %w", err) + } + } else { + var err error + remoteTreeRef, err = getTreeRef(stderr, fmt.Sprintf("refs/remotes/%s/%s", forkRemoteName, remoteBranch), opts...) + if err != nil { + return fmt.Errorf("get remote tree ref: %w", err) + } + } + localTreeRef, err := getTreeRef(stderr, "HEAD", opts...) + if err != nil { + return fmt.Errorf("get local tree ref: %w", err) + } + + if dryrun { + logrus.Info("[Dryrun] Skip git push with: ") + logrus.Info(forkRemoteName, remoteBranch, stdout, stderr, "") + return nil + } + // Avoid doing metadata-only pushes that re-trigger tests and remove lgtm + if localTreeRef != remoteTreeRef { + if err := GitPush(forkRemoteName, remoteBranch, stdout, stderr, "", opts...); err != nil { + return err + } + } else { + logrus.Info("Not pushing as up-to-date remote branch already exists") + } + return nil +} + +// GitPush push the changes to the given remote and branch. +func GitPush(remote, remoteBranch string, stdout, stderr io.Writer, workingDir string, opts ...CallOption) error { + logrus.Info("Pushing to remote...") + gc := GitCommand{ + baseCommand: gitCmd, + args: []string{"push", "-f", remote, fmt.Sprintf("HEAD:%s", remoteBranch)}, + workingDir: workingDir, + } + if err := gc.Call(stdout, stderr, opts...); err != nil { + return fmt.Errorf("%s: %w", gc.getCommand(), err) + } + return nil +} +func generatePRBody(body, assignment string) string { + return body + assignment + "\n" +} + +func getAssignment(assignTo string) string { + if assignTo != "" { + return "/cc @" + assignTo + } + return "" +} + +func getTreeRef(stderr io.Writer, refname string, opts ...CallOption) (string, error) { + revParseStdout := &bytes.Buffer{} + if err := Call(revParseStdout, stderr, gitCmd, []string{"rev-parse", refname + ":"}, opts...); err != nil { + return "", fmt.Errorf("parse ref: %w", err) + } + fields := strings.Fields(revParseStdout.String()) + if n := len(fields); n < 1 { + return "", errors.New("got no output when trying to rev-parse") + } + return fields[0], nil +} diff --git a/cmd/image-autobumper/bumper/bumper_test.go b/cmd/image-autobumper/bumper/bumper_test.go new file mode 100644 index 000000000000..e4547fce67ef --- /dev/null +++ b/cmd/image-autobumper/bumper/bumper_test.go @@ -0,0 +1,227 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bumper + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "k8s.io/test-infra/prow/config/secret" +) + +func TestValidateOptions(t *testing.T) { + emptyStr := "" + cases := []struct { + name string + githubToken *string + githubOrg *string + githubRepo *string + gerrit *bool + gerritAuthor *string + gerritPRIdentifier *string + gerritHostRepo *string + gerritCookieFile *string + remoteName *string + skipPullRequest *bool + signoff *bool + err bool + upstreamBaseChanged bool + }{ + { + name: "Everything correct", + err: false, + }, + { + name: "GitHubToken must not be empty when SkipPullRequest is false", + githubToken: &emptyStr, + err: true, + }, + { + name: "remoteName must not be empty when SkipPullRequest is false", + remoteName: &emptyStr, + err: true, + }, + { + name: "GitHubOrg cannot be empty when SkipPullRequest is false", + githubOrg: &emptyStr, + err: true, + }, + { + name: "GitHubRepo cannot be empty when SkipPullRequest is false", + githubRepo: &emptyStr, + err: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + defaultOption := &Options{ + GitHubOrg: "whatever-org", + GitHubRepo: "whatever-repo", + GitHubLogin: "whatever-login", + GitHubToken: "whatever-token", + GitName: "whatever-name", + GitEmail: "whatever-email", + RemoteName: "whatever-name", + SkipPullRequest: false, + Signoff: false, + } + + if tc.skipPullRequest != nil { + defaultOption.SkipPullRequest = *tc.skipPullRequest + } + if tc.signoff != nil { + defaultOption.Signoff = *tc.signoff + } + if tc.githubToken != nil { + defaultOption.GitHubToken = *tc.githubToken + } + if tc.remoteName != nil { + defaultOption.RemoteName = *tc.remoteName + } + if tc.githubOrg != nil { + defaultOption.GitHubOrg = *tc.githubOrg + } + if tc.githubRepo != nil { + defaultOption.GitHubRepo = *tc.githubRepo + } + + err := validateOptions(defaultOption) + t.Logf("err is: %v", err) + if err == nil && tc.err { + t.Errorf("Expected to get an error for %#v but got nil", defaultOption) + } + if err != nil && !tc.err { + t.Errorf("Expected to not get an error for %#v but got %v", defaultOption, err) + } + }) + } +} + +type fakeWriter struct { + results []byte +} + +func (w *fakeWriter) Write(content []byte) (n int, err error) { + w.results = append(w.results, content...) + return len(content), nil +} + +func writeToFile(t *testing.T, path, content string) { + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Errorf("write file %s dir with error '%v'", path, err) + } +} + +func TestCallWithWriter(t *testing.T) { + dir := t.TempDir() + + file1 := filepath.Join(dir, "secret1") + file2 := filepath.Join(dir, "secret2") + + writeToFile(t, file1, "abc") + writeToFile(t, file2, "xyz") + + if err := secret.Add(file1, file2); err != nil { + t.Errorf("failed to start secrets agent; %v", err) + } + + var fakeOut fakeWriter + var fakeErr fakeWriter + + stdout := HideSecretsWriter{Delegate: &fakeOut, Censor: secret.Censor} + stderr := HideSecretsWriter{Delegate: &fakeErr, Censor: secret.Censor} + + testCases := []struct { + description string + command string + args []string + expectedOut string + expectedErr string + }{ + { + description: "no secret in stdout are working well", + command: "echo", + args: []string{"-n", "aaa: 123"}, + expectedOut: "aaa: 123", + }, + { + description: "secret in stdout are censored", + command: "echo", + args: []string{"-n", "abc: 123"}, + expectedOut: "XXX: 123", + }, + { + description: "secret in stderr are censored", + command: "ls", + args: []string{"/tmp/file-not-exist/abc/xyz/file-not-exist"}, + expectedErr: "/tmp/file-not-exist/XXX/XXX/file-not-exist", + }, + { + description: "no secret in stderr are working well", + command: "ls", + args: []string{"/tmp/file-not-exist/aaa/file-not-exist"}, + expectedErr: "/tmp/file-not-exist/aaa/file-not-exist", + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + fakeOut.results = []byte{} + fakeErr.results = []byte{} + _ = Call(stdout, stderr, tc.command, tc.args) + if full, want := string(fakeOut.results), tc.expectedOut; !strings.Contains(full, want) { + t.Errorf("stdout does not contain %q, got %q", full, want) + } + if full, want := string(fakeErr.results), tc.expectedErr; !strings.Contains(full, want) { + t.Errorf("stderr does not contain %q, got %q", full, want) + } + }) + } +} + +func TestGetAssignment(t *testing.T) { + cases := []struct { + description string + assignTo string + oncallURL string + oncallGroup string + oncallServerResponse string + expectResKeyword string + }{ + { + description: "AssignTo takes precedence over oncall settings", + assignTo: "some-user", + expectResKeyword: "/cc @some-user", + }, + { + description: "No assign to", + assignTo: "", + expectResKeyword: "", + }, + } + + for _, tc := range cases { + t.Run(tc.description, func(t *testing.T) { + res := getAssignment(tc.assignTo) + if !strings.Contains(res, tc.expectResKeyword) { + t.Errorf("Expect the result %q contains keyword %q but it does not", res, tc.expectResKeyword) + } + }) + } +} diff --git a/cmd/image-autobumper/imagebumper/imagebumper.go b/cmd/image-autobumper/imagebumper/imagebumper.go new file mode 100644 index 000000000000..f16aa252aa63 --- /dev/null +++ b/cmd/image-autobumper/imagebumper/imagebumper.go @@ -0,0 +1,322 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package imagebumper + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "regexp" + "strconv" + "strings" + "time" +) + +var ( + // imageRegexp matches image names with the following structure: + // - The registry part can be: + // - gcr.io or docker.pkg.dev (optionally preceded by a subdomain) + // - e.g., `gcr.io`, `us.gcr.io`, `docker.pkg.dev`, `eu.docker.pkg.dev` + // - The repository part: + // - Begins with a lowercase letter, followed by 5-29 lowercase letters, digits, or hyphens + // - e.g., `project-name`, `my-repo` + // - The image name: + // - Begins with an alphanumeric character and can contain alphanumeric characters, underscores, dots, hyphens, and slashes + // - e.g., `my_image`, `some.image/name` + // - The tag: + // - Contains alphanumeric characters, dots, underscores, or hyphens + // - e.g., `v1.0.0`, `latest`, `1.2.3-beta` + imageRegexp = regexp.MustCompile(`\b((?:[a-z0-9]+\.)?gcr\.io|(?:[a-z0-9-]+)?docker\.pkg\.dev)/([a-z][a-z0-9-]{5,29}/[a-zA-Z0-9][a-zA-Z0-9_./-]+):([a-zA-Z0-9_.-]+)\b`) + + // tagRegexp matches version tags with the following structure: + // - Version tags starting with an optional 'v' followed by an 8-digit date (YYYYMMDD) + // - Optionally followed by a dash, 'v', a version number, dash or dot-separated numeric parts, '-g', and a 6-10 character alphanumeric hash + // - e.g., `v20220714`, `20220714-v1.2.3-gabcdef1234` + // - Alternatively, matches the string "latest" + // - Optionally followed by a dash and any additional characters + // - e.g., `latest`, `v20220714-extra` + tagRegexp = regexp.MustCompile(`(v?\d{8}-(?:v\d(?:[.-]\d+)*-g)?[0-9a-f]{6,10}|latest)(-.+)?`) +) + +// Constants to indicate parts of the image and tag regex matches. +const ( + tagVersionPart = 1 // Index of the main version part of the tag in regex match + tagExtraPart = 2 // Index of the extra part of the tag in regex match +) + +type Client struct { + // Keys are /:. Values are corresponding tags. + tagCache map[string]string + httpClient *http.Client +} + +func NewClient(httpClient *http.Client) *Client { + // Shallow copy to adjust Timeout + httpClientCopy := *httpClient + httpClientCopy.Timeout = 1 * time.Minute + + return &Client{ + tagCache: map[string]string{}, + httpClient: &httpClientCopy, + } +} + +type manifest map[string]struct { + TimeCreatedMs string `json:"timeCreatedMs"` + Tags []string `json:"tag"` +} + +// DeconstructCommit separates a git describe commit into its parts. +// +// Examples: +// +// v0.0.30-14-gdeadbeef => (v0.0.30 14 deadbeef) +// v0.0.30 => (v0.0.30 0 "") +// deadbeef => ("", 0, deadbeef) +// +// See man git describe. +func DeconstructCommit(commit string) (string, int, string) { + // Split the commit string by '-' + parts := strings.Split(commit, "-") + + // If there's only one part, it can be a version tag or a commit hash + if len(parts) == 1 { + // Check if it's a commit hash + if len(parts[0]) == 40 || !strings.HasPrefix(parts[0], "v") { + return "", 0, strings.TrimPrefix(parts[0], "g") + } + // It's a tag + return parts[0], 0, "" + } + + // If there are two parts, we need to handle it based on the second part + if len(parts) == 2 { + // If the second part starts with 'g', it's a git commit hash + if strings.HasPrefix(parts[1], "g") { + return parts[0], 0, strings.TrimPrefix(parts[1], "g") + } + // Otherwise, it's a version tag + return parts[0], 0, "" + } + + // If there are three parts, it should be in the form v0.0.30-14-gdeadbeef + if len(parts) == 3 { + // Parse the middle part as an integer (number of commits) + n, err := strconv.Atoi(parts[1]) + if err != nil { + panic(err) + } + // The last part should start with 'g' and followed by a commit hash + return parts[0], n, strings.TrimPrefix(parts[2], "g") + } + + // Fallback for unexpected formats + return "", 0, "" +} + +// DeconstructTag separates the tag into its vDATE-COMMIT-VARIANT components +// +// COMMIT may be in the form vTAG-NEW-gCOMMIT, use PureCommit to further process +// this down to COMMIT. +func DeconstructTag(tag string) (date, commit, variant string) { + currentTagParts := tagRegexp.FindStringSubmatch(tag) + if currentTagParts == nil { + return "", "", "" + } + parts := strings.Split(currentTagParts[tagVersionPart], "-") + return parts[0][1:], parts[len(parts)-1], currentTagParts[tagExtraPart] +} + +// Constructs the URI for fetching the manifest +func constructManifestURI(imageHost, imageName string) string { + return fmt.Sprintf("https://%s/v2/%s/tags/list", imageHost, imageName) +} + +// Fetches the manifest for a given image from a registry +func (cli *Client) getManifest(imageHost, imageName string) (manifest, error) { + uri := constructManifestURI(imageHost, imageName) + resp, err := cli.httpClient.Get(uri) + if err != nil { + return nil, fmt.Errorf("couldn't fetch tag list: %w", err) + } + defer resp.Body.Close() + + var result struct { + Manifest manifest `json:"manifest"` + } + + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("couldn't parse tag information from registry: %w", err) + } + + return result.Manifest, nil +} + +// FindLatestTag returns the latest valid tag for the given image. +func (cli *Client) FindLatestTag(imageHost, imageName, currentTag string) (string, error) { + k := imageHost + "/" + imageName + ":" + currentTag + if result, ok := cli.tagCache[k]; ok { + return result, nil + } + + currentTagParts := tagRegexp.FindStringSubmatch(currentTag) + if currentTagParts == nil { + return "", fmt.Errorf("couldn't figure out the current tag in %q", currentTag) + } + if currentTagParts[tagVersionPart] == "latest" { + return currentTag, nil + } + + imageList, err := cli.getManifest(imageHost, imageName) + if err != nil { + return "", err + } + + latestTag, err := pickBestTag(currentTagParts, imageList) + if err != nil { + return "", err + } + + cli.tagCache[k] = latestTag + + return latestTag, nil +} + +func (cli *Client) TagExists(imageHost, imageName, currentTag string) (bool, error) { + imageList, err := cli.getManifest(imageHost, imageName) + if err != nil { + return false, err + } + + for _, v := range imageList { + for _, tag := range v.Tags { + if tag == currentTag { + return true, nil + } + } + } + + return false, nil +} + +// pickBestTag finds the most recently created image tag that matches the suffix of the current tag. +// If a tag called "latest" with the appropriate suffix is found, it is assumed to be the latest, +// regardless of its creation time. +func pickBestTag(currentTagParts []string, manifest manifest) (string, error) { + var latestTime int64 + var latestTag string + + for _, entry := range manifest { + var bestTag string + var isLatest bool + + for _, tag := range entry.Tags { + parts := tagRegexp.FindStringSubmatch(tag) + if parts == nil { + continue + } + if parts[tagExtraPart] != currentTagParts[tagExtraPart] { + continue + } + if parts[tagVersionPart] == "latest" { + isLatest = true + continue + } + if bestTag == "" || len(tag) < len(bestTag) { + bestTag = tag + } + } + + if bestTag == "" { + continue + } + + timeCreated, err := strconv.ParseInt(entry.TimeCreatedMs, 10, 64) + if err != nil { + return "", fmt.Errorf("couldn't parse timestamp %q: %w", entry.TimeCreatedMs, err) + } + + if isLatest || timeCreated > latestTime { + latestTime = timeCreated + latestTag = bestTag + if isLatest { + break + } + } + } + + if latestTag == "" { + return "", fmt.Errorf("failed to find a suitable tag") + } + + return latestTag, nil +} + +// AddToCache keeps track of changed tags +func (cli *Client) AddToCache(image, newTag string) { + cli.tagCache[image] = newTag +} + +// updateAllTags updates all image tags in the given content based on the tagPicker function. +// If imageFilter is provided, only images matching the filter will be updated. +func updateAllTags(tagPicker func(string, string, string) (string, error), content []byte, imageFilter *regexp.Regexp) []byte { + return imageRegexp.ReplaceAllFunc(content, func(image []byte) []byte { + matches := imageRegexp.FindSubmatch(image) + if len(matches) != 4 { + return image // Should not happen, but for safety + } + + imageHost := string(matches[1]) + imageName := string(matches[2]) + imageTag := string(matches[3]) + + // Apply image filter if provided + if imageFilter != nil && !imageFilter.Match(image) { + return image + } + + newTag, err := tagPicker(imageHost, imageName, imageTag) + if err != nil { + return image // If there is an error getting the new tag, return the original image + } + + // Construct the updated image with the new tag + return []byte(fmt.Sprintf("%s/%s:%s", imageHost, imageName, newTag)) + }) +} + +// UpdateFile updates a file in place. +func (cli *Client) UpdateFile(tagPicker func(imageHost, imageName, currentTag string) (string, error), + path string, imageFilter *regexp.Regexp) error { + content, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read %s: %w", path, err) + } + + newContent := updateAllTags(tagPicker, content, imageFilter) + + if err := os.WriteFile(path, newContent, 0644); err != nil { + return fmt.Errorf("failed to write %s: %w", path, err) + } + return nil +} + +// GetReplacements returns the tag replacements that have been made. +func (cli *Client) GetReplacements() map[string]string { + return cli.tagCache +} diff --git a/cmd/image-autobumper/imagebumper/imagebumper_test.go b/cmd/image-autobumper/imagebumper/imagebumper_test.go new file mode 100644 index 000000000000..7721f53231e6 --- /dev/null +++ b/cmd/image-autobumper/imagebumper/imagebumper_test.go @@ -0,0 +1,376 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package imagebumper + +import ( + "fmt" + "regexp" + "testing" +) + +func TestDeconstructCommit(t *testing.T) { + cases := []struct { + name string + commit string + tag string + num int + expectedCommit string + }{ + { + name: "basically works", + }, + { + name: "just commit works", + commit: "deadbeef", + expectedCommit: "deadbeef", + }, + { + name: "commit drops leading g", + commit: "gdeadbeef", + expectedCommit: "deadbeef", + }, + { + name: "just tag works", + commit: "v0.0.30", + tag: "v0.0.30", + }, + { + name: "commits past tags work", + commit: "v0.0.30-14-gdeadbeef", + tag: "v0.0.30", + num: 14, + expectedCommit: "deadbeef", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tag, num, commit := DeconstructCommit(tc.commit) + if tag != tc.tag { + t.Errorf("DeconstructCommit(%s) got tag %q, want %q", tc.commit, tag, tc.tag) + } + if num != tc.num { + t.Errorf("DeconstructCommit(%s) got tag %d, want %d", tc.commit, num, tc.num) + } + if commit != tc.expectedCommit { + t.Errorf("DeconstructCommit(%s) got commit %q, want %q", tc.commit, commit, tc.expectedCommit) + } + + }) + } +} + +func TestDeconstructTag(t *testing.T) { + cases := []struct { + tag string + date string + commit string + variant string + }{ + { + tag: "deadbeef", + // TODO(fejta): commit: "deadbeef", + }, + { + tag: "v0.0.30", + // TODO(fejta): commit: "v0.0.30", + }, + { + tag: "v20190404-65af07d", + date: "20190404", + commit: "65af07d", + }, + { + tag: "v20190330-811f79999-experimental", + date: "20190330", + commit: "811f79999", + variant: "-experimental", + }, + { + tag: "latest", + date: "atest", // TODO(fejta): empty + commit: "latest", + }, + { + tag: "latest-experimental", + date: "atest", // TODO(fejta): empty + commit: "latest", + variant: "-experimental", // TODO(fejta): no - + }, + { + tag: "v20210125-v0.0.41-8-gcb960c8", + date: "20210125", + commit: "gcb960c8", // TODO(fejta): "v0.0.41-8-gcb960c8", + }, + { + tag: "v20210125-v0.0.41-8-gcb960c8-fancy", + date: "20210125", + commit: "gcb960c8", // TODO(fejta): "v0.0.41-8-gcb960c8", + variant: "-fancy", // TODO(fejta): no - + }, + } + + for _, tc := range cases { + t.Run(tc.tag, func(t *testing.T) { + date, commit, variant := DeconstructTag(tc.tag) + if date != tc.date { + t.Errorf("DeconstructTag(%q) got date %s, want %s", tc.tag, date, tc.date) + } + if commit != tc.commit { + t.Errorf("DeconstructTag(%q) got commit %s, want %s", tc.tag, commit, tc.commit) + } + if variant != tc.variant { + t.Errorf("DeconstructTag(%q) got variant %s, want %s", tc.tag, variant, tc.variant) + } + }) + } +} + +func TestPickBestTag(t *testing.T) { + tests := []struct { + name string + tag string + manifest manifest + bestTag string + expectErr bool + }{ + { + name: "simple lookup", + tag: "v20190329-811f7954b", + manifest: manifest{ + "image1": { + TimeCreatedMs: "2000", + Tags: []string{"v20190404-65af07d"}, + }, + "image2": { + TimeCreatedMs: "1000", + Tags: []string{"v20190329-811f7954b"}, + }, + }, + bestTag: "v20190404-65af07d", + }, + { + name: "'latest' overrides date", + tag: "v20190329-811f7954b", + manifest: manifest{ + "image1": { + TimeCreatedMs: "2000", + Tags: []string{"v20190404-65af07d"}, + }, + "image2": { + TimeCreatedMs: "1000", + Tags: []string{"v20190330-811f79999", "latest"}, + }, + }, + bestTag: "v20190330-811f79999", + }, + { + name: "tags with suffixes only match other tags with the same suffix", + tag: "v20190329-811f7954b-experimental", + manifest: manifest{ + "image1": { + TimeCreatedMs: "2000", + Tags: []string{"v20190404-65af07d"}, + }, + "image2": { + TimeCreatedMs: "1000", + Tags: []string{"v20190330-811f79999-experimental"}, + }, + }, + bestTag: "v20190330-811f79999-experimental", + }, + { + name: "unsuffixed 'latest' has no effect on suffixed tags", + tag: "v20190329-811f7954b-experimental", + manifest: manifest{ + "image1": { + TimeCreatedMs: "2000", + Tags: []string{"v20190404-65af07d", "latest"}, + }, + "image2": { + TimeCreatedMs: "1000", + Tags: []string{"v20190330-811f79999-experimental"}, + }, + }, + bestTag: "v20190330-811f79999-experimental", + }, + { + name: "suffixed 'latest' has no effect on unsuffixed tags", + tag: "v20190329-811f7954b", + manifest: manifest{ + "image1": { + TimeCreatedMs: "2000", + Tags: []string{"v20190404-65af07d"}, + }, + "image2": { + TimeCreatedMs: "1000", + Tags: []string{"v20190330-811f79999-experimental", "latest-experimental"}, + }, + }, + bestTag: "v20190404-65af07d", + }, + { + name: "'latest' with the correct suffix overrides date", + tag: "v20190329-811f7954b-experimental", + manifest: manifest{ + "image1": { + TimeCreatedMs: "2000", + Tags: []string{"v20190404-65af07d-experimental"}, + }, + "image2": { + TimeCreatedMs: "1000", + Tags: []string{"v20190330-811f79999-experimental", "latest-experimental"}, + }, + }, + bestTag: "v20190330-811f79999-experimental", + }, + { + name: "it is an error when no tags are found", + tag: "v20190329-811f7954b-master", + manifest: manifest{ + "image1": { + TimeCreatedMs: "2000", + Tags: []string{"v20190404-65af07d-experimental"}, + }, + "image2": { + TimeCreatedMs: "1000", + Tags: []string{"v20190330-811f79999-experimental", "latest-experimental"}, + }, + }, + expectErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + tagParts := tagRegexp.FindStringSubmatch(test.tag) + bestTag, err := pickBestTag(tagParts, test.manifest) + if err != nil { + if !test.expectErr { + t.Fatalf("Unexpected error: %v", err) + } + return + } + if test.expectErr { + t.Fatalf("Expected an error, but got result %q", bestTag) + } + if bestTag != test.bestTag { + t.Fatalf("Expected tag %q, but got %q instead", test.bestTag, bestTag) + } + }) + } +} + +func TestUpdateAllTags(t *testing.T) { + tests := []struct { + name string + content string + expectedResult string + imageFilter *regexp.Regexp + newTags map[string]string + }{ + { + name: "file with no images does nothing", + content: "this is just a normal file", + expectedResult: "this is just a normal file", + }, + { + name: "file that has only an image replaces the image", + content: "gcr.io/k8s-testimages/some-image:v20190404-12345678", + expectedResult: "gcr.io/k8s-testimages/some-image:v20190405-123456789", + newTags: map[string]string{ + "gcr.io/k8s-testimages/some-image:v20190404-12345678": "v20190405-123456789", + }, + }, + { + name: "file that has content before and after an image still has it later", + content: `{"image": "gcr.io/k8s-testimages/some-image:v20190404-12345678"}`, + expectedResult: `{"image": "gcr.io/k8s-testimages/some-image:v20190405-123456789"}`, + newTags: map[string]string{ + "gcr.io/k8s-testimages/some-image:v20190404-12345678": "v20190405-123456789", + }, + }, + { + name: "file that has multiple different images replaces both of them", + content: `{"images": ["gcr.io/k8s-testimages/some-image:v20190404-12345678-master", "gcr.io/k8s-testimages/some-image:v20190404-12345678-experimental"]}`, + expectedResult: `{"images": ["gcr.io/k8s-testimages/some-image:v20190405-123456789-master", "gcr.io/k8s-testimages/some-image:v20190405-123456789-experimental"]}`, + newTags: map[string]string{ + "gcr.io/k8s-testimages/some-image:v20190404-12345678-master": "v20190405-123456789-master", + "gcr.io/k8s-testimages/some-image:v20190404-12345678-experimental": "v20190405-123456789-experimental", + }, + }, + { + name: "file with an error image is still otherwise updated", + content: `{"images": ["gcr.io/k8s-testimages/some-image:0.2", "gcr.io/k8s-testimages/some-image:v20190404-12345678"]}`, + expectedResult: `{"images": ["gcr.io/k8s-testimages/some-image:0.2", "gcr.io/k8s-testimages/some-image:v20190405-123456789"]}`, + newTags: map[string]string{ + "gcr.io/k8s-testimages/some-image:v20190404-12345678": "v20190405-123456789", + }, + }, + { + name: "gcr subdomains are supported", + content: `{"images": ["eu.gcr.io/k8s-testimages/some-image:v20190404-12345678"]}`, + expectedResult: `{"images": ["eu.gcr.io/k8s-testimages/some-image:v20190405-123456789"]}`, + newTags: map[string]string{ + "eu.gcr.io/k8s-testimages/some-image:v20190404-12345678": "v20190405-123456789", + }, + }, + { + name: "AR multi-regional subdomains are supported", + content: `{"images": ["us-docker.pkg.dev/k8s-testimages/some-image:v20190404-12345678"]}`, + expectedResult: `{"images": ["us-docker.pkg.dev/k8s-testimages/some-image:v20190405-123456789"]}`, + newTags: map[string]string{ + "us-docker.pkg.dev/k8s-testimages/some-image:v20190404-12345678": "v20190405-123456789", + }, + }, + { + name: "AR regional subdomains are supported", + content: `{"images": ["us-central1-docker.pkg.dev/k8s-testimages/some-image:v20190404-12345678"]}`, + expectedResult: `{"images": ["us-central1-docker.pkg.dev/k8s-testimages/some-image:v20190405-123456789"]}`, + newTags: map[string]string{ + "us-central1-docker.pkg.dev/k8s-testimages/some-image:v20190404-12345678": "v20190405-123456789", + }, + }, + { + name: "images not matching the filter regex are not updated", + content: `{"images": ["gcr.io/k8s-prow/pkg-thing:v20190404-12345678", "gcr.io/k8s-testimages/some-image:v20190404-12345678"]}`, + expectedResult: `{"images": ["gcr.io/k8s-prow/pkg-thing:v20190404-12345678", "gcr.io/k8s-testimages/some-image:v20190405-123456789"]}`, + newTags: map[string]string{ + "gcr.io/k8s-prow/pkg-thing:v20190404-12345678": "v20190405-123456789", + "gcr.io/k8s-testimages/some-image:v20190404-12345678": "v20190405-123456789", + }, + imageFilter: regexp.MustCompile("gcr.io/k8s-testimages"), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + tagPicker := func(imageHost string, imageName string, imageTag string) (string, error) { + result, ok := test.newTags[imageHost+"/"+imageName+":"+imageTag] + if !ok { + return "", fmt.Errorf("unknown image %s/%s:%s", imageHost, imageName, imageTag) + } + return result, nil + } + + newContent := updateAllTags(tagPicker, []byte(test.content), test.imageFilter) + if test.expectedResult != string(newContent) { + t.Fatalf("Expected content:\n%s\n\nActual content:\n%s\n\n", test.expectedResult, string(newContent)) + } + }) + } +} diff --git a/cmd/image-autobumper/main.go b/cmd/image-autobumper/main.go new file mode 100644 index 000000000000..fa024a06c9d4 --- /dev/null +++ b/cmd/image-autobumper/main.go @@ -0,0 +1,630 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + + "github.com/kyma-project/test-infra/cmd/image-autobumper/bumper" + "github.com/kyma-project/test-infra/cmd/image-autobumper/imagebumper" + "github.com/sirupsen/logrus" + flag "github.com/spf13/pflag" + "golang.org/x/oauth2/google" + "gopkg.in/yaml.v3" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/spf13/cobra" +) + +var ( + // AutoBumpConfig contains the path to the AutoBump config file + AutoBumpConfig string + + // GitHubToken contains the path to the GitHub token + GitHubToken string + + // tagRegexp is the regular expression to match a tag. + // This expression matches a string that starts with 'v', followed by exactly 8 digits, + // a hyphen, and then a hexadecimal string of 6 to 9 characters. + tagRegexp = regexp.MustCompile("v[0-9]{8}-[a-f0-9]{6,9}") + + // imageMatcher is the regular expression to match an image. + // This expression matches a string that starts with any character(s) (matched non-greedily due to (?s)^.), + // followed by the keyword 'image:', then captures any characters up to the next colon (:), + // and finally captures a version tag starting with 'v' and followed by any combination of alphanumeric + // characters, underscores, periods, or hyphens. + imageMatcher = regexp.MustCompile(`(?s)^.+image:(.+):(v[a-zA-Z0-9_.-]+)`) +) + +const ( + latestVersion = "latest" + upstreamVersion = "upstream" + upstreamStagingVersion = "upstream-staging" + tagVersion = "vYYYYMMDD-deadbeef" + defaultUpstreamURLBase = "https://raw.githubusercontent.com/kubernetes/test-infra/master" + googleImageRegistryAuth = "google" + cloudPlatformScope = "https://www.googleapis.com/auth/cloud-platform" +) + +// options is the options for autobumper operations. +type options struct { + // The URL where upstream image references are located. Only required if Target Version is "upstream" or "upstreamStaging". Use "https://raw.githubusercontent.com/{ORG}/{REPO}" + // Images will be bumped based off images located at the address using this URL and the refConfigFile or stagingRefConfigFile for each Prefix. + UpstreamURLBase string `yaml:"upstreamURLBase"` + // The config paths to be included in this bump, in which only .yaml files will be considered. By default, all files are included. + IncludedConfigPaths []string `yaml:"includedConfigPaths"` + // The config paths to be excluded in this bump, in which only .yaml files will be considered. + ExcludedConfigPaths []string `yaml:"excludedConfigPaths"` + // The extra non-yaml file to be considered in this bump. + ExtraFiles []string `yaml:"extraFiles"` + // The target version to bump images version to, which can be one of latest, upstream, upstream-staging and vYYYYMMDD-deadbeef. + TargetVersion string `yaml:"targetVersion"` + // List of prefixes that the autobumped is looking for, and other information needed to bump them. Must have at least 1 prefix. + Prefixes []prefix `yaml:"prefixes"` + // The oncall address where we can get the JSON file that stores the current oncall information. + SelfAssign bool `yaml:"selfAssign"` + // ImageRegistryAuth determines a way the autobumper with authenticate when talking to image registry. + // Allowed values: + // * "" (empty) -- uses no auth token + // * "google" -- uses Google's "Application Default Credentials" as defined on https://pkg.go.dev/golang.org/x/oauth2/google#hdr-Credentials. + ImageRegistryAuth string `yaml:"imageRegistryAuth"` + // AdditionalPRBody allows for generic, additional content in the body of the PR + AdditionalPRBody string `yaml:"additionalPRBody"` +} + +// prefix is the information needed for each prefix being bumped. +type prefix struct { + // Name of the tool being bumped + Name string `yaml:"name"` + // The image prefix that the autobumper should look for + Prefix string `yaml:"prefix"` + // File that is looked at to determine current upstream image when bumping to upstream. Required only if targetVersion is "upstream" + RefConfigFile string `yaml:"refConfigFile"` + // File that is looked at to determine current upstream staging image when bumping to upstream staging. Required only if targetVersion is "upstream-staging" + StagingRefConfigFile string `yaml:"stagingRefConfigFile"` + // The repo where the image source resides for the images with this prefix. Used to create the links to see comparisons between images in the PR summary. + Repo string `yaml:"repo"` + // Whether the format of the PR summary for this prefix should be summarised. + Summarise bool `yaml:"summarise"` + // Whether the prefix tags should be consistent after the bump + ConsistentImages bool `yaml:"consistentImages"` + // A list of images whose tags are not required to be consistent after the bump. Requires `consistentImages: true`. + ConsistentImageExceptions []string `yaml:"consistentImageExceptions"` +} + +// client is bumper client +type client struct { + o *options + images map[string]string + versions map[string][]string +} + +type imageBumper interface { + FindLatestTag(imageHost, imageName, currentTag string) (string, error) + UpdateFile(tagPicker func(imageHost, imageName, currentTag string) (string, error), path string, imageFilter *regexp.Regexp) error + GetReplacements() map[string]string + AddToCache(image, newTag string) + TagExists(imageHost, imageName, currentTag string) (bool, error) +} + +// Changes returns a slice of functions, each one does some stuff, and +// returns commit message for the changes +func (c *client) Changes() []func(context.Context) (string, error) { + return []func(context.Context) (string, error){ + func(ctx context.Context) (string, error) { + var err error + if c.images, err = updateReferencesWrapper(ctx, c.o); err != nil { + return "", fmt.Errorf("failed to update image references: %w", err) + } + + if c.versions, err = getVersionsAndCheckConsistency(c.o.Prefixes, c.images); err != nil { + return "", err + } + + var body string + var prefixNames []string + for _, prefix := range c.o.Prefixes { + prefixNames = append(prefixNames, prefix.Name) + body = body + generateSummary(prefix.Repo, prefix.Prefix, prefix.Summarise, c.images) + "\n\n" + } + + return fmt.Sprintf("Bumping %s\n\n%s", strings.Join(prefixNames, " and "), body), nil + }, + } +} + +// PRTitleBody returns the body of the PR, this function runs after each commit +func (c *client) PRTitleBody() (string, string) { + body := generatePRBody(c.images, c.o.Prefixes) + if c.o.AdditionalPRBody != "" { + body += c.o.AdditionalPRBody + "\n" + } + return makeCommitSummary(c.o.Prefixes, c.versions), body +} + +var rootCmd = &cobra.Command{ + Use: "image-autobumper", + Short: "Image AutoBumper CLI", + Long: "Command-Line tool to autobump images", + //nolint:revive + Run: func(cmd *cobra.Command, args []string) { + ctx := context.Background() + logrus.SetLevel(logrus.DebugLevel) + o, pro, err := parseOptions() + if err != nil { + logrus.WithError(err).Fatalf("Failed to run the bumper tool") + } + + if err := validateOptions(o); err != nil { + logrus.WithError(err).Fatalf("Failed validating flags") + } + + if err := bumper.Run(ctx, pro, &client{o: o}); err != nil { + logrus.WithError(err).Fatalf("failed to run the bumper tool") + } + }, +} + +func init() { + rootCmd.PersistentFlags().StringVar(&AutoBumpConfig, "autobump-config", "", "path to the Autobump config file") + rootCmd.PersistentFlags().StringVar(&GitHubToken, "github-token-path", "/etc/github/token", "path to github token for fetching inrepo config") +} + +func main() { + if err := rootCmd.Execute(); err != nil { + log.Fatalf("failed to run command: %s", err) + } +} + +func parseOptions() (*options, *bumper.Options, error) { + var config string + var labelsOverride []string + var skipPullRequest bool + var signoff bool + + var o options + flag.StringVar(&config, "autobump-config", "", "The path to the config file for the autobumber.") + flag.StringSliceVar(&labelsOverride, "labels-override", nil, "Override labels to be added to PR.") + flag.BoolVar(&skipPullRequest, "skip-pullrequest", false, "") + flag.BoolVar(&signoff, "signoff", false, "Signoff the commits.") + flag.Parse() + + var pro bumper.Options + data, err := os.ReadFile(config) + if err != nil { + return nil, nil, fmt.Errorf("read %q: %w", config, err) + } + + if err = yaml.Unmarshal(data, &o); err != nil { + return nil, nil, fmt.Errorf("unmarshal %q: %w", config, err) + } + + if err := yaml.Unmarshal(data, &pro); err != nil { + return nil, nil, fmt.Errorf("unmarshal %q: %w", config, err) + } + + if labelsOverride != nil { + pro.Labels = labelsOverride + } + pro.SkipPullRequest = skipPullRequest + pro.Signoff = signoff + return &o, &pro, nil +} + +func validateOptions(o *options) error { + if len(o.Prefixes) == 0 { + return errors.New("must have at least one Prefix specified") + } + for _, prefix := range o.Prefixes { + if len(prefix.ConsistentImageExceptions) > 0 && !prefix.ConsistentImages { + return fmt.Errorf("consistentImageExceptions requires consistentImages to be true, found in prefix %q", prefix.Name) + } + } + if len(o.IncludedConfigPaths) == 0 { + return errors.New("includedConfigPaths is mandatory") + } + if o.TargetVersion != latestVersion && o.TargetVersion != upstreamVersion && + o.TargetVersion != upstreamStagingVersion && !tagRegexp.MatchString(o.TargetVersion) { + logrus.WithField("allowed", []string{latestVersion, upstreamVersion, upstreamStagingVersion, tagVersion}).Warn( + "Warning: targetVersion mot in allowed so it might not work properly.") + } + if o.TargetVersion == upstreamVersion { + for _, prefix := range o.Prefixes { + if prefix.RefConfigFile == "" { + return fmt.Errorf("targetVersion can't be %q without refConfigFile for each prefix. %q is missing one", upstreamVersion, prefix.Name) + } + } + } + if o.TargetVersion == upstreamStagingVersion { + for _, prefix := range o.Prefixes { + if prefix.StagingRefConfigFile == "" { + return fmt.Errorf("targetVersion can't be %q without stagingRefConfigFile for each prefix. %q is missing one", upstreamStagingVersion, prefix.Name) + } + } + } + if (o.TargetVersion == upstreamVersion || o.TargetVersion == upstreamStagingVersion) && o.UpstreamURLBase == "" { + o.UpstreamURLBase = defaultUpstreamURLBase + logrus.Warnf("targetVersion can't be 'upstream' or 'upstreamStaging` without upstreamURLBase set. Default upstreamURLBase is %q", defaultUpstreamURLBase) + } + + if o.ImageRegistryAuth != "" && o.ImageRegistryAuth != googleImageRegistryAuth { + return fmt.Errorf("imageRegistryAuth has incorrect value: %q. Only \"\" and %q are allowed", o.ImageRegistryAuth, googleImageRegistryAuth) + } + + return nil +} + +// updateReferencesWrapper update the references of prow-images and/or boskos-images and/or testimages +// in the files in any of "subfolders" of the includeConfigPaths but not in excludeConfigPaths +// if the file is a yaml file (*.yaml) or extraFiles[file]=true +func updateReferencesWrapper(ctx context.Context, o *options) (map[string]string, error) { + logrus.Info("Bumping image references...") + var allPrefixes []string + for _, prefix := range o.Prefixes { + allPrefixes = append(allPrefixes, prefix.Prefix) + } + filterRegexp, err := regexp.Compile(strings.Join(allPrefixes, "|")) + if err != nil { + return nil, fmt.Errorf("bad regexp %q: %w", strings.Join(allPrefixes, "|"), err) + } + var client = http.DefaultClient + if o.ImageRegistryAuth == googleImageRegistryAuth { + var err error + client, err = google.DefaultClient(ctx, cloudPlatformScope) + if err != nil { + return nil, fmt.Errorf("failed to create authed client: %v", err) + } + } + imageBumperCli := imagebumper.NewClient(client) + return updateReferences(imageBumperCli, filterRegexp, o) +} + +func updateReferences(imageBumperCli imageBumper, filterRegexp *regexp.Regexp, o *options) (map[string]string, error) { + var tagPicker func(string, string, string) (string, error) + + switch o.TargetVersion { + case latestVersion: + tagPicker = imageBumperCli.FindLatestTag + case upstreamVersion, upstreamStagingVersion: + var err error + if tagPicker, err = upstreamImageVersionResolver(o, o.TargetVersion, parseUpstreamImageVersion, imageBumperCli); err != nil { + return nil, fmt.Errorf("failed to resolve the %s image version: %w", o.TargetVersion, err) + } + } + + updateFile := func(name string) error { + logrus.WithField("file", name).Info("Updating file") + if err := imageBumperCli.UpdateFile(tagPicker, name, filterRegexp); err != nil { + return fmt.Errorf("failed to update the file: %w", err) + } + return nil + } + updateYAMLFile := func(name string) error { + if (strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")) && !isUnderPath(name, o.ExcludedConfigPaths) { + return updateFile(name) + } + return nil + } + + // Updated all .yaml and .yml files under the included config paths but not under excluded config paths. + for _, path := range o.IncludedConfigPaths { + info, err := os.Stat(path) + if err != nil { + return nil, fmt.Errorf("failed to get the file info for %q: %w", path, err) + } + if info.IsDir() { + err := filepath.Walk(path, func(subpath string, _ os.FileInfo, _ error) error { + return updateYAMLFile(subpath) + }) + if err != nil { + return nil, fmt.Errorf("failed to update yaml files under %q: %w", path, err) + } + } else { + if err := updateYAMLFile(path); err != nil { + return nil, fmt.Errorf("failed to update the yaml file %q: %w", path, err) + } + } + } + + // Update the extra files in any case. + for _, file := range o.ExtraFiles { + if err := updateFile(file); err != nil { + return nil, fmt.Errorf("failed to update the extra file %q: %w", file, err) + } + } + + return imageBumperCli.GetReplacements(), nil +} + +// used by updateReferences +func upstreamImageVersionResolver( + o *options, upstreamVersionType string, parse func(upstreamAddress, prefix string) (string, error), imageBumperCli imageBumper) (func(imageHost, imageName, currentTag string) (string, error), error) { + upstreamVersions, err := upstreamConfigVersions(upstreamVersionType, o, parse) + if err != nil { + return nil, err + } + + return func(imageHost, imageName, currentTag string) (string, error) { + imageFullPath := imageHost + "/" + imageName + ":" + currentTag + for prefix, version := range upstreamVersions { + if !strings.HasPrefix(imageFullPath, prefix) { + continue + } + if exists, err := imageBumperCli.TagExists(imageHost, imageName, version); err != nil { + return "", err + } else if exists { + imageBumperCli.AddToCache(imageFullPath, version) + return version, nil + } + imageBumperCli.AddToCache(imageFullPath, currentTag) + return "", fmt.Errorf("unable to bump to %s, image tag %s does not exist for %s", imageFullPath, version, imageName) + } + return currentTag, nil + }, nil +} + +// used by upstreamImageVersionResolver +func upstreamConfigVersions(upstreamVersionType string, o *options, parse func(upstreamAddress, prefix string) (string, error)) (versions map[string]string, err error) { + versions = make(map[string]string) + var upstreamAddress string + for _, prefix := range o.Prefixes { + if upstreamVersionType == upstreamVersion { + upstreamAddress = o.UpstreamURLBase + "/" + prefix.RefConfigFile + } else if upstreamVersionType == upstreamStagingVersion { + upstreamAddress = o.UpstreamURLBase + "/" + prefix.StagingRefConfigFile + } else { + return nil, fmt.Errorf("unsupported upstream version type: %s, must be one of %v", + upstreamVersionType, []string{upstreamVersion, upstreamStagingVersion}) + } + version, err := parse(upstreamAddress, prefix.Prefix) + if err != nil { + return nil, err + } + versions[prefix.Prefix] = version + } + + return versions, nil +} + +// used by updateReferences +func parseUpstreamImageVersion(upstreamAddress, prefix string) (string, error) { + resp, err := http.Get(upstreamAddress) + if err != nil { + return "", fmt.Errorf("error sending GET request to %q: %w", upstreamAddress, err) + } + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + logrus.WithError(err).Error("failed to close the response body") + } + }(resp.Body) + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("HTTP error %d (%q) fetching upstream config file", resp.StatusCode, resp.Status) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("error reading the response body: %w", err) + } + for _, line := range strings.Split(strings.TrimSuffix(string(body), "\n"), "\n") { + res := imageMatcher.FindStringSubmatch(string(line)) + if len(res) > 2 && strings.Contains(res[1], prefix) { + return res[2], nil + } + } + return "", fmt.Errorf("unable to find match for %s in upstream refConfigFile", prefix) +} + +// Check whether the path is under the given path +func isUnderPath(name string, paths []string) bool { + for _, p := range paths { + if p != "" && strings.HasPrefix(name, p) { + return true + } + } + return false +} + +func generatePRBody(images map[string]string, prefixes []prefix) (body string) { + body = "" + for _, prefix := range prefixes { + body = body + generateSummary(prefix.Repo, prefix.Prefix, prefix.Summarise, images) + "\n\n" + } + return body + "\n" +} + +// Generate PR summary for github +func generateSummary(repo, prefix string, summarise bool, images map[string]string) string { + type delta struct { + oldCommit string + newCommit string + oldDate string + newDate string + variant string + component string + } + versions := map[string][]delta{} + for image, newTag := range images { + if !strings.HasPrefix(image, prefix) { + continue + } + if strings.HasSuffix(image, ":"+newTag) { + continue + } + oldDate, oldCommit, oldVariant := imagebumper.DeconstructTag(tagFromName(image)) + newDate, newCommit, _ := imagebumper.DeconstructTag(newTag) + oldCommit = commitToRef(oldCommit) + newCommit = commitToRef(newCommit) + k := oldCommit + ":" + newCommit + d := delta{ + oldCommit: oldCommit, + newCommit: newCommit, + oldDate: oldDate, + newDate: newDate, + variant: formatVariant(oldVariant), + component: componentFromName(image), + } + versions[k] = append(versions[k], d) + } + + switch { + case len(versions) == 0: + return fmt.Sprintf("No %s changes.", prefix) + case len(versions) == 1 && summarise: + for k, v := range versions { + s := strings.Split(k, ":") + return fmt.Sprintf("%s changes: %s/compare/%s...%s (%s → %s)", prefix, repo, s[0], s[1], formatTagDate(v[0].oldDate), formatTagDate(v[0].newDate)) + } + default: + changes := make([]string, 0, len(versions)) + for k, v := range versions { + s := strings.Split(k, ":") + names := make([]string, 0, len(v)) + for _, d := range v { + names = append(names, d.component+d.variant) + } + sort.Strings(names) + changes = append(changes, fmt.Sprintf("%s/compare/%s...%s | %s → %s | %s", + repo, s[0], s[1], formatTagDate(v[0].oldDate), formatTagDate(v[0].newDate), strings.Join(names, ", "))) + } + sort.Slice(changes, func(i, j int) bool { return strings.Split(changes[i], "|")[1] < strings.Split(changes[j], "|")[1] }) + return fmt.Sprintf("Multiple distinct %s changes:\n\nCommits | Dates | Images\n--- | --- | ---\n%s\n", prefix, strings.Join(changes, "\n")) + } + panic("unreachable!") +} + +func formatTagDate(d string) string { + if len(d) != 8 { + return d + } + // ‑ = U+2011 NON-BREAKING HYPHEN, to prevent line wraps. + return fmt.Sprintf("%s‑%s‑%s", d[0:4], d[4:6], d[6:8]) +} + +func commitToRef(commit string) string { + tag, _, commit := imagebumper.DeconstructCommit(commit) + if commit != "" { + return commit + } + return tag +} + +// Format variant for PR summary +func formatVariant(variant string) string { + if variant == "" { + return "" + } + return fmt.Sprintf("(%s)", strings.TrimPrefix(variant, "-")) +} + +// Extract image from image name +func imageFromName(name string) string { + parts := strings.Split(name, ":") + if len(parts) < 2 { + return "" + } + return parts[0] +} + +// Extract image tag from image name +func tagFromName(name string) string { + parts := strings.Split(name, ":") + if len(parts) < 2 { + return "" + } + return parts[1] +} + +// Extract prow component name from image +func componentFromName(name string) string { + s := strings.SplitN(strings.Split(name, ":")[0], "/", 3) + return s[len(s)-1] +} + +// makeCommitSummary takes a list of Prefixes and a map of new tags resulted +// from bumping : the images using those tags and returns a summary of what was +// bumped for use in the commit message +func makeCommitSummary(prefixes []prefix, versions map[string][]string) string { + var allPrefixes []string + for _, prefix := range prefixes { + allPrefixes = append(allPrefixes, prefix.Name) + } + if len(versions) == 0 { + return fmt.Sprintf("Update %s images as necessary", strings.Join(allPrefixes, ", ")) + } + var inconsistentBumps []string + var consistentBumps []string + for _, prefix := range prefixes { + tag, bumped := isBumpedPrefix(prefix, versions) + if !prefix.ConsistentImages && bumped { + inconsistentBumps = append(inconsistentBumps, prefix.Name) + } else if prefix.ConsistentImages && bumped { + consistentBumps = append(consistentBumps, fmt.Sprintf("%s to %s", prefix.Name, tag)) + } + } + var msgs []string + if len(consistentBumps) != 0 { + msgs = append(msgs, strings.Join(consistentBumps, ", ")) + } + if len(inconsistentBumps) != 0 { + msgs = append(msgs, fmt.Sprintf("%s as needed", strings.Join(inconsistentBumps, ", "))) + } + return fmt.Sprintf("Update %s", strings.Join(msgs, " and ")) + +} + +// isBumpedPrefix takes a prefix and a map of new tags resulted from bumping +// : the images using those tags and iterates over the map to find if the +// prefix is found. If it is, this means it has been bumped. +func isBumpedPrefix(prefix prefix, versions map[string][]string) (string, bool) { + for tag, imageList := range versions { + for _, image := range imageList { + if strings.HasPrefix(image, prefix.Prefix) { + return tag, true + } + } + } + return "", false +} + +// getVersionsAndCheckConisistency takes a list of Prefixes and a map of +// all the images found in the code before the bump : their versions after the bump +// For example {"gcr.io/k8s-prow/test1:tag": "newtag", "gcr.io/k8s-prow/test2:tag": "newtag"}, +// and returns a map of new versions resulted from bumping : the images using those versions. +// It will error if one of the Prefixes was bumped inconsistently when it was not supposed to +func getVersionsAndCheckConsistency(prefixes []prefix, images map[string]string) (map[string][]string, error) { + // Key is tag, value is full image. + versions := map[string][]string{} + for _, prefix := range prefixes { + exceptions := sets.NewString(prefix.ConsistentImageExceptions...) + var consistencyVersion, consistencySourceImage string + for k, v := range images { + if strings.HasPrefix(k, prefix.Prefix) { + image := imageFromName(k) + if prefix.ConsistentImages && !exceptions.Has(image) { + if consistencySourceImage != "" && (consistencyVersion != v) { + return nil, fmt.Errorf("%s -> %s not bumped consistently for prefix %s (%s), expected version %s based on bump of %s", k, v, prefix.Prefix, prefix.Name, consistencyVersion, consistencySourceImage) + } + if consistencySourceImage == "" { + consistencyVersion = v + consistencySourceImage = k + } + } + + //Only add bumped images to the new versions map + if !strings.Contains(k, v) { + versions[v] = append(versions[v], k) + } + + } + } + } + return versions, nil +} diff --git a/cmd/image-autobumper/updater/updater.go b/cmd/image-autobumper/updater/updater.go new file mode 100644 index 000000000000..13aba368fcd7 --- /dev/null +++ b/cmd/image-autobumper/updater/updater.go @@ -0,0 +1,162 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package updater handles creation and updates of GitHub PullRequests. +package updater + +import ( + "fmt" + + "github.com/sirupsen/logrus" + "k8s.io/test-infra/prow/github" +) + +// Constants to indicate whether maintainers can modify a pull request in a fork. +const ( + // AllowMods indicates that maintainers can modify the pull request. + AllowMods = true + + // PreventMods indicates that maintainers cannot modify the pull request. + PreventMods = false +) + +// Query constants used in the GitHub search query. +const ( + // queryState indicates the state of the pull requests to search for (open). + queryState = "is:open" + + // queryType indicates the type of the items to search for (pull requests). + queryType = "is:pr" + + // queryArchived excludes archived repositories from the search. + queryArchived = "archived:false" + + // querySortField indicates the field to sort the search results by (updated time). + querySortField = "updated" + + // firstIssueIndex is the index of the first issue in the search results. + firstIssueIndex = 0 +) + +type updateClient interface { + UpdatePullRequest(org, repo string, number int, title, body *string, open *bool, branch *string, canModify *bool) error + BotUser() (*github.UserData, error) + FindIssues(query, sort string, asc bool) ([]github.Issue, error) +} + +type ensureClient interface { + updateClient + AddLabel(org, repo string, number int, label string) error + CreatePullRequest(org, repo, title, body, head, base string, canModify bool) (int, error) + GetIssue(org, repo string, number int) (*github.Issue, error) +} + +// EnsurePRWithQueryTokens ensures that a pull request exists with the given parameters. +// It reuses an existing pull request if one matches the query tokens, otherwise it creates a new one. +func EnsurePRWithQueryTokens(org, repo, title, body, source, baseBranch, queryTokensString string, allowMods bool, gc ensureClient) (*int, error) { + prNumber, err := updatePRWithQueryTokens(org, repo, title, body, queryTokensString, gc) + if err != nil { + return nil, fmt.Errorf("update error: %w", err) + } + + if prNumber == nil { + pr, err := gc.CreatePullRequest(org, repo, title, body, source, baseBranch, allowMods) + if err != nil { + return nil, fmt.Errorf("create error: %w", err) + } + logrus.Infof("Created new PR with number: %d", pr) + prNumber = &pr + } else { + logrus.Infof("Reused existing PR with number: %d", *prNumber) + } + + return prNumber, nil +} + +// updatePRWithQueryTokens looks for an existing PR to reuse based on the provided query tokens. +// If found, it updates the PR; otherwise, it returns nil. +func updatePRWithQueryTokens(org, repo, title, body, queryTokensString string, gc updateClient) (*int, error) { + logrus.Info("Looking for a PR to reuse...") + + // Get the bot user + me, err := gc.BotUser() + if err != nil { + return nil, fmt.Errorf("bot name: %w", err) + } + + // Construct the query to find issues + query := fmt.Sprintf("%s %s %s repo:%s/%s author:%s %s", queryState, queryType, queryArchived, org, repo, me.Login, queryTokensString) + + // Find issues based on the query + issues, err := gc.FindIssues(query, querySortField, false) + if err != nil { + return nil, fmt.Errorf("find issues: %w", err) + } + + // If no reusable issues are found, return nil + if len(issues) == 0 { + logrus.Info("No reusable issues found") + return nil, nil + } + + // Pick the first issue (most recently updated) + prNumber := issues[firstIssueIndex].Number + logrus.Infof("Found PR #%d", prNumber) + + // Prepare to ignore certain fields in the update request + var ignoreOpen *bool + var ignoreBranch *string + var ignoreModify *bool + + // Update the pull request with the new title and body + if err := gc.UpdatePullRequest(org, repo, prNumber, &title, &body, ignoreOpen, ignoreBranch, ignoreModify); err != nil { + return nil, fmt.Errorf("update PR #%d: %w", prNumber, err) + } + + return &prNumber, nil +} + +func EnsurePRWithLabels(org, repo, title, body, source, baseBranch, headBranch string, allowMods bool, gc ensureClient, labels []string) (*int, error) { + return EnsurePRWithQueryTokensAndLabels(org, repo, title, body, source, baseBranch, "head:"+headBranch, allowMods, labels, gc) +} + +func EnsurePRWithQueryTokensAndLabels(org, repo, title, body, source, baseBranch, queryTokensString string, allowMods bool, labels []string, gc ensureClient) (*int, error) { + n, err := EnsurePRWithQueryTokens(org, repo, title, body, source, baseBranch, queryTokensString, allowMods, gc) + if err != nil { + return n, err + } + + if len(labels) == 0 { + return n, nil + } + + issue, err := gc.GetIssue(org, repo, *n) + if err != nil { + return n, fmt.Errorf("failed to get PR: %w", err) + } + + for _, label := range labels { + if issue.HasLabel(label) { + continue + } + + if err := gc.AddLabel(org, repo, *n, label); err != nil { + return n, fmt.Errorf("failed to add label %q: %w", label, err) + } + logrus.WithField("label", label).Info("Added label") + } + return n, nil +} diff --git a/cmd/image-autobumper/updater/updater_test.go b/cmd/image-autobumper/updater/updater_test.go new file mode 100644 index 000000000000..5d319d7f378d --- /dev/null +++ b/cmd/image-autobumper/updater/updater_test.go @@ -0,0 +1,169 @@ +package updater + +import ( + "fmt" + "testing" + + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/test-infra/prow/github" + "k8s.io/test-infra/prow/github/fakegithub" +) + +func TestEnsurePRWithLabels(t *testing.T) { + testCases := []struct { + name string + client *fakegithub.FakeClient + }{ + { + name: "pr is created", + client: fakegithub.NewFakeClient(), + }, + { + name: "pr is updated", + client: &fakegithub.FakeClient{ + PullRequests: map[int]*github.PullRequest{ + 22: {Number: 22, User: github.User{Login: "k8s-ci-robot"}}, + }, + Issues: map[int]*github.Issue{ + 22: {Number: 22}, + }, + }, + }, + { + name: "existing labels are considered", + client: &fakegithub.FakeClient{ + PullRequests: map[int]*github.PullRequest{ + 42: {Number: 42, User: github.User{Login: "k8s-ci-robot"}}, + }, + Issues: map[int]*github.Issue{ + 42: { + Number: 42, + Labels: []github.Label{{Name: "a"}}, + }, + }, + IssueLabelsAdded: []string{"org/repo#42:a"}, + }, + }, + } + + org, repo, labels := "org", "repo", []string{"a", "b"} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + prNumberPtr, err := EnsurePRWithLabels(org, repo, "title", "body", "source", "branch", "matchTitle", PreventMods, tc.client, labels) + if err != nil { + t.Fatalf("error: %v", err) + } + if n := len(tc.client.PullRequests); n != 1 { + t.Fatalf("expected to find one PR, got %d", n) + } + + expectedLabels := sets.NewString() + for _, label := range labels { + expectedLabels.Insert(fmt.Sprintf("%s/%s#%d:%s", org, repo, *prNumberPtr, label)) + } + + if diff := sets.NewString(tc.client.IssueLabelsAdded...).Difference(expectedLabels); len(diff) != 0 { + t.Errorf("found labels do not match expected, diff: %v", diff) + } + }) + } +} + +func TestEnsurePRWithQueryTokens(t *testing.T) { + testCases := []struct { + name string + client *fakegithub.FakeClient + expectedPR int + expectedErr bool + }{ + { + name: "create new PR if no match", + client: fakegithub.NewFakeClient(), + expectedPR: 0, + expectedErr: false, + }, + { + name: "update existing PR", + client: &fakegithub.FakeClient{ + PullRequests: map[int]*github.PullRequest{ + 1: {Number: 1, Title: "old title", Body: "old body", User: github.User{Login: "k8s-ci-robot"}}, + }, + Issues: map[int]*github.Issue{ + 1: {Number: 1}, + }, + }, + expectedPR: 1, + expectedErr: false, + }, + } + + org, repo, title, body, source, baseBranch := "org", "repo", "title", "body", "source", "baseBranch" + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + prNumber, err := EnsurePRWithQueryTokens(org, repo, title, body, source, baseBranch, "queryTokensString", AllowMods, tc.client) + if (err != nil) != tc.expectedErr { + t.Fatalf("error: %v, expected error: %v", err, tc.expectedErr) + } + if prNumber == nil { + t.Fatalf("prNumber is nil") + } + if *prNumber != tc.expectedPR { + t.Fatalf("expected PR number: %d, got: %d", tc.expectedPR, *prNumber) + } + // Dodatkowe logowanie + fmt.Printf("PR number: %d\n", *prNumber) + for number, pr := range tc.client.PullRequests { + fmt.Printf("Existing PR in client - Number: %d, Title: %s\n", number, pr.Title) + } + }) + } +} + +func TestUpdatePRWithQueryTokens(t *testing.T) { + testCases := []struct { + name string + client *fakegithub.FakeClient + expectedPR *int + expectedErr bool + }{ + { + name: "no existing PRs", + client: fakegithub.NewFakeClient(), + expectedPR: nil, + expectedErr: false, + }, + { + name: "update existing PR", + client: &fakegithub.FakeClient{ + PullRequests: map[int]*github.PullRequest{ + 1: {Number: 1, Title: "old title", Body: "old body", User: github.User{Login: "k8s-ci-robot"}}, + }, + Issues: map[int]*github.Issue{ + 1: {Number: 1}, + }, + }, + expectedPR: intPtr(1), + expectedErr: false, + }, + } + + org, repo, title, body, queryTokensString := "org", "repo", "title", "body", "queryTokensString" + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + prNumber, err := updatePRWithQueryTokens(org, repo, title, body, queryTokensString, tc.client) + if (err != nil) != tc.expectedErr { + t.Fatalf("error: %v, expected error: %v", err, tc.expectedErr) + } + if prNumber != tc.expectedPR && (prNumber == nil || tc.expectedPR == nil || *prNumber != *tc.expectedPR) { + t.Fatalf("expected PR number: %v, got: %v", tc.expectedPR, prNumber) + } + }) + } +} + +func intPtr(i int) *int { + return &i +} diff --git a/go.mod b/go.mod index 0b4c3fb59e94..8a0fd49e8193 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/avast/retry-go/v4 v4.6.0 github.com/cenkalti/backoff/v4 v4.3.0 github.com/cloudevents/sdk-go/v2 v2.15.2 - github.com/coreos/go-oidc/v3 v3.11.0 + github.com/coreos/go-oidc/v3 v3.10.0 github.com/forestgiant/sliceutil v0.0.0-20160425183142-94783f95db6c github.com/fsnotify/fsnotify v1.7.0 github.com/go-jose/go-jose/v4 v4.0.3 diff --git a/go.sum b/go.sum index 2b6df878694a..a43b6a98f54a 100644 --- a/go.sum +++ b/go.sum @@ -151,8 +151,8 @@ github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnht github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= -github.com/coreos/go-oidc/v3 v3.11.0 h1:Ia3MxdwpSw702YW0xgfmP1GVCMA9aEFWu12XUZ3/OtI= -github.com/coreos/go-oidc/v3 v3.11.0/go.mod h1:gE3LgjOgFoHi9a4ce4/tJczr0Ai2/BoDhf0r5lltWI0= +github.com/coreos/go-oidc/v3 v3.10.0 h1:tDnXHnLyiTVyT/2zLDGj09pFPkhND8Gl8lnTRhoEaJU= +github.com/coreos/go-oidc/v3 v3.10.0/go.mod h1:5j11xcw0D3+SGxn6Z/WFADsgcWVMyNAlSQupk0KK3ac= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=