Skip to content

Commit

Permalink
Removes dead code for conflict-test
Browse files Browse the repository at this point in the history
  • Loading branch information
schnie committed Jan 30, 2025
1 parent a8abaea commit e0230f8
Show file tree
Hide file tree
Showing 6 changed files with 14 additions and 147 deletions.
3 changes: 1 addition & 2 deletions airflow/container.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ type ContainerHandler interface {
ComposeExport(settingsFile, composeFile string) error
Pytest(pytestFile, customImageName, deployImageName, pytestArgsString, buildSecretString string) (string, error)
Parse(customImageName, deployImageName, buildSecretString string) error
UpgradeTest(runtimeVersion, deploymentID, newImageName, customImageName, buildSecretString string, dependencyTest, versionTest, dagTest bool, astroPlatformCore astroplatformcore.ClientWithResponsesInterface) error
UpgradeTest(runtimeVersion, deploymentID, newImageName, customImageName, buildSecretString string, versionTest, dagTest bool, astroPlatformCore astroplatformcore.ClientWithResponsesInterface) error
}

// RegistryHandler defines methods require to handle all operations with registry
Expand All @@ -54,7 +54,6 @@ type ImageHandler interface {
TagLocalImage(localImage string) error
Run(dagID, envFile, settingsFile, containerName, dagFile, executionDate string, taskLogs bool) error
Pytest(pytestFile, airflowHome, envFile, testHomeDirectory string, pytestArgs []string, htmlReport bool, config types.ImageBuildConfig) (string, error)
ConflictTest(workingDirectory, testHomeDirectory string, buildConfig types.ImageBuildConfig) (string, error)
CreatePipFreeze(altImageName, pipFreezeFile string) error
GetImageRepoSHA(registry string) (string, error)
}
Expand Down
37 changes: 1 addition & 36 deletions airflow/docker.go
Original file line number Diff line number Diff line change
Expand Up @@ -548,7 +548,7 @@ func (d *DockerCompose) Pytest(pytestFile, customImageName, deployImageName, pyt
return exitCode, errors.New("something went wrong while Pytesting your DAGs")
}

func (d *DockerCompose) UpgradeTest(newAirflowVersion, deploymentID, newImageName, customImage, buildSecretString string, conflictTest, versionTest, dagTest bool, astroPlatformCore astroplatformcore.CoreClient) error {
func (d *DockerCompose) UpgradeTest(newAirflowVersion, deploymentID, newImageName, customImage, buildSecretString string, versionTest, dagTest bool, astroPlatformCore astroplatformcore.CoreClient) error {
// figure out which tests to run
if !versionTest && !dagTest {
versionTest = true
Expand Down Expand Up @@ -598,13 +598,6 @@ func (d *DockerCompose) UpgradeTest(newAirflowVersion, deploymentID, newImageNam
}
newDockerFile := destFolder + "/Dockerfile"

// check for dependency conflicts
if conflictTest {
err = d.conflictTest(testHomeDirectory, newImageName, newAirflowVersion)
if err != nil {
return err
}
}
if versionTest {
err := d.versionTest(testHomeDirectory, currentAirflowVersion, deploymentImage, newDockerFile, newAirflowVersion, customImage, buildSecretString)
if err != nil {
Expand All @@ -620,9 +613,6 @@ func (d *DockerCompose) UpgradeTest(newAirflowVersion, deploymentID, newImageNam
}
fmt.Println("\nTest Summary:")
fmt.Printf("\tUpgrade Test Results Directory: %s\n", testHomeDirectory)
if conflictTest {
fmt.Printf("\tDependency Conflict Test Results file: %s\n", "conflict-test-results.txt")
}
if versionTest {
fmt.Printf("\tDependency Version Comparison Results file: %s\n", "dependency_compare.txt")
}
Expand Down Expand Up @@ -656,31 +646,6 @@ func (d *DockerCompose) pullImageFromDeployment(deploymentID string, platformCor
return nil
}

func (d *DockerCompose) conflictTest(testHomeDirectory, newImageName, newAirflowVersion string) error {
fmt.Println("\nChecking your 'requirments.txt' for dependency conflicts with the new version of Airflow")
fmt.Println("\nThis may take a few minutes...")

// create files needed for conflict test
err := initConflictTest(config.WorkingPath, newImageName, newAirflowVersion)
defer os.Remove("conflict-check.Dockerfile")
if err != nil {
return err
}

exitCode, conflictErr := d.imageHandler.ConflictTest(d.airflowHome, testHomeDirectory, airflowTypes.ImageBuildConfig{Path: d.airflowHome})
if conflictErr != nil {
return conflictErr
}
if strings.Contains(exitCode, "0") || exitCode == "" { // if the error code is 0 the pytests passed
fmt.Println("There were no dependency conflicts found")
} else {
fmt.Println("\nSomething went wrong while compiling your dependencies check the logs above for conflicts")
fmt.Println("If there are conflicts remove them from your 'requirments.txt' and rerun this test\nYou will see the best candidate in the 'conflict-test-results.txt' file")
return err
}
return nil
}

func (d *DockerCompose) versionTest(testHomeDirectory, currentAirflowVersion, deploymentImage, newDockerFile, newAirflowVersion, customImage, buildSecretString string) error {
fmt.Println("\nComparing dependency versions between current and upgraded environment")
// pip freeze old Airflow image
Expand Down
68 changes: 0 additions & 68 deletions airflow/docker_image.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ import (
)

const (
EchoCmd = "echo"
pushingImagePrompt = "Pushing image to Astronomer registry"
astroRunContainer = "astro-run"
pullingImagePrompt = "Pulling image from Astronomer registry"
Expand Down Expand Up @@ -264,73 +263,6 @@ func (d *DockerImage) Pytest(pytestFile, airflowHome, envFile, testHomeDirectory
return outb.String(), err
}

func (d *DockerImage) ConflictTest(workingDirectory, testHomeDirectory string, buildConfig airflowTypes.ImageBuildConfig) (string, error) {
containerRuntime, err := runtimes.GetContainerRuntimeBinary()
if err != nil {
return "", err
}
// delete container
err = cmdExec(containerRuntime, nil, nil, "rm", "astro-temp-container")
if err != nil {
logger.Debug(err)
}
// Change to location of Dockerfile
err = os.Chdir(buildConfig.Path)
if err != nil {
return "", err
}
args := []string{
"build",
"-t",
"conflict-check:latest",
"-f",
"conflict-check.Dockerfile",
".",
}

// Create a buffer to capture the command output
var stdout, stderr bytes.Buffer
multiStdout := io.MultiWriter(&stdout, os.Stdout)
multiStderr := io.MultiWriter(&stderr, os.Stdout)

// Start the command execution
err = cmdExec(containerRuntime, multiStdout, multiStderr, args...)
if err != nil {
return "", err
}
// Get the exit code
exitCode := ""
if _, ok := err.(*exec.ExitError); ok {
// The command exited with a non-zero status
exitCode = parseExitCode(stderr.String())
} else if err != nil {
// An error occurred while running the command
return "", err
}
// Run a temporary container to copy the file from the image
err = cmdExec(containerRuntime, nil, nil, "create", "--name", "astro-temp-container", "conflict-check:latest")
if err != nil {
return exitCode, err
}
// Copy the result.txt file from the container to the destination folder
err1 := cmdExec(containerRuntime, nil, nil, "cp", "astro-temp-container:/usr/local/airflow/conflict-test-results.txt", "./"+testHomeDirectory)
if err1 != nil {
// Remove the temporary container
err = cmdExec(containerRuntime, nil, nil, "rm", "astro-temp-container")
if err != nil {
return exitCode, err
}
return exitCode, err1
}

// Remove the temporary container
err = cmdExec(containerRuntime, nil, nil, "rm", "astro-temp-container")
if err != nil {
return exitCode, err
}
return exitCode, nil
}

func parseExitCode(logs string) string {
re := regexp.MustCompile(`exit code: (\d+)`)
match := re.FindStringSubmatch(logs)
Expand Down
10 changes: 5 additions & 5 deletions airflow/mocks/ContainerHandler.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

40 changes: 6 additions & 34 deletions airflow/mocks/ImageHandler.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 1 addition & 2 deletions cmd/airflow.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,6 @@ var (
envExport bool
noBrowser bool
compose bool
conflictTest bool
versionTest bool
dagTest bool
waitTime time.Duration
Expand Down Expand Up @@ -658,7 +657,7 @@ func airflowUpgradeTest(cmd *cobra.Command, platformCoreClient astroplatformcore

buildSecretString = util.GetbuildSecretString(buildSecrets)

err = containerHandler.UpgradeTest(defaultImageTag, deploymentID, defaultImageName, customImageName, buildSecretString, conflictTest, versionTest, dagTest, platformCoreClient)
err = containerHandler.UpgradeTest(defaultImageTag, deploymentID, defaultImageName, customImageName, buildSecretString, versionTest, dagTest, platformCoreClient)
if err != nil {
return err
}
Expand Down

0 comments on commit e0230f8

Please sign in to comment.