TOOLS-3941 - make mongodump_passthrough work with newer pinned mongosync

mongodump_passthrough/functions.yml

@@ -13,11 +13,21 @@
 # is changed here to point at the new mongosync commit hash.
 
 variables:
-  _migration_verifier_pin: &_migration_verifier_pin 9348607d61c4c865287828ce78b798a0f1b1cb89
+  _migration_verifier_pin: &_migration_verifier_pin "9431222bdc602e21b3cf2a64ee45d5eaceac6060"
   _resmoke_dir: &_resmoke_dir "src/resmoke"
   _src_dir: &src_dir src/mongosync
-  # Added for mongodump_passthrough
-  _mongosync_pin: &_mongosync_pin 55b77da9db675c8f92e288b684617ad8a6c158b3
+  #
+  # WARNING:
+  #
+  # There is a delicate relationship between the mongodump_passthrough evergreen
+  # configuration files, and the pinned mongosync which it fetches as an evergreen module.
+  # When you update this pin, you may need to make other changes to get mongodump_passthrough
+  # working again.
+  #
+  # And when you change the _mongosync_pin, you also need to update other pins
+  # which get passed as environment variables to scripts in mongosync/evergreen/scripts.
+  #
+  _mongosync_pin: &_mongosync_pin "7126c03d3f87950824b393b5b1157065767c6d44"
 
   f_expansions_write: &f_expansions_write
     command: expansions.write
@@ -36,7 +46,7 @@ variables:
   f_set_up_logger_credentials: &f_set_up_logger_credentials
     command: subprocess.exec
     params:
-      binary: "./src/mongosync/evergreen/logger_credentials_setup.sh"
+      binary: "./src/mongosync/evergreen/scripts/logger_credentials_setup.sh"
       add_expansions_to_env: true
 
   # Added for mongodump_passthrough
@@ -50,7 +60,7 @@ variables:
   f_resmoke_wheelhouse_setup: &f_resmoke_wheelhouse_setup
     command: subprocess.exec
     params:
-      binary: "./src/mongosync/evergreen/resmoke_wheelhouse_setup.sh"
+      binary: "./src/mongosync/evergreen/scripts/resmoke_wheelhouse_setup.sh"
       add_expansions_to_env: true
 
   # Function used to download particular bits of resmoke and binaries uploaded previously.
@@ -122,7 +132,7 @@ variables:
   f_resmoke_jobs_determine: &f_resmoke_jobs_determine
     command: subprocess.exec
     params:
-      binary: "./src/mongosync/evergreen/resmoke_jobs_determine.sh"
+      binary: "./src/mongosync/evergreen/scripts/resmoke_jobs_determine.sh"
       add_expansions_to_env: true
       env:
         resmoke_jobs_factor: ${resmoke_jobs_factor|1}
@@ -138,7 +148,7 @@ variables:
     command: subprocess.exec
     type: test
     params:
-      binary: "./src/mongosync/evergreen/resmoke_tests_execute.sh"
+      binary: "./src/mongosync/evergreen/scripts/resmoke_tests_execute.sh"
       add_expansions_to_env: true
       env:
         resmoke_jobs: ${resmoke_jobs|1}
@@ -151,13 +161,13 @@ variables:
     command: subprocess.exec
     type: setup
     params:
-      binary: "./src/mongosync/evergreen/infrastructure_failure_check.sh"
+      binary: "./src/mongosync/evergreen/scripts/infrastructure_failure_check.sh"
       add_expansions_to_env: true
 
   f_set_up_archive_credentials: &f_set_up_archive_credentials
     command: subprocess.exec
     params:
-      binary: "./src/mongosync/evergreen/archive_credentials_setup.sh"
+      binary: "./src/mongosync/evergreen/scripts/archive_credentials_setup.sh"
       env:
         private_key_remote_bash_var: ${private_key_remote}
       add_expansions_to_env: true
@@ -181,15 +191,27 @@ functions:
     # The mongo-tools repo is cloned into src/github.com/mongodb/mongo-tools, but
     # we also create a symbolic link at src/mongo-tools.
     - *f_link_to_mongo_tools
+    # Check that mongo-tools and mongosync both got cloned to the right place.
     - command: shell.exec
       params:
         shell: bash
         script: |
-          ls -l src
+          if [ ! -d "src/github.com/mongodb/mongo-tools" ]; then
+            echo "ERROR: git.get_project did not clone mongo-tools as required."
+            find . -name "mongo-tools"
+            exit 1
+          fi
+          if [ ! -d "src/mongosync" ]; then
+            echo "ERROR: git.get_project did not clone mongosync as required."
+            echo "       Check that buildvariant ${build_variant} lists module mongosync."
+            find . -name "mongosync"
+            exit 1
+          fi
+
     # Make an evergreen expansion file with dynamic values
     - command: subprocess.exec
       params:
-        binary: "./src/mongosync/evergreen/generate_mongosync_expansion.sh"
+        binary: "./src/mongosync/evergreen/scripts/generate_mongosync_expansion.sh"
         add_expansions_to_env: true
         env:
           mongosync_dir: "src/mongosync"
@@ -253,8 +275,23 @@ functions:
       params:
         directory: src/github.com/mongodb/mongo-tools
         revisions:
-          migration-verifier: *_migration_verifier_pin
           mongosync: *_mongosync_pin
+          migration-verifier: *_migration_verifier_pin
+    # Check that mongo-tools and mongosync both got cloned to the right place.
+    - command: shell.exec
+      params:
+        shell: bash
+        script: |
+          if [ ! -d "src/github.com/mongodb/mongo-tools" ]; then
+            echo "ERROR: git.get_project did not clone mongo-tools as required."
+            exit 1
+          fi
+          ln -s ${workdir}/src/github.com/mongodb/mongo-tools ${workdir}/src/mongo-tools
+          if [ ! -d "src/mongosync" ]; then
+            echo "ERROR: git.get_project did not clone mongosync as required."
+            echo "       Check that buildvariant ${build_variant} lists module mongosync."
+            exit 1
+          fi
 
   # Under src/mongosync/resmoke we have separate directories "suite-config"
   # and "mongodump-suite-config". To run mongodump passthrough tests we delete
@@ -286,7 +323,7 @@ functions:
     command: subprocess.exec
     params:
       # The evergreen project variables in mongo-tools are ${evg_key} and ${evg_user}.
-      binary: "./src/mongosync/evergreen/evergreen_credentials_setup.sh"
+      binary: "./src/mongosync/evergreen/scripts/evergreen_credentials_setup.sh"
       env:
         evergreen_api_key: ${evg_key}
         evergreen_api_user: ${evg_user}
@@ -297,7 +334,7 @@ functions:
     - *f_evergreen_credentials_setup
     - command: subprocess.exec
       params:
-        binary: "./src/mongosync/evergreen/resmoke_and_binaries_download.sh"
+        binary: "./src/mongosync/evergreen/scripts/resmoke_and_binaries_download.sh"
         add_expansions_to_env: true
 
   f_resmoke_wheelhouse_setup:
@@ -448,7 +485,7 @@ functions:
   f_mongo_coredumps_save:
     - command: subprocess.exec
       params:
-        binary: "./src/mongosync/evergreen/mongo_coredumps_gather.sh"
+        binary: "./src/mongosync/evergreen/scripts/mongo_coredumps_gather.sh"
         add_expansions_to_env: true
     - command: archive.targz_pack
       params:
@@ -578,7 +615,7 @@ functions:
           contents: read
     - command: subprocess.exec
       params:
-        binary: "./src/mongosync/evergreen/setup_jstestfuzz.sh"
+        binary: "./src/mongosync/evergreen/scripts/setup_jstestfuzz.sh"
         add_expansions_to_env: true
 
   "run jstestfuzz":
@@ -593,12 +630,12 @@ functions:
           contents: read
     - command: subprocess.exec
       params:
-        binary: "./src/mongosync/evergreen/clone_jstestfuzz_jstest_repos.sh"
+        binary: "./src/mongosync/evergreen/scripts/clone_jstestfuzz_jstest_repos.sh"
         add_expansions_to_env: true
     - command: subprocess.exec
       type: test
       params:
-        binary: "./src/mongosync/evergreen/run_jstestfuzz.sh"
+        binary: "./src/mongosync/evergreen/scripts/run_jstestfuzz.sh"
         add_expansions_to_env: true
     - command: archive.targz_pack
       params:
@@ -644,4 +681,4 @@ functions:
     - command: subprocess.exec
       display_name: "wait for resmoke to shutdown"
       params:
-        binary: "./src/mongosync/evergreen/wait_for_resmoke_to_shutdown.sh"
+        binary: "./src/mongosync/evergreen/scripts/wait_for_resmoke_to_shutdown.sh"

mongodump_passthrough/modules.yml

@@ -32,4 +32,4 @@ modules:
   - name: mongosync
     repo: [email protected]:10gen/mongosync.git
     prefix: ${workdir}/src
-    branch: rcownie/passthru4
+    branch: rcownie/md_passthru6

mongodump_passthrough/tasks.yml

@@ -22,6 +22,15 @@ tasks:
           script: |
             echo "This task is a no-op to work around evergreen's commit queue behavior"
 
+  - name: deliberate_fail_to_test_build_baron_context_rules
+    commands:
+      - command: shell.exec
+        params:
+          script: |
+            echo "Deliberate fail test should give BF on mongodump_passthrough context"
+            echo "exit 1"
+            exit 1
+
   # For mongodump passthrough, this builds and uploads the mongodump and mongorestore
   # binaries (but without coverage), and uploads them.  The name "compile_coverage"
   # is misleading, but that task name is used in the generated tasks from mongodump-task-gen
@@ -46,7 +55,7 @@ tasks:
       - func: f_repo_with_modules_fetch
       - func: f_resmoke_and_binaries_download
         vars:
-          platform: rhel80
+          platform: mongodump_passthru_v
       - func: f_resmoke_wheelhouse_setup
       - func: f_resmoke_archive
 
@@ -77,7 +86,7 @@ tasks:
     depends_on:
       - name: generate_mongodump_resmoke_build_variants
       - name: compile_coverage
-        variant: rhel80
+        variant: mongodump_passthru_v
     commands:
       - func: "fetch source and install go"
       - func: "generate mongodump evergreen tasks"
@@ -90,7 +99,7 @@ tasks:
     depends_on:
       - name: generate_mongodump_resmoke_build_variants
       - name: compile_coverage
-        variant: rhel80
+        variant: mongodump_passthru_v
     commands:
       - func: "fetch source and install go"
       - func: "generate mongodump evergreen tasks"
@@ -99,11 +108,14 @@ tasks:
           json_filename: mongodump_fsm_tasks.json
 
   - name: generate_mongodump_fuzz_tasks
-    tags: ["mongodump_passthrough_task_generator"]
+    # TOOLS-3951 mongodump_fuzz_tasks not yet working against recent mongosync.
+    # The "run jstestfuzz" function is failing, so I'm temporarily disabling
+    # the jstestfuzz tests on the waterfall by commenting out this tag.
+    # tags: ["mongodump_passthrough_task_generator"]
     depends_on:
       - name: generate_mongodump_resmoke_build_variants
       - name: compile_coverage
-        variant: rhel80
+        variant: mongodump_passthru_v
     commands:
       - func: "fetch source and install go"
       - func: "generate mongodump evergreen tasks"
@@ -121,6 +133,6 @@ tasks:
           script: |
             ${PREPARE_SHELL}
             set -ex
-            mise install
+            mise install go
             mise exec -- go run ./cmd/mongodump-suite-gen/cmd write --all --quiet
             git diff --name-status --exit-code resmoke/mongodump-suite-config/*/suites

mongodump_passthrough/variants.yml

@@ -1,22 +1,28 @@
-# buildvariant "rhel80" is used only for mongodump passthrough testing
+# buildvariant "mongodump_passthrough_v" is used only for mongodump passthrough testing
 #
-# This is the same variant name used for mongosync passthrough testing,
-# and we try to set it up the same way.
+# This maps to the same amazon2-arm64 platform used for mongosync passthrough testing.
 
 buildvariants:
-  - name: rhel80
+  # We use a strange name here so that the Build Baron context rules can steer
+  # the mongodump passthrough tests, and only those tests, to context "mongodump_passthrough".
+  #
+  # This name must also match some name in mongo-tools/release/platform/platform.go,
+  # so if you change the name here you also need to change that.
+  - name: mongodump_passthru_v
     # Use this name to get it sorted near the top of the list in spruce UI.
-    display_name: "# RHEL 8.x (mongodump passthrough)"
+    display_name: "# V mongodump_passthru_v (arm64)"
     cron: 0 3 * * *
     tasks:
+      - name: deliberate_fail_to_test_build_baron_context_rules
       - name: compile_coverage
       - name: compile_verifier
       - name: mongodump_suites_up_to_date
       - name: .mongodump_passthrough_task_generator
     run_on:
-      - rhel80-xlarge
+      - amazon2-arm64-latest-small
     expansions:
       python_override: /opt/mongodbtoolchain/v4/bin/python3
     modules:
+      # git.get_project will only fetch modules that are declared in the variant.
       - migration-verifier
       - mongosync

release/platform/platform.go

@@ -530,6 +530,20 @@ var platforms = []Platform{
 		BuildTags:          defaultBuildTags,
 		ServerVariantNames: mapset.NewSet("enterprise-macos"),
 	},
+	{
+		// mongodump_passthru_v is the evergreen variant name used for
+		// the passthrough tests. It currently maps to an amazon2-aarch64,
+		// but having a distinct variant name helps in managing Evergreen
+		// and Build Baron.
+		Name:                  "mongodump_passthru_v",
+		VariantName:           "mongodump_passthru_v",
+		Arch:                  ArchAarch64,
+		OS:                    OSLinux,
+		Pkg:                   PkgRPM,
+		Repos:                 []Repo{RepoEnterprise, RepoOrg},
+		BuildTags:             defaultBuildTags,
+		MaxLinuxServerVersion: &version.Version{Major: 7, Minor: 0, Patch: 0},
+	},
 	{
 		Name:                  "rhel70",
 		Arch:                  ArchX86_64,

release/platform/platform_test.go

@@ -40,9 +40,10 @@ func TestPlatformsMatchCI(t *testing.T) {
 	}
 
 	for _, v := range config.Variants {
-		// Variant "rhel80" has been added to support mongodump passthrough testing.
-		if v.Name == "release" || v.Name == "static" || v.Name == "rhel80" ||
-			v.Name == "rhel88-race" {
+		// Variant "mongodump_passthru_v" has been added to support mongodump
+		// passthrough testing.
+		if v.Name == "release" || v.Name == "static" || v.Name == "rhel88-race" ||
+			v.Name == "mongodump_passthru_v" {
 			continue
 		}
 
@@ -63,6 +64,9 @@ func TestPlatformsMatchCI(t *testing.T) {
 	}
 
 	for name, seen := range releasePlatforms {
+		if name == "mongodump_passthru_v" {
+			continue
+		}
 		assert.True(seen, "%s from the list of known platforms is in the evergreen config", name)
 	}
 }