Speed up BATS tests

solr/core/src/test/org/apache/solr/cli/PackageToolTest.java

@@ -373,6 +373,63 @@ public static void testForResponseElement(
         success);
   }
 
+  /** Validates that collection existence is checked before package resolution. */
+  @Test
+  public void testDeployValidationMessages() throws Exception {
+    String solrUrl = cluster.getJettySolrRunner(0).getBaseUrl().toString();
+
+    withBasicAuth(CollectionAdminRequest.createCollection("validation-test", "conf1", 1, 1))
+        .processAndWait(cluster.getSolrClient(), 10);
+
+    CLITestHelper.TestingRuntime captureRuntime = new CLITestHelper.TestingRuntime(true);
+    PackageTool tool = new PackageTool(captureRuntime);
+
+    // Collection exists but package does not — collection validation should pass,
+    // package lookup should fail.
+    tool.runTool(
+        SolrCLI.processCommandLineArgs(
+            tool,
+            new String[] {
+              "--solr-url",
+              solrUrl,
+              "deploy",
+              "NONEXISTENT_PKG",
+              "--collections",
+              "validation-test",
+              "--credentials",
+              SecurityJson.USER_PASS
+            }));
+    String deployOut = captureRuntime.getOutput();
+    assertFalse(
+        "Should not complain about invalid collection", deployOut.contains("Invalid collection"));
+    assertTrue(
+        "Should report missing package",
+        deployOut.contains("Package instance doesn't exist: NONEXISTENT_PKG:null"));
+
+    captureRuntime.clearOutput();
+
+    // Undeploy of a package that was never deployed should give a clear message.
+    tool.runTool(
+        SolrCLI.processCommandLineArgs(
+            tool,
+            new String[] {
+              "--solr-url",
+              solrUrl,
+              "undeploy",
+              "NONEXISTENT_PKG",
+              "--collections",
+              "validation-test",
+              "--credentials",
+              SecurityJson.USER_PASS
+            }));
+    String undeployOut = captureRuntime.getOutput();
+    assertFalse(
+        "Should not complain about invalid collection", undeployOut.contains("Invalid collection"));
+    assertTrue(
+        "Should report package not deployed",
+        undeployOut.contains("Package NONEXISTENT_PKG not deployed on collection validation-test"));
+  }
+
   private void run(PackageTool tool, String[] args) throws Exception {
     int res = tool.runTool(SolrCLI.processCommandLineArgs(tool, args));
     assertEquals("Non-zero status returned for: " + Arrays.toString(args), 0, res);

solr/packaging/build.gradle

@@ -281,6 +281,10 @@ task integrationTests(type: BatsTask) {
     }
   }
 
+  if (!Boolean.parseBoolean((String) project.findProperty('solr.bats.nightly'))) {
+    filterTags = '!nightly'
+  }
+
   inputs.dir(distDir)
   inputs.dir('test')  // Track .bats test files and helper as inputs
   outputs.dir(integrationTestOutput)
@@ -320,6 +324,10 @@ class BatsTask extends Exec {
   @Input
   var testFiles = []
 
+  @Input
+  @Optional
+  String filterTags = null
+
   @Option(option = "tests", description = "Sets test cases to be included")
   public void setTestNamePatterns(List<String> tests) {
     // TODO: bats --filter <regex>
@@ -335,6 +343,9 @@ class BatsTask extends Exec {
     if (logger.isInfoEnabled()) {
       batsArgs << '--verbose-run'
     }
+    if (filterTags) {
+      batsArgs += ['--filter-tags', filterTags]
+    }
     batsArgs += ['-T', '--print-output-on-failure', '--report-formatter', 'junit', '--output', "$project.buildDir/test-output"]
     // Note: tests to run must be listed after all other arguments
     batsArgs += testFiles.empty ? [testDir] : testFiles

solr/packaging/test/README.md

@@ -55,6 +55,24 @@ Some tests will start clusters or create collections,
 
 It is recommended that you install and run `shellcheck` to verify your test scripts and catch common mistakes before committing your changes.
 
+## Nightly Tests
+
+Some BATS tests are slow or require network access (e.g. downloading models) and are
+excluded from regular runs by default. These tests use BATS native tagging via
+`# bats test_tags=nightly` and are filtered out with `--filter-tags !nightly`.
+
+To include nightly tests, pass the Gradle property `solr.bats.nightly`:
+
+    ./gradlew integrationTests -Psolr.bats.nightly=true
+
+When writing a new test that should only run in nightly builds, add the BATS tag comment
+directly before the `@test` declaration:
+
+    # bats test_tags=nightly
+    @test "my slow test" {
+      ...
+    }
+
 ## Limitations
 
 1. Currently this test suite is only available for \*nix environments. Although, newer

solr/packaging/test/test_example.bats

@@ -17,25 +17,36 @@
 
 load bats_helper
 
-setup() {
+setup_file() {
   common_clean_setup
+  solr start -e cloud --no-prompt --jvm-opts "-Dcustom.prop=helloworld"
+  solr assert --started http://localhost:${SOLR_PORT} --cloud http://localhost:${SOLR_PORT} --timeout 60000
+  solr assert --started http://localhost:${SOLR2_PORT} --cloud http://localhost:${SOLR2_PORT} --timeout 60000
+}
+
+teardown_file() {
+  common_setup
+  solr stop --all >/dev/null 2>&1
+}
+
+setup() {
+  common_setup
 }
 
 teardown() {
   # save a snapshot of SOLR_HOME for failed tests
   save_home_on_failure
+}
 
-  solr stop --all >/dev/null 2>&1
+@test "start -e cloud works with --no-prompt" {
+  solr assert --started http://localhost:${SOLR_PORT} --cloud http://localhost:${SOLR_PORT} --timeout 10000
+  solr assert --started http://localhost:${SOLR2_PORT} --cloud http://localhost:${SOLR2_PORT} --timeout 10000
 }
 
 @test "start -e cloud works with --jvm-opts" {
-  solr start -e cloud --no-prompt --jvm-opts "-Dcustom.prop=helloworld"
-  solr assert --started http://localhost:${SOLR_PORT} --cloud http://localhost:${SOLR_PORT} --timeout 60000
-  solr assert --started http://localhost:${SOLR2_PORT} --cloud http://localhost:${SOLR2_PORT} --timeout 60000
-
   run curl "http://localhost:${SOLR_PORT}/solr/admin/info/properties"
   assert_output --partial 'helloworld'
-  
+
   run curl "http://localhost:${SOLR2_PORT}/solr/admin/info/properties"
   assert_output --partial 'helloworld'
 }

solr/packaging/test/test_healthcheck.bats

@@ -29,22 +29,18 @@ teardown() {
 }
 
 @test "healthcheck on cloud solr" {
-  solr start -e films
-  run solr healthcheck -c films
+  solr start
+  solr create -c healthcheck_test -d _default
+  # Local
+  run solr healthcheck -c healthcheck_test
   refute_output --partial 'error'
-
-}
-
-@test "healthcheck on remote cloud solr" {
-  solr start -e films
-  run solr healthcheck -c films --solr-connection http://localhost:${SOLR_PORT}/solr
+  # Remote
+  run solr healthcheck -c healthcheck_test --solr-connection http://localhost:${SOLR_PORT}/solr
   refute_output --partial 'error'
-
 }
 
 @test "healthcheck errors on standalone solr" {
-  solr start --user-managed -e films
-  run solr healthcheck -c films
+  solr start --user-managed
+  run solr healthcheck -c healthcheck_test
   assert_output --partial 'Healthcheck tool only works in Solr Cloud mode'
-
 }

solr/packaging/test/test_opennlp.bats

@@ -42,8 +42,8 @@ teardown() {
 # long test as an "integration" test is something we decide is okay?
 # I also have dreams of incorporating this as code snippets in a Tutorial via the ascii doc tags
 # like we use for the SolrJ code snippets.  That way we know the snippets continue to work!
+# bats test_tags=nightly
 @test "Check lifecycle of sentiment classification" {
-
   echo "Downloading onnx model and vocab..."
 
   mkdir -p ${SOLR_TIP}/models/sentiment/

solr/packaging/test/test_ssl.bats

@@ -572,26 +572,26 @@ teardown() {
     # Replace server1 keystore with client's
     cp cert2.keystore.p12 server1.keystore.p12
   )
-  # Give some time for the server reload
-  sleep 6
+  # Wait for Jetty keystore-reload scan to pick up the new certificate
+  wait_for 30 1 solr api --solr-url "https://localhost:${SOLR_PORT}/solr/admin/info/system"
 
-  run solr healthcheck --solr-url https://localhost:${SOLR_PORT}
+  run solr healthcheck -c test --solr-url https://localhost:${SOLR_PORT}
 
   # Server 2 still uses the cert1, so this request should fail
   run ! solr api --solr-url "https://localhost:${SOLR2_PORT}/solr/test/select?q=query2"
 
-  run ! solr healthcheck --solr-url https://localhost:${SOLR2_PORT}
+  run ! solr healthcheck -c test --solr-url https://localhost:${SOLR2_PORT}
 
   (
     cd "$ssl_dir"
     # Replace server2 keystore with client's
     cp cert2.keystore.p12 server2.keystore.p12
   )
-  # Give some time for the server reload
-  sleep 6
+  # Wait for Jetty keystore-reload scan to pick up the new certificate
+  wait_for 30 1 solr api --solr-url "https://localhost:${SOLR2_PORT}/solr/admin/info/system"
 
-  run solr healthcheck --solr-url https://localhost:${SOLR_PORT}
-  run solr healthcheck --solr-url https://localhost:${SOLR2_PORT}
+  run solr healthcheck -c test --solr-url https://localhost:${SOLR_PORT}
+  run solr healthcheck -c test --solr-url https://localhost:${SOLR2_PORT}
 
   run solr api --solr-url "https://localhost:${SOLR_PORT}/solr/test/select?q=query3"
   assert_output --partial '"numFound":0'

solr/packaging/test/test_start_solr.bats

@@ -25,7 +25,7 @@ teardown() {
   # save a snapshot of SOLR_HOME for failed tests
   save_home_on_failure
 
-  solr stop --all >/dev/null 2>&1
+  SOLR_STOP_WAIT=30 solr stop --all >/dev/null 2>&1
 }
 
 @test "SOLR-11740 check 'solr stop' connection" {