Add a step to exhaustive tests for observabilitySRE accetpance testing

.buildkite/scripts/exhaustive-tests/generate-steps.py

@@ -168,6 +168,24 @@ def acceptance_docker_steps()-> list[typing.Any]:
 
     return steps
 
+def fips_test_runner_step() -> dict[str, typing.Any]:
+    step = {
+        "label": "Observability SRE Acceptance Tests",
+        "key": "observabilitySRE-acceptance-tests",
+        "agents": {
+            "provider": "aws",
+            "instanceType": "m6i.xlarge",
+            "diskSizeGb": 60,
+            "instanceMaxAge": 1440,
+            "imagePrefix": "logstash-ubuntu-2204-fips"
+        },
+        "retry": {"automatic": [{"limit": 1}]},
+        "command": LiteralScalarString("""#!/usr/bin/env bash
+./gradlew observabilitySREacceptanceTests --stacktrace
+"""),
+    }
+    return step
+
 if __name__ == "__main__":
     LINUX_OS_ENV_VAR_OVERRIDE = os.getenv("LINUX_OS")
     WINDOWS_OS_ENV_VAR_OVERRIDE = os.getenv("WINDOWS_OS")
@@ -215,5 +233,12 @@ def acceptance_docker_steps()-> list[typing.Any]:
             "steps": acceptance_docker_steps(),
     })
 
+    structure["steps"].append({
+        "group": "Observability SRE Acceptance Tests",
+        "key": "acceptance-observability-sre",
+        "depends_on": ["testing-phase"],
+        "steps": [fips_test_runner_step()],
+    })
+
     print('# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json')
     YAML().dump(structure, sys.stdout)

x-pack/build.gradle

@@ -75,6 +75,7 @@ tasks.register("buildFipsValidationGem") {
     rake(rootProject.projectDir, rootProject.buildDir, 'plugin:build-fips-validation-plugin')
   }
 }
+
 tasks.register("observabilitySREsmokeTests", Test) {
     description = "Run ObservabilitySRE smoke tests using docker-compose and RSpec"
     // Need to have set up the ruby environment for rspec even through we are running in container
@@ -114,3 +115,25 @@ tasks.register("observabilitySREsmokeTests", Test) {
         delete fileTree("distributions/internal/observabilitySRE/qa/smoke/docker/certs").include("*.key", "*.crt", "*.csr", "*.srl")
     }
 }
+
+tasks.register("observabilitySREacceptanceTests", Test) {
+    description = "Run ObservabilitySRE acceptance tests"
+    // Need to have set up the ruby environment for rspec even through we are running in container
+    dependsOn(":bootstrap", ":logstash-core:assemble", ":installDevelopmentGems")
+
+    inputs.files fileTree("${projectDir}/distributions/internal/observabilitySRE/qa/smoke")
+    doFirst {
+        // Generate the certificates first
+        exec {
+            workingDir file("distributions/internal/observabilitySRE/qa/acceptance/docker/certs")
+            commandLine 'bash', './generate.sh'
+            ignoreExitValue = false
+        }
+    }
+    systemProperty 'logstash.root.dir', projectDir.parent
+    include '**/org/logstash/xpack/test/RSpecObservabilitySREAcceptanceTests.class'
+    doLast {
+        // Clean up the generated certificates
+        delete fileTree("distributions/internal/observabilitySRE/qa/acceptance/docker/certs").include("*.key", "*.crt", "*.csr", "*.srl")
+    }
+}

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/certs/.gitignore

@@ -0,0 +1,3 @@
+*.crt
+*.csr
+*.key

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/certs/generate.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+echo "Generating CA certificate"
+openssl req -x509 -newkey rsa:3072 -days 365 -nodes -keyout ca.key -out ca.crt -subj "/CN=Elastic-CA" -sha256
+
+echo "Generating Elasticsearch certificate"
+openssl req -newkey rsa:3072 -nodes -keyout elasticsearch.key -out elasticsearch.csr -subj "/CN=elasticsearch" -sha256
+openssl x509 -req -in elasticsearch.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out elasticsearch.crt -days 365 -sha256
+
+echo "Generating Logstash certificate"
+openssl req -newkey rsa:3072 -nodes -keyout logstash.key -out logstash.csr -subj "/CN=logstash" -sha256
+openssl x509 -req -in logstash.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out logstash.crt -days 365 -sha256
+
+echo "Generating Filebeat certificate"
+openssl req -newkey rsa:3072 -nodes -keyout filebeat.key -out filebeat.csr -subj "/CN=filebeat" -sha256
+openssl x509 -req -in filebeat.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out filebeat.crt -days 365 -sha256
+
+chmod 644 *.crt *.key

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/docker-compose.yml

@@ -0,0 +1,45 @@
+services:
+  logstash:
+    image: docker.elastic.co/logstash/logstash-observability-sre:${OBSERVABILITY_SRE_IMAGE_VERSION:-8.19.0-SNAPSHOT}
+    container_name: fips_test_logstash
+    ports:
+      - "5044:5044"
+    volumes:
+      - ./logstash/config/${LOGSTASH_CONFIG:-logstash-fips.yml}:/usr/share/logstash/config/logstash.yml
+      - ./logstash/pipeline/${LOGSTASH_PIPELINE:-logstash-to-elasticsearch.conf}:/usr/share/logstash/pipeline/logstash.conf
+      - ./certs:/usr/share/logstash/config/certs
+    networks:
+      - elastic
+    depends_on:
+      - elasticsearch
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch-fips:${ELASTICSEARCH_IMAGE_VERSION:-8.19.0-SNAPSHOT}
+    container_name: fips_test_elasticsearch
+    ports:
+      - "9200:9200"
+    volumes:
+      - ./elasticsearch/config/${ELASTICSEARCH_CONFIG:-elasticsearch-fips.yml}:/usr/share/elasticsearch/config/elasticsearch.yml
+      - ./certs:/usr/share/elasticsearch/config/certs
+    environment:
+      - discovery.type=single-node
+      - ES_JAVA_OPTS=-Xms1g -Xmx1g
+      - ELASTIC_PASSWORD=changeme
+    networks:
+      - elastic
+# Filebeat is not yet used in tests, but this is included to show that including it in the compose network
+# will not adversely affect startup time etc for testing interactions between other components.
+  filebeat:
+    image: docker.elastic.co/elasticsearch/elasticsearch-fips:${FILEBEAT_IMAGE_VERSION:-8.19.0-SNAPSHOT}
+    container_name: fips_test_filebeat
+    entrypoint: ["filebeat", "-e", "--strict.perms=false", "-c", "/usr/share/filebeat/filebeat.yml"]
+    volumes:
+      - ./filebeat/config/${FILEBEAT_CONFIG:-filebeat-fips.yml}:/usr/share/filebeat/filebeat.yml
+      - ./filebeat/data:/data
+      - ./certs:/usr/share/filebeat/certs
+    profiles:
+      - filebeat
+    networks:
+      - elastic
+networks:
+  elastic:
+    driver: bridge

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/elasticsearch/config/elasticsearch-fips.yml

@@ -0,0 +1,15 @@
+# Elasticsearch settings
+discovery.type: single-node
+http.port: 9200
+network.host: 0.0.0.0
+# Security settings
+xpack.security.enabled: true
+xpack.security.transport.ssl.enabled: true
+xpack.security.transport.ssl.verification_mode: certificate
+xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certs/elasticsearch.key
+xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certs/elasticsearch.crt
+xpack.security.transport.ssl.certificate_authorities: ["/usr/share/elasticsearch/config/certs/ca.crt"]
+xpack.security.http.ssl.enabled: true
+xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certs/elasticsearch.key
+xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certs/elasticsearch.crt
+xpack.security.http.ssl.certificate_authorities: ["/usr/share/elasticsearch/config/certs/ca.crt"]

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/filebeat/config/filebeat-fips.yml

@@ -0,0 +1,13 @@
+filebeat.inputs:
+- type: log
+  enabled: true
+  paths:
+    - /test-logs/*.log
+
+output.logstash:
+  hosts: ["logstash:5044"]
+  ssl.enabled: true
+  ssl.certificate: "/usr/share/elasticsearch/config/certs/filebeat.crt"
+  ssl.key: "/usr/share/elasticsearch/config/certs/filebeat.key"
+  ssl.certificate_authorities: ["/usr/share/elasticsearch/config/certs/ca.crt"]
+

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/filebeat/data/sample_logs.json

@@ -0,0 +1,3 @@
+{"message":"FIPS compliance test message 1","timestamp":"2025-05-01T12:00:00Z","level":"info"}
+{"message":"FIPS compliance test message 2","timestamp":"2025-05-01T12:01:00Z","level":"debug"}
+{"message":"FIPS compliance test message 3","timestamp":"2025-05-01T12:02:00Z","level":"info"}

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/logstash/config/logstash-fips.yml

@@ -0,0 +1,6 @@
+api.http.host: "0.0.0.0"
+xpack.monitoring.enabled: false
+
+pipeline.ordered: false
+pipeline.workers: 2
+pipeline.buffer.type: heap

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/logstash/pipeline/logstash-to-elasticsearch-weak.conf

@@ -0,0 +1,28 @@
+input {
+  generator {
+    # Generate this message indefinitely to give ES container time to come online
+    count => -1
+    lines => ["FIPS weak protocol test heartbeat"]
+  }
+}
+
+filter {
+  mutate {
+    add_field => {
+      "fips_test" => "true"
+    }
+  }
+}
+
+output {
+  elasticsearch {
+    hosts => ["https://elasticsearch:9200"]
+    user => "elastic"
+    password => "changeme"
+    ssl_enabled => true
+    ssl_verification_mode => "none"
+    ssl_supported_protocols => ["TLSv1.1"]
+    ssl_certificate_authorities => ["/usr/share/logstash/config/certs/ca.crt"]
+    index => "logstash-weak-ssl-test-%{+YYYY.MM.dd}"
+  }
+}

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/logstash/pipeline/logstash-to-elasticsearch.conf

@@ -0,0 +1,28 @@
+input {
+  generator {
+    # Generate this message indefinitely to give ES container time to come online
+    count => -1
+    lines => ["FIPS compliance test heartbeat"]
+  }
+}
+
+filter {
+  mutate {
+    add_field => {
+      "fips_test" => "true"
+    }
+  }
+}
+
+output {
+  elasticsearch {
+    hosts => ["https://elasticsearch:9200"]
+    user => "elastic"
+    password => "changeme"
+    ssl_enabled => true
+    ssl_verification_mode => "full"
+    ssl_certificate_authorities => ["/usr/share/logstash/config/certs/ca.crt"]
+    index => "logstash-fips-test-%{+YYYY.MM.dd}"
+    ssl_supported_protocols => ["TLSv1.2"]
+  }
+}

x-pack/distributions/internal/observabilitySRE/qa/acceptance/spec/acceptance_tests_spec.rb

@@ -0,0 +1,118 @@
+require 'net/http'
+require 'uri'
+require 'json'
+require 'timeout'
+
+describe "ObservabilitySRE FIPS container" do
+  def es_request(path, body = nil)
+    es_url = "https://localhost:9200"
+    es_user = 'elastic'
+    es_password = 'changeme'
+    uri = URI.parse(es_url + path)
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = true
+    http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+
+    request = body ? Net::HTTP::Post.new(uri.request_uri) : Net::HTTP::Get.new(uri.request_uri)
+    request.basic_auth(es_user, es_password)q
+    request["Content-Type"] = "application/json"
+    request.body = body if body
+
+    http.request(request)
+  end
+
+  def wait_until(timeout: 30, interval: 1, message: nil)
+    Timeout.timeout(timeout) do
+      loop do
+        break if yield
+        sleep interval
+      end
+    end
+  rescue Timeout::Error
+    raise message || "Condition not met within #{timeout} seconds"
+  end
+
+  def wait_for_elasticsearch(max_retries = 120)
+    retries = 0
+    ready = false
+
+    while !ready && retries < max_retries
+      begin
+        response = es_request("/_cluster/health")
+        if response.code == "200"
+          health = JSON.parse(response.body)
+          if ["green", "yellow"].include?(health["status"])
+            ready = true
+          end
+        end
+      rescue => e
+        puts "Waiting for Elasticsearch: #{e.message}"
+      ensure
+        unless ready
+          retries += 1
+          sleep 1
+          puts "Retry #{retries}/#{max_retries}"
+        end
+      end
+    end
+
+    raise "System not ready after #{max_retries} seconds" unless ready
+  end
+
+  context "when running with FIPS-compliant configuration" do
+    before(:all) do
+      system("cd #{__dir__}/../docker && docker-compose up -d") or fail "Failed to start Docker Compose environment"
+      wait_for_elasticsearch
+    end
+
+    after(:all) do
+      system("cd #{__dir__}/../docker && docker-compose down -v")
+    end
+
+    it "data flows from Logstash to Elasticsearch using FIPS-approved SSL" do
+      # Wait for index to appear, indicating data is flowing
+      wait_until(timeout: 30, message: "Index logstash-fips-test not found") do
+        response = es_request("/_cat/indices?v")
+        response.code == "200" && response.body.include?("logstash-fips-test")
+      end
+      # Wait until specific data from logstash generator/mutate filters are observed
+      query = { query: { match_all: {} } }.to_json
+      result = nil
+      wait_until(timeout: 30, message: "Index logstash-fips-test not found") do
+          response = es_request("/logstash-fips-test-*/_search", query)
+          result = JSON.parse(response.body)
+          response.code == "200" && result["hits"]["total"]["value"] > 0
+        end
+      expect(result["hits"]["hits"].first["_source"]).to include("fips_test")
+    end
+  end
+
+  context "when running with non-FIPS compliant configuration" do
+    before(:all) do
+      system("cd #{__dir__}/../docker && LOGSTASH_PIPELINE=logstash-to-elasticsearch-weak.conf docker-compose up -d") or fail "Failed to start Docker Compose with weak SSL"
+      wait_for_elasticsearch
+    end
+
+    after(:all) do
+      system("cd #{__dir__}/../docker && docker-compose down -v")
+    end
+
+    it "prevents data flow when using TLSv1.1 which is not FIPS-compliant" do
+        # Allow time for Logstash to attempt connections (and fail)
+        sleep 15
+
+        # Verify that no index has been created that would indicate successful data flow
+        response = es_request("/_cat/indices?v")
+        today_pattern = "logstash-weak-ssl-test-#{Time.now.strftime('%Y.%m.%d')}"
+        expect(response.body).not_to include(today_pattern)
+
+        # Check logs for the specific BouncyCastle FIPS error we expect
+        logs = `docker logs fips_test_logstash 2>&1`
+
+        # Verify the logs contain the FIPS-mode TLS protocol error
+        expect(logs).to include("No usable protocols enabled")
+        expect(logs).to include("IllegalStateException")
+        expect(logs).to include("org.bouncycastle")
+      end
+  end
+end

x-pack/distributions/internal/observabilitySRE/qa/smoke/docker/docker-compose.yml

@@ -2,7 +2,7 @@ version: '3'
 
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch-fips:8.19.0-SNAPSHOT
+    image: docker.elastic.co/cloud-release/elasticsearch-cloud-ess-fips:8.19.0-SNAPSHOT
     environment:
       - discovery.type=single-node
       - xpack.security.enabled=true

x-pack/src/test/java/org/logstash/xpack/test/RSpecObservabilitySREAcceptanceTests.java

@@ -0,0 +1,18 @@
+package org.logstash.xpack.test;
+
+import org.junit.Test;
+import java.util.Arrays;
+import java.util.List;
+
+public class RSpecObservabilitySREAcceptanceTests extends RSpecTests {
+    @Override
+    protected List<String> rspecArgs() {
+        return Arrays.asList("-fd", "distributions/internal/observabilitySRE/qa/acceptance/spec");
+    }
+
+    @Test
+    @Override
+    public void rspecTests() throws Exception {
+        super.rspecTests();
+    }
+}