WIP: Idea for how to handle multipe container configs for acceptance tests

This commit shows the rough structure for how I am planning on handling docker
compose networks for acceptance tests. The main idea is to use interpolation in
the docker compose file to point to different configuration files for
filebeat/logstash/elasticsearch. This is mainly due to the nature of these tests
showing behavior when the system is and is not configured properly for FIPS. The
breakdown in responsibility is:

1. Gradle handles cert generation (similar to smoke test, this avoids checking
in PKI)
2. Rspec handles stopping/starting docker compose and managing environment vars
for intperolation in docker compose manifests (different from smoke tests where
a single static docker compose is started in gradle)
3. Rspec handles deciding when containers are ready and querying state about
data flowing through the system
4. Gradle cleans up certs

THis is just a rough sketch, there are still bugs to be worked out but before i
get too far in to it I want to get the idea out there.

Author: donoghuc

x-pack/build.gradle

@@ -120,5 +120,20 @@ tasks.register("observabilitySREacceptanceTests", Test) {
     description = "Run ObservabilitySRE acceptance tests"
     // Need to have set up the ruby environment for rspec even through we are running in container
     dependsOn(":bootstrap", ":logstash-core:assemble", ":installDevelopmentGems")
-    // TODO: hook in to rspec
+
+    inputs.files fileTree("${projectDir}/distributions/internal/observabilitySRE/qa/smoke")
+    doFirst {
+        // Generate the certificates first
+        exec {
+            workingDir file("distributions/internal/observabilitySRE/qa/acceptance/docker/certs")
+            commandLine 'bash', './generate.sh'
+            ignoreExitValue = false
+        }
+    }
+    systemProperty 'logstash.root.dir', projectDir.parent
+    include '**/org/logstash/xpack/test/RSpecObservabilitySREAcceptanceTests.class'
+    doLast {
+        // Clean up the generated certificates
+        delete fileTree("distributions/internal/observabilitySRE/qa/acceptance/docker/certs").include("*.key", "*.crt", "*.csr", "*.srl")
+    }
 }

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/certs/.gitignore

@@ -0,0 +1,3 @@
+*.crt
+*.csr
+*.key

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/certs/generate.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+echo "Generating CA certificate"
+openssl req -x509 -newkey rsa:3072 -days 365 -nodes -keyout ca.key -out ca.crt -subj "/CN=Elastic-CA" -sha256
+
+echo "Generating Elasticsearch certificate"
+openssl req -newkey rsa:3072 -nodes -keyout elasticsearch.key -out elasticsearch.csr -subj "/CN=elasticsearch" -sha256
+openssl x509 -req -in elasticsearch.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out elasticsearch.crt -days 365 -sha256
+
+echo "Generating Logstash certificate"
+openssl req -newkey rsa:3072 -nodes -keyout logstash.key -out logstash.csr -subj "/CN=logstash" -sha256
+openssl x509 -req -in logstash.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out logstash.crt -days 365 -sha256
+
+echo "Generating Filebeat certificate"
+openssl req -newkey rsa:3072 -nodes -keyout filebeat.key -out filebeat.csr -subj "/CN=filebeat" -sha256
+openssl x509 -req -in filebeat.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out filebeat.crt -days 365 -sha256
+
+chmod 644 *.crt *.key

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/docker-compose.yml

@@ -0,0 +1,51 @@
+services:
+  logstash:
+    image: docker.elastic.co/logstash/logstash-observability-sre:${OBSERVABILITY_SRE_IMAGE_VERSION:-8.19.0-SNAPSHOT}
+    container_name: fips_test_logstash
+    ports:
+      - "9600:9600"
+      - "5044:5044"
+    volumes:
+      - ./logstash/config/${LOGSTASH_CONFIG:-logstash-fips.yml}:/usr/share/logstash/config/logstash.yml
+      - ./logstash/pipeline/${LOGSTASH_PIPELINE:-logstash-to-elasticsearch.conf}:/usr/share/logstash/pipeline/logstash.conf
+      - ./certs:/usr/share/logstash/config/certs
+    environment:
+      - ELASTIC_STACK_SECURITY_ENABLED=true
+      - ELASTIC_SSL_CERTIFICATE_AUTHORITIES=/usr/share/logstash/config/certs/ca.crt
+      - ELASTIC_SSL_CERTIFICATE=/usr/share/logstash/config/certs/logstash.crt
+      - ELASTIC_SSL_KEY=/usr/share/logstash/config/certs/logstash.key
+    networks:
+      - elastic
+    depends_on:
+      - elasticsearch
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch-fips:${ELASTICSEARCH_IMAGE_VERSION:-8.19.0-SNAPSHOT}
+    container_name: fips_test_elasticsearch
+    ports:
+      - "9200:9200"
+    volumes:
+      - ./elasticsearch/config/${ELASTICSEARCH_CONFIG:-elasticsearch-fips.yml}:/usr/share/elasticsearch/config/elasticsearch.yml
+      - ./certs:/usr/share/elasticsearch/config/certs
+    environment:
+      - discovery.type=single-node
+      - ES_JAVA_OPTS=-Xms1g -Xmx1g
+      - ELASTIC_PASSWORD=changeme
+    networks:
+      - elastic
+# Filebeat is not yet used in tests, but this is included to show that including it in the compose network
+# will not adversely affect startup time etc for testing interactions between other components.
+  filebeat:
+    image: docker.elastic.co/elasticsearch/elasticsearch-fips:${FILEBEAT_IMAGE_VERSION:-8.19.0-SNAPSHOT}
+    container_name: fips_test_filebeat
+    entrypoint: ["filebeat", "-e", "--strict.perms=false", "-c", "/usr/share/filebeat/filebeat.yml"]
+    volumes:
+      - ./filebeat/config/${FILEBEAT_CONFIG:-filebeat-fips.yml}:/usr/share/filebeat/filebeat.yml
+      - ./filebeat/data:/data
+      - ./certs:/usr/share/filebeat/certs
+    profiles:
+      - filebeat
+    networks:
+      - elastic
+networks:
+  elastic:
+    driver: bridge

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/elasticsearch/config/elasticsearch-fips.yml

@@ -0,0 +1,15 @@
+# Elasticsearch settings
+discovery.type: single-node
+http.port: 9200
+network.host: 0.0.0.0
+# Security settings
+xpack.security.enabled: true
+xpack.security.transport.ssl.enabled: true
+xpack.security.transport.ssl.verification_mode: certificate
+xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certs/elasticsearch.key
+xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certs/elasticsearch.crt
+xpack.security.transport.ssl.certificate_authorities: ["/usr/share/elasticsearch/config/certs/ca.crt"]
+xpack.security.http.ssl.enabled: true
+xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certs/elasticsearch.key
+xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certs/elasticsearch.crt
+xpack.security.http.ssl.certificate_authorities: ["/usr/share/elasticsearch/config/certs/ca.crt"]

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/filebeat/config/filebeat-fips.yml

@@ -0,0 +1,13 @@
+filebeat.inputs:
+- type: log
+  enabled: true
+  paths:
+    - /test-logs/*.log
+
+output.logstash:
+  hosts: ["logstash:5044"]
+  ssl.enabled: true
+  ssl.certificate: "/usr/share/elasticsearch/config/certs/filebeat.crt"
+  ssl.key: "/usr/share/elasticsearch/config/certs/filebeat.key"
+  ssl.certificate_authorities: ["/usr/share/elasticsearch/config/certs/ca.crt"]
+

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/filebeat/data/sample_logs.json

@@ -0,0 +1,3 @@
+{"message":"FIPS compliance test message 1","timestamp":"2025-05-01T12:00:00Z","level":"info"}
+{"message":"FIPS compliance test message 2","timestamp":"2025-05-01T12:01:00Z","level":"debug"}
+{"message":"FIPS compliance test message 3","timestamp":"2025-05-01T12:02:00Z","level":"info"}

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/logstash/config/logstash-fips.yml

@@ -0,0 +1,6 @@
+api.http.host: "0.0.0.0"
+xpack.monitoring.enabled: false
+
+pipeline.ordered: false
+pipeline.workers: 2
+pipeline.buffer.type: heap

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/logstash/pipeline/logstash-to-elasticsearch.conf

@@ -0,0 +1,28 @@
+input {
+  generator {
+    # Generate this message indefinitely to give ES container time to come online
+    count => -1
+    lines => ["FIPS compliance test heartbeat"]
+  }
+}
+
+filter {
+  mutate {
+    add_field => {
+      "fips_test" => "true"
+    }
+  }
+}
+
+output {
+  elasticsearch {
+    hosts => ["https://elasticsearch:9200"]
+    user => "elastic"
+    password => "changeme"
+    ssl_enabled => true
+    ssl_verification_mode => "full"
+    ssl_certificate_authorities => ["/usr/share/logstash/config/certs/ca.crt"]
+    index => "logstash-fips-test-%{+YYYY.MM.dd}"
+    ssl_supported_protocols => ["TLSv1.2"]
+  }
+}

x-pack/distributions/internal/observabilitySRE/qa/acceptance/spec/acceptance_tests_spec.rb

@@ -0,0 +1,88 @@
+require 'net/http'
+require 'uri'
+require 'json'
+require 'timeout'
+
+describe "ObservabilitySRE FIPS container running on FIPS vm" do
+  
+  before(:all) do
+    # Start docker-compose and wait for ES
+    system("cd #{__dir__}/../docker && docker-compose up -d") or fail "Failed to start Docker Compose environment"
+    max_retries = 120
+    retries = 0
+    ready = false
+    
+    while !ready && retries < max_retries
+      begin
+        # Wait for elasticsearch to be ready
+        response = es_request("/_cluster/health")
+        if response.code == "200"
+          health = JSON.parse(response.body)
+          if ["green", "yellow"].include?(health["status"])
+            ready = true
+          end
+        end
+      rescue => e
+        puts "Waiting for Elasticsearch: #{e.message}"
+      ensure
+        unless ready
+          retries += 1
+          sleep 1
+          puts "Retry #{retries}/#{max_retries}"
+        end
+      end
+    end
+    
+    raise "System not ready after #{max_retries} seconds" unless ready
+  end
+  
+  after(:all) do
+    # stop docker network
+    system("cd #{__dir__}/../docker && docker-compose down -v")
+  end
+  
+  it "data flows from Logstash to Elasticsearch using FIPS-approved SSL" do
+    # Wait for index to appear, indicating data is flowing
+    wait_until(timeout: 30, message: "Index logstash-fips-test not found") do
+      response = es_request("/_cat/indices?v")
+      response.code == "200" && response.body.include?("logstash-fips-test")
+    end
+    # Wait until specific data from logstash generator/mutate filters are observed
+    query = { query: { match_all: {} } }.to_json
+    result = nil
+    wait_until(timeout: 30, message: "Index logstash-fips-test not found") do
+        response = es_request("/logstash-fips-test-*/_search", query)
+        result = JSON.parse(response.body)
+        response.code == "200" && result["hits"]["total"]["value"] > 0
+      end
+    expect(result["hits"]["hits"].first["_source"]).to include("fips_test")
+  end
+
+  def wait_until(timeout: 30, interval: 1, message: nil)
+    Timeout.timeout(timeout) do
+      loop do
+        break if yield
+        sleep interval
+      end
+    end
+  rescue Timeout::Error
+    raise message || "Condition not met within #{timeout} seconds"
+  end
+
+  def es_request(path, body = nil)
+    es_url = "https://localhost:9200"
+    es_user = 'elastic'
+    es_password = 'changeme'
+    uri = URI.parse(es_url + path)
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = true
+    http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    
+    request = body ? Net::HTTP::Post.new(uri.request_uri) : Net::HTTP::Get.new(uri.request_uri)
+    request.basic_auth(es_user, es_password)
+    request["Content-Type"] = "application/json"
+    request.body = body if body
+    
+    http.request(request)
+  end
+end

x-pack/src/test/java/org/logstash/xpack/test/RSpecObservabilitySREAcceptanceTests.java

@@ -0,0 +1,18 @@
+package org.logstash.xpack.test;
+
+import org.junit.Test;
+import java.util.Arrays;
+import java.util.List;
+
+public class RSpecObservabilitySREAcceptanceTests extends RSpecTests {
+    @Override
+    protected List<String> rspecArgs() {
+        return Arrays.asList("-fd", "distributions/internal/observabilitySRE/qa/acceptance/spec");
+    }
+
+    @Test
+    @Override
+    public void rspecTests() throws Exception {
+        super.rspecTests();
+    }
+}