Add a step to exhaustive tests for observabilitySRE accetpance testing

.buildkite/scripts/exhaustive-tests/generate-steps.py

@@ -168,6 +168,26 @@ def acceptance_docker_steps()-> list[typing.Any]:
 
     return steps
 
+def fips_test_runner_step() -> dict[str, typing.Any]:
+    step = {
+        "label": "Observability SRE Acceptance Tests",
+        "key": "observabilitySRE-acceptance-tests",
+        "agents": {
+            "provider": "aws",
+            "instanceType": "m6i.xlarge",
+            "diskSizeGb": 60,
+            "instanceMaxAge": 1440,
+            "imagePrefix": "platform-ingest-logstash-ubuntu-2204-fips"
+        },
+        "retry": {"automatic": [{"limit": 1}]},
+        "command": LiteralScalarString("""#!/usr/bin/env bash
+set -euo pipefail
+source .buildkite/scripts/common/vm-agent.sh
+./gradlew observabilitySREacceptanceTests --stacktrace
+"""),
+    }
+    return step
+
 if __name__ == "__main__":
     LINUX_OS_ENV_VAR_OVERRIDE = os.getenv("LINUX_OS")
     WINDOWS_OS_ENV_VAR_OVERRIDE = os.getenv("WINDOWS_OS")
@@ -215,5 +235,12 @@ def acceptance_docker_steps()-> list[typing.Any]:
             "steps": acceptance_docker_steps(),
     })
 
+    structure["steps"].append({
+        "group": "Observability SRE Acceptance Tests",
+        "key": "acceptance-observability-sre",
+        "depends_on": ["testing-phase"],
+        "steps": [fips_test_runner_step()],
+    })
+
     print('# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json')
     YAML().dump(structure, sys.stdout)

x-pack/build.gradle

@@ -75,6 +75,7 @@ tasks.register("buildFipsValidationGem") {
     rake(rootProject.projectDir, rootProject.buildDir, 'plugin:build-fips-validation-plugin')
   }
 }
+
 tasks.register("observabilitySREsmokeTests", Test) {
     description = "Run ObservabilitySRE smoke tests using docker-compose and RSpec"
     // Need to have set up the ruby environment for rspec even through we are running in container
@@ -114,3 +115,25 @@ tasks.register("observabilitySREsmokeTests", Test) {
         delete fileTree("distributions/internal/observabilitySRE/qa/smoke/docker/certs").include("*.key", "*.crt", "*.csr", "*.srl")
     }
 }
+
+tasks.register("observabilitySREacceptanceTests", Test) {
+    description = "Run ObservabilitySRE acceptance tests"
+    // Need to have set up the ruby environment for rspec even through we are running in container
+    dependsOn(":bootstrap", ":logstash-core:assemble", ":installDevelopmentGems")
+
+    inputs.files fileTree("${projectDir}/distributions/internal/observabilitySRE/qa/smoke")
+    doFirst {
+        // Generate the certificates first
+        exec {
+            workingDir file("distributions/internal/observabilitySRE/qa/acceptance/docker/certs")
+            commandLine 'bash', './generate.sh'
+            ignoreExitValue = false
+        }
+    }
+    systemProperty 'logstash.root.dir', projectDir.parent
+    include '**/org/logstash/xpack/test/RSpecObservabilitySREAcceptanceTests.class'
+    doLast {
+        // Clean up the generated certificates
+        delete fileTree("distributions/internal/observabilitySRE/qa/acceptance/docker/certs").include("*.key", "*.crt", "*.csr", "*.srl")
+    }
+}

x-pack/distributions/internal/observabilitySRE/plugin/logstash-integration-fips_validation/lib/logstash/fips_validation.rb

@@ -42,7 +42,7 @@ def before_bootstrap_checks(runner)
       # ensure Bouncycastle is configured and ready
       begin
         if Java::org.bouncycastle.crypto.CryptoServicesRegistrar.isInApprovedOnlyMode
-          accumulator.success "Bouncycastle Crytpo is in `approved-only` mode"
+          accumulator.success "Bouncycastle Crypto is in `approved-only` mode"
         else
           accumulator.failure "Bouncycastle Crypto is not in 'approved-only' mode"
         end

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/certs/.gitignore

@@ -0,0 +1,3 @@
+*.crt
+*.csr
+*.key

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/certs/generate.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+echo "Generating CA certificate"
+openssl req -x509 -newkey rsa:3072 -days 365 -nodes -keyout ca.key -out ca.crt -subj "/CN=Elastic-CA" -sha256
+
+echo "Generating Elasticsearch certificate"
+openssl req -newkey rsa:3072 -nodes -keyout elasticsearch.key -out elasticsearch.csr -subj "/CN=elasticsearch" -sha256
+openssl x509 -req -in elasticsearch.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out elasticsearch.crt -days 365 -sha256
+
+echo "Generating Logstash certificate"
+openssl req -newkey rsa:3072 -nodes -keyout logstash.key -out logstash.csr -subj "/CN=logstash" -sha256
+openssl x509 -req -in logstash.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out logstash.crt -days 365 -sha256
+
+echo "Generating Filebeat certificate"
+openssl req -newkey rsa:3072 -nodes -keyout filebeat.key -out filebeat.csr -subj "/CN=filebeat" -sha256
+openssl x509 -req -in filebeat.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out filebeat.crt -days 365 -sha256
+
+chmod 644 *.crt *.key

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/docker-compose.yml

@@ -0,0 +1,56 @@
+services:
+  logstash:
+    image: docker.elastic.co/logstash/logstash-observability-sre:${OBSERVABILITY_SRE_IMAGE_VERSION:-8.19.0-SNAPSHOT}
+    container_name: fips_test_logstash
+    ports:
+      - "5044:5044"
+    volumes:
+      - ./logstash/config/${LOGSTASH_CONFIG:-logstash-fips.yml}:/usr/share/logstash/config/logstash.yml
+      - ./logstash/pipeline/${LOGSTASH_PIPELINE:-logstash-to-elasticsearch.conf}:/usr/share/logstash/pipeline/logstash.conf
+      - ./certs:/usr/share/logstash/config/certs
+    networks:
+      - elastic
+    depends_on:
+      - elasticsearch
+
+  elasticsearch:
+    image: docker.elastic.co/cloud-release/elasticsearch-cloud-ess-fips:${ELASTICSEARCH_IMAGE_VERSION:-8.19.0-SNAPSHOT}
+    container_name: fips_test_elasticsearch
+    ports:
+      - "9200:9200"
+    volumes:
+      - ./elasticsearch/config/${ELASTICSEARCH_CONFIG:-elasticsearch-fips.yml}:/usr/share/elasticsearch/config/elasticsearch.yml
+      - ./certs:/usr/share/elasticsearch/config/certs
+    environment:
+      - discovery.type=single-node
+      - ES_JAVA_OPTS=-Xms1g -Xmx1g
+      - ELASTIC_PASSWORD=changeme
+    networks:
+      - elastic
+
+  filebeat:
+    # The filebeat shipped with the elasticsearch-fips container is built for FIPS support
+    # There is no stand alone distribution. This uses the shipped version for testing. 
+    image: docker.elastic.co/cloud-release/elasticsearch-cloud-ess-fips:${FILEBEAT_IMAGE_VERSION:-8.19.0-SNAPSHOT}
+    container_name: fips_test_filebeat
+    working_dir: /usr/share/filebeat
+    entrypoint: ["/bin/bash", "-c"]
+    # Start Filebeat with /tmp for data (always writable)
+    command: 
+      - |
+        exec /opt/filebeat/filebeat -e \
+          --strict.perms=false \
+          -c /usr/share/filebeat/filebeat.yml \
+          --path.data /tmp/filebeat_data
+    volumes:
+      - ./filebeat/config/${FILEBEAT_CONFIG:-filebeat-fips.yml}:/usr/share/filebeat/filebeat.yml:ro
+      - ./filebeat/data:/data/logs:ro
+      - ./certs:/usr/share/filebeat/certs:ro
+    networks:
+      - elastic
+    depends_on:
+      - logstash
+
+networks:
+  elastic:
+    driver: bridge

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/elasticsearch/config/elasticsearch-fips.yml

@@ -0,0 +1,15 @@
+# Elasticsearch settings
+discovery.type: single-node
+http.port: 9200
+network.host: 0.0.0.0
+# Security settings
+xpack.security.enabled: true
+xpack.security.transport.ssl.enabled: true
+xpack.security.transport.ssl.verification_mode: certificate
+xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certs/elasticsearch.key
+xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certs/elasticsearch.crt
+xpack.security.transport.ssl.certificate_authorities: ["/usr/share/elasticsearch/config/certs/ca.crt"]
+xpack.security.http.ssl.enabled: true
+xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certs/elasticsearch.key
+xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certs/elasticsearch.crt
+xpack.security.http.ssl.certificate_authorities: ["/usr/share/elasticsearch/config/certs/ca.crt"]

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/filebeat/config/filebeat-fips.yml

@@ -0,0 +1,20 @@
+filebeat.inputs:
+- type: log
+  enabled: true
+  paths:
+    - /data/logs/sample_logs.txt
+
+output.logstash:
+  hosts: ["logstash:5044"]
+  ssl.enabled: true
+  ssl.certificate: "/usr/share/filebeat/certs/filebeat.crt"
+  ssl.key: "/usr/share/filebeat/certs/filebeat.key"
+  ssl.certificate_authorities: ["/usr/share/filebeat/certs/ca.crt"]
+  ssl.verification_mode: "certificate"
+
+# Add debugging
+logging.level: debug
+logging.to_stderr: true
+
+# Keep registry in the anonymous volume to avoid host pollution
+path.data: /tmp/filebeat_data

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/filebeat/data/sample_logs.txt

@@ -0,0 +1 @@
+TEST-LOG: FIPS filebeat test message

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/logstash/config/logstash-fips.yml

@@ -0,0 +1,6 @@
+api.http.host: "0.0.0.0"
+xpack.monitoring.enabled: false
+
+pipeline.ordered: false
+pipeline.workers: 2
+pipeline.buffer.type: heap

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/logstash/pipeline/filebeat-to-ls-to-es.conf

@@ -0,0 +1,26 @@
+input {
+  beats {
+    port => 5044
+    ssl_enabled => true
+    ssl_certificate => "/usr/share/logstash/config/certs/logstash.crt"
+    ssl_key => "/usr/share/logstash/config/certs/logstash.key"
+    ssl_certificate_authorities => ["/usr/share/logstash/config/certs/ca.crt"]
+  }
+}
+
+filter {
+  mutate {
+    add_tag => ["filebeat"]
+  }
+}
+
+output {
+  elasticsearch {
+    hosts => ["https://elasticsearch:9200"]
+    user => "elastic"
+    password => "changeme"
+    ssl_enabled => true
+    ssl_certificate_authorities => ["/usr/share/logstash/config/certs/ca.crt"]
+    index => "filebeat-test-%{+YYYY.MM.dd}"
+  }
+}

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/logstash/pipeline/filebeat-to-ls-weak.conf

@@ -0,0 +1,27 @@
+input {
+  beats {
+    port => 5044
+    ssl_enabled => true
+    ssl_certificate => "/usr/share/logstash/config/certs/logstash.crt"
+    ssl_key => "/usr/share/logstash/config/certs/logstash.key"
+    ssl_certificate_authorities => ["/usr/share/logstash/config/certs/ca.crt"]
+    ssl_supported_protocols => ["TLSv1.1"]
+  }
+}
+
+filter {
+  mutate {
+    add_tag => ["filebeat"]
+  }
+}
+
+output {
+  elasticsearch {
+    hosts => ["https://elasticsearch:9200"]
+    user => "elastic"
+    password => "changeme"
+    ssl_enabled => true
+    ssl_certificate_authorities => ["/usr/share/logstash/config/certs/ca.crt"]
+    index => "filebeat-weak-ssl-test-%{+YYYY.MM.dd}"
+  }
+}

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/logstash/pipeline/logstash-to-elasticsearch-weak.conf

@@ -0,0 +1,26 @@
+input {
+  generator {
+    lines => ["FIPS weak protocol test message"]
+  }
+}
+
+filter {
+  mutate {
+    add_field => {
+      "fips_test" => "true"
+    }
+  }
+}
+
+output {
+  elasticsearch {
+    hosts => ["https://elasticsearch:9200"]
+    user => "elastic"
+    password => "changeme"
+    ssl_enabled => true
+    ssl_verification_mode => "none"
+    ssl_supported_protocols => ["TLSv1.1"]
+    ssl_certificate_authorities => ["/usr/share/logstash/config/certs/ca.crt"]
+    index => "logstash-weak-ssl-test-%{+YYYY.MM.dd}"
+  }
+}

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/logstash/pipeline/logstash-to-elasticsearch.conf

@@ -0,0 +1,30 @@
+input {
+  generator {
+    lines => ["FIPS compliance test message"]
+  }
+}
+
+filter {
+  mutate {
+    add_field => {
+      "fips_test" => "true"
+    }
+  }
+}
+
+output {
+  elasticsearch {
+    hosts => ["https://elasticsearch:9200"]
+    user => "elastic"
+    password => "changeme"
+    ssl_enabled => true
+    ssl_verification_mode => "full"
+    ssl_certificate_authorities => ["/usr/share/logstash/config/certs/ca.crt"]
+    index => "logstash-fips-test-%{+YYYY.MM.dd}"
+    ssl_supported_protocols => ["TLSv1.2"]
+  }
+
+  stdout {
+    codec => rubydebug
+  }
+}