[BBOT] New integration for Black Lantern Security scanner (#9651)

Author: CarsonHrusovsky

.github/CODEOWNERS

@@ -97,6 +97,7 @@
 /packages/azure_metrics/data_stream/storage_account @elastic/obs-ds-hosted-services
 /packages/barracuda @elastic/security-service-integrations
 /packages/barracuda_cloudgen_firewall @elastic/security-service-integrations
+/packages/bbot @elastic/security-service-integrations
 /packages/beaconing @elastic/ml-ui @elastic/sec-applied-ml
 /packages/beat @elastic/stack-monitoring
 /packages/bitdefender @elastic/security-service-integrations

packages/bbot/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 carsonhrusovsky nicpenning zaynsnyder nickprokop williambakeberg
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

packages/bbot/_dev/build/build.yml

@@ -0,0 +1,4 @@
+dependencies:
+  ecs:
+    reference: [email protected]
+    import_mappings: true

packages/bbot/_dev/build/docs/README.md

@@ -0,0 +1,37 @@
+# BBOT integration
+
+Please read this page in its entirety as this integration requires some setup.
+
+This integration is for [BBOT](https://www.blacklanternsecurity.com/bbot/), an Attack Surface Management (ASM) Open Source Inteligence (OSINT) Tool. BBOT itself stands for Bighuge BLS OSINT Tool (BBOT).
+
+This integration requires the external use of BBOT. You will have to download and run the tool apart from this integration. Once your scan is complete, this integration will ingest the results into Elastic.
+
+This tool is used to enhance your external knowledge of your environment. This is done through the integration of many tools into BBOT providing a overview of your attack surface. Here is [how it works](https://www.blacklanternsecurity.com/bbot/how_it_works/).
+
+**Important Note** - You will have to provide the following parameter in your BBOT scan for your output.ndjson to be formatted correctly.
+```
+-c output_modules.json.siem_friendly=true
+```
+**Example BBOT Scan**
+```
+bbot -t elastic.co --strict-scope -f safe passive -c output_modules.json.siem_friendly=true -om json
+```
+
+You will have to configure the path for the output file within the integration settings. A common and popular path that could work here is:
+
+**Example BBOT Path**
+```
+/home/<user>/.bbot/scans/*/output.ndjson
+```
+
+BBOT Scanning [Documentation](https://www.blacklanternsecurity.com/bbot/scanning/).
+
+- `bbot` dataset: Made up of the findings found in the BBOT Scans.
+
+## Logs
+
+### ASM Findings
+
+{{event "asm_intel"}}
+
+{{fields "asm_intel"}}

packages/bbot/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,7 @@
+services:
+  bbot-logfile:
+    image: alpine
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+      - ${SERVICE_LOGS_DIR}:/var/log
+    command: /bin/sh -c "cp /sample_logs/* /var/log/"

packages/bbot/_dev/deploy/docker/sample_logs/log.log

@@ -0,0 +1,7 @@
+{"type": "SCAN", "id": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "data": {"SCAN": "heinous_hermione (SCAN:725368977d3a680e579707504e59428a7e3acc9d)"}, "scope_distance": 0, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170919.403808, "source": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "tags": ["in-scope"], "module": "TARGET", "module_sequence": "TARGET"}
+{"type": "DNS_NAME", "id": "DNS_NAME:f57ba0828becd7bf94faa616db081ed06f31bd3d", "data": {"DNS_NAME": "example.com"}, "scope_distance": 0, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170907.779394, "resolved_hosts": ["123.123.123.123"], "source": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "tags": ["resolved", "a-record", "target", "in-scope", "subdomain"], "module": "TARGET", "module_sequence": "TARGET"}
+{"type": "ORG_STUB", "id": "ORG_STUB:4452d71687b6bc2c9389c3349fdc17fbd73b833b", "data": {"ORG_STUB": "ex"}, "scope_distance": 1, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170932.070682, "source": "DNS_NAME:f57ba0828becd7bf94faa616db081ed06f31bd3d", "tags": ["distance-1"], "module": "speculate", "module_sequence": "speculate"}
+{"type": "DNS_NAME", "id": "DNS_NAME:f57ba0828becd7bf94faa616db081ed06f31bd3d", "data": {"DNS_NAME": "example.com"}, "scope_distance": 0, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170932.40485, "resolved_hosts": ["123.123.123.123"], "source": "IP_ADDRESS:605bb7cd2537bcf0fd94ce6b120bfb56cf3e0753", "tags": ["resolved", "a-record", "in-scope", "subdomain"], "module": "PTR", "module_sequence": "PTR"}
+{"type": "IP_ADDRESS", "id": "IP_ADDRESS:605bb7cd2537bcf0fd94ce6b120bfb56cf3e0753", "data": {"IP_ADDRESS": "123.123.123.123"}, "scope_distance": 1, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170932.033501, "resolved_hosts": ["123.123.123.123"], "source": "DNS_NAME:f57ba0828becd7bf94faa616db081ed06f31bd3d", "tags": ["ipv4", "resolved", "distance-1", "internal", "ptr-record"], "module": "A", "module_sequence": "A"}
+{"type": "DNS_NAME", "id": "DNS_NAME:32102fa4a314358e17370aaec3b0f32641dd9ab3", "data": {"DNS_NAME": "example.com"}, "scope_distance": 1, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170933.584077, "resolved_hosts": ["123.123.123.123"], "source": "OPEN_TCP_PORT:62743609c2dcda87da01a4faa9ddd1b7b362939f", "tags": ["soa-record", "affiliate", "resolved", "distance-1", "domain", "a-record", "txt-record", "mx-record", "ns-record"], "module": "sslcert", "module_sequence": "sslcert"}
+{"type": "AZURE_TENANT", "id": "AZURE_TENANT:db22e942c8a51294e1abc874daa076caecccbf01", "data": {"AZURE_TENANT": {"tenant-names": ["exampletenant"], "domains": ["example.smtp.online", "example.com", "example1.com", "example.org", "example2.com", "example.gov", "example3.com"], "tenant-id": "70af547c-69ab-416d-b4a6-543b5ce52b99"}}, "scope_distance": 1, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170932.947014, "source": "DNS_NAME:f57ba0828becd7bf94faa616db081ed06f31bd3d", "tags": ["distance-1"], "module": "azure_tenant", "module_sequence": "azure_tenant"}

packages/bbot/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.1.0"
+  changes:
+    - description: Initial release of the bbot package
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/9651

packages/bbot/data_stream/asm_intel/_dev/test/pipeline/test-bbot-ndjson.log

@@ -0,0 +1,10 @@
+{"type": "DNS_NAME", "id": "DNS_NAME:9d26e95592fabe5d7756f89b8ca33c8c0a0bfc88", "data": {"DNS_NAME": "elastic.co"}, "scope_distance": 0, "scan": "SCAN:c0c85de7152b616ba28b9d20b4f009df2dc3d20e", "timestamp": 1710970699.038803, "resolved_hosts": ["2600:1901:0:1f6d::", "34.107.161.234"], "source": "SCAN:c0c85de7152b616ba28b9d20b4f009df2dc3d20e", "tags": ["target", "soa-record", "mx-record", "ns-record", "resolved", "aaaa-record", "in-scope", "a-record", "txt-record", "domain", "cloud-google"], "module": "TARGET", "module_sequence": "TARGET"}
+{"type": "ORG_STUB", "id": "ORG_STUB:988eb4980904d42862f7dc9d4eed52469312ac20", "data": {"ORG_STUB": "elastic"}, "scope_distance": 1, "scan": "SCAN:c0c85de7152b616ba28b9d20b4f009df2dc3d20e", "timestamp": 1710970733.812364, "source": "DNS_NAME:9d26e95592fabe5d7756f89b8ca33c8c0a0bfc88", "tags": ["distance-1"], "module": "speculate", "module_sequence": "speculate"}
+{"type": "PROTOCOL", "id": "PROTOCOL:d24ec95c89ca37a123377fb3727fab7d49215709", "data": {"PROTOCOL": {"host": "elastic.co", "protocol": "HTTPS", "port": 443}}, "scope_distance": 0, "scan": "SCAN:c0c85de7152b616ba28b9d20b4f009df2dc3d20e", "timestamp": 1710970734.23711, "resolved_hosts": ["2600:1901:0:1f6d::", "34.107.161.234"], "source": "OPEN_TCP_PORT:a65b60678a7b6f30b1bd040f731582402baac2b8", "tags": ["ip-34-107-161-234", "cloud-google", "in-scope"], "module": "fingerprintx", "module_sequence": "fingerprintx"}
+{"type": "DNS_NAME", "id": "DNS_NAME:9999999999999999999999", "data": {"DNS_NAME": "test.com"}, "scope_distance": 0, "scan": "SCAN:79d977fa01a9f53c31301b3d2c28a738b569e1fa", "timestamp": 1709244163.717503, "resolved_hosts": ["192.168.1.1"], "source": "IP_ADDRESS:9999999999999999999999", "tags": ["ns-record", "soa-record", "resolved", "a-record", "domain", "in-scope"], "module": "PTR", "module_sequence": "PTR"}
+{"type": "TECHNOLOGY", "id": "TECHNOLOGY:84bddf65d64caa76bc0afcc889bfb442a33d9a5a", "data": {"TECHNOLOGY": {"host": "elastic.co", "technology": "google cloud", "url": "https://elastic.co/"}}, "scope_distance": 0, "scan": "SCAN:c0c85de7152b616ba28b9d20b4f009df2dc3d20e", "timestamp": 1710970734.669938, "resolved_hosts": ["2600:1901:0:1f6d::", "34.107.161.234"], "source": "URL:61d1d0467d7e841c13fff70dff85bb305cac7e1a", "tags": ["in-scope", "cloud-google"], "module": "wappalyzer", "module_sequence": "httpx->wappalyzer"}
+{"type": "FINDING", "id": "FINDING:769d4c9101436d8a2d4cc2d2033643c14e1a4a1d", "data": {"FINDING": {"host": "login.windows.net", "description": "Potentially Sprayable OAUTH Endpoint (domain: elastic.co) at https://login.windows.net/4fa94b7d-a743-486f-abcc-6c276c44cf4b/oauth2/token", "url": "https://login.windows.net/4fa94b7d-a743-486f-abcc-6c276c44cf4b/oauth2/token"}}, "scope_distance": 1, "scan": "SCAN:c0c85de7152b616ba28b9d20b4f009df2dc3d20e", "timestamp": 1710970742.029721, "resolved_hosts": [], "source": "DNS_NAME:9d26e95592fabe5d7756f89b8ca33c8c0a0bfc88", "tags": ["affiliate", "distance-1"], "module": "oauth", "module_sequence": "oauth->oauth"}
+{"type": "PROTOCOL", "id": "PROTOCOL:d24ec95c89ca37a123377fb3727fab7d49215709", "data": {"PROTOCOL": {"host": "elastic.co", "protocol": "HTTPS", "port": 443}}, "scope_distance": 0, "scan": "SCAN:c0c85de7152b616ba28b9d20b4f009df2dc3d20e", "timestamp": 1710970734.23711, "resolved_hosts": ["2600:1901:0:1f6d::", "34.107.161.234"], "source": "OPEN_TCP_PORT:a65b60678a7b6f30b1bd040f731582402baac2b8", "tags": ["ip-34-107-161-234", "cloud-google", "in-scope"], "module": "fingerprintx", "module_sequence": "fingerprintx"}
+{"type": "ASN", "id": "ASN:6af45683f00c3b8f6f32d0acf47ebeb2d4bdda3e", "data": {"ASN": {"asn": "396982", "subnet": "34.96.0.0/12", "name": "GOOGLE-CLOUD-PLATFORM", "description": "Google LLC", "country": "US"}}, "scope_distance": 2, "scan": "SCAN:c0c85de7152b616ba28b9d20b4f009df2dc3d20e", "timestamp": 1710970737.966555, "source": "IP_ADDRESS:54cdd6c3f5decd0856fd40523d6b1ce78c3f7f09", "tags": ["distance-2"], "module": "asn", "module_sequence": "asn"}
+{"type": "STORAGE_BUCKET", "id": "STORAGE_BUCKET:53ea3e7d2e3faa5d6d27fa912b04e30d25e1e59f", "data": {"STORAGE_BUCKET": {"name": "elastic-co", "url": "https://www.googleapis.com/storage/v1/b/elastic-co"}}, "web_spider_distance": 0, "scope_distance": 1, "scan": "SCAN:c0c85de7152b616ba28b9d20b4f009df2dc3d20e", "timestamp": 1710970736.836363, "resolved_hosts": ["2607:f8b0:4009:804::200a", "172.217.5.10", "142.250.191.170", "142.250.191.138", "142.250.191.106", "142.251.32.10", "142.250.191.202", "142.250.191.234", "2607:f8b0:4009:81c::200a", "2607:f8b0:4009:81b::200a", "2607:f8b0:4009:803::200a", "172.217.0.170"], "source": "DNS_NAME:9d26e95592fabe5d7756f89b8ca33c8c0a0bfc88", "tags": ["cloud-google", "distance-1"], "module": "bucket_google", "module_sequence": "bucket_google"}
+{"type": "WAF", "id": "WAF:99999999999999999", "data": {"WAF": {"url": "https://www.testsite.test/", "host": "www.testsite.test", "waf": "WAF DATA"}}, "scope_distance": 0, "scan": "SCAN:79d977fa01a9f53c31301b3d2c28a738b569e1fa", "timestamp": 1709244216.155426, "resolved_hosts": [], "source": "URL:9999999999999999999", "tags": ["in-scope"], "module": "wafw00f", "module_sequence": "wafw00f"}