[Rule Tuning] ESQL Query Field Dynamic Field Standardization

rules/integrations/azure/credential_access_entra_signin_brute_force_microsoft_365_repeat_source.toml → rules/_deprecated/credential_access_entra_signin_brute_force_microsoft_365_repeat_source.toml

@@ -1,8 +1,9 @@
 [metadata]
 creation_date = "2024/09/06"
+deprecation_date = "2025/07/16"
 integration = ["azure"]
-maturity = "production"
-updated_date = "2025/06/06"
+maturity = "deprecated"
+updated_date = "2025/07/16"
 
 [rule]
 author = ["Elastic"]

rules/cross-platform/execution_potential_widespread_malware_infection.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2024/05/08"
 maturity = "production"
-updated_date = "2025/03/20"
+updated_date = "2025/07/16"
 
 [rule]
 author = ["Elastic"]
@@ -67,8 +67,8 @@ query = '''
 from logs-endpoint.alerts-*
 | where event.code in ("malicious_file", "memory_signature", "shellcode_thread") and rule.name is not null
 | keep host.id, rule.name, event.code
-| stats hosts = count_distinct(host.id) by rule.name, event.code
-| where hosts >= 3
+| stats Esql.host.id.count_distinct = count_distinct(host.id) by rule.name, event.code
+| where Esql.host.id.count_distinct >= 3
 '''
 
 

rules/cross-platform/initial_access_azure_o365_with_network_alert.toml

@@ -2,7 +2,7 @@
 creation_date = "2025/04/29"
 integration = ["azure", "o365"]
 maturity = "production"
-updated_date = "2025/07/02"
+updated_date = "2025/07/16"
 
 [rule]
 author = ["Elastic"]
@@ -78,21 +78,47 @@ type = "esql"
 
 query = '''
 FROM logs-*, .alerts-security.*
-// query runs every 1 hour looking for activities occured during last 8 hours to match on disparate events
+// query runs every 1 hour looking for activities occurred during last 8 hours to match on disparate events
 | where @timestamp > NOW() - 8 hours
 // filter for Azure or M365 sign-in and External Alerts with source.ip not null
-| where TO_IP(source.ip) is not null and (event.dataset in ("o365.audit", "azure.signinlogs") or kibana.alert.rule.name == "External Alerts") and
-// exclude private IP ranges
-  not CIDR_MATCH(TO_IP(source.ip), "10.0.0.0/8", "127.0.0.0/8", "169.254.0.0/16", "172.16.0.0/12", "192.0.0.0/24", "192.0.0.0/29", "192.0.0.8/32", "192.0.0.9/32", "192.0.0.10/32", "192.0.0.170/32", "192.0.0.171/32", "192.0.2.0/24", "192.31.196.0/24", "192.52.193.0/24", "192.168.0.0/16", "192.88.99.0/24", "224.0.0.0/4", "100.64.0.0/10", "192.175.48.0/24","198.18.0.0/15", "198.51.100.0/24", "203.0.113.0/24", "240.0.0.0/4", "::1","FE80::/10", "FF00::/8")
+| where TO_IP(source.ip) is not null
+  and (event.dataset in ("o365.audit", "azure.signinlogs") or kibana.alert.rule.name == "External Alerts")
+  and not CIDR_MATCH(
+    TO_IP(source.ip),
+    "10.0.0.0/8", "127.0.0.0/8", "169.254.0.0/16", "172.16.0.0/12", "192.0.0.0/24", "192.0.0.0/29",
+    "192.0.0.8/32", "192.0.0.9/32", "192.0.0.10/32", "192.0.0.170/32", "192.0.0.171/32", "192.0.2.0/24",
+    "192.31.196.0/24", "192.52.193.0/24", "192.168.0.0/16", "192.88.99.0/24", "224.0.0.0/4",
+    "100.64.0.0/10", "192.175.48.0/24", "198.18.0.0/15", "198.51.100.0/24", "203.0.113.0/24",
+    "240.0.0.0/4", "::1", "FE80::/10", "FF00::/8"
+  )
+
+// capture relevant raw fields
 | keep source.ip, event.action, event.outcome, event.dataset, kibana.alert.rule.name, event.category
-// split alerts to 3 buckets -  M365 mail access, azure sign-in and network related external alerts like NGFW and IDS
-| eval mail_access_src_ip = case(event.dataset == "o365.audit" and event.action == "MailItemsAccessed" and event.outcome == "success", TO_IP(source.ip), null),
-  azure_src_ip = case(event.dataset == "azure.signinlogs" and event.outcome == "success", TO_IP(source.ip), null),
-  network_alert_src_ip = case(kibana.alert.rule.name == "External Alerts" and not event.dataset in ("o365.audit", "azure.signinlogs"), TO_IP(source.ip), null)
-// aggregated alerts count by bucket and by source.ip
-| stats total_alerts = count(*), is_mail_access = COUNT_DISTINCT(mail_access_src_ip), is_azure = COUNT_DISTINCT(azure_src_ip), unique_dataset = COUNT_DISTINCT(event.dataset),is_network_alert = COUNT_DISTINCT(network_alert_src_ip), datasets = VALUES(event.dataset), rules = VALUES(kibana.alert.rule.name), cat = VALUES(event.category) by source_ip = TO_IP(source.ip)
-// filter for cases where there is a successful sign-in to azure or m365 mail and the source.ip is reported by a network external alert.
-| where is_network_alert  > 0 and unique_dataset >= 2 and (is_mail_access > 0 or is_azure > 0) and total_alerts <= 100
+
+// classify each source IP based on alert type
+| eval
+  Esql.source.ip.mail_access.case = case(event.dataset == "o365.audit" and event.action == "MailItemsAccessed" and event.outcome == "success", TO_IP(source.ip), null),
+  Esql.source.ip.azure_signin.case = case(event.dataset == "azure.signinlogs" and event.outcome == "success", TO_IP(source.ip), null),
+  Esql.source.ip.network_alert.case = case(kibana.alert.rule.name == "External Alerts" and not event.dataset in ("o365.audit", "azure.signinlogs"), TO_IP(source.ip), null)
+
+// aggregate by source IP
+| stats
+    Esql.event.count = count(*),
+    Esql.source.ip.mail_access.case.count_distinct = COUNT_DISTINCT(Esql.source.ip.mail_access.case),
+    Esql.source.ip.azure_signin.case.count_distinct = COUNT_DISTINCT(Esql.source.ip.azure_signin.case),
+    Esql.source.ip.network_alert.case.count_distinct = COUNT_DISTINCT(Esql.source.ip.network_alert.case),
+    Esql.event.dataset.count_distinct = COUNT_DISTINCT(event.dataset),
+    Esql.event.dataset.values = VALUES(event.dataset),
+    Esql.kibana.alert.rule.name.values = VALUES(kibana.alert.rule.name),
+    Esql.event.category.values = VALUES(event.category)
+  by Esql.source.ip = TO_IP(source.ip)
+
+// correlation condition
+| where
+  Esql.source.ip.network_alert.case.count_distinct > 0
+  and Esql.event.dataset.count_distinct >= 2
+  and (Esql.source.ip.mail_access.case.count_distinct > 0 or Esql.source.ip.azure_signin.case.count_distinct > 0)
+  and Esql.event.count <= 100
 '''
 
 

rules/integrations/aws/discovery_ec2_multi_region_describe_instances.toml

@@ -2,7 +2,7 @@
 creation_date = "2024/08/26"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2025/01/10"
+updated_date = "2025/07/16"
 
 [rule]
 author = ["Elastic"]
@@ -86,25 +86,30 @@ timestamp_override = "event.ingested"
 type = "esql"
 
 query = '''
-from logs-aws.cloudtrail-*
+FROM logs-aws.cloudtrail-*
 
 // filter for DescribeInstances API calls
-| where event.dataset == "aws.cloudtrail" and event.provider == "ec2.amazonaws.com" and event.action == "DescribeInstances"
+| where event.dataset == "aws.cloudtrail"
+  and event.provider == "ec2.amazonaws.com"
+  and event.action == "DescribeInstances"
 
 // truncate the timestamp to a 30-second window
-| eval target_time_window = DATE_TRUNC(30 seconds, @timestamp)
+| eval Esql.time_window.date_trunc = DATE_TRUNC(30 seconds, @timestamp)
 
-// keep only the relevant fields
-| keep target_time_window, aws.cloudtrail.user_identity.arn, cloud.region
+// keep only the relevant raw fields
+| keep Esql.time_window.date_trunc, aws.cloudtrail.user_identity.arn, cloud.region
 
 // count the number of unique regions and total API calls within the 30-second window
-| stats region_count = count_distinct(cloud.region), window_count = count(*) by target_time_window, aws.cloudtrail.user_identity.arn
+| stats
+    Esql.cloud.region.count_distinct = COUNT_DISTINCT(cloud.region),
+    Esql.event.count = COUNT(*)
+  by Esql.time_window.date_trunc, aws.cloudtrail.user_identity.arn
 
 // filter for resources making DescribeInstances API calls in more than 10 regions within the 30-second window
-| where region_count >= 10 and window_count >= 10
+| where Esql.cloud.region.count_distinct >= 10 and Esql.event.count >= 10
 
-// sort the results by time windows in descending order
-| sort target_time_window desc
+// sort the results by time window in descending order
+| sort Esql.time_window.date_trunc desc
 '''
 
 [rule.investigation_fields]

rules/integrations/aws/discovery_ec2_multiple_discovery_api_calls_via_cli.toml

@@ -2,7 +2,7 @@
 creation_date = "2024/11/04"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2025/03/20"
+updated_date = "2025/07/16"
 
 [rule]
 author = ["Elastic"]
@@ -80,14 +80,14 @@ timestamp_override = "event.ingested"
 type = "esql"
 
 query = '''
-from logs-aws.cloudtrail*
+FROM logs-aws.cloudtrail*
 
 // create time window buckets of 10 seconds
-| eval time_window = date_trunc(10 seconds, @timestamp)
+| eval Esql.time_window.date_trunc = DATE_TRUNC(10 seconds, @timestamp)
 | where
     event.dataset == "aws.cloudtrail"
 
-    // filter on CloudTrail audit logs for IAM, EC2, and S3 events only
+    // filter on CloudTrail audit logs for IAM, EC2, S3, etc.
     and event.provider in (
       "iam.amazonaws.com",
       "ec2.amazonaws.com",
@@ -97,8 +97,7 @@ from logs-aws.cloudtrail*
       "dynamodb.amazonaws.com",
       "kms.amazonaws.com",
       "cloudfront.amazonaws.com",
-      "elasticloadbalancing.amazonaws.com",
-      "cloudfront.amazonaws.com"
+      "elasticloadbalancing.amazonaws.com"
     )
 
     // ignore AWS service actions
@@ -112,24 +111,29 @@ from logs-aws.cloudtrail*
 
 // filter for Describe, Get, List, and Generate API calls
 | where true in (
-    starts_with(event.action, "Describe"),
-    starts_with(event.action, "Get"),
-    starts_with(event.action, "List"),
-    starts_with(event.action, "Generate")
+    STARTS_WITH(event.action, "Describe"),
+    STARTS_WITH(event.action, "Get"),
+    STARTS_WITH(event.action, "List"),
+    STARTS_WITH(event.action, "Generate")
 )
+
 // extract owner, identity type, and actor from the ARN
-| dissect aws.cloudtrail.user_identity.arn "%{}::%{owner}:%{identity_type}/%{actor}"
-| where starts_with(actor, "AWSServiceRoleForConfig") != true
-| keep @timestamp, time_window, event.action, aws.cloudtrail.user_identity.arn
+| dissect aws.cloudtrail.user_identity.arn "%{}::%{Esql.aws.cloudtrail.user_identity.arn.owner}:%{Esql.aws.cloudtrail.user_identity.arn.type}/%{Esql.aws.cloudtrail.user_identity.arn.roles}"
+| where STARTS_WITH(Esql.aws.cloudtrail.user_identity.arn.roles, "AWSServiceRoleForConfig") != true
+
+// keep relevant fields (preserving ECS fields and computed time window)
+| keep @timestamp, Esql.time_window.date_trunc, event.action, aws.cloudtrail.user_identity.arn
+
+// count the number of unique API calls per time window and actor
 | stats
-    // count the number of unique API calls per time window and actor
-    unique_api_count = count_distinct(event.action) by time_window, aws.cloudtrail.user_identity.arn
+    Esql.event.action.count_distinct = COUNT_DISTINCT(event.action)
+  by Esql.time_window.date_trunc, aws.cloudtrail.user_identity.arn
 
-// filter for more than 5 unique API calls per time window
-| where unique_api_count > 5
+// filter for more than 5 unique API calls per 10s window
+| where Esql.event.action.count_distinct > 5
 
 // sort the results by the number of unique API calls in descending order
-| sort unique_api_count desc
+| sort Esql.event.action.count_distinct desc
 '''
 
 

rules/integrations/aws/discovery_servicequotas_multi_region_service_quota_requests.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2024/08/26"
 maturity = "production"
-updated_date = "2025/01/15"
+updated_date = "2025/07/16"
 
 [rule]
 author = ["Elastic"]
@@ -35,31 +35,44 @@ timestamp_override = "event.ingested"
 type = "esql"
 
 query = '''
-from logs-aws.cloudtrail-*
+FROM logs-aws.cloudtrail-*
 
 // filter for GetServiceQuota API calls
-| where event.dataset == "aws.cloudtrail" and event.provider == "servicequotas.amazonaws.com" and event.action == "GetServiceQuota"
+| where
+  event.dataset == "aws.cloudtrail"
+  and event.provider == "servicequotas.amazonaws.com"
+  and event.action == "GetServiceQuota"
 
 // truncate the timestamp to a 30-second window
-| eval target_time_window = DATE_TRUNC(30 seconds, @timestamp)
+| eval Esql.time_window.date_trunc = DATE_TRUNC(30 seconds, @timestamp)
 
-// pre-process the request parameters to extract the service code and quota code
-| dissect aws.cloudtrail.request_parameters "{%{?service_code_key}=%{service_code}, %{?quota_code_key}=%{quota_code}}"
+// dissect request parameters to extract service and quota code
+| dissect aws.cloudtrail.request_parameters "{%{?Esql.aws.cloudtrail.request_parameters.service_code.key}=%{Esql.aws.cloudtrail.request_parameters.service_code}, %{?quota_code_key}=%{Esql.aws.cloudtrail.request_parameters.quota_code}}"
 
 // filter for EC2 service quota L-1216C47A (vCPU on-demand instances)
-| where service_code == "ec2" and quota_code == "L-1216C47A"
+| where Esql.aws.cloudtrail.request_parameters.service_code == "ec2" and Esql.aws.cloudtrail.request_parameters.quota_code == "L-1216C47A"
 
 // keep only the relevant fields
-| keep target_time_window, aws.cloudtrail.user_identity.arn, cloud.region, service_code, quota_code
-
-// count the number of unique regions and total API calls within the 30-second window
-| stats region_count = count_distinct(cloud.region), window_count = count(*) by target_time_window, aws.cloudtrail.user_identity.arn
-
-// filter for resources making DescribeInstances API calls in more than 10 regions within the 30-second window
-| where region_count >= 10 and window_count >= 10
-
-// sort the results by time windows in descending order
-| sort target_time_window desc
+| keep
+    Esql.time_window.date_trunc,
+    aws.cloudtrail.user_identity.arn,
+    cloud.region,
+    Esql.aws.cloudtrail.request_parameters.service_code,
+    Esql.aws.cloudtrail.request_parameters.quota_code
+
+// count the number of unique regions and total API calls within the time window
+| stats
+    Esql.cloud.region.count_distinct = COUNT_DISTINCT(cloud.region),
+    Esql.event.count = COUNT(*)
+  by Esql.time_window.date_trunc, aws.cloudtrail.user_identity.arn
+
+// filter for API calls in more than 10 regions within the 30-second window
+| where
+  Esql.cloud.region.count_distinct >= 10
+  and Esql.event.count >= 10
+
+// sort by time window descending
+| sort Esql.time_window.date_trunc desc
 '''
 note = """## Triage and analysis
 

rules/integrations/aws/exfiltration_ec2_ebs_snapshot_shared_with_another_account.toml

@@ -2,7 +2,7 @@
 creation_date = "2024/04/16"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2025/06/02"
+updated_date = "2025/07/16"
 
 [rule]
 author = ["Elastic"]
@@ -80,11 +80,32 @@ timestamp_override = "event.ingested"
 type = "esql"
 
 query = '''
-from logs-aws.cloudtrail-* metadata _id, _version, _index
-| where event.provider == "ec2.amazonaws.com" and event.action == "ModifySnapshotAttribute" and event.outcome == "success"
-| dissect aws.cloudtrail.request_parameters "{%{?snapshotId}=%{snapshotId},%{?attributeType}=%{attributeType},%{?createVolumePermission}={%{operationType}={%{?items}=[{%{?userId}=%{userId}}]}}}"
-| where operationType == "add" and cloud.account.id != userId
-| keep @timestamp, aws.cloudtrail.user_identity.arn, cloud.account.id, event.action, snapshotId, attributeType, operationType, userId, source.address
+FROM logs-aws.cloudtrail-* METADATA _id, _version, _index
+| where
+  event.provider == "ec2.amazonaws.com"
+  and event.action == "ModifySnapshotAttribute"
+  and event.outcome == "success"
+
+// Extract snapshotId, attribute type, operation type, and userId
+| dissect aws.cloudtrail.request_parameters
+  "{%{?snapshotId}=%{Esql.aws.cloudtrail.request_parameters.snapshot.id},%{?attributeType}=%{Esql.aws.cloudtrail.request_parameters.attribute.type},%{?createVolumePermission}={%{Esql.aws.cloudtrail.request_parameters.operation.type}={%{?items}=[{%{?userId}=%{Esql.aws.cloudtrail.request_parameters.user.id}}]}}}"
+
+// Check for snapshot permission added for another AWS account
+| where
+  Esql.aws.cloudtrail.request_parameters.operation.type == "add"
+  and cloud.account.id != Esql.aws.cloudtrail.request_parameters.user.id
+
+// Keep ECS and derived fields
+| keep
+  @timestamp,
+  aws.cloudtrail.user_identity.arn,
+  cloud.account.id,
+  event.action,
+  Esql.aws.cloudtrail.request_parameters.snapshot.id,
+  Esql.aws.cloudtrail.request_parameters.attribute.type,
+  Esql.aws.cloudtrail.request_parameters.operation.type,
+  Esql.aws.cloudtrail.request_parameters.user.id,
+  source.ip
 '''
 
 

rules/integrations/aws/impact_aws_s3_bucket_enumeration_or_brute_force.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2024/05/01"
 maturity = "production"
-updated_date = "2025/03/20"
+updated_date = "2025/07/16"
 
 [rule]
 author = ["Elastic"]
@@ -85,14 +85,30 @@ timestamp_override = "event.ingested"
 type = "esql"
 
 query = '''
-from logs-aws.cloudtrail*
-| where event.provider == "s3.amazonaws.com" and aws.cloudtrail.error_code == "AccessDenied"
-// keep only relevant fields
-| keep tls.client.server_name, source.address, cloud.account.id
-| stats failed_requests = count(*) by tls.client.server_name, source.address, cloud.account.id
-  // can modify the failed request count or tweak time window to fit environment
-  // can add `not cloud.account.id in (KNOWN)` or specify in exceptions
-| where failed_requests > 40
+FROM logs-aws.cloudtrail*
+
+| where
+  event.provider == "s3.amazonaws.com"
+  and aws.cloudtrail.error_code == "AccessDenied"
+  and tls.client.server_name IS NOT NULL
+  and cloud.account.id IS NOT NULL
+
+// Keep only relevant ECS fields
+| keep
+  tls.client.server_name,
+  source.address,
+  cloud.account.id
+
+// Count access denied requests per server_name, source, and account
+| stats
+    Esql.event.count = COUNT(*)
+  by
+    tls.client.server_name,
+    source.address,
+    cloud.account.id
+
+// Threshold: more than 40 denied requests
+| where Esql.event.count > 40
 '''