rebase - allowing any order and structure

Author: frederikb96

detection_rules/rule.py

@@ -935,8 +935,15 @@ def validates_esql_data(self, data: dict[str, Any], **_: Any) -> None:
         query_lower = data["query"].lower()
 
         # Combine both patterns using an OR operator and compile the regex
+        # The first part matches the metadata fields in the from clause by allowing one or 
+        # multiple indices and any order of the metadata fields
+        # The second part matches the stats command with the by clause
         combined_pattern = re.compile(
-            r"(from\s+\S+\s+metadata\s+_id,\s*_version,\s*_index)|(\bstats\b.*?\bby\b)", re.DOTALL
+            r"(from\s+(?:\S+\s*,\s*)*\S+\s+metadata\s+"
+            r"(_id,\s*_version,\s*_index|_id,\s*_index,\s*_version|_version,\s*_id,\s*_index|"
+            r"_version,\s*_index,\s*_id|_index,\s*_id,\s*_version|_index,\s*_version,\s*_id))"
+            r"|(\bstats\b.*?\bby\b)",
+            re.DOTALL
         )
 
         # Ensure that non-aggregate queries have metadata
@@ -948,9 +955,12 @@ def validates_esql_data(self, data: dict[str, Any], **_: Any) -> None:
             )
 
         # Enforce KEEP command for ESQL rules
-        if "| keep" not in query_lower:
+        # Match | followed by optional whitespace/newlines and then 'keep'
+        keep_pattern = re.compile(r"\|\s*keep\b", re.IGNORECASE | re.DOTALL)
+        if not keep_pattern.search(query_lower):
             raise ValidationError(
-                f"Rule: {data['name']} does not contain a 'keep' command -> Add a 'keep' command to the query."
+                f"Rule: {data['name']} does not contain a 'keep' command ->"
+                f" Add a 'keep' command to the query."
             )