[backend] Define a logic for saving structured outputs

openbas-api/src/main/java/io/openbas/migration/V3_90__Add_StructuredOutput_ExecutionTraces.java

@@ -0,0 +1,18 @@
+package io.openbas.migration;
+
+import java.sql.Connection;
+import java.sql.Statement;
+import org.flywaydb.core.api.migration.BaseJavaMigration;
+import org.flywaydb.core.api.migration.Context;
+import org.springframework.stereotype.Component;
+
+@Component
+public class V3_90__Add_StructuredOutput_ExecutionTraces extends BaseJavaMigration {
+
+  @Override
+  public void migrate(Context context) throws Exception {
+    Connection connection = context.getConnection();
+    Statement select = connection.createStatement();
+    select.execute("ALTER TABLE execution_traces ADD COLUMN execution_structured_output TEXT;");
+  }
+}

openbas-api/src/main/java/io/openbas/rest/finding/FindingService.java

@@ -2,9 +2,7 @@
 
 import static io.openbas.helper.StreamHelper.fromIterable;
 import static io.openbas.injector_contract.outputs.ContractOutputUtils.getContractOutputs;
-import static io.openbas.utils.InjectExecutionUtils.convertExecutionAction;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -15,12 +13,14 @@
 import io.openbas.database.repository.UserRepository;
 import io.openbas.injector_contract.outputs.ContractOutputElement;
 import io.openbas.injector_contract.outputs.ContractOutputUtils;
-import io.openbas.rest.inject.form.InjectExecutionInput;
 import io.openbas.rest.inject.service.InjectService;
 import jakarta.annotation.Resource;
 import jakarta.persistence.EntityNotFoundException;
 import jakarta.validation.constraints.NotBlank;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.java.Log;
 import org.jetbrains.annotations.NotNull;
@@ -82,48 +82,26 @@
     this.findingRepository.deleteById(id);
   }
 
-  // -- EXTRACTION FINDINGS --
+  // -- Extract findings from strctured output : Here we compute the findings from structured output
+  // from ExecutionInjectInput sent by injectors
+  // This structrued output is generated based on injectorcontract where we can find the node
+  // Outputs and with that the injector generate this structure output--
 
-  public void computeFindings(InjectExecutionInput input, Inject inject, Agent agent) {
-    // Used for inject with payload
-    if (agent != null) {
-      extractFindingsFromRawOutput(input, inject, agent);
-    }
-    // Used for injectors
-    extractFindingsFromStructuredOutput(input, inject);
-  }
-
-  // -- STRUCTURED OUTPUT --
-
-  public void extractFindingsFromStructuredOutput(InjectExecutionInput input, Inject inject) {
+  public void extractFindingsFromInjectorContract(Inject inject, ObjectNode structuredOutput) {
     // NOTE: do it in every call to callback ? (reflexion on implant mechanism)
-    if (input.getOutputStructured() != null) {
-      try {
-        List<Finding> findings = new ArrayList<>();
-        // Get the contract
-        InjectorContract injectorContract = inject.getInjectorContract().orElseThrow();
-        List<ContractOutputElement> contractOutputs =
-            getContractOutputs(injectorContract.getConvertedContent(), mapper);
-        ObjectNode values = mapper.readValue(input.getOutputStructured(), ObjectNode.class);
-        if (!contractOutputs.isEmpty()) {
-          contractOutputs.forEach(
-              contractOutput -> {
-                if (contractOutput.isFindingCompatible()) {
-                  if (contractOutput.isMultiple()) {
-                    JsonNode jsonNodes = values.get(contractOutput.getField());
-                    if (jsonNodes != null && jsonNodes.isArray()) {
-                      for (JsonNode jsonNode : jsonNodes) {
-                        if (!contractOutput.getType().validate.apply(jsonNode)) {
-                          throw new IllegalArgumentException("Finding not correctly formatted");
-                        }
-                        Finding finding = ContractOutputUtils.createFinding(contractOutput);
-                        finding.setValue(contractOutput.getType().toFindingValue.apply(jsonNode));
-                        Finding linkedFinding = linkFindings(contractOutput, jsonNode, finding);
-                        findings.add(linkedFinding);
-                      }
-                    }
-                  } else {
-                    JsonNode jsonNode = values.get(contractOutput.getField());
+    List<Finding> findings = new ArrayList<>();
+    // Get the contract
+    InjectorContract injectorContract = inject.getInjectorContract().orElseThrow();
+    List<ContractOutputElement> contractOutputs =
+        getContractOutputs(injectorContract.getConvertedContent(), mapper);
+    if (!contractOutputs.isEmpty()) {
+      contractOutputs.forEach(
+          contractOutput -> {
+            if (contractOutput.isFindingCompatible()) {
+              if (contractOutput.isMultiple()) {
+                JsonNode jsonNodes = structuredOutput.get(contractOutput.getField());
+                if (jsonNodes != null && jsonNodes.isArray()) {
+                  for (JsonNode jsonNode : jsonNodes) {
                     if (!contractOutput.getType().validate.apply(jsonNode)) {
                       throw new IllegalArgumentException("Finding not correctly formatted");
                     }
@@ -133,13 +111,20 @@
                     findings.add(linkedFinding);
                   }
                 }
-              });
-        }
-        this.createFindings(findings, inject.getId());
-      } catch (JsonProcessingException e) {
-        throw new RuntimeException(e);
-      }
+              } else {
+                JsonNode jsonNode = structuredOutput.get(contractOutput.getField());
+                if (!contractOutput.getType().validate.apply(jsonNode)) {
+                  throw new IllegalArgumentException("Finding not correctly formatted");
+                }
+                Finding finding = ContractOutputUtils.createFinding(contractOutput);
+                finding.setValue(contractOutput.getType().toFindingValue.apply(jsonNode));
+                Finding linkedFinding = linkFindings(contractOutput, jsonNode, finding);
+                findings.add(linkedFinding);
+              }
+            }
+          });
     }
+    this.createFindings(findings, inject.getId());
   }
 
   private Finding linkFindings(
@@ -172,47 +157,34 @@
     return finding;
   }
 
-  // -- RAW OUTPUT --
-
-  public void extractFindingsFromRawOutput(InjectExecutionInput input, Inject inject, Agent agent) {
-    if (ExecutionTraceAction.EXECUTION.equals(convertExecutionAction(input.getAction()))) {
-      inject
-          .getPayload()
-          .ifPresent(
-              payload -> {
-                if (payload.getOutputParsers() != null && !payload.getOutputParsers().isEmpty()) {
-                  extractFindings(inject, agent.getAsset(), input.getMessage());
-                } else {
-                  log.info(
-                      "No output parsers available for payload used in inject:" + inject.getId());
-                }
-              });
-    }
-  }
-
-  private void extractFindings(Inject inject, Asset asset, String trace) {
-    inject
-        .getPayload()
-        .map(Payload::getOutputParsers)
-        .ifPresent(
-            outputParsers ->
-                outputParsers.forEach(
-                    outputParser -> {
-                      String rawOutputByMode =
-                          findingUtils.extractRawOutputByMode(trace, outputParser.getMode());
-                      if (rawOutputByMode == null) {
-                        return;
-                      }
-                      switch (outputParser.getType()) {
-                        case REGEX:
-                        default:
-                          findingUtils.computeFindingUsingRegexRules(
+  /** Extracts findings from structured output that was generated using output parsers. */
+  public void extractFindingsFromOutputParsers(
+      Inject inject, Agent agent, Set<OutputParser> outputParsers, JsonNode structuredOutput) {
+
+    outputParsers.forEach(
+        outputParser -> {
+          outputParser
+              .getContractOutputElements()
+              .forEach(
+                  contractOutputElement -> {
+                    if (contractOutputElement.isFinding()) {
+                      JsonNode jsonNodes = structuredOutput.get(contractOutputElement.getKey());
+                      if (jsonNodes != null && jsonNodes.isArray()) {
+                        for (JsonNode jsonNode : jsonNodes) {
+                          // Validate finding format
+                          if (!contractOutputElement.getType().validate.apply(jsonNode)) {
+                            throw new IllegalArgumentException("Finding not correctly formatted");
+                          }
+                          // Build and save the finding
+                          findingUtils.buildFinding(
                               inject,
-                              asset,
-                              rawOutputByMode,
-                              outputParser.getContractOutputElements());
-                          break;
+                              agent.getAsset(),
+                              contractOutputElement,
+                              contractOutputElement.getType().toFindingValue.apply(jsonNode));
+                        }
                       }
-                    }));
+                    }
+                  });
+        });
   }
 }

openbas-api/src/main/java/io/openbas/rest/finding/FindingUtils.java

@@ -2,19 +2,10 @@
 
 import static io.openbas.database.model.ContractOutputType.*;
 
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
 import io.openbas.database.model.*;
 import io.openbas.database.repository.FindingRepository;
-import jakarta.annotation.Resource;
 import java.util.*;
 import java.util.logging.Level;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.regex.PatternSyntaxException;
-import java.util.stream.Collectors;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.java.Log;
 import org.springframework.dao.DataIntegrityViolationException;
@@ -25,133 +16,8 @@
 @Component
 public class FindingUtils {
 
-  @Resource private final ObjectMapper mapper;
-
   private final FindingRepository findingRepository;
 
-  public void computeFindingUsingRegexRules(
-      Inject inject,
-      Asset asset,
-      String rawOutputByMode,
-      Set<io.openbas.database.model.ContractOutputElement> contractOutputElements) {
-    Map<String, Pattern> patternCache = new HashMap<>();
-
-    contractOutputElements.stream()
-        .filter(io.openbas.database.model.ContractOutputElement::isFinding)
-        .forEach(
-            contractOutputElement -> {
-              String regex = contractOutputElement.getRule();
-
-              // Check regex
-              Pattern pattern =
-                  patternCache.computeIfAbsent(
-                      regex,
-                      r -> {
-                        try {
-                          return Pattern.compile(
-                              r,
-                              Pattern.MULTILINE
-                                  | Pattern.CASE_INSENSITIVE
-                                  | Pattern.UNICODE_CHARACTER_CLASS);
-                        } catch (PatternSyntaxException e) {
-                          log.log(Level.INFO, "Invalid regex pattern: " + r, e.getMessage());
-                          return null;
-                        }
-                      });
-
-              if (pattern == null) return;
-
-              Matcher matcher = pattern.matcher(rawOutputByMode);
-
-              while (matcher.find()) {
-                String finalValue = buildValue(contractOutputElement, matcher);
-                if (isValid(finalValue)) {
-                  buildFinding(inject, asset, contractOutputElement, finalValue);
-                }
-              }
-            });
-  }
-
-  private static boolean isValid(String finalValue) {
-    return finalValue != null && !finalValue.isEmpty();
-  }
-
-  public String extractRawOutputByMode(String rawOutput, ParserMode mode) {
-    if (rawOutput == null || rawOutput.isEmpty()) {
-      return "";
-    }
-
-    try {
-      ObjectMapper objectMapper = new ObjectMapper();
-      JsonNode rootNode = objectMapper.readTree(rawOutput);
-
-      if (mode == ParserMode.STDOUT && rootNode.has("stdout")) {
-        return rootNode.get("stdout").asText();
-      } else if (mode == ParserMode.STDERR && rootNode.has("stderr")) {
-        return rootNode.get("stderr").asText();
-      }
-    } catch (Exception e) {
-      log.log(Level.WARNING, e.getMessage(), e);
-    }
-
-    return "";
-  }
-
-  public String buildValue(ContractOutputElement contractOutputElement, Matcher matcher) {
-    JsonNode resultNode;
-    if (contractOutputElement.getType().fields == null) {
-      List<String> extractedValues = extractValues(contractOutputElement.getRegexGroups(), matcher);
-      resultNode = mapper.valueToTree(extractedValues);
-    } else {
-      ObjectNode objectNode = mapper.createObjectNode();
-      for (ContractOutputField field : contractOutputElement.getType().fields) {
-        List<String> extractedValues =
-            extractValues(
-                contractOutputElement.getRegexGroups().stream()
-                    .filter(regexGroup -> field.getKey().equals(regexGroup.getField()))
-                    .collect(Collectors.toSet()),
-                matcher);
-        ArrayNode arrayNode = mapper.valueToTree(extractedValues);
-        objectNode.set(field.getKey(), arrayNode);
-      }
-      resultNode = objectNode;
-    }
-
-    return contractOutputElement.getType().toFindingValue.apply(resultNode);
-  }
-
-  private List<String> extractValues(Set<RegexGroup> regexGroups, Matcher matcher) {
-    List<String> extractedValues = new ArrayList<>();
-
-    for (RegexGroup regexGroup : regexGroups) {
-      String[] indexes =
-          Arrays.stream(regexGroup.getIndexValues().split("\\$", -1))
-              .filter(index -> !index.isEmpty())
-              .toArray(String[]::new);
-
-      for (String index : indexes) {
-        try {
-          int groupIndex = Integer.parseInt(index);
-          if (groupIndex > matcher.groupCount()) {
-            log.log(Level.WARNING, "Skipping invalid group index: " + groupIndex);
-            continue;
-          }
-          String extracted = matcher.group(groupIndex);
-          if (extracted == null || extracted.isEmpty()) {
-            log.log(Level.WARNING, "Skipping invalid extracted value");
-            continue;
-          }
-          if (extracted != null) {
-            extractedValues.add(extracted.trim());
-          }
-        } catch (NumberFormatException | IllegalStateException e) {
-          log.log(Level.SEVERE, "Invalid regex group index: " + index, e);
-        }
-      }
-    }
-    return extractedValues;
-  }
-
   public void buildFinding(
       Inject inject,
       Asset asset,