Stable Config file: target system properties in process_arguments and support template variables in YamlParser (#8690)

mtoffl01 · mcculls · web-flow · commit 7169626143df · 2025-05-06T09:13:48.000Z
* migrate CLIHelper to use a Map, parse system properties in key-vals

* Fix processTemplateVar logic to remove superfluous ':'

* nits: javadocs

* Move writeFileRaw helper to testutils

* Add YamlParser tests

* Revert CLIHelper to use List&lt;String&gt;; lazily load a Map cache for querying key-vals

* reuse logic for adding to the vm args cache

* apply github suggestions

* Move processTemplate yaml helper fns into StableConfigParser, along with tests

* Remove changes to CLIHelper; rely on System.getProperty instead

* optimize: return from processTemplate early if no {{ found

* Add more test coverage to StableConfigParserTest

* Optimize template processing to reduce use of substrings

* Introduce constants for repeated strings

* Remove FileUtils and all its references

---------

Co-authored-by: Stuart McCulloch &lt;stuart.mcculloch@datadoghq.com&gt;
diff --git a/components/yaml/src/main/java/datadog/yaml/YamlParser.java b/components/yaml/src/main/java/datadog/yaml/YamlParser.java
@@ -1,19 +1,15 @@
 package datadog.yaml;
 
-import java.io.FileInputStream;
-import java.io.IOException;
 import org.yaml.snakeyaml.Yaml;
 
 public class YamlParser {
   // Supports clazz == null for default yaml parsing
-  public static <T> T parse(String filePath, Class<T> clazz) throws IOException {
+  public static <T> T parse(String content, Class<T> clazz) {
     Yaml yaml = new Yaml();
-    try (FileInputStream fis = new FileInputStream(filePath)) {
-      if (clazz == null) {
-        return yaml.load(fis);
-      } else {
-        return yaml.loadAs(fis, clazz);
-      }
+    if (clazz == null) {
+      return yaml.load(content);
+    } else {
+      return yaml.loadAs(content, clazz);
     }
   }
 }
diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/config/provider/StableConfigParser.java b/internal-api/src/main/java/datadog/trace/bootstrap/config/provider/StableConfigParser.java
@@ -1,25 +1,27 @@
 package datadog.trace.bootstrap.config.provider;
 
-import datadog.cli.CLIHelper;
 import datadog.trace.bootstrap.config.provider.stableconfigyaml.ConfigurationMap;
 import datadog.trace.bootstrap.config.provider.stableconfigyaml.Rule;
 import datadog.trace.bootstrap.config.provider.stableconfigyaml.Selector;
 import datadog.trace.bootstrap.config.provider.stableconfigyaml.StableConfigYaml;
 import datadog.yaml.YamlParser;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 import java.util.function.BiPredicate;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class StableConfigParser {
   private static final Logger log = LoggerFactory.getLogger(StableConfigParser.class);
 
-  private static final Set<String> VM_ARGS = new HashSet<>(CLIHelper.getVmArgs());
+  private static final String ENVIRONMENT_VARIABLES_PREFIX = "environment_variables['";
+  private static final String PROCESS_ARGUMENTS_PREFIX = "process_arguments['";
+  private static final String UNDEFINED_VALUE = "UNDEFINED";
 
   /**
    * Parses a configuration file and returns a stable configuration object.
@@ -37,7 +39,9 @@ public class StableConfigParser {
    */
   public static StableConfigSource.StableConfig parse(String filePath) throws IOException {
     try {
-      StableConfigYaml data = YamlParser.parse(filePath, StableConfigYaml.class);
+      String content = new String(Files.readAllBytes(Paths.get(filePath)), StandardCharsets.UTF_8);
+      String processedContent = processTemplate(content);
+      StableConfigYaml data = YamlParser.parse(processedContent, StableConfigYaml.class);
 
       String configId = data.getConfig_id();
       ConfigurationMap configMap = data.getApm_configuration_default();
@@ -66,7 +70,9 @@ public static StableConfigSource.StableConfig parse(String filePath) throws IOEx
 
     } catch (IOException e) {
       log.debug(
-          "Stable configuration file either not found or not readable at filepath {}", filePath);
+          "Stable configuration file either not found or not readable at filepath {}. Error: {}",
+          filePath,
+          e.getMessage());
     }
     return StableConfigSource.StableConfig.EMPTY;
   }
@@ -166,14 +172,100 @@ static boolean selectorMatch(String origin, List<String> matches, String operato
             return false;
         }
       case "process_arguments":
-        // For now, always return true if `key` exists in the JVM Args
         // TODO: flesh out the meaning of each operator for process_arguments
-        return VM_ARGS.contains(key);
+        if (!key.startsWith("-D")) {
+          log.warn(
+              "Ignoring unsupported process_arguments entry in selector match, '{}'. Only system properties specified with the '-D' prefix are supported.",
+              key);
+          return false;
+        }
+        // Cut the -D prefix
+        return System.getProperty(key.substring(2)) != null;
       case "tags":
         // TODO: Support this down the line (Must define the source of "tags" first)
         return false;
       default:
         return false;
     }
   }
+
+  static String processTemplate(String content) throws IOException {
+    // Do nothing if there are no variables to process
+    int openIndex = content.indexOf("{{");
+    if (openIndex == -1) {
+      return content;
+    }
+
+    StringBuilder result = new StringBuilder(content.length());
+
+    // Add everything before the opening braces
+    result.append(content, 0, openIndex);
+
+    while (true) {
+
+      // Find the closing braces
+      int closeIndex = content.indexOf("}}", openIndex);
+      if (closeIndex == -1) {
+        throw new IOException("Unterminated template in config");
+      }
+
+      // Extract the template variable
+      String templateVar = content.substring(openIndex + 2, closeIndex).trim();
+
+      // Process the template variable and get its value
+      String value = processTemplateVar(templateVar);
+
+      // Add the processed value
+      result.append(value);
+
+      // Continue with the next template variable
+      openIndex = content.indexOf("{{", closeIndex);
+      if (openIndex == -1) {
+        // Stop and add everything left after the final closing braces
+        result.append(content, closeIndex + 2, content.length());
+        break;
+      } else {
+        // Add everything between the last braces and the next
+        result.append(content, closeIndex + 2, openIndex);
+      }
+    }
+
+    return result.toString();
+  }
+
+  private static String processTemplateVar(String templateVar) throws IOException {
+    if (templateVar.startsWith(ENVIRONMENT_VARIABLES_PREFIX) && templateVar.endsWith("']")) {
+      String envVar =
+          templateVar
+              .substring(ENVIRONMENT_VARIABLES_PREFIX.length(), templateVar.length() - 2)
+              .trim();
+      if (envVar.isEmpty()) {
+        throw new IOException("Empty environment variable name in template");
+      }
+      String value = System.getenv(envVar.toUpperCase());
+      if (value == null || value.isEmpty()) {
+        return UNDEFINED_VALUE;
+      }
+      return value;
+    } else if (templateVar.startsWith(PROCESS_ARGUMENTS_PREFIX) && templateVar.endsWith("']")) {
+      String processArg =
+          templateVar.substring(PROCESS_ARGUMENTS_PREFIX.length(), templateVar.length() - 2).trim();
+      if (processArg.isEmpty()) {
+        throw new IOException("Empty process argument in template");
+      }
+      if (!processArg.startsWith("-D")) {
+        log.warn(
+            "Ignoring unsupported process_arguments entry in template variable, '{}'. Only system properties specified with the '-D' prefix are supported.",
+            processArg);
+        return UNDEFINED_VALUE;
+      }
+      String value = System.getProperty(processArg.substring(2));
+      if (value == null || value.isEmpty()) {
+        return UNDEFINED_VALUE;
+      }
+      return value;
+    } else {
+      return UNDEFINED_VALUE;
+    }
+  }
 }
diff --git a/internal-api/src/test/groovy/datadog/trace/bootstrap/config/provider/StableConfigParserTest.groovy b/internal-api/src/test/groovy/datadog/trace/bootstrap/config/provider/StableConfigParserTest.groovy
@@ -1,14 +1,12 @@
 package datadog.trace.bootstrap.config.provider
-
 import datadog.trace.test.util.DDSpecification
-
 import java.nio.file.Files
 import java.nio.file.Path
 
 class StableConfigParserTest extends DDSpecification {
   def "test parse valid"() {
     when:
-    Path filePath = StableConfigSourceTest.tempFile()
+    Path filePath = Files.createTempFile("testFile_", ".yaml")
     if (filePath == null) {
       throw new AssertionError("Failed to create test file")
     }
@@ -42,7 +40,7 @@ apm_configuration_rules:
       KEY_FOUR: "ignored"
 """
     try {
-      StableConfigSourceTest.writeFileRaw(filePath, yaml)
+      Files.write(filePath, yaml.getBytes())
     } catch (IOException e) {
       throw new AssertionError("Failed to write to file: ${e.message}")
     }
@@ -82,6 +80,8 @@ apm_configuration_rules:
     "language" | ["java", "golang"] | "equals" | "" | true
     "language" | ["java"] | "starts_with" | "" | true
     "language" | ["golang"] | "equals" | "" | false
+    "language" | ["java"] | "exists" | "" | false
+    "language" | ["java"] | "something unexpected" | "" | false
     "environment_variables" | [] | "exists" | "DD_TAGS" | true
     "environment_variables" | ["team:apm"] | "contains" | "DD_TAGS" | true
     "ENVIRONMENT_VARIABLES" | ["TeAm:ApM"] | "CoNtAiNs" | "Dd_TaGs" | true // check case insensitivity
@@ -96,13 +96,12 @@ apm_configuration_rules:
     "environment_variables" | ["svc"] | "contains" | "DD_SERVICE" | true
     "environment_variables" | ["other"] | "contains" | "DD_SERVICE" | false
     "environment_variables" | [null] | "contains" | "DD_SERVICE" | false
-    //    "process_arguments" | null | "equals" | "-DCustomKey" | true
   }
 
   def "test duplicate entries"() {
     // When duplicate keys are encountered, snakeyaml preserves the last value by default
     when:
-    Path filePath = StableConfigSourceTest.tempFile()
+    Path filePath = Files.createTempFile("testFile_", ".yaml")
     if (filePath == null) {
       throw new AssertionError("Failed to create test file")
     }
@@ -116,7 +115,7 @@ apm_configuration_rules:
   """
 
     try {
-      StableConfigSourceTest.writeFileRaw(filePath, yaml)
+      Files.write(filePath, yaml.getBytes())
     } catch (IOException e) {
       throw new AssertionError("Failed to write to file: ${e.message}")
     }
@@ -134,10 +133,38 @@ apm_configuration_rules:
     cfg.get("DD_KEY") == "value_2"
   }
 
+  def "test config_id only"() {
+    when:
+    Path filePath = Files.createTempFile("testFile_", ".yaml")
+    if (filePath == null) {
+      throw new AssertionError("Failed to create test file")
+    }
+    String yaml = """
+  config_id: 12345
+  """
+    try {
+      Files.write(filePath, yaml.getBytes())
+    } catch (IOException e) {
+      throw new AssertionError("Failed to write to file: ${e.message}")
+    }
+
+    StableConfigSource.StableConfig cfg
+    try {
+      cfg = StableConfigParser.parse(filePath.toString())
+    } catch (Exception e) {
+      throw new AssertionError("Failed to parse the file: ${e.message}")
+    }
+
+    then:
+    cfg != null
+    cfg.getConfigId() == "12345"
+    cfg.getKeys().size() == 0
+  }
+
   def "test parse invalid"() {
     // If any piece of the file is invalid, the whole file is rendered invalid and an exception is thrown
     when:
-    Path filePath = StableConfigSourceTest.tempFile()
+    Path filePath = Files.createTempFile("testFile_", ".yaml")
     if (filePath == null) {
       throw new AssertionError("Failed to create test file")
     }
@@ -159,7 +186,7 @@ apm_configuration_rules:
   something-else-irrelevant: value-irrelevant
   """
     try {
-      StableConfigSourceTest.writeFileRaw(filePath, yaml)
+      Files.write(filePath, yaml.getBytes())
     } catch (IOException e) {
       throw new AssertionError("Failed to write to file: ${e.message}")
     }
@@ -177,4 +204,39 @@ apm_configuration_rules:
     cfg == null
     Files.delete(filePath)
   }
+
+  def "test processTemplate valid cases"() {
+    when:
+    if (envKey != null) {
+      injectEnvConfig(envKey, envVal)
+    }
+
+    then:
+    StableConfigParser.processTemplate(templateVar) == expect
+
+    where:
+    templateVar | envKey | envVal | expect
+    "{{environment_variables['DD_KEY']}}" | "DD_KEY" | "value" | "value"
+    "{{environment_variables['DD_KEY']}}" | null | null | "UNDEFINED"
+    "{{}}" | null | null | "UNDEFINED"
+    "{}" | null | null | "{}"
+    "{{environment_variables['dd_key']}}" | "DD_KEY" | "value" | "value"
+    "{{environment_variables['DD_KEY}}" | "DD_KEY" | "value" | "UNDEFINED"
+    "header-{{environment_variables['DD_KEY']}}-footer" | "DD_KEY" | "value" | "header-value-footer"
+    "{{environment_variables['HEADER']}}{{environment_variables['DD_KEY']}}{{environment_variables['FOOTER']}}" | "DD_KEY" | "value" | "UNDEFINEDvalueUNDEFINED"
+  }
+
+  def "test processTemplate error cases"() {
+    when:
+    StableConfigParser.processTemplate(templateVar)
+
+    then:
+    def e = thrown(IOException)
+    e.message == expect
+
+    where:
+    templateVar | expect
+    "{{environment_variables['']}}" | "Empty environment variable name in template"
+    "{{environment_variables['DD_KEY']}" | "Unterminated template in config"
+  }
 }
diff --git a/internal-api/src/test/groovy/datadog/trace/bootstrap/config/provider/StableConfigSourceTest.groovy b/internal-api/src/test/groovy/datadog/trace/bootstrap/config/provider/StableConfigSourceTest.groovy
@@ -33,7 +33,7 @@ class StableConfigSourceTest extends DDSpecification {
 
   def "test empty file"() {
     when:
-    Path filePath = tempFile()
+    Path filePath = Files.createTempFile("testFile_", ".yaml")
     if (filePath == null) {
       throw new AssertionError("Failed to create test file")
     }
@@ -47,7 +47,7 @@ class StableConfigSourceTest extends DDSpecification {
   def "test file invalid format"() {
     // StableConfigSource must handle the exception thrown by StableConfigParser.parse(filePath) gracefully
     when:
-    Path filePath = tempFile()
+    Path filePath = Files.createTempFile("testFile_", ".yaml")
     if (filePath == null) {
       throw new AssertionError("Failed to create test file")
     }
@@ -73,7 +73,7 @@ class StableConfigSourceTest extends DDSpecification {
 
   def "test file valid format"() {
     when:
-    Path filePath = tempFile()
+    Path filePath = Files.createTempFile("testFile_", ".yaml")
     if (filePath == null) {
       throw new AssertionError("Failed to create test file")
     }
@@ -133,16 +133,6 @@ class StableConfigSourceTest extends DDSpecification {
   def static sampleNonMatchingRule = new Rule(Arrays.asList(new Selector("origin", "language", Arrays.asList("Golang"), null)), new ConfigurationMap(singletonMap("DD_KEY_FOUR", new ConfigurationValue("four"))))
 
   // Helper functions
-  static Path tempFile() {
-    try {
-      return Files.createTempFile("testFile_", ".yaml")
-    } catch (IOException e) {
-      println "Error creating file: ${e.message}"
-      e.printStackTrace()
-      return null // or throw new RuntimeException("File creation failed", e)
-    }
-  }
-
   def stableConfigYamlWriter = getStableConfigYamlWriter()
 
   Yaml getStableConfigYamlWriter() {
@@ -166,7 +156,6 @@ class StableConfigSourceTest extends DDSpecification {
   }
 
   def writeFileYaml(Path filePath, StableConfigYaml stableConfigs) {
-    //    Yaml yaml = getStableConfigYaml();
     try (FileWriter writer = new FileWriter(filePath.toString())) {
       stableConfigYamlWriter.dump(stableConfigs, writer)
     } catch (IOException e) {
@@ -180,10 +169,4 @@ class StableConfigSourceTest extends DDSpecification {
     StandardOpenOption[] openOpts = [StandardOpenOption.WRITE] as StandardOpenOption[]
     Files.write(filePath, data.getBytes(), openOpts)
   }
-
-  // Use this for writing a string directly into a file
-  static writeFileRaw(Path filePath, String data) {
-    StandardOpenOption[] openOpts = [StandardOpenOption.WRITE] as StandardOpenOption[]
-    Files.write(filePath, data.getBytes(), openOpts)
-  }
 }
