feat: generate another .conf-file when conf parameter is used

splunk_add_on_ucc_framework/generators/conf_files/create_inputs_conf.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+from collections import defaultdict
 from typing import Any, Dict, List
 
 from splunk_add_on_ucc_framework.generators.file_generator import FileGenerator
@@ -24,47 +25,79 @@ class InputsConf(FileGenerator):
         "file for the services mentioned in globalConfig"
     )
 
+    def _conf_file_name(self, conf_name: str) -> str:
+        return f"{conf_name}.conf"
+
+    def _spec_file_name(self, conf_name: str) -> str:
+        return f"{self._conf_file_name(conf_name)}.spec"
+
     def _set_attributes(self, **kwargs: Any) -> None:
-        self.conf_file = "inputs.conf"
-        self.conf_spec_file = f"{self.conf_file}.spec"
-        self.input_names: List[Dict[str, List[str]]] = []
-        self.disable = False
-        self.service_name = ""
-        self.default_value_info: Dict[str, Dict[str, str]] = {}
-        if self._global_config.has_inputs():
-            for service in self._global_config.inputs:
-                properties = []
-                self.default_value_info[service["name"]] = {}
-                if service.get("disableNewInput"):
-                    self.disable = True
-                    self.service_name = service["name"]
-                if service.get("conf") is not None:
-                    # Add data input of self defined conf to inputs.conf.spec
-                    self.input_names.append(
-                        {service["name"]: ["placeholder = placeholder"]}
-                    )
+        self.conf_file = self._conf_file_name("inputs")
+
+        # A list of service names from globalConfig that will be in inputs.conf
+        self.inputs_conf_names: List[str] = []
+        # A dictionary of dictionaries of default properties for each service in inputs.conf
+        self.inputs_conf_params: Dict[str, Dict[str, Any]] = defaultdict(
+            lambda: defaultdict(dict)
+        )
+
+        # A dictionary of lists of properties for each service in inputs.conf
+        self.inputs_conf_spec: Dict[str, List[str]] = defaultdict(list)
+        # A dictionary of lists of properties for each service in their own spec files
+        # (i.e. dict key is the spec file name)
+        self.other_spec_files: Dict[str, List[str]] = defaultdict(list)
+
+        if not self._global_config.has_inputs():
+            return
+
+        for service in self._global_config.inputs:
+            default_values = None
+
+            # If the service has a conf property, it will be in a separate spec file
+            # Otherwise, it will be in inputs.conf
+            if service.get("conf") is not None:
+                spec_properties = self.other_spec_files[service["conf"]]
+            else:
+                spec_properties = self.inputs_conf_spec[service["name"]]
+                default_values = self.inputs_conf_params[service["name"]]
+
+            # Always add the service name to the list of service names in inputs.conf
+            # to add at least the Python version
+            self.inputs_conf_names.append(service["name"])
+
+            if default_values is not None and service.get("disableNewInput"):
+                default_values["disabled"] = "true"
+
+            for entity in service.get("entity", []):
+                field_name = entity["field"]
+
+                # Skip the name field as it is already in the service name
+                if field_name == "name":
                     continue
-                for entity in service.get("entity", {"field": "name"}):
-                    # TODO: add the details and updates on what to skip and process
-                    if entity["field"] == "name":
-                        continue
-                    nl = "\n"  # hack for `f-string expression part cannot include a backslash`
-                    # TODO: enhance the message formation for inputs.conf.spec file
-                    properties.append(
-                        f"{entity['field']} = {entity.get('help', '').replace(nl, ' ')} "
-                        f"{'' if entity.get('defaultValue') is None else ' Default: ' + str(entity['defaultValue'])}"
-                    )
-                    if entity.get("defaultValue"):
-                        if type(entity["defaultValue"]) is bool:
-                            self.default_value_info[service["name"]].update(
-                                {entity["field"]: str(entity["defaultValue"]).lower()}
-                            )
-                        else:
-                            self.default_value_info[service["name"]].update(
-                                {f"{entity['field']}": f"{str(entity['defaultValue'])}"}
-                            )
-
-                self.input_names.append({service["name"]: properties})
+
+                # Construct the property spec description
+                field_value_parts = []
+
+                if entity.get("help"):
+                    field_value_parts.append(entity["help"].replace("\n", " "))
+
+                field_default_value = entity.get("defaultValue")
+
+                if field_default_value:
+                    # Convert boolean values to lowercase strings
+                    if type(field_default_value) is bool:
+                        value = str(field_default_value).lower()
+                    else:
+                        value = str(field_default_value)
+
+                    # Add the default value to the description and dictionary
+                    if default_values is not None:
+                        default_values[field_name] = value
+                    field_value_parts.append(f"(Default: {value})")
+
+                field_value = " ".join(field_value_parts)
+                prop = f"{field_name} = {field_value}".rstrip()
+                spec_properties.append(prop)
 
     def generate(self) -> Dict[str, str]:
         conf_files: Dict[str, str] = {}
@@ -73,22 +106,17 @@ def generate(self) -> Dict[str, str]:
         return conf_files
 
     def generate_conf(self) -> Dict[str, str]:
-        if not self.input_names:
-            return {"": ""}
+        if not self.inputs_conf_names:
+            return {}
 
         file_path = self.get_file_output_path(["default", self.conf_file])
-        stanzas: List[str] = []
-        for k in self.input_names:
-            stanzas.extend(k.keys())
         self.set_template_and_render(
             template_file_path=["conf_files"], file_name="inputs_conf.template"
         )
 
         rendered_content = self._template.render(
-            input_names=stanzas,
-            disabled=self.disable,
-            service_name=self.service_name,
-            default_values=self.default_value_info,
+            input_names=self.inputs_conf_names,
+            default_values=self.inputs_conf_params,
         )
         self.writer(
             file_name=self.conf_file,
@@ -97,21 +125,49 @@ def generate_conf(self) -> Dict[str, str]:
         )
         return {self.conf_file: file_path}
 
-    def generate_conf_spec(self) -> Dict[str, str]:
-        if not self.input_names:
-            return {"": ""}
+    def _generate_spec_inputs(self) -> Dict[str, str]:
+        if not self.inputs_conf_spec:
+            return {}
+
+        spec_file = self._spec_file_name("inputs")
+        file_path = self.get_file_output_path(["README", spec_file])
 
-        file_path = self.get_file_output_path(["README", self.conf_spec_file])
         self.set_template_and_render(
             template_file_path=["README"], file_name="inputs_conf_spec.template"
         )
 
         rendered_content = self._template.render(
-            input_stanzas=self.input_names,
+            input_names=self.inputs_conf_names,
+            input_stanzas=self.inputs_conf_spec,
         )
         self.writer(
-            file_name=self.conf_spec_file,
+            file_name=spec_file,
             file_path=file_path,
             content=rendered_content,
         )
-        return {self.conf_spec_file: file_path}
+        return {spec_file: file_path}
+
+    def _generate_spec_other(self, name: str, parameters: List[str]) -> Dict[str, str]:
+        spec_file = self._spec_file_name(name)
+        file_path = self.get_file_output_path(["README", spec_file])
+
+        content = ["[<name>]"]
+        content.extend(parameters)
+
+        self.writer(
+            file_name=spec_file,
+            file_path=file_path,
+            content="\n".join(content),
+        )
+        return {spec_file: file_path}
+
+    def generate_conf_spec(self) -> Dict[str, str]:
+        files = self._generate_spec_inputs()
+
+        for name, params in self.other_spec_files.items():
+            files.update(self._generate_spec_other(name, params))
+
+        if not files:
+            return {}
+
+        return files

splunk_add_on_ucc_framework/templates/README/inputs_conf_spec.template

@@ -1,6 +1,4 @@
-{% for stanza in input_stanzas %}
-{{ "[" ~ stanza.keys() | join("") ~ "://<name>]" }}
-    {% for property in stanza.values() -%}
-{{ property | sort() | join("\n") }}
-    {% endfor %}
+{% for name, properties in input_stanzas.items() %}
+{{ "[" ~ name | join("") ~ "://<name>]" }}
+{{ properties | sort() | join("\n") }}
 {% endfor %}

splunk_add_on_ucc_framework/templates/conf_files/inputs_conf.template

@@ -1,10 +1,7 @@
 {% for input_name in input_names %}
 {{ "[" ~ input_name ~ "]"}}
 python.version = python3
-{% if disabled and service_name == input_name%}
-disabled = true
-{% endif %}
-{% for item, value in default_values[input_name].items() %}
+{% for item, value in default_values.get(input_name, {}).items() %}
 {{item}} = {{value}}
 {% endfor %}
 {% endfor %}

tests/smoke/test_ucc_build.py

@@ -179,6 +179,7 @@ def test_ucc_generate_with_everything(caplog):
             ("bin", "test_alert.py"),
             ("README", "alert_actions.conf.spec"),
             ("README", "inputs.conf.spec"),
+            ("README", "some_conf.conf.spec"),
             ("README", "splunk_ta_uccexample_account.conf.spec"),
             ("README", "splunk_ta_uccexample_settings.conf.spec"),
             ("metadata", "default.meta"),

tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/README/inputs.conf.spec

@@ -4,34 +4,34 @@ example_help_link =
 example_textarea_field = Help message
 hard_disabled = 
 hide_in_ui = 
-index = Default: default
-input_one_checkbox = This is an example checkbox for the input one entity  Default: True
-input_one_radio = This is an example radio button for the input one entity  Default: yes
+index = (Default: default)
+input_one_checkbox = This is an example checkbox for the input one entity (Default: true)
+input_one_radio = This is an example radio button for the input one entity (Default: yes)
 interval = Time interval of the data input, in seconds.
-limit = The maximum number of results returned by the query.  Default: 1000
-multipleSelectTest = Default: a|b
+limit = The maximum number of results returned by the query. (Default: 1000)
+multipleSelectTest = (Default: a|b)
 object = The name of the object to query for.
 object_fields = Object fields from which to collect data. Delimit multiple fields using a comma.
-order_by = The datetime field by which to query results in ascending order for indexing.  Default: LastModifiedDate
+order_by = The datetime field by which to query results in ascending order for indexing. (Default: LastModifiedDate)
 singleSelectTest = 
 start_date = The datetime after which to query and index records, in this format: "YYYY-MM-DDThh:mm:ss.000z". Defaults to 90 days earlier from now.
-use_existing_checkpoint = Data input already exists. Select `No` if you want to reset the data collection.  Default: yes
+use_existing_checkpoint = Data input already exists. Select `No` if you want to reset the data collection. (Default: yes)
 
 [example_input_two://<name>]
 account = 
 apis = 
 example_help_link = 
 hard_disabled = 
 hide_in_ui = 
-index = Default: default
+index = (Default: default)
 input_two_checkbox = This is an example checkbox for the input two entity
 input_two_multiple_select = This is an example multipleSelect for input two entity
 input_two_radio = This is an example radio button for the input two entity
 input_two_text_hidden_for_cloud = Should be hidden for cloud
 input_two_text_hidden_for_enterprise = Should be hidden for enterprise
 interval = Time interval of the data input, in seconds.
 start_date = The date and time, in "YYYY-MM-DDThh:mm:ss.000z" format, after which to query and index records.  The default is 90 days before today.
-use_existing_checkpoint = Data input already exists. Select `No` if you want to reset the data collection.  Default: yes
+use_existing_checkpoint = Data input already exists. Select `No` if you want to reset the data collection. (Default: yes)
 
 [example_input_three://<name>]
 event_filters = 
@@ -46,9 +46,6 @@ interval = Time interval of the data input, in seconds.
 [service_hidden_for_enterprise://<name>]
 interval = Time interval of the data input, in seconds.
 
-[service_with_conf_param://<name>]
-placeholder = placeholder
-
 [service_inside_menu_one://<name>]
 interval = Time interval of the data input, in seconds.
 text_with_validators = 

tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/README/some_conf.conf.spec

@@ -0,0 +1,2 @@
+[<name>]
+interval = Time interval of the data input, in seconds.

tests/testdata/test_addons/package_global_config_everything/globalConfig.json

@@ -2091,7 +2091,7 @@
     "meta": {
         "name": "Splunk_TA_UCCExample",
         "restRoot": "splunk_ta_uccexample",
-        "version": "5.60.0+34ad0cb7a",
+        "version": "5.61.0+8593b0555",
         "displayName": "Splunk UCC test Add-on",
         "schemaVersion": "0.0.9",
         "supportedThemes": [