feat: Update ConfluenceParameters model and enhance validation methods for comma separated lists

Author: a-klos

admin-api-lib/pyproject.toml

@@ -29,7 +29,10 @@ per-file-ignores = """
   ./src/admin_api_lib/impl/admin_api.py: B008,
   ./src/admin_api_lib/dependency_container.py: CCE002,CCE001,
   ./src/admin_api_lib/apis/admin_api_base.py: WOT001,
-  ./tests/*: S101,
+  ./tests/*: S101,S106,D100,D103,PT011
+  ./src/admin_api_lib/impl/settings/confluence_settings.py: C901,N805,
+  ./src/admin_api_lib/impl/utils/comma_separated_bool_list.py: R505,
+  ./src/admin_api_lib/impl/utils/comma_separated_str_list.py: R505,
 """
 
 [tool.black]

admin-api-lib/src/admin_api_lib/extractor_api_client/openapi_client/models/confluence_parameters.py

@@ -25,18 +25,40 @@
 from typing import Optional, Set
 from typing_extensions import Self
 
+
 class ConfluenceParameters(BaseModel):
     """ """  # noqa: E501
 
     url: StrictStr = Field(description="url of the confluence space.")
     token: StrictStr = Field(description="api key to access confluence.")
     space_key: StrictStr = Field(description="the space key of the confluence pages.")
-    include_attachments: Optional[StrictBool] = Field(default=False, description="whether to include file attachments (e.g., images, documents) in the parsed content. Default is `false`.")
-    keep_markdown_format: Optional[StrictBool] = Field(default=True, description="whether to preserve markdown formatting in the output. Default is `true`.")
-    keep_newlines: Optional[StrictBool] = Field(default=True, description="whether to retain newline characters in the output for better readability. Default is `true`.")
-    document_name: StrictStr = Field(description="The name that will be used to store the confluence db in the key value db and the vectordatabase (metadata.document).")
-    confluence_kwargs: Optional[List[KeyValuePair]] = Field(default=None, description="Additional kwargs like verify_ssl")
-    __properties: ClassVar[List[str]] = ["url", "token", "space_key", "include_attachments", "keep_markdown_format", "keep_newlines", "document_name", "confluence_kwargs"]
+    include_attachments: Optional[StrictBool] = Field(
+        default=False,
+        description="whether to include file attachments (e.g., images, documents) in the parsed content. Default is `false`.",
+    )
+    keep_markdown_format: Optional[StrictBool] = Field(
+        default=True, description="whether to preserve markdown formatting in the output. Default is `true`."
+    )
+    keep_newlines: Optional[StrictBool] = Field(
+        default=True,
+        description="whether to retain newline characters in the output for better readability. Default is `true`.",
+    )
+    document_name: StrictStr = Field(
+        description="The name that will be used to store the confluence db in the key value db and the vectordatabase (metadata.document)."
+    )
+    confluence_kwargs: Optional[List[KeyValuePair]] = Field(
+        default=None, description="Additional kwargs like verify_ssl"
+    )
+    __properties: ClassVar[List[str]] = [
+        "url",
+        "token",
+        "space_key",
+        "include_attachments",
+        "keep_markdown_format",
+        "keep_newlines",
+        "document_name",
+        "confluence_kwargs",
+    ]
 
     model_config = ConfigDict(
         populate_by_name=True,
@@ -80,7 +102,7 @@ def to_dict(self) -> Dict[str, Any]:
             for _item_confluence_kwargs in self.confluence_kwargs:
                 if _item_confluence_kwargs:
                     _items.append(_item_confluence_kwargs.to_dict())
-            _dict['confluence_kwargs'] = _items
+            _dict["confluence_kwargs"] = _items
         return _dict
 
     @classmethod
@@ -92,16 +114,22 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
         if not isinstance(obj, dict):
             return cls.model_validate(obj)
 
-        _obj = cls.model_validate({
-            "url": obj.get("url"),
-            "token": obj.get("token"),
-            "space_key": obj.get("space_key"),
-            "include_attachments": obj.get("include_attachments") if obj.get("include_attachments") is not None else False,
-            "keep_markdown_format": obj.get("keep_markdown_format") if obj.get("keep_markdown_format") is not None else True,
-            "keep_newlines": obj.get("keep_newlines") if obj.get("keep_newlines") is not None else True,
-            "document_name": obj.get("document_name"),
-            "confluence_kwargs": [KeyValuePair.from_dict(_item) for _item in obj["confluence_kwargs"]] if obj.get("confluence_kwargs") is not None else None
-        })
+        _obj = cls.model_validate(
+            {
+                "url": obj.get("url"),
+                "token": obj.get("token"),
+                "space_key": obj.get("space_key"),
+                "include_attachments": obj.get("include_attachments")
+                if obj.get("include_attachments") is not None
+                else False,
+                "keep_markdown_format": obj.get("keep_markdown_format")
+                if obj.get("keep_markdown_format") is not None
+                else True,
+                "keep_newlines": obj.get("keep_newlines") if obj.get("keep_newlines") is not None else True,
+                "document_name": obj.get("document_name"),
+                "confluence_kwargs": [KeyValuePair.from_dict(_item) for _item in obj["confluence_kwargs"]]
+                if obj.get("confluence_kwargs") is not None
+                else None,
+            }
+        )
         return _obj
-
-

admin-api-lib/src/admin_api_lib/impl/settings/confluence_settings.py

@@ -14,18 +14,29 @@ class ConfluenceSettings(BaseSettings):
     """
     Contains configuration settings for the Confluence integration.
 
-    Attributes:
-      url: An optional list of Confluence URLs.
-      token: An optional list of authentication tokens.
-      space_key: An optional list of Confluence space keys.
-      document_name: An optional list of document names.
-      verify_ssl: An optional list of booleans indicating whether SSL verification is enabled.
-      include_attachments: Indicates whether to include attachments in the integration.
-      keep_markdown_format: Determines if markdown formatting is maintained.
-      keep_newlines: Indicates whether newlines are preserved.
+    Parameters
+    ----------
+    url : CommaSeparatedStrList, optional
+        List of Confluence URLs.
+    token : CommaSeparatedStrList, optional
+        List of authentication tokens.
+    space_key : CommaSeparatedStrList, optional
+        List of Confluence space keys.
+    document_name : CommaSeparatedStrList, optional
+        List of document names.
+    verify_ssl : CommaSeparatedBoolList, optional
+        List of booleans indicating whether SSL verification is enabled.
+    include_attachments : CommaSeparatedBoolList, optional
+        Indicates whether to include attachments in the integration.
+    keep_markdown_format : CommaSeparatedBoolList, optional
+        Determines if markdown formatting is maintained.
+    keep_newlines : CommaSeparatedBoolList, optional
+        Indicates whether newlines are preserved.
     """
 
     class Config:
+        """Config class for reading Fields from env."""
+
         env_prefix = "CONFLUENCE_"
         case_sensitive = False
 
@@ -41,22 +52,24 @@ class Config:
     @model_validator(mode="after")
     def check_lists_length_consistency(cls, values):
         """
-        Validate that all list-valued settings have the same length. If not, the list is adjusted accordingly.
+        Validate that all list-valued settings have the same length.
+
+        If not, the list is adjusted accordingly.
 
         Parameters
         ----------
         values : dict
-            Dictionary of configuration settings. Expected to contain keys with list values.
+            Dictionary of configuration settings.
 
         Returns
         -------
         dict
-            The original values dict if all checked lists have the same length.
+            The validated values dictionary with consistent list lengths.
 
         Raises
         ------
         ValueError
-            If any list has a different length compared to the others.
+            If any non-optional list has a different length compared to others.
         """
         # Define the keys to check
         keys = [
@@ -93,71 +106,65 @@ def check_lists_length_consistency(cls, values):
             try:
                 document_name = getattr(values, "document_name", None)
                 if not document_name or len(document_name) == 0:
-                    setattr(values, "document_name", CommaSeparatedStrList([""] * n))
+                    values.document_name = CommaSeparatedStrList([""] * n)
                 elif len(document_name) != n:
                     raise ValueError("document_name list length mismatch")
             except ValueError as e:
                 logger.error(f"Error setting document_name: {e}")
                 logger.warning("Setting document_name to default values")
                 document_name = getattr(values, "document_name", [])
-                setattr(values, "document_name", CommaSeparatedStrList(document_name + [""] * (n - len(document_name))))
+                values.document_name = CommaSeparatedStrList(document_name + [""] * (n - len(document_name)))
 
             try:
                 verify_ssl = getattr(values, "verify_ssl", None)
                 if not verify_ssl or len(verify_ssl) == 0:
-                    setattr(values, "verify_ssl", CommaSeparatedBoolList([True] * n))
+                    values.verify_ssl = CommaSeparatedBoolList([True] * n)
                 elif len(verify_ssl) != n:
                     raise ValueError("verify_ssl list length mismatch")
             except ValueError as e:
                 logger.error(f"Error setting verify_ssl: {e}")
                 logger.warning("Setting verify_ssl to default values")
                 verify_ssl = getattr(values, "verify_ssl", [])
-                setattr(values, "verify_ssl", CommaSeparatedBoolList(verify_ssl + [True] * (n - len(verify_ssl))))
+                values.verify_ssl = CommaSeparatedBoolList(verify_ssl + [True] * (n - len(verify_ssl)))
 
             try:
                 include_attachments = getattr(values, "include_attachments", None)
                 if not include_attachments or len(include_attachments) == 0:
-                    setattr(values, "include_attachments", CommaSeparatedBoolList([False] * n))
+                    values.include_attachments = CommaSeparatedBoolList([False] * n)
                 elif len(include_attachments) != n:
                     raise ValueError("include_attachments list length mismatch")
             except ValueError as e:
                 logger.error(f"Error setting include_attachments: {e}")
                 logger.warning("Setting include_attachments to default values")
                 include_attachments = getattr(values, "include_attachments", [])
-                setattr(
-                    values,
-                    "include_attachments",
-                    CommaSeparatedBoolList(include_attachments + [False] * (n - len(include_attachments))),
+                values.include_attachments = CommaSeparatedBoolList(
+                    include_attachments + [False] * (n - len(include_attachments))
                 )
 
             try:
                 keep_markdown_format = getattr(values, "keep_markdown_format", None)
                 if not keep_markdown_format or len(keep_markdown_format) == 0:
-                    setattr(values, "keep_markdown_format", CommaSeparatedBoolList([True] * n))
+                    values.keep_markdown_format = CommaSeparatedBoolList([True] * n)
                 elif len(keep_markdown_format) != n:
                     raise ValueError("keep_markdown_format list length mismatch")
             except ValueError as e:
                 logger.error(f"Error setting keep_markdown_format: {e}")
                 logger.warning("Setting keep_markdown_format to default values")
                 keep_markdown_format = getattr(values, "keep_markdown_format", [])
-                setattr(
-                    values,
-                    "keep_markdown_format",
-                    CommaSeparatedBoolList(keep_markdown_format + [True] * (n - len(keep_markdown_format))),
+                values.keep_markdown_format = CommaSeparatedBoolList(
+                    keep_markdown_format + [True] * (n - len(keep_markdown_format))
                 )
 
             try:
                 keep_newlines = getattr(values, "keep_newlines", None)
                 if not keep_newlines or len(keep_newlines) == 0:
-                    setattr(values, "keep_newlines", CommaSeparatedBoolList([True] * n))
+                    values.keep_newlines = CommaSeparatedBoolList([True] * n)
                 elif len(keep_newlines) != n:
                     raise ValueError("keep_newlines list length mismatch")
             except ValueError as e:
                 logger.error(f"Error setting keep_newlines: {e}")
                 logger.warning("Setting keep_newlines to default values")
                 keep_newlines = getattr(values, "keep_newlines", [])
-                setattr(
-                    values, "keep_newlines", CommaSeparatedBoolList(keep_newlines + [True] * (n - len(keep_newlines)))
-                )
+                values.keep_newlines = CommaSeparatedBoolList(keep_newlines + [True] * (n - len(keep_newlines)))
 
         return values

admin-api-lib/src/admin_api_lib/impl/utils/comma_separated_bool_list.py

@@ -15,10 +15,6 @@ class CommaSeparatedBoolList(list):
     - For list inputs, each element is converted to a boolean.
     """
 
-    @classmethod
-    def __get_validators__(cls):
-        yield cls.validate
-
     @classmethod
     def validate(cls, v: Any, info) -> list[bool]:
         """
@@ -52,3 +48,19 @@ def str_to_bool(s: str) -> bool:
         elif isinstance(v, list):
             return [bool(item) for item in v]
         raise ValueError("Not a valid comma separated boolean list")
+
+    @classmethod
+    def __get_validators__(cls):
+        """
+        Get validator functions for Pydantic to use with this data type.
+
+        This method is called by Pydantic during model initialization to collect
+        validator functions for fields using this custom data type.
+
+        Returns
+        -------
+        generator
+            A generator yielding validator functions, specifically `cls.validate`,
+            which will be applied to validate and convert input values.
+        """
+        yield cls.validate

admin-api-lib/src/admin_api_lib/impl/utils/comma_separated_str_list.py

@@ -1,6 +1,5 @@
 """
-Comma Separated String List Utility Module
-============================================
+Comma Separated String List Utility Module.
 
 This module provides a custom list type to validate and convert inputs into
 a list of strings. It splits comma separated strings and converts list elements
@@ -22,15 +21,12 @@ class CommaSeparatedStrList(list):
     - If input is a string: splits by commas and strips whitespace.
     - If input is a list: converts all elements to strings.
 
-    Raises:
-    -------
-      ValueError: For invalid input type.
+    Raises
+    ------
+    ValueError
+        For invalid input type.
     """
 
-    @classmethod
-    def __get_validators__(cls):
-        yield cls.validate
-
     @classmethod
     def validate(cls, v: Any, info) -> list[str]:
         """
@@ -60,3 +56,19 @@ def validate(cls, v: Any, info) -> list[str]:
         elif isinstance(v, list):
             return [str(item) for item in v]
         raise ValueError("Not a valid comma separated string list")
+
+    @classmethod
+    def __get_validators__(cls):
+        """
+        Get validator functions for Pydantic to use with this data type.
+
+        This method is called by Pydantic during model initialization to collect
+        validator functions for fields using this custom data type.
+
+        Returns
+        -------
+        generator
+            A generator yielding validator functions, specifically `cls.validate`,
+            which will be applied to validate and convert input values.
+        """
+        yield cls.validate