chore: Get documentation building. Fix simple mypy errors.

docs/api.rst

@@ -1,7 +1,6 @@
 API Reference
 =============
 
-
 .. automodule:: nightingale.loader
     :members:
     :undoc-members:
@@ -17,7 +16,12 @@ API Reference
     :undoc-members:
     :show-inheritance:
 
-.. automodule:: nightingale.mapping.v1.config
+.. automodule:: nightingale.codelists
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+.. automodule:: nightingale.mapping_template.v09
     :members:
     :undoc-members:
     :show-inheritance:

docs/tutorial.rst

@@ -130,7 +130,7 @@ Run the transformation using the CLI:
 This will produce an output file in the `output` directory.
 
 MappingTemplate Configuration
-----------------------
+-----------------------------
 
 Field-level mapping is specified in the `mapping.xlsx` file. It is formed from stardard `"OCDS Field Level MappingTemplate template" <https://www.open-contracting.org/resources/ocds-field-level-mapping-template/>`_.
 For more information about how to fill the mapping file, refer to the `OCDS Field Level MappingTemplate template guidance <https://www.open-contracting.org/resources/ocds-1-1-mapping-template-guidance/>`_.

nightingale/cli.py

@@ -6,11 +6,11 @@
 import click_pathlib
 from pydantic import TypeAdapter
 
-from .config import Config
-from .loader import DataLoader
-from .mapper import OCDSDataMapper
-from .publisher import DataPublisher
-from .writer import DataWriter
+from nightingale.config import Config
+from nightingale.loader import DataLoader
+from nightingale.mapper import OCDSDataMapper
+from nightingale.publisher import DataPublisher
+from nightingale.writer import DataWriter
 
 logger = logging.getLogger(__name__)
 

nightingale/config.py

@@ -1,6 +1,5 @@
 import tomllib
 from pathlib import Path
-from typing import Optional, Self
 
 from pydantic import TypeAdapter
 from pydantic.dataclasses import dataclass
@@ -33,8 +32,8 @@ class Mapping:
     file: Path
     ocid_prefix: str
     selector: str
-    force_publish: Optional[bool] = False
-    codelists: Optional[Path] = None
+    force_publish: bool | None = False
+    codelists: Path | None = None
 
 
 @dataclass(frozen=True)
@@ -45,7 +44,7 @@ class Config:
     output: Output
 
     @classmethod
-    def from_file(cls, config_file: Path) -> Self:
+    def from_file(cls, config_file: Path) -> "Config":
         with open(config_file, "rb") as f:
             data = tomllib.load(f)
         return TypeAdapter(Config).validate_python(data)

nightingale/mapper.py

@@ -1,30 +1,27 @@
 import logging
-from typing import Any, Optional
+from typing import Any
 
 import dict_hash
 
-from .codelists import CodelistsMapping
-from .config import Config
-from .mapping_template.v09 import MappingTemplate, MappingTemplateValidator
-from .utils import get_iso_now, is_new_array, remove_dicts_without_id
+from nightingale.codelists import CodelistsMapping
+from nightingale.config import Config
+from nightingale.mapping_template.v09 import MappingTemplate
+from nightingale.mapping_template.validator import MappingTemplateValidator
+from nightingale.utils import get_iso_now, is_new_array, remove_dicts_without_id
 
 logger = logging.getLogger(__name__)
 
 
 class OCDSDataMapper:
     """
     Maps data from a source to the OCDS format.
-
-    :param config: Configuration object containing settings for the mapper.
-    :type config: Config
     """
 
     def __init__(self, config: Config):
         """
         Initialize the OCDSDataMapper.
 
         :param config: Configuration object containing settings for the mapper.
-        :type config: Config
         """
         self.config = config
         self.mapping = MappingTemplate(config.mapping)
@@ -37,9 +34,7 @@ def produce_ocid(self, value: str) -> str:
         Produce an OCID based on the given value.
 
         :param value: The value to use for generating the OCID.
-        :type value: str
         :return: The produced OCID.
-        :rtype: str
         """
         prefix = self.config.mapping.ocid_prefix
         return f"{prefix}-{value}"
@@ -49,9 +44,7 @@ def map(self, loader: Any, validate_mapping: bool = False) -> list[dict[str, Any
         Map data from the loader to the OCDS format.
 
         :param loader: Data loader object.
-        :type loader: Any
         :return: List of mapped release dictionaries.
-        :rtype: list[dict[str, Any]]
         """
         config = self.config.mapping
 
@@ -67,17 +60,14 @@ def map(self, loader: Any, validate_mapping: bool = False) -> list[dict[str, Any
         return self.transform_data(data, self.mapping, codelists=self.codelists)
 
     def transform_data(
-        self, data: list[dict[Any, Any]], mapping: MappingTemplate, codelists: Optional[CodelistsMapping] = None
+        self, data: list[dict[Any, Any]], mapping: MappingTemplate, codelists: CodelistsMapping | None = None
     ) -> list[dict[str, Any]]:
         """
         Transform the input data to the OCDS format.
 
         :param data: List of input data dictionaries.
-        :type data: list[dict[Any, Any]]
         :param mapping: Mapping configuration object.
-        :type mapping: MappingTemplate
         :return: List of transformed release dictionaries.
-        :rtype: list[dict[str, Any]]
         """
         curr_ocid = ""
         curr_release = {}
@@ -125,24 +115,21 @@ def transform_row(
         input_data: dict[Any, Any],
         mapping_config: MappingTemplate,
         flattened_schema: dict[str, Any],
-        result: dict = None,
-        array_counters: dict = None,
-        codelists: Optional[CodelistsMapping] = None,
+        result: dict | None = None,
+        array_counters: dict | None = None,
+        codelists: CodelistsMapping | None = None,
     ) -> dict:
         """
         Transform a single row of input data to the OCDS format.
 
         :param input_data: Dictionary of input data.
-        :type input_data: dict[Any, Any]
         :param mapping_config: Mapping configuration object.
-        :type mapping_config: MappingTemplate
         :param flattened_schema: Flattened schema dictionary.
-        :type flattened_schema: dict[str, Any]
         :param result: Existing result dictionary to update.
-        :type result: dict, optional
         :return: Transformed row dictionary.
-        :rtype: dict
         """
+        if array_counters is None:
+            array_counters = {}
 
         # XXX: some duplication in code present maybe refactoring needed
         def set_nested_value(nested_dict, keys, value, schema, add_new=False, append_once=False):
@@ -243,7 +230,6 @@ def make_release_id(self, curr_row: dict) -> None:
         Generate and set a unique ID for the release based on its content.
 
         :param curr_row: The current release row dictionary.
-        :type curr_row: dict
         """
         id_ = dict_hash.sha256(curr_row)
         curr_row["id"] = id_
@@ -253,7 +239,6 @@ def date_release(self, curr_row: dict) -> None:
         Set the release date to the current date and time.
 
         :param curr_row: The current release row dictionary.
-        :type curr_row: dict
         """
         curr_row["date"] = get_iso_now()
 
@@ -262,7 +247,6 @@ def tag_initiation_type(self, curr_row: dict) -> None:
         Tag the initiation type of the release as 'tender' if applicable.
 
         :param curr_row: The current release row dictionary.
-        :type curr_row: dict
         """
         if "tender" in curr_row and "initiationType" not in curr_row:
             curr_row["initiationType"] = "tender"
@@ -272,9 +256,7 @@ def tag_ocid(self, curr_row: dict, curr_ocid: str) -> None:
         Set the OCID for the release.
 
         :param curr_row: The current release row dictionary.
-        :type curr_row: dict
         :param curr_ocid: The OCID value to set.
-        :type curr_ocid: str
         """
         curr_row["ocid"] = self.produce_ocid(curr_ocid)
 
@@ -293,7 +275,6 @@ def remove_empty_id_arrays(self, data: Any) -> Any:
         Recursively remove arrays that do not contain an 'id' field.
 
         :param data: The data dictionary to process.
-        :type data: dict[str, Any]
         """
 
         return remove_dicts_without_id(data)

nightingale/mapping_template/v09/__init__.py

@@ -2,7 +2,6 @@
 
 import openpyxl
 
-from nightingale.mapping_template.validator import MappingTemplateValidator  # noqa
 from nightingale.utils import get_longest_array_path
 
 logger = logging.getLogger(__name__)

nightingale/mapping_template/validator.py

@@ -10,7 +10,7 @@ def __init__(self, loader, mapping_template):
         self.loader = loader
         self.mapping_template = mapping_template
 
-    def validate_data_elements(self):
+    def validate_data_elements(self) -> None:
         """Match columns in the database and in data elements from mapping template"""
         cursor = self.loader.get_cursor()
         # XXX: postgersql support?
@@ -24,7 +24,7 @@ def validate_data_elements(self):
                 if column not in elements:
                     logger.warning(f"Column {table_name}/{column} is not described in data elements")
 
-    def validate_selector(self, row):
+    def validate_selector(self, row) -> None:
         """Check selected columns are used in mapping"""
         elements = self.mapping_template.get_data_elements()
         labels_for_mapping = [e["for_mapping"] for e in elements.values()]

nightingale/publisher.py

@@ -3,23 +3,20 @@
 
 from ocdskit.combine import package_releases
 
-from .utils import get_iso_now, produce_package_name
+from nightingale.config import Publishing
+from nightingale.utils import get_iso_now, produce_package_name
 
 
 class DataPublisher:
     """
     Packs array of releases into a release package.
-
-    :param config: Configuration object containing settings for the publisher.
-    :type config: Config
     """
 
-    def __init__(self, config, mapping):
+    def __init__(self, config: Publishing, mapping):
         """
         Initialize the DataPublisher.
 
         :param config: Configuration object containing settings for the publisher.
-        :type config: Config
         """
         self.config = config
         self.mapping = mapping
@@ -29,10 +26,7 @@ def produce_uri(self) -> str:
         """
         Produce a URI for the package based on the given date.
 
-        :param date: The date to use for generating the URI.
-        :type date: str
         :return: The produced URI.
-        :rtype: str
         """
         full_name = produce_package_name(self.date)
         return urljoin(self.config.base_uri, f"/{full_name}")
@@ -42,9 +36,7 @@ def package(self, data: list[dict[str, Any]]) -> dict[str, Any]:
         Package the given data into a release package.
 
         :param data: List of release dictionaries to be packaged.
-        :type data: list[dict[str, Any]]
         :return: A dictionary representing the release package.
-        :rtype: dict[str, Any]
         """
         kwargs = dict(
             uri=self.produce_uri(),

nightingale/utils.py

@@ -23,22 +23,16 @@ def get_iso_now():
     return now.strftime("%Y-%m-%dT%H:%M:%SZ")
 
 
-def is_new_array(array_counters, child_path, array_key, array_value, array_path):
+def is_new_array(array_counters: dict, child_path: str, array_key: str, array_value: str, array_path: str) -> bool:
     """
     Check if a new array should be created based on the given parameters.
 
     :param array_counters: Dictionary keeping track of array counters.
-    :type array_counters: dict
     :param child_path: The child path in the schema.
-    :type child_path: str
     :param array_key: The key in the array.
-    :type array_key: str
     :param array_value: The value associated with the array key.
-    :type array_value: str
     :param array_path: The path of the array.
-    :type array_path: str
     :return: True if a new array should be created, False otherwise.
-    :rtype: bool
 
     >>> array_counters = {'/object/field2/array_field': '1'}
     >>> child_path = '/id'

nightingale/writer.py

@@ -3,17 +3,16 @@
 
 import simplejson as json
 
-from .utils import produce_package_name
+from nightingale.config import Output
+from nightingale.utils import produce_package_name
 
 
 def new_name(package: dict | list) -> str:
     """
     Generate a new name for the package based on its published date.
 
     :param package: The release package dictionary.
-    :type package: dict
     :return: The generated package name.
-    :rtype: str
     """
 
     if isinstance(package, list):
@@ -26,17 +25,13 @@ def new_name(package: dict | list) -> str:
 class DataWriter:
     """
     Writes release package to disk.
-
-    :param config: Configuration object containing settings for the writer.
-    :type config: Config
     """
 
-    def __init__(self, config):
+    def __init__(self, config: Output):
         """
         Initialize the DataWriter.
 
         :param config: Configuration object containing settings for the writer.
-        :type config: Config
         """
         self.config = config
 
@@ -45,30 +40,26 @@ def make_dirs(self) -> Path:
         Create the necessary directories for storing the release package.
 
         :return: The base directory path.
-        :rtype: Path
         """
         base = Path(self.config.directory)
         base.mkdir(parents=True, exist_ok=True)
         return base
 
-    def get_output_path(self, package: dict) -> Path:
+    def get_output_path(self, package: dict | list) -> Path:
         """
         Get the output path for the release package.
 
         :param package: The release package dictionary.
-        :type package: dict
         :return: The path where the package will be written.
-        :rtype: Path
         """
         base = self.make_dirs()
         return base / new_name(package)
 
-    def write(self, package: list | dict) -> None:
+    def write(self, package: dict | list) -> None:
         """
         Write the release package to disk.
 
         :param package: The release package dictionary.
-        :type package: dict
         """
         path = self.get_output_path(package)
         with path.open("w") as f: