Implement common logic for saving to disk & writing to db for arcgis_feature_layer and comapeo_observations

f/common_logic/save_disk.py

@@ -21,26 +21,24 @@ def get_safe_file_path(storage_path: str, db_table_name: str, file_type: str):
     return file_path
 
 
-def save_export_file(
-    data, db_table_name: str, storage_path: str, file_type: str = "json"
-):
+def save_data_to_file(data, filename: str, storage_path: str, file_type: str = "json"):
     """
     Saves the provided data to a file in the specified format and storage path.
 
     Parameters
     ----------
     data : list or dict
         The data to be saved. For CSV, should be a list of rows including a header.
-    db_table_name : str
-        The name of the database table, used to name the output file.
+    filename : str
+        The name of the file to save the data to, without extension.
     storage_path : str
         The directory path where the file will be saved.
     file_type : str
         The format to save the file as: "json", "geojson", or "csv".
     """
     storage_path = Path(storage_path)
     storage_path.mkdir(parents=True, exist_ok=True)
-    file_path = get_safe_file_path(storage_path, db_table_name, file_type)
+    file_path = get_safe_file_path(storage_path, filename, file_type)
 
     if file_type in {"geojson", "json"}:
         with file_path.open("w") as f:

f/connectors/arcgis/arcgis_feature_layer.py

@@ -2,14 +2,14 @@
 # psycopg2-binary
 # requests~=2.32
 
-import json
 import logging
 import os
 from pathlib import Path
 
 import requests
 
 from f.common_logic.db_operations import postgresql
+from f.common_logic.save_disk import save_data_to_file
 from f.connectors.geojson.geojson_to_postgres import main as save_geojson_to_postgres
 
 # type names that refer to Windmill Resources
@@ -38,19 +38,26 @@ def main(
 
     features_with_global_ids = set_global_id(features_with_attachments)
 
-    save_geojson_file_to_disk(features_with_global_ids, storage_path)
+    geojson = {"type": "FeatureCollection", "features": features_with_global_ids}
 
     # At this point, the ArcGIS data is GeoJSON-compliant, and we don't need anything
-    # from the REST API anymore. The data can therefore be handled further using the
-    # existing GeoJSON connector.
+    # from the REST API anymore. We can save the data to a file and then to Postgres.
+
+    save_data_to_file(
+        geojson,
+        db_table_name,
+        storage_path,
+        file_type="geojson",
+    )
+
+    rel_geojson_path = Path(db_table_name) / f"{db_table_name}.geojson"
+
     save_geojson_to_postgres(
         db,
         db_table_name,
-        str(storage_path / "data.geojson"),
-        storage_path,
-        False,  # to not delete the GeoJSON file after its contents are written to the database.
-        # Users might like to have access to the GeoJSON file directly, in addition to the data
-        # in the database.
+        rel_geojson_path,
+        attachment_root,
+        False,  # do not delete the file after saving to Postgres
     )
 
 
@@ -246,29 +253,3 @@ def set_global_id(features: list):
         feature["id"] = feature["properties"]["globalid"]
 
     return features
-
-
-def save_geojson_file_to_disk(
-    features: list,
-    storage_path: str,
-):
-    """
-    Save the GeoJSON file to disk.
-
-    Parameters
-    ----------
-    features : list
-        A list of features to save.
-    storage_path : str
-        The directory where the GeoJSON file should be saved.
-    """
-    geojson = {"type": "FeatureCollection", "features": features}
-
-    geojson_filename = Path(storage_path) / "data.geojson"
-
-    geojson_filename.parent.mkdir(parents=True, exist_ok=True)
-
-    with open(geojson_filename, "w") as f:
-        json.dump(geojson, f)
-
-    logger.info(f"GeoJSON file saved to: {geojson_filename}")