make destinations request via function

github action runners in US cannot access smartraveller site
use functions to return updated destinations (syd1)

Author: kevle1

README.md

@@ -55,7 +55,6 @@ Get all available Smartraveller advisories. This is updated every 3 hours.
 {
     "last_updated": "2024-10-05T04:55:06.260067+00:00",
     "advisories": [
-      ...
         {
           "country": {
             "name": "Spain",
@@ -65,7 +64,6 @@ Get all available Smartraveller advisories. This is updated every 3 hours.
           "level": 1,
           "page_url": "https://www.smartraveller.gov.au/destinations/europe/spain"
         },
-      ...
     ]
 }
 ```
@@ -80,15 +78,13 @@ Get all available Smartraveller destinations. This is updated every 3 hours.
 {
     "last_updated": "2024-10-05T04:55:06.260067+00:00",
     "destinations": {
-      ...
       "ES": {
         "country": {
           "name": "Spain",
           "alpha_2": "ES"
         },
         "page_url": "/destinations/europe/spain"
       },
-      ...
     }
 }
 ```

api/api.py

@@ -3,6 +3,7 @@
 from flask import Flask, Response, request
 
 from smartraveller.advisory import get_advisory
+from smartraveller.destination import get_destinations
 
 app = Flask(__name__, static_url_path="")
 
@@ -36,15 +37,21 @@ def advisory():
 @app.route("/advisories")
 def advisories():
     with open("data/advisories.json", "r") as advisories_file:
-        json_advisories = json.load(advisories_file)
-    return _return_success(json_advisories)
+        advisories = json.load(advisories_file)
+    return _return_success(advisories)
 
 
 @app.route("/destinations")
 def destinations():
-    with open("data/destinations.json", "r") as destinations_file:
-        destinations = json.load(destinations_file)
-    return _return_success(destinations)
+    refetch = request.args.get("refetch") == "true"
+    destinations = get_destinations(refetch)
+
+    return _return_success(
+        {
+            country: destination.model_dump()
+            for country, destination in sorted(destinations.items())
+        }
+    )
 
 
 def _return_success(data: dict, cache_duration_min: int = 60) -> Response:

data/destinations.json

@@ -1,5 +1,5 @@
 {
-    "last_updated": "2024-10-05T06:11:20.824473+00:00",
+    "last_updated": "2024-10-05T06:40:46.331869+00:00",
     "destinations": {
         "AE": {
             "country": {

smartraveller/aggregate.py

@@ -12,20 +12,18 @@
 
 
 def _save_destinations():
-    destinations = {
-        country: destination.model_dump()
-        for country, destination in _fetch_destinations().items()
-    }
-
-    with open(f"{OUTPUT_PREFIX}/destinations.json", "w") as outfile:
-        json.dump(
-            {
-                "last_updated": datetime.now(timezone.utc).isoformat(),
-                "destinations": dict(sorted(destinations.items())),
-            },
-            outfile,
-            indent=4,
-        )
+    response = requests.get(f"{BASE_URL}/destinations?refetch=true")
+    if response.status_code == 200:
+        with open(f"{OUTPUT_PREFIX}/destinations.json", "w") as outfile:
+            json.dump(
+                {
+                    "last_updated": datetime.now(timezone.utc).isoformat(),
+                    "destinations": response.json(),
+                },
+                outfile,
+                indent=4,
+            )
+        logging.info("Saved destinations")
 
 
 def _aggregate_advisories():
@@ -47,6 +45,7 @@ def _aggregate_advisories():
             outfile,
             indent=4,
         )
+        logging.info("Saved advisories")
 
 
 if __name__ == "__main__":

smartraveller/destination.py

@@ -30,19 +30,25 @@ class SmartravellerDestination(BaseModel):
 }
 
 
-def get_destinations() -> dict[str, SmartravellerDestination]:
-    try:
-        with open("data/destinations.json", "r") as destinations:
-            destinations = json.load(destinations)["destinations"].items()
-            return {
-                country: SmartravellerDestination(**destination)
-                for country, destination in destinations
-            }
-    except FileNotFoundError:
-        return _fetch_destinations()
-
-
-def _fetch_destinations() -> list[dict[str, SmartravellerDestination]]:
+def get_destinations(refetch: bool = False) -> dict[str, SmartravellerDestination]:
+    if not refetch:
+        try:
+            with open("data/destinations.json", "r") as destinations:
+                destinations = json.load(destinations)["destinations"]
+        except FileNotFoundError:
+            destinations = _fetch_destinations()
+    else:
+        destinations = _fetch_destinations()
+
+    return {
+        country: SmartravellerDestination(**destination)
+        if isinstance(destination, dict)
+        else destination
+        for country, destination in destinations.items()
+    }
+
+
+def _fetch_destinations() -> dict[str, SmartravellerDestination]:
     response = requests.get("https://www.smartraveller.gov.au/api/publishedpages")
     if response.status_code != 200:
         logging.error("Failed to get destinations")