Merge pull request #27 from airavata-courses/feature/24-datasource-ca…

…ching

#24-milestone2: cache the nexrad object and added logging

datasource_service/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9-slim-buster
+FROM python:3.9.10-slim-buster
 
 LABEL maintainer="[email protected]"
 

datasource_service/README.md

@@ -5,6 +5,7 @@
 - Fetches the radar objects from nexradaws cloud storage
 - Plots the graph for the requested radar station
 - Returns the json as response with queried information and encoded image
+- [ENHANCEMENT] caching added to speed up the download of nexrad object
 
 ### The project is built using poetry packaging manager
 - Docs: [poetry](https://python-poetry.org/docs/)

datasource_service/datasource_service/main.py

@@ -8,9 +8,14 @@
 import pyart
 from matplotlib import pyplot as plt
 import base64
-
+from cachetools import cached, TTLCache
+import logging
 from datasource_service.response_model import ResponseModel
 
+logging.basicConfig()
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+
 desc = """ ### Datasource api helps you fetch the radar object from a specified radar station \n
            API uses a Py-ART library, an open source library developed by
            - JJ Helmus and SM Collis, JORS 2016, doi: 10.5334/jors.119     
@@ -19,14 +24,20 @@
 app = FastAPI(title="Datasource-api",
               description=desc)
 
+# set caching to one day
+ONE_DAY = 60 * 60 * 24
+
 
 @app.get("/api/v1/{year}/{month}/{day}/{radar}", response_model=ResponseModel, response_model_exclude_unset=True)
+@cached(cache=TTLCache(maxsize=5, ttl=ONE_DAY))
 def nexrad_data(year: int, month: int, day: int, radar: str):
     """ GET api to fetch data from nexradaws """
 
     json_object = {}
     encoded_image = ""
     try:
+        logger.info(f"Trying to fetch the {radar} radar station information, for year: {year}, month: {month}, "
+                    f"day: {day}")
         radar_object, scans = download_radar_object(year, month, day, radar)
         radar_file = scans[-1].filename.rstrip('.gz')
         for i, scan in enumerate(radar_object.iter_success(), start=1):
@@ -51,11 +62,13 @@ def nexrad_data(year: int, month: int, day: int, radar: str):
                 "radar": radar,
                 "encoded_image": encoded_image
             }
+        logger.info(f"Finished plotting the weather data for radar station: {radar}")
         return json_object
     except Exception as ex:
-        print(f"Error while fetching the data or plotting the graph: {ex}")
+        logger.error(f"Error while fetching the data or plotting the graph: {ex}", exc_info=True)
         raise HTTPException(status_code=404, detail="Radar station is not found")
     finally:
+        logger.debug("Removing the dangling files")
         dirs = os.listdir()
         cur_dir = os.getcwd()
         for file in dirs:
@@ -72,6 +85,7 @@ def download_radar_object(year: int, month: int, day: int, radar: str):
     """ Downloading the radar object """
 
     # nexradaws connection object
+    logger.info(f"Downloading the nexrad object for radar station: {radar}")
     ncon = nexradaws.NexradAwsInterface()
     scans = ncon.get_avail_scans(year, month, day, radar)
     download_dir = os.getcwd()
@@ -81,4 +95,5 @@ def download_radar_object(year: int, month: int, day: int, radar: str):
 
 def start():
     """ Start fastAPI using uvicorn """
+    logger.info("Starting the datasource api...")
     uvicorn.run("datasource_service.main:app", port=8000, host="0.0.0.0")