comments and cleanup

Author: nmdefries

integrations/server/test_flusurv.py

@@ -10,6 +10,8 @@ def localSetUp(self):
 
     def test_flusurv(self):
         """Basic integration test for flusurv endpoint"""
+        # TODO: Should test out the whole pipeline, where we mock the source
+        # API by pulling more realistic data from a local JSON.
         self.cur.execute(
             """
             INSERT INTO `flusurv`(

src/acquisition/flusurv/api.py

@@ -67,14 +67,13 @@ def fetch_json(path, payload, call_count=1, requests_impl=requests):
         "Accept-Encoding": "gzip",
         "User-Agent": USER_AGENT,
     }
-    if payload is not None:
-        headers["Content-Type"] = "application/json;charset=UTF-8"
 
     # send the request and read the response
     if payload is None:
         method = requests_impl.get
         data = None
     else:
+        headers["Content-Type"] = "application/json;charset=UTF-8"
         method = requests_impl.post
         data = json.dumps(payload)
     resp = method(flusurv_url, headers=headers, data=data)
@@ -100,30 +99,30 @@ def fetch_json(path, payload, call_count=1, requests_impl=requests):
 def mmwrid_to_epiweek(mmwrid):
     """Convert a CDC week index into an epiweek."""
 
-    # Add the difference in IDs, which are sequential, to a reference
-    # epiweek, which is 2003w40 in this case.
+    # Add the difference in IDs, which are sequential, to a reference epiweek,
+    # which is 2003w40 in this case. This is the earliest date we see in the
+    # returned data. The index-1 week on this scale is the first week of
+    # 1962.
     epiweek_200340 = EpiDate(2003, 9, 28)
     mmwrid_200340 = 2179
     return epiweek_200340.add_weeks(mmwrid - mmwrid_200340).get_ew()
 
 
 class FlusurvMetadata:
     def __init__(self, max_age_weeks):
-        self.metadata = self._fetch_flusurv_metadata()
-
-        self.issue = self._get_current_issue()
         self.max_age_weeks = max_age_weeks
-        self.seasonids = self._get_recent_seasonids()
+        self.id_to_group = ID_TO_LABEL_MAP
 
-        self.location_to_code = self._make_location_to_code_map()
-        self.locations = self.location_to_code.keys()
+        self._fetch_flusurv_metadata()
 
-        self.id_to_group = ID_TO_LABEL_MAP
-        self.id_to_season = self._make_id_season_map()
+        self._get_current_issue()
+        self._get_recent_seasonids()
+        self._make_location_to_code_map()
+        self._make_id_season_map()
 
     def _fetch_flusurv_metadata(self):
         """Return FluSurv JSON metadata object."""
-        return fetch_json(
+        self.metadata = fetch_json(
             "PostPhase03DataTool",
             {"appversion": "Public", "key": "", "injson": []}
         )
@@ -151,7 +150,8 @@ def _make_location_to_code_map(self):
             location_to_code[location_name] = (
                 int(location["networkid"]), int(location["catchmentid"])
             )
-        return location_to_code
+        self.location_to_code = location_to_code
+        self.locations = self.location_to_code.keys()
 
     def _get_current_issue(self):
         """
@@ -163,25 +163,24 @@ def _get_current_issue(self):
         # extract
         date = datetime.strptime(self.metadata["loaddatetime"], "%b %d, %Y")
 
-        # convert and return
-        return EpiDate(date.year, date.month, date.day).get_ew()
+        # convert
+        self.issue = EpiDate(date.year, date.month, date.day).get_ew()
 
     def _get_recent_seasonids(self):
-        # Ignore seasons with all dates older than one year
-        seasonids = {
+        # Ignore seasons with all dates older than `self.max_age_weeks` (from
+        # user command line argument `max_age`)
+        self.seasonids = {
             season_blob["seasonid"] for season_blob in self.metadata["seasons"]
             if delta_epiweeks(mmwrid_to_epiweek(season_blob["endweek"]), self.issue) < self.max_age_weeks
         }
 
-        return seasonids
-
     def _make_id_season_map(self):
         """Create a map from seasonid to season description, in the format "YYYY-YY" """
         id_to_label = defaultdict(lambda: defaultdict(lambda: None))
         for season in self.metadata["seasons"]:
             id_to_label[season["seasonid"]] = season["label"].strip()
 
-        return id_to_label
+        self.id_to_season = id_to_label
 
 
 class FlusurvLocationFetcher:

src/acquisition/flusurv/reference/new_grasp_location_result.json

@@ -1,4 +1,8 @@
 ### New location-specific API response from https://gis.cdc.gov/GRASP/Flu3/PostPhase03DataTool?appVersion=Public
+#
+# The JSON format is invalid as-is. To be able to read these reference files
+# into python, first remove the comments.
+#
 # with payload
 #     {"appversion": "Public", "key": "getdata", "injson": [
 #         {"networkid": 1, "catchmentid": 22, "seasonid": 30 },
@@ -19,16 +23,16 @@
     # Data!! Format is list of dicts. Each obs' fields are
     #  labelled, so we could easily convert this to dataframe.
     'default_data': [
-        {'networkid': 1, 'catchmentid': 22, 'seasonid': 49, 'ageid': 0, 'sexid': 0, 'raceid': 1, 'rate': 4.3, 'weeklyrate': 1.7, 'mmwrid': 2493}, 
-        {'networkid': 1, 'catchmentid': 22, 'seasonid': 49, 'ageid': 0, 'sexid': 0, 'raceid': 1, 'rate': 20.3, 'weeklyrate': 0.1, 'mmwrid': 2513}, 
-        {'networkid': 1, 'catchmentid': 22, 'seasonid': 49, 'ageid': 0, 'sexid': 0, 'raceid': 1, 'rate': 20.6, 'weeklyrate': 0.1, 'mmwrid': 2516}, 
-        {'networkid': 1, 'catchmentid': 22, 'seasonid': 49, 'ageid': 0, 'sexid': 0, 'raceid': 1, 'rate': 20.7, 'weeklyrate': 0.0, 'mmwrid': 2519}, 
-        {'networkid': 1, 'catchmentid': 22, 'seasonid': 49, 'ageid': 0, 'sexid': 0, 'raceid': 2, 'rate': 11.6, 'weeklyrate': 3.6, 'mmwrid': 2493}, 
-        {'networkid': 1, 'catchmentid': 22, 'seasonid': 49, 'ageid': 0, 'sexid': 0, 'raceid': 2, 'rate': 39.6, 'weeklyrate': 0.3, 'mmwrid': 2513}, 
-        {'networkid': 1, 'catchmentid': 22, 'seasonid': 49, 'ageid': 0, 'sexid': 0, 'raceid': 2, 'rate': 40.7, 'weeklyrate': 0.5, 'mmwrid': 2516}, 
-        {'networkid': 1, 'catchmentid': 22, 'seasonid': 49, 'ageid': 0, 'sexid': 0, 'raceid': 2, 'rate': 41.3, 'weeklyrate': 0.1, 'mmwrid': 2519}, 
-        {'networkid': 1, 'catchmentid': 22, 'seasonid': 49, 'ageid': 0, 'sexid': 0, 'raceid': 3, 'rate': 12.8, 'weeklyrate': 4.8, 'mmwrid': 2493}, 
-        {'networkid': 1, 'catchmentid': 22, 'seasonid': 49, 'ageid': 0, 'sexid': 0, 'raceid': 3, 'rate': 36.0, 'weeklyrate': 0.1, 'mmwrid': 2513}, 
+        {'networkid': 1, 'catchmentid': 22, 'seasonid': 49, 'ageid': 0, 'sexid': 0, 'raceid': 1, 'rate': 4.3, 'weeklyrate': 1.7, 'mmwrid': 2493},
+        {'networkid': 1, 'catchmentid': 22, 'seasonid': 49, 'ageid': 0, 'sexid': 0, 'raceid': 1, 'rate': 20.3, 'weeklyrate': 0.1, 'mmwrid': 2513},
+        {'networkid': 1, 'catchmentid': 22, 'seasonid': 49, 'ageid': 0, 'sexid': 0, 'raceid': 1, 'rate': 20.6, 'weeklyrate': 0.1, 'mmwrid': 2516},
+        {'networkid': 1, 'catchmentid': 22, 'seasonid': 49, 'ageid': 0, 'sexid': 0, 'raceid': 1, 'rate': 20.7, 'weeklyrate': 0.0, 'mmwrid': 2519},
+        {'networkid': 1, 'catchmentid': 22, 'seasonid': 49, 'ageid': 0, 'sexid': 0, 'raceid': 2, 'rate': 11.6, 'weeklyrate': 3.6, 'mmwrid': 2493},
+        {'networkid': 1, 'catchmentid': 22, 'seasonid': 49, 'ageid': 0, 'sexid': 0, 'raceid': 2, 'rate': 39.6, 'weeklyrate': 0.3, 'mmwrid': 2513},
+        {'networkid': 1, 'catchmentid': 22, 'seasonid': 49, 'ageid': 0, 'sexid': 0, 'raceid': 2, 'rate': 40.7, 'weeklyrate': 0.5, 'mmwrid': 2516},
+        {'networkid': 1, 'catchmentid': 22, 'seasonid': 49, 'ageid': 0, 'sexid': 0, 'raceid': 2, 'rate': 41.3, 'weeklyrate': 0.1, 'mmwrid': 2519},
+        {'networkid': 1, 'catchmentid': 22, 'seasonid': 49, 'ageid': 0, 'sexid': 0, 'raceid': 3, 'rate': 12.8, 'weeklyrate': 4.8, 'mmwrid': 2493},
+        {'networkid': 1, 'catchmentid': 22, 'seasonid': 49, 'ageid': 0, 'sexid': 0, 'raceid': 3, 'rate': 36.0, 'weeklyrate': 0.1, 'mmwrid': 2513},
         ...
     ]
 }