Update categorization of geo_types

Author: cchuong

src/acquisition/rvdss/run.py

@@ -8,7 +8,7 @@
 import argparse
 from datetime import datetime
 
-from delphi.epidata.acquisition.rvdss.utils import fetch_current_dashboard_data, check_most_recent_update_date,get_dashboard_update_date, combine_tables, duplicate_provincial_detections,expand_detections_columns
+from delphi.epidata.acquisition.rvdss.utils import create_geo_types, abbreviate_geo, fetch_current_dashboard_data, check_most_recent_update_date,get_dashboard_update_date, combine_tables, duplicate_provincial_detections,expand_detections_columns
 from delphi.epidata.acquisition.rvdss.constants import DASHBOARD_BASE_URL, RESP_DETECTIONS_OUTPUT_FILE, POSITIVE_TESTS_OUTPUT_FILE,UPDATE_DATES_FILE
 from delphi.epidata.acquisition.rvdss.pull_historic import fetch_report_data,fetch_historical_dashboard_data
 from delphi.epidata.acquisition.rvdss.database import update
@@ -88,6 +88,8 @@ def patch_seasons(season_start_years,logger):
 
         if start >=2024:
             data = pd.read_csv(f"{start}_{end}_respiratory_detections.csv")
+            data['geo_value'] = [abbreviate_geo(g) for g in data['geo_value']]
+            data["geo_type"] = [create_geo_types(g,"lab") for g in data["geo_value"]]
 
             # current dashboard only needs one table
             new_data = expand_detections_columns(data)
@@ -98,7 +100,10 @@ def patch_seasons(season_start_years,logger):
             update(new_data,logger)
         else:
             resp_data = pd.read_csv(f"{start}_{end}_respiratory_detections.csv")
+            resp_data['geo_value'] = [abbreviate_geo(g) for g in resp_data['geo_value']]
+            
             pos_data = pd.read_csv(f"{start}_{end}_positive_tests.csv")
+        
             data_dict={"positive":pos_data,"respiratory_detection":resp_data} 
 
             # Combine all rables into a single table

src/acquisition/rvdss/utils.py

@@ -51,6 +51,8 @@ def create_geo_types(geo,default_geo):
     lowercase_geo = geo.lower()
     if lowercase_geo in NATION:
         geo_type="nation"
+    elif geo in PROVINCES:
+        geo_type="province"
     elif geo in REGIONS:
         geo_type="region"
     else:
@@ -181,7 +183,7 @@ def get_positive_data(base_url,headers,update_date):
     df['province'] = [abbreviate_geo(g) for g in df['province']]
     df=df.rename(columns={'province':"geo_value",'date':'time_value',"detections":"positivetests"})
     df['time_value'] = [check_date_format(d) for d in df['time_value']]
-    df['geo_type'] = [create_geo_types(g,"province") for g in df['geo_value']]
+    df['geo_type'] = [create_geo_types(g,"lab") for g in df['geo_value']]
     df.insert(1,"issue",update_date)
     df['region'] = [abbreviate_geo(g) for g in df['region']]
 
@@ -304,12 +306,19 @@ def expand_detections_columns(new_data):
     return(new_data.set_index(['epiweek', 'time_value', 'issue', 'geo_type', 'geo_value']))
 
 def duplicate_provincial_detections(data):
+    '''
+    The data has geo_type = lab for all labs, geo_type = province for all province and territories
+    and geo_type region for prairies, territories and atlantic, but ontario, quebec and bc are
+    both regions and provinces, so duplicate the data for those three geos with the geo_type = region
+    for completeness
+    '''
     dat = data.copy(deep=True)
     dat = dat.reset_index()
 
     # provincial data
-    dat.loc[dat['geo_value'].isin(PROVINCES),'geo_type'] = "province" 
-    provincial_detections = dat.loc[dat['geo_value'].isin(PROVINCES)]
+    provincial_regions = ['bc','on','qc']
+    dat.loc[dat['geo_value'].isin(provincial_regions),'geo_type'] = "region" 
+    provincial_detections = dat.loc[dat['geo_value'].isin(provincial_regions)]
 
     if not provincial_detections.empty:
         #provincial_detections['geo_type']="province"
@@ -329,14 +338,14 @@ def combine_tables(data_dict):
         positive["epiweek"] = pd.to_numeric(positive["epiweek"],downcast="integer")
         positive["time_value"] = pd.to_datetime(positive["time_value"])
         positive["issue"] = pd.to_datetime(positive["issue"])
+        positive['geo_type'] = [create_geo_types(g,'lab') for g in positive['geo_value']]
 
         detections["epiweek"] = pd.to_numeric(detections["epiweek"],downcast="integer")
         detections["time_value"] = pd.to_datetime(detections["time_value"])
         detections["issue"] = pd.to_datetime(detections["issue"])
+        detections['geo_type'] = [create_geo_types(g,'lab') for g in detections['geo_value']]
 
         detections = expand_detections_columns(detections)
-        positive = positive.drop(['geo_type'], axis=1)
-        positive['geo_type'] = [create_geo_types(g,'lab') for g in positive['geo_value']]
         positive=positive.set_index(['epiweek', 'time_value', 'issue', 'geo_type', 'geo_value'])
 
         positive = positive.fillna(np.nan)

tests/acquisition/rvdss/test_utils.py

@@ -281,8 +281,9 @@
                   'issue': list(np.repeat(np.array([pd.to_datetime("2012-09-10"),
                                                          pd.to_datetime("2012-09-11")]), 
                                                [13, 3], axis=0)),
-                  'geo_type':["lab","lab","lab","lab","lab","lab","lab","lab","lab","lab","lab",
-                              "lab","lab","lab","lab","lab"],
+                  'geo_type':["province","province","province","province","province","province","province",
+                              "province","province","province","province","province","province","nation",
+                              "lab","region"],
                   'geo_value':['nl','pe','ns','nb','qc','on','mb','sk','ab','bc','yt','nt','nu','ca','phol-toronto',
                                'atlantic'],
                   'adv_positive_tests': [1,1,1,1,1,1,1,1,1,1,1,1,1,9,9,9],
@@ -306,12 +307,11 @@
                                                          pd.to_datetime("2012-09-11"),
                                                          pd.to_datetime("2012-09-10")]), 
                                                [13, 3,13], axis=0)),
-                  'geo_type':["lab","lab","lab","lab","lab","lab","lab","lab","lab","lab","lab",
-                              "lab","lab","lab","lab","lab","province","province","province","province",
-                              "province","province","province","province","province","province","province",
-                              "province","province"],
+                  'geo_type':["province","province","province","province","province","province","province",
+                              "province","province","province","province","province","province","nation",
+                              "lab","region",'region','region','region'],
                   'geo_value':['nl','pe','ns','nb','qc','on','mb','sk','ab','bc','yt','nt','nu','ca','phol-toronto',
-                               'atlantic','nl','pe','ns','nb','qc','on','mb','sk','ab','bc','yt','nt','nu'],
+                               'atlantic','qc','on','bc'],
                   'adv_positive_tests': [1,1,1,1,1,1,1,1,1,1,1,1,1,9,9,9,1,1,1,1,1,1,1,1,1,1,1,1,1],
                   }).set_index(['epiweek', 'time_value', 'issue', 'geo_type', 'geo_value']),
     pd.DataFrame({'epiweek': [1,2,3], 
@@ -338,14 +338,15 @@ def test_abbreviate_geo(self):
         assert abbreviate_geo("british columbia") == "bc"
         assert abbreviate_geo("québec") == "qc" # recognise accents in provinces
         assert abbreviate_geo("Région Nord-Est") == "région nord est" # remove dashes, make lowercase
-        assert abbreviate_geo("P.H.O.L. - Sault Ste. Marie") == "phol sault ste marie"
+        assert abbreviate_geo("P.H.O.L. - Sault Ste. Marie") == "sault ste marie phl"
         assert abbreviate_geo("random lab") == "random lab" #unknown geos remain unchanged     
         # only province names on their own should be abbreviated, not as part of a larger name
         assert abbreviate_geo("british columbia lab") == "british columbia lab"
 
     def test_create_geo_types(self):
         assert create_geo_types("canada","lab") == "nation"
-        assert create_geo_types("bc","lab") == "region"
+        assert create_geo_types("bc","lab") == "province"
+        assert create_geo_types("prairies","lab") == "region"
         assert create_geo_types("random lab","lab") == "lab"
         assert create_geo_types("Canada","province") == "nation"