Change issue and time_value to integers instead of dates, and updated tests

Author: cchuong

src/acquisition/rvdss/pull_historic.py

@@ -20,7 +20,7 @@
         ALTERNATIVE_SEASON_BASE_URL, SEASON_BASE_URL, FIRST_WEEK_OF_YEAR, DASHBOARD_BASE_URLS_2023_2024_SEASON
     )
 from delphi.epidata.acquisition.rvdss.utils import (
-        abbreviate_geo, create_geo_types, check_date_format,
+        abbreviate_geo, create_geo_types, check_date_format,convert_date_to_int,
         fetch_archived_dashboard_data, preprocess_table_columns, add_flu_prefix
     )
  #%% Functions
@@ -499,6 +499,17 @@ def fetch_one_season_from_report(url):
             del combined_positive_tables
             del pos_table
 
+    # convert dates to integers
+    all_respiratory_detection_tables = all_respiratory_detection_tables.reset_index()
+    all_respiratory_detection_tables['time_value'] = [convert_date_to_int(t) for t in all_respiratory_detection_tables['time_value']]
+    all_respiratory_detection_tables['issue'] = [convert_date_to_int(t) for t in all_respiratory_detection_tables['issue']]
+    all_respiratory_detection_tables = all_respiratory_detection_tables.set_index(['epiweek', 'time_value', 'issue', 'geo_type', 'geo_value'],verify_integrity=True)
+    
+    all_positive_tables = all_positive_tables.reset_index()
+    all_positive_tables['time_value'] = [convert_date_to_int(t) for t in all_positive_tables['time_value']]
+    all_positive_tables['issue'] = [convert_date_to_int(t) for t in all_positive_tables['issue']]
+    all_positive_tables = all_positive_tables.set_index(['epiweek', 'time_value', 'issue', 'geo_type', 'geo_value'],verify_integrity=True)
+    
     return {
         "respiratory_detection": all_respiratory_detection_tables,
         "positive": all_positive_tables
@@ -515,5 +526,4 @@ def fetch_historical_dashboard_data():
     # Update the end of the 2023-2024 season with the dashboard data
 
     dict_list = [fetch_archived_dashboard_data(url) for url in DASHBOARD_BASE_URLS_2023_2024_SEASON]
-
     return dict_list

src/acquisition/rvdss/run.py

@@ -8,7 +8,7 @@
 import argparse
 from datetime import datetime
 
-from delphi.epidata.acquisition.rvdss.utils import create_geo_types, abbreviate_geo, fetch_current_dashboard_data, check_most_recent_update_date,get_dashboard_update_date, combine_tables, duplicate_provincial_detections,expand_detections_columns
+from delphi.epidata.acquisition.rvdss.utils import create_geo_types, abbreviate_geo, fetch_current_dashboard_data, check_most_recent_update_date,get_dashboard_update_date, combine_tables, duplicate_provincial_detections,expand_detections_columns      
 from delphi.epidata.acquisition.rvdss.constants import DASHBOARD_BASE_URL, RESP_DETECTIONS_OUTPUT_FILE, POSITIVE_TESTS_OUTPUT_FILE,UPDATE_DATES_FILE
 from delphi.epidata.acquisition.rvdss.pull_historic import fetch_report_data,fetch_historical_dashboard_data
 from delphi.epidata.acquisition.rvdss.database import update
@@ -76,6 +76,7 @@ def update_historical_data(logger):
     data = combine_tables(hist_dict_list)
     data['time_type'] = "week"
 
+    
     #update database
     update(data,logger)
     logger.info("Finished updating historic data")

src/acquisition/rvdss/utils.py

@@ -73,6 +73,13 @@ def check_date_format(date_string):
 
     return(new_date)
 
+def convert_date_to_int(date_string):
+    if re.search("[0-9]{4}-[0-9]{2}-[0-9]{2}",str(date_string)):
+        new_date = int(re.sub("-","",str(date_string)))
+    else:
+        new_date=date_string
+    return(new_date)
+
 def get_dashboard_update_date(base_url,headers):
     # Get update date
     update_date_url =  base_url + "RVD_UpdateDate.csv"
@@ -206,6 +213,11 @@ def get_positive_data(base_url,headers,update_date):
             assert all([0 <= val <= 100 or math.isnan(val) for val in  df[df.columns[k]]]), "Percentage not from 0-100"
 
     df = df.reset_index()
+    
+    # convert dates to integers
+    df['time_value'] = [convert_date_to_int(t) for t in df['time_value']]
+    df['issue'] = [convert_date_to_int(t) for t in df['issue']]
+
     return(df.set_index(['epiweek', 'time_value', 'issue', 'geo_type', 'geo_value'],verify_integrity=True))
 
 def get_detections_data(base_url,headers,update_date):
@@ -243,6 +255,10 @@ def get_detections_data(base_url,headers,update_date):
     df_detections=df_detections.rename(columns={'reportinglaboratory':"geo_value",'date':"time_value"})
     df_detections['geo_value'] = [abbreviate_geo(g) for g in df_detections['geo_value']]
     df_detections['geo_type'] = [create_geo_types(g,"lab") for g in df_detections['geo_value']]
+    
+    # convert dates to integers
+    df_detections['time_value'] = [convert_date_to_int(t) for t in df_detections['time_value']]
+    df_detections['issue'] = [convert_date_to_int(t) for t in df_detections['issue']]
 
     return(df_detections.set_index(['epiweek', 'time_value', 'issue', 'geo_type', 'geo_value'],verify_integrity=True))
 
@@ -336,13 +352,13 @@ def combine_tables(data_dict):
         detections=data_dict["respiratory_detection"]
 
         positive["epiweek"] = pd.to_numeric(positive["epiweek"],downcast="integer")
-        positive["time_value"] = pd.to_datetime(positive["time_value"])
-        positive["issue"] = pd.to_datetime(positive["issue"])
+        #positive["time_value"] = pd.to_datetime(positive["time_value"])
+        #positive["issue"] = pd.to_datetime(positive["issue"])
         positive['geo_type'] = [create_geo_types(g,'lab') for g in positive['geo_value']]
 
         detections["epiweek"] = pd.to_numeric(detections["epiweek"],downcast="integer")
-        detections["time_value"] = pd.to_datetime(detections["time_value"])
-        detections["issue"] = pd.to_datetime(detections["issue"])
+        #detections["time_value"] = pd.to_datetime(detections["time_value"])
+        #detections["issue"] = pd.to_datetime(detections["issue"])
         detections['geo_type'] = [create_geo_types(g,'lab') for g in detections['geo_value']]
 
         detections = expand_detections_columns(detections)