@@ -47,62 +47,122 @@ def setUp(self):
4747 cur .execute ('delete from api_user' )
4848 cur .execute ('insert into api_user(api_key, email) values("key", "email")' )
4949
50- @freeze_time ("2021-03-16" )
51- def test_acquire_dataset (self ):
52- """Acquire a new dataset."""
50+ def get_modified_dataset (self , critical_staffing_shortage_today_yes , reporting_cutoff_start ):
51+ """Get a simplified version of a test dataset.
5352
54- # make sure the data does not yet exist
55- with self .subTest (name = 'no data yet' ):
56- response = Epidata .covid_hosp ('MA' , Epidata .range (20200101 , 20210101 ))
57- self .assertEqual (response ['result' ], - 2 , response )
53+ Only WY data is modified. The issue date is specified in the metadata file.
54+ """
55+ df = self .test_utils .load_sample_dataset ()
56+ df_new = pd .DataFrame (df [df ["state" ] == "WY" ], columns = df .columns ).reset_index (drop = True )
57+ df_new ["critical_staffing_shortage_today_yes" ] = critical_staffing_shortage_today_yes
58+ df_new ["reporting_cutoff_start" ] = reporting_cutoff_start
59+ return df_new
5860
59- # acquire sample data into local database
60- # mock out network calls to external hosts
61- with self .subTest (name = 'first acquisition' ), \
62- patch .object (Network , 'fetch_metadata' , return_value = self .test_utils .load_sample_metadata ()) as mock_fetch_meta , \
63- patch .object (Network , 'fetch_dataset' , side_effect = [self .test_utils .load_sample_dataset ("dataset0.csv" ), # dataset for 3/13
64- self .test_utils .load_sample_dataset ("dataset0.csv" ), # first dataset for 3/15
65- self .test_utils .load_sample_dataset ()] # second dataset for 3/15
66- ) as mock_fetch :
67- acquired = Update .run ()
68- self .assertTrue (acquired )
69- self .assertEqual (mock_fetch_meta .call_count , 1 )
70-
71- # make sure the data now exists
72- with self .subTest (name = 'initial data checks' ):
73- response = Epidata .covid_hosp ('WY' , Epidata .range (20200101 , 20210101 ))
74- self .assertEqual (response ['result' ], 1 )
75- self .assertEqual (len (response ['epidata' ]), 1 )
76- row = response ['epidata' ][0 ]
77- self .assertEqual (row ['state' ], 'WY' )
78- self .assertEqual (row ['date' ], 20201209 )
79- self .assertEqual (row ['issue' ], 20210315 )
80- self .assertEqual (row ['critical_staffing_shortage_today_yes' ], 8 )
81- self .assertEqual (row ['total_patients_hospitalized_confirmed_influenza_covid_coverage' ], 56 )
82- actual = row ['inpatient_bed_covid_utilization' ]
83- expected = 0.11729857819905214
84- self .assertAlmostEqual (actual , expected )
85- self .assertIsNone (row ['critical_staffing_shortage_today_no' ])
86-
87- # expect 61 fields per row (63 database columns, except `id` and `record_type`)
88- self .assertEqual (len (row ), 118 )
89-
90- with self .subTest (name = 'all date batches acquired' ):
91- response = Epidata .covid_hosp ('WY' , Epidata .range (20200101 , 20210101 ), issues = 20210313 )
92- self .assertEqual (response ['result' ], 1 )
93-
94- # re-acquisition of the same dataset should be a no-op
95- with self .subTest (name = 'second acquisition' ), \
96- patch .object (Network , 'fetch_metadata' , return_value = self .test_utils .load_sample_metadata ()) as mock_fetch_meta , \
97- patch .object (Network , 'fetch_dataset' , return_value = self .test_utils .load_sample_dataset ()) as mock_fetch :
98- acquired = Update .run ()
99- self .assertFalse (acquired )
61+ def test_acquire_dataset (self ):
62+ """Acquire a new dataset."""
10063
101- # make sure the data still exists
102- with self .subTest (name = 'final data checks' ):
103- response = Epidata .covid_hosp ('WY' , Epidata .range (20200101 , 20210101 ))
104- self .assertEqual (response ['result' ], 1 )
105- self .assertEqual (len (response ['epidata' ]), 1 )
64+ with freeze_time ("2021-03-15" ):
65+ # make sure the data does not yet exist
66+ with self .subTest (name = 'no data yet' ):
67+ response = Epidata .covid_hosp ('MA' , Epidata .range (20200101 , 20210101 ))
68+ self .assertEqual (response ['result' ], - 2 , response )
69+
70+ # acquire sample data into local database
71+ # mock out network calls to external hosts
72+ # issues: 3/13, 3/15
73+ with self .subTest (name = 'first acquisition' ), \
74+ patch .object (Network , 'fetch_metadata' ,
75+ return_value = self .test_utils .load_sample_metadata ("metadata.csv" )) as mock_fetch_meta , \
76+ patch .object (Network , 'fetch_dataset' , side_effect = [
77+ self .test_utils .load_sample_dataset (),
78+ self .test_utils .load_sample_dataset ()
79+ ]) as mock_fetch :
80+ acquired = Update .run ()
81+ self .assertTrue (acquired )
82+ self .assertEqual (mock_fetch_meta .call_count , 1 )
83+
84+ # make sure the data now exists
85+ with self .subTest (name = 'initial data checks' ):
86+ response = Epidata .covid_hosp ('WY' , Epidata .range (20200101 , 20210101 ))
87+ self .assertEqual (response ['result' ], 1 )
88+ self .assertEqual (len (response ['epidata' ]), 1 )
89+ row = response ['epidata' ][0 ]
90+ self .assertEqual (row ['state' ], 'WY' )
91+ self .assertEqual (row ['date' ], 20201209 )
92+ self .assertEqual (row ['issue' ], 20210315 ) # include today's data by default
93+ self .assertEqual (row ['critical_staffing_shortage_today_yes' ], 8 )
94+ self .assertEqual (row ['total_patients_hospitalized_confirmed_influenza_covid_coverage' ], 56 )
95+ self .assertIsNone (row ['critical_staffing_shortage_today_no' ])
96+
97+ # expect 61 fields per row (63 database columns, except `id` and `record_type`)
98+ self .assertEqual (len (row ), 118 )
99+
100+ with self .subTest (name = 'all date batches acquired' ):
101+ response = Epidata .covid_hosp ('WY' , Epidata .range (20200101 , 20210101 ), issues = 20210313 )
102+ self .assertEqual (response ['result' ], 1 )
103+
104+ # re-acquisition of the same dataset should be a no-op
105+ # issues: 3/13, 3/15
106+ with self .subTest (name = 'second acquisition' ), \
107+ patch .object (Network , 'fetch_metadata' ,
108+ return_value = self .test_utils .load_sample_metadata ("metadata.csv" )) as mock_fetch_meta , \
109+ patch .object (Network , 'fetch_dataset' , side_effect = [
110+ self .test_utils .load_sample_dataset (),
111+ self .test_utils .load_sample_dataset ()
112+ ]) as mock_fetch :
113+ acquired = Update .run ()
114+ self .assertFalse (acquired )
115+
116+ # make sure the data still exists
117+ response = Epidata .covid_hosp ('WY' , Epidata .range (20200101 , 20210101 ))
118+ self .assertEqual (response ['result' ], 1 )
119+ self .assertEqual (len (response ['epidata' ]), 1 )
120+
121+ with freeze_time ("2021-03-16" ):
122+ # simulate issue posted after yesterday's run
123+ with self .subTest (name = 'late issue posted' ), \
124+ patch .object (Network , 'fetch_metadata' ,
125+ return_value = self .test_utils .load_sample_metadata ("metadata2.csv" )) as mock_fetch_meta , \
126+ patch .object (Network , 'fetch_dataset' , side_effect = [
127+ self .get_modified_dataset (critical_staffing_shortage_today_yes = 9 , reporting_cutoff_start = "2020-12-09" ),
128+ self .get_modified_dataset (critical_staffing_shortage_today_yes = 10 , reporting_cutoff_start = "2020-12-09" ),
129+ self .get_modified_dataset (critical_staffing_shortage_today_yes = 11 , reporting_cutoff_start = "2020-12-10" ),
130+ self .get_modified_dataset (critical_staffing_shortage_today_yes = 12 , reporting_cutoff_start = "2020-12-10" ),
131+ ]) as mock_fetch :
132+ acquired = Update .run ()
133+ self .assertTrue (acquired )
134+ self .assertEqual (mock_fetch_meta .call_count , 1 )
135+
136+ # make sure everything was filed correctly
137+ with self .subTest (name = 'late issue data checks' ):
138+ response = Epidata .covid_hosp ('WY' , Epidata .range (20200101 , 20210101 ))
139+ self .assertEqual (response ['result' ], 1 )
140+ self .assertEqual (len (response ['epidata' ]), 2 )
141+
142+ # should have data from 03-15 00:00:01AM
143+ row = response ['epidata' ][0 ]
144+ self .assertEqual (row ['state' ], 'WY' )
145+ self .assertEqual (row ['date' ], 20201209 )
146+ self .assertEqual (row ['issue' ], 20210315 ) # include today's data by default
147+ self .assertEqual (row ['critical_staffing_shortage_today_yes' ], 10 )
148+ self .assertEqual (row ['total_patients_hospitalized_confirmed_influenza_covid_coverage' ], 56 )
149+ self .assertIsNone (row ['critical_staffing_shortage_today_no' ])
150+
151+ # should have data from 03-16 00:00:01AM
152+ row = response ['epidata' ][1 ]
153+ self .assertEqual (row ['state' ], 'WY' )
154+ self .assertEqual (row ['date' ], 20201210 )
155+ self .assertEqual (row ['issue' ], 20210316 ) # include today's data by default
156+ self .assertEqual (row ['critical_staffing_shortage_today_yes' ], 12 )
157+ self .assertEqual (row ['total_patients_hospitalized_confirmed_influenza_covid_coverage' ], 56 )
158+ self .assertIsNone (row ['critical_staffing_shortage_today_no' ])
159+
160+ # expect 61 fields per row (63 database columns, except `id` and `record_type`)
161+ self .assertEqual (len (row ), 118 )
162+
163+ with self .subTest (name = 'all date batches acquired' ):
164+ response = Epidata .covid_hosp ('WY' , Epidata .range (20200101 , 20210101 ), issues = 20210316 )
165+ self .assertEqual (response ['result' ], 1 )
106166
107167
108168 @freeze_time ("2021-03-16" )
@@ -121,7 +181,7 @@ def test_acquire_specific_issue(self):
121181 self .assertEqual (pre_max_issue , pd .Timestamp ('1900-01-01 00:00:00' ))
122182 with self .subTest (name = 'first acquisition' ), \
123183 patch .object (Network , 'fetch_metadata' , return_value = self .test_utils .load_sample_metadata ()) as mock_fetch_meta , \
124- patch .object (Network , 'fetch_dataset' , side_effect = [self .test_utils .load_sample_dataset ("dataset0.csv" )]
184+ patch .object (Network , 'fetch_dataset' , side_effect = [self .test_utils .load_sample_dataset ()]
125185 ) as mock_fetch :
126186 acquired = Utils .update_dataset (Database ,
127187 Network ,
0 commit comments