Run unit tests with one test database for all Pytest threads. #1436

Co-authored-by: Kyle MacMillan <[email protected]>
Co-authored-by: Michael Wellman <[email protected]>

.gitignore

@@ -91,3 +91,6 @@ user_flows_exit_code.txt
 # These files can get written in the notification_api directory if you run bash commands on a container with read-write access.
 .bash_history
 .python_history
+
+# pdb
+.local/

.talismanrc

@@ -1,50 +1,90 @@
-fileignoreconfig:
-- filename: app/va/mpi/mpi.py
-  checksum: d339cb418dcc278a42848c8307a9af271713c22a2691d595ebe779f3492bd978
-- filename: certs/VA-Internal-S2-ICA1-v1.cer
-  checksum: 286a7e438a844585e7d9653e17b1488a51867201d58400d324ea733678bdd92c
-- filename: certs/VA-Internal-S2-ICA10.cer
-  checksum: 207f80ceb7f76aefa88e4ba45674093dd8029df372f03a9b5d2b8fdf11c1f167
-- filename: certs/VA-Internal-S2-ICA12.cer
-  checksum: 489a015ee0ab38eaefd8f311f511df8489deca9aae4b0e6ca338f8bd2ff2f6d3
-- filename: certs/VA-Internal-S2-ICA13.cer
-  checksum: 52f517028038900a4a37b5232dda9972fcbf752f9763f16b4a2324866f7bdcd5
-- filename: certs/VA-Internal-S2-ICA14.cer
-  checksum: 9c3a935f9da7a0d6b1625b87834fa79c69c35b589cf4fd16b354867bd7c6f3aa
-- filename: certs/VA-Internal-S2-ICA15.cer
-  checksum: 0e7830ac6a0e3e1f6df3eb69dc3fece78e4c1a3092a2be7dcde33620200199d2
-- filename: certs/VA-Internal-S2-ICA16.cer
-  checksum: ea1dced1923ccb7246ec0e716720ce38f123b4efe879ab25f5036bc272a32e9f
-- filename: certs/VA-Internal-S2-ICA17.cer
-  checksum: 537f0486dc874faa86dfb4733167efbfbdcbaca9f8173f32f89316ce445f2d90
-- filename: certs/VA-Internal-S2-ICA18.cer
-  checksum: bedd1ac0f71844c68fba8798f88193bb0149725ab0337dee82292f3dcf11c989
-- filename: certs/VA-Internal-S2-ICA19.cer
-  checksum: 1c155d9d46baf3ae81a530f7ac66693afcad52ef31f8dc14eb0695ace51c9e0f
-- filename: certs/VA-Internal-S2-ICA2-v1.cer
-  checksum: 04748b31daa16fd4f75f38b13f00445b331ff0f380c985d6d6edfeaee92d956b
-- filename: certs/VA-Internal-S2-ICA20.cer
-  checksum: a8421fb7665aa58ced4b83ec26d8eeadb56058cf15048c2c81c22b07418d203d
-- filename: certs/VA-Internal-S2-ICA21.cer
-  checksum: fdfc26d44c7d6688f79b09ed3e22379e88c60fbfa891294f55cb2d7c2bbaf067
-- filename: certs/VA-Internal-S2-ICA3-v1.cer
-  checksum: 3db3c65e54219e10ae5ea675acf6a4679e879f864e49371f8209a358d788f606
-- filename: certs/VA-Internal-S2-ICA5.cer
-  checksum: 5ff6ad67daca90b034db642b1bf5bb7839a59e401403a59f29080cb96087d6ad
-- filename: certs/VA-Internal-S2-ICA6.cer
-  checksum: df2629b8c6e9bf64fec0261d1fc2e047fc5e22fb9bcb020c31f6d9dc0dd39e54
-- filename: certs/VA-Internal-S2-ICA7.cer
-  checksum: 89358e208a28fb4869f08773befd00b02963aafdd22057cb74a22ec53ae00068
-- filename: certs/VA-Internal-S2-ICA8.cer
-  checksum: 38a7c4c11ced82618867ad61af05c764add68dbd2c59c8c1515ee075dac2a5ae
-- filename: certs/VA-Internal-S2-ICA9.cer
-  checksum: 17557e726e93fb1983ec1dc340865d64fc039c8ec7113c12ade4b6c8f0c80758
-- filename: documents/postman/internal_api_developers/development.postman_environment.json
-  checksum: ca23e82ce6045c668de609cd48ee5bff986250c06198a262aebbd80e25c1ce65
-- filename: documents/postman/internal_api_developers/performance.postman_environment.json
-  checksum: a39e7244c02137f4c76df6fec596dbafeb3ba05796faa410e67ca6afff1f4099
-- filename: documents/postman/internal_api_developers/staging.postman_environment.json
-  checksum: a6b9dbff04ca357fed37bb800544195355555c5b5bc5f14e30965121403e5907
-- filename: documents/postman/staging-simplified.postman_environment.json
-  checksum: d69f7b3dfd46f9d8fe17c1ec8429fa1a90cd8acea452e2de2f0936f1c0013474
-version: "1.0"
+- filename: tests/app/dao/notification_dao/test_notification_dao_performance_platform.py
+  checksum: b3ffec0e69481a73091ea8388eff348842a82ce00d7f8c9a65dbd8498ef1b220
+- filename: tests/app/dao/notification_dao/test_notification_dao_delete_notifications.py
+  checksum: a31c6f7abed822d20b6371c96aaa3f2fd19200a8a2f8dac8240233986df80e0b
+- filename: tests/app/notifications/test_rest.py
+  checksum: 22a54f4187f7dfe404e4eefb677ff376933aff2f053362662c6d5b0b2eaad891
+- filename: tests/app/service/test_api_key_endpoints.py
+  checksum: a8268dd19d4394733a974db319c5a2353aad455ff1bab189949fa1a9a757e166
+- filename: tests/app/conftest.py
+  checksum: 96c0f09e85dc957d90da2728911d684744559fb08069c4d8763aa99329f26511
+- filename: tests/app/dao/test_api_key_dao.py
+  checksum: 9a97adf107dbbf0d821a5d210f19b55a27c8b22d0593be3224a10770cd78d4e1
+- filename: tests/app/service/test_rest.py
+  checksum: c4331cbc7d3c935ceca4fa94c40034348f00369e50540b02a36dbf6d98bc4fe9
+- filename: tests/app/dao/test_fido2_keys_dao.py
+  checksum: 35cba2fd6e81a4d9e55fa9ebe8fa6ad8fc1e597aede8495e386f8b7999b6f482
+- filename: tests/app/dao/notification_dao/test_notification_dao.py
+  checksum: b32000cfe4245fb5080d1744789913d53b73d68fb0573b6c2e6eaf424ae8255d
+- filename: tests/app/delivery/test_send_to_providers.py
+  checksum: 261192636154bcebca11e0991525270a124cd57f8c06d2f515c8077762662e3e
+- filename: tests/app/notifications/test_process_notifications.py
+  checksum: 4134fa90ed48750b6fc73ddd4cc0416ad21474296cdcc7ae75ada0df70e20a4c
+- filename: tests/app/notifications/test_validators.py
+  checksum: b81d29ae6ee3b88b429610261021069b388de1d50163c9f3176f7502c625a17a
+- filename: tests/app/notifications/test_receive_notification.py
+  checksum: 37abf7a3f23fc8acf022930b4fad1c42cb344b575837dec602bdedad51360190
+- filename: tests/app/dao/test_services_dao.py
+  checksum: 3ea2f8d01afd32d1c1586488aa442835b45895f7ee270b0da49d3134f37c7685
+- filename: tests/app/dao/test_fact_notification_status_dao.py
+  checksum: 2cad8d3b4a53bad53c3adeb9c81968d11b21e75d729976120bdd66ca91031738
+- filename: tests/app/dao/test_fido2_keys_dao.py
+  checksum: 35cba2fd6e81a4d9e55fa9ebe8fa6ad8fc1e597aede8495e386f8b7999b6f482
+- filename: tests/app/dao/notification_dao/test_notification_dao_performance_platform.py
+  checksum: b3ffec0e69481a73091ea8388eff348842a82ce00d7f8c9a65dbd8498ef1b220
+- filename: tests/app/dao/notification_dao/test_notification_dao_delete_notifications.py
+  checksum: a31c6f7abed822d20b6371c96aaa3f2fd19200a8a2f8dac8240233986df80e0b
+- filename: tests/app/notifications/test_rest.py
+  checksum: 22a54f4187f7dfe404e4eefb677ff376933aff2f053362662c6d5b0b2eaad891
+- filename: tests/app/notifications/test_process_notifications.py
+  checksum: 4134fa90ed48750b6fc73ddd4cc0416ad21474296cdcc7ae75ada0df70e20a4c
+- filename: tests/app/service/test_api_key_endpoints.py
+  checksum: a8268dd19d4394733a974db319c5a2353aad455ff1bab189949fa1a9a757e166
+- filename: tests/app/delivery/test_send_to_providers.py
+  checksum: 261192636154bcebca11e0991525270a124cd57f8c06d2f515c8077762662e3e
+- filename: tests/app/dao/test_api_key_dao.py
+  checksum: 9a97adf107dbbf0d821a5d210f19b55a27c8b22d0593be3224a10770cd78d4e1
+- filename: tests/app/notifications/test_validators.py
+  checksum: b81d29ae6ee3b88b429610261021069b388de1d50163c9f3176f7502c625a17a
+- filename: tests/app/dao/notification_dao/test_notification_dao.py
+  checksum: b32000cfe4245fb5080d1744789913d53b73d68fb0573b6c2e6eaf424ae8255d
+- filename: tests/app/service/test_rest.py
+  checksum: c4331cbc7d3c935ceca4fa94c40034348f00369e50540b02a36dbf6d98bc4fe9
+- filename: tests/app/notifications/test_receive_notification.py
+  checksum: 37abf7a3f23fc8acf022930b4fad1c42cb344b575837dec602bdedad51360190
+- filename: tests/app/conftest.py
+  checksum: 96c0f09e85dc957d90da2728911d684744559fb08069c4d8763aa99329f26511
+- filename: tests/app/dao/test_fact_notification_status_dao.py
+  checksum: 2cad8d3b4a53bad53c3adeb9c81968d11b21e75d729976120bdd66ca91031738
+- filename: tests/app/dao/test_services_dao.py
+  checksum: 3ea2f8d01afd32d1c1586488aa442835b45895f7ee270b0da49d3134f37c7685
+- filename: tests/app/dao/test_api_key_dao.py
+  checksum: 9a97adf107dbbf0d821a5d210f19b55a27c8b22d0593be3224a10770cd78d4e1
+- filename: tests/app/dao/test_fido2_keys_dao.py
+  checksum: 35cba2fd6e81a4d9e55fa9ebe8fa6ad8fc1e597aede8495e386f8b7999b6f482
+- filename: tests/app/dao/notification_dao/test_notification_dao_delete_notifications.py
+  checksum: a31c6f7abed822d20b6371c96aaa3f2fd19200a8a2f8dac8240233986df80e0b
+- filename: tests/app/conftest.py
+  checksum: 96c0f09e85dc957d90da2728911d684744559fb08069c4d8763aa99329f26511
+- filename: tests/app/service/test_rest.py
+  checksum: c4331cbc7d3c935ceca4fa94c40034348f00369e50540b02a36dbf6d98bc4fe9
+- filename: tests/app/notifications/test_rest.py
+  checksum: 22a54f4187f7dfe404e4eefb677ff376933aff2f053362662c6d5b0b2eaad891
+- filename: tests/app/service/test_api_key_endpoints.py
+  checksum: a8268dd19d4394733a974db319c5a2353aad455ff1bab189949fa1a9a757e166
+- filename: tests/app/dao/notification_dao/test_notification_dao_performance_platform.py
+  checksum: b3ffec0e69481a73091ea8388eff348842a82ce00d7f8c9a65dbd8498ef1b220
+- filename: tests/app/notifications/test_process_notifications.py
+  checksum: 4134fa90ed48750b6fc73ddd4cc0416ad21474296cdcc7ae75ada0df70e20a4c
+- filename: tests/app/dao/notification_dao/test_notification_dao.py
+  checksum: b32000cfe4245fb5080d1744789913d53b73d68fb0573b6c2e6eaf424ae8255d
+- filename: tests/app/delivery/test_send_to_providers.py
+  checksum: 261192636154bcebca11e0991525270a124cd57f8c06d2f515c8077762662e3e
+- filename: tests/app/notifications/test_validators.py
+  checksum: b81d29ae6ee3b88b429610261021069b388de1d50163c9f3176f7502c625a17a
+- filename: tests/app/dao/test_fact_notification_status_dao.py
+  checksum: 2cad8d3b4a53bad53c3adeb9c81968d11b21e75d729976120bdd66ca91031738
+- filename: tests/app/notifications/test_receive_notification.py
+  checksum: 37abf7a3f23fc8acf022930b4fad1c42cb344b575837dec602bdedad51360190
+- filename: tests/app/dao/test_services_dao.py
+  checksum: 3ea2f8d01afd32d1c1586488aa442835b45895f7ee270b0da49d3134f37c7685

app/__init__.py

@@ -89,26 +89,25 @@
 authenticated_service = LocalProxy(lambda: g.authenticated_service)
 
 
-def create_app(
-    application,
-    worker_id=None,
-):
+def create_app(application):
+    """
+    https://flask.palletsprojects.com/en/2.3.x/tutorial/factory/
+    """
+
     from app.config import configs
 
     notify_environment = os.getenv('NOTIFY_ENVIRONMENT', 'development')
 
     application.config.from_object(configs[notify_environment])
     if notify_environment == 'test':
-        assert worker_id is not None
-        application.config['SQLALCHEMY_DATABASE_URI'] += f'_{worker_id}'
-        # set read-db to be the same as write/default instance for testing
+        # Set the read-db to be the same as the write/default instance.
         application.config['SQLALCHEMY_BINDS'] = {'read-db': application.config['SQLALCHEMY_DATABASE_URI']}
-        assert (
-            'test_notification_api' in application.config['SQLALCHEMY_DATABASE_URI']
-        ), "Don't run tests against the main database."
-        assert (
-            'test_notification_api' in application.config['SQLALCHEMY_BINDS']['read-db']
-        ), "Don't run tests against the main database."
+        assert application.config['SQLALCHEMY_DATABASE_URI'].endswith(
+            'test_notification_api'
+        ), "Don't run tests against the main writer database."
+        assert application.config['SQLALCHEMY_BINDS']['read-db'].endswith(
+            'test_notification_api'
+        ), "Don't run tests against the main reader database."
 
     application.config['NOTIFY_APP_NAME'] = application.name
     init_app(application)

app/celery/nightly_tasks.py

@@ -107,7 +107,6 @@ def timeout_notifications():
     technical_failure_notifications, temporary_failure_notifications = dao_timeout_notifications(
         current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD')
     )
-
     notifications = technical_failure_notifications + temporary_failure_notifications
     for notification in notifications:
         # queue callback task only if the service_callback_api exists
@@ -225,13 +224,10 @@ def raise_alert_if_letter_notifications_still_sending():
     today = datetime.utcnow().date()
 
     # Do nothing on the weekend
-    if today.isoweekday() in [6, 7]:
+    if today.isoweekday() in (6, 7):
         return
 
-    if today.isoweekday() in [1, 2]:
-        offset_days = 4
-    else:
-        offset_days = 2
+    offset_days = 4 if (today.isoweekday() in [1, 2]) else 2
 
     stmt = (
         select([func.count()])
@@ -249,7 +245,8 @@ def raise_alert_if_letter_notifications_still_sending():
         message = "There are {} letters in the 'sending' state from {}".format(
             still_sending, (today - timedelta(days=offset_days)).strftime('%A %d %B')
         )
-        # Only send alerts in production
+
+        # Only send alerts in production.
         if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
             zendesk_client.create_ticket(
                 subject='[{}] Letters still sending'.format(current_app.config['NOTIFY_ENVIRONMENT']),

app/celery/scheduled_tasks.py

@@ -357,3 +357,37 @@ def send_scheduled_comp_and_pen_sms():
                 type(e),
                 e,
             )
+
+
+def _get_dynamodb_comp_pen_messages(table, message_limit: int) -> list:
+    """
+    Helper function to get the Comp and Pen data from our dynamodb cache table.
+
+    :param table: the dynamodb table to grab the data from
+    :param message_limit: the number of rows to search at a time and the max number of items that should be returned
+    :return: a list of entries from the table that have not been processed yet
+    """
+
+    results = table.scan(FilterExpression=boto3.dynamodb.conditions.Attr('is_processed').eq(False), Limit=message_limit)
+
+    items: list = results.get('Items')
+
+    if items is None:
+        current_app.logger.critical(
+            'Error in _get_dynamodb_comp_pen_messages trying to read "Items" from dynamodb table scan result. '
+            'Returned results does not include "Items" - results: %s',
+            results,
+        )
+        return []
+
+    # Keep getting items from the table until we have the number we want to send, or run out of items
+    while 'LastEvaluatedKey' in results and len(items) < message_limit:
+        results = table.scan(
+            FilterExpression=boto3.dynamodb.conditions.Attr('is_processed').eq(False),
+            Limit=message_limit,
+            ExclusiveStartKey=results['LastEvaluatedKey'],
+        )
+
+        items.extend(results['Items'])
+
+    return items[:message_limit]