Filter canvas etl to single course (#2368)

Author: abeglova

learning_resources/management/commands/backpopulate_canvas_courses.py

@@ -20,10 +20,25 @@ def add_arguments(self, parser):
             help="Force regenerate existing summaries/flashcards",
         )
 
+        parser.add_argument(
+            "--canvas-ids",
+            dest="canvas_ids",
+            required=False,
+            help="""
+              If set, backpopulate only the canvas courses with these ids.
+              The canvas id is the number in the url for the course on canvas
+              or the numerical part at the start of the readable_id.
+              Example: https://canvas.mit.edu/courses/1234567 -> 1234567
+              Example: readable_id 1234567-foobar -> 1234567
+            """,
+        )
+
     def handle(self, *args, **options):  # noqa: ARG002
         """Populate Canvas courses from S3"""
+        canvas_ids = options["canvas_ids"].split(",") if options["canvas_ids"] else None
 
         task = sync_canvas_courses.delay(
+            canvas_course_ids=canvas_ids,
             overwrite=options["force_overwrite"],
         )
         self.stdout.write(f"Started task {task} to get courses from Canvas")

learning_resources/tasks.py

@@ -481,7 +481,7 @@ def ingest_canvas_course(archive_path, overwrite):
 
 
 @app.task(acks_late=True)
-def sync_canvas_courses(overwrite):
+def sync_canvas_courses(canvas_course_ids, overwrite):
     """
     Sync all canvas course files
 
@@ -494,12 +494,15 @@ def sync_canvas_courses(overwrite):
     exports = bucket.objects.filter(Prefix=s3_prefix)
     log.info("syncing all canvas courses")
     latest_archives = {}
+
     for archive in exports:
         key = archive.key
         course_folder = key.lstrip(settings.CANVAS_COURSE_BUCKET_PREFIX).split("/")[0]
         log.info("processing course folder %s", course_folder)
-        if course_folder not in latest_archives or (
-            max(archive.last_modified, latest_archives[course_folder].last_modified)
+
+        if (not canvas_course_ids or course_folder in canvas_course_ids) and (
+            (course_folder not in latest_archives)
+            or max(archive.last_modified, latest_archives[course_folder].last_modified)
             == archive.last_modified
         ):
             latest_archives[course_folder] = archive
@@ -513,12 +516,14 @@ def sync_canvas_courses(overwrite):
             overwrite=overwrite,
         )
         canvas_readable_ids.append(resource_readable_id)
-    stale_courses = LearningResource.objects.filter(
-        etl_source=ETLSource.canvas.name
-    ).exclude(readable_id__in=canvas_readable_ids)
-    stale_courses.update(test_mode=False, published=False)
-    [resource_unpublished_actions(resource) for resource in stale_courses]
-    stale_courses.delete()
+
+    if not canvas_course_ids:
+        stale_courses = LearningResource.objects.filter(
+            etl_source=ETLSource.canvas.name
+        ).exclude(readable_id__in=canvas_readable_ids)
+        stale_courses.update(test_mode=False, published=False)
+        [resource_unpublished_actions(resource) for resource in stale_courses]
+        stale_courses.delete()
 
 
 @app.task(bind=True)

learning_resources/tasks_test.py

@@ -611,9 +611,8 @@ def test_scrape_marketing_pages(mocker, settings, mocked_celery):
     mock_group.assert_called_once()
 
 
-def test_sync_canvas_courses_removes_stale_resources(
-    settings, mocker, django_assert_num_queries
-):
+@pytest.mark.parametrize("canvas_ids", [["1"], None])
+def test_sync_canvas_courses(settings, mocker, django_assert_num_queries, canvas_ids):
     """
     sync_canvas_courses should unpublish and delete stale canvas LearningResources
     """
@@ -622,10 +621,10 @@ def test_sync_canvas_courses_removes_stale_resources(
     mocker.patch("learning_resources.tasks.get_learning_course_bucket")
     mock_bucket = mocker.Mock()
     mock_archive1 = mocker.Mock()
-    mock_archive1.key = "canvas/course1/archive1.zip"
+    mock_archive1.key = "canvas/1/archive1.zip"
     mock_archive1.last_modified = now_in_utc()
     mock_archive2 = mocker.Mock()
-    mock_archive2.key = "canvas/course2/archive2.zip"
+    mock_archive2.key = "canvas/2/archive2.zip"
     mock_archive2.last_modified = now_in_utc() - timedelta(days=1)
     mock_bucket.objects.filter.return_value = [mock_archive1, mock_archive2]
     mocker.patch(
@@ -657,14 +656,22 @@ def test_sync_canvas_courses_removes_stale_resources(
     )
 
     # Patch ingest_canvas_course to return the readable_ids for the two non-stale courses
-    mocker.patch(
+    mock_ingest_course = mocker.patch(
         "learning_resources.tasks.ingest_canvas_course",
         side_effect=[("course1", lr1.runs.first()), ("course2", lr2.runs.first())],
     )
-    sync_canvas_courses(overwrite=False)
+    sync_canvas_courses(canvas_course_ids=canvas_ids, overwrite=False)
 
     # The stale course should be unpublished and deleted
-    assert not LearningResource.objects.filter(id=lr_stale.id).exists()
+    if canvas_ids:
+        assert LearningResource.objects.filter(id=lr_stale.id).exists()
+    else:
+        assert not LearningResource.objects.filter(id=lr_stale.id).exists()
     # The non-stale courses should still exist
     assert LearningResource.objects.filter(id=lr1.id).exists()
     assert LearningResource.objects.filter(id=lr2.id).exists()
+
+    if canvas_ids:
+        assert mock_ingest_course.call_count == 1
+    else:
+        assert mock_ingest_course.call_count == 2