Implemented import status

Author: George Schneeloch

importer/tests/test_forms.py

@@ -5,6 +5,7 @@
 
 import logging
 
+from django.contrib.sessions.backends.cache import SessionStore
 from django.core.files.storage import default_storage
 from django.core.files.uploadedfile import InMemoryUploadedFile
 
@@ -36,7 +37,7 @@ def test_init(self):
             {"course_file": self.get_upload_file(".zip")},
         )
         self.assertTrue(form.is_valid())
-        form.save(self.user.id, self.repo.id)
+        form.save(self.user.id, self.repo.id, SessionStore())
 
     def test_extensions(self):
         """Only certain extensions are valid."""

rest/tasks.py

@@ -17,6 +17,7 @@
 TASK_KEY = 'tasks'
 EXPORT_TASK_TYPE = 'resource_export'
 EXPORTS_KEY = 'learning_resource_exports'
+IMPORT_TASK_TYPE = 'import_course'
 
 
 def create_initial_task_dict(task, task_type, task_info):

ui/forms.py

@@ -16,6 +16,7 @@
 
 from importer.tasks import import_file
 from learningresources.models import Repository
+from rest.tasks import track_task, IMPORT_TASK_TYPE
 
 log = logging.getLogger(__name__)
 
@@ -45,13 +46,14 @@ def clean_course_file(self):
         log.debug("got to end, so the file is bad")
         raise ValidationError("Unsupported file type.")
 
-    def save(self, user_id, repo_id):
+    def save(self, user_id, repo_id, session):
         """
         Receives the request.FILES from the view.
 
         Args:
             user_id (int): primary key of the user uploading the course.
-            repo_id (int): primary key of repository we're uploading to
+            repo_id (int): primary key of repository we're uploading to.
+            session (SessionStore): User's session to store task data.
         """
         # Assumes a single file, because we only accept
         # one at a time.
@@ -66,7 +68,14 @@ def save(self, user_id, repo_id):
             ),
             uploaded_file
         )
-        import_file.delay(path, repo_id, user_id)
+        task = import_file.delay(path, repo_id, user_id)
+
+        repo_slug = Repository.objects.get(id=repo_id).slug
+        # Save task data in session so we can keep track of it.
+        track_task(session, task, IMPORT_TASK_TYPE, {
+            "repo_slug": repo_slug,
+            "path": path
+        })
 
 
 class RepositoryForm(ModelForm):

ui/static/ui/js/import_status.jsx

@@ -0,0 +1,112 @@
+define("import_status", ["React", "utils"], function(React, Utils) {
+  var ImportStatus = React.createClass({
+    getInitialState: function() {
+      return {
+        importTasks: {},
+        hasRefreshed: false
+      };
+    },
+    updateImportStatus: function() {
+      var thiz = this;
+
+      var oldImportTasks = this.state.importTasks;
+
+      Utils.getCollection("/api/v1/tasks/").then(function(tasks) {
+        // Only deal with tasks on this repo that deal with imports.
+        var tasksMap = {};
+        _.each(tasks, function(task) {
+          if (task.task_type === 'import_course' &&
+            task.task_info.repo_slug === thiz.props.repoSlug) {
+            tasksMap[task.id] = task;
+          }
+        });
+        thiz.setState({importTasks: tasksMap});
+
+        var notProcessing = _.filter(tasks, function(task) {
+          return task.status === 'success' || task.status === 'failure';
+        });
+
+        var deferred = $.Deferred();
+
+        // Delete one after another to avoid race conditions with
+        // Django session.
+        _.each(notProcessing, function(task) {
+          deferred.then(function() {
+            return $.ajax({
+              method: 'DELETE',
+              url: '/api/v1/tasks/' + task.id + '/'
+            });
+          });
+        });
+        deferred.resolve();
+
+        var numProcessing = _.filter(tasks, function(task) {
+          return task.status === 'processing';
+        }).length;
+        if (numProcessing > 0) {
+          thiz.setState({hasRefreshed: true});
+          setTimeout(thiz.updateImportStatus, 3000);
+        } else {
+          if (thiz.state.hasRefreshed) {
+            // There were tasks and they're finished now so we should refresh.
+            thiz.props.refreshFromAPI();
+          }
+        }
+      });
+    },
+    componentDidMount: function() {
+      this.updateImportStatus();
+    },
+    render: function() {
+      var numSuccessful = _.filter(this.state.importTasks, function(task) {
+        return task.status === 'success';
+      }).length;
+      var numProcessing = _.filter(this.state.importTasks, function(task) {
+        return task.status === 'processing';
+      }).length;
+
+      var importWord = function(n) {
+        if (n === 1) {
+          return "import";
+        } else {
+          return "imports";
+        }
+      };
+
+      var messages = [];
+      if (numSuccessful !== 0) {
+        messages.push(
+          numSuccessful + ' ' + importWord(numSuccessful) + ' finished');
+      }
+      if (numProcessing !== 0) {
+        messages.push(
+          numProcessing + ' ' + importWord(numProcessing) +
+          ' still processing'
+        );
+      }
+
+      var failed = _.filter(this.state.importTasks, function(task) {
+        return task.status === 'failure';
+      });
+      var errorMessages = [];
+      _.each(failed, function(task) {
+        errorMessages.push("Import failed: " + task.result.error);
+        errorMessages.push(<br />);
+      });
+
+      var messageString = '';
+      if (messages.length > 0) {
+        messageString = messages.join(", ") + ".";
+      }
+
+      return <span>
+        <p>{messageString}</p>
+        <p>{errorMessages}</p>
+      </span>;
+    }
+  });
+
+  return {
+    ImportStatus: ImportStatus
+  };
+});

ui/static/ui/js/listing.js

@@ -1,11 +1,11 @@
 define('listing',
   ['csrf', 'react', 'jquery', 'lodash', 'uri', 'history', 'manage_taxonomies',
     'learning_resources', 'static_assets', 'utils',
-    'lr_exports', 'listing_resources', 'xml_panel',
+    'lr_exports', 'listing_resources', 'xml_panel', 'import_status',
     'bootstrap', 'icheck'],
   function (CSRF, React, $, _, URI, History,
             ManageTaxonomies, LearningResources, StaticAssets,
-            Utils, Exports, ListingResources, XmlPanel) {
+            Utils, Exports, ListingResources, XmlPanel, ImportStatusModule) {
     'use strict';
 
     var EMAIL_EXTENSION = '@mit.edu';
@@ -184,7 +184,13 @@ define('listing',
           facetCounts: this.state.facetCounts,
           ref: "listingResources"
         };
-        return React.createElement(ListingResources.ListingPage, options);
+        return React.DOM.div(null,
+          React.createElement(ImportStatusModule.ImportStatus, {
+            repoSlug: this.props.repoSlug,
+            refreshFromAPI: this.refreshFromAPI
+          }),
+          React.createElement(ListingResources.ListingPage, options)
+        );
       },
       /**
        * Clears exports on page. Assumes DELETE to clear on server already

ui/static/ui/js/require_config.js

@@ -5,7 +5,7 @@ var REQUIRE_PATHS = {
   bootstrap: "bootstrap/dist/js/bootstrap.min",
   icheck: "icheck/icheck.min",
   retina: "retina.js/dist/retina.min",
-  React: "react/react-with-addons.min",
+  React: "react/react-with-addons",
   react_infinite: "react-infinite/dist/react-infinite.min",
   react_datagrid: "../lib/js/react-datagrid.min",
   lodash: "lodash/lodash.min",
@@ -21,6 +21,7 @@ var REQUIRE_PATHS = {
   listing: "../ui/js/listing",
   manage_taxonomies: "../ui/js/manage_taxonomies.jsx?noext",
   xml_panel: "../ui/js/xml_panel.jsx?noext",
+  import_status: "../ui/js/import_status.jsx?noext",
   learning_resources: "../ui/js/learning_resources.jsx?noext",
   listing_resources: "../ui/js/listing_resources.jsx?noext",
   static_assets: "../ui/js/static_assets.jsx?noext",

ui/templates/repository.html

@@ -60,6 +60,9 @@
 <script type="text/jsx"
         src="{% static "ui/js/xml_panel.jsx" %}">
 </script>
+<script type="text/jsx"
+        src="{% static "ui/js/import_status.jsx" %}">
+</script>
 <script type="text/jsx"
         src="{% static "ui/js/pagination.jsx" %}">
 </script>

ui/views.py

@@ -63,7 +63,7 @@ def upload(request, repo_slug):
             data=request.POST, files=request.FILES)
         if form.is_valid():
             try:
-                form.save(request.user.id, repo.id)
+                form.save(request.user.id, repo.id, request.session)
                 return redirect("/repositories/{0}/".format(repo_slug))
             except ValueError as ex:
                 # Coverage exception reasoning: After successful upload,