ISSUE-195: CSV queue worker and nested CSVs for EAD + an EAD plugin

src/Form/AmiMultiStepIngest.php

@@ -74,19 +74,24 @@ public function __construct(PrivateTempStoreFactory $temp_store_factory, Session
    */
   public function buildForm(array $form, FormStateInterface $form_state) {
     $form = parent::buildForm($form, $form_state);
+
     $form['message-step'] = [
-      '#markup' => '<div class="step">' . $this->t('AMI step @step of @laststep',[
+      '#markup' => '<div class="step">' . $this->t('AMI step @step of @laststep', [
           '@step' => $this->step,
           '@laststep' => $this->lastStep,
         ]) . '</div>',
     ];
+    $plugin_instance = NULL;
+    $pluginValue = NULL;
     if ($this->step == 1) {
       $pluginValue = $this->store->get('plugin');
       $definitions = $this->importerManager->getDefinitions();
       $options = [];
       foreach ($definitions as $id => $definition) {
         $options[$id] = $definition['label'];
       }
+      // Reset the plugin instance in case of code change.
+      $this->store->set('plugininstance', NULL);
 
       $form['plugin'] = [
         '#type' => 'select',
@@ -98,29 +103,32 @@ public function buildForm(array $form, FormStateInterface $form_state) {
         '#empty_option' => $this->t('- Please select a plugin -'),
       ];
     }
+    if ($this->step > 1) {
+      //Only carry the plugin instance
+      /* @var $plugin_instance \Drupal\ami\Plugin\ImporterAdapterInterface | NULL */
+      $plugin_instance = $this->store->get('plugininstance');
+      $pluginValue = $this->store->get('plugin');
+    }
     if ($this->step == 2) {
       $parents = ['pluginconfig'];
       $form_state->setValue('pluginconfig', $this->store->get('pluginconfig'));
-      $pluginValue = $this->store->get('plugin');
       // Only create a new instance if we do not have the PluginInstace around
-      /* @var $plugin_instance \Drupal\ami\Plugin\ImporterAdapterInterface | NULL */
-      $plugin_instance = $this->store->get('plugininstance');
       if (!$plugin_instance || $plugin_instance->getPluginid() != $pluginValue || $pluginValue == NULL) {
         $configuration = [];
         $configuration['config'] = ImporterAdapter::create();
-        $plugin_instance = $this->importerManager->createInstance($pluginValue,$configuration);
-        $this->store->set('plugininstance',$plugin_instance);
+        $plugin_instance = $this->importerManager->createInstance($pluginValue, $configuration);
+        $this->store->set('plugininstance', $plugin_instance);
       }
       $form['pluginconfig'] = $plugin_instance->interactiveForm($parents, $form_state);
       $form['pluginconfig']['#tree'] = TRUE;
     }
-    // TO keep this discrete and easier to edit maybe move to it's own method?
+
+    // To keep this discrete and easier to edit maybe move to their own methods?
     if ($this->step == 3) {
       // We should never reach this point if data is not enough. Submit handler
       // will go back to Step 2 if so.
       $data = $this->store->get('data') ?? [];
       $pluginconfig = $this->store->get('pluginconfig');
-      $plugin_instance = $this->store->get('plugininstance');
       $op = $pluginconfig['op'];
       $column_keys = $plugin_instance->provideKeys($pluginconfig, $data);
       $mapping = $this->store->get('mapping');
@@ -129,13 +137,17 @@ public function buildForm(array $form, FormStateInterface $form_state) {
         'template' => 'Template',
       ];
       $template = $this->getMetadatadisplays();
-      // $webform = $this->getWebforms();
       $bundle = $this->getBundlesAndFields();
 
       $global_metadata_options = $metadata + ['custom' => 'Custom (Expert Mode)'];
       //Each row (based on its type column) can have its own approach setup(expert mode)
       $element_conditional = [];
       $element = [];
+      // Get all headers and check for a 'type' key first, if not allow the user to select one?
+      // Wonder if we can be strict about this and simply require always a "type"?
+      // @TODO WE need to check for 'type' always. Maybe even in the submit handler?
+      $alltypes = $plugin_instance->provideTypes($pluginconfig, $data);
+
       $element['bundle'] =[
         '#type' => 'select',
         '#title' => $this->t('Fields and Bundles'),
@@ -150,15 +162,6 @@ public function buildForm(array $form, FormStateInterface $form_state) {
         '#description' => $this->t('Columns will be casted to ADO metadata (JSON) using a Twig template setup for JSON output'),
       ];
 
-      /**
-       * $element_conditional['webform'] = [
-       *   '#type' => 'select',
-       *   '#title' => $this->t('Webform'),
-       *   '#options' => $webform,
-       *   '#description' => $this->t('Columns are casted to ADO metadata (JSON) by passing/validating Data through an existing Webform'),
-       * ];
-       */
-
       $form['ingestsetup']['globalmapping'] = [
         '#type' => 'select',
         '#title' => $this->t('Select the data transformation approach'),
@@ -202,10 +205,6 @@ public function buildForm(array $form, FormStateInterface $form_state) {
         ],
       ];
 
-      // Get all headers and check for a 'type' key first, if not allow the user to select one?
-      // Wonder if we can be strict about this and simply require always a "type"?
-      // @TODO WE need to check for 'type' always. Maybe even in the submit handler?
-      $alltypes = $plugin_instance->provideTypes($pluginconfig, $data);
       if (!empty($alltypes)) {
         $form['ingestsetup']['custommapping'] = [
           '#type' => 'fieldset',
@@ -296,7 +295,6 @@ public function buildForm(array $form, FormStateInterface $form_state) {
       $data = $this->store->get('data') ?? [];
       $pluginconfig = $this->store->get('pluginconfig');
       $op = $pluginconfig['op'];
-      $plugin_instance = $this->store->get('plugininstance');
       $column_keys = $plugin_instance->provideKeys($pluginconfig, $data);
       $column_options = array_combine($column_keys, $column_keys);
       $mapping = $this->store->get('mapping');
@@ -317,8 +315,6 @@ public function buildForm(array $form, FormStateInterface $form_state) {
         $node_description = $this->t('Columns that hold either other row <b>numbers</b> or <b>UUIDs</b>(an existing ADO) connecting ADOs between each other (e.g "ismemberof"). You can choose multiple.');
       }
 
-
-
       $form['ingestsetup']['adomapping']['parents'] = [
         '#type' => 'select',
         '#title' => $this->t('ADO Parent Columns'),
@@ -429,6 +425,10 @@ public function buildForm(array $form, FormStateInterface $form_state) {
         '#default_value' => 'AMI Set of ' . $this->currentUser()->getDisplayName()
       ];
     }
+    // Allow the plugin to alter the forms if needed in any way
+    if ($plugin_instance && $this->step > 1) {
+      $plugin_instance->stepFormAlter($form, $form_state, $this->store, $this->step);
+    }
     return $form;
   }
 
@@ -473,28 +473,26 @@ public function validateMapping(array &$form, FormStateInterface $form_state) {
   public function submitForm(array &$form, FormStateInterface $form_state) {
     parent::submitForm($form, $form_state);
     if ($form_state->getValue('plugin', NULL)) {
-
       if ($this->store->get('plugin') != $form_state->getValue('plugin', NULL)) {
         $this->store->set('pluginconfig',[]);
       }
       $this->store->set('plugin', $form_state->getValue('plugin'));
-
     }
     if ($form_state->getValue('pluginconfig', [])) {
       $this->store->set('pluginconfig', $form_state->getValue('pluginconfig'));
     }
+    /* @var $plugin_instance \Drupal\ami\Plugin\ImporterAdapterInterface| NULL */
+    $plugin_instance = $this->store->get('plugininstance');
     // First data fetch step
     if ($this->step == 3) {
       $this->store->delete('data');
-      /* @var $plugin_instance \Drupal\ami\Plugin\ImporterAdapterInterface| NULL */
-      $plugin_instance = $this->store->get('plugininstance');
       if ($plugin_instance) {
         $data = $plugin_instance->getInfo($this->store->get('pluginconfig'), $form_state,0,-1);
         // Check if the Plugin is ready processing or needs more data
         $ready = $form_state->getValue('pluginconfig')['ready'] ?? TRUE;
         $op = $form_state->getValue('pluginconfig')['op'] ?? 'create';
         if (!$ready) {
-          // Back yo Step 2 until the Plugin is ready doing its thing.
+          // Back to Step 2 until the Plugin is ready doing its thing.
           $this->step = 2;
           $form_state->setRebuild();
         }
@@ -504,6 +502,7 @@ public function submitForm(array &$form, FormStateInterface $form_state) {
           // Total rows contains data without headers So a single one is good enough.
           if (is_array($data) && !empty($data) and isset($data['headers']) && ((count($data['headers']) >= 3) || (count($data['headers']) >= 2 && $op != 'create')) && isset($data['totalrows']) && $data['totalrows'] >= 1) {
             $this->store->set('data', $data);
+            $plugin_instance->alterStepStore($form_state, $this->store, $this->step);
           }
           else {
             // Not the data we are looking for? Back to Step 2.
@@ -536,12 +535,18 @@ public function submitForm(array &$form, FormStateInterface $form_state) {
             ]
           ]
         ]);
+        if ($plugin_instance) {
+          $plugin_instance->alterStepStore($form_state, $this->store, $this->step);
+        }
       }
     }
     if ($this->step == 5) {
       if ($form_state->getTriggeringElement()['#name'] !== 'prev') {
         $adomapping = $form_state->getValue('adomapping');
         $this->store->set('adomapping', $adomapping);
+        if ($plugin_instance) {
+          $plugin_instance->alterStepStore($form_state, $this->store, $this->step);
+        }
       }
     }
     if ($this->step == 6) {
@@ -558,6 +563,9 @@ public function submitForm(array &$form, FormStateInterface $form_state) {
       } else {
         $this->store->set('zip', NULL);
       }
+      if ($plugin_instance) {
+        $plugin_instance->alterStepStore($form_state, $this->store, $this->step);
+      }
       $ami_set_label = $form_state->getValue('ami_set_label', NULL);
       $ami_set_label = $ami_set_label ? trim($ami_set_label) : $ami_set_label;
       $amisetdata = new \stdClass();
@@ -568,8 +576,6 @@ public function submitForm(array &$form, FormStateInterface $form_state) {
       $amisetdata->adomapping = $this->store->get('adomapping');
       $amisetdata->zip = $this->store->get('zip');
       $amisetdata->name = $ami_set_label;
-      /* @var $plugin_instance \Drupal\ami\Plugin\ImporterAdapterInterface| NULL */
-      $plugin_instance = $this->store->get('plugininstance');
       if ($plugin_instance) {
         if (!$plugin_instance->getPluginDefinition()['batch']) {
           $data = $plugin_instance->getData($this->store->get('pluginconfig'),
@@ -653,6 +659,9 @@ public function submitForm(array &$form, FormStateInterface $form_state) {
       $form_state->setRebuild(TRUE);
     } else {
       if (!empty($batch)) {
+        if ($plugin_instance) {
+          $plugin_instance->alterStepStore($form_state, $this->store, $this->step);
+        }
         batch_set($batch);
       }
     }

src/Form/amiSetEntityProcessForm.php

@@ -115,31 +115,6 @@ public function submitForm(array &$form, FormStateInterface $form_state) {
     }
     if ($file && $data !== new \stdClass()) {
       $invalid = [];
-      $info = $this->AmiUtilityService->preprocessAmiSet($file, $data, $invalid, FALSE);
-      // Means preprocess set
-      if (count($invalid)) {
-        $invalid_message = $this->formatPlural(count($invalid),
-          'Source data Row @row had an issue, common cause is an invalid parent.',
-          '@count rows, @row, had issues, common causes are invalid parents and/or non existing referenced rows.',
-          [
-            '@row' => implode(', ', array_keys($invalid)),
-          ]
-        );
-        $this->messenger()->addWarning($invalid_message);
-      }
-      if (!count($info)) {
-        $this->messenger()->addError(
-          $this->t(
-            'So Sorry. Ami Set @label produced no ADOs. Please correct your source CSV data.',
-            [
-              '@label' => $this->entity->label(),
-            ]
-          )
-        );
-        $form_state->setRebuild();
-        return;
-      }
-
       $SetURL = $this->entity->toUrl('canonical', ['absolute' => TRUE])
         ->toString();
 
@@ -169,34 +144,87 @@ public function submitForm(array &$form, FormStateInterface $form_state) {
         $op_secondary = $form_state->getValue(['ops_secondary','ops_secondary_update'], 'update');
         $ops_safefiles = $form_state->getValue(['ops_secondary','ops_safefiles'], TRUE);
       }
-      foreach ($info as $item) {
-        // We set current User here since we want to be sure the final owner of
-        // the object is this and not the user that runs the queue
-        $data->info = [
+      if ($notprocessnow) {
+        $data_csv = clone $data;
+        // Testing the CSV processor
+        $data_csv->info = [
           'zip_file' => $zip_file,
-          'row' => $item,
+          'csv_file' => $file,
           'set_id' => $this->entity->id(),
           'uid' => $this->currentUser()->id(),
           'status' => $statuses,
           'op_secondary' => $op_secondary,
-          'ops_safefiles' => $ops_safefiles ? TRUE: FALSE,
+          'ops_safefiles' => $ops_safefiles ? TRUE : FALSE,
           'log_jsonpatch' => FALSE,
           'set_url' => $SetURL,
           'attempt' => 1,
           'queue_name' => $queue_name,
-          'force_file_queue' => (bool) $form_state->getValue('force_file_queue', FALSE),
-          'force_file_process' => (bool) $form_state->getValue('force_file_process', FALSE),
+          'force_file_queue' => (bool)$form_state->getValue('force_file_queue', FALSE),
+          'force_file_process' => (bool)$form_state->getValue('force_file_process', FALSE),
           'manyfiles' => $manyfiles,
           'ops_skip_onmissing_file' => $ops_skip_onmissing_file,
           'ops_forcemanaged_destination_file' => $ops_forcemanaged_destination_file,
           'time_submitted' => $run_timestamp
         ];
-        $added[] = \Drupal::queue($queue_name)
-          ->createItem($data);
+        \Drupal::queue('ami_csv_ado')
+          ->createItem($data_csv);
+      }
+      else {
+        $info = $this->AmiUtilityService->preprocessAmiSet($file, $data, $invalid, FALSE);
+        // Means preprocess set
+        if (count($invalid)) {
+          $invalid_message = $this->formatPlural(count($invalid),
+            'Source data Row @row had an issue, common cause is an invalid parent.',
+            '@count rows, @row, had issues, common causes are invalid parents and/or non existing referenced rows.',
+            [
+              '@row' => implode(', ', array_keys($invalid)),
+            ]
+          );
+          $this->messenger()->addWarning($invalid_message);
+        }
+        if (!count($info)) {
+          $this->messenger()->addError(
+            $this->t(
+              'So Sorry. Ami Set @label produced no ADOs. Please correct your source CSV data.',
+              [
+                '@label' => $this->entity->label(),
+              ]
+            )
+          );
+          $form_state->setRebuild();
+          return;
+        }
+
+
+        foreach ($info as $item) {
+          // We set current User here since we want to be sure the final owner of
+          // the object is this and not the user that runs the queue
+          $data->info = [
+            'zip_file' => $zip_file,
+            'row' => $item,
+            'set_id' => $this->entity->id(),
+            'uid' => $this->currentUser()->id(),
+            'status' => $statuses,
+            'op_secondary' => $op_secondary,
+            'ops_safefiles' => $ops_safefiles ? TRUE : FALSE,
+            'log_jsonpatch' => FALSE,
+            'set_url' => $SetURL,
+            'attempt' => 1,
+            'queue_name' => $queue_name,
+            'force_file_queue' => (bool)$form_state->getValue('force_file_queue', FALSE),
+            'force_file_process' => (bool)$form_state->getValue('force_file_process', FALSE),
+            'manyfiles' => $manyfiles,
+            'ops_skip_onmissing_file' => $ops_skip_onmissing_file,
+            'ops_forcemanaged_destination_file' => $ops_forcemanaged_destination_file,
+            'time_submitted' => $run_timestamp
+          ];
+          $added[] = \Drupal::queue($queue_name)
+            ->createItem($data);
+        }
+        $count = count(array_filter($added));
       }
-      $count = count(array_filter($added));
 
-      if ($notprocessnow && $count) {
+      if ($notprocessnow) {
         $this->messenger()->addMessage(
           $this->t(
             'Set @label enqueued and processed .',
@@ -208,7 +236,8 @@ public function submitForm(array &$form, FormStateInterface $form_state) {
 
         $processed_set_status['processed'] =  0;
         $processed_set_status['errored'] =  0;
-        $processed_set_status['total'] = $count;
+        $processed_set_status['total'] = 0;
+        // So far here, with the new CSV enqueue plugin we have no idea how many. But the CSV queue entry will fill up the gap
         $this->statusStore->set('set_' . $this->entity->id(), $processed_set_status);
         $this->entity->setStatus(amiSetEntity::STATUS_ENQUEUED);
         $this->entity->save();