Skip to content

Commit

Permalink
Merge pull request #196 from esmero/ISSUE-195
Browse files Browse the repository at this point in the history 
ISSUE-195: CSV queue worker and nested CSVs for EAD + an EAD plugin
  • Loading branch information
@DiegoPino
DiegoPino authored Jun 20, 2024
2 parents c3e7be6 + dc070b1 commit 0dda14a
Show file tree
Hide file tree
Showing 9 changed files with 660 additions and 104 deletions.
73 changes: 41 additions & 32 deletions src/Form/AmiMultiStepIngest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,19 +74,24 @@ public function __construct(PrivateTempStoreFactory $temp_store_factory, Session
*/
public function buildForm(array $form, FormStateInterface $form_state) {
$form = parent::buildForm($form, $form_state);

$form['message-step'] = [
'#markup' => '<div class="step">' . $this->t('AMI step @step of @laststep',[
'#markup' => '<div class="step">' . $this->t('AMI step @step of @laststep', [
'@step' => $this->step,
'@laststep' => $this->lastStep,
]) . '</div>',
];
$plugin_instance = NULL;
$pluginValue = NULL;
if ($this->step == 1) {
$pluginValue = $this->store->get('plugin');
$definitions = $this->importerManager->getDefinitions();
$options = [];
foreach ($definitions as $id => $definition) {
$options[$id] = $definition['label'];
}
// Reset the plugin instance in case of code change.
$this->store->set('plugininstance', NULL);

$form['plugin'] = [
'#type' => 'select',
Expand All @@ -98,29 +103,32 @@ public function buildForm(array $form, FormStateInterface $form_state) {
'#empty_option' => $this->t('- Please select a plugin -'),
];
}
if ($this->step > 1) {
//Only carry the plugin instance
/* @var $plugin_instance \Drupal\ami\Plugin\ImporterAdapterInterface | NULL */
$plugin_instance = $this->store->get('plugininstance');
$pluginValue = $this->store->get('plugin');
}
if ($this->step == 2) {
$parents = ['pluginconfig'];
$form_state->setValue('pluginconfig', $this->store->get('pluginconfig'));
$pluginValue = $this->store->get('plugin');
// Only create a new instance if we do not have the PluginInstace around
/* @var $plugin_instance \Drupal\ami\Plugin\ImporterAdapterInterface | NULL */
$plugin_instance = $this->store->get('plugininstance');
if (!$plugin_instance || $plugin_instance->getPluginid() != $pluginValue || $pluginValue == NULL) {
$configuration = [];
$configuration['config'] = ImporterAdapter::create();
$plugin_instance = $this->importerManager->createInstance($pluginValue,$configuration);
$this->store->set('plugininstance',$plugin_instance);
$plugin_instance = $this->importerManager->createInstance($pluginValue, $configuration);
$this->store->set('plugininstance', $plugin_instance);
}
$form['pluginconfig'] = $plugin_instance->interactiveForm($parents, $form_state);
$form['pluginconfig']['#tree'] = TRUE;
}
// TO keep this discrete and easier to edit maybe move to it's own method?

// To keep this discrete and easier to edit maybe move to their own methods?
if ($this->step == 3) {
// We should never reach this point if data is not enough. Submit handler
// will go back to Step 2 if so.
$data = $this->store->get('data') ?? [];
$pluginconfig = $this->store->get('pluginconfig');
$plugin_instance = $this->store->get('plugininstance');
$op = $pluginconfig['op'];
$column_keys = $plugin_instance->provideKeys($pluginconfig, $data);
$mapping = $this->store->get('mapping');
Expand All @@ -129,13 +137,17 @@ public function buildForm(array $form, FormStateInterface $form_state) {
'template' => 'Template',
];
$template = $this->getMetadatadisplays();
// $webform = $this->getWebforms();
$bundle = $this->getBundlesAndFields();

$global_metadata_options = $metadata + ['custom' => 'Custom (Expert Mode)'];
//Each row (based on its type column) can have its own approach setup(expert mode)
$element_conditional = [];
$element = [];
// Get all headers and check for a 'type' key first, if not allow the user to select one?
// Wonder if we can be strict about this and simply require always a "type"?
// @TODO WE need to check for 'type' always. Maybe even in the submit handler?
$alltypes = $plugin_instance->provideTypes($pluginconfig, $data);

$element['bundle'] =[
'#type' => 'select',
'#title' => $this->t('Fields and Bundles'),
Expand All @@ -150,15 +162,6 @@ public function buildForm(array $form, FormStateInterface $form_state) {
'#description' => $this->t('Columns will be casted to ADO metadata (JSON) using a Twig template setup for JSON output'),
];

/**
* $element_conditional['webform'] = [
* '#type' => 'select',
* '#title' => $this->t('Webform'),
* '#options' => $webform,
* '#description' => $this->t('Columns are casted to ADO metadata (JSON) by passing/validating Data through an existing Webform'),
* ];
*/

$form['ingestsetup']['globalmapping'] = [
'#type' => 'select',
'#title' => $this->t('Select the data transformation approach'),
Expand Down Expand Up @@ -202,10 +205,6 @@ public function buildForm(array $form, FormStateInterface $form_state) {
],
];

// Get all headers and check for a 'type' key first, if not allow the user to select one?
// Wonder if we can be strict about this and simply require always a "type"?
// @TODO WE need to check for 'type' always. Maybe even in the submit handler?
$alltypes = $plugin_instance->provideTypes($pluginconfig, $data);
if (!empty($alltypes)) {
$form['ingestsetup']['custommapping'] = [
'#type' => 'fieldset',
Expand Down Expand Up @@ -296,7 +295,6 @@ public function buildForm(array $form, FormStateInterface $form_state) {
$data = $this->store->get('data') ?? [];
$pluginconfig = $this->store->get('pluginconfig');
$op = $pluginconfig['op'];
$plugin_instance = $this->store->get('plugininstance');
$column_keys = $plugin_instance->provideKeys($pluginconfig, $data);
$column_options = array_combine($column_keys, $column_keys);
$mapping = $this->store->get('mapping');
Expand All @@ -317,8 +315,6 @@ public function buildForm(array $form, FormStateInterface $form_state) {
$node_description = $this->t('Columns that hold either other row <b>numbers</b> or <b>UUIDs</b>(an existing ADO) connecting ADOs between each other (e.g "ismemberof"). You can choose multiple.');
}



$form['ingestsetup']['adomapping']['parents'] = [
'#type' => 'select',
'#title' => $this->t('ADO Parent Columns'),
Expand Down Expand Up @@ -429,6 +425,10 @@ public function buildForm(array $form, FormStateInterface $form_state) {
'#default_value' => 'AMI Set of ' . $this->currentUser()->getDisplayName()
];
}
// Allow the plugin to alter the forms if needed in any way
if ($plugin_instance && $this->step > 1) {
$plugin_instance->stepFormAlter($form, $form_state, $this->store, $this->step);
}
return $form;
}

Expand Down Expand Up @@ -473,28 +473,26 @@ public function validateMapping(array &$form, FormStateInterface $form_state) {
public function submitForm(array &$form, FormStateInterface $form_state) {
parent::submitForm($form, $form_state);
if ($form_state->getValue('plugin', NULL)) {

if ($this->store->get('plugin') != $form_state->getValue('plugin', NULL)) {
$this->store->set('pluginconfig',[]);
}
$this->store->set('plugin', $form_state->getValue('plugin'));

}
if ($form_state->getValue('pluginconfig', [])) {
$this->store->set('pluginconfig', $form_state->getValue('pluginconfig'));
}
/* @var $plugin_instance \Drupal\ami\Plugin\ImporterAdapterInterface| NULL */
$plugin_instance = $this->store->get('plugininstance');
// First data fetch step
if ($this->step == 3) {
$this->store->delete('data');
/* @var $plugin_instance \Drupal\ami\Plugin\ImporterAdapterInterface| NULL */
$plugin_instance = $this->store->get('plugininstance');
if ($plugin_instance) {
$data = $plugin_instance->getInfo($this->store->get('pluginconfig'), $form_state,0,-1);
// Check if the Plugin is ready processing or needs more data
$ready = $form_state->getValue('pluginconfig')['ready'] ?? TRUE;
$op = $form_state->getValue('pluginconfig')['op'] ?? 'create';
if (!$ready) {
// Back yo Step 2 until the Plugin is ready doing its thing.
// Back to Step 2 until the Plugin is ready doing its thing.
$this->step = 2;
$form_state->setRebuild();
}
Expand All @@ -504,6 +502,7 @@ public function submitForm(array &$form, FormStateInterface $form_state) {
// Total rows contains data without headers So a single one is good enough.
if (is_array($data) && !empty($data) and isset($data['headers']) && ((count($data['headers']) >= 3) || (count($data['headers']) >= 2 && $op != 'create')) && isset($data['totalrows']) && $data['totalrows'] >= 1) {
$this->store->set('data', $data);
$plugin_instance->alterStepStore($form_state, $this->store, $this->step);
}
else {
// Not the data we are looking for? Back to Step 2.
Expand Down Expand Up @@ -536,12 +535,18 @@ public function submitForm(array &$form, FormStateInterface $form_state) {
]
]
]);
if ($plugin_instance) {
$plugin_instance->alterStepStore($form_state, $this->store, $this->step);
}
}
}
if ($this->step == 5) {
if ($form_state->getTriggeringElement()['#name'] !== 'prev') {
$adomapping = $form_state->getValue('adomapping');
$this->store->set('adomapping', $adomapping);
if ($plugin_instance) {
$plugin_instance->alterStepStore($form_state, $this->store, $this->step);
}
}
}
if ($this->step == 6) {
Expand All @@ -558,6 +563,9 @@ public function submitForm(array &$form, FormStateInterface $form_state) {
} else {
$this->store->set('zip', NULL);
}
if ($plugin_instance) {
$plugin_instance->alterStepStore($form_state, $this->store, $this->step);
}
$ami_set_label = $form_state->getValue('ami_set_label', NULL);
$ami_set_label = $ami_set_label ? trim($ami_set_label) : $ami_set_label;
$amisetdata = new \stdClass();
Expand All @@ -568,8 +576,6 @@ public function submitForm(array &$form, FormStateInterface $form_state) {
$amisetdata->adomapping = $this->store->get('adomapping');
$amisetdata->zip = $this->store->get('zip');
$amisetdata->name = $ami_set_label;
/* @var $plugin_instance \Drupal\ami\Plugin\ImporterAdapterInterface| NULL */
$plugin_instance = $this->store->get('plugininstance');
if ($plugin_instance) {
if (!$plugin_instance->getPluginDefinition()['batch']) {
$data = $plugin_instance->getData($this->store->get('pluginconfig'),
Expand Down Expand Up @@ -653,6 +659,9 @@ public function submitForm(array &$form, FormStateInterface $form_state) {
$form_state->setRebuild(TRUE);
} else {
if (!empty($batch)) {
if ($plugin_instance) {
$plugin_instance->alterStepStore($form_state, $this->store, $this->step);
}
batch_set($batch);
}
}
Expand Down
105 changes: 67 additions & 38 deletions src/Form/amiSetEntityProcessForm.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,31 +115,6 @@ public function submitForm(array &$form, FormStateInterface $form_state) {
}
if ($file && $data !== new \stdClass()) {
$invalid = [];
$info = $this->AmiUtilityService->preprocessAmiSet($file, $data, $invalid, FALSE);
// Means preprocess set
if (count($invalid)) {
$invalid_message = $this->formatPlural(count($invalid),
'Source data Row @row had an issue, common cause is an invalid parent.',
'@count rows, @row, had issues, common causes are invalid parents and/or non existing referenced rows.',
[
'@row' => implode(', ', array_keys($invalid)),
]
);
$this->messenger()->addWarning($invalid_message);
}
if (!count($info)) {
$this->messenger()->addError(
$this->t(
'So Sorry. Ami Set @label produced no ADOs. Please correct your source CSV data.',
[
'@label' => $this->entity->label(),
]
)
);
$form_state->setRebuild();
return;
}

$SetURL = $this->entity->toUrl('canonical', ['absolute' => TRUE])
->toString();

Expand Down Expand Up @@ -169,34 +144,87 @@ public function submitForm(array &$form, FormStateInterface $form_state) {
$op_secondary = $form_state->getValue(['ops_secondary','ops_secondary_update'], 'update');
$ops_safefiles = $form_state->getValue(['ops_secondary','ops_safefiles'], TRUE);
}
foreach ($info as $item) {
// We set current User here since we want to be sure the final owner of
// the object is this and not the user that runs the queue
$data->info = [
if ($notprocessnow) {
$data_csv = clone $data;
// Testing the CSV processor
$data_csv->info = [
'zip_file' => $zip_file,
'row' => $item,
'csv_file' => $file,
'set_id' => $this->entity->id(),
'uid' => $this->currentUser()->id(),
'status' => $statuses,
'op_secondary' => $op_secondary,
'ops_safefiles' => $ops_safefiles ? TRUE: FALSE,
'ops_safefiles' => $ops_safefiles ? TRUE : FALSE,
'log_jsonpatch' => FALSE,
'set_url' => $SetURL,
'attempt' => 1,
'queue_name' => $queue_name,
'force_file_queue' => (bool) $form_state->getValue('force_file_queue', FALSE),
'force_file_process' => (bool) $form_state->getValue('force_file_process', FALSE),
'force_file_queue' => (bool)$form_state->getValue('force_file_queue', FALSE),
'force_file_process' => (bool)$form_state->getValue('force_file_process', FALSE),
'manyfiles' => $manyfiles,
'ops_skip_onmissing_file' => $ops_skip_onmissing_file,
'ops_forcemanaged_destination_file' => $ops_forcemanaged_destination_file,
'time_submitted' => $run_timestamp
];
$added[] = \Drupal::queue($queue_name)
->createItem($data);
\Drupal::queue('ami_csv_ado')
->createItem($data_csv);
}
else {
$info = $this->AmiUtilityService->preprocessAmiSet($file, $data, $invalid, FALSE);
// Means preprocess set
if (count($invalid)) {
$invalid_message = $this->formatPlural(count($invalid),
'Source data Row @row had an issue, common cause is an invalid parent.',
'@count rows, @row, had issues, common causes are invalid parents and/or non existing referenced rows.',
[
'@row' => implode(', ', array_keys($invalid)),
]
);
$this->messenger()->addWarning($invalid_message);
}
if (!count($info)) {
$this->messenger()->addError(
$this->t(
'So Sorry. Ami Set @label produced no ADOs. Please correct your source CSV data.',
[
'@label' => $this->entity->label(),
]
)
);
$form_state->setRebuild();
return;
}


foreach ($info as $item) {
// We set current User here since we want to be sure the final owner of
// the object is this and not the user that runs the queue
$data->info = [
'zip_file' => $zip_file,
'row' => $item,
'set_id' => $this->entity->id(),
'uid' => $this->currentUser()->id(),
'status' => $statuses,
'op_secondary' => $op_secondary,
'ops_safefiles' => $ops_safefiles ? TRUE : FALSE,
'log_jsonpatch' => FALSE,
'set_url' => $SetURL,
'attempt' => 1,
'queue_name' => $queue_name,
'force_file_queue' => (bool)$form_state->getValue('force_file_queue', FALSE),
'force_file_process' => (bool)$form_state->getValue('force_file_process', FALSE),
'manyfiles' => $manyfiles,
'ops_skip_onmissing_file' => $ops_skip_onmissing_file,
'ops_forcemanaged_destination_file' => $ops_forcemanaged_destination_file,
'time_submitted' => $run_timestamp
];
$added[] = \Drupal::queue($queue_name)
->createItem($data);
}
$count = count(array_filter($added));
}
$count = count(array_filter($added));

if ($notprocessnow && $count) {
if ($notprocessnow) {
$this->messenger()->addMessage(
$this->t(
'Set @label enqueued and processed .',
Expand All @@ -208,7 +236,8 @@ public function submitForm(array &$form, FormStateInterface $form_state) {

$processed_set_status['processed'] = 0;
$processed_set_status['errored'] = 0;
$processed_set_status['total'] = $count;
$processed_set_status['total'] = 0;
// So far here, with the new CSV enqueue plugin we have no idea how many. But the CSV queue entry will fill up the gap
$this->statusStore->set('set_' . $this->entity->id(), $processed_set_status);
$this->entity->setStatus(amiSetEntity::STATUS_ENQUEUED);
$this->entity->save();
Expand Down
Loading

0 comments on commit 0dda14a

Please sign in to comment.