Merge branch 'release/2.0.11' Ignoring Deleted Files

BD2KGenomics · Feb 15, 2018 · 39d9303 · 39d9303
2 parents 52fa59a + 03c61a7
commit 39d9303
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 2 deletions.
diff --git a/v2/es_filebrowser_index_v2.py b/v2/es_filebrowser_index_v2.py
@@ -11,6 +11,7 @@
 from elasticsearch import Elasticsearch
 from urllib2 import urlopen, Request
 import os
+from file_filters import filter_deleted_files
 
 es_service = os.environ.get("ES_SERVICE", "localhost")
 redwood_host = os.environ.get("REDWOOD_SERVER", "redwood.io")
@@ -162,7 +163,8 @@ def requires():
                  #TEST WORKFLOW CONCATENATION
                   workflow = workflow+':'+workflow_version #DELETE IF IT CRASHES
                   download_id = analys['bundle_uuid']
-                  for file in analys['workflow_outputs']:
+                  available_files = filter_deleted_files(analys['workflow_outputs'])
+                  for file in available_files:
                      #pull out file_type, title(file_path)
                      file_type = file['file_type']
                      title = file['file_path']
@@ -178,11 +180,20 @@ def requires():
                      indexing = str(indexing).replace("'",'"')
                      counter += 1
                      #add all stuff to dictionary
+                     try:
+                        file_id = bundle_uuid_filename_to_file_uuid[
+                            download_id+'_'+title]
+                     except KeyError:
+                        print "{} ({}) from Bundle {} ." \
+                              " Skipping file.".format(title,
+                                                       fileMd5sum or 'No Hash',
+                                                       download_id)
+                        continue
                      try:
                         udict = {'center_name': center_name, 'project': project, 
                         'program': program, 'donor': donor, 'specimen_type': specimen_type, 'analysis_type': analysis_type, 
                         'workflow': workflow, 'download_id': download_id, 'file_type': file_type, 'title': title, 
-                        'file_id':bundle_uuid_filename_to_file_uuid[download_id+'_'+title], 'experimentalStrategy': submitter_experimental_design,
+                        'file_id': file_id, 'experimentalStrategy': submitter_experimental_design,
                         'redwoodDonorUUID': redwoodDonorUUID, 'study':study, 'sampleId':sampleId, 'submittedSampleId':submittedSampleId,
                         'submittedDonorId': submittedDonorId, 'submittedSpecimenId':submittedSpecimenId,
                         'fileSize':fileSize, 'fileMd5sum':fileMd5sum, 'workflowVersion': workflow_version,

diff --git a/v2/file_filters.py b/v2/file_filters.py
@@ -0,0 +1,6 @@
+def file_exists(workflow_output_file):
+    return not bool(workflow_output_file.get('is_deleted', False))
+
+
+def filter_deleted_files(workflow_outputs):
+    return filter(file_exists, workflow_outputs)
diff --git a/v2/update_endpoint_metadata.sh b/v2/update_endpoint_metadata.sh
@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+icgc-storage-client download --output-dir endpoint_metadata --object-id $1 --output-layout bundle --force > logs/force_update_metadata.txt
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		#!/usr/bin/env bash
		icgc-storage-client download --output-dir endpoint_metadata --object-id $1 --output-layout bundle --force > logs/force_update_metadata.txt