add ability to fetch all files of a given directory (ensure compatibility with old py27 for tests)

Author: sjanssen2

qiita_client/qiita_client.py

@@ -14,6 +14,7 @@
 import pandas as pd
 from json import dumps
 from random import randint
+import fnmatch
 
 try:
     from itertools import zip_longest
@@ -842,16 +843,29 @@ def push_file_to_central(self, filepath):
             return filepath
 
         elif self._plugincoupling == 'https':
-            logger.debug('Submitting file %s to qiita server.' % filepath)
+            logger.debug('Submitting %s %s to qiita server.' % (
+                'directory' if os.path.isdir(filepath) else 'file', filepath))
 
             # target path, i.e. without filename
             dirpath = os.path.dirname(filepath)
             if dirpath == "":
                 dirpath = "/"
 
-            self.post(
-                '/cloud/push_file_to_central/',
-                files={dirpath: open(filepath, 'rb')})
+            if os.path.isdir(filepath):
+                # Pushing all files of a directory, not only a single file.
+                # Cannot use "glob" as it lacks the "recursive" parameter in
+                # py27. This is used e.g. in qp-target-gene
+                for root, dirnames, filenames in os.walk(filepath):
+                    for filename in fnmatch.filter(filenames, "*"):
+                        fp = os.path.join(root, filename)
+                        self.post('/cloud/push_file_to_central/',
+                                  files={os.path.join(
+                                      dirpath,
+                                      os.path.dirname(fp)): open(fp, 'rb')})
+            else:
+                self.post(
+                    '/cloud/push_file_to_central/',
+                    files={dirpath: open(filepath, 'rb')})
 
             return filepath
 

qiita_client/tests/test_qiita_client.py

@@ -8,11 +8,12 @@
 
 from unittest import TestCase, main
 import filecmp
-from os import remove, close
-from os.path import basename, exists, expanduser, join
+from os import remove, close, makedirs
+from os.path import basename, exists, expanduser, join, isdir
 from tempfile import mkstemp
 from json import dumps
 import pandas as pd
+from shutil import rmtree
 
 from qiita_client.qiita_client import (QiitaClient, _format_payload,
                                        ArtifactInfo)
@@ -110,7 +111,10 @@ def setUp(self):
     def tearDown(self):
         for fp in self.clean_up_files:
             if exists(fp):
-                remove(fp)
+                if isdir(fp):
+                    rmtree(fp)
+                else:
+                    remove(fp)
 
     def test_init(self):
         obs = QiitaClient(URL,
@@ -184,7 +188,7 @@ def test_patch(self):
         with open(fp, 'w') as f:
             f.write('\n')
         self.clean_up_files.append(fp)
-        obs = self.tester.patch(f'/qiita_db/artifacts/{artifact_id}/', 'add',
+        obs = self.tester.patch('/qiita_db/artifacts/%s/' % artifact_id, 'add',
                                 '/html_summary/', value=fp)
         self.assertIsNone(obs)
 
@@ -440,6 +444,50 @@ def test_push_file_to_central(self):
         fp_obs = self.tester.push_file_to_central(fp_source)
         self.assertEqual(fp_source, fp_obs)
 
+    def _create_test_dir(self, prefix=None):
+        """Creates a test directory with files and subdirs."""
+        # prefix
+        # |- testdir/
+        # |---- fileA.txt
+        # |---- subdirA_l1/
+        # |-------- fileB.fna
+        # |-------- subdirC_l2/
+        # |------------ fileC.log
+        # |------------ fileD.seq
+        # |---- subdirB_l1/
+        # |-------- fileE.sff
+        if (prefix is not None) and (prefix != ""):
+            prefix = join(prefix, 'testdir')
+        else:
+            prefix = 'testdir'
+
+        for dir in [join(prefix, 'subdirA_l1', 'subdirC_l2'),
+                    join(prefix, 'subdirB_l1')]:
+            if not exists(dir):
+                makedirs(dir)
+        for file, cont in [(join(prefix, 'fileA.txt'), 'contentA'),
+                           (join(prefix, 'subdirA_l1',
+                                 'fileB.fna'), 'this is B'),
+                           (join(prefix, 'subdirA_l1', 'subdirC_l2',
+                                 'fileC.log'), 'call me c'),
+                           (join(prefix, 'subdirA_l1', 'subdirC_l2',
+                                 'fileD.seq'), 'I d'),
+                           (join(prefix, 'subdirB_l1', 'fileE.sff'), 'oh e')]:
+            with open(file, "w") as f:
+                f.write(cont + "\n")
+        self.clean_up_files.append(prefix)
+
+        return prefix
+
+    def test_push_file_to_central_dir(self):
+        self.tester._plugincoupling = 'https'
+
+        fp_source = self._create_test_dir('/tmp/test_push_dir/')
+        fp_obs = self.tester.push_file_to_central(fp_source)
+        self.assertEqual(fp_source, fp_obs)
+        # As we don't necessarily know the QIITA_BASE_DIR, we cannot fetch one
+        # of the files to double check for it's content
+
 
 if __name__ == '__main__':
     main()