Merge pull request #3169 from antgonza/update-python

Python 3.9

Author: charles-cowart

.github/workflows/qiita-ci.yml

@@ -45,7 +45,7 @@ jobs:
         uses: conda-incubator/setup-miniconda@v2
         with:
           auto-update-conda: true
-          python-version: 3.6
+          python-version: '3.9'
 
       - name: Basic dependencies install
         env:
@@ -63,7 +63,10 @@ jobs:
 
           # Setting up main qiita conda environment
           conda config --add channels conda-forge
-          conda create -q --yes -n qiita python=3.6 pip==9.0.3 libgfortran numpy nginx cython redis
+          conda deactivate
+          conda install --quiet mamba
+          mamba create --quiet --yes -n qiita python=3.9 pip libgfortran numpy nginx cython redis
+          conda env list
           conda activate qiita
           pip install -U pip
           pip install sphinx sphinx-bootstrap-theme nose-timer Click coverage
@@ -91,6 +94,7 @@ jobs:
           export QIITA_SERVER_CERT=`pwd`/qiita_core/support_files/server.crt
           export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test.cfg
           export REDBIOM_HOST="http://localhost:7379"
+
           pip install . --no-binary redbiom
           pwd
           mkdir ~/.qiita_plugins
@@ -99,7 +103,7 @@ jobs:
         shell: bash -l {0}
         run: |
           wget https://data.qiime2.org/distro/core/qiime2-2019.4-py36-linux-conda.yml
-          conda env create -q -n qtp-biom --file qiime2-2019.4-py36-linux-conda.yml
+          mamba env create --quiet -n qtp-biom --file qiime2-2019.4-py36-linux-conda.yml
           rm qiime2-2019.4-py36-linux-conda.yml
           export QIITA_SERVER_CERT=`pwd`/qiita_core/support_files/server.crt
           export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test.cfg
@@ -210,7 +214,7 @@ jobs:
     - name: flake8
       uses: actions/setup-python@v2
       with:
-        python-version: 3.6
+        python-version: '3.9'
     - name: install dependencies
       run: python -m pip install --upgrade pip
     - name: Check out repository code

qiita_db/metadata_template/base_metadata_template.py

@@ -41,7 +41,7 @@
 
 import pandas as pd
 import numpy as np
-from skbio.util import find_duplicates
+from iteration_utilities import duplicates
 import warnings
 
 from qiita_core.exceptions import IncompetentQiitaDeveloperError
@@ -525,7 +525,7 @@ def _clean_validate_template(cls, md_template, study_id,
 
         if len(set(md_template.index)) != len(md_template.index):
             raise qdb.exceptions.QiitaDBDuplicateSamplesError(
-                find_duplicates(md_template.index))
+                set(duplicates(md_template.index)))
 
         # We are going to modify the md_template. We create a copy so
         # we don't modify the user one
@@ -584,7 +584,7 @@ def _clean_validate_template(cls, md_template, study_id,
         # Check that we don't have duplicate columns
         if len(set(md_template.columns)) != len(md_template.columns):
             raise qdb.exceptions.QiitaDBDuplicateHeaderError(
-                find_duplicates(md_template.columns))
+                set(duplicates(md_template.columns)))
 
         return md_template
 

qiita_db/metadata_template/prep_template.py

@@ -8,7 +8,7 @@
 from itertools import chain
 from os.path import join
 from copy import deepcopy
-from skbio.util import find_duplicates
+from iteration_utilities import duplicates
 
 from qiita_core.exceptions import IncompetentQiitaDeveloperError
 import qiita_db as qdb
@@ -34,7 +34,7 @@ def _check_duplicated_columns(prep_cols, sample_cols):
         If there are duplicated columns names in the sample and the prep
     """
     prep_cols.extend(sample_cols)
-    dups = find_duplicates(prep_cols)
+    dups = set(duplicates(prep_cols))
     if dups:
         raise qdb.exceptions.QiitaDBColumnError(
             'Duplicated column names in the sample and prep info '

qiita_db/metadata_template/util.py

@@ -10,7 +10,7 @@
 import pandas as pd
 import numpy as np
 import warnings
-from skbio.util import find_duplicates
+from iteration_utilities import duplicates
 
 import qiita_db as qdb
 
@@ -131,7 +131,7 @@ def load_template_to_dataframe(fn, index='sample_name'):
                     raise ValueError(
                         'Your file has empty columns headers.')
                 raise qdb.exceptions.QiitaDBDuplicateHeaderError(
-                    find_duplicates(newcols))
+                    set(duplicates(newcols)))
         else:
             # .strip will remove odd chars, newlines, tabs and multiple
             # spaces but we need to read a new line at the end of the

qiita_ware/ebi.py

@@ -6,6 +6,7 @@
 # The full license is in the file LICENSE, distributed with this software.
 # -----------------------------------------------------------------------------
 
+import hashlib
 from os.path import basename, join, isdir, isfile, exists
 from shutil import copyfile, rmtree
 from os import remove, listdir, makedirs
@@ -18,7 +19,6 @@
 from gzip import GzipFile
 from functools import partial
 from h5py import File
-from skbio.util import safe_md5
 from qiita_files.demux import to_per_sample_ascii
 
 from qiita_core.qiita_settings import qiita_config
@@ -584,7 +584,7 @@ def _add_file_subelement(self, add_file, file_type, sample_name,
 
         file_path = self.sample_demux_fps[sample_name] + suffix
         with open(file_path, 'rb') as fp:
-            md5 = safe_md5(fp).hexdigest()
+            md5 = hashlib.md5(fp.read()).hexdigest()
 
         file_details = {'filetype': file_type,
                         'quality_scoring_system': 'phred',

qiita_ware/test/test_ebi.py

@@ -16,7 +16,7 @@
 import pandas as pd
 import warnings
 from datetime import date
-from skbio.util import safe_md5
+import hashlib
 
 from h5py import File
 from qiita_files.demux import to_hdf5
@@ -558,8 +558,8 @@ def test_generate_run_xml(self):
 
         md5_sums = {}
         for s, fp in submission.sample_demux_fps.items():
-            md5_sums[s] = safe_md5(
-                open(fp + submission.FWD_READ_SUFFIX, 'rb')).hexdigest()
+            md5_sums[s] = hashlib.md5(
+                open(fp + submission.FWD_READ_SUFFIX, 'rb').read()).hexdigest()
 
         exp = RUNXML_NEWSTUDY % {
             'study_alias': submission._get_study_alias(),
@@ -1650,9 +1650,9 @@ def test_parse_EBI_reply(self):
     <EXPERIMENT_REF accession="ERX0000008" />
     <DATA_BLOCK>
       <FILES>
-        <FILE checksum="a32357beb845f5b598f1a712fb3b4c70" \
-checksum_method="MD5" filename="%(ebi_dir)s/1.SKB2.640194.R1.fastq.gz" \
-filetype="fastq" quality_scoring_system="phred" />
+        <FILE filetype="fastq" quality_scoring_system="phred" \
+checksum_method="MD5" checksum="a32357beb845f5b598f1a712fb3b4c70" \
+filename="%(ebi_dir)s/1.SKB2.640194.R1.fastq.gz" />
       </FILES>
     </DATA_BLOCK>
   </RUN>
@@ -1661,9 +1661,9 @@ def test_parse_EBI_reply(self):
     <EXPERIMENT_REF accession="ERX0000024" />
     <DATA_BLOCK>
       <FILES>
-        <FILE checksum="deb905ced92812a65a2158fdcfd0f84d" \
-checksum_method="MD5" filename="%(ebi_dir)s/1.SKB3.640195.R1.fastq.gz" \
-filetype="fastq" quality_scoring_system="phred" />
+        <FILE filetype="fastq" quality_scoring_system="phred" \
+checksum_method="MD5" checksum="deb905ced92812a65a2158fdcfd0f84d" \
+filename="%(ebi_dir)s/1.SKB3.640195.R1.fastq.gz" />
       </FILES>
     </DATA_BLOCK>
   </RUN>
@@ -1672,9 +1672,9 @@ def test_parse_EBI_reply(self):
     <EXPERIMENT_REF accession="ERX0000025" />
     <DATA_BLOCK>
       <FILES>
-        <FILE checksum="847ba142770397a2fae3a8acfbc70640" \
-checksum_method="MD5" filename="%(ebi_dir)s/1.SKB6.640176.R1.fastq.gz" \
-filetype="fastq" quality_scoring_system="phred" />
+        <FILE filetype="fastq" quality_scoring_system="phred" \
+checksum_method="MD5" checksum="847ba142770397a2fae3a8acfbc70640" \
+filename="%(ebi_dir)s/1.SKB6.640176.R1.fastq.gz" />
       </FILES>
     </DATA_BLOCK>
   </RUN>
@@ -1683,9 +1683,9 @@ def test_parse_EBI_reply(self):
     <EXPERIMENT_REF accession="ERX0000004" />
     <DATA_BLOCK>
       <FILES>
-        <FILE checksum="0dc19bc7ad4ab613c3f738cc9eb57e2c" \
-checksum_method="MD5" filename="%(ebi_dir)s/1.SKM4.640180.R1.fastq.gz" \
-filetype="fastq" quality_scoring_system="phred" />
+        <FILE filetype="fastq" quality_scoring_system="phred" \
+checksum_method="MD5" checksum="0dc19bc7ad4ab613c3f738cc9eb57e2c" \
+filename="%(ebi_dir)s/1.SKM4.640180.R1.fastq.gz" />
       </FILES>
     </DATA_BLOCK>
   </RUN>
@@ -1701,9 +1701,9 @@ def test_parse_EBI_reply(self):
 %(study_id)s.Sample1" />
     <DATA_BLOCK>
       <FILES>
-        <FILE checksum="%(sample_1)s" \
-checksum_method="MD5" filename="%(ebi_dir)s/%(study_id)s.Sample1.R1.fastq.gz" \
-filetype="fastq" quality_scoring_system="phred" />
+        <FILE filetype="fastq" quality_scoring_system="phred" \
+checksum_method="MD5" checksum="%(sample_1)s" \
+filename="%(ebi_dir)s/%(study_id)s.Sample1.R1.fastq.gz" />
       </FILES>
     </DATA_BLOCK>
   </RUN>
@@ -1713,9 +1713,9 @@ def test_parse_EBI_reply(self):
 %(study_id)s.Sample2" />
     <DATA_BLOCK>
       <FILES>
-        <FILE checksum="%(sample_2)s" \
-checksum_method="MD5" filename="%(ebi_dir)s/%(study_id)s.Sample2.R1.fastq.gz" \
-filetype="fastq" quality_scoring_system="phred" />
+        <FILE filetype="fastq" quality_scoring_system="phred" \
+checksum_method="MD5" checksum="%(sample_2)s" \
+filename="%(ebi_dir)s/%(study_id)s.Sample2.R1.fastq.gz" />
       </FILES>
     </DATA_BLOCK>
   </RUN>
@@ -1725,9 +1725,9 @@ def test_parse_EBI_reply(self):
 %(study_id)s.Sample3" />
     <DATA_BLOCK>
       <FILES>
-        <FILE checksum="%(sample_3)s" \
-checksum_method="MD5" filename="%(ebi_dir)s/%(study_id)s.Sample3.R1.fastq.gz" \
-filetype="fastq" quality_scoring_system="phred" />
+        <FILE filetype="fastq" quality_scoring_system="phred" \
+checksum_method="MD5" checksum="%(sample_3)s" \
+filename="%(ebi_dir)s/%(study_id)s.Sample3.R1.fastq.gz" />
       </FILES>
     </DATA_BLOCK>
   </RUN>

setup.py

@@ -105,13 +105,13 @@
       install_requires=['psycopg2', 'click', 'bcrypt', 'pandas',
                         'biom-format', 'tornado<6.0', 'toredis', 'redis',
                         'scp', 'pyparsing', 'h5py',  'natsort', 'nose', 'pep8',
-                        'networkx', 'humanize', 'scikit-bio', 'wtforms<3.0.0',
+                        'networkx', 'humanize', 'wtforms<3.0.0',
                         'openpyxl', 'sphinx-bootstrap-theme', 'Sphinx', 'nltk',
                         'gitpython', 'redbiom', 'pyzmq', 'sphinx_rtd_theme',
                         'paramiko', 'seaborn',  'matplotlib', 'scipy', 'nose',
                         'flake8', 'six', 'qiita-files @ https://github.com/'
                         'qiita-spots/qiita-files/archive/master.zip', 'mock',
-                        'python-jose', 'markdown2',
+                        'python-jose', 'markdown2', 'iteration_utilities',
                         'supervisor @ https://github.com/Supervisor/'
                         'supervisor/archive/master.zip', 'joblib'],
       classifiers=classifiers