Drop python3.6 in favor of python3.9. (DataBiosphere#3929)

* Drop 3.6, add 3.9, and add/make 3.10 as the main py version. * Update .gitlab-ci.yml * We apparently need python3.9-distutils. * Switch to py3.9 as latest. * Add Dockerfile dep. Co-authored-by: Michael R. Crusoe <[email protected]>
douglowe · Dec 16, 2021 · d62dc5b · d62dc5b
1 parent 537aef4
commit d62dc5b
Show file tree

Hide file tree

Showing 3 changed files with 54 additions and 51 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -5,7 +5,7 @@ variables:
   PYTHONIOENCODING: "utf-8"
   DEBIAN_FRONTEND: "noninteractive"
   TOIL_OWNER_TAG: "shared"
-  MAIN_PYTHON_PKG: "python3.8"
+  MAIN_PYTHON_PKG: "python3.9"
 
 before_script:
   # Configure Docker to use a mirror for Docker Hub and restart the daemon
@@ -33,7 +33,7 @@ before_script:
   - sudo apt-get install -y software-properties-common build-essential virtualenv
   - sudo add-apt-repository -y ppa:deadsnakes/ppa
   - sudo apt-get update
-  - sudo apt-get install -y tzdata jq python3.6 python3.6-dev python3.7 python3.7-dev python3.8 python3.8-dev
+  - sudo apt-get install -y tzdata jq python3.7 python3.7-dev python3.8 python3.8-dev python3.9 python3.9-dev python3.9-distutils  # python3.10 python3.10-dev python3.10-venv
 
 after_script:
   # We need to clean up any files that Toil may have made via Docker that
@@ -82,27 +82,69 @@ wdl_dependency_is_stand_alone:
 quick_test_offline:
   stage: basic_tests
   script:
-    - virtualenv -p python3.6 venv
+    - virtualenv -p ${MAIN_PYTHON_PKG} venv
     - . venv/bin/activate
     - pip install -U pip wheel
     - make prepare
     - make develop extras=[aws,google,wdl]
     - TOIL_TEST_QUICK=True make test_offline
 
-# Python3.6
-py36_main:
+# Python3.7
+py37_main:
+  stage: basic_tests
+  script:
+    - pwd
+    - virtualenv -p python3.7 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor awscli'
+    - make test tests=src/toil/test/src
+    - make test tests=src/toil/test/utils
+    - make test tests=src/toil/test/lib/test_ec2.py
+    - make test tests=src/toil/test/lib/aws
+    - make test tests=src/toil/test/lib/test_conversions.py
+
+py37_appliance_build:
+  stage: basic_tests
+  script:
+    - pwd
+    - virtualenv -p python3.7 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor awscli'
+    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
+    - python setup_gitlab_docker.py
+    - make push_docker
+
+# Python3.8
+py38_main:
+  stage: basic_tests
+  script:
+    - pwd
+    - virtualenv -p python3.8 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor awscli'
+    - make test tests=src/toil/test/src
+    - make test tests=src/toil/test/utils
+    - make test tests=src/toil/test/lib/test_ec2.py
+    - make test tests=src/toil/test/lib/aws
+    - make test tests=src/toil/test/lib/test_conversions.py
+
+py38_appliance_build:
+  stage: basic_tests
+  script:
+    - pwd
+    - virtualenv -p python3.8 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor awscli'
+    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
+    - python setup_gitlab_docker.py
+    - make push_docker
+
+# Python3.9
+py39_main:
   stage: basic_tests
   script:
     - pwd
-    - virtualenv -p python3.6 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages=htcondor
+    - virtualenv -p python3.9 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages=htcondor
     - make test tests=src/toil/test/src
     - make test tests=src/toil/test/utils
 
-py36_appliance_build:
+py39_appliance_build:
   stage: basic_tests
   script:
     - pwd
-    - virtualenv -p python3.6 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor awscli'
+    - virtualenv -p python3.9 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor awscli'
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
     - make push_docker
@@ -242,24 +284,6 @@ jobstore_and_provisioning:
 # https://ucsc-ci.com/databiosphere/toil/-/jobs/38672
 # guessing decorators are masking class as function?  ^  also, abstract class is run as normal test?  should hide.
 
-main:
-  stage: basic_tests
-  script:
-    - pwd
-    - virtualenv -p ${MAIN_PYTHON_PKG} venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages=htcondor
-    - make test tests=src/toil/test/src
-    - make test tests=src/toil/test/utils
-#    - make test tests=src/toil/test/docs/scriptsTest.py::ToilDocumentationTest::testDocker
-
-appliance_build:
-  stage: basic_tests
-  script:
-    - pwd
-    - virtualenv -p ${MAIN_PYTHON_PKG} venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor awscli'
-    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
-    - python setup_gitlab_docker.py
-    - make push_docker
-
 integration:
   stage: integration
   script:
@@ -305,29 +329,6 @@ google_jobstore:
     - export TOIL_GOOGLE_PROJECTID=toil-dev
     - make test tests=src/toil/test/jobStores/jobStoreTest.py::GoogleJobStoreTest
 
-# Python3.7
-py37_main:
-  stage: basic_tests
-  script:
-    - pwd
-    - virtualenv -p python3.7 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor awscli'
-    - make test tests=src/toil/test/src
-    - make test tests=src/toil/test/utils
-    - make test tests=src/toil/test/lib/test_ec2.py
-    - make test tests=src/toil/test/lib/aws
-    - make test tests=src/toil/test/lib/test_conversions.py
-
-py37_appliance_build:
-  stage: basic_tests
-  script:
-    - pwd
-    - virtualenv -p python3.7 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor awscli'
-    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
-    - python setup_gitlab_docker.py
-    - make push_docker
-
-# Python 3.8 is the "main" python already and doesn't need its own tests.
-
 # Cactus-on-Kubernetes integration (as a script and not a pytest test)
 cactus_integration:
   stage: integration

diff --git a/docker/Dockerfile.py b/docker/Dockerfile.py
@@ -29,6 +29,8 @@
                          python,
                          f'{python}-dev',
                          'python3.8-distutils' if python == 'python3.8' else '',
+                         'python3.9-distutils' if python == 'python3.9' else '',
+                         # 'python3.9-venv' if python == 'python3.9' else '',
                          'python3-pip',
                          'libcurl4-openssl-dev',
                          'libssl-dev',

diff --git a/setup.py b/setup.py
@@ -80,9 +80,9 @@ def run_setup():
           'Operating System :: MacOS :: MacOS X',
           'Operating System :: POSIX',
           'Operating System :: POSIX :: Linux',
-          'Programming Language :: Python :: 3.6',
           'Programming Language :: Python :: 3.7',
           'Programming Language :: Python :: 3.8',
+          'Programming Language :: Python :: 3.9',
           'Topic :: Scientific/Engineering',
           'Topic :: Scientific/Engineering :: Bio-Informatics',
           'Topic :: Scientific/Engineering :: Astronomy',
@@ -92,7 +92,7 @@ def run_setup():
           'Topic :: System :: Distributed Computing',
           'Topic :: Utilities'],
         license="Apache License v2.0",
-        python_requires=">=3.6",
+        python_requires=">=3.7",
         install_requires=install_requires,
         extras_require=extras_require,
         package_dir={'': 'src'},