Merge branch 'master' into jobstack

.gitlab-ci.yml

@@ -5,9 +5,11 @@ variables:
   PYTHONIOENCODING: "utf-8"
   DEBIAN_FRONTEND: "noninteractive"
   TOIL_OWNER_TAG: "shared"
-  MAIN_PYTHON_PKG: "python3.8"
+  MAIN_PYTHON_PKG: "python3.9"
 
 before_script:
+  # Log where we are running, in case some Kubernetes hosts are busted. IPs are assigned per host.
+  - ip addr
   # Configure Docker to use a mirror for Docker Hub and restart the daemon
   # Set the registry as insecure because it is probably cluster-internal over plain HTTP.
   - |
@@ -33,7 +35,7 @@ before_script:
   - sudo apt-get install -y software-properties-common build-essential virtualenv
   - sudo add-apt-repository -y ppa:deadsnakes/ppa
   - sudo apt-get update
-  - sudo apt-get install -y tzdata jq python3.6 python3.6-dev python3.7 python3.7-dev python3.8 python3.8-dev
+  - sudo apt-get install -y tzdata jq python3.7 python3.7-dev python3.8 python3.8-dev python3.9 python3.9-dev python3.9-distutils  # python3.10 python3.10-dev python3.10-venv
 
 after_script:
   # We need to clean up any files that Toil may have made via Docker that
@@ -76,33 +78,72 @@ wdl_dependency_is_stand_alone:
     - virtualenv -p ${MAIN_PYTHON_PKG} venv && . venv/bin/activate && make prepare && make develop extras=[wdl]
     - make test tests=src/toil/test/wdl/toilwdlTest.py::ToilWdlIntegrationTest::testMD5sum
 
-# Tests that used to be on Travis
-# The Makefile sets "TRAVIS=true", which is now a misnomer, but we don't need
-# to also do it here.
 quick_test_offline:
   stage: basic_tests
   script:
-    - virtualenv -p python3.6 venv
+    - virtualenv -p ${MAIN_PYTHON_PKG} venv
     - . venv/bin/activate
     - pip install -U pip wheel
     - make prepare
     - make develop extras=[aws,google,wdl]
     - TOIL_TEST_QUICK=True make test_offline
 
-# Python3.6
-py36_main:
+# Python3.7
+py37_main:
+  stage: basic_tests
+  script:
+    - pwd
+    - virtualenv -p python3.7 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor awscli'
+    - make test tests=src/toil/test/src
+    - make test tests=src/toil/test/utils
+    - make test tests=src/toil/test/lib/test_ec2.py
+    - make test tests=src/toil/test/lib/aws
+    - make test tests=src/toil/test/lib/test_conversions.py
+
+py37_appliance_build:
+  stage: basic_tests
+  script:
+    - pwd
+    - virtualenv -p python3.7 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor awscli'
+    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
+    - python setup_gitlab_docker.py
+    - make push_docker
+
+# Python3.8
+py38_main:
+  stage: basic_tests
+  script:
+    - pwd
+    - virtualenv -p python3.8 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor awscli'
+    - make test tests=src/toil/test/src
+    - make test tests=src/toil/test/utils
+    - make test tests=src/toil/test/lib/test_ec2.py
+    - make test tests=src/toil/test/lib/aws
+    - make test tests=src/toil/test/lib/test_conversions.py
+
+py38_appliance_build:
+  stage: basic_tests
+  script:
+    - pwd
+    - virtualenv -p python3.8 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor awscli'
+    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
+    - python setup_gitlab_docker.py
+    - make push_docker
+
+# Python3.9
+py39_main:
   stage: basic_tests
   script:
     - pwd
-    - virtualenv -p python3.6 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages=htcondor
+    - virtualenv -p python3.9 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages=htcondor
     - make test tests=src/toil/test/src
     - make test tests=src/toil/test/utils
 
-py36_appliance_build:
+py39_appliance_build:
   stage: basic_tests
   script:
     - pwd
-    - virtualenv -p python3.6 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor awscli'
+    - virtualenv -p python3.9 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor awscli'
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
     - make push_docker
@@ -243,24 +284,6 @@ jobstore_and_provisioning:
 # https://ucsc-ci.com/databiosphere/toil/-/jobs/38672
 # guessing decorators are masking class as function?  ^  also, abstract class is run as normal test?  should hide.
 
-main:
-  stage: basic_tests
-  script:
-    - pwd
-    - virtualenv -p ${MAIN_PYTHON_PKG} venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages=htcondor
-    - make test tests=src/toil/test/src
-    - make test tests=src/toil/test/utils
-#    - make test tests=src/toil/test/docs/scriptsTest.py::ToilDocumentationTest::testDocker
-
-appliance_build:
-  stage: basic_tests
-  script:
-    - pwd
-    - virtualenv -p ${MAIN_PYTHON_PKG} venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor awscli'
-    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
-    - python setup_gitlab_docker.py
-    - make push_docker
-
 integration:
   stage: integration
   script:
@@ -306,29 +329,6 @@ google_jobstore:
     - export TOIL_GOOGLE_PROJECTID=toil-dev
     - make test tests=src/toil/test/jobStores/jobStoreTest.py::GoogleJobStoreTest
 
-# Python3.7
-py37_main:
-  stage: basic_tests
-  script:
-    - pwd
-    - virtualenv -p python3.7 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor awscli'
-    - make test tests=src/toil/test/src
-    - make test tests=src/toil/test/utils
-    - make test tests=src/toil/test/lib/test_ec2.py
-    - make test tests=src/toil/test/lib/aws
-    - make test tests=src/toil/test/lib/test_conversions.py
-
-py37_appliance_build:
-  stage: basic_tests
-  script:
-    - pwd
-    - virtualenv -p python3.7 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor awscli'
-    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
-    - python setup_gitlab_docker.py
-    - make push_docker
-
-# Python 3.8 is the "main" python already and doesn't need its own tests.
-
 # Cactus-on-Kubernetes integration (as a script and not a pytest test)
 cactus_integration:
   stage: integration

Makefile

@@ -83,7 +83,7 @@ tests=src/toil/test
 cov="--cov=toil"
 extras=
 # You can say make develop packages=xxx to install packages in the same Python
-# environemnt as Toil itself without creating dependency conflicts with Toil
+# environment as Toil itself without creating dependency conflicts with Toil
 packages=
 sdist_name:=toil-$(shell python version_template.py distVersion).tar.gz
 
@@ -118,25 +118,21 @@ clean_sdist:
 	- rm -rf dist
 	- rm src/toil/version.py
 
-# We always claim to be Travis, so that local test runs will not skip Travis tests.
 # Setting SET_OWNER_TAG will tag cloud resources so that UCSC's cloud murder bot won't kill them.
 test: check_venv check_build_reqs
-	TRAVIS=true TOIL_OWNER_TAG="shared" \
+	TOIL_OWNER_TAG="shared" \
 	    python -m pytest --durations=0 --log-level DEBUG --log-cli-level INFO -r s $(cov) $(tests)
 
 
 # This target will skip building docker and all docker based tests
-# these are our travis tests; rename?
 test_offline: check_venv check_build_reqs
 	@printf "$(cyan)All docker related tests will be skipped.$(normal)\n"
 	TOIL_SKIP_DOCKER=True \
-	TRAVIS=true \
 	    python -m pytest -vv --timeout=600 --log-level DEBUG --log-cli-level INFO $(cov) $(tests)
 
 ifdef TOIL_DOCKER_REGISTRY
 
 docker_image:=$(TOIL_DOCKER_REGISTRY)/$(TOIL_DOCKER_NAME)
-
 grafana_image:=$(TOIL_DOCKER_REGISTRY)/toil-grafana
 prometheus_image:=$(TOIL_DOCKER_REGISTRY)/toil-prometheus
 mtail_image:=$(TOIL_DOCKER_REGISTRY)/toil-mtail

attic/README.md

@@ -245,8 +245,8 @@ other jobs in a simple way.
 
 The basic pattern provided by toil is as follows:
 
-1. You have a job running on your cluster which requires further parallelisation.
-2. You create a list of jobs to perform this parallelisation. These are the 'child' jobs of your process, we call them collectively the 'children'.
+1. You have a job running on your cluster which requires further parallelization.
+2. You create a list of jobs to perform this parallelization. These are the 'child' jobs of your process, we call them collectively the 'children'.
 3. You create a 'follow-on' job, to be performed after all the children have successfully completed. This job is responsible for cleaning up the input files created for the children and doing any further processing. Children should not cleanup files created by parents, in case of a batch system failure which requires the child to be re-run (see 'Atomicity' below).
 4. You end your current job successfully.
 5. The batch system runs the children. These jobs may in turn have children and follow-on jobs.
@@ -312,7 +312,7 @@ Job.makeJobFnJob(setup, (fileToSort, N))
 
 Notice that the child and follow-on jobs have also been refactored as functions, hence the methods **[addChildJobFn](https://github.com/benedictpaten/toil/blob/development/scriptTree/job.py#L82)** and **[setFollowOnFn](https://github.com/benedictpaten/toil/blob/development/scriptTree/job.py#L67)**, which take functions as opposed to Job objects.
 
-Note, there are two types of functions you can wrap - **job functions**, whose first argument must be the wrapping job object (the setup function above is an excample of a job function), and plain functions that do not have a reference to the wrapping job.
+Note, there are two types of functions you can wrap - **job functions**, whose first argument must be the wrapping job object (the setup function above is an example of a job function), and plain functions that do not have a reference to the wrapping job.
 
 ##Creating a scriptTree script:
 
@@ -392,7 +392,7 @@ toil replicates the environment in which toil or scriptTree is invoked and provi
 
     Toil checkpoints its state on disk, so that it or the job manager can be wiped out and restarted. There is some gnarly test code to show how this works, it will keep crashing everything, at random points, but eventually everything will complete okay. As a user you needn't worry about any of this, but your child jobs must be atomic (as with all batch systems), and must follow the convention regarding input files.
 
-* _How scaleable?_
+* _How scalable?_
 
     We have tested having 1000 concurrent jobs running on our cluster. This will depend on the underlying batch system being used.
 

common.mk

@@ -67,7 +67,6 @@ export TOIL_AWS_NODE_DEBUG?=False  # Don't shut down EC2 instances that fail so
 export TOIL_TEST_INTEGRATIVE?=False  # If ``True``, this allows the integration tests to run.
 export TOIL_TEST_QUICK?=False  # If ``True``, long running tests are skipped.
 export TOIL_SKIP_DOCKER?=False  # Skip docker dependent tests
-export TRAVIS?=True  # Run tests for travis (shorter unit tests)
 
 export TOIL_AWS_KEYNAME?=id_rsa  # SSH key to use for tests in AWS.
 export TOIL_GOOGLE_KEYNAME?=id_rsa  # SSH key to use for tests in google.

contrib/admin/mypy-with-ignore.py

@@ -28,7 +28,6 @@ def main():
         'docs/vendor/sphinxcontrib/__init__.py',
         'src/toil/job.py',
         'src/toil/leader.py',
-        'src/toil/common.py',
         'src/toil/__init__.py',
         'src/toil/resource.py',
         'src/toil/deferred.py',
@@ -41,7 +40,6 @@ def main():
         'src/toil/wdl/versions/draft2.py',
         'src/toil/wdl/versions/v1.py',
         'src/toil/wdl/versions/dev.py',
-        'src/toil/provisioners/clusterScaler.py',
         'src/toil/provisioners/abstractProvisioner.py',
         'src/toil/provisioners/gceProvisioner.py',
         'src/toil/provisioners/__init__.py',