Merge branch 'master' into issues/5152-fix-bug-in-coordination-dir-ov…

…erride

.gitlab-ci.yml

@@ -5,7 +5,7 @@ variables:
   PYTHONIOENCODING: "utf-8"
   DEBIAN_FRONTEND: "noninteractive"
   TOIL_OWNER_TAG: "shared"
-  MAIN_PYTHON_PKG: "python3.12"
+  MAIN_PYTHON_PKG: "python3.13"
   # Used to tell pytest which tests to be run by specifying markers,
   # Allows partitioning of tests to prevent duplicate running of tests in different jobs.
   # Currently specifies special tests that are not run by quick_test_offline.
@@ -191,6 +191,12 @@ py311_main:
     - TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib
 
 py312_appliance_build:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_COMMIT_TAG
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
+    - if: $CI_COMMIT_BRANCH =~ /.*3\.12.*/
   stage: basic_tests
   script:
     - pwd
@@ -201,14 +207,39 @@ py312_appliance_build:
 
 py312_main:
   rules:
-    - if: $CI_PIPELINE_SOURCE != "schedule"
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_COMMIT_TAG
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
+    - if: $CI_COMMIT_BRANCH =~ /.*3\.12.*/
   stage: basic_tests
   script:
     - pwd
     - python3.12 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==23.6.1'
     - make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils"
     - TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib
 
+py313_appliance_build:
+  stage: basic_tests
+  script:
+    - pwd
+    - python3.13 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.13 &&  pip install -U pip wheel && pip install -U build && make prepare && pip install pycparser && make develop extras=[all]
+    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
+    - python setup_gitlab_docker.py
+    - make push_docker
+
+py313_main:
+  rules:
+    - if: $CI_PIPELINE_SOURCE != "schedule"
+  stage: basic_tests
+  script:
+    - pwd
+    # todo: htcondor is not out for python 3.13 yet, we don't actively test htcondor batchsystems but should still test an htcondor install
+    - python3.13 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.13 && pip install -U pip wheel && make prepare && make develop extras=[all]
+    - make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils"
+    - TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib
+
+
 slurm_test:
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"

contrib/slurm-test/slurm_test.sh

@@ -12,14 +12,13 @@ docker cp -L sort.py ${LEADER}:/home/admin
 docker cp fileToSort.txt ${LEADER}:/home/admin
 docker cp toil_workflow.py ${LEADER}:/home/admin
 GIT_COMMIT=$(git rev-parse HEAD)
-docker exec ${LEADER} sudo apt install python3-pip -y
-docker exec ${LEADER} pip3 install "git+https://github.com/DataBiosphere/toil.git@${GIT_COMMIT}"
+docker exec ${LEADER} python3.9 -m pip install "git+https://github.com/DataBiosphere/toil.git@${GIT_COMMIT}"
 docker exec ${LEADER} sinfo -N -l
 # Test 1: A really basic workflow to check Slurm is working correctly
-docker exec ${LEADER} python3 /home/admin/toil_workflow.py file:my-job-store --batchSystem slurm --disableCaching --retryCount 0 --batchLogsDir ./nonexistent/paths
+docker exec ${LEADER} python3.9 /home/admin/toil_workflow.py file:my-job-store --batchSystem slurm --disableCaching --retryCount 0 --batchLogsDir ./nonexistent/paths
 docker cp ${LEADER}:/home/admin/output.txt output_Docker.txt
 # Test 2: Make sure that "sort" workflow runs under slurm
-docker exec ${LEADER} python3 /home/admin/sort.py file:my-job-store --batchSystem slurm --disableCaching --retryCount 0
+docker exec ${LEADER} python3.9 /home/admin/sort.py file:my-job-store --batchSystem slurm --disableCaching --retryCount 0
 docker cp ${LEADER}:/home/admin/sortedFile.txt sortedFile.txt
 docker compose stop
 ./check_out.sh

contrib/toil-ci-prebake/Dockerfile

@@ -36,12 +36,13 @@ RUN sudo add-apt-repository -y ppa:deadsnakes/ppa && \
         python3.9 python3.9-dev python3.9-venv python3.9-distutils \
         python3.10 python3.10-dev python3.10-venv python3.10-distutils \
         python3.11 python3.11-dev python3.11-venv python3.11-distutils \
-        python3.12 python3.12-dev python3.12-venv python3.12-distutils
+        python3.12 python3.12-dev python3.12-venv python3.12-distutils \
+        python3.13 python3.13-dev python3.13-venv
 
 RUN rm -f /usr/bin/python
 RUN ln -s /usr/bin/python3.11 /usr/bin/python
-RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3.8 get-pip.py && python3.9 get-pip.py && python3.10 get-pip.py && python3.11 get-pip.py && python3.12 get-pip.py # for python 3.12, get-pip seems to remove other pip versions
-RUN python3.12 -m pip install virtualenv -U && python3.8 -m pip install virtualenv && python3.9 -m pip install virtualenv && python3.10 -m pip install virtualenv && python3.11 -m pip install virtualenv # python3.12 needs latest version of virtualenv
+RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3.8 get-pip.py && python3.9 get-pip.py && python3.10 get-pip.py && python3.11 get-pip.py && python3.12 get-pip.py && python3.13 get-pip.py # for python 3.12, get-pip seems to remove other pip versions, python 3.13 needs get-pip.py or ensure-pip as older versions of pip relies on distutils which is removed in 3.13
+RUN python3.13 -m pip install virtualenv -U && python3.12 -m pip install virtualenv -U && python3.8 -m pip install virtualenv && python3.9 -m pip install virtualenv && python3.10 -m pip install virtualenv && python3.11 -m pip install virtualenv # python3.12 needs latest version of virtualenv
 
 
 LABEL maintainers="[email protected] [email protected]"

docker/Dockerfile.py

@@ -28,7 +28,8 @@
 python_packages = {'python3.9': ['python3.9-distutils', 'python3.9-venv'],
                    'python3.10': ['python3.10-distutils', 'python3.10-venv'],
                    'python3.11': ['python3.11-distutils', 'python3.11-venv'],
-                   'python3.12': ['python3.12-distutils', 'python3.12-venv']}
+                   'python3.12': ['python3.12-distutils', 'python3.12-venv'],
+                   'python3.13': ['python3.13-venv']}  # python3.13 removed distutils
 
 dependencies = ' '.join(python_packages[python] +
                         ['libffi-dev',  # For client side encryption for extras with PyNACL
@@ -110,11 +111,12 @@ def heredoc(s):
     # Find a repo with a Mesos build.
     # This one was archived like:
     # mkdir mesos-repo && cd mesos-repo
-    # wget --recursive --restrict-file-names=windows -k --convert-links --no-parent --page-requisites https://rpm.aventer.biz/Ubuntu/ https://www.aventer.biz/assets/support_aventer.asc https://rpm.aventer.biz/README.txt
+    # wget --recursive --restrict-file-names=windows -k --convert-links --no-parent --page-requisites -m https://rpm.aventer.biz/Ubuntu/ https://www.aventer.biz/assets/support_aventer.asc https://rpm.aventer.biz/README.txt
     # ipfs add -r .
-    RUN echo "deb https://public.gi.ucsc.edu/~anovak/outbox/toil/ipfs/QmeaErHzK4Dajz2mCMd36eUDQp7GX2bSECVRpGfrqdragR/rpm.aventer.biz/Ubuntu/focal focal main" \
+    # It contains a GPG key that will expire 2026-09-28
+    RUN echo "deb https://public.gi.ucsc.edu/~anovak/outbox/toil/ipfs/QmRXnGNiWk523zgNkuamENVkghMJ2zJtinVfgjHbc4Dcpr/rpm.aventer.biz/Ubuntu/focal focal main" \
         > /etc/apt/sources.list.d/mesos.list \
-        && curl https://public.gi.ucsc.edu/~anovak/outbox/toil/ipfs/QmeaErHzK4Dajz2mCMd36eUDQp7GX2bSECVRpGfrqdragR/www.aventer.biz/assets/support_aventer.asc | apt-key add -
+        && curl https://public.gi.ucsc.edu/~anovak/outbox/toil/ipfs/QmRXnGNiWk523zgNkuamENVkghMJ2zJtinVfgjHbc4Dcpr/www.aventer.biz/assets/support_aventer.asc | apt-key add -
 
     RUN apt-get -y update --fix-missing && \
         DEBIAN_FRONTEND=noninteractive apt-get -y upgrade && \

requirements-cwl.txt

@@ -1,4 +1,4 @@
-cwltool==3.1.20241024121129
+cwltool==3.1.20241112140730
 schema-salad>=8.4.20230128170514,<9
 galaxy-tool-util<25
 galaxy-util<25

requirements-dev.txt

@@ -14,7 +14,7 @@ types-aws-xray-sdk
 types-boto<2.49.18.20241020
 types-Flask-Cors
 types-requests
-types-psutil
+types-psutil>=6.1.0.20241102
 types-python-dateutil
 types-setuptools
 types-xmltodict

requirements.txt

@@ -3,7 +3,7 @@ requests<=2.31.0
 docker>=6.1.0, <8
 urllib3>=1.26.0,<3
 python-dateutil
-psutil >= 3.0.1, < 7
+psutil >= 6.1.0, < 7
 PyPubSub >=4.0.3, <5
 addict>=2.2.1, <2.5
 backports.zoneinfo[tzdata];python_version<"3.9"

setup.py

@@ -95,6 +95,7 @@ def run_setup():
             "Programming Language :: Python :: 3.10",
             "Programming Language :: Python :: 3.11",
             "Programming Language :: Python :: 3.12",
+            "Programming Language :: Python :: 3.13",
             "Topic :: Scientific/Engineering",
             "Topic :: Scientific/Engineering :: Bio-Informatics",
             "Topic :: Scientific/Engineering :: Astronomy",

src/toil/common.py

@@ -777,29 +777,47 @@ def check_arguments(typ: str) -> None:
 
     # if cwl is set, format the namespace for cwl and check that wdl options are not set on the command line
     if cwl:
-        parser.add_argument("cwltool", type=str, help="CWL file to run.")
-        parser.add_argument(
-            "cwljob",
-            nargs="*",
-            help="Input file or CWL options. If CWL workflow takes an input, "
-            "the name of the input can be used as an option. "
-            'For example: "%(prog)s workflow.cwl --file1 file". '
-            "If an input has the same name as a Toil option, pass '--' before it.",
-        )
+        # So we can manually write out the help for this and the inputs
+        # file/workflow options in the argument parser description, we suppress
+        # help for this option.
+        parser.add_argument("cwltool", metavar="WORKFLOW", type=str, help=SUPPRESS)
+        # We also need a "cwljob" command line argument, holding possibly a
+        # positional input file and possibly a whole string of option flags
+        # only known to the workflow.
+        #
+        # We don't want to try and parse out the positional argument here
+        # since, on Python 3.12, we can grab what's really supposed to be an
+        # argument to a workflow-defined option.
+        #
+        # We don't want to use the undocumented argparse.REMAINDER, since that
+        # will eat any Toil-defined option flags after the first positional
+        # argument.
+        #
+        # So we just use parse_known_args and dump all unknown args into it,
+        # and manually write help text in the argparse description. So don't
+        # define it here.
         check_arguments(typ="cwl")
 
     # if wdl is set, format the namespace for wdl and check that cwl options are not set on the command line
     if wdl:
         parser.add_argument("wdl_uri", type=str, help="WDL document URI")
+        # We want to have an inputs_url that can be either a positional or a flag.
+        # We can't just have them share a single-item dest in Python 3.12;
+        # argparse does not guarantee that will work, and we can get the
+        # positional default value clobbering the flag. See
+        # <https://stackoverflow.com/a/60531838>.
+        # So we make them accumulate to the same list.
+        # Note that we will get a None in the list when there's no positional inputs.
         parser.add_argument(
-            "inputs_uri", type=str, nargs="?", help="WDL input JSON URI"
+            "inputs_uri", type=str, nargs='?', action="append", help="WDL input JSON URI"
         )
         parser.add_argument(
             "--input",
             "--inputs",
             "-i",
             dest="inputs_uri",
             type=str,
+            action="append",
             help="WDL input JSON URI",
         )
         check_arguments(typ="wdl")