Adapt Mia to populse_db V3 (#17)

.pre-commit-config.yaml

@@ -14,7 +14,6 @@ repos:
     -   id: check-yaml
     -   id: debug-statements
     -   id: end-of-file-fixer
-    -   id: fix-encoding-pragma
     -   id: name-tests-test
     -   id: trailing-whitespace
 
@@ -45,4 +44,10 @@ repos:
         name: flake8 under python3
         language_version: python3
 
+- repo: https://github.com/asottile/pyupgrade
+  rev: v3.19.1
+  hooks:
+    - id: pyupgrade
+      args: ["--py39-plus"]  # Ensures compatibility with Python 3.9+
+
 exclude: 'docs/doctrees/.*|docs/html/.*|processes/.*'

appveyor.yml

@@ -6,18 +6,22 @@ build: off
 environment:
   GIT_CLONE_PROTECTION_ACTIVE: false
   GIT_LFS_SKIP_SMUDGE: 1
+  POPULSE_DB_BRANCH: 3.0
+  CAPSUL_BRANCH: populsedb3
+  MIA_PROCESSES_BRANCH: db_v3
+  MIA_UT_DATA_BRANCH: bv_v3
 
   matrix:
     - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
-      PYTHON: "C:\\Python39-x64"
+      PYTHON: "C:\\Python310-x64"
 #      APPVEYOR_RDP_PASSWORD: 5fh'D*
 
     - APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2204
-      PYTHON: "$HOME/venv3.9"
+      PYTHON: "$HOME/venv3.10"
 #      APPVEYOR_SSH_KEY: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCinsCtblP8Dlo0IyeE3chxy7Opt1nrrhLWQCUFJBtWr1EjtvZSme00z56yV2c0WWlhKyVUbz56w893WHTjvqgeZuezzGM1vdeD+fQDPBn3WhhOQHNmL2VxDo+f1CFE7ybT6epYPrvdz4B7MqORX8lIOIVVL82GclpDxnJkM7LzMWbOvJ7Gjf12Ob45Nx2ESZxHVv5Z9GeAmMyX2gHu3dlhJQoq6YQ0U/aqkFzfYLbEI5/H78Xmd2YzlxDnzNPB9xbjNqub1KTS+p3JhhSXJgWHeRn/kqwwkOdTnjG1yZoCWgSzCHsQx9oacYciY49Lo5falgIIw3ORJWgk/vjedxDFjBLgYj5NPzGcaxgzsFmMDWyynTSZyysb3HKbydODXV9i40dEnC9gfBQER0m9/cXPcxBK7oVZdZwrxXm0Ke3QGYtRLmh9N2lTUm51/IPBRaDH6/L1se4NgVuT2uytHEYN2t30r14CJhzJ2QGCoas8sxFS9Pt0BWzC4C1WQyZlDN0= econdami@irmage-pteric
 
     - APPVEYOR_BUILD_WORKER_IMAGE: macos-monterey
-      PYTHON: "~/venv3.9"
+      PYTHON: "~/venv3.10"
 #      APPVEYOR_SSH_KEY: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCdDYk1yZyk/W/2UtL1oBEDzX9Mk6YUXJKS3gTrZaK+ScAzPUkm7UlXwwpzSYZjKIkYzpi3WKSRKuDc7VRVk43hIhmwaIzcmpHAXtbgxAGRYembgz+kLDGtkhlP4i8n5GfNWvkkfbS0pu2w2dZQB7EGFFtopfvzBokKltjAYOnP89GCAo4MTfx6kmv1tpm1SDbCAGHJPuN+81B47+v0uGw3KG+AX+1ezPv1MCBQl5V2J4MI6vomyaHz6/UNBZ6rpsrZBYRljwWbyH5E5fqLdep/kO9sEuiXxTrCTLnUPSrR0+h0f8HlgWCaqxO+OUyDoQMI3xsUn4sMpoASi8jkDgd51vGbzkQHyjy2hYzjs47GAvLLtGfTJYQBVcfXEoZaKQUJhqjR7TUdY4bSBOxIPR9qlA9fLFwlQsC3++0m4roM/0SyoDVTz0I6lp1TsYRMKxEJSM90ewCAK8UnN6gEqmOz5ezOFpVsnpz4bZ7vCGaJa/6ODfOhEvl5bFUK40+GxSk= otavio@otavio-UX430UAR
 
 install:
@@ -28,41 +32,123 @@ install:
 #  - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
 #  - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e -
 
-  - cmd: SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%
-  - sh: export PATH=~/venv3.9/bin:~/venv3.9/lib:$PATH
+  - cmd: set PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%
+  - sh: export PATH=~/venv3.10/bin:~/venv3.10/lib:$PATH
   - sh: if [[ $(uname -s) == Linux ]]; then export QT_LOGGING_RULES='*.debug=false;qt.qpa.*=false'; fi
 
+  - cd ..
+
+  # Force recent SQLite version
+  - sh: |
+      if [[ $(uname -s) == Linux ]]; then
+        echo "     ** Removing older SQLite versions **"
+        sudo apt-get remove --purge -y sqlite3 libsqlite3-dev
+        sudo apt-get autoremove -y
+        sudo apt-get update
+        echo "     ** Installing SQLite 3.47 for linux **"
+        sudo apt-get install -y build-essential wget
+        wget https://www.sqlite.org/2024/sqlite-autoconf-3470100.tar.gz
+        tar -xvzf sqlite-autoconf-3470100.tar.gz
+        cd sqlite-autoconf-3470100
+        ./configure --prefix=/usr/local
+        make
+        sudo make install
+        export PATH=/usr/local/bin:$PATH
+        export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
+        cd ..
+      fi
+  - sh: |
+      if [[ $(uname -s) == Darwin ]]; then
+        echo "** Installing SQLite 3.47 for macOS **"
+        curl -L https://www.sqlite.org/2024/sqlite-autoconf-3470100.tar.gz -o sqlite-autoconf-3470100.tar.gz
+        tar -xvzf sqlite-autoconf-3470100.tar.gz
+        cd sqlite-autoconf-3470100
+        ./configure --prefix=/usr/local
+        make
+        sudo make install
+        export PATH=/usr/local/bin:$PATH
+        export DYLD_LIBRARY_PATH=/usr/local/lib:$DYLD_LIBRARY_PATH
+        sqlite3 --version
+        cd ..
+        # Now download and rebuild Python
+        echo "** Rebuilding Python with new SQLite **"
+
+        # Determine current Python version
+        PYTHON_VERSION=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}')")
+        PYTHON_MAJOR_MINOR=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")
+
+        # Download Python source of the same version
+        curl -L https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz -o Python-${PYTHON_VERSION}.tgz
+        tar -xzf Python-${PYTHON_VERSION}.tgz
+        cd Python-${PYTHON_VERSION}
+
+        # Configure with SQLite paths and optimizations
+        ./configure --prefix=/usr/local/opt/python-rebuild \
+        --with-openssl=/usr/local/opt/openssl \
+        --enable-optimizations \
+        --with-system-ffi \
+        LDFLAGS="-L/usr/local/lib" \
+        CPPFLAGS="-I/usr/local/include"
+
+        # Build and install Python
+        make -j $(sysctl -n hw.ncpu)
+        sudo make install
+
+        # Update PATH to use the rebuilt Python
+        export PATH=/usr/local/opt/python-rebuild/bin:$PATH
+
+        # Create symlinks for python3 and pip3
+        sudo ln -sf /usr/local/opt/python-rebuild/bin/python${PYTHON_MAJOR_MINOR} /usr/local/bin/python3
+        sudo ln -sf /usr/local/opt/python-rebuild/bin/pip${PYTHON_MAJOR_MINOR} /usr/local/bin/pip3
+
+        # Verify the SQLite version
+        python3 -c "import sqlite3; print(f'Python is now using SQLite version: {sqlite3.sqlite_version}')"
+
+        # Remove old venv and create a new one with the rebuilt Python
+        echo "** Creating new virtual environment with rebuilt Python **"
+        rm -rf ~/venv3.10
+        /usr/local/opt/python-rebuild/bin/python3 -m venv ~/venv3.10
+        source ~/venv3.10/bin/activate
+
+        cd ..
+      fi
+  - cmd: |
+      echo "     ** Installing SQLite 3.47 for Windows **"
+      mkdir C:\sqlite
+      curl -L -o sqlite-tools-win-x64-3470100.zip https://www.sqlite.org/2024/sqlite-tools-win-x64-3470100.zip
+      tar -xf sqlite-tools-win-x64-3470100.zip -C C:\sqlite
+      set PATH=C:\sqlite;%PATH%
+
   - python --version
   - python -m pip install pip --upgrade
-  - pip -V
-  - pip install --upgrade setuptools wheel
-  - pip install cython --upgrade
-  - pip install psutil
+  - python -m pip -V
+  - python -m pip install --upgrade setuptools wheel
+  - python -m pip install cython --upgrade
+  - python -m pip install psutil
+  - python -m pip show psutil
+  - python -c "import psutil; print(psutil.__path__)"
   - cmd: pip install pypiwin32
-  - pip install pyparsing --upgrade
+  - python -m pip install pyparsing --upgrade
 
-# This is a patch to avoid the "AttributeError: 'PathMetadata' object has no attribute 'isdir'" observed since pip-22.0.2 (see for example UT for commit 50187200b6):
-  - pip install capsul soma-workflow
+  - cd populse-mia
+  - python -m pip install .
 
-  - pip install .
-#  - git lfs install
-#  - git lfs pull
-  - echo y | pip uninstall PyQt5
-  - echo y | pip uninstall PyQt5-sip
-  - pip install PyQt5
+  - echo y | python -m pip uninstall PyQt5
+  - echo y | python -m pip uninstall PyQt5-sip
+  - python -m pip install PyQt5
 
   # We want to use the last github versions instead of the pypi version:
-  - pip uninstall -y populse_db soma-base soma-workflow capsul mia_processes
+  - python -m pip uninstall -y populse_db soma-base soma-workflow capsul mia_processes
 
-  - cmd: git clone --depth=50 --branch=master https://github.com/populse/populse_db.git C:\\projects\\populse_db
-  - sh: if [[ $(uname -s) == Linux ]]; then git clone --depth=50 --branch=master https://github.com/populse/populse_db.git /home/appveyor/projects/populse_db; fi
-  - sh: if [[ $(uname -s) == Darwin ]]; then git clone --depth=50 --branch=master https://github.com/populse/populse_db.git /Users/appveyor/projects/populse_db; fi
+  - cmd: git clone --depth=50 --branch=%POPULSE_DB_BRANCH% https://github.com/populse/populse_db.git C:\\projects\\populse_db
+  - sh: if [[ $(uname -s) == Linux ]]; then git clone --depth=50 --branch=$POPULSE_DB_BRANCH https://github.com/populse/populse_db.git /home/appveyor/projects/populse_db; fi
+  - sh: if [[ $(uname -s) == Darwin ]]; then git clone --depth=50 --branch=$POPULSE_DB_BRANCH https://github.com/populse/populse_db.git /Users/appveyor/projects/populse_db; fi
 
   - cmd: cd C:\\projects\\populse_db
   - sh: if [[ $(uname -s) == Linux ]]; then cd /home/appveyor/projects/populse_db; fi
   - sh: if [[ $(uname -s) == Darwin ]]; then cd /Users/appveyor/projects/populse_db; fi
 
-  - pip install -e .[postgres]
+  - python -m pip install -e .[postgres]
   - cd ..
 
   - cmd: git clone --depth=50 --branch=master https://github.com/populse/soma-base.git C:\\projects\\soma-base
@@ -73,18 +159,18 @@ install:
   - sh: if [[ $(uname -s) == Linux ]]; then git clone --depth=50 --branch=master https://github.com/populse/soma-workflow.git /home/appveyor/projects/soma-workflow; fi
   - sh: if [[ $(uname -s) == Darwin ]]; then git clone --depth=50 --branch=master https://github.com/populse/soma-workflow.git /Users/appveyor/projects/soma-workflow; fi
 
-  - cmd: git clone --depth=50 --branch=master https://github.com/populse/capsul.git C:\\projects\\capsul
-  - sh: if [[ $(uname -s) == Linux ]]; then git clone --depth=50 --branch=master https://github.com/populse/capsul.git /home/appveyor/projects/capsul; fi
-  - sh: if [[ $(uname -s) == Darwin ]]; then git clone --depth=50 --branch=master https://github.com/populse/capsul.git /Users/appveyor/projects/capsul; fi
+  - cmd: git clone --depth=50 --branch=%CAPSUL_BRANCH% https://github.com/populse/capsul.git C:\\projects\\capsul
+  - sh: if [[ $(uname -s) == Linux ]]; then git clone --depth=50 --branch=$CAPSUL_BRANCH https://github.com/populse/capsul.git /home/appveyor/projects/capsul; fi
+  - sh: if [[ $(uname -s) == Darwin ]]; then git clone --depth=50 --branch=$CAPSUL_BRANCH https://github.com/populse/capsul.git /Users/appveyor/projects/capsul; fi
 
-  - cmd: git clone --depth=50 --branch=master https://github.com/populse/mia_processes.git C:\\projects\\mia_processes
-  - sh: if [[ $(uname -s) == Linux ]]; then git clone --depth=50 --branch=master https://github.com/populse/mia_processes.git /home/appveyor/projects/mia_processes; fi
-  - sh: if [[ $(uname -s) == Darwin ]]; then git clone --depth=50 --branch=master https://github.com/populse/mia_processes.git /Users/appveyor/projects/mia_processes; fi
+  - cmd: git clone --depth=50 --branch=%MIA_PROCESSES_BRANCH% https://github.com/populse/mia_processes.git C:\\projects\\mia_processes
+  - sh: if [[ $(uname -s) == Linux ]]; then git clone --depth=50 --branch=$MIA_PROCESSES_BRANCH https://github.com/populse/mia_processes.git /home/appveyor/projects/mia_processes; fi
+  - sh: if [[ $(uname -s) == Darwin ]]; then git clone --depth=50 --branch=$MIA_PROCESSES_BRANCH https://github.com/populse/mia_processes.git /Users/appveyor/projects/mia_processes; fi
 
 # Download data for unit tests
-  - cmd: git clone --depth=50 --branch=main https://gricad-gitlab.univ-grenoble-alpes.fr/mia/mia_ut_data.git C:\\projects\\populse-mia\\mia_ut_data
-  - sh: if [[ $(uname -s) == Linux ]]; then git clone --depth=50 --branch=main https://gricad-gitlab.univ-grenoble-alpes.fr/mia/mia_ut_data.git /home/appveyor/projects/populse-mia/mia_ut_data; fi
-  - sh: if [[ $(uname -s) == Darwin ]]; then git clone --depth=50 --branch=main https://gricad-gitlab.univ-grenoble-alpes.fr/mia/mia_ut_data.git /Users/appveyor/projects/populse-mia/mia_ut_data; fi
+  - cmd: git clone --depth=50 --branch=%MIA_UT_DATA_BRANCH% https://gricad-gitlab.univ-grenoble-alpes.fr/mia/mia_ut_data.git C:\\projects\\populse-mia\\mia_ut_data
+  - sh: if [[ $(uname -s) == Linux ]]; then git clone --depth=50 --branch=$MIA_UT_DATA_BRANCH https://gricad-gitlab.univ-grenoble-alpes.fr/mia/mia_ut_data.git /home/appveyor/projects/populse-mia/mia_ut_data; fi
+  - sh: if [[ $(uname -s) == Darwin ]]; then git clone --depth=50 --branch=$MIA_UT_DATA_BRANCH https://gricad-gitlab.univ-grenoble-alpes.fr/mia/mia_ut_data.git /Users/appveyor/projects/populse-mia/mia_ut_data; fi
 
   - cd populse-mia
 
@@ -97,13 +183,13 @@ install:
 #  - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
 
 test_script:
-  - cmd: python .\\populse_mia\\test.py -v
+  - cmd: python .\\populse_mia\\tests\\run_mia_test.py -v
 
-  - sh: if [[ $(uname -s) == Darwin ]]; then python3 ./populse_mia/test.py -v; fi
+  - sh: if [[ $(uname -s) == Darwin ]]; then python3 ./populse_mia/tests/run_mia_test.py -v; fi
 
   - sh: if [[ $(uname -s) == Linux ]]; then if [ -z $XDG_RUNTIME_DIR ]; then export XDG_RUNTIME_DIR="/tmp/runtime-$(id -un)"; if [ ! -d "/tmp/runtime-$(id -un)" ]; then mkdir "/tmp/runtime-$(id -un)"; chmod 0700 "/tmp/runtime-$(id -un)"; fi; fi; fi
-  - sh: if [[ $(uname -s) == Linux ]]; then xvfb-run python3 ./populse_mia/test.py -v; fi
-  - sh: if [[ $(uname -s) == Linux ]]; then xvfb-run coverage run ./populse_mia/test.py -v; fi
+  - sh: if [[ $(uname -s) == Linux ]]; then xvfb-run python ./populse_mia/tests/run_mia_test.py -v; fi
+  - sh: if [[ $(uname -s) == Linux ]]; then xvfb-run coverage run ./populse_mia/tests/run_mia_test.py -v; fi
   - sh: if [[ $(uname -s) == Linux ]]; then coverage xml -i; fi
 
 

docs/source/conf.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 """Configuration file for the Sphinx documentation builder."""
 
 # This file does only contain a selection of the most common options. For a

populse_mia/__init__.py

@@ -1,39 +1,39 @@
-# -*- coding: utf-8 -*-
 """
-MIA is shorthand for “Multiparametric Image Analysis”. It is intended to be
-a complete image processing environment mainly targeted at the analysis and
-visualization of large amounts of MRI data.
-
-MRI data analysis often requires a complex succession of data processing
-pipelines applied to a set of data acquired in an MRI exam or over several
-MRI exams. This analysis may need to be repeated a large number of times in
-studies involving a large number of acquisition sessions. Such that manual
-execution of the processing modules or simple ad-hoc scripting of the
-process may become error-prone, cumbersome and difficult to reproduce. Data
-processing pipelines exist in separate heterogeneous toolboxes, developed
-in-house or by other researchers in the field. This heterogeneity adds to
-the complexity of the modules are to be invoked manually.
-
-MIA (populse_mia) aims to provide easy tools to perform complex data processing
-based on a definition of the inputs and outputs of the individual pipelines
-on a conceptual level, and implies identifying data with respect to their
-role in an analysis project: “the scan type”, “the subject being scanned”,
-“the group of this subject is part of”, etc.
-
-
-:Contains:
-    :Packages:
-      - data_manager: handles the projects and their database
-      - sources_images: a collection of images used in Mia
-      - user_interface: mainly intended for the graphic interface of Mia
-      - utils: various utilities for the Mia operation
-
-    :Modules:
-      - info.py: define software version, description and requirements
-      - main.py: the first module used at the runtime of mia
-      - __main__.py: make populse_mia accessible as a module
-      - software_properties.py: handles the configuration of the software
-      - test.py: unitary tests
+MIA (Multiparametric Image Analysis) is a comprehensive image processing
+environment primarily designed for the analysis and visualization of large
+MRI datasets.
+
+MRI data analysis often requires executing complex processing pipelines on
+datasets acquired during single or multiple MRI exams. In large-scale studies,
+where data processing must be repeated across numerous acquisition sessions,
+manually running processing modules or relying on simple ad-hoc scripts can
+become error-prone, cumbersome, and difficult to reproduce. Additionally, data
+processing pipelines are often distributed across various heterogeneous
+toolboxes, developed either in-house or by other researchers, further
+increasing the complexity of manually invoking these modules.
+
+MIA (populse_mia) aims to simplify complex data processing by providing
+intuitive tools that define inputs and outputs at a conceptual level. It
+organizes data based on their roles in an analysis project, such
+as “scan type,” “subject,” or “subject group,” making it easier to structure
+and automate workflows.
+
+Contents:
+
+  - Packages
+
+    - data_manager: Manages projects and their associated databases.
+    - sources_images: Contains a collection of images used in MIA.
+    - user_interface: Handles the graphical user interface of MIA.
+    - utils: Provides various utility functions for MIA operations.
+
+  - Modules
+
+    - info.py: Defines software version, description, and requirements.
+    - main.py: The primary module executed at runtime.
+    - __main__.py: Allows populse_mia to be executed as a module.
+    - software_properties.py: Manages the software’s configuration.
+    - test.py: Contains unit tests.
 
 """
 

populse_mia/__main__.py

@@ -1,7 +1,10 @@
-# -*- coding: utf-8 -*-
-"""Make populse_mia accessible as a module.
+"""
+Enable populse_mia to be run as a module.
+
+This allows running populse_mia using:
+    python3 -m populse_mia
 
-For example by using python3 -m populse_mia.
+This will execute the main entry point of the application.
 
 """
 
@@ -17,4 +20,9 @@
 from populse_mia.main import main
 
 if __name__ == "__main__":
-    main()
+
+    try:
+        main()
+
+    except Exception as e:
+        print(f"Error while running populse_mia: {e}")