.gitlab-ci.yml

default:
  image: condaforge/linux-anvil-cos7-x86_64:latest

stages:
  - custom
  - lint
  - build
  - test
  - doc
  - deploy

# === Variables ===

variables:
  PACKAGE_VERSION: "0.1.7"
  PYTHON_VERSION: "3.8"
  GIT_LFS_SKIP_SMUDGE: 1

# === Configurations ===

.skip-custom-pipelines:
  except:
    variables:
      - $UPDATE_PAGES
      - $BUILD_IMAGE_WITH_TAG
      - $MIRROR_TO_GITHUB
      - $TEST_PIP_INSTALL

.configure-conda:
  # Set conda envs and pkgs dirs
  script: &configure-conda
    - |
      cat <<EOF > ~/.condarc
      channel_priority: true
      channels:
        - pytorch
        - conda-forge
        - defaults
        - kimlab
        - ostrokach-forge
        - bioconda
        - salilab
        - https://${KIMLAB_CONDA_LOGIN}@conda.proteinsolver.org
        - https://${PYROSETTA_CONDA_LOGIN}@conda.graylab.jhu.edu
        - omnia
      EOF
    - conda install -yq mamba

# === Lint ===

lint:
  stage: lint
  extends:
    - .skip-custom-pipelines
  before_script:
    - *configure-conda
  script:
    - mamba create -n lint -yq "python=${PYTHON_VERSION}" isort toml flake8 mypy black
    - source activate lint
    - python -m isort -p elaspic2 -c .
    - python -m flake8
    - python -m black --config pyproject.toml --check .
    # MyPy does not support namespace packages until this issue gets resolved:
    # https://github.com/python/mypy/issues/1645
    - python -m mypy src || true

# === Build ===

build:
  stage: build
  extends:
    - .skip-custom-pipelines
  before_script:
    - *configure-conda
  script:
    - mamba install -yq conda conda-build conda-verify conda-forge-pinning
    - cd "${CI_PROJECT_DIR}/.gitlab/conda"
    - >
      mamba build .
      --no-test
      --variant-config-files /opt/conda/conda_build_config.yaml
      --variants "{python: [$PYTHON_VERSION], numpy: [1.16], python_impl: [cpython]}"
      --output-folder "$CI_PROJECT_DIR/conda-bld"
  artifacts:
    paths:
      - conda-bld

# === Test ===

.install-ssh-client:
  script: &install-ssh-client
    - "which ssh-agent || ( apt-get install -y -qq -o=Dpkg::Use-Pty=0 openssh-client -y )"
    - eval $(ssh-agent -s)
    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh
    - echo "$KNOWN_HOSTS" >> ~/.ssh/known_hosts

test:
  stage: test
  image: ubuntu:20.04
  extends:
    - .skip-custom-pipelines
  before_script:
    # Install global dependencies
    - apt-get update -y -qq -o=Dpkg::Use-Pty=0
    - apt-get install -y -qq -o=Dpkg::Use-Pty=0 curl rsync gettext-base

    # Install ssh client
    - *install-ssh-client

    # Test that ssh client works
    - ssh strokach@conda-envs.proteinsolver.org "echo hello"

    # Install conda
    - curl -s -L https://repo.anaconda.com/miniconda/Miniconda3-py38_4.8.3-Linux-x86_64.sh > miniconda.sh
    - md5sum miniconda.sh | grep d63adf39f2c220950a063e0529d4ff74
    - sh miniconda.sh -b -p /opt/conda
    - source /opt/conda/etc/profile.d/conda.sh
    - *configure-conda
    - conda activate root
    - conda install -n root -c conda-forge -yq conda-pack
  script:
    # Create conda environment
    - envsubst < .gitlab/conda-env/environment.yml > .gitlab/conda-env/environment-subst.yml
    - cat .gitlab/conda-env/environment-subst.yml
    - mamba env create -q -n ${CI_PROJECT_NAME} -f .gitlab/conda-env/environment-subst.yml

    # Install pypi packages
    - conda activate ${CI_PROJECT_NAME}
    - mamba install -c file://${CI_PROJECT_DIR}/conda-bld -yq ${CI_PROJECT_NAME}
    - pip install --no-use-pep517 -U -r .gitlab/conda-env/requirements.txt
    - mamba list -n ${CI_PROJECT_NAME}

    # Tests
    - export SKIP_SLOW_TESTS=true
    - PKG_INSTALL_DIR=$(python -c "import elaspic2; print(elaspic2.__path__[0])")
    - mkdir coverage
    - python -c "import torch_sparse"
    - python -m pytest
      -c setup.cfg
      --cov="${PKG_INSTALL_DIR}"
      --cov-config=setup.cfg
      --color=yes
      --ignore="tests/plugins/modeller"
      --ignore="tests/plugins/rosetta_ddg"
      "tests/"
    - mv .coverage coverage/.coverage.base

    # Export the environment if building a release
    - |
      if [[ "${CI_BUILD_REF_NAME}" == "master" || ${CI_COMMIT_TAG} == v* ]]; then
        conda deactivate
        if [[ ${CI_COMMIT_TAG} == v* ]] ; then
          ENV_VERSION=${CI_COMMIT_TAG}
        else
          ENV_VERSION="latest"
        fi
        ENVIRONMENT_FILE=${CI_PROJECT_NAME}-${ENV_VERSION}.tar.gz
        conda pack -q -n ${CI_PROJECT_NAME} -o ${ENVIRONMENT_FILE}
        ls -lSh ${ENVIRONMENT_FILE}
        rsync -rv --info=progress2 -p --chmod=ug=rwX,o=rX "${ENVIRONMENT_FILE}" strokach@conda-envs.proteinsolver.org:/share/conda-envs/${CI_PROJECT_NAME}/
        conda activate ${CI_PROJECT_NAME}
      fi

    # Install optional dependencies (try conda, mamba was having difficulties...)
    - conda install "modeller=10.1" "pyrosetta=2020.37+release.3ba1aaa" "rosetta-ddg=2020.37" "anaconda::cudatoolkit=10.1"

    # Test optional plugins
    - python -m pytest
      -c setup.cfg
      --cov="${PKG_INSTALL_DIR}"
      --cov-config=setup.cfg
      --color=yes
      "tests/plugins/modeller"
      "tests/plugins/rosetta_ddg"
    - mv .coverage coverage/.coverage.extra

    # Need to to change folder file permissions if files are created in a different container
    - chmod ugo+rwX -R coverage/
  dependencies:
    - build
  artifacts:
    paths:
      - coverage

# === Document ===

# NB: Has to be called "docs" for the pages script to work.
docs:
  stage: doc
  extends:
    - .skip-custom-pipelines
  before_script:
    - *configure-conda
  script:
    # Create conda environment for testing
    - mamba create -n test -yq -c file://${CI_PROJECT_DIR}/conda-bld ${CI_PROJECT_NAME}
      "python=${PYTHON_VERSION}" ${CI_PROJECT_NAME} nbconvert ipython ipykernel pandoc || true
    - source activate test
    - pip install -q 'sphinx>=3.4' sphinx_rtd_theme msmb_theme nbsphinx coverage
      'recommonmark>=0.5' sphinx-markdown-tables
    # Build docs
    - sphinx-build ${CI_PROJECT_DIR}/docs public
    - ln -s . public/docs
    # Coverage
    - coverage combine coverage/
    - coverage report
    - coverage html
    - mv htmlcov public/
  coverage: /^TOTAL.* (\d+\%)/
  dependencies:
    - build
    - test
  artifacts:
    paths:
      - public
    when: always

# === Deploy ===

deploy:
  stage: deploy
  extends:
    - .skip-custom-pipelines
  script:
    - anaconda -t $ANACONDA_TOKEN upload $CI_PROJECT_DIR/conda-bld/*/*.tar.bz2 -u ostrokach-forge --no-progress --force
  only:
    - tags
  dependencies:
    - build

deploy-pypi:
  stage: deploy
  extends:
    - .skip-custom-pipelines
  script:
    - python -m pip install -q twine wheel
    - python setup.py sdist bdist_wheel
    - twine upload dist/*
  only:
    - tags

trigger-custom-pipelines:
  stage: deploy
  extends:
    - .skip-custom-pipelines
  image:
    name: ubuntu:20.04
  before_script:
    - apt-get -y -qq update
    - apt-get -y -qq install curl
  script:
    # Update pages
    - curl --request POST
      --form token="${CI_JOB_TOKEN}"
      --form ref=${CI_COMMIT_TAG}
      --form "variables[UPDATE_PAGES]=true"
      https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/trigger/pipeline
    # Build a docker image
    - curl --request POST
      --form token="${CI_JOB_TOKEN}"
      --form ref=${CI_COMMIT_TAG}
      --form "variables[BUILD_IMAGE_WITH_TAG]=${CI_COMMIT_TAG}"
      https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/trigger/pipeline
    # Mirror to GitHub
    - curl --request POST
      --form token="${CI_JOB_TOKEN}"
      --form ref=${CI_COMMIT_TAG}
      --form "variables[MIRROR_TO_GITHUB]=true"
      https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/trigger/pipeline
  only:
    - tags

# === Custom pipelines ===

pages:
  stage: custom
  before_script:
    - sudo yum update -y -q
    - sudo yum install -y -q unzip
    - pip install jinja2 python-gitlab
  script:
    # Set environment variables
    - export OUTPUT_DIR="./public"
    - mkdir -p ${OUTPUT_DIR}
    # Download all previous docs
    - python .gitlab/pages/download_docs.py
      --project-id ${CI_PROJECT_ID}
      --job-name docs
      --private-token ${CI_DOCS_TOKEN}
      --output-dir ${OUTPUT_DIR}
  artifacts:
    paths:
      - public
  only:
    variables:
      - $UPDATE_PAGES

build-image:
  stage: custom
  image:
    name: docker:latest
  # tags:
  #   - east-cloud
  services:
    - docker:dind
  script:
    - IMAGE_TAG=${BUILD_IMAGE_WITH_TAG}
    - echo "${CI_REGISTRY_IMAGE}:${IMAGE_TAG}"
    - docker login registry.gitlab.com -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD"
    - docker build --build-arg ENV_VERSION=${IMAGE_TAG} -t "${CI_REGISTRY_IMAGE}:${IMAGE_TAG}" .gitlab/docker
    - docker push "${CI_REGISTRY_IMAGE}:${IMAGE_TAG}"
  only:
    variables:
      - $BUILD_IMAGE_WITH_TAG

mirror-to-github:
  stage: custom
  image:
    name: ubuntu:20.04
  before_script:
    # Install global dependencies
    - apt-get update -y -qq -o=Dpkg::Use-Pty=0
    - apt-get install -y -qq -o=Dpkg::Use-Pty=0 curl rsync gettext-base git git-lfs

    # Install ssh client
    - export SSH_PRIVATE_KEY="${GITHUB_SSH_PRIVATE_KEY}"
    - export KNOWN_HOSTS="${GITHUB_KNOWN_HOSTS}"
    - *install-ssh-client
  script:
    - git fetch --all
    - git remote add mirror git@github.com:elaspic/elaspic2.git
    - git checkout master
    - git push mirror master --force
    - git push mirror master --tags --force
  only:
    variables:
      - $MIRROR_TO_GITHUB

test-pip-install:
  stage: custom
  image:
    name: python:3.8-buster
  script:
    - PIP_INSTALL_CMD=$(./scripts/get_pip_install_cmd.py)
    - eval "${PIP_INSTALL_CMD}"
    - python -m elaspic2 --help
  only:
    variables:
      - $TEST_PIP_INSTALL