Bring repo up to date with the current version of cookiecutter-datapa…

…ckage

.cookiecutter.yaml

@@ -0,0 +1,13 @@
+default_context:
+    full_name: "Alexey Strokach"
+    email: "[email protected]"
+    host_url: "https://gitlab.com"
+    project_namespace: "datapkg"
+    project_name: "adjacency-net"
+    project_slug: "adjacency-net"
+    project_short_description: "Train and validate the Protein Adjacency Graph Neural Network (PAGNN)."
+    docs_secret_key: "SWKhaBJiG8O53MTcw97h1vRjVgimQCb4mzTIH4H8dD10vrb6NWOUantGWhqHMoY7"
+    version: "0.1"
+    create_author_file: "y"
+    open_source_license: "MIT License"
+    _copy_without_render: ['docs/output_toggle.tpl', 'docs/_templates/logo-image.html']

.editorconfig

@@ -0,0 +1,24 @@
+# http://editorconfig.org
+
+root = true
+
+[*]
+indent_style = space
+indent_size = 4
+trim_trailing_whitespace = true
+insert_final_newline = true
+charset = utf-8
+end_of_line = lf
+
+[*.bat]
+indent_style = tab
+end_of_line = crlf
+
+[*.yml]
+indent_size = 2
+
+[LICENSE]
+insert_final_newline = false
+
+[Makefile]
+indent_style = tab

.gitignore

@@ -1,13 +1,73 @@
-.*
-!.gitignore
-!.gitattributes
-!.gitlab-ci.yml
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
 
-# Documentation
-docs/_build
-docs/notebooks
-docs/README.md
-public
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# pyenv python configuration file
+.python-version
+
+
+# === AS ===
+
+# Cache folders
+.mypy_cache/
+.ipynb_checkpoints/
+
+# Temporary files
+.~*
+*.bak
 
 # Keep downloads
 downloads/*
@@ -23,14 +83,3 @@ notebooks/*
 # Keep output
 output/*
 # !output/*.parquet
-
-# Temporary files
-.~*
-*.bak
-
-# Other files
-.ipynb_checkpoints
-__pycache__
-
-# Big data
-output_mysql

.gitlab-ci.yml

@@ -1,86 +1,166 @@
-image: condaforge/linux-anvil:latest
+image: registry.gitlab.com/ostrokach-docker/conda:latest
 
 stages:
-  - build
-  - test
+  - init
+  - notebooks-01
   - deploy
 
 
 # === Variables ===
 
 variables:
-  PACKAGE_VERSION: "0.1.0"
-  DOCS_SECRET_KEY: "QQRcgz67PxJNFopTWGygv1Y9iqKle0Mb2rxFh22a9cc/"
+  PROJECT_NAME: "adjacency-net"
+  PROJECT_VERSION: "0.1"
+  DOCS_SECRET_KEY: "SWKhaBJiG8O53MTcw97h1vRjVgimQCb4mzTIH4H8dD10vrb6NWOUantGWhqHMoY7"
 
+.niagara-envs: &niagara-envs
+  SBATCH_ARGS: "--account=def-pmkim"
+  # SBATCH_EXTRA_ARGS: "--time 12:00:00 --array=1-1"
 
-# === Pages ===
+.graham-envs: &graham-envs
+  SBATCH_ARGS: "--account=def-pmkim --exclusive --mem=0"
+  # SBATCH_EXTRA_ARGS: "--time 12:00:00 --array=1-1"
 
-pages:
-  stage: test
-  before_script:
-    # Create conda environment
-    # (nice to use an environment in case we use `gitlab-runner exec shell`)
-    - conda remove -n pages --all -yq || true
-    - conda create -n pages -yq || true
-    - conda install -n pages -yq
-      git python pip ipython jupyter ipykernel pypandoc
-    - source activate pages
-    # Install pip packages
-    - pip install -q
-      git+https://github.com/arximboldi/sinusoidal-sphinx-theme.git
-      sphinx sphinx_rtd_theme guzzle_sphinx_theme recommonmark
+.cedar-envs: &cedar-envs
+  SBATCH_ARGS: "--account=rrg-pmkim --exclusive --mem=0"
+  # SBATCH_EXTRA_ARGS: "--time 12:00:00 --array=1-1"
+
+
+# === Basic GitLab runner tests ===
+
+.test: &test
+  stage: init
   script:
-    # Generate notebooks.csv
-    - ./scripts/create_notebook_table.py -i notebooks/ -o docs/notebooks.csv
-    # Build pages
-    - sphinx-build docs public/$DOCS_SECRET_KEY
-    # Convert notebooks
-    - mkdir -p public/$DOCS_SECRET_KEY/notebooks
-    - ./scripts/convert_notebooks.sh notebooks/ public/$DOCS_SECRET_KEY/notebooks/
+    - echo "starting test..."
+    - find .
+    - sleep 1
+    - mkdir -p some-folder
+    - echo "hello world" > some-folder/greeting.txt
+    - echo "$(date --iso-8601=seconds)" > some-folder/time.txt
+    - echo "done"
   artifacts:
     paths:
-    - public
+      - some-folder
 
+test-niagara:
+  tags:
+    - niagara
+  <<: *test
 
-# === Deploy ===
+test-ssh:
+  tags:
+    - compute-canada
+    - ssh
+  <<: *test
 
-.exec-notebook: &exec-notebook
+
+# === Conda environment ===
+
+create-conda-environment:
+  stage: init
+  script:
+    - conda env remove -q -y -n ${CI_PROJECT_NAME}-ci || true
+    - conda env create -q -n ${CI_PROJECT_NAME}-ci -f environment.yaml
+
+
+# === Batch jobs ===
+
+# sbatch
+
+.sbatch-setup: &sbatch-setup
   before_script:
-    - if [[ -z ${NOTEBOOK_NAME} || -z ${DATABIN_DIR} || -z ${CI_PROJECT_NAME} || -z ${CI_COMMIT_SHA} || -z ${CI_JOB_NAME} ]] ; then
+    # Make sure all required environment variables have been set
+    - if [[ -z ${NOTEBOOK_NAME} || -z ${DATA_DIR} || -z ${DATABIN_DIR} || -z ${CI_PROJECT_NAME} || -z ${CI_COMMIT_SHA} || -z ${CI_JOB_NAME} || -z ${CI_PROJECT_PATH_SLUG} ]] ; then
       echo "Not all required environment variables have been set!" ;
       exit -1 ;
-      done
-    - conda create
-      -n ${CI_PROJECT_PATH_SLUG}-${CI_COMMIT_SHA}-${CI_JOB_NAME}
-      -f environment.yaml
-    - conda activate ${CI_PROJECT_PATH_SLUG}-${CI_COMMIT_SHA}-${CI_JOB_NAME}
+      fi
+    # Create conda environment
+    - conda activate ${CI_PROJECT_NAME}-ci
+    # Set output directory
     - export OUTPUT_DIR=${DATABIN_DIR}/${CI_PROJECT_NAME}/${CI_COMMIT_SHA}/${CI_JOB_NAME}
-    - export SYSTEM_COMMAND="
-      jupyter nbconvert 'notebooks/${NOTEBOOK_NAME}.ipynb'
-      --to=html_ch
-      --output='${NOTEBOOK_NAME}.html'
-      --output-dir='${OUTPUT_DIR}'
-      --template=docs/output_toggle.tpl
-      --execute
+    - mkdir -p "${OUTPUT_DIR}"
+    - echo "export OUTPUT_DIR=${OUTPUT_DIR}" >> env.sh
+
+.sbatch: &sbatch
   script:
     - sbatch
+      --time=12:00:00
+      --nodes=1
+      --job-name=${NOTEBOOK_NAME}
+      --chdir=`pwd`
+      --output=${OUTPUT_DIR}/sbatch-%A-%a.log
+      --export=ALL
+      --wait
+      ${SBATCH_ARGS}
+      ${SBATCH_EXTRA_ARGS}
+      ${SBATCH_SCRIPT} |
+      tee sbatch.out
+
+.sbatch-teardown: &sbatch-teardown
   after_script:
-    - conda remove
-      -n ${CI_PROJECT_PATH_SLUG}-${CI_COMMIT_SHA}-${CI_JOB_NAME}
-      --all
-
+    # Set JOB_ID
+    - JOB_ID=$(rg "Submitted batch job" sbatch.out | rg -o '\d+$')
+    - echo "export JOB_ID=${JOB_ID}" >> env.sh
+    # Load environment variables from previous stages
+    - source env.sh
+    # Remove conda environment
+    - conda deactivate
+    - conda env remove -q -y -n ${CI_PROJECT_NAME}-ci
+    # Save job output
+    - mkdir -p output
+    - cp -al ${OUTPUT_DIR} output/$(basename ${OUTPUT_DIR})
+
+.sbatch-output: &sbatch-output
+  script:
+    - cat env.sh
+    - source env.sh
+    - ./scripts/wait-for-job-to-finish.sh
+    - mkdir -p output
+    - cp -al ${OUTPUT_DIR} output/$(basename ${OUTPUT_DIR})
+    - (cd output; ln -s )
+  artifacts:
+    paths:
+      - output
 
+# Jobs
 
-deploy:
-  stage: deploy
+basic_tests:
+  stage: notebooks-01
+  tags:
+    - niagara
   variables:
-    NOTEBOOK_NAME: deploy
-  <<: *exec-notebook
+    <<: *niagara-envs
+    SBATCH_SCRIPT: "./scripts/run-notebook.sh"
+    SBATCH_EXTRA_ARGS: "--array=1-2"
+    NOTEBOOK_NAME: "01-basic_tests"
+  <<: [*sbatch-setup, *sbatch, *sbatch-teardown]
+  artifacts:
+    paths:
+      - env.sh
+      - output
+    when: always
   when: manual
 
 
+# === Pages ===
 
-
-destroy-environment:
+pages:
+  stage: deploy
+  before_script:
+    # Install conda packages
+    - conda install -yq
+      git python pip ipython jupyter ipykernel pypandoc
+    # Install pip packages
+    - pip install -q
+      sphinx recommonmark sphinx_bootstrap_theme
   script:
-
+    # Generate notebooks.csv
+    - ./scripts/create_notebook_table.py -i notebooks/ -o docs/notebooks.csv
+    # Build pages
+    - sphinx-build docs public/$DOCS_SECRET_KEY
+    # Convert notebooks
+    - mkdir -p public/$DOCS_SECRET_KEY/notebooks
+    - ./scripts/convert_notebooks.sh notebooks/ public/$DOCS_SECRET_KEY/notebooks/
+  artifacts:
+    paths:
+    - public

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "python.pythonPath": "/usr/bin/python3"
+}