first commit

Author: claire-peters

.github/workflows/test.yml

@@ -0,0 +1,39 @@
+name: integration-test
+
+on:
+  push:
+    branches: [ main, dev ]
+  pull_request:
+    branches: [ main ]
+jobs:
+  integration-test:
+    runs-on: ubuntu-latest
+    services:
+      docker:
+        image: docker:24.0.5
+        options: --privileged
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Compose
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y docker-compose
+
+      - name: Build and start containers
+        run: docker-compose up -d --build
+
+      - name: Wait for slurmctl to be ready
+        run: |
+          for i in {1..30}; do
+            if docker-compose exec slurmctl curl -sf http://localhost:6820/slurm/v0.0.36/clusters; then
+              break
+            fi
+            sleep 5
+          done
+      - name: Install package
+        run: pip install 
+
+      - name: Run integration tests
+        run: docker-compose run --rm app pytest tests/integration

LICENSE

@@ -0,0 +1,76 @@
+# Slurm integration for ColdFront
+
+ColdFront django plugin providing Slurm REST API integration for ColdFront.
+
+A command line tool is also provided with this app that allows an administrator
+to check the consistency between ColdFront and Slurm and optionally remove any
+associations that should not be in Slurm according to ColdFront.
+
+## Design
+
+Resources in ColdFront map to Clusters (or partitions within a cluster) in
+Slurm. The name of the Slurm cluster is taken from a resource attribute in
+ColdFront named "slurm\_cluster".  You can optionally provide Slurm
+specifications for a cluster using a resource attribute named "slurm\_specs".
+The value of this attribute must conform to the Slurm specification format and
+are colon separated.
+
+Allocations in ColdFront map to Accounts in Slurm. The name of the Slurm
+account is taken from a allocation attribute in ColdFront named
+"slurm\_account\_name" . You can optionally provide Slurm specifications for
+the account using a allocation attribute named "slurm\_specs". The value of
+this attribute must conform to the Slurm specification format and are colon
+separated.
+
+Allocation users in ColdFront map to Users in Slurm. You can optionally
+provide Slurm specifications for each user in a allocation using a
+allocation attribute named "slurm\_user\_specs". The value of this attribute
+must conform to the Slurm specification format and are colon separated. Setting
+specifications on an individual user basis is not currently supported.
+
+## Usage
+
+To enable this plugin set the following environment variables:
+
+```
+PLUGIN_SLURMREST=True
+SLURMREST_CLUSTERS=production # add the names of the resources that correspond to the Slurm Clusters to use, separated by commas
+# For each cluster listed in slurm_clusters:
+# Set SLURM_{clustername}_ENDPOINT to the IP or URL, e.g.:
+SLURM_PRODUCTION_ENDPOINT=http://production_cluster_url.com
+# Set SLURM_{clustername}_TOKEN to the JWT token used for authentication
+SLURM_PRODUCTION_TOKEN=$SLURMTEST_TOKEN
+```
+
+
+## Special cases
+
+Resources in ColdFront can optionally be organized into parent/child
+relationships. This is useful in the context of Slurm for example if you have a
+single Slurm controller with multiple partitions. Each partition represents a
+separate resource in ColdFront that users can subscribe to. In this case you
+would create a parent resource that represents your Slurm cluster (or
+controller) with a resource type of "Cluster". Each Slurm partition would be a
+separate resource in ColdFront with a resource type of "Partition" and have
+their parent resource set to the Slurm cluster. Users wouldn't subscribe to the
+parent Slurm cluster resource but only subscribe to the partition resources. Here
+you would only set the "slurm\_cluster" resource attribute on the Slurm cluster
+resource and not on the partitions. Also, "slurm\_specs" resource attribute on
+partitions are merged with the allocation "slurm\_specs" and set on the Slurm
+account association instead of the cluster.
+
+## CLI Usage
+
+To check the consistency between ColdFront and Slurm run the following command:
+
+```
+    $ sacctmgr dump file=/output_dir/tux.cfg
+    $ coldfront slurm_check -i /output_dir/tux.cfg
+```
+
+This will compare active allocations in ColdFront to Slurm accounts and
+associations. Any differences in the Slurm association and ColdFront user lists
+will be reported. You can optionally provide the '--slurm-sync' flag to
+automatically add and remove Slurm associations to match the allocationuser lists
+in ColdFront, or the '--coldfront-sync' flag to automatically match the
+allocationuser lists to Slurm's associations.

README.md

@@ -0,0 +1,46 @@
+services:
+  slurmctl:
+    image: ghcr.io/fasrc/slurmrest-container
+    hostname: slurmctl
+    container_name: slurmctl
+    volumes:
+      - share:/share
+    cap_add:
+      - sys_admin
+    tty: true          # -t   keep bash interactive
+    stdin_open: true   # -i   keep STDIN open
+    networks:
+      - compute
+  coldfront:
+    image: ghcr.io/fasrc/coldfront-builder
+    command: ["serve"]
+    hostname: coldfront
+    container_name: coldfront-test
+    volumes: 
+      - ./coldfront/slurm_rest_coldfront:/opt/slurmrest
+      - share:/share
+    expose:
+      - "22"
+    networks:
+      - compute
+  mysql:
+    image: mariadb:10.4
+    hostname: mysql
+    container_name: mysql
+    environment:
+      MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
+    volumes:
+      - ./database:/docker-entrypoint-initdb.d
+      - ./database:/etc/mysql/conf.d
+      - var_lib_mysql:/var/lib/mysql
+    expose:
+      - "3306"
+    networks:
+      - compute
+    
+networks:
+  compute:
+
+volumes:
+  var_lib_mysql:
+  share:

docker-compose.yml

@@ -0,0 +1,7 @@
+[build-system]
+requires = [
+    "setuptools>=42",
+    "git+https://github.com/fasrc/slurmrest_python_sdk.git",
+    "wheel"
+]
+build-backend = "setuptools.build_meta"

pyproject.toml

@@ -0,0 +1,26 @@
+[metadata]
+name = coldfront_plugin_slurmrest
+version = 0.0.1
+author = Claire Peters
+author_email = [email protected]
+description = Slurm REST API plugin for ColdFront
+long_description = file: README.md
+long_description_content_type = text/markdown
+url = https://github.com/coldfront/coldfront-slurm-plugin
+project_urls =
+    Bug Tracker = https://github.com/coldfront/coldfront-slurm-plugin/issues
+classifiers =
+    Programming Language :: Python :: 3
+    License :: OSI Approved :: MIT License
+    Operating System :: OS Independent
+
+[options]
+package_dir =
+    = src
+packages = find:
+python_requires = >=3.9
+install_requires =
+    coldfront >= 1.1.0
+
+[options.packages.find]
+where = src

setup.cfg

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+import setuptools
+
+if __name__ == "__main__":
+    setuptools.setup()

setup.py

@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: (C) ColdFront Authors
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+default_app_config = "coldfront.plugins.slurm.apps.SlurmConfig"

src/coldfront_plugin_slurmrest/__init__.py

@@ -0,0 +1,9 @@
+# SPDX-FileCopyrightText: (C) ColdFront Authors
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+from django.apps import AppConfig
+
+
+class SlurmRestConfig(AppConfig):
+    name = "coldfront.plugins.slurmrest"

src/coldfront_plugin_slurmrest/apps.py

@@ -0,0 +1,45 @@
+from coldfront.config.base import INSTALLED_APPS
+from coldfront.config.env import ENV  # noqa: F401
+from coldfront.config.logging import LOGGING
+
+
+if "coldfront_plugin_slurmrest" not in INSTALLED_APPS:
+    INSTALLED_APPS += [
+        "coldfront_plugin_slurmrest",
+    ]
+
+SLURMREST_CLUSTER_ATTRIBUTE_NAME = ENV.str("SLURM_CLUSTER_ATTRIBUTE_NAME", default="slurm_cluster")
+SLURMREST_ACCOUNT_ATTRIBUTE_NAME = ENV.str(
+        "SLURM_ACCOUNT_ATTRIBUTE_NAME", default="slurm_account_name")
+SLURMREST_SPECS_ATTRIBUTE_NAME = ENV.str(
+        "SLURM_SPECS_ATTRIBUTE_NAME", default="slurm_specs")
+SLURMREST_USER_SPECS_ATTRIBUTE_NAME = ENV.str(
+        "SLURM_USER_SPECS_ATTRIBUTE_NAME", default="slurm_user_specs")
+
+SLURMREST_NOOP = ENV.bool('SLURM_NOOP', default=False)
+SLURMREST_IGNORE_USERS = ENV.list('SLURM_IGNORE_USERS', default=['root'])
+SLURMREST_IGNORE_ACCOUNTS = ENV.list('SLURM_IGNORE_ACCOUNTS', default=[])
+
+SLURMREST_CLUSTERS = {}
+for cluster in ENV.str('SLURMREST_CLUSTERS', '').split(','):
+    cluster_name = f"SLURM_{cluster.upper()}"
+    cluster_type = ENV.str(f"{cluster_name}_TYPE")
+    SLURMREST_CLUSTERS[cluster] = {
+        'name': cluster,
+        'base_url': ENV.str(f"{cluster_name}_ENDPOINT"),
+        'token': ENV.str(f"{cluster_name}_TOKEN"),
+    }
+
+
+LOGGING['handlers']['slurmrest'] = {
+            'class': 'logging.handlers.TimedRotatingFileHandler',
+            'filename': 'logs/slurmrest.log',
+            'when': 'D',
+            'backupCount': 10, # how many backup files to keep
+            'formatter': 'default',
+            'level': 'DEBUG',
+        }
+
+LOGGING['loggers']['coldfront.plugins.slurmrest'] = {
+            'handlers': ['slurmrest'],
+        }