Enable ParallelCluster access from outside the cluster (#137)

Create CfnOutputs and scripts to configure a submission host to access
a ParallelCluster without having to ssh to it.

Install the same version of munge as used by ParallelCluster.

Copy ParallelCluster's munge key to the submission host.
In the future will test changing ParallelCluster to use a key from an
SSM parameter.

Create a command to mount the Slurm shared file system on the submission host.

Create a command to run a script to configure the submission host.

Create a modulefile to configure the environment to use the ParallelCluster
from the submission host.

Resolves #136

Author: cartalla

source/cdk/cdk_slurm_stack.py

@@ -711,6 +711,20 @@ def create_parallel_cluster_assets(self):
         self.assets_bucket = self.playbooks_asset.s3_bucket_name
         self.assets_base_key = self.config['slurm']['ClusterName']
 
+        self.parallel_cluster_munge_key_write_policy = iam.ManagedPolicy(
+            self, "ParallelClusterMungeKeyWritePolicy",
+            managed_policy_name = f"{self.stack_name}-ParallelClusterMungeKeyWritePolicy",
+            statements = [
+                iam.PolicyStatement(
+                    effect=iam.Effect.ALLOW,
+                    actions=[
+                        's3:PutObject',
+                    ],
+                    resources=[f"arn:aws:s3:::{self.assets_bucket}/{self.config['slurm']['ClusterName']}/config/munge.key"]
+                )
+            ]
+        )
+
         s3_client = boto3.client('s3', region_name=self.config['Region'])
 
         template_vars = {
@@ -719,14 +733,15 @@ def create_parallel_cluster_assets(self):
             'playbooks_s3_url': self.playbooks_s3_url,
         }
         files_to_upload = [
+            'config/bin/config_submitter.sh',
             'config/bin/create_users_groups_json.py',
             'config/bin/create_users_groups.py',
-            'config/users_groups.json',
             'config/bin/on_head_node_start.sh',
             'config/bin/on_head_node_configured.sh',
             'config/bin/on_head_node_updated.sh',
             'config/bin/on_compute_node_start.sh',
             'config/bin/on_compute_node_configured.sh',
+            'config/users_groups.json',
         ]
         self.custom_action_s3_urls = {}
         for file_to_upload in files_to_upload:
@@ -2416,7 +2431,10 @@ def get_instance_template_vars(self, instance_role):
                 "Region": self.config['Region'],
                 "TimeZone": self.config['TimeZone'],
             }
+            instance_template_vars['DefaultPartition'] = 'batch'
             instance_template_vars['FileSystemMountPath'] = '/opt/slurm'
+            instance_template_vars['ModulefilesBaseDir'] = '/opt/slurm/config/modules/modulefiles'
+            instance_template_vars['ParallelClusterVersion'] = self.config['slurm']['ParallelClusterConfig']['Version']
             instance_template_vars['SlurmBaseDir'] = '/opt/slurm'
             instance_template_vars['SlurmOSDir'] = '/opt/slurm'
             instance_template_vars['SlurmVersion'] =  self.config['slurm']['SlurmVersion']
@@ -3538,6 +3556,7 @@ def create_parallel_cluster_config(self):
                         {'Policy': 'arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore'},
                         {'Policy': self.parallel_cluster_asset_read_policy.managed_policy_arn},
                         {'Policy': self.parallel_cluster_jwt_write_policy.managed_policy_arn},
+                        {'Policy': self.parallel_cluster_munge_key_write_policy.managed_policy_arn},
                     ],
                 },
                 'Imds': {
@@ -4178,3 +4197,11 @@ def create_parallel_cluster_config(self):
         CfnOutput(self, "PlaybookS3Url",
             value = self.playbooks_asset.s3_object_url
         )
+        region = self.config['Region']
+        cluster_name = self.config['slurm']['ClusterName']
+        CfnOutput(self, "SubmitterMountHeadNodeCommand",
+            value = f"head_ip=$(aws ec2 describe-instances --region {region} --filters 'Name=tag:parallelcluster:cluster-name,Values={cluster_name}' 'Name=tag:parallelcluster:node-type,Values=HeadNode' --query 'Reservations[0].Instances[0].PrivateIpAddress' --output text) && sudo mkdir -p /opt/slurm && sudo mount $head_ip:/opt/slurm /opt/slurm"
+        )
+        CfnOutput(self, "SubmitterConfigureCommand",
+            value = f"sudo /opt/slurm/config/bin/config_submitter.sh"
+        )

source/cdk/config_schema.py

@@ -57,7 +57,8 @@
 DEFAULT_PARALLEL_CLUSTER_VERSION = parse_version('3.6.1')
 DEFAULT_PARALLEL_CLUSTER_PYTHON_VERSION = '3.9.16'
 DEFAULT_PARALLEL_CLUSTER_PYTHON_VERSIONS = {
-    '3.7.0b1': str(DEFAULT_PARALLEL_CLUSTER_PYTHON_VERSION),
+    '3.6.1':   '3.9.16',
+    '3.7.0b1': '3.9.16',
 }
 DEFAULT_PARALLEL_CLUSTER_SLURM_VERSION = '23-02-3-1'
 DEFAULT_PARALLEL_CLUSTER_SLURM_VERSIONS = {

source/resources/parallel-cluster/config/bin/config_submitter.sh

@@ -0,0 +1,33 @@
+#!/bin/bash -xe
+
+full_script=$(realpath $0)
+script_dir=$(dirname $full_script)
+base_script=$(basename $full_script)
+
+date
+echo "Started config_submitter.sh: $full_script"
+
+config_dir=/opt/slurm/config
+config_bin_dir=$config_dir/bin
+
+assets_bucket={{assets_bucket}}
+assets_base_key={{assets_base_key}}
+
+# Configure using ansible
+if ! yum list installed ansible &> /dev/null; then
+    yum install -y ansible || amazon-linux-extras install -y ansible2
+fi
+
+ANSIBLE_PATH=$config_dir/ansible
+PLAYBOOKS_PATH=$ANSIBLE_PATH/playbooks
+
+pushd $PLAYBOOKS_PATH
+ansible-playbook $PLAYBOOKS_PATH/ParallelClusterSubmitter.yml \
+    -i inventories/local.yml \
+    -e @$ANSIBLE_PATH/ansible_head_node_vars.yml
+popd
+
+date
+echo "Finished config_submitter.sh: $full_script"
+
+exit 0

source/resources/parallel-cluster/config/bin/on_head_node_start.sh

@@ -33,6 +33,7 @@ chmod -R 0700 $config_dir
 
 # Download all of the config scripts
 config_scripts=(\
+    config_submitter.sh \
     create_users_groups_json.py \
     create_users_groups.py \
     on_head_node_start.sh \
@@ -95,6 +96,8 @@ if ! [ -e $jwt_key ]; then
     chmod 0600 $jwt_key
 fi
 
+/usr/bin/cp -f /etc/munge/munge.key $config_dir/munge.key
+
 date
 echo "Finished on_head_node_start.sh: $full_script"
 

source/resources/playbooks/ParallelClusterSubmitter.yml

@@ -0,0 +1,7 @@
+---
+- name: Configure ParallelCluster Submitter
+  hosts: ParallelClusterHeadNode
+  become_user: root
+  become: yes
+  roles:
+    - ParallelClusterSubmitter

source/resources/playbooks/roles/ParallelClusterHeadNode/tasks/config-modulefiles.yml

@@ -0,0 +1,58 @@
+---
+
+- name: Show vars used in this playbook
+  debug:
+    msg: |
+      Architecture:               {{Architecture}}
+      ClusterName:                {{ClusterName}}
+      DefaultPartition:           {{DefaultPartition}}
+      distribution:               {{distribution}}
+      distribution_major_version: {{distribution_major_version}}
+      ModulefilesBaseDir:         {{ModulefilesBaseDir}}
+      ParallelClusterVersion:     {{ParallelClusterVersion}}
+      SlurmBaseDir:               {{SlurmBaseDir}}
+      SlurmConfigDir:             {{SlurmConfigDir}}
+      SlurmVersion:               {{SlurmVersion}}
+
+- name: Fix permissions on config dir so users can access it.
+  file:
+    path: "{{SlurmConfigDir}}"
+    state: directory
+    owner: root
+    group: root
+    mode: 0755
+
+- name: Create {{ModulefilesBaseDir}}/{{distribution}}/{{distribution_major_version}}/{{Architecture}}/{{ClusterName}}
+  file:
+    path: "{{ModulefilesBaseDir}}/{{distribution}}/{{distribution_major_version}}/{{Architecture}}/{{ClusterName}}"
+    state: directory
+    owner: root
+    group: root
+    mode: 0755
+
+- name: Create slurm modulefile .template
+  template:
+    dest: "{{ModulefilesBaseDir}}/{{distribution}}/{{distribution_major_version}}/{{Architecture}}/{{ClusterName}}/.template"
+    src:  opt/slurm/modules/modulefiles/slurm/.template
+    owner: root
+    group: root
+    mode: 0664
+    force: yes
+
+- name: Create slurm modulefile
+  file:
+    path: "{{ModulefilesBaseDir}}/{{distribution}}/{{distribution_major_version}}/{{Architecture}}/{{ClusterName}}/{{ParallelClusterVersion}}"
+    src:  "{{ModulefilesBaseDir}}/{{distribution}}/{{distribution_major_version}}/{{Architecture}}/{{ClusterName}}/.template"
+    state: link
+    owner: root
+    group: root
+    mode: 0664
+
+- name: Create slurm modulefile .version
+  template:
+    dest: "{{ModulefilesBaseDir}}/{{distribution}}/{{distribution_major_version}}/{{Architecture}}/{{ClusterName}}/.version"
+    src:   opt/slurm/modules/modulefiles/slurm/.version
+    owner: root
+    group: root
+    mode: 0664
+    force: yes

source/resources/playbooks/roles/ParallelClusterHeadNode/tasks/main.yml

@@ -6,3 +6,4 @@
 - { include: config-licenses.yml, tags: licenses }
 - { include: config-slurmrestd.yml, tags: slurmrestd }
 - { include: config-sshd.yml, tags: sshd }
+- { include: config-modulefiles.yml, tags: modulefiles }

source/resources/playbooks/roles/ParallelClusterHeadNode/templates/opt/slurm/modules/modulefiles/slurm/.template

@@ -0,0 +1,146 @@
+#%Module1.0
+########################################
+##
+## SLURM modulefile
+##
+## Docs: https://modules.readthedocs.io/en/latest/modulefile.html
+########################################
+
+set nicename "SLURM"
+
+set thisname [ module-info name ]
+set namelen  [llength [split $thisname "/"]]
+set toolname [lindex [split $thisname "/" ] $namelen-2 ]
+set version  [lindex [split $thisname "/" ] end ]
+
+proc ModulesHelp { } {
+  global thisname toolname nicename
+  puts stderr "\t$thisname - loads the env for $nicename\n"
+}
+
+module-whatis "loads the env for $toolname version $version"
+
+prepend-path PATH {{SlurmBaseDir}}/bin
+
+setenv SLURM_CLUSTER_NAME {{ClusterName}}
+
+# sbatch defaults
+if { [ module-info mode load ] || [ module-info mode display ] } {
+    if { ! [ info exists ::env(SBATCH_MEM_PER_NODE) ] } {
+        setenv SBATCH_MEM_PER_NODE 100M
+        setenv SBATCH_MEM_PER_NODE_SET ""
+    }
+    if { ! ( [ info exists ::env(SBATCH_REQUEUE) ] || [ info exists ::env(SBATCH_NO_REQUEUE) ] ) } {
+        setenv SBATCH_REQUEUE ""
+        setenv SBATCH_REQUEUE_SET ""
+    }
+    if { ! [ info exists ::env(SBATCH_TIMELIMIT) ] } {
+        setenv SBATCH_TIMELIMIT "1:0:0"
+        setenv SBATCH_TIMELIMIT_SET ""
+    }
+    if { ! [ info exists ::env(SBATCH_PARTITION) ] } {
+        setenv SBATCH_PARTITION "{{DefaultPartition}}"
+        setenv SBATCH_PARTITION_SET ""
+    }
+} elseif { [ module-info mode remove ] } {
+    if { [ info exists ::env(SBATCH_MEM_PER_NODE_SET) ] } {
+        unsetenv SBATCH_MEM_PER_NODE
+        unsetenv SBATCH_MEM_PER_NODE_SET
+    }
+    if { [ info exists ::env(SBATCH_REQUEUE_SET) ] } {
+        unsetenv SBATCH_REQUEUE
+        unsetenv SBATCH_REQUEUE_SET
+    }
+    if { [ info exists ::env(SBATCH_TIMELIMIT_SET) ] } {
+        unsetenv SBATCH_TIMELIMIT
+        unsetenv SBATCH_TIMELIMIT_SET
+    }
+    if { [ info exists ::env(SBATCH_PARTITION_SET) ] } {
+        unsetenv SBATCH_PARTITION
+        unsetenv SBATCH_PARTITION_SET
+    }
+}
+
+# srun defaults
+if { [ module-info mode load ] || [ module-info mode display ] } {
+    if { ! [ info exists ::env(SLURM_CPUS_PER_TASK) ] } {
+        setenv SLURM_CPUS_PER_TASK 1
+        setenv SLURM_CPUS_PER_TASK_SET ""
+    }
+    if { ! [ info exists ::env(SLURM_MEM_PER_NODE) ] } {
+        setenv SLURM_MEM_PER_NODE 100M
+        setenv SLURM_MEM_PER_NODE_SET ""
+    }
+    if { ! [ info exists ::env(SLURM_PARTITION) ] } {
+        setenv SLURM_PARTITION "{{DefaultPartition}}"
+        setenv SLURM_PARTITION_SET ""
+    }
+    if { ! [ info exists ::env(SLURM_TIMELIMIT) ] } {
+        setenv SLURM_TIMELIMIT "1:0:0"
+        setenv SLURM_TIMELIMIT_SET ""
+    }
+} elseif { [ module-info mode remove ] } {
+    if { [ info exists ::env(SLURM_CPUS_PER_TASK_SET) ] } {
+        unsetenv SLURM_CPUS_PER_TASK
+        unsetenv SLURM_CPUS_PER_TASK_SET
+    }
+    if { [ info exists ::env(SLURM_MEM_PER_NODE_SET) ] } {
+        unsetenv SLURM_MEM_PER_NODE
+        unsetenv SLURM_MEM_PER_NODE_SET
+    }
+    if { ! [ info exists ::env(SLURM_PARTITION_SET) ] } {
+        unsetenv SLURM_PARTITION
+        unsetenv SLURM_PARTITION_SET
+    }
+    if { [ info exists ::env(SLURM_TIMELIMIT_SET) ] } {
+        unsetenv SLURM_TIMELIMIT
+        unsetenv SLURM_TIMELIMIT_SET
+    }
+}
+
+# squeue defaults
+#
+# Output format
+# If SQUEUE_FORMAT or SQUEUE_FORMAT2 already set then doen't change them.
+# This is to avoid overwriting a user's settings.
+#
+# SQUEUE_FORMAT doesn't allow all fields so prefer SQUEUE_FORMAT2
+# default: "%.18i %9P %8j %8u %12T %.10M %.6D %R"
+# JOBID PARTITION     NAME     USER ST       TIME  NODES NODELIST(REASON)
+# --long:  "%.18i %.9P %.8j %.8u %.2t %.10M %.6D %R"
+# JOBID PARTITION     NAME     USER    STATE       TIME TIME_LIMI  NODES NODELIST(REASON)
+#setenv SQUEUE_FORMAT  '%.9i %.19P %.9j %.8u %.10T %R'
+#
+# SQUEUE_FORMAT2 has all available fields
+# BatchHost: Similar to NodeList
+# ClusterFeature
+#
+# Priority
+# Reason
+#
+# TimeUsed
+# tres-alloc
+# UserId
+if { [ module-info mode load ] || [ module-info mode display ] } {
+    if { ! [ info exists ::env(SQUEUE_SORT) ] } {
+        # Sort by state, priority
+        setenv SQUEUE_SORT "t,-p"
+        setenv SQUEUE_SORT_SET ""
+    }
+    if { ! ( [ info exists ::env(SQUEUE_FORMAT) ] || [ info exists ::env(SQUEUE_FORMAT2) ] ) } {
+        setenv SQUEUE_FORMAT2 "Cluster:16 ,Partition:15 ,JobArrayId:16 ,Priority:12 ,State:11 ,UserName:8 ,Name:16 ,NumNodes:.5 ,NumCPUs:.4 ,MinMemory:.10 ,Feature:15 ,Dependency:10 ,Licenses:8 ,ReasonList:35"
+        #
+        # Time and priority information
+        #setenv SQUEUE_FORMAT2 "JobId:.6 ,Partition:9 ,State:7 ,UserName:8 ,Name:16 ,SubmitTime:16 ,PendingTime:12 ,TimeLimit:18 ,EndTime:18 ,ReasonList"
+        setenv SQUEUE_FORMAT2_SET ""
+    }
+} elseif { [ module-info mode remove ] } {
+    if { [ info exists ::env(SQUEUE_SORT_SET) ] } {
+        unsetenv SQUEUE_SORT
+        unsetenv SQUEUE_SORT_SET
+    }
+    if { [ info exists ::env(SQUEUE_FORMAT2_SET) ] } {
+        unsetenv SQUEUE_FORMAT2
+        unsetenv SQUEUE_FORMAT2_SET
+    }
+}

source/resources/playbooks/roles/ParallelClusterHeadNode/templates/opt/slurm/modules/modulefiles/slurm/.version

@@ -0,0 +1,5 @@
+#%Module1.0
+##
+##  Default version of this module
+##
+set ModulesVersion "{{ParallelClusterVersion}}"

source/resources/playbooks/roles/ParallelClusterSubmitter/README.md

@@ -0,0 +1,38 @@
+Role Name
+=========
+
+A brief description of the role goes here.
+
+Requirements
+------------
+
+Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
+
+Role Variables
+--------------
+
+A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
+
+Dependencies
+------------
+
+A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
+
+Example Playbook
+----------------
+
+Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
+
+    - hosts: servers
+      roles:
+         - { role: username.rolename, x: 42 }
+
+License
+-------
+
+BSD
+
+Author Information
+------------------
+
+An optional section for the role authors to include contact information, or a website (HTML is not allowed).