Replace create.py with spack stack extension

Move create.py functionality into Spack extension in the jcsda_emc_spack_stack branch of Spack. This uses Spack Python code directly for interacting with YAML and composing environments. It also offers more flexibility into adding more custom commands such as meta_module creation and stack management.

To use: `spack stack create env/container -h`.

Author: kgerheiser

.github/actions/setup-spack-stack/action.yaml

@@ -163,7 +163,7 @@ runs:
       fi
 
       source setup.sh
-      ./create.py environment --site default --app ${{ inputs.app }} --name ${{ inputs.app }}
+      spack stack create env --site default --app ${{ inputs.app }} --name ${{ inputs.app }}
       spack env activate envs/${{ inputs.app }}
 
       # LLVM Clang not in PATH, search for it specifically

.github/workflows/unit-tests.yaml

@@ -0,0 +1,21 @@
+name: unit-tests
+
+on:
+  pull_request:
+  push:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - name: run-unit-tests
+        run: |
+          source ./setup.sh
+          spack unit-test --extension=stack

.gitignore

@@ -2,3 +2,4 @@
 envs
 __pycache__
 cache
+.vscode

.spackstack

@@ -0,0 +1 @@
+Lets spack stack command find top-level spack-stack directory for config location

README.md

@@ -4,8 +4,8 @@ spack-stack is a collaborative effort between the NOAA Environmental Modeling Ce
 
 [spack](https://github.com/spack/spack) is a community-supported, multi-platform, Python-based package manager originally developed by the Lawrence Livermore National Laboratory (LLNL; https://computing.llnl.gov/projects/spack-hpc-package-manager). It is provided as a submodule so that a stable version can be referenced. [See the Spack Documentation for more information](https://spack.readthedocs.io/en/latest/)
 
-spack-stack is mainly a collection of Spack configuration files, but provides a few Python scripts to simplify the installation process:
-- `create.py` is provided to copy common, site-specific, and application-specific configuration files into a coherent Spack environment and to create container recipes
+spack-stack is mainly a collection of Spack configuration files, but provides a Spack extension to simplify the installation process:
+- `spack stack create` is provided to copy common, site-specific, and application-specific configuration files into a coherent Spack environment and to create container recipes
 - `meta_modules/setup_meta_modules.py` creates compiler, MPI and Python meta-modules for a convenient setup of a user environment using modules (lua and tcl)
 
 spack-stack is maintained by:
@@ -42,17 +42,17 @@ cd spack-stack
 source setup.sh
 
 # Basic usage of create.py
-./create.py -h
+spack stack create -h
 ```
 
 ### Create local environment
 ```
 # See a list of sites and apps
-./create.py environment -h
+spack stack create env -h
 
 # Create a pre-configured Spack environment in envs/<app>.<site>
 # (copies site-specific, application-specific, and common config files into the environment directory)
-./create.py environment --site hera --app jedi-fv3 --name jedi-fv3.hera
+spack stack create env --site hera --app jedi-fv3 --name jedi-fv3.hera
 
 # Activate the newly created environment
 # Optional: decorate the command line prompt using -p
@@ -81,13 +81,13 @@ spack module lmod refresh
 ### Create container
 ```
 # See a list of preconfigured containers
-./create.py container -h
+spack stack create container -h
 
 # Create container spack definition (spack.yaml) in directory envs/<spec>.<config>
-./create.py container --config docker-ubuntu-gcc-openmpi --spec esmf
+spack stack create container docker-ubuntu-gcc-openmpi --app ufs-weather-model
 
 # Descend into container environment directory
-cd envs/esmf.docker-ubuntu-gcc-openmpi
+cd envs/docker-ubuntu-gcc-openmpi.ufs-weather-model
 
 # Optionally edit config file
 emacs spack.yaml
@@ -102,7 +102,7 @@ docker run -it myimage
 ## Generating new site config
 Recommended: Start with an empty (default) site config. Then run `spack external find` to locate common external packages such as git, Perl, CMake, etc., and run `spack compiler find` to locate compilers in your path. Compilers or external packages with modules need to be added manually.
 ```
-./create.py environment --site default --app jedi-ufs --name jedi-ufs.mysite
+spack stack create env --site default --app jedi-ufs --name jedi-ufs.mysite
 
 # Descend into site config directory
 cd envs/jedi-ufs.mysite/site

configs/apps/empty/spack.yaml

@@ -2,6 +2,6 @@ spack:
   concretization: separately
   view: false
 
-  @CONFIG_INCLUDES@
+  include: []
 
   specs: []

configs/apps/jedi-ewok/spack.yaml

@@ -2,7 +2,7 @@ spack:
   concretization: separately
   view: false
 
-  @CONFIG_INCLUDES@
+  include: []
 
   specs:
     - jedi-ewok-env

configs/apps/jedi-fv3/spack.yaml

@@ -2,7 +2,7 @@ spack:
   concretization: separately
   view: false
 
-  @CONFIG_INCLUDES@
+  include: []
 
   specs:
   - jedi-fv3-bundle-env

configs/apps/jedi-tools/spack.yaml

@@ -2,7 +2,7 @@ spack:
   concretization: separately
   view: false
 
-  @CONFIG_INCLUDES@
+  include: []
 
   specs:
   - jedi-tools-env

configs/apps/jedi-ufs/spack.yaml

@@ -2,7 +2,7 @@ spack:
   concretization: separately
   view: false
 
-  @CONFIG_INCLUDES@
+  include: []
 
   specs:
   - jedi-ufs-bundle-env

configs/apps/jedi-um/spack.yaml

@@ -2,7 +2,7 @@ spack:
   concretization: separately
   view: false
 
-  @CONFIG_INCLUDES@
+  include: []
 
   specs:
   - jedi-um-bundle-env

configs/apps/nceplibs/spack.yaml

@@ -3,7 +3,7 @@ spack:
   concretization: separately
   view: false
 
-  @CONFIG_INCLUDES@
+  include: []
 
   specs:
     - nceplibs-bundle-env

configs/apps/soca/spack.yaml

@@ -2,7 +2,7 @@ spack:
   concretization: separately
   view: false
 
-  @CONFIG_INCLUDES@
+  include: []
 
   specs:
   - soca-bundle-env

configs/apps/spack.template.yaml

@@ -3,7 +3,9 @@ spack:
   concretization: separately
   view: false
 
-  @CONFIG_INCLUDES@
+  # Custom includes which can be a directory containg
+  # Spack config files or a file itself.
+  include: [] 
 
 # File in specs
   specs:  

configs/apps/ufs-weather-model/spack.yaml

@@ -1,9 +1,8 @@
-# Template spack.yaml for apps
 spack:
   concretization: separately
   view: false
 
-  @CONFIG_INCLUDES@
+  include: []
 
   specs:
     - ufs-weather-model-env

configs/containers/docker-ubuntu-gcc-openmpi.yaml

@@ -22,8 +22,8 @@ spack:
       extra_rpaths: []
 
   # Basic package config from configs/common/packages.yaml
-  @PACKAGE_CONFIG@
   # Additional package config for container
+  packages:
     gcc:
       buildable: False
       externals:
@@ -40,22 +40,7 @@ spack:
       - spec: git-lfs@2
         prefix: /usr
 
-  specs:
-    ### Not yet tested
-    #- jedi-ewok-env
-    #- jedi-um-bundle-env
-    #- jedi-tools-env
-    #- ufs-weather-model-env
-    #- nceplibs-bundle-env
-    ### Tested to work - increasing level of complexity
-    #- wget
-    #- esmf
-    #- base-env
-    #- jedi-base-env
-    #- jedi-fv3-bundle-env
-    #- jedi-ufs-bundle-env
-    #- jedi-um-bundle-env
-    - @SPEC@
+  specs: []
 
   container:
 
@@ -91,15 +76,13 @@ spack:
       - g++-10
       - gfortran-10
       - cpp-10
-      #- libgomp
       - git
       - git-lfs
       final:
       - gcc-10
       - g++-10
       - gfortran-10
       - cpp-10
-      #- libgomp
       - git
       - git-lfs
 
@@ -114,5 +97,5 @@ spack:
 
     # Labels for the image
     labels:
-      app: "@SPEC@"
+      app: ""
       mpi: "openmpi"