Feature/flux cluster (LLNL#402)

* Add flux native worker launch.

* Add flux native to changlog.

* Run black.

* Run black.

* Run fix-style

* Add try except for check_for_flux function when flux is not present.

* Add function doc.

* Add flux native run-workers test.

* Added PBS flux launch support
Added check_for_flux, check_for_slurm, check_for_lsf, and check_for_pbs
utility functions

* Return None from get_batch_type instead of slurm, now there are check
functions, there should be an error condition returned.

* Fix block comment

* Fix comment.

* Fix isort in Makefile.
Run fix-style
Add - to pbs for reading from stdin.

* Fix SYS_TYPE check

* fix-style

* Re fix-style with new isort.

* New qsub version uses -- for stdin.

* Chnage celery_regex ito celery_slurm_regex
Remove extra nodes variable in flux_par_native_test.yaml

* Change celery_regex ito celery_slurm_regex
Remove extra nodes variable in flux_par_native_test.yaml

CHANGELOG.md

@@ -11,6 +11,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Now loads np.arrays of dtype='object', allowing mix-type sample npy
 - Added a singularity container openfoam_wf example
+- Added flux native worker launch support
+- Added PBS flux launch support
+- Added check_for_flux, check_for_slurm, check_for_lsf, and check_for_pbs utility functions
+
+### Changed
+- Changed celery_regex to celery_slurm_regex in test_definitions.py
 
 ## [1.9.1]
 ### Fixed

Makefile

@@ -130,9 +130,9 @@ check-black:
 
 check-isort:
 	. $(VENV)/bin/activate; \
-	$(PYTHON) -m isort --check -w $(MAX_LINE_LENGTH) merlin; \
-	$(PYTHON) -m isort --check -w $(MAX_LINE_LENGTH) tests; \
-	$(PYTHON) -m isort --check -w $(MAX_LINE_LENGTH) *.py; \
+	$(PYTHON) -m isort --check --line-length $(MAX_LINE_LENGTH) $(MRLN); \
+	$(PYTHON) -m isort --check --line-length $(MAX_LINE_LENGTH) $(TEST); \
+	$(PYTHON) -m isort --check --line-length $(MAX_LINE_LENGTH) *.py; \
 
 
 check-pylint:

docs/source/conf.py

@@ -16,17 +16,17 @@
 import os
 import sys
 
-sys.path.insert(0, os.path.abspath('../..'))
+sys.path.insert(0, os.path.abspath("../.."))
 
 MERLIN_VERSION = __import__("merlin").VERSION
 
 # -- Project information -----------------------------------------------------
 
 _year = date.today().year
 
-project = u'Merlin'
-copyright = '{}, LLNL: LLNL-CODE-797170'.format(_year)
-author = u'Lawrence Livermore National Laboratory'
+project = "Merlin"
+copyright = "{}, LLNL: LLNL-CODE-797170".format(_year)
+author = "Lawrence Livermore National Laboratory"
 
 # The short X.Y version
 version = MERLIN_VERSION
@@ -47,19 +47,19 @@
 #    'sphinx.ext.autodoc',
 #    'sphinx.ext.intersphinx',
 # ]
-extensions = ['sphinx.ext.autodoc']
+extensions = ["sphinx.ext.autodoc"]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
 #
 # source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -74,15 +74,15 @@
 exclude_patterns = []
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 
 # -- Options for HTML output -------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'alabaster'
+html_theme = "alabaster"
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -99,9 +99,9 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
-html_css_files = ['custom.css']
+html_css_files = ["custom.css"]
 
 # Custom sidebar templates, must be a dictionary that maps document names
 # to template names.
@@ -117,7 +117,7 @@
 # -- Options for HTMLHelp output ---------------------------------------------
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'Merlindoc'
+htmlhelp_basename = "Merlindoc"
 
 
 # -- Options for LaTeX output ------------------------------------------------
@@ -126,15 +126,12 @@
     # The paper size ('letterpaper' or 'a4paper').
     #
     # 'papersize': 'letterpaper',
-
     # The font size ('10pt', '11pt' or '12pt').
     #
     # 'pointsize': '10pt',
-
     # Additional stuff for the LaTeX preamble.
     #
     # 'preamble': '',
-
     # Latex figure (float) alignment
     #
     # 'figure_align': 'htbp',
@@ -144,19 +141,15 @@
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'Merlin.tex', u'Merlin Documentation',
-     u'The Merlin Development Team', 'manual'),
+    (master_doc, "Merlin.tex", "Merlin Documentation", "The Merlin Development Team", "manual"),
 ]
 
 
 # -- Options for manual page output ------------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, 'merlin', u'Merlin Documentation',
-     [author], 1)
-]
+man_pages = [(master_doc, "merlin", "Merlin Documentation", [author], 1)]
 
 
 # -- Options for Texinfo output ----------------------------------------------
@@ -165,19 +158,17 @@
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'Merlin', u'Merlin Documentation',
-     author, 'Merlin', 'One line description of project.',
-     'Miscellaneous'),
+    (master_doc, "Merlin", "Merlin Documentation", author, "Merlin", "One line description of project.", "Miscellaneous"),
 ]
 
 
 # -- Extension configuration -------------------------------------------------
 
-primary_domain = 'py'
+primary_domain = "py"
 
-highlight_language = 'bash'
+highlight_language = "bash"
 
-intersphinx_mapping = {'python': ('https://docs.python.org/3', None)}
+intersphinx_mapping = {"python": ("https://docs.python.org/3", None)}
 
 
 def setup(app):

merlin/ascii_art.py

@@ -120,7 +120,6 @@
 
 
 def _make_banner():
-
     name_lines = merlin_name_small.split("\n")
     hat_lines = merlin_hat_small.split("\n")
 

merlin/common/tasks.py

@@ -141,7 +141,6 @@ def merlin_step(self, *args: Any, **kwargs: Any) -> Optional[ReturnCode]:  # noq
         elif result == ReturnCode.SOFT_FAIL:
             LOG.warning(f"*** Step '{step_name}' in '{step_dir}' soft failed. Continuing with workflow.")
         elif result == ReturnCode.HARD_FAIL:
-
             # stop all workers attached to this queue
             step_queue = step.get_task_queue()
             LOG.error(f"*** Step '{step_name}' in '{step_dir}' hard failed. Quitting workflow.")
@@ -260,7 +259,6 @@ def add_merlin_expanded_chain_to_chord(
         ]
         LOG.debug(f"recursing grandparent with relative paths {relative_paths}")
         for step in chain_:
-
             # Make a list of new task objects with modified cmd and workspace
             # based off of the parameter substitutions and relative_path for
             # a given sample.
@@ -325,7 +323,6 @@ def add_simple_chain_to_chord(self, task_type, chain_, adapter_config):
     LOG.debug(f"simple chain with {chain_}")
     all_chains = []
     for step in chain_:
-
         # Make a list of new task signatures with modified cmd and workspace
         # based off of the parameter substitutions and relative_path for
         # a given sample.

merlin/examples/workflows/flux/flux_par_native_test.yaml

@@ -0,0 +1,76 @@
+description:
+  description: A simple ensemble of parallel MPI jobs run by flux.
+  name: flux_par
+
+batch:
+  type: flux
+  flux_exec: flux exec -r "0-1"
+  flux_start_opts: -o,-S,log-filename=flux_par.out
+  nodes: 1
+
+env:
+  variables:
+    OUTPUT_PATH: ./studies
+    N_SAMPLES: 10
+
+study:
+- description: Build the code
+  name: build
+  run:
+    cmd: mpicc -o mpi_hello $(SPECROOT)/scripts/hello.c >& build.out
+    task_queue: flux_par
+- description: Echo the params
+  name: runs 
+  run:
+    cmd: |
+     if [  ! -z ${FLUX_PMI_LIBRARY_PATH+x} ]; then
+        FPMI2LIB=`dirname ${FLUX_PMI_LIBRARY_PATH}`/libpmi2.so
+        if [ -e ${FPMI2LIB} ]; then 
+          if [  ! -z ${LD_PRELOAD+x} ]; then
+            export LD_PRELOAD=${LD_PRELOAD}:${FPMI2LIB}
+          else
+            export LD_PRELOAD=${FPMI2LIB}
+          fi
+        fi
+     fi
+     $(LAUNCHER) $(build.workspace)/mpi_hello $(V1) $(V2) > flux_run.out
+    depends: [build]
+    task_queue: flux_par
+    nodes: 1
+    procs: 4
+    cores per task: 1
+
+- description: Dump flux info
+  name: data
+  run:
+    cmd: |
+      $(SPECROOT)/scripts/flux_info.py > flux_timings.out
+    depends: [runs_*]
+    task_queue: flux_par
+
+- description: Stop workers
+  name: stop_workers
+  run:
+    cmd: |
+      exit $(MERLIN_STOP_WORKERS)
+    depends: [data]
+    task_queue: flux_par
+
+global.parameters:
+  STUDY:
+    label: STUDY.%%
+    values:
+    - FLUXTEST
+
+merlin:
+  resources:
+    task_server: celery
+    workers:
+      simworkers:
+        args: -l INFO --concurrency 1 --prefetch-multiplier 1 -Ofair
+        steps: [runs, data]
+  samples:
+    column_labels: [V1, V2]
+    file: $(MERLIN_INFO)/samples.npy
+    generate:
+      cmd: python3 $(SPECROOT)/scripts/make_samples.py -dims 2 -n $(N_SAMPLES) -outfile=$(MERLIN_INFO)/samples.npy 

merlin/examples/workflows/flux/scripts/flux_test/flux

@@ -0,0 +1,2 @@
+#!/usr/bin/env python3
+print("Nodes")

merlin/examples/workflows/flux/scripts/pbs_test/qsub

@@ -0,0 +1,2 @@
+#!/bin/sh
+echo "pbs_version = 19.0.0"

merlin/examples/workflows/optimization/scripts/visualizer.py

@@ -1,16 +1,15 @@
 import argparse
-
-import matplotlib
-
-
-matplotlib.use("pdf")
 import ast
 import pickle
 
+import matplotlib
 import matplotlib.pyplot as plt
 import numpy as np
 
 
+matplotlib.use("pdf")
+
+
 plt.style.use("seaborn-white")
 
 

merlin/server/server_config.py

@@ -39,12 +39,6 @@
 
 import yaml
 
-
-try:
-    import importlib.resources as resources
-except ImportError:
-    import importlib_resources as resources
-
 from merlin.server.server_util import (
     CONTAINER_TYPES,
     MERLIN_CONFIG_DIR,
@@ -57,6 +51,12 @@
 )
 
 
+try:
+    import importlib.resources as resources
+except ImportError:
+    import importlib_resources as resources
+
+
 LOG = logging.getLogger("merlin")
 
 # Default values for configuration