Add support for GCAFS on MSU HPC

dev/ci/cases/pr/C96_gcafs_cycled.yaml

@@ -15,9 +15,7 @@ experiment:
   yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/gcafs_cycled_defaults_ci.yaml
 
 skip_ci_on_hosts:
-  - orion
   - gaeac5
-  - hercules
   - awsepicglobalworkflow
   - ursa
 

dev/parm/config/gcafs/config.fetch

@@ -17,7 +17,12 @@ if [[ "${PDY}${cyc}" -gt "${SDATE}" ]]; then
        gdas_version="prod"
    fi
    export gdas_version
-   export FETCH_YAML_TMPL_LIST="${PARMgfs}/fetch/${NET}_${APP}_gdas-anl.yaml.j2,${PARMgfs}/fetch/${NET}_${APP}_gdas-dtfanl.yaml.j2"
+   if [[ "${machine}" == "ORION" || "${machine}" == "HERCULES" ]]; then
+       FETCH_YAML_TMPL_LIST="${PARMgfs}/fetch/${NET}_${APP}_gdas-anl_msu.yaml.j2,${PARMgfs}/fetch/${NET}_${APP}_gdas-dtfanl_msu.yaml.j2"
+   else
+       FETCH_YAML_TMPL_LIST="${PARMgfs}/fetch/${NET}_${APP}_gdas-anl.yaml.j2,${PARMgfs}/fetch/${NET}_${APP}_gdas-dtfanl.yaml.j2"
+   fi
+   export FETCH_YAML_TMPL_LIST
 else
    # fetch based on first cycle
    # Determine start type

dev/ush/fetch_gdas_for_gcafs.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+# This script fetches GDAS data for GCAFS from HPSS
+# and places it in a new tar file.
+set -e
+set -u
+
+# Get arguments from command line
+if [[ "${#}" -ne 2 ]]; then
+    echo "Usage: ${0} <YYYYMMDDHH> <output_directory>"
+    exit 1
+fi
+YYYYMMDDHH=${1}
+OUTPUT_DIR=${2}
+# Create output directory if it doesn't exist
+mkdir -p "${OUTPUT_DIR}"
+mkdir -p "${OUTPUT_DIR}/tmp"
+
+# determine GDAS version based on date
+if [[ "${YYYYMMDDHH}" -ge "2022112900" ]]; then
+    gdas_version="v16.3"
+elif [[ "${YYYYMMDDHH}" -ge "2022062700" ]]; then
+    gdas_version="v16.2"
+else
+    gdas_version="prod"
+fi
+
+# break date and time into components
+cycle_Y=${YYYYMMDDHH:0:4}
+cycle_M=${YYYYMMDDHH:4:2}
+cycle_D=${YYYYMMDDHH:6:2}
+cyc=${YYYYMMDDHH:8:2}
+# cycle_YM is YYYYMM
+cycle_YM="${cycle_Y}${cycle_M}"
+# PDY is YYYYMMDD
+PDY="${cycle_Y}${cycle_M}${cycle_D}"
+
+# HPSS path for the two tar files
+hpss_path_root="/NCEPPROD/hpssprod/runhistory/rh${cycle_Y}/${cycle_YM}/${PDY}"
+hpss_file_nc="com_gfs_${gdas_version}_gdas.${PDY}_${cyc}.gdas_nc.tar"
+hpss_file_restart="com_gfs_${gdas_version}_gdas.${PDY}_${cyc}.gdas_restart.tar"
+
+# get the names of the files to extract
+atmanl="./gdas.${PDY}/${cyc}/atmos/gdas.t${cyc}z.atmanl.nc"
+dtfanl="./gdas.${PDY}/${cyc}/atmos/gdas.t${cyc}z.dtfanl.nc"
+
+# Fetch the tar files from HPSS
+cd "${OUTPUT_DIR}/tmp"
+
+htar -xvf "${hpss_path_root}/${hpss_file_nc}" "${atmanl}"
+htar -xvf "${hpss_path_root}/${hpss_file_restart}" "${dtfanl}"
+
+# create the output tar files
+echo "creating output tar files"
+tar cvf "${hpss_file_nc}" "${atmanl}"
+tar cvf "${hpss_file_restart}" "${dtfanl}"
+
+# Move the tar files to the output directory
+echo "moving tar files to ${OUTPUT_DIR}"
+mv "${hpss_file_nc}" "${OUTPUT_DIR}/"
+mv "${hpss_file_restart}" "${OUTPUT_DIR}/"
+
+# Clean up temporary directory
+rm -rf "${OUTPUT_DIR}/tmp"
+echo "GDAS data for ${YYYYMMDDHH} has been successfully fetched and stored in ${OUTPUT_DIR}."
+# End of script
+exit 0

parm/fetch/gcafs_ATMA_gdas-anl_msu.yaml.j2

@@ -0,0 +1,12 @@
+{% set cycle_YMDH = current_cycle | to_YMDH %}
+{% set cycle_Y = current_cycle | strftime("%Y") %}
+{% set cycle_YM = current_cycle | strftime("%Y%m") %}
+{% set cycle_YMD = current_cycle | to_YMD %}
+{% set cycle_HH = current_cycle | strftime("%H") %}
+target:
+    tarball: "/work/noaa/da/cmartin/june2025/gcafs/gcafs-tars/com_gfs_{{ gdas_version }}_gdas.{{ cycle_YMD }}_{{ cycle_HH }}.gdas_nc.tar"
+    on_hpss: False
+    contents:
+        # Atmospheric analysis
+        - ./gdas.{{ cycle_YMD }}/{{ cycle_HH }}/atmos/gdas.t{{ cycle_HH }}z.atmanl.nc
+    destination: "{{ ROTDIR }}"

parm/fetch/gcafs_ATMA_gdas-dtfanl_msu.yaml.j2

@@ -0,0 +1,12 @@
+{% set cycle_YMDH = current_cycle | to_YMDH %}
+{% set cycle_Y = current_cycle | strftime("%Y") %}
+{% set cycle_YM = current_cycle | strftime("%Y%m") %}
+{% set cycle_YMD = current_cycle | to_YMD %}
+{% set cycle_HH = current_cycle | strftime("%H") %}
+target:
+    tarball: "/work/noaa/da/cmartin/june2025/gcafs/gcafs-tars/com_gfs_{{ gdas_version }}_gdas.{{ cycle_YMD }}_{{ cycle_HH }}.gdas_restart.tar"
+    on_hpss: False
+    contents:
+        # NSST analysis
+        - ./gdas.{{ cycle_YMD }}/{{ cycle_HH }}/atmos/gdas.t{{ cycle_HH }}z.dtfanl.nc
+    destination: "{{ ROTDIR }}"

ush/python/pygfs/task/fetch.py

@@ -3,6 +3,7 @@
 import os
 from logging import getLogger
 from typing import Any, Dict
+import tarfile
 
 from wxflow import (Task, htar,
                     logit, parse_j2yaml, chdir)
@@ -85,12 +86,10 @@ def execute_pull_data(self, fetchdir_set: Dict[str, Any]) -> None:
             if on_hpss is True:  # htar all files in fnames
                 htar_obj = htar.Htar()
                 htar_obj.xvf(tarball, f_names)
-            else:  # tar all files in fnames
-                raise NotImplementedError("The fetch job does not yet support pulling from local archives")
-
-#                with tarfile.open(dest, "w") as tar:
-#                    for filename in f_names:
-#                        tar.add(filename)
+            else:  # extract from a specified tarball
+                with tarfile.open(tarball, "r") as tar:
+                    members = [m for m in tar.getmembers() if m.name in f_names]
+                    tar.extractall(members=members)
             # Verify all data files were extracted
             missing_files = []
             for f in f_names: