From 6c909cae74f18edc9bba63e794a115e8e3ea6cef Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Thu, 4 Apr 2024 11:02:03 -0500 Subject: [PATCH 1/5] Upgrading to SCORPIO v1.6.2 SCORPIO v1.6.2 includes the following fixes/enhancements, * Support for standalone EAMXX/SCREAM builds * A new rearranger, PIO_REARR_ANY SCORPIO v1.6.1 includes the following fixes, * Fixes for ADIOS to NetCDF conversion tool hangs/crashes in certain scenarios * Fixes for ultra high res simulations using the SUBSET rearranger SCORPIO v1.6.0 includes the following fixes/enhancements, * Support for reading ADIOS BP files and updates to the BP to NetCDF conversion tool to integrate with CIME/E3SM * Adding a SCORPIO CMake package during install * Support for new CDF5 types in the C interface * Support for pio_set_fill() * Fix for performance issues while writing large output files (hang during enddef() call) * Misc bug fixes --- externals/scorpio | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/scorpio b/externals/scorpio index de0b1ca2200f..dc6f1eb24f3d 160000 --- a/externals/scorpio +++ b/externals/scorpio @@ -1 +1 @@ -Subproject commit de0b1ca2200f62c6eb5e3fd40147965409e97123 +Subproject commit dc6f1eb24f3dae1dcb0ebb13130c4941dc9bd445 From 353ab56ebf294f4b0158d2273b776a0deda4b4f9 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Thu, 4 Apr 2024 11:32:08 -0500 Subject: [PATCH 2/5] Reset MPICH GPU support for conversion tool Disable MPICH/MPI GPU support for ADIOS to NetCDF conversion tool. Without this change the conversion tool crashes (with message "MPIDI_CRAY_init: GPU_SUPPORT_ENABLED is requested, but GTL library is not linked" - most likely due to the way SCREAM builds/links SCORPIO) on Frontier. --- cime_config/customize/case_post_run_io.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cime_config/customize/case_post_run_io.py b/cime_config/customize/case_post_run_io.py index bb44477ebb57..83419aa81348 100755 --- a/cime_config/customize/case_post_run_io.py +++ b/cime_config/customize/case_post_run_io.py @@ -74,6 +74,12 @@ def _convert_adios_to_nc(case): # Load the environment case.load_env(reset=True) + # Reset MPICH/MPI GPU support, if enabled + is_mpich_gpu_enabled = os.environ.get('MPICH_GPU_SUPPORT_ENABLED') + if int(0 if is_mpich_gpu_enabled is None else is_mpich_gpu_enabled) == 1: + logger.info("Resetting support for GPU in MPICH/MPI library (since its not used by the tool)") + os.environ['MPICH_GPU_SUPPORT_ENABLED'] = str(0); + run_func = lambda: run_cmd(cmd, from_dir=rundir)[0] # Run the modified case From 909af120f29d7b8c30dbbdeadf6e2c32cc0acbc3 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Thu, 4 Apr 2024 11:40:17 -0500 Subject: [PATCH 3/5] Adding support for PIO_REARR_ANY Adding support for new SCORPIO rearranger, PIO_REARR_ANY --- driver-mct/cime_config/config_component.xml | 4 ++-- driver-mct/cime_config/namelist_definition_modelio.xml | 4 ++-- driver-moab/cime_config/config_component.xml | 4 ++-- driver-moab/cime_config/namelist_definition_modelio.xml | 4 ++-- share/util/shr_pio_mod.F90 | 3 ++- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/driver-mct/cime_config/config_component.xml b/driver-mct/cime_config/config_component.xml index a76311dbfa7a..d037770ed5c5 100644 --- a/driver-mct/cime_config/config_component.xml +++ b/driver-mct/cime_config/config_component.xml @@ -2692,10 +2692,10 @@ integer - 1,2 + 1,2,3 run_pio env_run.xml - pio rearranger choice box=1, subset=2 + pio rearranger choice box=1, subset=2, any=3 $PIO_VERSION $PIO_VERSION diff --git a/driver-mct/cime_config/namelist_definition_modelio.xml b/driver-mct/cime_config/namelist_definition_modelio.xml index ce0275e59bf3..2860ffd7106f 100644 --- a/driver-mct/cime_config/namelist_definition_modelio.xml +++ b/driver-mct/cime_config/namelist_definition_modelio.xml @@ -90,9 +90,9 @@ integer pio pio_inparm - -99,1,2 + -99,1,2,3 - Rearranger method for pio 1=box, 2=subset. + Rearranger method for pio 1=box, 2=subset, 3=any. $CPL_PIO_REARRANGER diff --git a/driver-moab/cime_config/config_component.xml b/driver-moab/cime_config/config_component.xml index 8e4e686aa1c5..bcd9cb05008e 100644 --- a/driver-moab/cime_config/config_component.xml +++ b/driver-moab/cime_config/config_component.xml @@ -2684,10 +2684,10 @@ integer - 1,2 + 1,2,3 run_pio env_run.xml - pio rearranger choice box=1, subset=2 + pio rearranger choice box=1, subset=2, any=3 $PIO_VERSION $PIO_VERSION diff --git a/driver-moab/cime_config/namelist_definition_modelio.xml b/driver-moab/cime_config/namelist_definition_modelio.xml index ce0275e59bf3..2860ffd7106f 100644 --- a/driver-moab/cime_config/namelist_definition_modelio.xml +++ b/driver-moab/cime_config/namelist_definition_modelio.xml @@ -90,9 +90,9 @@ integer pio pio_inparm - -99,1,2 + -99,1,2,3 - Rearranger method for pio 1=box, 2=subset. + Rearranger method for pio 1=box, 2=subset, 3=any. $CPL_PIO_REARRANGER diff --git a/share/util/shr_pio_mod.F90 b/share/util/shr_pio_mod.F90 index ed7dfdaa2ed8..0935bce351a5 100644 --- a/share/util/shr_pio_mod.F90 +++ b/share/util/shr_pio_mod.F90 @@ -730,7 +730,8 @@ subroutine shr_pio_namelist_set(npes,mycomm, pio_stride, pio_root, pio_numiotask if(pio_stride == 1 .and. .not. pio_async_interface) then pio_root = 0 endif - if(pio_rearranger .ne. PIO_REARR_SUBSET .and. pio_rearranger .ne. PIO_REARR_BOX) then + if(pio_rearranger .ne. PIO_REARR_SUBSET .and. pio_rearranger .ne. PIO_REARR_BOX .and.& + pio_rearranger .ne. PIO_REARR_ANY) then write(shr_log_unit,*) 'pio_rearranger value, ',pio_rearranger,& ', not supported - using PIO_REARR_BOX' pio_rearranger = PIO_REARR_BOX From 9fa0ce63dd75f7b14cfce56fd1d5f85db1fae6e1 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Thu, 11 Apr 2024 09:18:21 -0500 Subject: [PATCH 4/5] Turn on BP to NC on close for tests Enable conversion of the ADIOS BP output files to NetCDF during file close for tests. This is required for tests like ERIO tests that require the NetCDF output before the dependent conversion job (which runs after the current job completes) gets a chance to perform the conversion. --- components/cmake/modules/FindPIO.cmake | 3 +++ share/build/buildlib.spio | 3 +++ 2 files changed, 6 insertions(+) diff --git a/components/cmake/modules/FindPIO.cmake b/components/cmake/modules/FindPIO.cmake index 7df8a9ba1062..0277918ac8e0 100644 --- a/components/cmake/modules/FindPIO.cmake +++ b/components/cmake/modules/FindPIO.cmake @@ -18,6 +18,9 @@ endif() if (PIO_VERSION STREQUAL 2) # This is a pio2 library set(PIOLIBS "${PIO_LIBDIR}/libpiof.a;${PIO_LIBDIR}/libpioc.a") + if (DEFINED ENV{ADIOS2_ROOT}) + list(APPEND PIOLIBS "${PIO_LIBDIR}/libadios2pio-nm-lib.a") + endif() else() # This is a pio1 library set(PIOLIBS "${PIO_LIBDIR}/libpio.a") diff --git a/share/build/buildlib.spio b/share/build/buildlib.spio index d65d55479361..49b898202f72 100755 --- a/share/build/buildlib.spio +++ b/share/build/buildlib.spio @@ -106,6 +106,9 @@ def buildlib(bldroot, installpath, case): if "ADIOS2_ROOT" in os.environ: cmake_opts += "-DWITH_ADIOS2:BOOL=ON " + if "FROM_CREATE_TEST" in os.environ and os.environ["FROM_CREATE_TEST"] == "True": + cmake_opts += "-DADIOS_BP2NC_TEST:BOOL=ON " + if debug: cmake_opts += "-DPIO_ENABLE_LOGGING=ON " # Case changes for NetCDF/NETCDF forces us to do this. For other packages From 7f5de8b8866eaaa434b4e7c27200bec5320ae609 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Wed, 1 May 2024 12:19:34 -0500 Subject: [PATCH 5/5] Updating to SCORPIO v1.6.3 SCORPIO v1.6.3 includes the following fixes, * Fix for build issues with the Intel compiler * Fix for issues writing/reading ADIOS string attributes * Misc bug fixes --- externals/scorpio | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/scorpio b/externals/scorpio index dc6f1eb24f3d..cdd541e0cd70 160000 --- a/externals/scorpio +++ b/externals/scorpio @@ -1 +1 @@ -Subproject commit dc6f1eb24f3dae1dcb0ebb13130c4941dc9bd445 +Subproject commit cdd541e0cd708bece13b1f3ee42f66dfd6440aa7