Merge branch 'multiple_icesheets_evolve'

Support one ice sheet evolving and one not evolving

Changes needed to allow a mix of evolving and non-evolving ice sheets
in a single run. The primary need for this was moving zero_gcm_fluxes
from cism_in into the config file, but I also made some other small
changes to support this scenario. I have also added an ERI test of
this scenario to make sure that hybrid, branch and restart runs all
work when you have a mix of evolving and non-evolving ice sheets.

Also some other minor fixes.

Resolves #53 (Change handling of some ice
sheet-specific variables that are currently in cism_in)

Resolves ESCOMP/CISM-warpper#54 (Do some testing with one ice sheet
evolving and the other not evolving)

Author: billsacks

ChangeLog

@@ -2,6 +2,49 @@
 This file describes what tags were created on master and why
 ================================================================================
 
+================================================================================
+Originator: sacks
+Date: December 28, 2021
+Version: cismwrap_2_1_95
+One-line summary: Support one ice sheet evolving and one not evolving
+
+Purpose of changes:
+
+   Changes needed to allow a mix of evolving and non-evolving ice sheets
+   in a single run. The primary need for this was moving zero_gcm_fluxes
+   from cism_in into the config file, but I also made some other small
+   changes to support this scenario. I have also added an ERI test of
+   this scenario to make sure that hybrid, branch and restart runs all
+   work when you have a mix of evolving and non-evolving ice sheets.
+
+   Also some other minor fixes.
+
+Standalone checkout supported in this tag (Yes/No): Yes
+   (If yes, this implies that we expect to be able to build and run a
+   case from a standalone checkout using manage_externals, and for this
+   to continue to work long-term. The answer may be "No" if the set of
+   externals pointed to by manage_externals is broken, or if at least
+   one external points to a temporary branch that is may be deleted in
+   the near future.)
+
+   If No: Notes on externals used for testing:
+
+Changes answers relative to previous tag: NO
+
+Bugs fixed (include github issue number):
+
+   Resolves ESCOMP/CISM-wrapper#53 (Change handling of some ice
+   sheet-specific variables that are currently in cism_in)
+   
+   Resolves ESCOMP/CISM-warpper#54 (Do some testing with one ice sheet
+   evolving and the other not evolving)
+
+Summary of testing:
+
+   Full aux_glc test suite on cheyenne and izumi.
+
+   All tests passed and were bit-for-bit.
+
 ================================================================================
 Originator: sacks
 Date: December 27, 2021

Externals_CISM.cfg

@@ -2,7 +2,7 @@
 local_path = source_cism
 protocol = git
 repo_url = https://github.com/ESCOMP/cism
-tag = cism_main_2.01.010
+tag = cism_main_2.01.011
 required = True
 
 [externals_description]

cime_config/buildnml

@@ -284,15 +284,6 @@ def _icesheet_consistency_checks(case, icesheet_names, glc_grid_names):
         expect(found_evolving_icesheet,
                "CISM_EVOLVE is TRUE, but no ice sheets are set to evolve")
 
-    # TODO(wjs, 2021-02-26) This is a temporary check that we can remove once we resolve
-    # https://github.com/ESCOMP/CISM-wrapper/issues/54
-    # For now, ensure that all ice sheets are either evolving or non-evolving (having one
-    # evolving and one non-evolving is untested)
-    icesheet_evolution = _is_icesheet_evolving(case, icesheet_names[0])
-    for icesheet in icesheet_names[1:]:
-        expect(_is_icesheet_evolving(case, icesheet) == icesheet_evolution,
-               "For now, ice sheets must either all be evolving or all be non-evolving")
-
 ####################################################################################
 def _check_compset_grid_consistency(case, icesheet_names, glc_grid_names):
 ####################################################################################
@@ -383,7 +374,6 @@ def _create_cism_in(case, confdir, inst_string, icesheet_names, infile, nmlgen,
     # Note that the cism_evolve value used in cism_in is the overall CISM_EVOLVE, which is
     # true if *any* ice sheet is evolving.
     config['cism_evolve'] = '.true.' if case.get_value('CISM_EVOLVE') else '.false.'
-    config['glc_two_way_coupling'] = '.true.' if case.get_value('GLC_TWO_WAY_COUPLING') else '.false.'
     config['calendar'] = case.get_value("CALENDAR")
 
     run_type = _get_effective_run_type(case)
@@ -421,8 +411,6 @@ def _create_cism_in(case, confdir, inst_string, icesheet_names, infile, nmlgen,
     output_file = os.path.join(confdir, "cism_in")
     nmlgen.write_output_file(output_file, data_list_path, groups=groups, sorted_groups=False)
 
-    _cism_in_final_consistency_checks(nmlgen, config)
-
 ####################################################################################
 def _create_cism_config(case, confdir, inst_string, icesheet, glc_grid,
                         infile_overall, infile_this_icesheet, nmlgen, data_list_path):
@@ -449,6 +437,10 @@ def _create_cism_config(case, confdir, inst_string, icesheet, glc_grid,
     config['icesheet'] = icesheet
     config['glc_grid']  = glc_grid
     config['cism_phys'] = case.get_value("CISM_PHYS")
+    if case.get_value('GLC_TWO_WAY_COUPLING'):
+        config['glc_two_way_coupling'] = '.true.'
+    else:
+        config['glc_two_way_coupling'] = '.false.'
     if _is_icesheet_evolving(case, icesheet):
         config['cism_evolve_this_icesheet'] = '.true.'
     else:
@@ -478,7 +470,6 @@ def _create_cism_config(case, confdir, inst_string, icesheet, glc_grid,
                          icesheet=icesheet,
                          run_type=run_type,
                          cism_observed_ic=cism_observed_ic,
-                         cism_evolve_this_icesheet=config['cism_evolve_this_icesheet'],
                          inst_string=inst_string)
 
     #----------------------------------------------------
@@ -514,7 +505,7 @@ def _create_cism_config(case, confdir, inst_string, icesheet, glc_grid,
     # ------------------------------------------------------------------------
     # Final consistency checks
     # ------------------------------------------------------------------------
-    _cism_config_final_consistency_checks(nmlgen)
+    _cism_config_final_consistency_checks(nmlgen, icesheet)
 
 ###############################################################################
 def _initial_consistency_checks(case):
@@ -538,9 +529,19 @@ def _get_effective_run_type(case):
 
     comp_interface = case.get_value("COMP_INTERFACE")
     cism_evolve = case.get_value("CISM_EVOLVE")
-    # NOTE: with the nuopc interface, the CISM run phase is never called when running
-    # in noevolve mode, so a restart file will never be written for cism - only initial
-    # information will be sent back to CTSM.
+    # NOTE: with the nuopc interface, the CISM run phase is never called when running in
+    # noevolve mode, so a restart file will never be written for cism - only initial
+    # information will be sent back to CTSM. Since we don't have a restart file to start
+    # from, we need to tell CISM to start in startup mode in this situation.
+    #
+    # Note that this looks at the overall cism_evolve, which is only False if no ice
+    # sheets are evolving: In the situation where one ice sheet is evolving but another is
+    # not, the overall cism_evolve will be True and restart files will still be written
+    # for every ice sheet (even non-evolving ones), so we can safely restart from these
+    # restart files (so we do not need to force run_type to "startup"). (Also note that,
+    # because all ice sheets share a single time manager in the CISM-wrapper layer, it
+    # might currently be awkward or impossible for one ice sheet to start in startup mode
+    # and another to start in branch mode, for example.)
     if (comp_interface == 'nuopc' and not cism_evolve):
         logger.debug("Interface is nuopc, cism is not evolving: CISM run_type is always startup")
         run_type = "startup"
@@ -549,7 +550,7 @@ def _get_effective_run_type(case):
 
 ###############################################################################
 def _set_restart_options(nmlgen, case, icesheet, run_type, cism_observed_ic,
-                         cism_evolve_this_icesheet, inst_string):
+                         inst_string):
 ###############################################################################
     """
     Sets defaults for variables related to restart in cism_in
@@ -560,7 +561,6 @@ def _set_restart_options(nmlgen, case, icesheet, run_type, cism_observed_ic,
     - icesheet: name of this ice sheet
     - run_type: string - 'startup', 'hybrid' or 'branch'
     - cism_observed_ic: logical
-    - cism_evolve_this_icesheet: string - '.true.' or '.false.'
     - inst_string: string
     """
     if run_type == 'startup':
@@ -570,7 +570,7 @@ def _set_restart_options(nmlgen, case, icesheet, run_type, cism_observed_ic,
             expect(False,"CISM_OBSERVED_IC=TRUE not allowed for branch runs - only for hybrid runs")
         else:
             restart = 0
-    elif run_type == 'hybrid' and _value_is_false(cism_evolve_this_icesheet):
+    elif run_type == 'hybrid' and not case.get_value('CISM_EVOLVE'):
         # We could use the hybrid refcase's CISM restart file in this case. However, we
         # often run into problems where a change in CISM makes it incompatible with old
         # restart files, and this breaks configurations that are set up as hybrid runs
@@ -588,9 +588,12 @@ def _set_restart_options(nmlgen, case, icesheet, run_type, cism_observed_ic,
         # observed initial conditions. Thus, the user would need to explicitly set
         # 'cisminputfile' and 'restart' in this case.
         #
-        # Note that this logic is currently based on the ice sheet-specific CISM_EVOLVE
-        # value; this may or may not be the right thing to do (see
-        # https://github.com/ESCOMP/CISM-wrapper/issues/54).
+        # Note that this logic is based on the overall CISM_EVOLVE, so it is not triggered
+        # if one ice sheet is evolving but another is not. Also note that this logic is
+        # currently redundant with the NUOPC-specific logic that sets run_type to
+        # "startup" if CISM_EVOLVE is FALSE (in _get_effective_run_type). But we're
+        # keeping this redundancy for the sake of MCT runs and in case the NUOPC-specific
+        # logic is ever changed.
         restart = 0
     elif run_type == 'branch' or run_type == 'hybrid':
         run_refcase = case.get_value('RUN_REFCASE')
@@ -692,41 +695,28 @@ def _add_conditional_section_to_cism_config(nmlgen, output_file, group,
         nmlgen.confirm_group_is_empty(group, errmsg)
 
 ###############################################################################
-def _cism_in_final_consistency_checks(nmlgen, config):
+def _cism_config_final_consistency_checks(nmlgen, icesheet):
 ###############################################################################
     num_errors = 0
+    num_errors += _check_cism_dt(float(nmlgen.get_value("dt")), icesheet)
 
-    # TODO(wjs, 2021-02-25) We will move zero_gcm_fluxes to the cism.icesheet.config file,
-    # making it ice sheet-specific. Then this needs to be changed to check the ice
-    # sheet-specific zero_gcm_fluxes against the ice sheet-specific evolve_ice. See also
-    # https://github.com/ESCOMP/CISM-wrapper/issues/53.
-    if _value_is_false(config["cism_evolve"]):
-        # The important condition is if evolve_ice = 0. But since evolve_ice is in
-        # cism.icesheet.config, we don't have access to that value when doing consistency
-        # checks on cism_in. We force evolve_ice to match CISM_EVOLVE, so we can check
-        # this by checking CISM_EVOLVE.
+    if int(nmlgen.get_value("evolve_ice")) == 0:
         if _value_is_false(nmlgen.get_value("zero_gcm_fluxes")):
             errmsg = """\
-ERROR: For CISM_EVOLVE FALSE, you must also set zero_gcm_fluxes = .true.
+ERROR for ice sheet {}:
+For evolve_ice = 0 (set from CISM_EVOLVE_ICESHEET for this icesheet),
+zero_gcm_fluxes must be set to .true.
 (This is because evolve_ice = 0 implies that there will be no fluxes,
-and you must explicitly set zero_gcm_fluxes = .true. for the sake of logic
+and so zero_gcm_fluxes must also be set to .true. for the sake of logic
 that depends on whether these fluxes will be zero - particularly, the creation
-of icemask_coupled_fluxes used by CTSM)."""
+of icemask_coupled_fluxes used by CTSM).""".format(icesheet)
             print(errmsg)
             num_errors += 1
 
-    expect(num_errors == 0, "Errors in cism_in final consistency checks (see above)")
-
-###############################################################################
-def _cism_config_final_consistency_checks(nmlgen):
-###############################################################################
-    num_errors = 0
-    num_errors += _check_cism_dt(float(nmlgen.get_value("dt")))
-
-    expect(num_errors == 0, "Errors in cism.config final consistency checks (see above)")
+    expect(num_errors == 0, "Errors in cism.{}.config final consistency checks (see above)".format(icesheet))
 
 ###############################################################################
-def _check_cism_dt(dt):
+def _check_cism_dt(dt, icesheet):
 ###############################################################################
     """Checks CISM's dt value: i.e., the dt variable in the time section of cism.icesheet.config
 
@@ -742,9 +732,10 @@ def _check_cism_dt(dt):
     # value near machine epsilon
     if (abs(dt_hours - dt_hours_int)/dt_hours > 1e-15):
         errmsg = """\
-ERROR: dt (in years) must translate into an integer number of hours
+ERROR for ice sheet {}:
+dt (in years) must translate into an integer number of hours
 dt = {}
-dt (hours) = {}""".format(dt, dt_hours)
+dt (hours) = {}""".format(icesheet, dt, dt_hours)
         print(errmsg)
         num_errors += 1
 

cime_config/config_component.xml

@@ -129,9 +129,8 @@
      <default_value>FALSE</default_value>
      <values>
        <!-- If ANY ice sheet is evolving, then set the overall
-            CISM_EVOLVE to TRUE. This is needed for some logic in CMEPS,
-            but note that combining an evolving and non-evolving ice
-            sheet has not been tested yet and may not work correctly. -->
+            CISM_EVOLVE to TRUE. This is needed for some logic in CMEPS.
+       -->
        <value compset="_CISM[^_]*-EVOLVE">TRUE</value>
 
        <!-- BACKWARDS_COMPATIBILITY(wjs, 2021-02-25) Backwards

cime_config/namelist_definition_cism.xml

@@ -142,25 +142,6 @@ Sub-elements of each entry include:
     </desc>
   </entry>
 
-  <!-- TODO(wjs, 2021-02-25) I think we'll need to move this to
-       cism.icesheet.config, since we will allow cism_evolve to differ
-       for each ice sheet -->
-  <entry id="zero_gcm_fluxes">
-    <type>logical</type>
-    <category>cism</category>
-    <group>cism_params</group>
-    <values>
-      <value glc_two_way_coupling=".true." cism_evolve=".true.">.false.</value>
-      <value glc_two_way_coupling=".false." cism_evolve=".true.">.true.</value>
-      <value glc_two_way_coupling=".true." cism_evolve=".false.">.true.</value>
-      <value glc_two_way_coupling=".false." cism_evolve=".false.">.true.</value>
-    </values>
-    <desc>
-      If true, zero out all fluxes sent to the GCM
-      Default: Depends on GLC_TWO_WAY_COUPLING xml variable
-    </desc>
-  </entry>
-
   <entry id="test_coupling">
     <type>logical</type>
     <category>cism</category>
@@ -186,6 +167,9 @@ Sub-elements of each entry include:
       Whether to enable overrides of the glc fraction sent to the coupler.
       If this is true, then settings in glc_override_nml are used.
       ONLY MEANT FOR TESTING - SHOULD NOT BE USED FOR SCIENCE RUNS.
+      Note that, if running with multiple ice sheets, this and the
+      various override settings apply to all ice sheets: there is no way
+      to do overrides for one ice sheet but not the others.
       Default: .false.
     </desc>
   </entry>
@@ -626,6 +610,23 @@ Sub-elements of each entry include:
     </desc>
   </entry>
 
+  <entry id="zero_gcm_fluxes">
+    <type>logical</type>
+    <category>cism_config_climate</category>
+    <group>glad_climate</group>
+    <values>
+      <value glc_two_way_coupling=".true." cism_evolve_this_icesheet=".true.">.false.</value>
+      <value glc_two_way_coupling=".false." cism_evolve_this_icesheet=".true.">.true.</value>
+      <value glc_two_way_coupling=".true." cism_evolve_this_icesheet=".false.">.true.</value>
+      <value glc_two_way_coupling=".false." cism_evolve_this_icesheet=".false.">.true.</value>
+    </values>
+    <desc>
+      If true, zero out all fluxes sent to the GCM
+      Default: Depends on GLC_TWO_WAY_COUPLING xml variable and whether
+      this ice sheet is evolving
+    </desc>
+  </entry>
+
   <entry id="ice_tstep_multiply" skip_default_entry="true">
     <type>integer</type>
     <category>cism_config_climate</category>

cime_config/testdefs/testlist_cism.xml

@@ -840,6 +840,19 @@
     </machines>
   </test>
 
+  <test name="ERI_Ly9" grid="f09_g17_ais8gris4" compset="T1850Gag" testmods="cism-noevolve_ais">
+    <machines>
+
+      <machine name="cheyenne" compiler="gnu" category="aux_glc">
+        <options>
+          <option name="wallclock">0:50:00</option>
+          <option name="comment">ERI test with one ice sheet evolving and the other not, to make sure restart, branch and hybrid runs all work when one ice sheet is evolving and another is not.</option>
+        </options>
+      </machine>
+
+    </machines>
+  </test>
+
   <test name="IRT_Ly5_P24x1" grid="f09_g17_ais8gris4" compset="T1850Gag">
     <machines>
 

cime_config/testdefs/testmods_dirs/cism/noevolve_ais/README

@@ -0,0 +1,2 @@
+This testmod turns off ice evolution for Antarctica but leaves it on for
+other ice sheets. It is meant to be used in a multi-ice sheet test.

cime_config/testdefs/testmods_dirs/cism/noevolve_ais/shell_commands

@@ -0,0 +1 @@
+./xmlchange CISM_EVOLVE_ANTARCTICA=FALSE

drivers/cpl/mct/glc_comp_mct.F90

@@ -18,7 +18,7 @@ module glc_comp_mct
   use glc_import_export
   use glc_cpl_indices
   use glc_constants,       only: verbose, stdout, stderr, radius
-  use glc_constants,       only: zero_gcm_fluxes, model_doi_url, icesheet_names
+  use glc_constants,       only: zero_gcm_fluxes_for_all_icesheets, model_doi_url, icesheet_names
   use glc_InitMod,         only: glc_initialize
   use glc_RunMod,          only: glc_run
   use glc_FinalMod,        only: glc_final
@@ -209,7 +209,7 @@ subroutine glc_init_mct( EClock, cdata, x2g, g2x, NLFilename )
 
     ! Set flags in infodata
 
-    glc_coupled_fluxes = (.not. zero_gcm_fluxes)
+    glc_coupled_fluxes = (.not. zero_gcm_fluxes_for_all_icesheets)
     call seq_infodata_PutData(infodata, &
          glc_present= .true., &
          glclnd_present = .true., &

drivers/cpl/mct/glc_import_export.F90

@@ -3,7 +3,7 @@ module glc_import_export
   use shr_sys_mod
   use shr_kind_mod,        only: IN=>SHR_KIND_IN, R8=>SHR_KIND_R8
   use shr_kind_mod,        only: CS=>SHR_KIND_CS, CL=>SHR_KIND_CL
-  use glc_constants,       only: verbose, stdout, stderr, tkfrz, zero_gcm_fluxes, enable_frac_overrides
+  use glc_constants,       only: verbose, stdout, stderr, tkfrz, enable_frac_overrides
   use glc_communicate,     only: my_task, master_task
   use glc_cpl_indices
 
@@ -58,7 +58,7 @@ subroutine glc_export(g2x)
 
     !-------------------------------------------------------------------
     use glc_indexing, only : get_nx, get_ny, spatial_to_vector
-    use glc_fields, only : cpl_bundles
+    use glc_fields, only : cpl_bundles, ice_sheet
     use glc_route_ice_runoff, only: route_ice_runoff
     use glc_override_frac   , only: do_frac_overrides
 
@@ -127,7 +127,7 @@ subroutine glc_export(g2x)
     allocate(rofi_to_ocn(nx, ny))
     allocate(rofi_to_ice(nx, ny))
 
-    if (zero_gcm_fluxes) then
+    if (ice_sheet%instances(1)%zero_gcm_fluxes) then
        icemask_coupled_fluxes = 0._r8
        hflx_to_cpl = 0._r8
        rofl_to_cpl = 0._r8