Upgrade 1-Degree to 025-Degree RYF Configuration (#48)

* Update configuration to quarter-degree resolution and MOM6 namelist parameters

* Set max_blocks based on output from ice.log

* Set pio tasks for cice to one per node.

This is a reasonable default, I haven't investgated in detail.

* Update config.yaml

Co-authored-by: Dougie Squire <[email protected]>

* Correct alignment issues in ice_in

Co-authored-by: Dougie Squire <[email protected]>

* Restore wave coupler time step

* Remove unused variable RESTART_CONTROL

* Change DT_THERM to 1350

* Revert to original cpl dt values

* update block_size_x=30, block_size_y=27, max_blocks=10 from https://github.com/COSIMA/ACCESS-OM2-1-025-010deg-report

---------

Co-authored-by: anton-climate <[email protected]>
Co-authored-by: Dougie Squire <[email protected]>
Co-authored-by: minghangli <[email protected]>

Author: 4 people

MOM_input

@@ -12,11 +12,11 @@ THICKNESSDIFFUSE = True         !   [Boolean] default = False
 THICKNESSDIFFUSE_FIRST = True   !   [Boolean] default = False
                                 ! If true, do thickness diffusion or interface height smoothing before dynamics.
                                 ! This is only used if THICKNESSDIFFUSE or APPLY_INTERFACE_FILTER is true.
-DT = 1800.0                     !   [s]
+DT = 1350.0                     !   [s]
                                 ! The (baroclinic) dynamics time step.  The time-step that is actually used will
                                 ! be an integer fraction of the forcing time-step (DT_FORCING in ocean-only mode
                                 ! or the coupling timestep in coupled mode.)
-DT_THERM = 3600.0               !   [s] default = 1800.0
+DT_THERM = 1350.0               !   [s] default = 1800.0
                                 ! The thermodynamic and tracer advection time step. Ideally DT_THERM should be
                                 ! an integer multiple of DT and less than the forcing or coupling time-step,
                                 ! unless THERMO_SPANS_COUPLING is true, in which case DT_THERM can be an integer
@@ -63,10 +63,10 @@ SAVE_INITIAL_CONDS = True       !   [Boolean] default = False
 TRIPOLAR_N = True               !   [Boolean] default = False
                                 ! Use tripolar connectivity at the northern edge of the domain.  With
                                 ! TRIPOLAR_N, NIGLOBAL must be even.
-NIGLOBAL = 360                  !
+NIGLOBAL = 1440                 !
                                 ! The total number of thickness grid points in the x-direction in the physical
                                 ! domain. With STATIC_MEMORY_ this is set in MOM_memory.h at compile time.
-NJGLOBAL = 300                  !
+NJGLOBAL = 1080                 !
                                 ! The total number of thickness grid points in the y-direction in the physical
                                 ! domain. With STATIC_MEMORY_ this is set in MOM_memory.h at compile time.
 
@@ -119,7 +119,7 @@ MINIMUM_DEPTH = 0.5             !   [m] default = 0.0
                                 ! MASKING_DEPTH are rounded to MINIMUM_DEPTH.
 MAXIMUM_DEPTH = 6000.0          !   [m]
                                 ! The maximum depth of the ocean.
-CHANNEL_CONFIG = "global_1deg"  ! default = "none"
+CHANNEL_CONFIG = none           ! default = "none"
                                 ! A parameter that determines which set of channels are
                                 ! restricted to specific  widths.  Options are:
                                 !     none - All channels have the grid width.
@@ -588,11 +588,6 @@ HBD_LINEAR_TRANSITION = True    !   [Boolean] default = False
 ! === module ocean_stochastics_init ===
 
 ! === module ocean_model_init ===
-RESTART_CONTROL = 3             ! default = 1
-                                ! An integer whose bits encode which restart files are written. Add 2 (bit 1)
-                                ! for a time-stamped file, and odd (bit 0) for a non-time-stamped file.  A
-                                ! restart file will be saved at the end of the run segment for any non-negative
-                                ! value.
 OCEAN_SURFACE_STAGGER = "A"     ! default = "C"
                                 ! A case-insensitive character string to indicate the staggering of the surface
                                 ! velocity field that is returned to the coupler.  Valid values include 'A',

README.md

@@ -1,4 +1,4 @@
-# MOM6-CICE6 1 deg JRA55-do RYF ACCESS-OM3 configuration
+# MOM6-CICE6 025 deg JRA55-do RYF ACCESS-OM3 configuration
 
 **WARNING: This configuration is still under development and should not be used for production.**
 

config.yaml

@@ -9,20 +9,20 @@
 # shortpath: /scratch/v45
 
 queue: normal
-ncpus: 48
+ncpus: 240
 jobfs: 10GB
-mem: 192GB
+mem: 960GB
 
-walltime: 01:00:00
-jobname: 1deg_jra55do_ryf
+walltime: 48:00:00
+jobname: 025deg_jra55do_ryf
 
 model: access-om3
 
 exe: /g/data/ik11/spack/0.20.1/opt/linux-rocky8-cascadelake/intel-2021.6.0/access-om3-main-lxbeca7/bin/access-om3-MOM6-CICE6
 input: 
-    - /g/data/ik11/inputs/access-om3/0.x.0/1deg/share # shared grids and topography
-    - /g/data/ik11/inputs/access-om3/0.x.0/1deg/mom # grids, ICs etc
-    - /g/data/ik11/inputs/access-om3/0.x.0/1deg/cice # grids, ICs etc
+    - /g/data/ik11/inputs/access-om3/0.x.0/025deg/share # shared grids and topography
+    - /g/data/ik11/inputs/access-om3/0.x.0/025deg/mom # grids, ICs etc
+    - /g/data/ik11/inputs/access-om3/0.x.0/025deg/cice # grids, ICs etc
     - /g/data/ik11/inputs/access-om3/0.x.0/share/meshes/JRA55do-ESMFmesh.nc # mesh for JRA55-do stream
     - /g/data/ik11/inputs/JRA-55/RYF/v1-4 # datm and drof JRA55-do streams data
 

datm_in

@@ -8,10 +8,10 @@
   flds_preso3 = .false.
   flds_wiso = .false.
   iradsw = 1
-  model_maskfile = "./input/access-om2-1deg-nomask-ESMFmesh.nc"
-  model_meshfile = "./input/access-om2-1deg-nomask-ESMFmesh.nc"
-  nx_global = 360
-  ny_global = 300
+  model_maskfile = "./input/access-om2-025deg-nomask-ESMFmesh.nc"
+  model_meshfile = "./input/access-om2-025deg-nomask-ESMFmesh.nc"
+  nx_global = 1440
+  ny_global = 1080
   restfilm = "null"
   skip_restart_read = .false.
 /

drof_in

@@ -1,9 +1,9 @@
 &drof_nml
   datamode = "copyall"
-  model_maskfile = "./input/access-om2-1deg-nomask-ESMFmesh.nc"
-  model_meshfile = "./input/access-om2-1deg-nomask-ESMFmesh.nc"
-  nx_global = 360
-  ny_global = 300
+  model_maskfile = "./input/access-om2-025deg-nomask-ESMFmesh.nc"
+  model_meshfile = "./input/access-om2-025deg-nomask-ESMFmesh.nc"
+  nx_global = 1440
+  ny_global = 1080
   restfilm = "null"
   skip_restart_read = .false.
 /

ice_in

@@ -8,7 +8,7 @@
   dump_last = .true.
   histfreq = "d", "m", "x", "x", "x"
   history_precision = 8
-  ice_ic = "./input/iced.1900-01-01-10800.nc"
+  ice_ic = 'default' 
   lcdf64 = .false.
   npt = 35040
   pointer_file      = './rpointer.ice'
@@ -65,18 +65,18 @@
   ustar_min = 0.0005
 /
 &domain_nml
-  block_size_x = 16
-  block_size_y = 15
-  distribution_type = "cartesian"
+  block_size_x = 30
+  block_size_y = 27
+  distribution_type = "roundrobin"
   distribution_wght = "latitude"
   maskhalo_bound = .true.
   maskhalo_dyn = .true.
   maskhalo_remap = .true.
   max_blocks = 10
   ns_boundary_type = "tripole"
-  nx_global = 360
-  ny_global = 300
-  processor_shape = "slenderX2"
+  nx_global = 1440
+  ny_global = 1080
+  processor_shape = "square-ice"
 /
 &ice_prescribed_nml
 /
@@ -157,4 +157,4 @@
   f_taubx = 'x' , f_tauby = 'x'
   f_Tref = 'x', f_Qref = 'x'
   f_uocn = 'x' , f_vocn = 'x'
-/
+/

manifests/input.yaml

@@ -66,31 +66,26 @@ work/input/RYF.vas.1990_1991.nc:
   hashes:
     binhash: 17224eecb745b6ce72604b41201c1fe7
     md5: 88dc8c70338bbc8f5efc48ed0009a99f
-work/input/access-om2-1deg-ESMFmesh.nc:
-  fullpath: /g/data/ik11/inputs/access-om3/0.x.0/1deg/share/access-om2-1deg-ESMFmesh.nc
+work/input/access-om2-025deg-ESMFmesh.nc:
+  fullpath: /g/data/ik11/inputs/access-om3/0.x.0/025deg/share/access-om2-025deg-ESMFmesh.nc
   hashes:
-    binhash: c741e080783ac65f2a2e67a9cfdb7645
-    md5: 44e508ac57b244fc357faf5b12933bed
-work/input/access-om2-1deg-nomask-ESMFmesh.nc:
-  fullpath: /g/data/ik11/inputs/access-om3/0.x.0/1deg/share/access-om2-1deg-nomask-ESMFmesh.nc
+    binhash: 5f29f55788f9fc9732c3bd713847d27e
+    md5: c006851eea2a64e881c41f3e9426e4ac
+work/input/access-om2-025deg-nomask-ESMFmesh.nc:
+  fullpath: /g/data/ik11/inputs/access-om3/0.x.0/025deg/share/access-om2-025deg-nomask-ESMFmesh.nc
   hashes:
-    binhash: 491167fde697289421ff25be582d001d
-    md5: 9b7120a42b5cb587492e7c31791eb549
+    binhash: 7ae247d56106c51c1f7a58deda307210
+    md5: db5407804d759435c6846d2d2c661a6e
 work/input/grid.nc:
-  fullpath: /g/data/ik11/inputs/access-om3/0.x.0/1deg/cice/grid.nc
+  fullpath: /g/data/ik11/inputs/access-om3/0.x.0/025deg/cice/grid.nc
   hashes:
-    binhash: 0952424dafa216e7c5d0f29ef96a7cb8
-    md5: 1213e346055ee073fe33dc12578d99c6
-work/input/iced.1900-01-01-10800.nc:
-  fullpath: /g/data/ik11/inputs/access-om3/0.x.0/1deg/cice/iced.1900-01-01-10800.nc
-  hashes:
-    binhash: 2dfb4fa5231df7e352155c266adb29c5
-    md5: 87c012d60c58c65bb56caa98779e5e51
+    binhash: f1239d0326cebb637d0eabc6834e00c9
+    md5: 6946bd45cb5cbf932f869b39eb68bee2
 work/input/kmt.nc:
-  fullpath: /g/data/ik11/inputs/access-om3/0.x.0/1deg/cice/kmt.nc
+  fullpath: /g/data/ik11/inputs/access-om3/0.x.0/025deg/cice/kmt.nc
   hashes:
-    binhash: cb7ff26d756687fedc897920818df02a
-    md5: 1f9806c646a620378e5257e480bc9df7
+    binhash: 52a74c8ec631ee045cf308a16af0d46c
+    md5: d87ca7229f1a579af6f3b308af16e3d1
 work/input/make_rhuss/.git/FETCH_HEAD:
   fullpath: /g/data/ik11/inputs/JRA-55/RYF/v1-4/make_rhuss/.git/FETCH_HEAD
   hashes:
@@ -652,27 +647,27 @@ work/input/make_ryf/make_ryf.py:
     binhash: 9fbb6a9f6bec50ccb2afafecae9a5ff7
     md5: c5ef2aeff9516ec76c80e76aef08d352
 work/input/ocean_hgrid.nc:
-  fullpath: /g/data/ik11/inputs/access-om3/0.x.0/1deg/mom/ocean_hgrid.nc
+  fullpath: /g/data/ik11/inputs/access-om3/0.x.0/025deg/mom/ocean_hgrid.nc
   hashes:
-    binhash: 7f9cf6930cda20f2ba304eef45c2a97d
-    md5: 51f58be0f4ea6da2cb438a893f95c689
+    binhash: e0b2f3bb2d76badb35e2dfc07788738f
+    md5: 38b6f324ae16cc13c180699123ad85b5
 work/input/ocean_temp_salt.res.nc:
-  fullpath: /g/data/ik11/inputs/access-om3/0.x.0/1deg/mom/ocean_temp_salt.res.nc
+  fullpath: /g/data/ik11/inputs/access-om3/0.x.0/025deg/mom/ocean_temp_salt.res.nc
   hashes:
-    binhash: 4812809ecd2abcc47351263cd2c66821
-    md5: c5f7e60b5427a4442f111adc65a2d067
+    binhash: 1c27f9fbe2c7755e4657ed14e5e14145
+    md5: 1797b25e6339595a320076080145fd5a
 work/input/ocean_vgrid.nc:
-  fullpath: /g/data/ik11/inputs/access-om3/0.x.0/1deg/mom/ocean_vgrid.nc
+  fullpath: /g/data/ik11/inputs/access-om3/0.x.0/025deg/mom/ocean_vgrid.nc
   hashes:
-    binhash: 30af5058d31d9133770757144f6bd7cf
-    md5: a1e0b0b0adc363506ff17a362e83f64a
+    binhash: a10a26a7c361279ae0032afc8c83fadd
+    md5: e56e2936499d3ad53429d5940821118d
 work/input/salt_sfc_restore.nc:
-  fullpath: /g/data/ik11/inputs/access-om3/0.x.0/1deg/mom/salt_sfc_restore.nc
+  fullpath: /g/data/ik11/inputs/access-om3/0.x.0/025deg/mom/salt_sfc_restore.nc
   hashes:
-    binhash: d1546a2b957a23e34f1154ad180df60d
-    md5: b2bd35c44017597ba99b85fb61ed4d72
+    binhash: abaea44efc3b0b31ab1a8eb9f46da744
+    md5: 434b4bdb323d17e9714d02519627fea2
 work/input/topog.nc:
-  fullpath: /g/data/ik11/inputs/access-om3/0.x.0/1deg/share/topog.nc
+  fullpath: /g/data/ik11/inputs/access-om3/0.x.0/025deg/share/topog.nc
   hashes:
-    binhash: 049d8b297efe931a1c1d3f40af23afea
-    md5: 4e13d88001b646f3cf4f1c3b8db59f91
+    binhash: a46aeba8e75a3b4c8cd599e17c4dd0eb
+    md5: fa71dfdf4a91198d651d657bcf9aadb6

nuopc.runconfig

@@ -27,11 +27,11 @@ DRIVER_attributes::
 ::
 
 PELAYOUT_attributes::
-     atm_ntasks = 48
+     atm_ntasks = 240
      atm_nthreads = 1
      atm_pestride = 1
      atm_rootpe = 0
-     cpl_ntasks = 48
+     cpl_ntasks = 240
      cpl_nthreads = 1
      cpl_pestride = 1
      cpl_rootpe = 0
@@ -40,31 +40,31 @@ PELAYOUT_attributes::
      esp_nthreads = 1
      esp_pestride = 1
      esp_rootpe = 0
-     glc_ntasks = 48
+     glc_ntasks = 240
      glc_nthreads = 1
      glc_pestride = 1
      glc_rootpe = 0
-     ice_ntasks = 48
+     ice_ntasks = 240
      ice_nthreads = 1
      ice_pestride = 1
      ice_rootpe = 0
-     lnd_ntasks = 48
+     lnd_ntasks = 240
      lnd_nthreads = 1
      lnd_pestride = 1
      lnd_rootpe = 0
      ninst = 1
-     ocn_ntasks = 48
+     ocn_ntasks = 240
      ocn_nthreads = 1
      ocn_pestride = 1
      ocn_rootpe = 0
      pio_asyncio_ntasks = 0
      pio_asyncio_rootpe = 1
      pio_asyncio_stride = 0
-     rof_ntasks = 48
+     rof_ntasks = 240
      rof_nthreads = 1
      rof_pestride = 1
      rof_rootpe = 0
-     wav_ntasks = 48
+     wav_ntasks = 240
      wav_nthreads = 1
      wav_pestride = 1
      wav_rootpe = 0
@@ -116,12 +116,12 @@ ALLCOMP_attributes::
      hostname = gadi
      ice_ncat = 5
      mediator_present = true
-     mesh_atm = ./input/access-om2-1deg-nomask-ESMFmesh.nc
+     mesh_atm = ./input/access-om2-025deg-nomask-ESMFmesh.nc
      mesh_glc = UNSET
-     mesh_ice = ./input/access-om2-1deg-ESMFmesh.nc
+     mesh_ice = ./input/access-om2-025deg-ESMFmesh.nc
      mesh_lnd = UNSET
-     mesh_mask = ./input/access-om2-1deg-ESMFmesh.nc
-     mesh_ocn = ./input/access-om2-1deg-ESMFmesh.nc
+     mesh_mask = ./input/access-om2-025deg-ESMFmesh.nc
+     mesh_ocn = ./input/access-om2-025deg-ESMFmesh.nc
      model_version = unknown
      ocn2glc_coupling = .false.
      ocn2glc_levels = 1:10:19:26:30:33:35
@@ -147,8 +147,8 @@ MED_attributes::
      atm2lnd_map = unset
      atm2ocn_map = unset
      atm2wav_map = unset
-     atm_nx = 360
-     atm_ny = 300
+     atm_nx = 1440
+     atm_ny = 1080
      budget_ann = 1
      budget_daily = 0
      budget_inst = 0
@@ -253,8 +253,8 @@ MED_attributes::
      history_option_wav_inst = never
      ice2atm_map = unset
      ice2wav_smapname = unset
-     ice_nx = 360
-     ice_ny = 300
+     ice_nx = 1440
+     ice_ny = 1080
      info_debug = 1
      lnd2atm_map = unset
      lnd2rof_map = unset
@@ -263,15 +263,15 @@ MED_attributes::
      mapuv_with_cart3d = .true.
      ocn2atm_map = unset
      ocn2wav_smapname = unset
-     ocn_nx = 360
-     ocn_ny = 300
+     ocn_nx = 1440
+     ocn_ny = 1080
      ocn_surface_flux_scheme = 0
      rof2lnd_map = unset
      rof2ocn_fmapname = unset
      rof2ocn_ice_rmapname = unset
      rof2ocn_liq_rmapname = unset
-     rof_nx = 360
-     rof_ny = 300
+     rof_nx = 1440
+     rof_ny = 1080
      wav2ocn_smapname = unset
      wav_nx = 0
      wav_ny = 0
@@ -288,13 +288,13 @@ CLOCK_attributes::
      lnd_cpl_dt = 3600
      ocn_cpl_dt = 3600
      restart_n = 1
-     restart_option = nmonths
+     restart_option = nyears
      restart_ymd = -999
      rof_cpl_dt = 3600
      start_tod = 0
      start_ymd = 19000101
      stop_n = 1
-     stop_option = nmonths
+     stop_option = nyears
      stop_tod = 0
      stop_ymd = -999
      tprof_n = -999
@@ -328,7 +328,7 @@ OCN_attributes::
 
 ROF_attributes::
      Verbosity = off
-     mesh_rof = ./input/access-om2-1deg-nomask-ESMFmesh.nc
+     mesh_rof = ./input/access-om2-025deg-nomask-ESMFmesh.nc
 ::
 
 WAV_attributes::
@@ -377,9 +377,9 @@ ICE_modelio::
      logfile = ice.log
      pio_async_interface = .false.
      pio_netcdf_format = nothing
-     pio_numiotasks = 1
+     pio_numiotasks = 5
      pio_rearranger = 1
-     pio_root = 0
+     pio_root = 1
      pio_stride = 48
      pio_typename = netcdf4p
 ::

nuopc.runseq

@@ -1,5 +1,5 @@
 runSeq:: 
-@3600 
+@1350 
   MED med_phases_aofluxes_run
   MED med_phases_prep_ocn_accum
   MED med_phases_ocnalb_run