diff --git a/.circleci/config.yml b/.circleci/config.yml index 5b107ce77773..fcb61785c368 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2.1 jobs: build: machine: - image: ubuntu-2004:202010-01 + image: ubuntu-2004:2024.01.1 steps: - checkout - run: bash .circleci/setup.sh diff --git a/.circleci/setup.sh b/.circleci/setup.sh index 498d9c6441ec..db0657f26158 100755 --- a/.circleci/setup.sh +++ b/.circleci/setup.sh @@ -11,6 +11,10 @@ function download return $? } +# Install Singularity deps: + +sudo apt-get update && sudo apt-get -y install uuid-dev + # Install Singularity download && \ tar -xzf singularity-${VERSION}.tar.gz && \ diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 8b6ddf3e51f5..4fb88c2dc67b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,3 +11,5 @@ updates: reviewers: - "mahf708" - "bartgol" + labels: + - "AT: Integrate Without Testing" diff --git a/.github/workflows/e3sm-gh-ci-cime-tests.yml b/.github/workflows/e3sm-gh-ci-cime-tests.yml index 7963e6bf1fff..366fef018f20 100644 --- a/.github/workflows/e3sm-gh-ci-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-cime-tests.yml @@ -3,12 +3,25 @@ name: gh on: pull_request: branches: [ master ] + paths: + # first, yes to these + - 'cime_config/**' + - 'components/eam/**' + - 'components/elm/**' + - 'driver-moab/**' + - 'driver-mct/**' + # second, no to these + - '!components/eam/docs/**' + - '!components/eam/mkdocs.yml' + - '!components/elm/docs/**' + - '!components/elm/mkdocs.yml' workflow_dispatch: jobs: ci: + if: ${{ github.event.repository.name == 'e3sm' }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -18,6 +31,7 @@ jobs: - SMS_P4.ne4pg2_oQU480.F2010.singularity_gnu - REP_P4.ne4pg2_oQU480.F2010.singularity_gnu - ERS_P4.ne4pg2_oQU480.F2010.singularity_gnu + - ERS_P4.ne4pg2_oQU480.F2010.singularity_gnu.eam-wcprod_F2010 - ERP_P4.ne4pg2_oQU480.F2010.singularity_gnu - PET_P4.ne4pg2_oQU480.F2010.singularity_gnu - PEM_P4.ne4pg2_oQU480.F2010.singularity_gnu diff --git a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml new file mode 100644 index 000000000000..595518c326d7 --- /dev/null +++ b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml @@ -0,0 +1,59 @@ +name: gh + +on: + pull_request: + branches: [ master ] + paths-ignore: + - 'mkdocs.yaml' + - 'docs/**' + - 'components/*/docs/**' + - 'components/*/mkdocs.yml' + + workflow_dispatch: + +jobs: + + ci-w: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + test: + - SMS_D_P8.ne4pg2_oQU480.WCYCL2010NS.singularity_gnu + - SMS_P8.ne4pg2_oQU480.WCYCL2010NS.singularity_gnu + - REP_P8.ne4pg2_oQU480.WCYCL2010NS.singularity_gnu + - ERS_P8.ne4pg2_oQU480.WCYCL2010NS.singularity_gnu + - ERS_P8.ne4pg2_oQU480.WCYCL2010NS.singularity_gnu.allactive-wcprod_1850 + - ERP_P8.ne4pg2_oQU480.WCYCL2010NS.singularity_gnu + - PET_P8.ne4pg2_oQU480.WCYCL2010NS.singularity_gnu + - PEM_P8.ne4pg2_oQU480.WCYCL2010NS.singularity_gnu + container: + image: ghcr.io/mahf708/e3sm-imgs:v0.0.6 + + steps: + - + name: Checkout + uses: actions/checkout@v4 + with: + show-progress: false + submodules: recursive + - + name: CIME + working-directory: cime/scripts + run: | + mkdir -p $HOME/projects/e3sm/cesm-inputdata/atm/cam/physprops/ + wget https://web.lcrc.anl.gov/public/e3sm/inputdata/atm/cam/physprops/p3_lookup_table_1.dat-v4.1.2 + mv p3_lookup_table_1.dat-v4.1.2 $HOME/projects/e3sm/cesm-inputdata/atm/cam/physprops/ + export USER=test + ./create_test ${{ matrix.test }} --wait --debug + - + name: Artifacts + uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: ${{ matrix.test }} + path: | + ~/projects/e3sm/scratch/${{ matrix.test }}*/TestStatus.log + ~/projects/e3sm/scratch/${{ matrix.test }}*/bld/*.bldlog.* + ~/projects/e3sm/scratch/${{ matrix.test }}*/run/*.log.* + ~/projects/e3sm/scratch/${{ matrix.test }}*/run/*.cprnc.out diff --git a/.github/workflows/e3sm-gh-md-linter.yml b/.github/workflows/e3sm-gh-md-linter.yml new file mode 100644 index 000000000000..6484335213dd --- /dev/null +++ b/.github/workflows/e3sm-gh-md-linter.yml @@ -0,0 +1,28 @@ +name: markdown + +# if .md files are touched in a PR, lint them! + +on: + pull_request: + branches: ["master"] + paths: + - '**/*.md' + +jobs: + linter: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: tj-actions/changed-files@v44 + id: changed-files + with: + files: '**/*.md' + separator: "," + - uses: DavidAnson/markdownlint-cli2-action@v16 + if: steps.changed-files.outputs.any_changed == 'true' + with: + config: 'docs/.markdownlint.json' + globs: ${{ steps.changed-files.outputs.all_changed_files }} + separator: "," diff --git a/.github/workflows/e3sm-gh-pages.yml b/.github/workflows/e3sm-gh-pages.yml index 8b8da284c76f..ccca0c479f26 100644 --- a/.github/workflows/e3sm-gh-pages.yml +++ b/.github/workflows/e3sm-gh-pages.yml @@ -15,7 +15,7 @@ concurrency: jobs: Build-and-Deploy-docs: - if: ${{ github.event.repository.name != 'scream' }} + if: ${{ github.event.repository.name == 'e3sm' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -31,7 +31,7 @@ jobs: - name: Show action trigger run: echo "= The job was automatically triggered by a ${{github.event_name}} event on repo ${{github.event.repository.name}}." - name: Set up Python 3.10 - uses: actions/setup-python@v5.0.0 + uses: actions/setup-python@v5.1.0 with: python-version: "3.10" - name: Install python deps @@ -53,7 +53,7 @@ jobs: folder: ./site/ # If it's a PR from within the same repo, deploy to a preview page # For security reasons, PRs from forks cannot write into gh-pages for now - - if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} + - if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' }} name: Preview docs uses: rossjrw/pr-preview-action@v1 with: diff --git a/.github/workflows/eamxx-gh-pages.yml b/.github/workflows/eamxx-gh-pages.yml index 0144f2abb7c7..91f591a32ba6 100644 --- a/.github/workflows/eamxx-gh-pages.yml +++ b/.github/workflows/eamxx-gh-pages.yml @@ -54,7 +54,7 @@ jobs: echo "= The job was automatically triggered by a ${{github.event_name}} event." - name: Set up Python 3.10 - uses: actions/setup-python@v5.0.0 + uses: actions/setup-python@v5.1.0 with: python-version: "3.10" diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml new file mode 100644 index 000000000000..5ecdf6dec00c --- /dev/null +++ b/.github/workflows/eamxx_default_files.yml @@ -0,0 +1,62 @@ +name: inputdata + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + schedule: + - cron: '00 00 * * *' + workflow_dispatch: + +jobs: + scream-defaults: + runs-on: ubuntu-latest + outputs: + event_name: ${{ github.event_name }} + steps: + - name: Check out the repository + uses: actions/checkout@v4 + with: + show-progress: false + submodules: false + - name: Set up Python 3.11 + uses: actions/setup-python@v5.1.0 + with: + python-version: "3.11" + - name: Run unit tests + working-directory: components/eamxx/cime_config/ + run: | + python -m unittest tests/eamxx_default_files.py -v + + notify-scream-defaults: + needs: scream-defaults + if: ${{ failure() && needs.scream-defaults.outputs.event_name != 'pull_request' }} + runs-on: ubuntu-latest + steps: + - name: Create issue + run: | + previous_issue_number=$(gh issue list \ + --label "$LABELS" \ + --json number \ + --jq '.[0].number') + if [[ -n $previous_issue_number ]]; then + gh issue comment "$previous_issue_number" \ + --body "$BODY" + else + gh issue create \ + --title "$TITLE" \ + --assignee "$ASSIGNEES" \ + --label "$LABELS" \ + --body "$BODY" + fi + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + TITLE: Inputdata server file missing + ASSIGNEES: mahf708,bartgol + LABELS: bug,input file,notify-file-gh-action + BODY: | + Workflow failed! There's likely a missing file specified in the configs! For more information, please see: + - Workflow URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} (number ${{ github.run_number }}, attempt ${{ github.run_attempt }}) + - Workflow SHA: ${{ github.sha }} diff --git a/.gitignore b/.gitignore index c4d2a64bc994..8c1fa12b87ca 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,13 @@ site # Ignore emacs backup files *~ +#Ignore Mac folder files +.DS_Store + +#Ignore vscode dir +.vscode + + # Ignore mkdocs site-generated files in eamxx components/eamxx/site/* # Ignore auto-generated eamxx_params.md file diff --git a/CITATION.cff b/CITATION.cff index bf980f74f6d3..9542a7d01ef4 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -8,8 +8,8 @@ type: software authors: - given-names: E3SM family-names: Project -version: 2.1.0 -doi: 10.11578/E3SM/dc.20230110.5 +version: 3.0.0 +doi: 10.11578/E3SM/dc.20240301.3 repository-code: 'https://github.com/E3SM-Project/E3SM' url: 'https://e3sm.org' license: BSD-3-Clause diff --git a/README.md b/README.md index 84eb15d8e186..192c288d78e1 100644 --- a/README.md +++ b/README.md @@ -9,12 +9,14 @@ the most challenging and demanding climate-change research problems and Department of Energy mission needs while efficiently using DOE Leadership Computing Facilities. -DOI: [10.11578/E3SM/dc.20230110.5](http://dx.doi.org/10.11578/E3SM/dc.20230110.5) +DOI: [10.11578/E3SM/dc.20240301.3](http://dx.doi.org/10.11578/E3SM/dc.20240301.3) Please visit the [project website](https://e3sm.org) or our [Confluence site](https://acme-climate.atlassian.net/wiki/spaces/DOC/overview) for further details. -For questions about the model, use [Github Discussions](https://github.com/E3SM-Project/E3SM/discussions) +For questions about the model, use [Github Discussions](https://github.com/E3SM-Project/E3SM/discussions). + +See our Github-hosted documentation at [https://e3sm-project.github.io/E3SM/](https://e3sm-project.github.io/E3SM/). Table of Contents -------------------------------------------------------------------------------- @@ -27,7 +29,7 @@ Table of Contents Quick Start -------------------------------------------------------------------------------- -The [Quick Start](https://e3sm.org/model/running-e3sm/e3sm-quick-start/) page +The [Quick Start](https://e3sm.org/model/running-e3sm/e3sm-quick-start/) page includes instructions on obtaining the necessary code and input data for model setup and execution on a supported machine. @@ -42,7 +44,7 @@ To run E3SM, it is recommended that you obtain time on a Running -------------------------------------------------------------------------------- -Please refer to [Running E3SM](https://e3sm.org/model/running-e3sm/) +Please refer to [Running E3SM](https://e3sm.org/model/running-e3sm/) for instructions on running the model. Contributing @@ -64,11 +66,11 @@ the following BibTeX entry is provided. author = {{E3SM Project}}, abstractNote = {{E3SM} is a state-of-the-art fully coupled model of the {E}arth's climate including important biogeochemical and cryospheric processes.}, - howpublished = {[Computer Software] \url{https://dx.doi.org/10.11578/E3SM/dc.20230110.5}}, - url = {https://dx.doi.org/10.11578/E3SM/dc.20230110.5}, - doi = {10.11578/E3SM/dc.20230110.5}, - year = 2023, - month = jan, + howpublished = {[Computer Software] \url{https://dx.doi.org/10.11578/E3SM/dc.20240301.3}}, + url = {https://dx.doi.org/10.11578/E3SM/dc.20240301.3}, + doi = {10.11578/E3SM/dc.20240301.3}, + year = 2024, + month = mar, } ``` diff --git a/cime b/cime index 12142ee0c003..4388509869bd 160000 --- a/cime +++ b/cime @@ -1 +1 @@ -Subproject commit 12142ee0c003c135e66265528247518339c0b067 +Subproject commit 4388509869bd5988d6315e2da65b1a2fbfa604fa diff --git a/cime_config/allactive/config_compsets.xml b/cime_config/allactive/config_compsets.xml index b0587d3f7ae8..66edcf617c0d 100755 --- a/cime_config/allactive/config_compsets.xml +++ b/cime_config/allactive/config_compsets.xml @@ -70,9 +70,15 @@ 1850_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV + + + WCYCL2010NS + 2010_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV + + WCYCL1950 - 1950SOI_EAM%CMIP6_ELM%SPBC_MPASSI_MPASO_MOSART_SGLC_SWAV + 1950SOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV @@ -113,12 +119,17 @@ WCYCLSSP585 - SSP585SOI_EAM%CMIP6_ELM%SPBC_MPASSI_MPASO_MOSART_SGLC_SWAV + SSP585SOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV WCYCLSSP370 - SSP370SOI_EAM%CMIP6_ELM%SPBC_MPASSI_MPASO_MOSART_SGLC_SWAV + SSP370SOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV + + + + WCYCLSSP245 + SSP245SOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV @@ -393,7 +404,7 @@ BGWCYCL1850 - 1850_EAM%CMIP6_ELM%SPBC_MPASSI_MPASO_MOSART_MALI%STATIC_SWAV + 1850_EAM%CMIP6_ELM%SPBC_MPASSI_MPASO_MOSART_MALI_SWAV @@ -443,6 +454,7 @@ 1850-01-01 2015-01-01 2015-01-01 + 2015-01-01 diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml index 67a74a3f60c0..0db40bd57982 100644 --- a/cime_config/allactive/config_pesall.xml +++ b/cime_config/allactive/config_pesall.xml @@ -325,7 +325,7 @@ - + "anvil, GPMPAS-JRA compset, 6 nodes" -6 @@ -338,7 +338,7 @@ - + "crusher, GPMPAS-JRA compset, 2 nodes" -4 @@ -351,7 +351,7 @@ - + summit|ascent: GPMPAS-JRA compset on ne30np4 grid -4 @@ -468,7 +468,7 @@ - + -compset A_WCYCL* -res ne30_oEC* on 27 nodes pure-MPI 900 @@ -487,7 +487,7 @@ 0 - + -compset A_WCYCL* -res ne30_oEC* on 40 nodes pure-MPI 1350 @@ -506,7 +506,7 @@ 0 - + -compset A_WCYCL* -res ne30_oEC* on 80 nodes pure-MPI 2700 @@ -525,7 +525,7 @@ 0 - + -compset A_WCYCL* -res ne30_oEC* on 160 nodes pure-MPI 5400 @@ -548,7 +548,7 @@ - + ne120-wcycl on 42 nodes 128x1 ~0.7 sypd 128 @@ -584,7 +584,7 @@ - + ne120-wcycl on 145 nodes, MPI-only 64 64 @@ -609,7 +609,7 @@ 0 - + ne120-wcycl on 145 nodes, threaded 256 64 @@ -644,7 +644,7 @@ 0 - + ne120 coupled-compset on 466 nodes 64 64 @@ -669,7 +669,7 @@ 0 - + ne120-wcycl on 863 nodes, MPI-only 64 64 @@ -694,7 +694,7 @@ 0 - + ne120-wcycl on 863 nodes, threaded 128 64 @@ -729,7 +729,7 @@ 0 - + ne120-wcycl on 825 nodes, threaded, 32 tasks/node 128 32 @@ -764,7 +764,7 @@ 0 - + ne120-wcycl on 800 nodes, threaded, 32 tasks/node 128 32 @@ -801,7 +801,7 @@ - + compy ne120 W-cycle on 310 nodes, 40x1, sypd=1.2 9600 @@ -824,7 +824,7 @@ - + --compset WCYCL* --res ne30pg2_EC30to60E2r2 on 25 nodes pure-MPI, ~5.4 sypd 675 @@ -843,7 +843,7 @@ 0 - + --compset WCYCL* --res ne30pg2_EC30to60E2r2 on 48 nodes pure-MPI, ~9.4 sypd 1350 @@ -862,7 +862,7 @@ 0 - + --compset WCYCL* --res ne30pg2_EC30to60E2r2 on 90 nodes pure-MPI, ~12 sypd 2700 @@ -917,7 +917,7 @@ - + -compset A_WCYCL* -res ne30pg2_EC30to60* on 27 nodes pure-MPI, ~15.5 sypd 2700 @@ -936,7 +936,7 @@ 0 - + -compset A_WCYCL* -res ne30pg2_EC30to60* on 54 nodes pure-MPI, ~25.5 sypd 5400 @@ -957,7 +957,7 @@ - + -compset A_WCYCL* -res ne30pg2_EC30to60* on 11 nodes pure-MPI, ~2.8 sypd 320 @@ -976,7 +976,7 @@ 0 - + -compset A_WCYCL* -res ne30pg2_EC30to60* on 21 nodes pure-MPI, ~5.5 sypd 600 @@ -995,7 +995,7 @@ 0 - + -compset A_WCYCL* -res ne30pg2_EC30to60* on 46 nodes pure-MPI, ~11 sypd 1350 @@ -1014,7 +1014,7 @@ 0 - + -compset A_WCYCL* -res ne30pg2_EC30to60* on 90 nodes pure-MPI, ~18 sypd 2700 @@ -1037,7 +1037,7 @@ - + -compset WCYCL*/CRYO* -res SOwISC12to60E2r4* on 75 nodes pure-MPI, ~5 sypd 900 @@ -1058,7 +1058,7 @@ - + -compset WCYCL*/CRYO* -res ne30pg*SOwISC12to60E2r4* on 105 nodes pure-MPI, ~18.5 sypd 64 @@ -1078,7 +1078,7 @@ 0 - + -compset WCYCL*/CRYO* -res ne30pg*SOwISC12to60E2r4* on 54 nodes pure-MPI, ~11 sypd 64 @@ -1102,7 +1102,7 @@ - + -compset WCYCL*/CRYO* -res ECwISC30to60E2r1* on 48 nodes pure-MPI, ~8.5 sypd 1350 @@ -1123,7 +1123,7 @@ - + -compset WCYCL*/CRYO* -res ne30pg*ECwISC30to60E2r1* on 105 nodes pure-MPI, ~40 sypd 64 @@ -1143,7 +1143,7 @@ 0 - + -compset WCYCL*/CRYO* -res ne30pg*ECwISC30to60E2r1* on 55 nodes pure-MPI, ~25 sypd 64 @@ -1163,7 +1163,7 @@ 0 - + -compset WCYCL*/CRYO* -res ne30pg*ECwISC30to60E2r1* on 28 nodes pure-MPI, ~15 sypd 64 @@ -1187,7 +1187,7 @@ - + gcp12 -compset A_WCYCL* -res ne30pg2_oECv3 with MPASO on 7 nodes 280 @@ -1216,7 +1216,7 @@ - + gcp10 -compset A_WCYCL* -res ne30pg2_oECv3 with MPASO on 11 nodes 240 @@ -1245,7 +1245,7 @@ - + -compset A_WCYCL* -res ne30pg2_oECv3 with MPASO on 7 nodes, 128x1 128 @@ -1273,7 +1273,7 @@ 0 - + -compset A_WCYCL* -res ne30pg2_oECv3 with MPASO on 58 nodes, 128x1, ~20 sypd 128 @@ -1379,10 +1379,38 @@ + + + improv: any compset on ne30np4 grid + + -4 + -4 + -4 + -4 + -4 + -4 + -4 + -4 + + + + improv: BGC compset on ne30np4 grid + + -8 + -8 + -8 + -8 + -8 + -8 + -8 + -8 + + + - + -compset A_WCYCL* -res ne30pg2_oECv3 on 11 nodes pure-MPI, ~2.8 sypd 320 @@ -1401,7 +1429,7 @@ 0 - + -compset A_WCYCL* -res ne30pg2_oECv3 on 21 nodes pure-MPI, ~5.5 sypd 600 @@ -1420,7 +1448,7 @@ 0 - + -compset A_WCYCL* -res ne30pg2_oECv3 on 46 nodes pure-MPI, ~11 sypd 1350 @@ -1439,7 +1467,7 @@ 0 - + -compset A_WCYCL* -res ne30pg2_oECv3 on 90 nodes pure-MPI, ~18 sypd 2700 @@ -1531,10 +1559,97 @@ + + + gcp12 --compset WCYCL* --res ne30pg2_r05_IcoswISC30E3r5 on 4 nodes + + -4 + -4 + -4 + -4 + -4 + -4 + + + + + + pm-cpu --compset WCYCL* --res ne30pg2_r05_IcoswISC30E3r5 on 4 nodes + + -4 + -4 + -4 + -4 + -4 + -4 + + + + + + + + --compset WCYCL* --res ne30pg2_r05_IcosXISC30E3r7 on 20 nodes pure-MPI, ~7.25 sypd + + 1024 + 1024 + 256 + 640 + 384 + 385 + + + 0 + 0 + 1024 + 0 + 640 + 640 + + + + --compset WCYCL* --res ne30pg2_r05_IcosXISC30E3r7 on 54 nodes pure-MPI, ~17.5 sypd + + 2752 + 2752 + 704 + 2048 + 704 + 704 + + + 0 + 0 + 2752 + 0 + 2048 + 2048 + + + + --compset WCYCL* --res ne30pg2_r05_IcosXISC30E3r7 on 105 nodes pure-MPI, ~27.7 sypd + + 5440 + 5440 + 1280 + 4352 + 1088 + 1088 + + + 0 + 0 + 5440 + 0 + 4352 + 4352 + + + - + -compset WCYCL* -res ne30pg2_EC30to60E2r2 on 14 nodes pure-MPI, ~6 sypd 704 @@ -1553,7 +1668,7 @@ 0 - + -compset WCYCL* -res ne30pg2_EC30to60E2r2 on 27 nodes pure-MPI, ~12 sypd 1408 @@ -1572,7 +1687,7 @@ 0 - + -compset WCYCL* -res ne30pg2_EC30to60E2r2 on 53 nodes pure-MPI, ~21 sypd 2752 @@ -1591,7 +1706,7 @@ 0 - + -compset WCYCL* -res ne30pg2_EC30to60E2r2 on 70 nodes pure-MPI, ~24 sypd 3648 @@ -1610,7 +1725,7 @@ 0 - + -compset WCYCL* -res ne30pg2EC30to60E2r2 on 100 nodes pure-MPI, ~30 sypd 5440 @@ -1631,7 +1746,7 @@ - + -compset A_WCYCL* -res ne30pg*EC30to60* on 8 debug Q nodes threaded, 0.8 sypd 384 @@ -1658,7 +1773,7 @@ 0 - + -compset A_WCYCL* -res ne30pg*EC30to60* on 128 default Q nodes pure-MPI, 3.8 sypd 5400 @@ -1681,7 +1796,7 @@ - + -compset WCYCL* -res ne30pg*WCAtl* on 80 nodes pure-MPI, ~8.5 sypd 2700 @@ -1879,7 +1994,7 @@ - + rmod025a 40 40 @@ -1900,7 +2015,7 @@ 0 - + rmod077a 40 40 @@ -1921,7 +2036,7 @@ 0 - + rmod111a 40 40 @@ -1942,7 +2057,7 @@ 0 - + rmod037a 20 40 @@ -1963,7 +2078,7 @@ 2 - + rmod074a 20 40 @@ -1992,7 +2107,7 @@ 0 - + rmod115b 20 40 @@ -2023,7 +2138,7 @@ - + RRM-WCYCL: 64 nodes 2.133 sypd 1800 @@ -2044,7 +2159,7 @@ - + cmod016b64x1 s=2.4 64 128 @@ -2065,7 +2180,7 @@ 0 - + cmod040c64x1 s=5.6 64 128 @@ -2086,7 +2201,7 @@ 0 - + cmod060d64x1 s=8.0 64 128 @@ -2107,7 +2222,7 @@ 0 - + cmod080c64x1 s=10.1 64 128 @@ -2128,7 +2243,7 @@ 0 - + cmod100b64x1 s=12.3 64 128 @@ -2151,7 +2266,7 @@ - + 8 nodes, 128x1 640 @@ -2204,7 +2319,7 @@ - + --compset WCYCL* --res ne30pg2_EC30to60E2r2_wQU225EC30to60E2r2 on 48 nodes pure-MPI, ~8.8 sypd 1350 @@ -2238,7 +2353,7 @@ - + none 924 @@ -2275,7 +2390,7 @@ - + none 2768 @@ -2450,5 +2565,102 @@ + + + pm-cpu, conus 2 nodes + + -2 + -2 + -2 + -2 + -2 + -2 + + + 1 + 1 + 1 + 1 + 1 + 1 + + + + + + + + GIS 1-to-10km (high-res) baseline config + 128 + 128 + + 1350 + 960 + 960 + 1350 + 320 + 960 + 2310 + 2310 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 1350 + 1350 + 0 + 2310 + 1350 + 0 + 0 + + + + + + GIS 1-to-10km (high-res) baseline config + 64 + 64 + + 1350 + 960 + 960 + 1408 + 384 + 960 + 1 + 2368 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 1408 + 1408 + 0 + 2368 + 1408 + 0 + 0 + + + diff --git a/cime_config/config_archive.xml b/cime_config/config_archive.xml index 028a11e84393..84de6e0f171a 100644 --- a/cime_config/config_archive.xml +++ b/cime_config/config_archive.xml @@ -119,7 +119,6 @@ rst - rst.am.timeSeriesStatsMonthly hist unset @@ -129,9 +128,8 @@ rpointer.glc casename.mali.rst.1976-01-01_00000.nc - casename.mali.rst.am.timeSeriesStatsMonthly.1976-01-01_00000.nc - casename.mali.hist.am.globalStats.1976-01-01.nc - casename.mali.hist.am.highFrequencyOutput.1976-01-01_00.00.00.nc + casename.mali.hist.1976-01-01_00000.nc + casename.mali.hist.am.1976-01-01.nc diff --git a/cime_config/config_files.xml b/cime_config/config_files.xml index 97d73759ac4d..71de51964426 100644 --- a/cime_config/config_files.xml +++ b/cime_config/config_files.xml @@ -98,6 +98,7 @@ char $CIMEROOT/CIME/data/config/config_tests.xml + $COMP_ROOT_DIR_OCN/cime_config/config_tests.xml test env_test.xml @@ -321,6 +322,7 @@ char $CIMEROOT/CIME/SystemTests + $COMP_ROOT_DIR_OCN/cime_config/SystemTests test env_test.xml diff --git a/cime_config/config_grids.xml b/cime_config/config_grids.xml index 1894ef66fa5d..ccfb3d908d86 100755 --- a/cime_config/config_grids.xml +++ b/cime_config/config_grids.xml @@ -38,9 +38,6 @@ r05 r05 null - gland5UM - gland4 - gland4 null null null @@ -489,6 +486,36 @@ WC14to60E2r3 + + TL319 + r05 + WC14to60E2r3 + r05 + null + null + WC14to60E2r3 + + + + TL319 + r05 + EC30to60E2r2 + r05 + null + null + EC30to60E2r2 + + + + TL319 + r05 + ARRM10to60E2r1 + r05 + null + null + ARRM10to60E2r1 + + TL319 TL319 @@ -509,6 +536,46 @@ SOwISC12to60E2r4 + + TL319 + TL319 + FRISwISC08to60E3r1 + JRA025 + null + null + FRISwISC08to60E3r1 + + + + TL319 + TL319 + FRISwISC04to60E3r1 + JRA025 + null + null + FRISwISC04to60E3r1 + + + + TL319 + TL319 + FRISwISC02to60E3r1 + JRA025 + null + null + FRISwISC02to60E3r1 + + + + TL319 + TL319 + FRISwISC01to60E3r1 + JRA025 + null + null + FRISwISC01to60E3r1 + + TL319 TL319 @@ -529,6 +596,16 @@ IcoswISC30E3r5 + + TL319 + TL319 + IcosXISC30E3r7 + JRA025 + null + null + IcosXISC30E3r7 + + TL319 TL319 @@ -681,6 +758,16 @@ oEC60to30v3 + + r025 + r025 + r025 + r025 + null + null + oEC60to30v3 + + r0125 r0125 @@ -711,6 +798,16 @@ oRRS18to6v3 + + r025 + r025 + IcoswISC30E3r5 + null + null + null + IcoswISC30E3r5 + + @@ -963,6 +1060,16 @@ oEC60to30v3 + + ne0np4_conus_x4v1_lowcon.pg2 + r05 + IcoswISC30E3r5 + r05 + null + null + IcoswISC30E3r5 + + ne0np4_northamericax4v1 r0125 @@ -973,6 +1080,16 @@ oRRS15to5 + + ne0np4_northamericax4v1.pg2 + r025 + IcoswISC30E3r5 + r025 + null + null + IcoswISC30E3r5 + + ne0np4_northamericax4v1.pg2 r0125 @@ -1376,6 +1493,26 @@ oRRS18to6v3 + + ne120np4.pg2 + r025 + IcoswISC30E3r5 + null + null + null + IcoswISC30E3r5 + + + + ne30np4.pg2 + r025 + IcoswISC30E3r5 + null + null + null + IcoswISC30E3r5 + + ne60np4 ne60np4 @@ -1658,6 +1795,36 @@ oEC60to30v3 + + ne30np4.pg2 + r05 + EC30to60E2r2 + r05 + mpas.gis20km + null + EC30to60E2r2 + + + + ne30np4.pg2 + r05 + IcoswISC30E3r5 + r05 + mpas.gis20km + null + IcoswISC30E3r5 + + + + TL319 + TL319 + IcoswISC30E3r5 + JRA025 + mpas.gis20km + null + IcoswISC30E3r5 + + ne30np4.pg2 r05 @@ -1678,6 +1845,16 @@ oEC60to30v3 + + ne30np4.pg2 + r05 + EC30to60E2r2 + r05 + mpas.gis1to10km + null + EC30to60E2r2 + + ne30np4.pg2 r0125 @@ -1688,6 +1865,36 @@ EC30to60E2r2 + + ne30np4.pg2 + r05 + EC30to60E2r2 + r05 + mpas.gis1to10kmR2 + null + EC30to60E2r2 + + + + ne30np4.pg2 + r05 + IcoswISC30E3r5 + r05 + mpas.gis1to10kmR2 + null + IcoswISC30E3r5 + + + + TL319 + TL319 + IcoswISC30E3r5 + JRA025 + mpas.gis1to10kmR2 + null + IcoswISC30E3r5 + + ne120np4.pg2 r0125 @@ -2054,6 +2261,16 @@ IcoswISC30E3r5 + + ne30np4.pg2 + r05 + IcosXISC30E3r7 + r05 + null + null + IcosXISC30E3r7 + + ne30np4.pg2 r05 @@ -2404,10 +2621,20 @@ $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_WCAtl12to45E2r4.210318.nc $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_SOwISC12to60E2r4.210119.nc $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_SOwISC12to60E2r4.210119.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_FRISwISC08to60E3r1.240214.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_FRISwISC08to60E3r1.240214.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_FRISwISC04to60E3r1.240214.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_FRISwISC04to60E3r1.240214.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_FRISwISC02to60E3r1.240214.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_FRISwISC02to60E3r1.240214.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_FRISwISC01to60E3r1.240216.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_FRISwISC01to60E3r1.240216.nc $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_ECwISC30to60E2r1.201007.nc $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_ECwISC30to60E2r1.201007.nc $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_IcoswISC30E3r5.231121.nc $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_IcoswISC30E3r5.231121.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_IcosXISC30E3r7.240326.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_oRRS18to6v3.220124.nc $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_oRRS18to6v3.220124.nc TL319 is JRA lat/lon grid: @@ -2517,6 +2744,8 @@ $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_oRRS18to6v3.211101.nc $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_IcoswISC30E3r5.231121.nc $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_IcoswISC30E3r5.231121.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_IcosXISC30E3r7.240326.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_gx1v6.190806.nc $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_gx1v6.190806.nc ne30np4.pg2 is Spectral Elem 1-deg grid w/ 2x2 FV physics grid per element: @@ -2777,6 +3006,34 @@ SOwISC12to60E2r4 is a MPAS ice/ocean grid with enhanced resolution of 12km in the Southern Ocean around Antarctica. The high resolution regions smoothly transition to the background resolution of the standard low resolution 60to30km grid: + + 605169 + 1 + $DIN_LOC_ROOT/share/domains/domain.ocn.FRISwISC08to60E3r1.240214.nc + FRISwISC08to60E3r1 is a MPAS ice/ocean grid with enhanced resolution of 12km in the Southern Ocean around Antarctica and 4 km beneath the Filchner-Ronne Ice Shelf. The high resolution regions smoothly transition to the background resolution of the standard low resolution 60to30km grid: + + + + 718054 + 1 + $DIN_LOC_ROOT/share/domains/domain.ocn.FRISwISC04to60E3r1.240214.nc + FRISwISC04to60E3r1 is a MPAS ice/ocean grid with enhanced resolution of 12km in the Southern Ocean around Antarctica and 4 km beneath the Filchner-Ronne Ice Shelf. The high resolution regions smoothly transition to the background resolution of the standard low resolution 60to30km grid: + + + + 1055045 + 1 + $DIN_LOC_ROOT/share/domains/domain.ocn.FRISwISC02to60E3r1.240214.nc + FRISwISC02to60E3r1 is a MPAS ice/ocean grid with enhanced resolution of 12km in the Southern Ocean around Antarctica and 4 km beneath the Filchner-Ronne Ice Shelf. The high resolution regions smoothly transition to the background resolution of the standard low resolution 60to30km grid: + + + + 2162616 + 1 + $DIN_LOC_ROOT/share/domains/domain.ocn.FRISwISC01to60E3r1.240216.nc + FRISwISC01to60E3r1 is a MPAS ice/ocean grid with enhanced resolution of 12km in the Southern Ocean around Antarctica and 4 km beneath the Filchner-Ronne Ice Shelf. The high resolution regions smoothly transition to the background resolution of the standard low resolution 60to30km grid: + + 237984 1 @@ -2791,6 +3048,13 @@ IcoswISC30E3r5 is a MPAS ocean grid generated with the jigsaw/compass process using a dual mesh that is a subdivided icosahedron, resulting in a nearly uniform resolution of 30 km. Additionally, it has ocean in ice-shelf cavities: + + 463013 + 1 + $DIN_LOC_ROOT/share/domains/domain.ocn.IcosXISC30E3r7.240326.nc + IcosXISC30E3r7 is a MPAS ocean grid generated with the jigsaw/compass process using a dual mesh that is a subdivided icosahedron, resulting in a nearly uniform resolution of 30 km.: + + @@ -2823,6 +3087,8 @@ $DIN_LOC_ROOT/share/domains/domain.lnd.r05_WC14to60E2r3.200929.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_IcoswISC30E3r5.231121.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_IcoswISC30E3r5.231121.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.r05_IcosXISC30E3r7.240326.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.r05_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_gx1v6.191014.nc r05 is 1/2 degree river routing grid: @@ -2852,6 +3118,12 @@ r0125 is 1/8 degree river routing grid: + + 1440 + 720 + $DIN_LOC_ROOT/share/domains/domain.lnd.r025_IcoswISC30E3r5.240129.nc + + 28993 1 @@ -2885,6 +3157,12 @@ mpas.gis1to10km is a variable-resolution, from 1- to 10-km, MALI grid of the Greenland Ice Sheet. + + 427386 + 1 + mpas.gis1to10kmR2 is an updated (fewer grid cells; improved optimization) variable-resolution, from 1- to 10-km, MALI grid of the Greenland Ice Sheet. + + 45675 1 @@ -2950,6 +3228,8 @@ 1 $DIN_LOC_ROOT/share/domains/domain.lnd.conusx4v1pg2_oEC60to30v3.200518.nc $DIN_LOC_ROOT/share/domains/domain.ocn.conusx4v1pg2_oEC60to30v3.200518.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.conusx4v1pg2_IcoswISC30E3r5.240205.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.conusx4v1pg2_IcoswISC30E3r5.240205.nc 1-deg with 1/4-deg over CONUS (version 1): @@ -2972,6 +3252,8 @@ $DIN_LOC_ROOT/share/domains/domain.ocn.northamericax4v1pg2_WC14to60E2r3.200929.nc $DIN_LOC_ROOT/share/domains/domain.lnd.northamericax4v1pg2_EC30to60E2r2.220428.nc $DIN_LOC_ROOT/share/domains/domain.ocn.northamericax4v1pg2_EC30to60E2r2.220428.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.northamericax4v1pg2_IcoswISC30E3r5.240416.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.northamericax4v1pg2_IcoswISC30E3r5.240416.nc 1-deg with 1/4-deg over North America (version 1) pg2: @@ -3283,6 +3565,18 @@ cpl/gridmaps/ne30pg2/map_ne30pg2_to_IcoswISC30E3r5-nomask_trbilin.20231121.nc cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_ne30pg2_traave.20231121.nc cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_ne30pg2_traave.20231121.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_IcoswISC30E3r5_trfvnp2.20231121.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_IcoswISC30E3r5_trfvnp2.20231121.nc + + + + cpl/gridmaps/ne30pg2/map_ne30pg2_to_IcosXISC30E3r7_traave.20240326.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_IcosXISC30E3r7_trbilin.20240326.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_IcosXISC30E3r7_trbilin.20240326.nc + cpl/gridmaps/IcosXISC30E3r7/map_IcosXISC30E3r7_to_ne30pg2_traave.20240326.nc + cpl/gridmaps/IcosXISC30E3r7/map_IcosXISC30E3r7_to_ne30pg2_traave.20240326.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_IcosXISC30E3r7_trfvnp2.20240326.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_IcosXISC30E3r7_trfvnp2.20240326.nc @@ -3340,6 +3634,7 @@ cpl/gridmaps/ne30pg2/map_ne30pg2_to_r05_traave.20231130.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_r05_trfvnp2.230516.nc cpl/gridmaps/ne30pg2/map_ne30pg2_to_r05_trbilin.20231130.nc cpl/gridmaps/ne30pg2/map_r05_to_ne30pg2_traave.20231130.nc cpl/gridmaps/ne30pg2/map_r05_to_ne30pg2_traave.20231130.nc @@ -3458,6 +3753,22 @@ cpl/gridmaps/ne120pg2/map_ne120pg2_to_r0125_mono.200707.nc + + cpl/gridmaps/ne120pg2/map_ne120pg2_to_r025_traave.20240206.nc + cpl/gridmaps/ne120pg2/map_ne120pg2_to_r025_trfv2.20240206.nc + cpl/gridmaps/ne120pg2/map_ne120pg2_to_r025_esmfbilin.20240206.nc + cpl/gridmaps/ne120pg2/map_r025_to_ne120pg2_traave.20240206.nc + cpl/gridmaps/ne120pg2/map_r025_to_ne120pg2_traave.20240206.nc + + + + cpl/gridmaps/ne120pg2/map_ne30pg2_to_r025_traave.20240206.nc + cpl/gridmaps/ne120pg2/map_ne30pg2_to_r025_trfv2.20240206.nc + cpl/gridmaps/ne120pg2/map_ne30pg2_to_r025_esmfbilin.20240206.nc + cpl/gridmaps/ne120pg2/map_r025_to_ne30pg2_traave.20240206.nc + cpl/gridmaps/ne120pg2/map_r025_to_ne30pg2_traave.20240206.nc + + cpl/gridmaps/ne120np4/map_ne120np4_to_oRRS18to6v3_mono.20200702.nc cpl/gridmaps/ne120np4/map_ne120np4_to_oRRS18to6v3_mono.20200702.nc @@ -3674,15 +3985,22 @@ cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_conusx4v1pg2_mono.200514.nc cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_conusx4v1pg2_mono.200514.nc + + cpl/gridmaps/conusx4v1pg2/map_conusx4v1pg2_to_IcoswISC30E3r5_traave.20240205.nc + cpl/gridmaps/conusx4v1pg2/map_conusx4v1pg2_to_IcoswISC30E3r5_trbilin.20240205.nc + cpl/gridmaps/conusx4v1pg2/map_conusx4v1pg2_to_IcoswISC30E3r5-nomask_trbilin.20240205.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_conusx4v1pg2_traave.20240205.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_conusx4v1pg2_traave.20240205.nc + - cpl/gridmaps/conusx4v1pg2/map_conusx4v1pg2_to_r05_mono.200514.nc - cpl/gridmaps/conusx4v1pg2/map_conusx4v1pg2_to_r05_bilin.200514.nc - cpl/gridmaps/conusx4v1pg2/map_r05_to_conusx4v1pg2_mono.200514.nc - cpl/gridmaps/conusx4v1pg2/map_r05_to_conusx4v1pg2_mono.200514.nc + cpl/gridmaps/conusx4v1pg2/map_conusx4v1pg2_to_r05_traave.20240205.nc + cpl/gridmaps/conusx4v1pg2/map_conusx4v1pg2_to_r05_trbilin.20240205.nc + cpl/gridmaps/conusx4v1pg2/map_r05_to_conusx4v1pg2_traave.20240205.nc + cpl/gridmaps/conusx4v1pg2/map_r05_to_conusx4v1pg2_traave.20240205.nc - cpl/gridmaps/conusx4v1pg2/map_conusx4v1pg2_to_r05_mono.200514.nc - cpl/gridmaps/conusx4v1pg2/map_conusx4v1pg2_to_r05_bilin.200514.nc + cpl/gridmaps/conusx4v1pg2/map_conusx4v1pg2_to_r05_traave.20240205.nc + cpl/gridmaps/conusx4v1pg2/map_conusx4v1pg2_to_r05_trbilin.20240205.nc @@ -3747,6 +4065,16 @@ cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_northamericax4v1pg2_mono.220428.nc + + cpl/gridmaps/northamericax4v1np4/map_northamericax4v1pg2_to_IcoswISC30E3r5_traave.20240411.nc + cpl/gridmaps/northamericax4v1np4/map_northamericax4v1pg2_to_IcoswISC30E3r5_trbilin.20240331.nc + cpl/gridmaps/northamericax4v1np4/map_northamericax4v1pg2_to_IcoswISC30E3r5-nomask_trbilin.20240331.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_northamericax4v1pg2_traave.20240331.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_northamericax4v1pg2_traave.20240331.nc + cpl/gridmaps/northamericax4v1np4/map_northamericax4v1pg2_to_IcoswISC30E3r5_trfvnp2.20240331.nc + cpl/gridmaps/northamericax4v1np4/map_northamericax4v1pg2_to_IcoswISC30E3r5_trfvnp2.20240331.nc + + cpl/gridmaps/northamericax4v1np4/map_northamericax4v1pg2_to_r0125_mono.20200401.nc cpl/gridmaps/northamericax4v1np4/map_northamericax4v1pg2_to_r0125_bilin.20200401.nc @@ -3754,6 +4082,14 @@ cpl/gridmaps/r0125/map_r0125_to_northamericax4v1pg2_mono.20200401.nc + + cpl/gridmaps/northamericax4v1np4/map_northamericax4v1pg2_to_r025_traave.20240331.nc + cpl/gridmaps/northamericax4v1np4/map_northamericax4v1pg2_to_r025_trfvnp2.20240331.nc + cpl/gridmaps/northamericax4v1np4/map_northamericax4v1pg2_to_r025_trbilin.20240331.nc + cpl/gridmaps/r025/map_r025_to_northamericax4v1pg2_traave.20240331.nc + cpl/gridmaps/r025/map_r025_to_northamericax4v1pg2_traave.20240331.nc + + cpl/gridmaps/northamericax4v1np4/map_northamericax4v1pg2_to_r0125_mono.20200401.nc cpl/gridmaps/northamericax4v1np4/map_northamericax4v1pg2_to_r0125_bilin.20200401.nc @@ -3764,6 +4100,12 @@ cpl/gridmaps/northamericax4v1np4/map_northamericax4v1pg2_to_r05_bilin.220428.nc + + cpl/gridmaps/northamericax4v1np4/map_northamericax4v1pg2_to_r025_traave.20240331.nc + cpl/gridmaps/northamericax4v1np4/map_northamericax4v1pg2_to_r025_trfvnp2.20240331.nc + cpl/gridmaps/northamericax4v1np4/map_northamericax4v1pg2_to_r025_trbilin.20240331.nc + + cpl/gridmaps/arcticx4v1pg2/map_arcticx4v1pg2_to_oARRM60to10_mono.20210407.nc cpl/gridmaps/arcticx4v1pg2/map_arcticx4v1pg2_to_oARRM60to10_mono.20210407.nc @@ -4084,6 +4426,38 @@ cpl/gridmaps/SOwISC12to60E2r4/map_SOwISC12to60E2r4_to_TL319_aave.210119.nc + + cpl/gridmaps/TL319/map_TL319_to_FRISwISC08to60E3r1_traave.20240214.nc + cpl/gridmaps/TL319/map_TL319_to_FRISwISC08to60E3r1-nomask_trbilin.20240214.nc + cpl/gridmaps/TL319/map_TL319_to_FRISwISC08to60E3r1_esmfpatch.20240214.nc + cpl/gridmaps/FRISwISC08to60E3r1/map_FRISwISC08to60E3r1_to_TL319_traave.20240214.nc + cpl/gridmaps/FRISwISC08to60E3r1/map_FRISwISC08to60E3r1_to_TL319_traave.20240214.nc + + + + cpl/gridmaps/TL319/map_TL319_to_FRISwISC04to60E3r1_traave.20240214.nc + cpl/gridmaps/TL319/map_TL319_to_FRISwISC04to60E3r1-nomask_trbilin.20240214.nc + cpl/gridmaps/TL319/map_TL319_to_FRISwISC04to60E3r1_esmfpatch.20240214.nc + cpl/gridmaps/FRISwISC04to60E3r1/map_FRISwISC04to60E3r1_to_TL319_traave.20240214.nc + cpl/gridmaps/FRISwISC04to60E3r1/map_FRISwISC04to60E3r1_to_TL319_traave.20240214.nc + + + + cpl/gridmaps/TL319/map_TL319_to_FRISwISC02to60E3r1_traave.20240214.nc + cpl/gridmaps/TL319/map_TL319_to_FRISwISC02to60E3r1-nomask_trbilin.20240214.nc + cpl/gridmaps/TL319/map_TL319_to_FRISwISC02to60E3r1_esmfpatch.20240214.nc + cpl/gridmaps/FRISwISC02to60E3r1/map_FRISwISC02to60E3r1_to_TL319_traave.20240214.nc + cpl/gridmaps/FRISwISC02to60E3r1/map_FRISwISC02to60E3r1_to_TL319_traave.20240214.nc + + + + cpl/gridmaps/TL319/map_TL319_to_FRISwISC01to60E3r1_traave.20240216.nc + cpl/gridmaps/TL319/map_TL319_to_FRISwISC01to60E3r1-nomask_trbilin.20240216.nc + cpl/gridmaps/TL319/map_TL319_to_FRISwISC01to60E3r1_esmfpatch.20240216.nc + cpl/gridmaps/FRISwISC01to60E3r1/map_FRISwISC01to60E3r1_to_TL319_traave.20240216.nc + cpl/gridmaps/FRISwISC01to60E3r1/map_FRISwISC01to60E3r1_to_TL319_traave.20240216.nc + + cpl/gridmaps/TL319/map_TL319_to_ECwISC30to60E2r1_aave.201006.nc cpl/gridmaps/TL319/map_TL319_to_ECwISC30to60E2r1-nomask_bilin.201006.nc @@ -4100,6 +4474,14 @@ cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_TL319_traave.20231121.nc + + cpl/gridmaps/TL319/map_TL319_to_IcosXISC30E3r7_traave.20240326.nc + cpl/gridmaps/TL319/map_TL319_to_IcosXISC30E3r7_trbilin.20240326.nc + cpl/gridmaps/TL319/map_TL319_to_IcosXISC30E3r7_esmfpatch.20240326.nc + cpl/gridmaps/IcosXISC30E3r7/map_IcosXISC30E3r7_to_TL319_traave.20240326.nc + cpl/gridmaps/IcosXISC30E3r7/map_IcosXISC30E3r7_to_TL319_traave.20240326.nc + + cpl/gridmaps/TL319/map_TL319_to_oRRS18to6v3_aave.220124.nc cpl/gridmaps/TL319/map_TL319_to_oRRS18to6v3_bilin.220124.nc @@ -4108,6 +4490,13 @@ cpl/gridmaps/oRRS18to6v3/map_oRRS18to6v3_to_TL319_aave.220124.nc + + cpl/gridmaps/TL319/map_TL319_to_r05_traave.20240212.nc + cpl/gridmaps/TL319/map_TL319_to_r05_trbilin.20240212.nc + cpl/gridmaps/TL319/map_r05_to_TL319_traave.20240212.nc + cpl/gridmaps/TL319/map_r05_to_TL319_traave.20240212.nc + + cpl/cpl6/map_T31_to_gx3v7_aave_da_090903.nc cpl/cpl6/map_T31_to_gx3v7_patch_090903.nc @@ -4184,7 +4573,8 @@ cpl/gridmaps/ne30pg2/map_ne30pg2_to_r05_traave.20231130.nc - cpl/gridmaps/ne30pg2/map_ne30pg2_to_r05_traave.20231130.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_r05_trfvnp2.230516.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_r05_trbilin.20231130.nc @@ -4222,6 +4612,11 @@ lnd/clm2/mappingdata/maps/ne120np4/map_ne120np4_to_0.125_nomask_aave.160613.nc + + cpl/gridmaps/TL319/map_TL319_to_r05_traave.20240212.nc + cpl/gridmaps/TL319/map_TL319_to_r05_trbilin.20240212.nc + + @@ -4287,6 +4682,11 @@ lnd/clm2/mappingdata/maps/ne240np4/map_0.1x0.1_nomask_to_ne240np4_nomask_aave_da_c120706.nc + + cpl/gridmaps/ne256pg2/map_ne256pg2_to_r0125_mono.200212.nc + cpl/gridmaps/ne256pg2/map_r0125_to_ne256pg2_mono.200212.nc + + cpl/gridmaps/ne1024pg2/map_ne1024pg2_to_r0125_mono.200212.nc cpl/gridmaps/ne1024pg2/map_r0125_to_ne1024pg2_mono.200212.nc @@ -4454,10 +4854,22 @@ + + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_r05_traave.20231121.nc + + + + cpl/gridmaps/IcosXISC30E3r7/map_IcosXISC30E3r7_to_r05_traave.20240326.nc + + cpl/cpl6/map_EC30to60E2r2_to_r05_neareststod.220728.nc + + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_r025_traave.20240401.nc + + @@ -4610,6 +5022,26 @@ cpl/cpl6/map_JRA025_to_SOwISC12to60E2r4_smoothed.r150e300.210119.nc + + cpl/cpl6/map_JRA025_to_FRISwISC08to60E3r1_cstmnn.r150e300.20230926.nc + cpl/cpl6/map_JRA025_to_FRISwISC08to60E3r1_cstmnn.r150e300.20230926.nc + + + + cpl/cpl6/map_JRA025_to_FRISwISC04to60E3r1_cstmnn.r150e300.20230912.nc + cpl/cpl6/map_JRA025_to_FRISwISC04to60E3r1_cstmnn.r150e300.20230912.nc + + + + cpl/cpl6/map_JRA025_to_FRISwISC02to60E3r1_cstmnn.r150e300.20230926.nc + cpl/cpl6/map_JRA025_to_FRISwISC02to60E3r1_cstmnn.r150e300.20230926.nc + + + + cpl/cpl6/map_JRA025_to_FRISwISC01to60E3r1_cstmnn.r150e300.20230926.nc + cpl/cpl6/map_JRA025_to_FRISwISC01to60E3r1_cstmnn.r150e300.20230926.nc + + cpl/cpl6/map_JRA025_to_ECwISC30to60E2r1_smoothed.r150e300.201006.nc cpl/cpl6/map_JRA025_to_ECwISC30to60E2r1_smoothed.r150e300.201006.nc @@ -4620,6 +5052,11 @@ cpl/cpl6/map_JRA025_to_IcoswISC30E3r5_cstmnn.r150e300.20231121.nc + + cpl/cpl6/map_JRA025_to_IcosXISC30E3r7_cstmnn.r150e300.20240326.nc + cpl/cpl6/map_JRA025_to_IcosXISC30E3r7_cstmnn.r150e300.20240326.nc + + cpl/cpl6/map_JRA025_to_oRRS18to6v3_smoothed.r50e100.220124.nc cpl/cpl6/map_JRA025_to_oRRS18to6v3_smoothed.r50e100.220124.nc @@ -4705,6 +5142,16 @@ cpl/cpl6/map_r05_to_IcoswISC30E3r5_cstmnn.r150e300.20231121.nc + + cpl/cpl6/map_r05_to_IcosXISC30E3r7_cstmnn.r150e300.20240326.nc + cpl/cpl6/map_r05_to_IcosXISC30E3r7_cstmnn.r150e300.20240326.nc + + + + cpl/cpl6/map_r025_to_IcoswISC30E3r5_cstmnn.r150e300.20240401.nc + cpl/cpl6/map_r025_to_IcoswISC30E3r5_cstmnn.r150e300.20240401.nc + + cpl/cpl6/map_r0125_to_WC14to60E2r3_smoothed.r150e300.200929.nc cpl/cpl6/map_r0125_to_WC14to60E2r3_smoothed.r150e300.200929.nc @@ -4823,6 +5270,49 @@ cpl/gridmaps/mpas.gis20km/map_mpas.gis20km_to_oEC60to30v3_aave.181115.nc + + cpl/gridmaps/ne30pg2/map_ne30pg2_to_gis20km_traave.20240403.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_gis20km_trbilin.20240403.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_ne30pg2_traave.20240403.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_ne30pg2_traave.20240403.nc + + + + cpl/gridmaps/r05/map_r05_to_gis20km_traave.20240403.nc + cpl/gridmaps/r05/map_r05_to_gis20km_trbilin.20240403.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_r05_traave.20240403.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_r05_traave.20240403.nc + + + + cpl/gridmaps/TL319/map_TL319_to_gis20km_traave.20240404.nc + cpl/gridmaps/TL319/map_TL319_to_gis20km_trbilin.20240404.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_TL319_traave.20240404.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_TL319_traave.20240404.nc + + + + cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis20km_aave.230510.nc + cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis20km_bilin.230510.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_EC30to60E2r2_aave.230510.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_EC30to60E2r2_aave.230510.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_EC30to60E2r2_aave.230510.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_EC30to60E2r2_aave.230510.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_EC30to60E2r2_aave.230510.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_EC30to60E2r2_aave.230510.nc + + + + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_gis20km_esmfaave.20240403.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_gis20km_esmfbilin.20240403.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_IcoswISC30E3r5_esmfaave.20240403.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_IcoswISC30E3r5_esmfaave.20240403.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_IcoswISC30E3r5_esmfaave.20240403.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_IcoswISC30E3r5_esmfaave.20240403.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_IcoswISC30E3r5_esmfaave.20240403.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_IcoswISC30E3r5_esmfaave.20240403.nc + + @@ -4870,6 +5360,53 @@ cpl/gridmaps/mpas.gis1to10km/map_gis1to10km_to_EC30to60E2r2_aave.210304.nc + + + + + + cpl/gridmaps/r05/map_r05_to_gis1to10kmR2_traave.20240403.nc + cpl/gridmaps/r05/map_r05_to_gis1to10kmR2_trbilin.20240403.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_r05_traave.20240403.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_r05_traave.20240403.nc + + + + cpl/gridmaps/ne30pg2/map_ne30pg2_to_gis1to10kmR2_traave.20240403.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_gis1to10kmR2_trbilin.20240403.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_ne30pg2_traave.20240403.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_ne30pg2_traave.20240403.nc + + + + cpl/gridmaps/TL319/map_TL319_to_gis1to10kmR2_traave.20240404.nc + cpl/gridmaps/TL319/map_TL319_to_gis1to10kmR2_trbilin.20240404.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_TL319_traave.20240404.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_TL319_traave.20240404.nc + + + + cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis1to10r02_aave.230725.nc + cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis1to10r02_bilin.230725.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10r02_to_EC30to60E2r2_aave.230725.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10r02_to_EC30to60E2r2_aave.230725.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10r02_to_EC30to60E2r2_aave.230725.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10r02_to_EC30to60E2r2_aave.230725.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10r02_to_EC30to60E2r2_aave.230725.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10r02_to_EC30to60E2r2_aave.230725.nc + + + + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_gis1to10kmR2_esmfaave.20240403.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_gis1to10kmR2_esmfbilin.20240403.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_IcoswISC30E3r5_esmfaave.20240403.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_IcoswISC30E3r5_esmfaave.20240403.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_IcoswISC30E3r5_esmfaave.20240403.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_IcoswISC30E3r5_esmfaave.20240403.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_IcoswISC30E3r5_esmfaave.20240403.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_IcoswISC30E3r5_esmfaave.20240403.nc + + diff --git a/cime_config/customize/case_post_run_io.py b/cime_config/customize/case_post_run_io.py index bb44477ebb57..83419aa81348 100755 --- a/cime_config/customize/case_post_run_io.py +++ b/cime_config/customize/case_post_run_io.py @@ -74,6 +74,12 @@ def _convert_adios_to_nc(case): # Load the environment case.load_env(reset=True) + # Reset MPICH/MPI GPU support, if enabled + is_mpich_gpu_enabled = os.environ.get('MPICH_GPU_SUPPORT_ENABLED') + if int(0 if is_mpich_gpu_enabled is None else is_mpich_gpu_enabled) == 1: + logger.info("Resetting support for GPU in MPICH/MPI library (since its not used by the tool)") + os.environ['MPICH_GPU_SUPPORT_ENABLED'] = str(0); + run_func = lambda: run_cmd(cmd, from_dir=rundir)[0] # Run the modified case diff --git a/cime_config/machines/Depends.chicoma-cpu.gnu.cmake b/cime_config/machines/Depends.chicoma-cpu.gnu.cmake new file mode 100644 index 000000000000..9ebd88ff8762 --- /dev/null +++ b/cime_config/machines/Depends.chicoma-cpu.gnu.cmake @@ -0,0 +1,13 @@ +# For this file, fixes non-BFB behavior of stealth feature on pm-cpu with -O2 +set(NOOPT + eam/src/physics/cam/zm_conv.F90) + +if (NOT DEBUG) + foreach(ITEM IN LISTS NOOPT) + e3sm_deoptimize_file("${ITEM}") + endforeach() +endif() + + + + diff --git a/cime_config/machines/Depends.chicoma-gpu.nvidia.cmake b/cime_config/machines/Depends.chicoma-gpu.nvidia.cmake new file mode 100644 index 000000000000..89235ac5efd1 --- /dev/null +++ b/cime_config/machines/Depends.chicoma-gpu.nvidia.cmake @@ -0,0 +1,10 @@ +list(APPEND REDUCE_OPT_LIST + homme/src/share/derivative_mod_base.F90 +) + +# Can use this flag to avoid internal compiler error for this file (with nvidia/21.11) +if (NOT DEBUG) + foreach(ITEM IN LISTS REDUCE_OPT_LIST) + e3sm_add_flags("${ITEM}" " -Mnovect") + endforeach() +endif() diff --git a/cime_config/machines/Depends.chicoma-gpu.nvidiagpu.cmake b/cime_config/machines/Depends.chicoma-gpu.nvidiagpu.cmake new file mode 100644 index 000000000000..89235ac5efd1 --- /dev/null +++ b/cime_config/machines/Depends.chicoma-gpu.nvidiagpu.cmake @@ -0,0 +1,10 @@ +list(APPEND REDUCE_OPT_LIST + homme/src/share/derivative_mod_base.F90 +) + +# Can use this flag to avoid internal compiler error for this file (with nvidia/21.11) +if (NOT DEBUG) + foreach(ITEM IN LISTS REDUCE_OPT_LIST) + e3sm_add_flags("${ITEM}" " -Mnovect") + endforeach() +endif() diff --git a/cime_config/machines/Depends.oneapi-ifx.cmake b/cime_config/machines/Depends.oneapi-ifx.cmake index 8b51086df4a9..5a958df26eba 100644 --- a/cime_config/machines/Depends.oneapi-ifx.cmake +++ b/cime_config/machines/Depends.oneapi-ifx.cmake @@ -1,3 +1,5 @@ # compile mpas_seaice_core_interface.f90 with ifort, not ifx -e3sm_add_flags("${CMAKE_BINARY_DIR}/core_seaice/model_forward/mpas_seaice_core_interface.f90" "-fc=ifort") +if (NOT MPILIB STREQUAL "openmpi") + e3sm_add_flags("${CMAKE_BINARY_DIR}/core_seaice/model_forward/mpas_seaice_core_interface.f90" "-fc=ifort") +endif() diff --git a/cime_config/machines/Depends.pm-cpu.nvidia.cmake b/cime_config/machines/Depends.pm-cpu.nvidia.cmake index 2947c73aacc4..f3514d118804 100644 --- a/cime_config/machines/Depends.pm-cpu.nvidia.cmake +++ b/cime_config/machines/Depends.pm-cpu.nvidia.cmake @@ -10,6 +10,17 @@ if (NOT DEBUG) endforeach() endif() +list(APPEND NO_STACK_ARRAY_LIST + eam/src/physics/cam/phys_grid_ctem.F90 +) + +# Remove -Mstack_arrays for this one file to avoid error (likely compiler bug) +# As we can't remove flag, try adding -Mnostack_arrays https://github.com/E3SM-Project/E3SM/issues/6350 +# Tried with nvidia/22.7 and 23.9 on pm-cpu +foreach(ITEM IN LISTS NO_STACK_ARRAY_LIST) + e3sm_add_flags("${ITEM}" " -Mnostack_arrays") +endforeach() + # Use -O2 for a few files already found to benefit from increased optimization in Intel Depends file set(PERFOBJS homme/src/share/prim_advection_base.F90 diff --git a/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake index 8cc85b92cc6a..afcca8f479e5 100644 --- a/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake +++ b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake @@ -34,5 +34,5 @@ if (COMP_NAME STREQUAL gptl) endif() set(PIO_FILESYSTEM_HINTS "lustre") -string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_VEGA90A=On -DCMAKE_CXX_FLAGS='-std=gnu++14'") +string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_VEGA90A=On -DCMAKE_CXX_FLAGS='-std=gnu++14' -DKokkos_ENABLE_OPENMP=OFF") set(USE_HIP "TRUE") diff --git a/cime_config/machines/cmake_macros/crayclang_frontier.cmake b/cime_config/machines/cmake_macros/crayclang_frontier.cmake index 6bda90a4187c..6c8d4164cb28 100644 --- a/cime_config/machines/cmake_macros/crayclang_frontier.cmake +++ b/cime_config/machines/cmake_macros/crayclang_frontier.cmake @@ -15,3 +15,6 @@ string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero") # -em -ef generates modulename.mod (lowercase files) to support # Scorpio installs string(APPEND CMAKE_Fortran_FLAGS " -em -ef") + +# to support Fortran specific compiler intrinsic functions +set(E3SM_LINK_WITH_FORTRAN "TRUE") diff --git a/cime_config/machines/cmake_macros/gnu9.cmake b/cime_config/machines/cmake_macros/gnu9.cmake deleted file mode 100644 index 02662c7d4c77..000000000000 --- a/cime_config/machines/cmake_macros/gnu9.cmake +++ /dev/null @@ -1,34 +0,0 @@ -string(APPEND CMAKE_C_FLAGS " -mcmodel=medium") -if (compile_threaded) - string(APPEND CMAKE_C_FLAGS " -fopenmp") -endif() -string(APPEND CMAKE_C_FLAGS_DEBUG " -g -Wall -fbacktrace -fcheck=bounds -ffpe-trap=invalid,zero,overflow") -string(APPEND CMAKE_C_FLAGS_RELEASE " -O") -if (COMP_NAME STREQUAL csm_share) - string(APPEND CMAKE_C_FLAGS " -std=c99") -endif() -string(APPEND CMAKE_CXX_FLAGS " -std=c++14") -if (compile_threaded) - string(APPEND CMAKE_CXX_FLAGS " -fopenmp") -endif() -string(APPEND CMAKE_CXX_FLAGS_DEBUG " -g -Wall -fbacktrace") -string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O") -string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRGNU") -string(APPEND CPPDEFS_DEBUG " -DYAKL_DEBUG") -string(APPEND CMAKE_Fortran_FLAGS " -mcmodel=medium -fconvert=big-endian -ffree-line-length-none -ffixed-line-length-none") -if (compile_threaded) - string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") -endif() -string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -g -Wall -fbacktrace -fcheck=bounds -ffpe-trap=zero,overflow") -string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O") -string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -ffixed-form") -string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -ffree-form") -if (compile_threaded) - string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") -endif() -set(MPICC "mpicc") -set(MPICXX "mpicxx") -set(MPIFC "mpif90") -set(SCC "gcc") -set(SCXX "g++") -set(SFC "gfortran") diff --git a/cime_config/machines/cmake_macros/gnu9_mappy.cmake b/cime_config/machines/cmake_macros/gnu9_mappy.cmake deleted file mode 100644 index 945113b5dc84..000000000000 --- a/cime_config/machines/cmake_macros/gnu9_mappy.cmake +++ /dev/null @@ -1,8 +0,0 @@ -if (COMP_NAME STREQUAL gptl) - string(APPEND CPPDEFS " -DHAVE_SLASHPROC") -endif() -string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") -if (MPILIB STREQUAL mpi-serial AND NOT compile_threaded) - set(PFUNIT_PATH "$ENV{SEMS_PFUNIT_ROOT}") -endif() diff --git a/cime_config/machines/cmake_macros/gnu_chicoma-gpu.cmake b/cime_config/machines/cmake_macros/gnu_chicoma-gpu.cmake new file mode 100644 index 000000000000..807c7d0211eb --- /dev/null +++ b/cime_config/machines/cmake_macros/gnu_chicoma-gpu.cmake @@ -0,0 +1,19 @@ +string(APPEND CONFIG_ARGS " --host=cray") +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY") +endif() +string(APPEND SLIBS " -lblas -llapack") +set(CXX_LINKER "FORTRAN") +if (NOT DEBUG) + string(APPEND CFLAGS " -O2 -g") +endif() +if (NOT DEBUG) + string(APPEND FFLAGS " -O2 -g") +endif() +string(APPEND CXX_LIBS " -lstdc++") +set(MPICC "cc") +set(MPICXX "CC") +set(MPIFC "ftn") +set(SCC "gcc") +set(SCXX "g++") +set(SFC "gfortran") diff --git a/cime_config/machines/cmake_macros/gnu_chrysalis.cmake b/cime_config/machines/cmake_macros/gnu_chrysalis.cmake index d18626a3ef24..d3f99e211552 100644 --- a/cime_config/machines/cmake_macros/gnu_chrysalis.cmake +++ b/cime_config/machines/cmake_macros/gnu_chrysalis.cmake @@ -2,4 +2,3 @@ if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY") endif() set(PIO_FILESYSTEM_HINTS "gpfs") -set(MOAB_PATH "/lcrc/soft/climate/moab/chrysalis/gnu") diff --git a/cime_config/machines/cmake_macros/gnu_improv.cmake b/cime_config/machines/cmake_macros/gnu_improv.cmake new file mode 100644 index 000000000000..3dd45bc09c10 --- /dev/null +++ b/cime_config/machines/cmake_macros/gnu_improv.cmake @@ -0,0 +1,8 @@ +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DHAVE_SLASHPROC") +endif() +string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") +string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") +set(PIO_FILESYSTEM_HINTS "gpfs") +string(APPEND SLIBS " /lcrc/group/e3sm/soft/improv/netlib-lapack/3.12.0/gcc-12.3.0/liblapack.a /lcrc/group/e3sm/soft/improv/netlib-lapack/3.12.0/gcc-12.3.0/libblas.a") + diff --git a/cime_config/machines/cmake_macros/gnugpu_chicoma-gpu.cmake b/cime_config/machines/cmake_macros/gnugpu_chicoma-gpu.cmake new file mode 100644 index 000000000000..b696962159cf --- /dev/null +++ b/cime_config/machines/cmake_macros/gnugpu_chicoma-gpu.cmake @@ -0,0 +1,20 @@ +string(APPEND CONFIG_ARGS " --host=cray") +set(USE_CUDA "TRUE") +string(APPEND CPPDEFS " -DGPU") +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY") +endif() +string(APPEND CPPDEFS " -DTHRUST_IGNORE_CUB_VERSION_CHECK") +string(APPEND SLIBS " -lblas -llapack") +string(APPEND CUDA_FLAGS " -ccbin CC -O2 -arch sm_80 --use_fast_math") +string(APPEND KOKKOS_OPTIONS " -DKokkos_ARCH_AMPERE80=On -DKokkos_ENABLE_CUDA=On -DKokkos_ENABLE_CUDA_LAMBDA=On -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_OPENMP=Off") +if (NOT DEBUG) + string(APPEND CFLAGS " -O2") + string(APPEND FFLAGS " -O2") +endif() +set(MPICC "cc") +set(MPICXX "CC") +set(MPIFC "ftn") +set(SCC "cc") +set(SCXX "CC") +set(SFC "ftn") diff --git a/cime_config/machines/cmake_macros/gnugpu_frontier.cmake b/cime_config/machines/cmake_macros/gnugpu_frontier.cmake index dbbe7ba2b2e9..174f8207d0d0 100644 --- a/cime_config/machines/cmake_macros/gnugpu_frontier.cmake +++ b/cime_config/machines/cmake_macros/gnugpu_frontier.cmake @@ -18,7 +18,7 @@ string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") set(E3SM_LINK_WITH_FORTRAN "TRUE") string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc-libs -lgfortran -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa ") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc/11.2.0/snos/lib64/ -lgfortran -L/opt/rocm-5.4.0/lib -lhsa-runtime64 -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa ") string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On -DKokkos_ENABLE_OPENMP=Off") diff --git a/cime_config/machines/cmake_macros/intel_chrysalis.cmake b/cime_config/machines/cmake_macros/intel_chrysalis.cmake index a38fe5226cdc..fa68bc6075df 100644 --- a/cime_config/machines/cmake_macros/intel_chrysalis.cmake +++ b/cime_config/machines/cmake_macros/intel_chrysalis.cmake @@ -12,7 +12,6 @@ string(APPEND CMAKE_Fortran_FLAGS " -axCORE-AVX2") string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O3 -qno-opt-dynamic-align") set(PIO_FILESYSTEM_HINTS "gpfs") string(APPEND CMAKE_EXE_LINKER_FLAGS " -static-intel") -set(MOAB_PATH "/lcrc/soft/climate/moab/chrysalis/intel") if (MPILIB STREQUAL impi) set(MPICC "mpiicc") set(MPICXX "mpiicpc") diff --git a/cime_config/machines/cmake_macros/nvidia_chicoma-gpu.cmake b/cime_config/machines/cmake_macros/nvidia_chicoma-gpu.cmake new file mode 100644 index 000000000000..5c55dc6bb2f4 --- /dev/null +++ b/cime_config/machines/cmake_macros/nvidia_chicoma-gpu.cmake @@ -0,0 +1,24 @@ +string(APPEND CONFIG_ARGS " --host=cray") +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY") +endif() +string(APPEND SLIBS " -lblas -llapack") +set(CXX_LINKER "FORTRAN") +if (NOT DEBUG) + string(APPEND CFLAGS " -O2") +endif() +if (NOT DEBUG) + string(APPEND CXXFLAGS " -O2") +endif() +if (NOT DEBUG) + string(APPEND FFLAGS " -O2") +endif() +if (compile_threaded) + string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_OPENMP=Off") # work-around for nvidia as kokkos is not passing "-mp" for threaded build +endif() +set(MPICC "cc") +set(MPICXX "CC") +set(MPIFC "ftn") +set(SCC "cc") +set(SCXX "CC") +set(SFC "ftn") diff --git a/cime_config/machines/cmake_macros/nvidiagpu_chicoma-gpu.cmake b/cime_config/machines/cmake_macros/nvidiagpu_chicoma-gpu.cmake new file mode 100644 index 000000000000..e9606e066060 --- /dev/null +++ b/cime_config/machines/cmake_macros/nvidiagpu_chicoma-gpu.cmake @@ -0,0 +1,13 @@ +string(APPEND CONFIG_ARGS " --host=cray") +set(USE_CUDA "TRUE") +string(APPEND CPPDEFS " -DGPU") +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY") +endif() +string(APPEND CPPDEFS " -DTHRUST_IGNORE_CUB_VERSION_CHECK") +string(APPEND CUDA_FLAGS " -ccbin CC -O2 -arch sm_80 --use_fast_math") +string(APPEND SLIBS " -lblas -llapack") +set(CXX_LINKER "FORTRAN") +set(SCC "cc") +set(SCXX "CC") +set(SFC "ftn") diff --git a/cime_config/machines/config_batch.xml b/cime_config/machines/config_batch.xml index 61d2cd364a69..5ba8e38f2e79 100644 --- a/cime_config/machines/config_batch.xml +++ b/cime_config/machines/config_batch.xml @@ -291,7 +291,6 @@ --job-name={{ job_id }} --nodes={{ num_nodes }} --output={{ job_id }}.%j - --exclusive @@ -423,6 +422,16 @@ + + + -l select={{ num_nodes }}:mpiprocs={{ tasks_per_node }} + + + debug + compute + + + collaboration @@ -433,17 +442,16 @@ --constraint=gpu - - --gpus-per-task=1 - + --gpus-per-node=4 --gpu-bind=none + --gpus-per-task=1 --gpu-bind=map_gpu:0,1,2,3 - --gpus-per-task=1 + --gpus-per-node=4 --gpu-bind=none @@ -475,17 +483,16 @@ --constraint=gpu - - --gpus-per-task=1 - + --gpus-per-node=4 --gpu-bind=none + --gpus-per-task=1 --gpu-bind=map_gpu:0,1,2,3 - --gpus-per-task=1 + --gpus-per-node=4 --gpu-bind=none @@ -645,35 +652,40 @@ - - - --nodes={{ num_nodes }} - --ntasks-per-node={{ tasks_per_node }} - --qos=standard - - - standard - - - - - - --nodes={{ num_nodes }} - --ntasks-per-node={{ tasks_per_node }} - --qos=standard - - - standard - - - --partition=standard --qos=standard - standard + standard + + + + + + --partition=gpu + + + --gpu-bind=none + + + --gpu-bind=none + + + --gpu-bind=map_gpu:0,1,2,3 + + + --gpu-bind=none + + + -G 0 + + + -G 0 + + + gpu diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index f3aa5cfa6149..f13901830891 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -206,6 +206,7 @@ aocc cudatoolkit climate-utils + matlab craype-accel-nvidia80 craype-accel-host perftools-base @@ -263,8 +264,6 @@ /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch software MPI_Bcast - $SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/albany-e3sm-serial-release-gcc; else echo "$Albany_ROOT"; fi} - $SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/trilinos-e3sm-serial-release-gcc; else echo "$Trilinos_ROOT"; fi} $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} 4000MB @@ -275,6 +274,9 @@ $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi} Generic + $SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/albany-e3sm-serial-release-gcc-cmake-fix; else echo "$Albany_ROOT"; fi} + $SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/trilinos-e3sm-serial-release-gcc; else echo "$Trilinos_ROOT"; fi} + $SHELL{if [ -z "$Kokkos_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/trilinos-e3sm-serial-release-gcc; else echo "$Kokkos_ROOT"; fi} $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/nvidia-22.7; else echo "$ADIOS2_ROOT"; fi} @@ -290,6 +292,12 @@ $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/aocc-4.0.0; else echo "$ADIOS2_ROOT"; fi} + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/intel; else echo "$MOAB_ROOT"; fi} + + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/gnu; else echo "$MOAB_ROOT"; fi} + -1 @@ -361,6 +369,7 @@ aocc cudatoolkit climate-utils + matlab craype-accel-nvidia80 craype-accel-host perftools-base @@ -504,6 +513,7 @@ aocc cudatoolkit climate-utils + matlab craype-accel-nvidia80 craype-accel-host perftools-base @@ -666,6 +676,7 @@ aocc cudatoolkit climate-utils + matlab craype-accel-nvidia80 craype-accel-host perftools-base @@ -817,6 +828,7 @@ aocc cudatoolkit climate-utils + matlab craype-accel-nvidia80 craype-accel-host perftools-base @@ -1773,7 +1785,7 @@ mappy LINUX proxy.sandia.gov:80 - gnu,gnu9,intel + gnu openmpi /sems-data-store/ACME/mappy/timings .* @@ -1812,21 +1824,12 @@ acme-cmake/3.26.3 - acme-gcc/8.1.0 - - - sems-archive-gcc/9.2.0 - - - sems-archive-intel/19.0.5 + acme-gcc/11.2.0 - + acme-netcdf/4.4.1/exo_acme acme-pfunit/3.2.8/base - - sems-archive-netcdf/4.7.3/base - acme-openmpi/4.1.4 acme-netcdf/4.7.4/acme @@ -1843,6 +1846,7 @@ spread threads Generic + 4000MB @@ -2066,7 +2070,7 @@ ANL CELS General Computing Environment (Linux) workstation (Ubuntu 22.04) - compute-386-01|compute-386-02 + compute-386-(01|02|03|05|07|08)|compute-240-(15) LINUX gnu mpich,openmpi @@ -2127,6 +2131,7 @@ /nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/hdf5/1.12.2/mpich-4.1.2/gcc-12.1.0 /nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-parallel/mpich-4.1.2/gcc-12.1.0 /nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/pnetcdf/1.12.3/mpich-4.1.2/gcc-12.1.0 + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /nfs/gce/projects/climate/software/moab/devel/mpich-4.1.2/gcc-12.1.0; else echo "$MOAB_ROOT"; fi} @@ -2136,6 +2141,7 @@ /nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/hdf5/1.12.2/openmpi-4.1.6/gcc-12.1.0 /nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-parallel/openmpi-4.1.6/gcc-12.1.0 /nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/pnetcdf/1.12.3/openmpi-4.1.6/gcc-12.1.0 + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /nfs/gce/projects/climate/software/moab/devel/openmpi-4.1.6/gcc-12.1.0; else echo "$MOAB_ROOT"; fi} 64M @@ -2218,7 +2224,6 @@ /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-parallel/mpich-4.0/gcc-11.1.0 /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/pnetcdf/1.12.2/mpich-4.0/gcc-11.1.0 $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /nfs/gce/projects/climate/software/moab/devel/mpich-4.0/gcc-11.1.0; else echo "$MOAB_ROOT"; fi} - /nfs/gce/projects/climate/software/moab/devel/mpich-4.0/gcc-11.1.0 @@ -2565,12 +2570,12 @@ intel-mkl/2020.4.304-g2qaxzf - openmpi/4.1.3-pin4k7o - hdf5/1.10.7-eewgp6v - netcdf-c/4.4.1-ihoo4zq - netcdf-cxx/4.2-soitsxm - netcdf-fortran/4.4.4-tplolxh - parallel-netcdf/1.11.0-gvcfihh + openmpi/4.1.6-2mm63n2 + hdf5/1.10.7-4cghwvq + netcdf-c/4.4.1-a4hji6e + netcdf-cxx/4.2-ldoxr43 + netcdf-fortran/4.4.4-husened + parallel-netcdf/1.11.0-icrpxty intel-mpi/2019.9.304-tkzvizk @@ -2616,12 +2621,14 @@ $CIME_OUTPUT_ROOT/$CASE/bld 0.05 0.05 - 1000 + 0 /lcrc/group/e3sm/soft/perl/chrys/lib/perl5 $SHELL{dirname $(dirname $(which nc-config))} $SHELL{dirname $(dirname $(which nf-config))} $SHELL{dirname $(dirname $(which pnetcdf_version))} + ^lockedfile,individual + ^xpmem 128M @@ -2641,6 +2648,11 @@ $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /lcrc/soft/climate/moab/chrysalis/gnu; else echo "$MOAB_ROOT"; fi} + + $SHELL{if [ -z "$Albany_ROOT" ]; then echo /lcrc/group/e3sm/ac.jwatkins/LandIce/AlbanyBuilds/build-gcc-sfad12-e3sm/install; else echo "$Albany_ROOT"; fi} + $SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /lcrc/group/e3sm/ac.jwatkins/LandIce/TrilinosBuilds/build-gcc-e3sm/install; else echo "$Trilinos_ROOT"; fi} + $SHELL{if [ -z "$Kokkos_ROOT" ]; then echo /lcrc/group/e3sm/ac.jwatkins/LandIce/TrilinosBuilds/build-gcc-e3sm/install; else echo "$Kokkos_ROOT"; fi} + @@ -2698,7 +2710,7 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 - 1000 + 0 $SHELL{dirname $(dirname $(which nc-config))} $SHELL{dirname $(dirname $(which nf-config))} @@ -2777,7 +2789,7 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 - 1000 + 0 $SHELL{dirname $(dirname $(which nc-config))} $SHELL{dirname $(dirname $(which nf-config))} @@ -2897,80 +2909,65 @@ - - LLNL Linux Cluster, Linux, 4 V100 GPUs/node, 44 IBM P9 cpu cores/node - lassen.* + + ANL LCRC cluster 825-node AMD 7713 2-sockets 128-cores per node + ilogin(1|2|3|4).lcrc.anl.gov LINUX - gnugpu - spectrum-mpi - cbronze - /usr/workspace/$USER/e3sm_scratch - /usr/gdata/climdat/ccsm3data/inputdata - /usr/gdata/climdat/ccsm3data/inputdata/atm/datm7 - /usr/workspace/$USER/archive/$CASE - /usr/gdata/climdat/baselines/$COMPILER - 16 - lsf - donahue5 -at- llnl.gov - 40 - 40 - - - - - jsrun + gnu + openmpi + e3sm + /lcrc/group/e3sm/$USER/scratch/improv + /lcrc/group/e3sm/data/inputdata + /lcrc/group/e3sm/data/inputdata/atm/datm7 + /lcrc/group/e3sm/$USER/scratch/improv/archive/$CASE + /lcrc/group/e3sm/baselines/improv/$COMPILER + /lcrc/group/e3sm/tools/cprnc/cprnc.improv + 8 + e3sm_integration + 8 + pbspro + E3SM + 128 + 128 + FALSE + + mpirun - -X 1 - $SHELL{if [ {{ total_tasks }} -eq 1 ];then echo --nrs 1 --rs_per_host 1;else echo --nrs $NUM_RS --rs_per_host $RS_PER_NODE;fi} - --tasks_per_rs $SHELL{echo "({{ tasks_per_node }} + $RS_PER_NODE - 1)/$RS_PER_NODE"|bc} - -d plane:$SHELL{echo "({{ tasks_per_node }} + $RS_PER_NODE - 1)/$RS_PER_NODE"|bc} - --cpu_per_rs $ENV{CPU_PER_RS} - --gpu_per_rs $ENV{GPU_PER_RS} - --bind packed:smt:$ENV{OMP_NUM_THREADS} - --latency_priority $ENV{LTC_PRT} - --stdio_mode prepended - $ENV{JSRUN_THREAD_VARS} - $ENV{SMPIARGS} + --tag-output -n {{ total_tasks }} + --map-by ppr:1:core:PE=$ENV{OMP_NUM_THREADS} --bind-to core --oversubscribe - /usr/share/lmod/lmod/init/env_modules_python.py - /usr/share/lmod/lmod/init/perl - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/init/csh - module + /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/lmod-8.3-5be73rg/lmod/lmod/init/sh + /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/lmod-8.3-5be73rg/lmod/lmod/init/csh + /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/lmod-8.3-5be73rg/lmod/lmod/init/env_modules_python.py + /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/lmod-8.3-5be73rg/lmod/lmod/libexec/lmod python module - /usr/share/lmod/lmod/libexec/lmod python - /usr/share/lmod/lmod/libexec/lmod perl - - - git - gcc/8.3.1 - cuda/11.8.0 - cmake/3.16.8 - spectrum-mpi - python/3.7.2 + module + + + cmake/3.27.4 + + + gcc/12.3.0 - /p/gpfs1/$USER/e3sm_scratch/$CASE/run + $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - - + 0.05 + 0 + + /lcrc/group/e3sm/soft/improv/netcdf-c/4.9.2b/gcc-12.3.0/openmpi-4.1.6 + /lcrc/group/e3sm/soft/improv/netcdf-fortran/4.6.1b/gcc-12.3.0/openmpi-4.1.6 + /lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6 + /lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/netcdf-fortran/4.6.1b/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/netcdf-c/4.9.2b/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/openmpi/4.1.6/gcc-12.3.0/bin:/lcrc/group/e3sm/soft/perl/improv/bin:$ENV{PATH} + $SHELL{lp=/lcrc/group/e3sm/soft/improv/netlib-lapack/3.12.0/gcc-12.3.0:/lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6/lib:/lcrc/group/e3sm/soft/improv/netcdf-fortran/4.6.1b/gcc-12.3.0/openmpi-4.1.6/lib:/lcrc/group/e3sm/soft/improv/netcdf-c/4.9.2b/gcc-12.3.0/openmpi-4.1.6/lib:/opt/pbs/lib:/lcrc/group/e3sm/soft/improv/openmpi/4.1.6/gcc-12.3.0/lib; if [ -z "$LD_LIBRARY_PATH" ]; then echo $lp; else echo "$lp:$LD_LIBRARY_PATH"; fi} - -E OMP_NUM_THREADS=$ENV{OMP_NUM_THREADS} -E OMP_PROC_BIND=spread -E OMP_PLACES=threads -E OMP_STACKSIZE=256M + 128M - - y - /usr/gdata/climdat/netcdf/bin:$ENV{PATH} - /usr/gdata/climdat/netcdf/lib:$ENV{LD_LIBRARY_PATH} - /usr/gdata/climdat/netcdf - 2 - 20 - 2 - gpu-cpu - $SHELL{echo "2*((`./xmlquery --value TOTAL_TASKS` + `./xmlquery --value TASKS_PER_NODE` - 1)/`./xmlquery --value TASKS_PER_NODE`)"|bc} - --smpiargs="-gpu" + + cores @@ -2981,11 +2978,11 @@ mpich cbronze /p/lustre2/$USER/e3sm_scratch/ruby - /usr/gdata/climdat/ccsm3data/inputdata - /usr/gdata/climdat/ccsm3data/inputdata/atm/datm7 + /usr/gdata/e3sm/ccsm3data/inputdata + /usr/gdata/e3sm/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER - /usr/gdata/climdat/tools/cprnc + /usr/gdata/e3sm/tools/cprnc 8 lc_slurm donahue5 -at- llnl.gov @@ -3013,18 +3010,19 @@ intel-classic/2021.6.0-magic mvapich2/2.3.7 cmake/3.19.2 - /usr/gdata/climdat/install/quartz/modulefiles + /usr/gdata/e3sm/install/quartz/modulefiles hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 screamML-venv/0.0.1 + subversion $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /usr/gdata/climdat/install/quartz/netcdf-fortran/ + /usr/gdata/e3sm/install/quartz/netcdf-fortran/ /usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0 @@ -3036,11 +3034,11 @@ mpich cbronze /p/lustre2/$USER/e3sm_scratch/quartz - /usr/gdata/climdat/ccsm3data/inputdata - /usr/gdata/climdat/ccsm3data/inputdata/atm/datm7 + /usr/gdata/e3sm/ccsm3data/inputdata + /usr/gdata/e3sm/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER - /usr/gdata/climdat/tools/cprnc + /usr/gdata/e3sm/tools/cprnc 8 lc_slurm donahue5 -at- llnl.gov @@ -3068,18 +3066,19 @@ intel-classic/2021.6.0-magic mvapich2/2.3.7 cmake/3.19.2 - /usr/gdata/climdat/install/quartz/modulefiles + /usr/gdata/e3sm/install/quartz/modulefiles hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 screamML-venv/0.0.1 + subversion $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /usr/gdata/climdat/install/quartz/netcdf-fortran/ + /usr/gdata/e3sm/install/quartz/netcdf-fortran/ /usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0 @@ -4213,181 +4212,153 @@ - - LANL Linux Cluster, 36 pes/node, batch system slurm - gr-fe.*.lanl.gov - LINUX - intel,gnu - openmpi,impi,mvapich - climateacme - /lustre/scratch4/turquoise/$ENV{USER}/E3SM/scratch - /lustre/scratch4/turquoise/$ENV{USER}/E3SM/input_data - /lustre/scratch4/turquoise/$ENV{USER}/E3SM/input_data/atm/datm7 - /lustre/scratch4/turquoise/$ENV{USER}/E3SM/archive/$CASE - /lustre/scratch4/turquoise/$ENV{USER}/E3SM/input_data/ccsm_baselines/$COMPILER - /turquoise/usr/projects/climate/SHARED_CLIMATE/software/wolf/cprnc/v0.40/cprnc - 4 - e3sm_developer - slurm - luke.vanroekel @ gmail.com - 36 - 32 - TRUE - - srun - - -n {{ total_tasks }} - - - - - - - /usr/share/Modules/init/perl.pm - /usr/share/Modules/init/python.py - /etc/profile.d/z00_lmod.sh - /etc/profile.d/z00_lmod.csh - /usr/share/lmod/lmod/libexec/lmod perl - /usr/share/lmod/lmod/libexec/lmod python - module - module - - - cmake/3.16.2 - - - gcc/6.4.0 - openmpi/2.1.2 - - - gcc/6.4.0 - mvapich2/2.3 - - - intel/19.0.4 - intel-mpi/2019.4 - - - intel/18.0.2 - mvapich2/2.2 - - - intel/19.0.4 - openmpi/2.1.2 - - - friendly-testing - hdf5-parallel/1.8.16 - pnetcdf/1.11.2 - netcdf-h5parallel/4.7.3 - mkl/2019.0.4 - - - $CIME_OUTPUT_ROOT/$CASE/run - $CIME_OUTPUT_ROOT/$CASE/bld - - $ENV{MKLROOT} - romio_ds_write=disable;romio_ds_read=disable;romio_cb_write=enable;romio_cb_read=enable - - - - - LANL Linux Cluster, 36 pes/node, batch system slurm - ba-fe.*.lanl.gov - LINUX - intel,gnu - openmpi,impi,mvapich - climateacme - /lustre/scratch4/turquoise/$ENV{USER}/E3SM/scratch - /lustre/scratch4/turquoise/$ENV{USER}/E3SM/input_data - /lustre/scratch4/turquoise/$ENV{USER}/E3SM/input_data/atm/datm7 - /lustre/scratch4/turquoise/$ENV{USER}/E3SM/archive/$CASE - /lustre/scratch4/turquoise/$ENV{USER}/E3SM/input_data/ccsm_baselines/$COMPILER - /turquoise/usr/projects/climate/SHARED_CLIMATE/software/wolf/cprnc/v0.40/cprnc - 4 + + Chicoma CPU-only nodes at LANL IC. Each node has 2 AMD EPYC 7H12 64-Core (Milan) 512GB + ch-fe* + Linux + gnu,intel,nvidia,amdclang + mpich + /lustre/scratch5/$ENV{USER}/E3SM/scratch/chicoma-cpu + /usr/projects/e3sm/inputdata + /usr/projects/e3sm/inputdata/atm/datm7 + /lustre/scratch5/$ENV{USER}/E3SM/archive/$CASE + /lustre/scratch5/$ENV{USER}/E3SM/input_data/ccsm_baselines/$COMPILER + /usr/projects/climate/SHARED_CLIMATE/software/badger/cprnc + 10 e3sm_developer + 4 slurm e3sm - 36 - 32 + 256 + 128 TRUE srun - -n {{ total_tasks }} - - - - + --label + -n {{ total_tasks }} -N {{ num_nodes }} + -c $SHELL{echo 256/`./xmlquery --value MAX_MPITASKS_PER_NODE`|bc} + $SHELL{if [ 128 -ge `./xmlquery --value MAX_MPITASKS_PER_NODE` ]; then echo "--cpu_bind=cores"; else echo "--cpu_bind=threads";fi;} + -m plane=$SHELL{echo `./xmlquery --value MAX_MPITASKS_PER_NODE`} + - - /usr/share/Modules/init/perl.pm - /usr/share/Modules/init/python.py - /etc/profile.d/z00_lmod.sh - /etc/profile.d/z00_lmod.csh + + /usr/share/lmod/8.3.1/init/perl + + /usr/share/lmod/8.3.1/init/python + /usr/share/lmod/8.3.1/init/sh + /usr/share/lmod/8.3.1/init/csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module module + - - cmake/3.16.2 - - - gcc/6.4.0 - openmpi/2.1.2 + cray-hdf5-parallel + cray-netcdf-hdf5parallel + cray-parallel-netcdf + cray-netcdf + cray-hdf5 + PrgEnv-gnu + PrgEnv-intel + PrgEnv-nvidia + PrgEnv-cray + PrgEnv-aocc + intel + intel-oneapi + nvidia + aocc + cudatoolkit + climate-utils + craype-accel-nvidia80 + craype-accel-host + perftools-base + perftools + darshan - - gcc/6.4.0 - mvapich2/2.3 + + + PrgEnv-gnu/8.4.0 + gcc/12.2.0 + cray-libsci/23.05.1.4 - - intel/19.0.4 - intel-mpi/2019.4 + + + PrgEnv-nvidia/8.4.0 + nvidia/22.7 + cray-libsci/23.05.1.4 - - intel/18.0.2 - mvapich2/2.2 + + + PrgEnv-intel/8.4.0 + intel-classic/2023.2.0 - - intel/19.0.4 - openmpi/2.1.2 + + + PrgEnv-aocc/8.4.0 + aocc/3.2.0 + cray-libsci/23.05.1.4 + - friendly-testing - hdf5-parallel/1.8.16 - pnetcdf/1.11.2 - netcdf-h5parallel/4.7.3 - mkl/2019.0.4 + craype-accel-host + craype/2.7.21 + cray-mpich/8.1.26 + libfabric/1.15.2.0 + cray-hdf5-parallel/1.12.2.3 + cray-netcdf-hdf5parallel/4.9.0.3 + cray-parallel-netcdf/1.12.3.3 + cmake/3.25.1 + $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld + 0.1 + - $ENV{MKLROOT} + 1 + 1 + 128M + spread + threads + FALSE + /usr/projects/climate/SHARED_CLIMATE/software/chicoma-cpu/perl5-only-switch/lib/perl5 romio_ds_write=disable;romio_ds_read=disable;romio_cb_write=enable;romio_cb_read=enable + software + MPI_Bcast + $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} + $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + + -1 + - - Chicoma CPU-only nodes at LANL IC. Each node has 2 AMD EPYC 7H12 64-Core (Milan) 512GB + + Chicoma GPU nodes at LANL IC. Each GPU node has single +AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' ch-fe* Linux - gnu,nvidia,intel,aocc,amdclang + gnugpu,gnu,nvidiagpu,nvidia mpich - /lustre/scratch4/turquoise/$ENV{USER}/E3SM/scratch/chicoma-cpu + /lustre/scratch5/$ENV{USER}/E3SM/scratch/chicoma-gpu /usr/projects/e3sm/inputdata /usr/projects/e3sm/inputdata/atm/datm7 - /lustre/scratch4/turquoise/$ENV{USER}/E3SM/archive/$CASE - /lustre/scratch4/turquoise/$ENV{USER}/E3SM/input_data/ccsm_baselines/$COMPILER + /lustre/scratch5/$ENV{USER}/E3SM/archive/$CASE + /lustre/scratch5/$ENV{USER}/E3SM/input_data/ccsm_baselines/$COMPILER /usr/projects/climate/SHARED_CLIMATE/software/badger/cprnc 10 e3sm_developer 4 slurm e3sm - 256 - 64 + 128 + 256 + 256 + 4 + 64 + 64 TRUE srun @@ -4395,7 +4366,7 @@ --label -n {{ total_tasks }} -N {{ num_nodes }} -c $ENV{OMP_NUM_THREADS} - $SHELL{if [ 128 -ge `./xmlquery --value MAX_MPITASKS_PER_NODE` ]; then echo "--cpu_bind=cores"; else echo "--cpu_bind=threads";fi;} + $SHELL{if [ 128 -ge `./xmlquery --value MAX_MPITASKS_PER_NODE` ]; then echo "--cpu_bind=cores"; else echo "--cpu_bind=threads";fi;} -m plane=$SHELL{echo `./xmlquery --value MAX_MPITASKS_PER_NODE`} @@ -4414,50 +4385,64 @@ cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf + cray-netcdf + cray-hdf5 PrgEnv-gnu + PrgEnv-intel PrgEnv-nvidia PrgEnv-cray PrgEnv-aocc + intel + intel-oneapi + nvidia + aocc + cudatoolkit + climate-utils craype-accel-nvidia80 craype-accel-host - cce + perftools-base + perftools + darshan - + PrgEnv-gnu/8.4.0 - gcc/12.2.0 + gcc/11.2.0 - + PrgEnv-nvidia/8.4.0 nvidia/22.7 - - PrgEnv-intel/8.4.0 - intel-classic/2023.2.0 + + cudatoolkit/22.7_11.7 + craype-accel-nvidia80 - - PrgEnv-aocc/8.4.0 - aocc/3.2.0 + + cudatoolkit/22.7_11.7 + craype-accel-nvidia80 + gcc-mixed/11.2.0 - - PrgEnv-aocc/8.4.0 - aocc/3.2.0 + + craype-accel-host - + craype-accel-host - cray-libsci - craype + + + + cray-libsci/23.05.1.4 + craype/2.7.21 cray-mpich/8.1.26 libfabric/1.15.2.0 cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 - cmake/3.22.3 + cmake/3.25.1 @@ -4478,6 +4463,7 @@ MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + /usr/projects/e3sm/cudatoolkit:$ENV{PKG_CONFIG_PATH} -1 @@ -4828,12 +4814,12 @@ cli115 /gpfs/alpine/proj-shared/cli115 .* - /gpfs/alpine/$PROJECT/proj-shared/$ENV{USER}/e3sm_scratch - /gpfs/alpine/cli115/world-shared/e3sm/inputdata - /gpfs/alpine/cli115/world-shared/e3sm/inputdata/atm/datm7 + /gpfs/alpine2/$PROJECT/proj-shared/$ENV{USER}/e3sm_scratch + /gpfs/alpine2/atm146/world-shared/e3sm/inputdata + /gpfs/alpine2/atm146/world-shared/e3sm/inputdata/atm/datm7 /gpfs/alpine/$PROJECT/proj-shared/$ENV{USER}/archive/$CASE - /gpfs/alpine/cli115/world-shared/e3sm/baselines/$COMPILER - /gpfs/alpine/cli115/world-shared/e3sm/tools/cprnc.summit/cprnc + /gpfs/alpine2/atm146/world-shared/e3sm/baselines/$COMPILER + /gpfs/alpine2/atm146/world-shared/e3sm/tools/cprnc.summit/cprnc 8 e3sm_developer 4 @@ -4843,7 +4829,7 @@ 18 42 42 - + 84 18 @@ -4877,7 +4863,7 @@ module - DefApps + DefApps-2023 python/3.7-anaconda3 subversion/1.14.0 git/2.31.1 @@ -4925,6 +4911,7 @@ $ENV{OLCF_PARALLEL_NETCDF_ROOT} 0 + True @@ -4988,15 +4975,6 @@ mlx5_3:1,mlx5_0:1 mlx5_0:1,mlx5_3:1 - - $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /gpfs/alpine/cli115/world-shared/3rdparty/adios2/2.9.1/spectrum-mpi-10.4.0.3/xl-16.1.1-10; else echo "$ADIOS2_ROOT"; fi} - - - $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /gpfs/alpine/cli115/world-shared/3rdparty/adios2/2.9.1/spectrum-mpi-10.4.0.3/nvhpc-21.11; else echo "$ADIOS2_ROOT"; fi} - - - $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /gpfs/alpine/cli115/world-shared/3rdparty/adios2/2.9.1/spectrum-mpi-10.4.0.3/gcc-9.1.0; else echo "$ADIOS2_ROOT"; fi} - diff --git a/cime_config/machines/config_pio.xml b/cime_config/machines/config_pio.xml index e1784b1618d1..742e82bd7ef7 100644 --- a/cime_config/machines/config_pio.xml +++ b/cime_config/machines/config_pio.xml @@ -63,9 +63,8 @@ netcdf netcdf netcdf - netcdf - netcdf netcdf + netcdf netcdf netcdf diff --git a/cime_config/testmods_dirs/allactive/nlmaps/shell_commands b/cime_config/testmods_dirs/allactive/nlmaps/shell_commands index 1d90d0938a66..637b4f3099b6 100644 --- a/cime_config/testmods_dirs/allactive/nlmaps/shell_commands +++ b/cime_config/testmods_dirs/allactive/nlmaps/shell_commands @@ -1,19 +1,15 @@ alg=trfvnp2 -# This line is separate from the nonlinear maps. It corrects an oversight in the -# existing default trigrid grid configuration. -./xmlchange ATM2ROF_SMAPNAME=cpl/gridmaps/ne30pg2/map_ne30pg2_to_r05_bilin.200220.nc - # We want these in v3. a2l=cpl/gridmaps/ne30pg2/map_ne30pg2_to_r05_trfvnp2.230516.nc -a2o=cpl/gridmaps/ne30pg2/map_ne30pg2_to_EC30to60E2r2_trfvnp2.230516.nc +a2o=cpl/gridmaps/ne30pg2/map_ne30pg2_to_IcoswISC30E3r5_trfvnp2.20231121.nc ./xmlchange ATM2LND_FMAPNAME_NONLINEAR=$a2l ./xmlchange ATM2ROF_FMAPNAME_NONLINEAR=$a2l ./xmlchange ATM2OCN_FMAPNAME_NONLINEAR=$a2o -# These surface->atm maps are not needed, but we want to test the capability. +# These surface->atm maps are not needed in v3, but we want to test the capability. l2a=cpl/gridmaps/ne30pg2/map_r05_to_ne30pg2_${alg}.230516.nc -o2a=cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_ne30pg2_trfvnp2.230516.nc +o2a=cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_ne30pg2_trfv2.20240222.nc ./xmlchange LND2ATM_FMAPNAME_NONLINEAR=$l2a ./xmlchange LND2ATM_SMAPNAME_NONLINEAR=$l2a ./xmlchange OCN2ATM_FMAPNAME_NONLINEAR=$o2a diff --git a/cime_config/testmods_dirs/allactive/wcprod/README b/cime_config/testmods_dirs/allactive/wcprod/README index 020bbf93e9f2..329e3e6ea7ab 100644 --- a/cime_config/testmods_dirs/allactive/wcprod/README +++ b/cime_config/testmods_dirs/allactive/wcprod/README @@ -3,4 +3,3 @@ water cycle production sims Run these for at least 1 day to see all output. Also use the CMIP6 compsets. -If running longer, change the nhtfrq for the first history file. diff --git a/cime_config/testmods_dirs/allactive/wcprod/user_nl_eam b/cime_config/testmods_dirs/allactive/wcprod/user_nl_eam index eeac1d647b1d..095c6946f5de 100644 --- a/cime_config/testmods_dirs/allactive/wcprod/user_nl_eam +++ b/cime_config/testmods_dirs/allactive/wcprod/user_nl_eam @@ -1,11 +1,59 @@ - nhtfrq = -24,-24,-6,-6,-3,-24,-24 - mfilt = 1,30,120,120,240,30,1 - avgflag_pertape = 'A','A','I','A','A','A','I' - fexcl1 = 'CFAD_SR532_CAL', 'LINOZ_DO3', 'LINOZ_DO3_PSC', 'LINOZ_O3CLIM', 'LINOZ_O3COL', 'LINOZ_SSO3', 'hstobie_linoz' - fincl1 = 'extinct_sw_inp','extinct_lw_bnd7','extinct_lw_inp','CLD_CAL', 'TREFMNAV', 'TREFMXAV' - fincl2 = 'FLUT','PRECT','U200','V200','U850','V850','Z500','OMEGA500','UBOT','VBOT','TREFHT','TREFHTMN:M','TREFHTMX:X','QREFHT','TS','PS','TMQ','TUQ','TVQ','TOZ', 'FLDS', 'FLNS', 'FSDS', 'FSNS', 'SHFLX', 'LHFLX', 'TGCLDCWP', 'TGCLDIWP', 'TGCLDLWP', 'CLDTOT', 'T250', 'T200', 'T150', 'T100', 'T050', 'T025', 'T010', 'T005', 'T002', 'T001', 'TTOP', 'U250', 'U150', 'U100', 'U050', 'U025', 'U010', 'U005', 'U002', 'U001', 'UTOP', 'FSNT', 'FLNT' - fincl3 = 'PSL','T200','T500','U850','V850','UBOT','VBOT','TREFHT', 'Z700', 'TBOT:M' - fincl4 = 'FLUT','U200','U850','PRECT','OMEGA500' - fincl5 = 'PRECT','PRECC','TUQ','TVQ','QFLX','SHFLX','U90M','V90M' - fincl6 = 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANTAU_ISCCP','MEANPTOP_ISCCP','MEANTB_ISCCP','CLDTOT_CAL','CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN','CLDHGH_CAL','CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN','CLDMED_CAL','CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN','CLDLOW_CAL','CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN' - fincl7 = 'O3', 'PS', 'TROP_P' +cosp_lite = .true. + +empty_htapes = .true. + +avgflag_pertape = 'A','A','A','A','I','I' +nhtfrq = -24,-24,-6,-3,-1,-24 +mfilt = 1,30,120,240,720,1 + +fincl1 = 'AODALL','AODBC','AODDUST','AODPOM','AODSO4','AODSOA','AODSS','AODVIS', + 'CLDLOW','CLDMED','CLDHGH','CLDTOT', + 'CLDHGH_CAL','CLDLOW_CAL','CLDMED_CAL','CLD_MISR','CLDTOT_CAL', + 'CLMODIS','FISCCP1_COSP','FLDS','FLNS','FLNSC','FLNT','FLUT', + 'FLUTC','FSDS','FSDSC','FSNS','FSNSC','FSNT','FSNTOA','FSNTOAC','FSNTC', + 'ICEFRAC','LANDFRAC','LWCF','OCNFRAC','OMEGA','PRECC','PRECL','PRECSC','PRECSL','PS','PSL','Q', + 'QFLX','QREFHT','RELHUM','SCO','SHFLX','SOLIN','SWCF','T','TAUX','TAUY','TCO', + 'TGCLDLWP','TMQ','TREFHT','TREFMNAV','TREFMXAV','TS','U','U10','V','Z3', + 'dst_a1DDF','dst_a3DDF','dst_c1DDF','dst_c3DDF','dst_a1SFWET','dst_a3SFWET','dst_c1SFWET','dst_c3SFWET', + 'O3','LHFLX', + 'O3_2DTDA_trop','O3_2DTDB_trop','O3_2DTDD_trop','O3_2DTDE_trop','O3_2DTDI_trop','O3_2DTDL_trop', + 'O3_2DTDN_trop','O3_2DTDO_trop','O3_2DTDS_trop','O3_2DTDU_trop','O3_2DTRE_trop','O3_2DTRI_trop', + 'O3_SRF','NO_2DTDS','NO_TDLgt','NO2_2DTDD','NO2_2DTDS','NO2_TDAcf','CO_SRF','TROPE3D_P','TROP_P', + 'CDNUMC','SFDMS','so4_a1_sfgaex1','so4_a2_sfgaex1','so4_a3_sfgaex1','so4_a5_sfgaex1','soa_a1_sfgaex1', + 'soa_a2_sfgaex1','soa_a3_sfgaex1','GS_soa_a1','GS_soa_a2','GS_soa_a3','AQSO4_H2O2','AQSO4_O3', + 'SFSO2','SO2_CLXF','SO2','DF_SO2','AQ_SO2','GS_SO2','WD_SO2','ABURDENSO4_STR','ABURDENSO4_TRO', + 'ABURDENSO4','ABURDENBC','ABURDENDUST','ABURDENMOM','ABURDENPOM','ABURDENSEASALT', + 'ABURDENSOA','AODSO4_STR','AODSO4_TRO', + 'EXTINCT','AODABS','AODABSBC','CLDICE','CLDLIQ','CLD_CAL_TMPLIQ','CLD_CAL_TMPICE','Mass_bc_srf', + 'Mass_dst_srf','Mass_mom_srf','Mass_ncl_srf','Mass_pom_srf','Mass_so4_srf','Mass_soa_srf','Mass_bc_850', + 'Mass_dst_850','Mass_mom_850','Mass_ncl_850','Mass_pom_850','Mass_so4_850','Mass_soa_850','Mass_bc_500', + 'Mass_dst_500','Mass_mom_500','Mass_ncl_500','Mass_pom_500','Mass_so4_500','Mass_soa_500','Mass_bc_330', + 'Mass_dst_330','Mass_mom_330','Mass_ncl_330','Mass_pom_330','Mass_so4_330','Mass_soa_330','Mass_bc_200', + 'Mass_dst_200','Mass_mom_200','Mass_ncl_200','Mass_pom_200','Mass_so4_200','Mass_soa_200', + 'O3_2DTDD','O3_2DCIP','O3_2DCIL','CO_2DTDS','CO_2DTDD','CO_2DCEP','CO_2DCEL','NO_2DTDD', + 'FLNTC','SAODVIS', + 'H2OLNZ', + 'dst_a1SF','dst_a3SF', + 'PHIS','CLOUD','TGCLDIWP','TGCLDCWP','AREL', + 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANPTOP_ISCCP','CLD_CAL', + 'CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN', + 'CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN', + 'CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN', + 'CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN', + 'CLWMODIS','CLIMODIS' + +fincl2 = 'PS', 'FLUT','PRECT','U200','V200','U850','V850', + 'TCO','SCO','TREFHTMN','TREFHTMX','TREFHT','QREFHT' +fincl3 = 'PS', 'PSL','PRECT','TUQ','TVQ','UBOT','VBOT','TREFHT','FLUT','OMEGA500','TBOT','U850','V850','U200','V200','T200','T500','Z700' +fincl4 = 'PRECT' +fincl5 = 'O3_SRF' +fincl6 = 'CO_2DMSD','NO2_2DMSD','NO_2DMSD','O3_2DMSD','O3_2DMSD_trop' + +! -- chemUCI settings ------------------ +history_chemdyg_summary = .true. +history_gaschmbudget_2D = .false. +history_gaschmbudget_2D_levels = .false. +history_gaschmbudget_num = 6 !! no impact if history_gaschmbudget_2D = .false. + +! -- MAM5 settings ------------------ +is_output_interactive_volc = .true. diff --git a/cime_config/testmods_dirs/allactive/wcprod/user_nl_elm b/cime_config/testmods_dirs/allactive/wcprod/user_nl_elm index a93edc10b690..cd1adab77404 100644 --- a/cime_config/testmods_dirs/allactive/wcprod/user_nl_elm +++ b/cime_config/testmods_dirs/allactive/wcprod/user_nl_elm @@ -1,6 +1,40 @@ - finidat = ' ' - hist_dov2xy = .true.,.true. - hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' - hist_mfilt = 1,365 - hist_nhtfrq = -24,-24 - hist_avgflag_pertape = 'A','A' +hist_dov2xy = .true.,.true. +hist_fexcl1 = 'AGWDNPP','ALTMAX_LASTYEAR','AVAIL_RETRANSP','AVAILC','BAF_CROP', + 'BAF_PEATF','BIOCHEM_PMIN_TO_PLANT','CH4_SURF_AERE_SAT','CH4_SURF_AERE_UNSAT','CH4_SURF_DIFF_SAT', + 'CH4_SURF_DIFF_UNSAT','CH4_SURF_EBUL_SAT','CH4_SURF_EBUL_UNSAT','CMASS_BALANCE_ERROR','cn_scalar', + 'COL_PTRUNC','CONC_CH4_SAT','CONC_CH4_UNSAT','CONC_O2_SAT','CONC_O2_UNSAT', + 'cp_scalar','CWDC_HR','CWDC_LOSS','CWDC_TO_LITR2C','CWDC_TO_LITR3C', + 'CWDC_vr','CWDN_TO_LITR2N','CWDN_TO_LITR3N','CWDN_vr','CWDP_TO_LITR2P', + 'CWDP_TO_LITR3P','CWDP_vr','DWT_CONV_CFLUX_DRIBBLED','F_CO2_SOIL','F_CO2_SOIL_vr', + 'F_DENIT_vr','F_N2O_DENIT','F_N2O_NIT','F_NIT_vr','FCH4_DFSAT', + 'FINUNDATED_LAG','FPI_P_vr','FPI_vr','FROOTC_LOSS','HR_vr', + 'LABILEP_TO_SECONDP','LABILEP_vr','LAND_UPTAKE','LEAF_MR','leaf_npimbalance', + 'LEAFC_LOSS','LEAFC_TO_LITTER','LFC2','LITR1_HR','LITR1C_TO_SOIL1C', + 'LITR1C_vr','LITR1N_TNDNCY_VERT_TRANS','LITR1N_TO_SOIL1N','LITR1N_vr','LITR1P_TNDNCY_VERT_TRANS', + 'LITR1P_TO_SOIL1P','LITR1P_vr','LITR2_HR','LITR2C_TO_SOIL2C','LITR2C_vr', + 'LITR2N_TNDNCY_VERT_TRANS','LITR2N_TO_SOIL2N','LITR2N_vr','LITR2P_TNDNCY_VERT_TRANS','LITR2P_TO_SOIL2P', + 'LITR2P_vr','LITR3_HR','LITR3C_TO_SOIL3C','LITR3C_vr','LITR3N_TNDNCY_VERT_TRANS', + 'LITR3N_TO_SOIL3N','LITR3N_vr','LITR3P_TNDNCY_VERT_TRANS','LITR3P_TO_SOIL3P','LITR3P_vr', + 'M_LITR1C_TO_LEACHING','M_LITR2C_TO_LEACHING','M_LITR3C_TO_LEACHING','M_SOIL1C_TO_LEACHING','M_SOIL2C_TO_LEACHING', + 'M_SOIL3C_TO_LEACHING','M_SOIL4C_TO_LEACHING','NDEPLOY','NEM','nlim_m', + 'o2_decomp_depth_unsat','OCCLP_vr','PDEPLOY','PLANT_CALLOC','PLANT_NDEMAND', + 'PLANT_NDEMAND_COL','PLANT_PALLOC','PLANT_PDEMAND','PLANT_PDEMAND_COL','plim_m', + 'POT_F_DENIT','POT_F_NIT','POTENTIAL_IMMOB','POTENTIAL_IMMOB_P','PRIMP_TO_LABILEP', + 'PRIMP_vr','PROD1P_LOSS','QOVER_LAG','RETRANSN_TO_NPOOL','RETRANSP_TO_PPOOL', + 'SCALARAVG_vr','SECONDP_TO_LABILEP','SECONDP_TO_OCCLP','SECONDP_vr','SMIN_NH4_vr', + 'SMIN_NO3_vr','SMINN_TO_SOIL1N_L1','SMINN_TO_SOIL2N_L2','SMINN_TO_SOIL2N_S1','SMINN_TO_SOIL3N_L3', + 'SMINN_TO_SOIL3N_S2','SMINN_TO_SOIL4N_S3','SMINP_TO_SOIL1P_L1','SMINP_TO_SOIL2P_L2','SMINP_TO_SOIL2P_S1', + 'SMINP_TO_SOIL3P_L3','SMINP_TO_SOIL3P_S2','SMINP_TO_SOIL4P_S3','SMINP_vr','SOIL1_HR','SOIL1C_TO_SOIL2C','SOIL1C_vr','SOIL1N_TNDNCY_VERT_TRANS','SOIL1N_TO_SOIL2N','SOIL1N_vr', + 'SOIL1P_TNDNCY_VERT_TRANS','SOIL1P_TO_SOIL2P','SOIL1P_vr','SOIL2_HR','SOIL2C_TO_SOIL3C', + 'SOIL2C_vr','SOIL2N_TNDNCY_VERT_TRANS','SOIL2N_TO_SOIL3N','SOIL2N_vr','SOIL2P_TNDNCY_VERT_TRANS', + 'SOIL2P_TO_SOIL3P','SOIL2P_vr','SOIL3_HR','SOIL3C_TO_SOIL4C','SOIL3C_vr', + 'SOIL3N_TNDNCY_VERT_TRANS','SOIL3N_TO_SOIL4N','SOIL3N_vr','SOIL3P_TNDNCY_VERT_TRANS','SOIL3P_TO_SOIL4P', + 'SOIL3P_vr','SOIL4_HR','SOIL4C_vr','SOIL4N_TNDNCY_VERT_TRANS','SOIL4N_TO_SMINN', + 'SOIL4N_vr','SOIL4P_TNDNCY_VERT_TRANS','SOIL4P_TO_SMINP','SOIL4P_vr','SOLUTIONP_vr', + 'TCS_MONTH_BEGIN','TCS_MONTH_END','TOTCOLCH4','water_scalar','WF', + 'wlim_m','WOODC_LOSS','WTGQ' +hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC' +hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' +hist_mfilt = 1,365 +hist_nhtfrq = -24,-24 +hist_avgflag_pertape = 'A','A' diff --git a/cime_config/testmods_dirs/allactive/wcprod_1850/README b/cime_config/testmods_dirs/allactive/wcprod_1850/README index 020bbf93e9f2..329e3e6ea7ab 100644 --- a/cime_config/testmods_dirs/allactive/wcprod_1850/README +++ b/cime_config/testmods_dirs/allactive/wcprod_1850/README @@ -3,4 +3,3 @@ water cycle production sims Run these for at least 1 day to see all output. Also use the CMIP6 compsets. -If running longer, change the nhtfrq for the first history file. diff --git a/cime_config/testmods_dirs/allactive/wcprod_1850/user_nl_eam b/cime_config/testmods_dirs/allactive/wcprod_1850/user_nl_eam index eeac1d647b1d..095c6946f5de 100644 --- a/cime_config/testmods_dirs/allactive/wcprod_1850/user_nl_eam +++ b/cime_config/testmods_dirs/allactive/wcprod_1850/user_nl_eam @@ -1,11 +1,59 @@ - nhtfrq = -24,-24,-6,-6,-3,-24,-24 - mfilt = 1,30,120,120,240,30,1 - avgflag_pertape = 'A','A','I','A','A','A','I' - fexcl1 = 'CFAD_SR532_CAL', 'LINOZ_DO3', 'LINOZ_DO3_PSC', 'LINOZ_O3CLIM', 'LINOZ_O3COL', 'LINOZ_SSO3', 'hstobie_linoz' - fincl1 = 'extinct_sw_inp','extinct_lw_bnd7','extinct_lw_inp','CLD_CAL', 'TREFMNAV', 'TREFMXAV' - fincl2 = 'FLUT','PRECT','U200','V200','U850','V850','Z500','OMEGA500','UBOT','VBOT','TREFHT','TREFHTMN:M','TREFHTMX:X','QREFHT','TS','PS','TMQ','TUQ','TVQ','TOZ', 'FLDS', 'FLNS', 'FSDS', 'FSNS', 'SHFLX', 'LHFLX', 'TGCLDCWP', 'TGCLDIWP', 'TGCLDLWP', 'CLDTOT', 'T250', 'T200', 'T150', 'T100', 'T050', 'T025', 'T010', 'T005', 'T002', 'T001', 'TTOP', 'U250', 'U150', 'U100', 'U050', 'U025', 'U010', 'U005', 'U002', 'U001', 'UTOP', 'FSNT', 'FLNT' - fincl3 = 'PSL','T200','T500','U850','V850','UBOT','VBOT','TREFHT', 'Z700', 'TBOT:M' - fincl4 = 'FLUT','U200','U850','PRECT','OMEGA500' - fincl5 = 'PRECT','PRECC','TUQ','TVQ','QFLX','SHFLX','U90M','V90M' - fincl6 = 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANTAU_ISCCP','MEANPTOP_ISCCP','MEANTB_ISCCP','CLDTOT_CAL','CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN','CLDHGH_CAL','CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN','CLDMED_CAL','CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN','CLDLOW_CAL','CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN' - fincl7 = 'O3', 'PS', 'TROP_P' +cosp_lite = .true. + +empty_htapes = .true. + +avgflag_pertape = 'A','A','A','A','I','I' +nhtfrq = -24,-24,-6,-3,-1,-24 +mfilt = 1,30,120,240,720,1 + +fincl1 = 'AODALL','AODBC','AODDUST','AODPOM','AODSO4','AODSOA','AODSS','AODVIS', + 'CLDLOW','CLDMED','CLDHGH','CLDTOT', + 'CLDHGH_CAL','CLDLOW_CAL','CLDMED_CAL','CLD_MISR','CLDTOT_CAL', + 'CLMODIS','FISCCP1_COSP','FLDS','FLNS','FLNSC','FLNT','FLUT', + 'FLUTC','FSDS','FSDSC','FSNS','FSNSC','FSNT','FSNTOA','FSNTOAC','FSNTC', + 'ICEFRAC','LANDFRAC','LWCF','OCNFRAC','OMEGA','PRECC','PRECL','PRECSC','PRECSL','PS','PSL','Q', + 'QFLX','QREFHT','RELHUM','SCO','SHFLX','SOLIN','SWCF','T','TAUX','TAUY','TCO', + 'TGCLDLWP','TMQ','TREFHT','TREFMNAV','TREFMXAV','TS','U','U10','V','Z3', + 'dst_a1DDF','dst_a3DDF','dst_c1DDF','dst_c3DDF','dst_a1SFWET','dst_a3SFWET','dst_c1SFWET','dst_c3SFWET', + 'O3','LHFLX', + 'O3_2DTDA_trop','O3_2DTDB_trop','O3_2DTDD_trop','O3_2DTDE_trop','O3_2DTDI_trop','O3_2DTDL_trop', + 'O3_2DTDN_trop','O3_2DTDO_trop','O3_2DTDS_trop','O3_2DTDU_trop','O3_2DTRE_trop','O3_2DTRI_trop', + 'O3_SRF','NO_2DTDS','NO_TDLgt','NO2_2DTDD','NO2_2DTDS','NO2_TDAcf','CO_SRF','TROPE3D_P','TROP_P', + 'CDNUMC','SFDMS','so4_a1_sfgaex1','so4_a2_sfgaex1','so4_a3_sfgaex1','so4_a5_sfgaex1','soa_a1_sfgaex1', + 'soa_a2_sfgaex1','soa_a3_sfgaex1','GS_soa_a1','GS_soa_a2','GS_soa_a3','AQSO4_H2O2','AQSO4_O3', + 'SFSO2','SO2_CLXF','SO2','DF_SO2','AQ_SO2','GS_SO2','WD_SO2','ABURDENSO4_STR','ABURDENSO4_TRO', + 'ABURDENSO4','ABURDENBC','ABURDENDUST','ABURDENMOM','ABURDENPOM','ABURDENSEASALT', + 'ABURDENSOA','AODSO4_STR','AODSO4_TRO', + 'EXTINCT','AODABS','AODABSBC','CLDICE','CLDLIQ','CLD_CAL_TMPLIQ','CLD_CAL_TMPICE','Mass_bc_srf', + 'Mass_dst_srf','Mass_mom_srf','Mass_ncl_srf','Mass_pom_srf','Mass_so4_srf','Mass_soa_srf','Mass_bc_850', + 'Mass_dst_850','Mass_mom_850','Mass_ncl_850','Mass_pom_850','Mass_so4_850','Mass_soa_850','Mass_bc_500', + 'Mass_dst_500','Mass_mom_500','Mass_ncl_500','Mass_pom_500','Mass_so4_500','Mass_soa_500','Mass_bc_330', + 'Mass_dst_330','Mass_mom_330','Mass_ncl_330','Mass_pom_330','Mass_so4_330','Mass_soa_330','Mass_bc_200', + 'Mass_dst_200','Mass_mom_200','Mass_ncl_200','Mass_pom_200','Mass_so4_200','Mass_soa_200', + 'O3_2DTDD','O3_2DCIP','O3_2DCIL','CO_2DTDS','CO_2DTDD','CO_2DCEP','CO_2DCEL','NO_2DTDD', + 'FLNTC','SAODVIS', + 'H2OLNZ', + 'dst_a1SF','dst_a3SF', + 'PHIS','CLOUD','TGCLDIWP','TGCLDCWP','AREL', + 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANPTOP_ISCCP','CLD_CAL', + 'CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN', + 'CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN', + 'CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN', + 'CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN', + 'CLWMODIS','CLIMODIS' + +fincl2 = 'PS', 'FLUT','PRECT','U200','V200','U850','V850', + 'TCO','SCO','TREFHTMN','TREFHTMX','TREFHT','QREFHT' +fincl3 = 'PS', 'PSL','PRECT','TUQ','TVQ','UBOT','VBOT','TREFHT','FLUT','OMEGA500','TBOT','U850','V850','U200','V200','T200','T500','Z700' +fincl4 = 'PRECT' +fincl5 = 'O3_SRF' +fincl6 = 'CO_2DMSD','NO2_2DMSD','NO_2DMSD','O3_2DMSD','O3_2DMSD_trop' + +! -- chemUCI settings ------------------ +history_chemdyg_summary = .true. +history_gaschmbudget_2D = .false. +history_gaschmbudget_2D_levels = .false. +history_gaschmbudget_num = 6 !! no impact if history_gaschmbudget_2D = .false. + +! -- MAM5 settings ------------------ +is_output_interactive_volc = .true. diff --git a/cime_config/testmods_dirs/allactive/wcprod_1850/user_nl_elm b/cime_config/testmods_dirs/allactive/wcprod_1850/user_nl_elm index 0bbfbeea61bc..cd1adab77404 100644 --- a/cime_config/testmods_dirs/allactive/wcprod_1850/user_nl_elm +++ b/cime_config/testmods_dirs/allactive/wcprod_1850/user_nl_elm @@ -1,7 +1,40 @@ -! Finidat to be updated, The one below not compatible with v3 lnd config (with TOP and BGC mode, new grid) -! finidat = '$DIN_LOC_ROOT/lnd/clm2/initdata_map/clmi.WCYCL1850.ne30pg2_EC30to60E2r2.SMS_Ld1.c20230213.nc' - hist_dov2xy = .true.,.true. - hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' - hist_mfilt = 1,365 - hist_nhtfrq = -24,-24 - hist_avgflag_pertape = 'A','A' +hist_dov2xy = .true.,.true. +hist_fexcl1 = 'AGWDNPP','ALTMAX_LASTYEAR','AVAIL_RETRANSP','AVAILC','BAF_CROP', + 'BAF_PEATF','BIOCHEM_PMIN_TO_PLANT','CH4_SURF_AERE_SAT','CH4_SURF_AERE_UNSAT','CH4_SURF_DIFF_SAT', + 'CH4_SURF_DIFF_UNSAT','CH4_SURF_EBUL_SAT','CH4_SURF_EBUL_UNSAT','CMASS_BALANCE_ERROR','cn_scalar', + 'COL_PTRUNC','CONC_CH4_SAT','CONC_CH4_UNSAT','CONC_O2_SAT','CONC_O2_UNSAT', + 'cp_scalar','CWDC_HR','CWDC_LOSS','CWDC_TO_LITR2C','CWDC_TO_LITR3C', + 'CWDC_vr','CWDN_TO_LITR2N','CWDN_TO_LITR3N','CWDN_vr','CWDP_TO_LITR2P', + 'CWDP_TO_LITR3P','CWDP_vr','DWT_CONV_CFLUX_DRIBBLED','F_CO2_SOIL','F_CO2_SOIL_vr', + 'F_DENIT_vr','F_N2O_DENIT','F_N2O_NIT','F_NIT_vr','FCH4_DFSAT', + 'FINUNDATED_LAG','FPI_P_vr','FPI_vr','FROOTC_LOSS','HR_vr', + 'LABILEP_TO_SECONDP','LABILEP_vr','LAND_UPTAKE','LEAF_MR','leaf_npimbalance', + 'LEAFC_LOSS','LEAFC_TO_LITTER','LFC2','LITR1_HR','LITR1C_TO_SOIL1C', + 'LITR1C_vr','LITR1N_TNDNCY_VERT_TRANS','LITR1N_TO_SOIL1N','LITR1N_vr','LITR1P_TNDNCY_VERT_TRANS', + 'LITR1P_TO_SOIL1P','LITR1P_vr','LITR2_HR','LITR2C_TO_SOIL2C','LITR2C_vr', + 'LITR2N_TNDNCY_VERT_TRANS','LITR2N_TO_SOIL2N','LITR2N_vr','LITR2P_TNDNCY_VERT_TRANS','LITR2P_TO_SOIL2P', + 'LITR2P_vr','LITR3_HR','LITR3C_TO_SOIL3C','LITR3C_vr','LITR3N_TNDNCY_VERT_TRANS', + 'LITR3N_TO_SOIL3N','LITR3N_vr','LITR3P_TNDNCY_VERT_TRANS','LITR3P_TO_SOIL3P','LITR3P_vr', + 'M_LITR1C_TO_LEACHING','M_LITR2C_TO_LEACHING','M_LITR3C_TO_LEACHING','M_SOIL1C_TO_LEACHING','M_SOIL2C_TO_LEACHING', + 'M_SOIL3C_TO_LEACHING','M_SOIL4C_TO_LEACHING','NDEPLOY','NEM','nlim_m', + 'o2_decomp_depth_unsat','OCCLP_vr','PDEPLOY','PLANT_CALLOC','PLANT_NDEMAND', + 'PLANT_NDEMAND_COL','PLANT_PALLOC','PLANT_PDEMAND','PLANT_PDEMAND_COL','plim_m', + 'POT_F_DENIT','POT_F_NIT','POTENTIAL_IMMOB','POTENTIAL_IMMOB_P','PRIMP_TO_LABILEP', + 'PRIMP_vr','PROD1P_LOSS','QOVER_LAG','RETRANSN_TO_NPOOL','RETRANSP_TO_PPOOL', + 'SCALARAVG_vr','SECONDP_TO_LABILEP','SECONDP_TO_OCCLP','SECONDP_vr','SMIN_NH4_vr', + 'SMIN_NO3_vr','SMINN_TO_SOIL1N_L1','SMINN_TO_SOIL2N_L2','SMINN_TO_SOIL2N_S1','SMINN_TO_SOIL3N_L3', + 'SMINN_TO_SOIL3N_S2','SMINN_TO_SOIL4N_S3','SMINP_TO_SOIL1P_L1','SMINP_TO_SOIL2P_L2','SMINP_TO_SOIL2P_S1', + 'SMINP_TO_SOIL3P_L3','SMINP_TO_SOIL3P_S2','SMINP_TO_SOIL4P_S3','SMINP_vr','SOIL1_HR','SOIL1C_TO_SOIL2C','SOIL1C_vr','SOIL1N_TNDNCY_VERT_TRANS','SOIL1N_TO_SOIL2N','SOIL1N_vr', + 'SOIL1P_TNDNCY_VERT_TRANS','SOIL1P_TO_SOIL2P','SOIL1P_vr','SOIL2_HR','SOIL2C_TO_SOIL3C', + 'SOIL2C_vr','SOIL2N_TNDNCY_VERT_TRANS','SOIL2N_TO_SOIL3N','SOIL2N_vr','SOIL2P_TNDNCY_VERT_TRANS', + 'SOIL2P_TO_SOIL3P','SOIL2P_vr','SOIL3_HR','SOIL3C_TO_SOIL4C','SOIL3C_vr', + 'SOIL3N_TNDNCY_VERT_TRANS','SOIL3N_TO_SOIL4N','SOIL3N_vr','SOIL3P_TNDNCY_VERT_TRANS','SOIL3P_TO_SOIL4P', + 'SOIL3P_vr','SOIL4_HR','SOIL4C_vr','SOIL4N_TNDNCY_VERT_TRANS','SOIL4N_TO_SMINN', + 'SOIL4N_vr','SOIL4P_TNDNCY_VERT_TRANS','SOIL4P_TO_SMINP','SOIL4P_vr','SOLUTIONP_vr', + 'TCS_MONTH_BEGIN','TCS_MONTH_END','TOTCOLCH4','water_scalar','WF', + 'wlim_m','WOODC_LOSS','WTGQ' +hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC' +hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' +hist_mfilt = 1,365 +hist_nhtfrq = -24,-24 +hist_avgflag_pertape = 'A','A' diff --git a/cime_config/testmods_dirs/allactive/wcprod_1850_1pctCO2/README b/cime_config/testmods_dirs/allactive/wcprod_1850_1pctCO2/README index 020bbf93e9f2..329e3e6ea7ab 100644 --- a/cime_config/testmods_dirs/allactive/wcprod_1850_1pctCO2/README +++ b/cime_config/testmods_dirs/allactive/wcprod_1850_1pctCO2/README @@ -3,4 +3,3 @@ water cycle production sims Run these for at least 1 day to see all output. Also use the CMIP6 compsets. -If running longer, change the nhtfrq for the first history file. diff --git a/cime_config/testmods_dirs/allactive/wcprod_1850_1pctCO2/user_nl_eam b/cime_config/testmods_dirs/allactive/wcprod_1850_1pctCO2/user_nl_eam index eeac1d647b1d..095c6946f5de 100644 --- a/cime_config/testmods_dirs/allactive/wcprod_1850_1pctCO2/user_nl_eam +++ b/cime_config/testmods_dirs/allactive/wcprod_1850_1pctCO2/user_nl_eam @@ -1,11 +1,59 @@ - nhtfrq = -24,-24,-6,-6,-3,-24,-24 - mfilt = 1,30,120,120,240,30,1 - avgflag_pertape = 'A','A','I','A','A','A','I' - fexcl1 = 'CFAD_SR532_CAL', 'LINOZ_DO3', 'LINOZ_DO3_PSC', 'LINOZ_O3CLIM', 'LINOZ_O3COL', 'LINOZ_SSO3', 'hstobie_linoz' - fincl1 = 'extinct_sw_inp','extinct_lw_bnd7','extinct_lw_inp','CLD_CAL', 'TREFMNAV', 'TREFMXAV' - fincl2 = 'FLUT','PRECT','U200','V200','U850','V850','Z500','OMEGA500','UBOT','VBOT','TREFHT','TREFHTMN:M','TREFHTMX:X','QREFHT','TS','PS','TMQ','TUQ','TVQ','TOZ', 'FLDS', 'FLNS', 'FSDS', 'FSNS', 'SHFLX', 'LHFLX', 'TGCLDCWP', 'TGCLDIWP', 'TGCLDLWP', 'CLDTOT', 'T250', 'T200', 'T150', 'T100', 'T050', 'T025', 'T010', 'T005', 'T002', 'T001', 'TTOP', 'U250', 'U150', 'U100', 'U050', 'U025', 'U010', 'U005', 'U002', 'U001', 'UTOP', 'FSNT', 'FLNT' - fincl3 = 'PSL','T200','T500','U850','V850','UBOT','VBOT','TREFHT', 'Z700', 'TBOT:M' - fincl4 = 'FLUT','U200','U850','PRECT','OMEGA500' - fincl5 = 'PRECT','PRECC','TUQ','TVQ','QFLX','SHFLX','U90M','V90M' - fincl6 = 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANTAU_ISCCP','MEANPTOP_ISCCP','MEANTB_ISCCP','CLDTOT_CAL','CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN','CLDHGH_CAL','CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN','CLDMED_CAL','CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN','CLDLOW_CAL','CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN' - fincl7 = 'O3', 'PS', 'TROP_P' +cosp_lite = .true. + +empty_htapes = .true. + +avgflag_pertape = 'A','A','A','A','I','I' +nhtfrq = -24,-24,-6,-3,-1,-24 +mfilt = 1,30,120,240,720,1 + +fincl1 = 'AODALL','AODBC','AODDUST','AODPOM','AODSO4','AODSOA','AODSS','AODVIS', + 'CLDLOW','CLDMED','CLDHGH','CLDTOT', + 'CLDHGH_CAL','CLDLOW_CAL','CLDMED_CAL','CLD_MISR','CLDTOT_CAL', + 'CLMODIS','FISCCP1_COSP','FLDS','FLNS','FLNSC','FLNT','FLUT', + 'FLUTC','FSDS','FSDSC','FSNS','FSNSC','FSNT','FSNTOA','FSNTOAC','FSNTC', + 'ICEFRAC','LANDFRAC','LWCF','OCNFRAC','OMEGA','PRECC','PRECL','PRECSC','PRECSL','PS','PSL','Q', + 'QFLX','QREFHT','RELHUM','SCO','SHFLX','SOLIN','SWCF','T','TAUX','TAUY','TCO', + 'TGCLDLWP','TMQ','TREFHT','TREFMNAV','TREFMXAV','TS','U','U10','V','Z3', + 'dst_a1DDF','dst_a3DDF','dst_c1DDF','dst_c3DDF','dst_a1SFWET','dst_a3SFWET','dst_c1SFWET','dst_c3SFWET', + 'O3','LHFLX', + 'O3_2DTDA_trop','O3_2DTDB_trop','O3_2DTDD_trop','O3_2DTDE_trop','O3_2DTDI_trop','O3_2DTDL_trop', + 'O3_2DTDN_trop','O3_2DTDO_trop','O3_2DTDS_trop','O3_2DTDU_trop','O3_2DTRE_trop','O3_2DTRI_trop', + 'O3_SRF','NO_2DTDS','NO_TDLgt','NO2_2DTDD','NO2_2DTDS','NO2_TDAcf','CO_SRF','TROPE3D_P','TROP_P', + 'CDNUMC','SFDMS','so4_a1_sfgaex1','so4_a2_sfgaex1','so4_a3_sfgaex1','so4_a5_sfgaex1','soa_a1_sfgaex1', + 'soa_a2_sfgaex1','soa_a3_sfgaex1','GS_soa_a1','GS_soa_a2','GS_soa_a3','AQSO4_H2O2','AQSO4_O3', + 'SFSO2','SO2_CLXF','SO2','DF_SO2','AQ_SO2','GS_SO2','WD_SO2','ABURDENSO4_STR','ABURDENSO4_TRO', + 'ABURDENSO4','ABURDENBC','ABURDENDUST','ABURDENMOM','ABURDENPOM','ABURDENSEASALT', + 'ABURDENSOA','AODSO4_STR','AODSO4_TRO', + 'EXTINCT','AODABS','AODABSBC','CLDICE','CLDLIQ','CLD_CAL_TMPLIQ','CLD_CAL_TMPICE','Mass_bc_srf', + 'Mass_dst_srf','Mass_mom_srf','Mass_ncl_srf','Mass_pom_srf','Mass_so4_srf','Mass_soa_srf','Mass_bc_850', + 'Mass_dst_850','Mass_mom_850','Mass_ncl_850','Mass_pom_850','Mass_so4_850','Mass_soa_850','Mass_bc_500', + 'Mass_dst_500','Mass_mom_500','Mass_ncl_500','Mass_pom_500','Mass_so4_500','Mass_soa_500','Mass_bc_330', + 'Mass_dst_330','Mass_mom_330','Mass_ncl_330','Mass_pom_330','Mass_so4_330','Mass_soa_330','Mass_bc_200', + 'Mass_dst_200','Mass_mom_200','Mass_ncl_200','Mass_pom_200','Mass_so4_200','Mass_soa_200', + 'O3_2DTDD','O3_2DCIP','O3_2DCIL','CO_2DTDS','CO_2DTDD','CO_2DCEP','CO_2DCEL','NO_2DTDD', + 'FLNTC','SAODVIS', + 'H2OLNZ', + 'dst_a1SF','dst_a3SF', + 'PHIS','CLOUD','TGCLDIWP','TGCLDCWP','AREL', + 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANPTOP_ISCCP','CLD_CAL', + 'CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN', + 'CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN', + 'CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN', + 'CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN', + 'CLWMODIS','CLIMODIS' + +fincl2 = 'PS', 'FLUT','PRECT','U200','V200','U850','V850', + 'TCO','SCO','TREFHTMN','TREFHTMX','TREFHT','QREFHT' +fincl3 = 'PS', 'PSL','PRECT','TUQ','TVQ','UBOT','VBOT','TREFHT','FLUT','OMEGA500','TBOT','U850','V850','U200','V200','T200','T500','Z700' +fincl4 = 'PRECT' +fincl5 = 'O3_SRF' +fincl6 = 'CO_2DMSD','NO2_2DMSD','NO_2DMSD','O3_2DMSD','O3_2DMSD_trop' + +! -- chemUCI settings ------------------ +history_chemdyg_summary = .true. +history_gaschmbudget_2D = .false. +history_gaschmbudget_2D_levels = .false. +history_gaschmbudget_num = 6 !! no impact if history_gaschmbudget_2D = .false. + +! -- MAM5 settings ------------------ +is_output_interactive_volc = .true. diff --git a/cime_config/testmods_dirs/allactive/wcprod_1850_1pctCO2/user_nl_elm b/cime_config/testmods_dirs/allactive/wcprod_1850_1pctCO2/user_nl_elm index c8d35999c0a5..cd1adab77404 100644 --- a/cime_config/testmods_dirs/allactive/wcprod_1850_1pctCO2/user_nl_elm +++ b/cime_config/testmods_dirs/allactive/wcprod_1850_1pctCO2/user_nl_elm @@ -1,7 +1,40 @@ -! Finidat to be updated, The one below not compatible with v3 lnd config (with TOP and BGC mode, new grid) -! finidat = '$DIN_LOC_ROOT/lnd/clm2/initdata_map/clmi.WCYCL1850-1pctCO2.ne30pg2_EC30to60E2r2.SMS_Ld1.c20230213.nc' - hist_dov2xy = .true.,.true. - hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' - hist_mfilt = 1,365 - hist_nhtfrq = -24,-24 - hist_avgflag_pertape = 'A','A' +hist_dov2xy = .true.,.true. +hist_fexcl1 = 'AGWDNPP','ALTMAX_LASTYEAR','AVAIL_RETRANSP','AVAILC','BAF_CROP', + 'BAF_PEATF','BIOCHEM_PMIN_TO_PLANT','CH4_SURF_AERE_SAT','CH4_SURF_AERE_UNSAT','CH4_SURF_DIFF_SAT', + 'CH4_SURF_DIFF_UNSAT','CH4_SURF_EBUL_SAT','CH4_SURF_EBUL_UNSAT','CMASS_BALANCE_ERROR','cn_scalar', + 'COL_PTRUNC','CONC_CH4_SAT','CONC_CH4_UNSAT','CONC_O2_SAT','CONC_O2_UNSAT', + 'cp_scalar','CWDC_HR','CWDC_LOSS','CWDC_TO_LITR2C','CWDC_TO_LITR3C', + 'CWDC_vr','CWDN_TO_LITR2N','CWDN_TO_LITR3N','CWDN_vr','CWDP_TO_LITR2P', + 'CWDP_TO_LITR3P','CWDP_vr','DWT_CONV_CFLUX_DRIBBLED','F_CO2_SOIL','F_CO2_SOIL_vr', + 'F_DENIT_vr','F_N2O_DENIT','F_N2O_NIT','F_NIT_vr','FCH4_DFSAT', + 'FINUNDATED_LAG','FPI_P_vr','FPI_vr','FROOTC_LOSS','HR_vr', + 'LABILEP_TO_SECONDP','LABILEP_vr','LAND_UPTAKE','LEAF_MR','leaf_npimbalance', + 'LEAFC_LOSS','LEAFC_TO_LITTER','LFC2','LITR1_HR','LITR1C_TO_SOIL1C', + 'LITR1C_vr','LITR1N_TNDNCY_VERT_TRANS','LITR1N_TO_SOIL1N','LITR1N_vr','LITR1P_TNDNCY_VERT_TRANS', + 'LITR1P_TO_SOIL1P','LITR1P_vr','LITR2_HR','LITR2C_TO_SOIL2C','LITR2C_vr', + 'LITR2N_TNDNCY_VERT_TRANS','LITR2N_TO_SOIL2N','LITR2N_vr','LITR2P_TNDNCY_VERT_TRANS','LITR2P_TO_SOIL2P', + 'LITR2P_vr','LITR3_HR','LITR3C_TO_SOIL3C','LITR3C_vr','LITR3N_TNDNCY_VERT_TRANS', + 'LITR3N_TO_SOIL3N','LITR3N_vr','LITR3P_TNDNCY_VERT_TRANS','LITR3P_TO_SOIL3P','LITR3P_vr', + 'M_LITR1C_TO_LEACHING','M_LITR2C_TO_LEACHING','M_LITR3C_TO_LEACHING','M_SOIL1C_TO_LEACHING','M_SOIL2C_TO_LEACHING', + 'M_SOIL3C_TO_LEACHING','M_SOIL4C_TO_LEACHING','NDEPLOY','NEM','nlim_m', + 'o2_decomp_depth_unsat','OCCLP_vr','PDEPLOY','PLANT_CALLOC','PLANT_NDEMAND', + 'PLANT_NDEMAND_COL','PLANT_PALLOC','PLANT_PDEMAND','PLANT_PDEMAND_COL','plim_m', + 'POT_F_DENIT','POT_F_NIT','POTENTIAL_IMMOB','POTENTIAL_IMMOB_P','PRIMP_TO_LABILEP', + 'PRIMP_vr','PROD1P_LOSS','QOVER_LAG','RETRANSN_TO_NPOOL','RETRANSP_TO_PPOOL', + 'SCALARAVG_vr','SECONDP_TO_LABILEP','SECONDP_TO_OCCLP','SECONDP_vr','SMIN_NH4_vr', + 'SMIN_NO3_vr','SMINN_TO_SOIL1N_L1','SMINN_TO_SOIL2N_L2','SMINN_TO_SOIL2N_S1','SMINN_TO_SOIL3N_L3', + 'SMINN_TO_SOIL3N_S2','SMINN_TO_SOIL4N_S3','SMINP_TO_SOIL1P_L1','SMINP_TO_SOIL2P_L2','SMINP_TO_SOIL2P_S1', + 'SMINP_TO_SOIL3P_L3','SMINP_TO_SOIL3P_S2','SMINP_TO_SOIL4P_S3','SMINP_vr','SOIL1_HR','SOIL1C_TO_SOIL2C','SOIL1C_vr','SOIL1N_TNDNCY_VERT_TRANS','SOIL1N_TO_SOIL2N','SOIL1N_vr', + 'SOIL1P_TNDNCY_VERT_TRANS','SOIL1P_TO_SOIL2P','SOIL1P_vr','SOIL2_HR','SOIL2C_TO_SOIL3C', + 'SOIL2C_vr','SOIL2N_TNDNCY_VERT_TRANS','SOIL2N_TO_SOIL3N','SOIL2N_vr','SOIL2P_TNDNCY_VERT_TRANS', + 'SOIL2P_TO_SOIL3P','SOIL2P_vr','SOIL3_HR','SOIL3C_TO_SOIL4C','SOIL3C_vr', + 'SOIL3N_TNDNCY_VERT_TRANS','SOIL3N_TO_SOIL4N','SOIL3N_vr','SOIL3P_TNDNCY_VERT_TRANS','SOIL3P_TO_SOIL4P', + 'SOIL3P_vr','SOIL4_HR','SOIL4C_vr','SOIL4N_TNDNCY_VERT_TRANS','SOIL4N_TO_SMINN', + 'SOIL4N_vr','SOIL4P_TNDNCY_VERT_TRANS','SOIL4P_TO_SMINP','SOIL4P_vr','SOLUTIONP_vr', + 'TCS_MONTH_BEGIN','TCS_MONTH_END','TOTCOLCH4','water_scalar','WF', + 'wlim_m','WOODC_LOSS','WTGQ' +hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC' +hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' +hist_mfilt = 1,365 +hist_nhtfrq = -24,-24 +hist_avgflag_pertape = 'A','A' diff --git a/cime_config/testmods_dirs/allactive/wcprod_1850_4xCO2/README b/cime_config/testmods_dirs/allactive/wcprod_1850_4xCO2/README index 020bbf93e9f2..329e3e6ea7ab 100644 --- a/cime_config/testmods_dirs/allactive/wcprod_1850_4xCO2/README +++ b/cime_config/testmods_dirs/allactive/wcprod_1850_4xCO2/README @@ -3,4 +3,3 @@ water cycle production sims Run these for at least 1 day to see all output. Also use the CMIP6 compsets. -If running longer, change the nhtfrq for the first history file. diff --git a/cime_config/testmods_dirs/allactive/wcprod_1850_4xCO2/user_nl_eam b/cime_config/testmods_dirs/allactive/wcprod_1850_4xCO2/user_nl_eam index eeac1d647b1d..095c6946f5de 100644 --- a/cime_config/testmods_dirs/allactive/wcprod_1850_4xCO2/user_nl_eam +++ b/cime_config/testmods_dirs/allactive/wcprod_1850_4xCO2/user_nl_eam @@ -1,11 +1,59 @@ - nhtfrq = -24,-24,-6,-6,-3,-24,-24 - mfilt = 1,30,120,120,240,30,1 - avgflag_pertape = 'A','A','I','A','A','A','I' - fexcl1 = 'CFAD_SR532_CAL', 'LINOZ_DO3', 'LINOZ_DO3_PSC', 'LINOZ_O3CLIM', 'LINOZ_O3COL', 'LINOZ_SSO3', 'hstobie_linoz' - fincl1 = 'extinct_sw_inp','extinct_lw_bnd7','extinct_lw_inp','CLD_CAL', 'TREFMNAV', 'TREFMXAV' - fincl2 = 'FLUT','PRECT','U200','V200','U850','V850','Z500','OMEGA500','UBOT','VBOT','TREFHT','TREFHTMN:M','TREFHTMX:X','QREFHT','TS','PS','TMQ','TUQ','TVQ','TOZ', 'FLDS', 'FLNS', 'FSDS', 'FSNS', 'SHFLX', 'LHFLX', 'TGCLDCWP', 'TGCLDIWP', 'TGCLDLWP', 'CLDTOT', 'T250', 'T200', 'T150', 'T100', 'T050', 'T025', 'T010', 'T005', 'T002', 'T001', 'TTOP', 'U250', 'U150', 'U100', 'U050', 'U025', 'U010', 'U005', 'U002', 'U001', 'UTOP', 'FSNT', 'FLNT' - fincl3 = 'PSL','T200','T500','U850','V850','UBOT','VBOT','TREFHT', 'Z700', 'TBOT:M' - fincl4 = 'FLUT','U200','U850','PRECT','OMEGA500' - fincl5 = 'PRECT','PRECC','TUQ','TVQ','QFLX','SHFLX','U90M','V90M' - fincl6 = 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANTAU_ISCCP','MEANPTOP_ISCCP','MEANTB_ISCCP','CLDTOT_CAL','CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN','CLDHGH_CAL','CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN','CLDMED_CAL','CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN','CLDLOW_CAL','CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN' - fincl7 = 'O3', 'PS', 'TROP_P' +cosp_lite = .true. + +empty_htapes = .true. + +avgflag_pertape = 'A','A','A','A','I','I' +nhtfrq = -24,-24,-6,-3,-1,-24 +mfilt = 1,30,120,240,720,1 + +fincl1 = 'AODALL','AODBC','AODDUST','AODPOM','AODSO4','AODSOA','AODSS','AODVIS', + 'CLDLOW','CLDMED','CLDHGH','CLDTOT', + 'CLDHGH_CAL','CLDLOW_CAL','CLDMED_CAL','CLD_MISR','CLDTOT_CAL', + 'CLMODIS','FISCCP1_COSP','FLDS','FLNS','FLNSC','FLNT','FLUT', + 'FLUTC','FSDS','FSDSC','FSNS','FSNSC','FSNT','FSNTOA','FSNTOAC','FSNTC', + 'ICEFRAC','LANDFRAC','LWCF','OCNFRAC','OMEGA','PRECC','PRECL','PRECSC','PRECSL','PS','PSL','Q', + 'QFLX','QREFHT','RELHUM','SCO','SHFLX','SOLIN','SWCF','T','TAUX','TAUY','TCO', + 'TGCLDLWP','TMQ','TREFHT','TREFMNAV','TREFMXAV','TS','U','U10','V','Z3', + 'dst_a1DDF','dst_a3DDF','dst_c1DDF','dst_c3DDF','dst_a1SFWET','dst_a3SFWET','dst_c1SFWET','dst_c3SFWET', + 'O3','LHFLX', + 'O3_2DTDA_trop','O3_2DTDB_trop','O3_2DTDD_trop','O3_2DTDE_trop','O3_2DTDI_trop','O3_2DTDL_trop', + 'O3_2DTDN_trop','O3_2DTDO_trop','O3_2DTDS_trop','O3_2DTDU_trop','O3_2DTRE_trop','O3_2DTRI_trop', + 'O3_SRF','NO_2DTDS','NO_TDLgt','NO2_2DTDD','NO2_2DTDS','NO2_TDAcf','CO_SRF','TROPE3D_P','TROP_P', + 'CDNUMC','SFDMS','so4_a1_sfgaex1','so4_a2_sfgaex1','so4_a3_sfgaex1','so4_a5_sfgaex1','soa_a1_sfgaex1', + 'soa_a2_sfgaex1','soa_a3_sfgaex1','GS_soa_a1','GS_soa_a2','GS_soa_a3','AQSO4_H2O2','AQSO4_O3', + 'SFSO2','SO2_CLXF','SO2','DF_SO2','AQ_SO2','GS_SO2','WD_SO2','ABURDENSO4_STR','ABURDENSO4_TRO', + 'ABURDENSO4','ABURDENBC','ABURDENDUST','ABURDENMOM','ABURDENPOM','ABURDENSEASALT', + 'ABURDENSOA','AODSO4_STR','AODSO4_TRO', + 'EXTINCT','AODABS','AODABSBC','CLDICE','CLDLIQ','CLD_CAL_TMPLIQ','CLD_CAL_TMPICE','Mass_bc_srf', + 'Mass_dst_srf','Mass_mom_srf','Mass_ncl_srf','Mass_pom_srf','Mass_so4_srf','Mass_soa_srf','Mass_bc_850', + 'Mass_dst_850','Mass_mom_850','Mass_ncl_850','Mass_pom_850','Mass_so4_850','Mass_soa_850','Mass_bc_500', + 'Mass_dst_500','Mass_mom_500','Mass_ncl_500','Mass_pom_500','Mass_so4_500','Mass_soa_500','Mass_bc_330', + 'Mass_dst_330','Mass_mom_330','Mass_ncl_330','Mass_pom_330','Mass_so4_330','Mass_soa_330','Mass_bc_200', + 'Mass_dst_200','Mass_mom_200','Mass_ncl_200','Mass_pom_200','Mass_so4_200','Mass_soa_200', + 'O3_2DTDD','O3_2DCIP','O3_2DCIL','CO_2DTDS','CO_2DTDD','CO_2DCEP','CO_2DCEL','NO_2DTDD', + 'FLNTC','SAODVIS', + 'H2OLNZ', + 'dst_a1SF','dst_a3SF', + 'PHIS','CLOUD','TGCLDIWP','TGCLDCWP','AREL', + 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANPTOP_ISCCP','CLD_CAL', + 'CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN', + 'CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN', + 'CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN', + 'CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN', + 'CLWMODIS','CLIMODIS' + +fincl2 = 'PS', 'FLUT','PRECT','U200','V200','U850','V850', + 'TCO','SCO','TREFHTMN','TREFHTMX','TREFHT','QREFHT' +fincl3 = 'PS', 'PSL','PRECT','TUQ','TVQ','UBOT','VBOT','TREFHT','FLUT','OMEGA500','TBOT','U850','V850','U200','V200','T200','T500','Z700' +fincl4 = 'PRECT' +fincl5 = 'O3_SRF' +fincl6 = 'CO_2DMSD','NO2_2DMSD','NO_2DMSD','O3_2DMSD','O3_2DMSD_trop' + +! -- chemUCI settings ------------------ +history_chemdyg_summary = .true. +history_gaschmbudget_2D = .false. +history_gaschmbudget_2D_levels = .false. +history_gaschmbudget_num = 6 !! no impact if history_gaschmbudget_2D = .false. + +! -- MAM5 settings ------------------ +is_output_interactive_volc = .true. diff --git a/cime_config/testmods_dirs/allactive/wcprod_1850_4xCO2/user_nl_elm b/cime_config/testmods_dirs/allactive/wcprod_1850_4xCO2/user_nl_elm index 4379fd5e4fe1..cd1adab77404 100644 --- a/cime_config/testmods_dirs/allactive/wcprod_1850_4xCO2/user_nl_elm +++ b/cime_config/testmods_dirs/allactive/wcprod_1850_4xCO2/user_nl_elm @@ -1,7 +1,40 @@ -! Finidat to be updated, The one below not compatible with v3 lnd config (with TOP and BGC mode, new grid) -! finidat = '$DIN_LOC_ROOT/lnd/clm2/initdata_map/clmi.WCYCL1850-4xCO2.ne30pg2_EC30to60E2r2.SMS_Ld1.c20230213.nc' - hist_dov2xy = .true.,.true. - hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' - hist_mfilt = 1,365 - hist_nhtfrq = -24,-24 - hist_avgflag_pertape = 'A','A' +hist_dov2xy = .true.,.true. +hist_fexcl1 = 'AGWDNPP','ALTMAX_LASTYEAR','AVAIL_RETRANSP','AVAILC','BAF_CROP', + 'BAF_PEATF','BIOCHEM_PMIN_TO_PLANT','CH4_SURF_AERE_SAT','CH4_SURF_AERE_UNSAT','CH4_SURF_DIFF_SAT', + 'CH4_SURF_DIFF_UNSAT','CH4_SURF_EBUL_SAT','CH4_SURF_EBUL_UNSAT','CMASS_BALANCE_ERROR','cn_scalar', + 'COL_PTRUNC','CONC_CH4_SAT','CONC_CH4_UNSAT','CONC_O2_SAT','CONC_O2_UNSAT', + 'cp_scalar','CWDC_HR','CWDC_LOSS','CWDC_TO_LITR2C','CWDC_TO_LITR3C', + 'CWDC_vr','CWDN_TO_LITR2N','CWDN_TO_LITR3N','CWDN_vr','CWDP_TO_LITR2P', + 'CWDP_TO_LITR3P','CWDP_vr','DWT_CONV_CFLUX_DRIBBLED','F_CO2_SOIL','F_CO2_SOIL_vr', + 'F_DENIT_vr','F_N2O_DENIT','F_N2O_NIT','F_NIT_vr','FCH4_DFSAT', + 'FINUNDATED_LAG','FPI_P_vr','FPI_vr','FROOTC_LOSS','HR_vr', + 'LABILEP_TO_SECONDP','LABILEP_vr','LAND_UPTAKE','LEAF_MR','leaf_npimbalance', + 'LEAFC_LOSS','LEAFC_TO_LITTER','LFC2','LITR1_HR','LITR1C_TO_SOIL1C', + 'LITR1C_vr','LITR1N_TNDNCY_VERT_TRANS','LITR1N_TO_SOIL1N','LITR1N_vr','LITR1P_TNDNCY_VERT_TRANS', + 'LITR1P_TO_SOIL1P','LITR1P_vr','LITR2_HR','LITR2C_TO_SOIL2C','LITR2C_vr', + 'LITR2N_TNDNCY_VERT_TRANS','LITR2N_TO_SOIL2N','LITR2N_vr','LITR2P_TNDNCY_VERT_TRANS','LITR2P_TO_SOIL2P', + 'LITR2P_vr','LITR3_HR','LITR3C_TO_SOIL3C','LITR3C_vr','LITR3N_TNDNCY_VERT_TRANS', + 'LITR3N_TO_SOIL3N','LITR3N_vr','LITR3P_TNDNCY_VERT_TRANS','LITR3P_TO_SOIL3P','LITR3P_vr', + 'M_LITR1C_TO_LEACHING','M_LITR2C_TO_LEACHING','M_LITR3C_TO_LEACHING','M_SOIL1C_TO_LEACHING','M_SOIL2C_TO_LEACHING', + 'M_SOIL3C_TO_LEACHING','M_SOIL4C_TO_LEACHING','NDEPLOY','NEM','nlim_m', + 'o2_decomp_depth_unsat','OCCLP_vr','PDEPLOY','PLANT_CALLOC','PLANT_NDEMAND', + 'PLANT_NDEMAND_COL','PLANT_PALLOC','PLANT_PDEMAND','PLANT_PDEMAND_COL','plim_m', + 'POT_F_DENIT','POT_F_NIT','POTENTIAL_IMMOB','POTENTIAL_IMMOB_P','PRIMP_TO_LABILEP', + 'PRIMP_vr','PROD1P_LOSS','QOVER_LAG','RETRANSN_TO_NPOOL','RETRANSP_TO_PPOOL', + 'SCALARAVG_vr','SECONDP_TO_LABILEP','SECONDP_TO_OCCLP','SECONDP_vr','SMIN_NH4_vr', + 'SMIN_NO3_vr','SMINN_TO_SOIL1N_L1','SMINN_TO_SOIL2N_L2','SMINN_TO_SOIL2N_S1','SMINN_TO_SOIL3N_L3', + 'SMINN_TO_SOIL3N_S2','SMINN_TO_SOIL4N_S3','SMINP_TO_SOIL1P_L1','SMINP_TO_SOIL2P_L2','SMINP_TO_SOIL2P_S1', + 'SMINP_TO_SOIL3P_L3','SMINP_TO_SOIL3P_S2','SMINP_TO_SOIL4P_S3','SMINP_vr','SOIL1_HR','SOIL1C_TO_SOIL2C','SOIL1C_vr','SOIL1N_TNDNCY_VERT_TRANS','SOIL1N_TO_SOIL2N','SOIL1N_vr', + 'SOIL1P_TNDNCY_VERT_TRANS','SOIL1P_TO_SOIL2P','SOIL1P_vr','SOIL2_HR','SOIL2C_TO_SOIL3C', + 'SOIL2C_vr','SOIL2N_TNDNCY_VERT_TRANS','SOIL2N_TO_SOIL3N','SOIL2N_vr','SOIL2P_TNDNCY_VERT_TRANS', + 'SOIL2P_TO_SOIL3P','SOIL2P_vr','SOIL3_HR','SOIL3C_TO_SOIL4C','SOIL3C_vr', + 'SOIL3N_TNDNCY_VERT_TRANS','SOIL3N_TO_SOIL4N','SOIL3N_vr','SOIL3P_TNDNCY_VERT_TRANS','SOIL3P_TO_SOIL4P', + 'SOIL3P_vr','SOIL4_HR','SOIL4C_vr','SOIL4N_TNDNCY_VERT_TRANS','SOIL4N_TO_SMINN', + 'SOIL4N_vr','SOIL4P_TNDNCY_VERT_TRANS','SOIL4P_TO_SMINP','SOIL4P_vr','SOLUTIONP_vr', + 'TCS_MONTH_BEGIN','TCS_MONTH_END','TOTCOLCH4','water_scalar','WF', + 'wlim_m','WOODC_LOSS','WTGQ' +hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC' +hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' +hist_mfilt = 1,365 +hist_nhtfrq = -24,-24 +hist_avgflag_pertape = 'A','A' diff --git a/cime_config/testmods_dirs/allactive/wcprod_1850_r05/README b/cime_config/testmods_dirs/allactive/wcprod_1850_r05/README deleted file mode 100644 index 020bbf93e9f2..000000000000 --- a/cime_config/testmods_dirs/allactive/wcprod_1850_r05/README +++ /dev/null @@ -1,6 +0,0 @@ -These modifications should result in a case that has the same namelist settings as the -water cycle production sims - -Run these for at least 1 day to see all output. -Also use the CMIP6 compsets. -If running longer, change the nhtfrq for the first history file. diff --git a/cime_config/testmods_dirs/allactive/wcprod_1850_r05/shell_commands b/cime_config/testmods_dirs/allactive/wcprod_1850_r05/shell_commands deleted file mode 100644 index 6e8ae38ac8e9..000000000000 --- a/cime_config/testmods_dirs/allactive/wcprod_1850_r05/shell_commands +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -./xmlchange --append CAM_CONFIG_OPTS='-cosp' - diff --git a/cime_config/testmods_dirs/allactive/wcprod_1850_r05/user_nl_eam b/cime_config/testmods_dirs/allactive/wcprod_1850_r05/user_nl_eam deleted file mode 100644 index eeac1d647b1d..000000000000 --- a/cime_config/testmods_dirs/allactive/wcprod_1850_r05/user_nl_eam +++ /dev/null @@ -1,11 +0,0 @@ - nhtfrq = -24,-24,-6,-6,-3,-24,-24 - mfilt = 1,30,120,120,240,30,1 - avgflag_pertape = 'A','A','I','A','A','A','I' - fexcl1 = 'CFAD_SR532_CAL', 'LINOZ_DO3', 'LINOZ_DO3_PSC', 'LINOZ_O3CLIM', 'LINOZ_O3COL', 'LINOZ_SSO3', 'hstobie_linoz' - fincl1 = 'extinct_sw_inp','extinct_lw_bnd7','extinct_lw_inp','CLD_CAL', 'TREFMNAV', 'TREFMXAV' - fincl2 = 'FLUT','PRECT','U200','V200','U850','V850','Z500','OMEGA500','UBOT','VBOT','TREFHT','TREFHTMN:M','TREFHTMX:X','QREFHT','TS','PS','TMQ','TUQ','TVQ','TOZ', 'FLDS', 'FLNS', 'FSDS', 'FSNS', 'SHFLX', 'LHFLX', 'TGCLDCWP', 'TGCLDIWP', 'TGCLDLWP', 'CLDTOT', 'T250', 'T200', 'T150', 'T100', 'T050', 'T025', 'T010', 'T005', 'T002', 'T001', 'TTOP', 'U250', 'U150', 'U100', 'U050', 'U025', 'U010', 'U005', 'U002', 'U001', 'UTOP', 'FSNT', 'FLNT' - fincl3 = 'PSL','T200','T500','U850','V850','UBOT','VBOT','TREFHT', 'Z700', 'TBOT:M' - fincl4 = 'FLUT','U200','U850','PRECT','OMEGA500' - fincl5 = 'PRECT','PRECC','TUQ','TVQ','QFLX','SHFLX','U90M','V90M' - fincl6 = 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANTAU_ISCCP','MEANPTOP_ISCCP','MEANTB_ISCCP','CLDTOT_CAL','CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN','CLDHGH_CAL','CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN','CLDMED_CAL','CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN','CLDLOW_CAL','CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN' - fincl7 = 'O3', 'PS', 'TROP_P' diff --git a/cime_config/testmods_dirs/allactive/wcprod_1850_r05/user_nl_elm b/cime_config/testmods_dirs/allactive/wcprod_1850_r05/user_nl_elm deleted file mode 100644 index 9974e1edeb95..000000000000 --- a/cime_config/testmods_dirs/allactive/wcprod_1850_r05/user_nl_elm +++ /dev/null @@ -1,7 +0,0 @@ -! Finidat to be updated, The one below not compatible with v3 lnd config (with TOP and BGC mode, new grid) -! finidat = '${DIN_LOC_ROOT}/lnd/clm2/initdata_map/clmi.WCYCL1850.ne30pg2_r05_EC30to60E2r2.SMS_Ld1.c20230213.nc' - hist_dov2xy = .true.,.true. - hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' - hist_mfilt = 1,365 - hist_nhtfrq = -24,-24 - hist_avgflag_pertape = 'A','A' diff --git a/cime_config/testmods_dirs/allactive/wcprod_1850_r05/user_nl_mosart b/cime_config/testmods_dirs/allactive/wcprod_1850_r05/user_nl_mosart deleted file mode 100644 index b0a170bcec25..000000000000 --- a/cime_config/testmods_dirs/allactive/wcprod_1850_r05/user_nl_mosart +++ /dev/null @@ -1,4 +0,0 @@ - rtmhist_fincl2 = 'RIVER_DISCHARGE_OVER_LAND_LIQ' - rtmhist_mfilt = 1,365 - rtmhist_ndens = 2 - rtmhist_nhtfrq = -24,-24 diff --git a/cime_config/testmods_dirs/allactive/wcprodrrm/README b/cime_config/testmods_dirs/allactive/wcprodrrm/README index 22610563ff40..5a23081a3db5 100644 --- a/cime_config/testmods_dirs/allactive/wcprodrrm/README +++ b/cime_config/testmods_dirs/allactive/wcprodrrm/README @@ -1,5 +1,2 @@ -The namelist (user_nl_*) and shell commands in this drectory should result in a case -which is considered as a V2 candidate at this moment (01/26/2020) -The user_nl_* files should be replaced with "use case" files once v2 configuration -is finalized. +These mods should result in output for an RRM production case diff --git a/cime_config/testmods_dirs/allactive/wcprodrrm/user_nl_eam b/cime_config/testmods_dirs/allactive/wcprodrrm/user_nl_eam index eeac1d647b1d..095c6946f5de 100644 --- a/cime_config/testmods_dirs/allactive/wcprodrrm/user_nl_eam +++ b/cime_config/testmods_dirs/allactive/wcprodrrm/user_nl_eam @@ -1,11 +1,59 @@ - nhtfrq = -24,-24,-6,-6,-3,-24,-24 - mfilt = 1,30,120,120,240,30,1 - avgflag_pertape = 'A','A','I','A','A','A','I' - fexcl1 = 'CFAD_SR532_CAL', 'LINOZ_DO3', 'LINOZ_DO3_PSC', 'LINOZ_O3CLIM', 'LINOZ_O3COL', 'LINOZ_SSO3', 'hstobie_linoz' - fincl1 = 'extinct_sw_inp','extinct_lw_bnd7','extinct_lw_inp','CLD_CAL', 'TREFMNAV', 'TREFMXAV' - fincl2 = 'FLUT','PRECT','U200','V200','U850','V850','Z500','OMEGA500','UBOT','VBOT','TREFHT','TREFHTMN:M','TREFHTMX:X','QREFHT','TS','PS','TMQ','TUQ','TVQ','TOZ', 'FLDS', 'FLNS', 'FSDS', 'FSNS', 'SHFLX', 'LHFLX', 'TGCLDCWP', 'TGCLDIWP', 'TGCLDLWP', 'CLDTOT', 'T250', 'T200', 'T150', 'T100', 'T050', 'T025', 'T010', 'T005', 'T002', 'T001', 'TTOP', 'U250', 'U150', 'U100', 'U050', 'U025', 'U010', 'U005', 'U002', 'U001', 'UTOP', 'FSNT', 'FLNT' - fincl3 = 'PSL','T200','T500','U850','V850','UBOT','VBOT','TREFHT', 'Z700', 'TBOT:M' - fincl4 = 'FLUT','U200','U850','PRECT','OMEGA500' - fincl5 = 'PRECT','PRECC','TUQ','TVQ','QFLX','SHFLX','U90M','V90M' - fincl6 = 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANTAU_ISCCP','MEANPTOP_ISCCP','MEANTB_ISCCP','CLDTOT_CAL','CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN','CLDHGH_CAL','CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN','CLDMED_CAL','CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN','CLDLOW_CAL','CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN' - fincl7 = 'O3', 'PS', 'TROP_P' +cosp_lite = .true. + +empty_htapes = .true. + +avgflag_pertape = 'A','A','A','A','I','I' +nhtfrq = -24,-24,-6,-3,-1,-24 +mfilt = 1,30,120,240,720,1 + +fincl1 = 'AODALL','AODBC','AODDUST','AODPOM','AODSO4','AODSOA','AODSS','AODVIS', + 'CLDLOW','CLDMED','CLDHGH','CLDTOT', + 'CLDHGH_CAL','CLDLOW_CAL','CLDMED_CAL','CLD_MISR','CLDTOT_CAL', + 'CLMODIS','FISCCP1_COSP','FLDS','FLNS','FLNSC','FLNT','FLUT', + 'FLUTC','FSDS','FSDSC','FSNS','FSNSC','FSNT','FSNTOA','FSNTOAC','FSNTC', + 'ICEFRAC','LANDFRAC','LWCF','OCNFRAC','OMEGA','PRECC','PRECL','PRECSC','PRECSL','PS','PSL','Q', + 'QFLX','QREFHT','RELHUM','SCO','SHFLX','SOLIN','SWCF','T','TAUX','TAUY','TCO', + 'TGCLDLWP','TMQ','TREFHT','TREFMNAV','TREFMXAV','TS','U','U10','V','Z3', + 'dst_a1DDF','dst_a3DDF','dst_c1DDF','dst_c3DDF','dst_a1SFWET','dst_a3SFWET','dst_c1SFWET','dst_c3SFWET', + 'O3','LHFLX', + 'O3_2DTDA_trop','O3_2DTDB_trop','O3_2DTDD_trop','O3_2DTDE_trop','O3_2DTDI_trop','O3_2DTDL_trop', + 'O3_2DTDN_trop','O3_2DTDO_trop','O3_2DTDS_trop','O3_2DTDU_trop','O3_2DTRE_trop','O3_2DTRI_trop', + 'O3_SRF','NO_2DTDS','NO_TDLgt','NO2_2DTDD','NO2_2DTDS','NO2_TDAcf','CO_SRF','TROPE3D_P','TROP_P', + 'CDNUMC','SFDMS','so4_a1_sfgaex1','so4_a2_sfgaex1','so4_a3_sfgaex1','so4_a5_sfgaex1','soa_a1_sfgaex1', + 'soa_a2_sfgaex1','soa_a3_sfgaex1','GS_soa_a1','GS_soa_a2','GS_soa_a3','AQSO4_H2O2','AQSO4_O3', + 'SFSO2','SO2_CLXF','SO2','DF_SO2','AQ_SO2','GS_SO2','WD_SO2','ABURDENSO4_STR','ABURDENSO4_TRO', + 'ABURDENSO4','ABURDENBC','ABURDENDUST','ABURDENMOM','ABURDENPOM','ABURDENSEASALT', + 'ABURDENSOA','AODSO4_STR','AODSO4_TRO', + 'EXTINCT','AODABS','AODABSBC','CLDICE','CLDLIQ','CLD_CAL_TMPLIQ','CLD_CAL_TMPICE','Mass_bc_srf', + 'Mass_dst_srf','Mass_mom_srf','Mass_ncl_srf','Mass_pom_srf','Mass_so4_srf','Mass_soa_srf','Mass_bc_850', + 'Mass_dst_850','Mass_mom_850','Mass_ncl_850','Mass_pom_850','Mass_so4_850','Mass_soa_850','Mass_bc_500', + 'Mass_dst_500','Mass_mom_500','Mass_ncl_500','Mass_pom_500','Mass_so4_500','Mass_soa_500','Mass_bc_330', + 'Mass_dst_330','Mass_mom_330','Mass_ncl_330','Mass_pom_330','Mass_so4_330','Mass_soa_330','Mass_bc_200', + 'Mass_dst_200','Mass_mom_200','Mass_ncl_200','Mass_pom_200','Mass_so4_200','Mass_soa_200', + 'O3_2DTDD','O3_2DCIP','O3_2DCIL','CO_2DTDS','CO_2DTDD','CO_2DCEP','CO_2DCEL','NO_2DTDD', + 'FLNTC','SAODVIS', + 'H2OLNZ', + 'dst_a1SF','dst_a3SF', + 'PHIS','CLOUD','TGCLDIWP','TGCLDCWP','AREL', + 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANPTOP_ISCCP','CLD_CAL', + 'CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN', + 'CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN', + 'CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN', + 'CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN', + 'CLWMODIS','CLIMODIS' + +fincl2 = 'PS', 'FLUT','PRECT','U200','V200','U850','V850', + 'TCO','SCO','TREFHTMN','TREFHTMX','TREFHT','QREFHT' +fincl3 = 'PS', 'PSL','PRECT','TUQ','TVQ','UBOT','VBOT','TREFHT','FLUT','OMEGA500','TBOT','U850','V850','U200','V200','T200','T500','Z700' +fincl4 = 'PRECT' +fincl5 = 'O3_SRF' +fincl6 = 'CO_2DMSD','NO2_2DMSD','NO_2DMSD','O3_2DMSD','O3_2DMSD_trop' + +! -- chemUCI settings ------------------ +history_chemdyg_summary = .true. +history_gaschmbudget_2D = .false. +history_gaschmbudget_2D_levels = .false. +history_gaschmbudget_num = 6 !! no impact if history_gaschmbudget_2D = .false. + +! -- MAM5 settings ------------------ +is_output_interactive_volc = .true. diff --git a/cime_config/testmods_dirs/allactive/wcprodrrm/user_nl_elm b/cime_config/testmods_dirs/allactive/wcprodrrm/user_nl_elm index f6e484326140..cd1adab77404 100644 --- a/cime_config/testmods_dirs/allactive/wcprodrrm/user_nl_elm +++ b/cime_config/testmods_dirs/allactive/wcprodrrm/user_nl_elm @@ -1,33 +1,40 @@ -!---------------------------------------------------------------------------------- -! Users should add all user specific namelist changes below in the form of -! namelist_var = new_namelist_value -! -! Include namelist variables for drv_flds_in ONLY if -megan and/or -drydep options -! are set in the CLM_NAMELIST_OPTS env variable. -! -! EXCEPTIONS: -! Set use_cndv by the compset you use and the CLM_BLDNML_OPTS -dynamic_vegetation setting -! Set use_vichydro by the compset you use and the CLM_BLDNML_OPTS -vichydro setting -! Set use_cn by the compset you use and CLM_BLDNML_OPTS -bgc setting -! Set use_crop by the compset you use and CLM_BLDNML_OPTS -crop setting -! Set spinup_state by the CLM_BLDNML_OPTS -bgc_spinup setting -! Set irrigate by the CLM_BLDNML_OPTS -irrig setting -! Set co2_ppmv with CCSM_CO2_PPMV option -! Set dtime with L_NCPL option -! Set fatmlndfrc with LND_DOMAIN_PATH/LND_DOMAIN_FILE options -! Set finidat with RUN_REFCASE/RUN_REFDATE/RUN_REFTOD options for hybrid or branch cases -! (includes $inst_string for multi-ensemble cases) -! Set glc_grid with CISM_GRID option -! Set glc_smb with GLC_SMB option -! Set maxpatch_glcmec with GLC_NEC option -! Set glc_do_dynglacier with GLC_TWO_WAY_COUPLING env variable -!---------------------------------------------------------------------------------- - - check_finidat_year_consistency = .false. - hist_dov2xy = .true.,.true. check_finidat_year_consistency = .false. - hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE' - hist_mfilt = 1,365 - hist_nhtfrq = 0,-24 - hist_avgflag_pertape = 'A','A' - check_finidat_year_consistency = .false. - !finidat = '${case_scripts_dir}/../init/remap_to_naRRMpg2_20201217.beta1_01.piControlSI.compy.elm.r.0121-01-01-00000.nc' \ No newline at end of file +hist_dov2xy = .true.,.true. +hist_fexcl1 = 'AGWDNPP','ALTMAX_LASTYEAR','AVAIL_RETRANSP','AVAILC','BAF_CROP', + 'BAF_PEATF','BIOCHEM_PMIN_TO_PLANT','CH4_SURF_AERE_SAT','CH4_SURF_AERE_UNSAT','CH4_SURF_DIFF_SAT', + 'CH4_SURF_DIFF_UNSAT','CH4_SURF_EBUL_SAT','CH4_SURF_EBUL_UNSAT','CMASS_BALANCE_ERROR','cn_scalar', + 'COL_PTRUNC','CONC_CH4_SAT','CONC_CH4_UNSAT','CONC_O2_SAT','CONC_O2_UNSAT', + 'cp_scalar','CWDC_HR','CWDC_LOSS','CWDC_TO_LITR2C','CWDC_TO_LITR3C', + 'CWDC_vr','CWDN_TO_LITR2N','CWDN_TO_LITR3N','CWDN_vr','CWDP_TO_LITR2P', + 'CWDP_TO_LITR3P','CWDP_vr','DWT_CONV_CFLUX_DRIBBLED','F_CO2_SOIL','F_CO2_SOIL_vr', + 'F_DENIT_vr','F_N2O_DENIT','F_N2O_NIT','F_NIT_vr','FCH4_DFSAT', + 'FINUNDATED_LAG','FPI_P_vr','FPI_vr','FROOTC_LOSS','HR_vr', + 'LABILEP_TO_SECONDP','LABILEP_vr','LAND_UPTAKE','LEAF_MR','leaf_npimbalance', + 'LEAFC_LOSS','LEAFC_TO_LITTER','LFC2','LITR1_HR','LITR1C_TO_SOIL1C', + 'LITR1C_vr','LITR1N_TNDNCY_VERT_TRANS','LITR1N_TO_SOIL1N','LITR1N_vr','LITR1P_TNDNCY_VERT_TRANS', + 'LITR1P_TO_SOIL1P','LITR1P_vr','LITR2_HR','LITR2C_TO_SOIL2C','LITR2C_vr', + 'LITR2N_TNDNCY_VERT_TRANS','LITR2N_TO_SOIL2N','LITR2N_vr','LITR2P_TNDNCY_VERT_TRANS','LITR2P_TO_SOIL2P', + 'LITR2P_vr','LITR3_HR','LITR3C_TO_SOIL3C','LITR3C_vr','LITR3N_TNDNCY_VERT_TRANS', + 'LITR3N_TO_SOIL3N','LITR3N_vr','LITR3P_TNDNCY_VERT_TRANS','LITR3P_TO_SOIL3P','LITR3P_vr', + 'M_LITR1C_TO_LEACHING','M_LITR2C_TO_LEACHING','M_LITR3C_TO_LEACHING','M_SOIL1C_TO_LEACHING','M_SOIL2C_TO_LEACHING', + 'M_SOIL3C_TO_LEACHING','M_SOIL4C_TO_LEACHING','NDEPLOY','NEM','nlim_m', + 'o2_decomp_depth_unsat','OCCLP_vr','PDEPLOY','PLANT_CALLOC','PLANT_NDEMAND', + 'PLANT_NDEMAND_COL','PLANT_PALLOC','PLANT_PDEMAND','PLANT_PDEMAND_COL','plim_m', + 'POT_F_DENIT','POT_F_NIT','POTENTIAL_IMMOB','POTENTIAL_IMMOB_P','PRIMP_TO_LABILEP', + 'PRIMP_vr','PROD1P_LOSS','QOVER_LAG','RETRANSN_TO_NPOOL','RETRANSP_TO_PPOOL', + 'SCALARAVG_vr','SECONDP_TO_LABILEP','SECONDP_TO_OCCLP','SECONDP_vr','SMIN_NH4_vr', + 'SMIN_NO3_vr','SMINN_TO_SOIL1N_L1','SMINN_TO_SOIL2N_L2','SMINN_TO_SOIL2N_S1','SMINN_TO_SOIL3N_L3', + 'SMINN_TO_SOIL3N_S2','SMINN_TO_SOIL4N_S3','SMINP_TO_SOIL1P_L1','SMINP_TO_SOIL2P_L2','SMINP_TO_SOIL2P_S1', + 'SMINP_TO_SOIL3P_L3','SMINP_TO_SOIL3P_S2','SMINP_TO_SOIL4P_S3','SMINP_vr','SOIL1_HR','SOIL1C_TO_SOIL2C','SOIL1C_vr','SOIL1N_TNDNCY_VERT_TRANS','SOIL1N_TO_SOIL2N','SOIL1N_vr', + 'SOIL1P_TNDNCY_VERT_TRANS','SOIL1P_TO_SOIL2P','SOIL1P_vr','SOIL2_HR','SOIL2C_TO_SOIL3C', + 'SOIL2C_vr','SOIL2N_TNDNCY_VERT_TRANS','SOIL2N_TO_SOIL3N','SOIL2N_vr','SOIL2P_TNDNCY_VERT_TRANS', + 'SOIL2P_TO_SOIL3P','SOIL2P_vr','SOIL3_HR','SOIL3C_TO_SOIL4C','SOIL3C_vr', + 'SOIL3N_TNDNCY_VERT_TRANS','SOIL3N_TO_SOIL4N','SOIL3N_vr','SOIL3P_TNDNCY_VERT_TRANS','SOIL3P_TO_SOIL4P', + 'SOIL3P_vr','SOIL4_HR','SOIL4C_vr','SOIL4N_TNDNCY_VERT_TRANS','SOIL4N_TO_SMINN', + 'SOIL4N_vr','SOIL4P_TNDNCY_VERT_TRANS','SOIL4P_TO_SMINP','SOIL4P_vr','SOLUTIONP_vr', + 'TCS_MONTH_BEGIN','TCS_MONTH_END','TOTCOLCH4','water_scalar','WF', + 'wlim_m','WOODC_LOSS','WTGQ' +hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC' +hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' +hist_mfilt = 1,365 +hist_nhtfrq = -24,-24 +hist_avgflag_pertape = 'A','A' diff --git a/cime_config/testmods_dirs/allactive/wcprodrrm_1850/README b/cime_config/testmods_dirs/allactive/wcprodrrm_1850/README index 22610563ff40..e701b223a6a9 100644 --- a/cime_config/testmods_dirs/allactive/wcprodrrm_1850/README +++ b/cime_config/testmods_dirs/allactive/wcprodrrm_1850/README @@ -1,5 +1,2 @@ -The namelist (user_nl_*) and shell commands in this drectory should result in a case -which is considered as a V2 candidate at this moment (01/26/2020) -The user_nl_* files should be replaced with "use case" files once v2 configuration -is finalized. +These mods should result in production output for an 1850 RRM case diff --git a/cime_config/testmods_dirs/allactive/wcprodrrm_1850/user_nl_eam b/cime_config/testmods_dirs/allactive/wcprodrrm_1850/user_nl_eam index eeac1d647b1d..095c6946f5de 100644 --- a/cime_config/testmods_dirs/allactive/wcprodrrm_1850/user_nl_eam +++ b/cime_config/testmods_dirs/allactive/wcprodrrm_1850/user_nl_eam @@ -1,11 +1,59 @@ - nhtfrq = -24,-24,-6,-6,-3,-24,-24 - mfilt = 1,30,120,120,240,30,1 - avgflag_pertape = 'A','A','I','A','A','A','I' - fexcl1 = 'CFAD_SR532_CAL', 'LINOZ_DO3', 'LINOZ_DO3_PSC', 'LINOZ_O3CLIM', 'LINOZ_O3COL', 'LINOZ_SSO3', 'hstobie_linoz' - fincl1 = 'extinct_sw_inp','extinct_lw_bnd7','extinct_lw_inp','CLD_CAL', 'TREFMNAV', 'TREFMXAV' - fincl2 = 'FLUT','PRECT','U200','V200','U850','V850','Z500','OMEGA500','UBOT','VBOT','TREFHT','TREFHTMN:M','TREFHTMX:X','QREFHT','TS','PS','TMQ','TUQ','TVQ','TOZ', 'FLDS', 'FLNS', 'FSDS', 'FSNS', 'SHFLX', 'LHFLX', 'TGCLDCWP', 'TGCLDIWP', 'TGCLDLWP', 'CLDTOT', 'T250', 'T200', 'T150', 'T100', 'T050', 'T025', 'T010', 'T005', 'T002', 'T001', 'TTOP', 'U250', 'U150', 'U100', 'U050', 'U025', 'U010', 'U005', 'U002', 'U001', 'UTOP', 'FSNT', 'FLNT' - fincl3 = 'PSL','T200','T500','U850','V850','UBOT','VBOT','TREFHT', 'Z700', 'TBOT:M' - fincl4 = 'FLUT','U200','U850','PRECT','OMEGA500' - fincl5 = 'PRECT','PRECC','TUQ','TVQ','QFLX','SHFLX','U90M','V90M' - fincl6 = 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANTAU_ISCCP','MEANPTOP_ISCCP','MEANTB_ISCCP','CLDTOT_CAL','CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN','CLDHGH_CAL','CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN','CLDMED_CAL','CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN','CLDLOW_CAL','CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN' - fincl7 = 'O3', 'PS', 'TROP_P' +cosp_lite = .true. + +empty_htapes = .true. + +avgflag_pertape = 'A','A','A','A','I','I' +nhtfrq = -24,-24,-6,-3,-1,-24 +mfilt = 1,30,120,240,720,1 + +fincl1 = 'AODALL','AODBC','AODDUST','AODPOM','AODSO4','AODSOA','AODSS','AODVIS', + 'CLDLOW','CLDMED','CLDHGH','CLDTOT', + 'CLDHGH_CAL','CLDLOW_CAL','CLDMED_CAL','CLD_MISR','CLDTOT_CAL', + 'CLMODIS','FISCCP1_COSP','FLDS','FLNS','FLNSC','FLNT','FLUT', + 'FLUTC','FSDS','FSDSC','FSNS','FSNSC','FSNT','FSNTOA','FSNTOAC','FSNTC', + 'ICEFRAC','LANDFRAC','LWCF','OCNFRAC','OMEGA','PRECC','PRECL','PRECSC','PRECSL','PS','PSL','Q', + 'QFLX','QREFHT','RELHUM','SCO','SHFLX','SOLIN','SWCF','T','TAUX','TAUY','TCO', + 'TGCLDLWP','TMQ','TREFHT','TREFMNAV','TREFMXAV','TS','U','U10','V','Z3', + 'dst_a1DDF','dst_a3DDF','dst_c1DDF','dst_c3DDF','dst_a1SFWET','dst_a3SFWET','dst_c1SFWET','dst_c3SFWET', + 'O3','LHFLX', + 'O3_2DTDA_trop','O3_2DTDB_trop','O3_2DTDD_trop','O3_2DTDE_trop','O3_2DTDI_trop','O3_2DTDL_trop', + 'O3_2DTDN_trop','O3_2DTDO_trop','O3_2DTDS_trop','O3_2DTDU_trop','O3_2DTRE_trop','O3_2DTRI_trop', + 'O3_SRF','NO_2DTDS','NO_TDLgt','NO2_2DTDD','NO2_2DTDS','NO2_TDAcf','CO_SRF','TROPE3D_P','TROP_P', + 'CDNUMC','SFDMS','so4_a1_sfgaex1','so4_a2_sfgaex1','so4_a3_sfgaex1','so4_a5_sfgaex1','soa_a1_sfgaex1', + 'soa_a2_sfgaex1','soa_a3_sfgaex1','GS_soa_a1','GS_soa_a2','GS_soa_a3','AQSO4_H2O2','AQSO4_O3', + 'SFSO2','SO2_CLXF','SO2','DF_SO2','AQ_SO2','GS_SO2','WD_SO2','ABURDENSO4_STR','ABURDENSO4_TRO', + 'ABURDENSO4','ABURDENBC','ABURDENDUST','ABURDENMOM','ABURDENPOM','ABURDENSEASALT', + 'ABURDENSOA','AODSO4_STR','AODSO4_TRO', + 'EXTINCT','AODABS','AODABSBC','CLDICE','CLDLIQ','CLD_CAL_TMPLIQ','CLD_CAL_TMPICE','Mass_bc_srf', + 'Mass_dst_srf','Mass_mom_srf','Mass_ncl_srf','Mass_pom_srf','Mass_so4_srf','Mass_soa_srf','Mass_bc_850', + 'Mass_dst_850','Mass_mom_850','Mass_ncl_850','Mass_pom_850','Mass_so4_850','Mass_soa_850','Mass_bc_500', + 'Mass_dst_500','Mass_mom_500','Mass_ncl_500','Mass_pom_500','Mass_so4_500','Mass_soa_500','Mass_bc_330', + 'Mass_dst_330','Mass_mom_330','Mass_ncl_330','Mass_pom_330','Mass_so4_330','Mass_soa_330','Mass_bc_200', + 'Mass_dst_200','Mass_mom_200','Mass_ncl_200','Mass_pom_200','Mass_so4_200','Mass_soa_200', + 'O3_2DTDD','O3_2DCIP','O3_2DCIL','CO_2DTDS','CO_2DTDD','CO_2DCEP','CO_2DCEL','NO_2DTDD', + 'FLNTC','SAODVIS', + 'H2OLNZ', + 'dst_a1SF','dst_a3SF', + 'PHIS','CLOUD','TGCLDIWP','TGCLDCWP','AREL', + 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANPTOP_ISCCP','CLD_CAL', + 'CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN', + 'CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN', + 'CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN', + 'CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN', + 'CLWMODIS','CLIMODIS' + +fincl2 = 'PS', 'FLUT','PRECT','U200','V200','U850','V850', + 'TCO','SCO','TREFHTMN','TREFHTMX','TREFHT','QREFHT' +fincl3 = 'PS', 'PSL','PRECT','TUQ','TVQ','UBOT','VBOT','TREFHT','FLUT','OMEGA500','TBOT','U850','V850','U200','V200','T200','T500','Z700' +fincl4 = 'PRECT' +fincl5 = 'O3_SRF' +fincl6 = 'CO_2DMSD','NO2_2DMSD','NO_2DMSD','O3_2DMSD','O3_2DMSD_trop' + +! -- chemUCI settings ------------------ +history_chemdyg_summary = .true. +history_gaschmbudget_2D = .false. +history_gaschmbudget_2D_levels = .false. +history_gaschmbudget_num = 6 !! no impact if history_gaschmbudget_2D = .false. + +! -- MAM5 settings ------------------ +is_output_interactive_volc = .true. diff --git a/cime_config/testmods_dirs/allactive/wcprodrrm_1850/user_nl_elm b/cime_config/testmods_dirs/allactive/wcprodrrm_1850/user_nl_elm index 8570ad3cf598..cd1adab77404 100644 --- a/cime_config/testmods_dirs/allactive/wcprodrrm_1850/user_nl_elm +++ b/cime_config/testmods_dirs/allactive/wcprodrrm_1850/user_nl_elm @@ -1,34 +1,40 @@ -!---------------------------------------------------------------------------------- -! Users should add all user specific namelist changes below in the form of -! namelist_var = new_namelist_value -! -! Include namelist variables for drv_flds_in ONLY if -megan and/or -drydep options -! are set in the CLM_NAMELIST_OPTS env variable. -! -! EXCEPTIONS: -! Set use_cndv by the compset you use and the CLM_BLDNML_OPTS -dynamic_vegetation setting -! Set use_vichydro by the compset you use and the CLM_BLDNML_OPTS -vichydro setting -! Set use_cn by the compset you use and CLM_BLDNML_OPTS -bgc setting -! Set use_crop by the compset you use and CLM_BLDNML_OPTS -crop setting -! Set spinup_state by the CLM_BLDNML_OPTS -bgc_spinup setting -! Set irrigate by the CLM_BLDNML_OPTS -irrig setting -! Set co2_ppmv with CCSM_CO2_PPMV option -! Set dtime with L_NCPL option -! Set fatmlndfrc with LND_DOMAIN_PATH/LND_DOMAIN_FILE options -! Set finidat with RUN_REFCASE/RUN_REFDATE/RUN_REFTOD options for hybrid or branch cases -! (includes $inst_string for multi-ensemble cases) -! Set glc_grid with CISM_GRID option -! Set glc_smb with GLC_SMB option -! Set maxpatch_glcmec with GLC_NEC option -! Set glc_do_dynglacier with GLC_TWO_WAY_COUPLING env variable -!---------------------------------------------------------------------------------- - - check_finidat_year_consistency = .false. - hist_dov2xy = .true.,.true. check_finidat_year_consistency = .false. - hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE' - hist_mfilt = 1,365 - hist_nhtfrq = 0,-24 - hist_avgflag_pertape = 'A','A' - check_finidat_year_consistency = .false. -! Finidat to be updated, The one below not compatible with v3 lnd config (with TOP and BGC mode, new grid) -! finidat = '${DIN_LOC_ROOT}/lnd/clm2/initdata_map/clmi.WCYCL1850.northamericax4v1pg2_WC14to60E2r3.SMS_PS.c20230213.nc' +hist_dov2xy = .true.,.true. +hist_fexcl1 = 'AGWDNPP','ALTMAX_LASTYEAR','AVAIL_RETRANSP','AVAILC','BAF_CROP', + 'BAF_PEATF','BIOCHEM_PMIN_TO_PLANT','CH4_SURF_AERE_SAT','CH4_SURF_AERE_UNSAT','CH4_SURF_DIFF_SAT', + 'CH4_SURF_DIFF_UNSAT','CH4_SURF_EBUL_SAT','CH4_SURF_EBUL_UNSAT','CMASS_BALANCE_ERROR','cn_scalar', + 'COL_PTRUNC','CONC_CH4_SAT','CONC_CH4_UNSAT','CONC_O2_SAT','CONC_O2_UNSAT', + 'cp_scalar','CWDC_HR','CWDC_LOSS','CWDC_TO_LITR2C','CWDC_TO_LITR3C', + 'CWDC_vr','CWDN_TO_LITR2N','CWDN_TO_LITR3N','CWDN_vr','CWDP_TO_LITR2P', + 'CWDP_TO_LITR3P','CWDP_vr','DWT_CONV_CFLUX_DRIBBLED','F_CO2_SOIL','F_CO2_SOIL_vr', + 'F_DENIT_vr','F_N2O_DENIT','F_N2O_NIT','F_NIT_vr','FCH4_DFSAT', + 'FINUNDATED_LAG','FPI_P_vr','FPI_vr','FROOTC_LOSS','HR_vr', + 'LABILEP_TO_SECONDP','LABILEP_vr','LAND_UPTAKE','LEAF_MR','leaf_npimbalance', + 'LEAFC_LOSS','LEAFC_TO_LITTER','LFC2','LITR1_HR','LITR1C_TO_SOIL1C', + 'LITR1C_vr','LITR1N_TNDNCY_VERT_TRANS','LITR1N_TO_SOIL1N','LITR1N_vr','LITR1P_TNDNCY_VERT_TRANS', + 'LITR1P_TO_SOIL1P','LITR1P_vr','LITR2_HR','LITR2C_TO_SOIL2C','LITR2C_vr', + 'LITR2N_TNDNCY_VERT_TRANS','LITR2N_TO_SOIL2N','LITR2N_vr','LITR2P_TNDNCY_VERT_TRANS','LITR2P_TO_SOIL2P', + 'LITR2P_vr','LITR3_HR','LITR3C_TO_SOIL3C','LITR3C_vr','LITR3N_TNDNCY_VERT_TRANS', + 'LITR3N_TO_SOIL3N','LITR3N_vr','LITR3P_TNDNCY_VERT_TRANS','LITR3P_TO_SOIL3P','LITR3P_vr', + 'M_LITR1C_TO_LEACHING','M_LITR2C_TO_LEACHING','M_LITR3C_TO_LEACHING','M_SOIL1C_TO_LEACHING','M_SOIL2C_TO_LEACHING', + 'M_SOIL3C_TO_LEACHING','M_SOIL4C_TO_LEACHING','NDEPLOY','NEM','nlim_m', + 'o2_decomp_depth_unsat','OCCLP_vr','PDEPLOY','PLANT_CALLOC','PLANT_NDEMAND', + 'PLANT_NDEMAND_COL','PLANT_PALLOC','PLANT_PDEMAND','PLANT_PDEMAND_COL','plim_m', + 'POT_F_DENIT','POT_F_NIT','POTENTIAL_IMMOB','POTENTIAL_IMMOB_P','PRIMP_TO_LABILEP', + 'PRIMP_vr','PROD1P_LOSS','QOVER_LAG','RETRANSN_TO_NPOOL','RETRANSP_TO_PPOOL', + 'SCALARAVG_vr','SECONDP_TO_LABILEP','SECONDP_TO_OCCLP','SECONDP_vr','SMIN_NH4_vr', + 'SMIN_NO3_vr','SMINN_TO_SOIL1N_L1','SMINN_TO_SOIL2N_L2','SMINN_TO_SOIL2N_S1','SMINN_TO_SOIL3N_L3', + 'SMINN_TO_SOIL3N_S2','SMINN_TO_SOIL4N_S3','SMINP_TO_SOIL1P_L1','SMINP_TO_SOIL2P_L2','SMINP_TO_SOIL2P_S1', + 'SMINP_TO_SOIL3P_L3','SMINP_TO_SOIL3P_S2','SMINP_TO_SOIL4P_S3','SMINP_vr','SOIL1_HR','SOIL1C_TO_SOIL2C','SOIL1C_vr','SOIL1N_TNDNCY_VERT_TRANS','SOIL1N_TO_SOIL2N','SOIL1N_vr', + 'SOIL1P_TNDNCY_VERT_TRANS','SOIL1P_TO_SOIL2P','SOIL1P_vr','SOIL2_HR','SOIL2C_TO_SOIL3C', + 'SOIL2C_vr','SOIL2N_TNDNCY_VERT_TRANS','SOIL2N_TO_SOIL3N','SOIL2N_vr','SOIL2P_TNDNCY_VERT_TRANS', + 'SOIL2P_TO_SOIL3P','SOIL2P_vr','SOIL3_HR','SOIL3C_TO_SOIL4C','SOIL3C_vr', + 'SOIL3N_TNDNCY_VERT_TRANS','SOIL3N_TO_SOIL4N','SOIL3N_vr','SOIL3P_TNDNCY_VERT_TRANS','SOIL3P_TO_SOIL4P', + 'SOIL3P_vr','SOIL4_HR','SOIL4C_vr','SOIL4N_TNDNCY_VERT_TRANS','SOIL4N_TO_SMINN', + 'SOIL4N_vr','SOIL4P_TNDNCY_VERT_TRANS','SOIL4P_TO_SMINP','SOIL4P_vr','SOLUTIONP_vr', + 'TCS_MONTH_BEGIN','TCS_MONTH_END','TOTCOLCH4','water_scalar','WF', + 'wlim_m','WOODC_LOSS','WTGQ' +hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC' +hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' +hist_mfilt = 1,365 +hist_nhtfrq = -24,-24 +hist_avgflag_pertape = 'A','A' diff --git a/cime_config/testmods_dirs/allactive/wcprodssp/README b/cime_config/testmods_dirs/allactive/wcprodssp/README index 65ebc5afc662..66e4a6d06eaa 100644 --- a/cime_config/testmods_dirs/allactive/wcprodssp/README +++ b/cime_config/testmods_dirs/allactive/wcprodssp/README @@ -1,9 +1,8 @@ Modifications (in shell_commands) to enable a hybrid run for SSP compsets -(e.g., SSp370 or SSP585), starting from 2015-01-01 of v2.LR.historical_0101 +(e.g., SSP370 or SSP585), starting from end of v3 historical run Other modifications (same as for wcprod) should result in a case that has the same namelist settings as the water cycle production sims Run these for at least 1 day to see all output. Also use the CMIP6 compsets. -If running longer, change the nhtfrq for the first history file. diff --git a/cime_config/testmods_dirs/allactive/wcprodssp/shell_commands b/cime_config/testmods_dirs/allactive/wcprodssp/shell_commands index 0ef3c8cc8efa..91b8f207046c 100644 --- a/cime_config/testmods_dirs/allactive/wcprodssp/shell_commands +++ b/cime_config/testmods_dirs/allactive/wcprodssp/shell_commands @@ -6,9 +6,9 @@ ./xmlchange RUN_TYPE="hybrid" ./xmlchange GET_REFCASE="TRUE" - ./xmlchange RUN_REFCASE="v2.LR.historical_0101" + ./xmlchange RUN_REFCASE="v3.LR.historical_0101" ./xmlchange RUN_REFDATE="2015-01-01" - ./xmlchange RUN_REFDIR=${INPUTDATA_ROOT}"/e3sm_init/V2.SSP370_SSP585.ne30pg2_EC30to60E2r2/v2.LR.historical_0101/2015-01-01-00000" + ./xmlchange RUN_REFDIR=${INPUTDATA_ROOT}"/e3sm_init/v3.SSP.ne30pg2_r05_IcoswISC30E3r5/v3.LR.historical_0101/2015-01-01-00000" exit diff --git a/cime_config/testmods_dirs/allactive/wcprodssp/user_nl_eam b/cime_config/testmods_dirs/allactive/wcprodssp/user_nl_eam index b83a8c6ca22d..7ef3e3782a32 100644 --- a/cime_config/testmods_dirs/allactive/wcprodssp/user_nl_eam +++ b/cime_config/testmods_dirs/allactive/wcprodssp/user_nl_eam @@ -1,16 +1,59 @@ - nhtfrq = -24,-24,-6,-6,-3,-24,-24 - mfilt = 1,30,120,120,240,30,1 - avgflag_pertape = 'A','A','I','A','A','A','I' -!temporarily remove LINOZ O3 related fields from the list until updated linoz v3 style inputdata is ready -!fexcl1 = 'CFAD_SR532_CAL', 'LINOZ_DO3', 'LINOZ_DO3_PSC', 'LINOZ_O3CLIM', 'LINOZ_O3COL', 'LINOZ_SSO3', 'hstobie_linoz' - fexcl1 = 'CFAD_SR532_CAL', 'hstobie_linoz' - fincl1 = 'extinct_sw_inp','extinct_lw_bnd7','extinct_lw_inp','CLD_CAL', 'TREFMNAV', 'TREFMXAV' - fincl2 = 'FLUT','PRECT','U200','V200','U850','V850','Z500','OMEGA500','UBOT','VBOT','TREFHT','TREFHTMN:M','TREFHTMX:X','QREFHT','TS','PS','TMQ','TUQ','TVQ','TOZ', 'FLDS', 'FLNS', 'FSDS', 'FSNS', 'SHFLX', 'LHFLX', 'TGCLDCWP', 'TGCLDIWP', 'TGCLDLWP', 'CLDTOT', 'T250', 'T200', 'T150', 'T100', 'T050', 'T025', 'T010', 'T005', 'T002', 'T001', 'TTOP', 'U250', 'U150', 'U100', 'U050', 'U025', 'U010', 'U005', 'U002', 'U001', 'UTOP', 'FSNT', 'FLNT' - fincl3 = 'PSL','T200','T500','U850','V850','UBOT','VBOT','TREFHT', 'Z700', 'TBOT:M' - fincl4 = 'FLUT','U200','U850','PRECT','OMEGA500' - fincl5 = 'PRECT','PRECC','TUQ','TVQ','QFLX','SHFLX','U90M','V90M' - fincl6 = 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANTAU_ISCCP','MEANPTOP_ISCCP','MEANTB_ISCCP','CLDTOT_CAL','CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN','CLDHGH_CAL','CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN','CLDMED_CAL','CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN','CLDLOW_CAL','CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN' - fincl7 = 'O3', 'PS', 'TROP_P' + cosp_lite = .true. -! Specify an L80 IC to override eam.i from reference case, which is still for L72 - ncdata = '$DIN_LOC_ROOT/atm/cam/inic/homme/eami_mam4_Linoz_ne30np4_L80_c20231010.nc' + empty_htapes = .true. + + avgflag_pertape = 'A','A','A','A','I','I' + nhtfrq = -24,-24,-6,-3,-1,0 + mfilt = 1,30,120,240,720,1 + + fincl1 = 'AODALL','AODBC','AODDUST','AODPOM','AODSO4','AODSOA','AODSS','AODVIS', + 'CLDLOW','CLDMED','CLDHGH','CLDTOT', + 'CLDHGH_CAL','CLDLOW_CAL','CLDMED_CAL','CLD_MISR','CLDTOT_CAL', + 'CLMODIS','FISCCP1_COSP','FLDS','FLNS','FLNSC','FLNT','FLUT', + 'FLUTC','FSDS','FSDSC','FSNS','FSNSC','FSNT','FSNTOA','FSNTOAC','FSNTC', + 'ICEFRAC','LANDFRAC','LWCF','OCNFRAC','OMEGA','PRECC','PRECL','PRECSC','PRECSL','PS','PSL','Q', + 'QFLX','QREFHT','RELHUM','SCO','SHFLX','SOLIN','SWCF','T','TAUX','TAUY','TCO', + 'TGCLDLWP','TMQ','TREFHT','TREFMNAV','TREFMXAV','TS','U','U10','V','Z3', + 'dst_a1DDF','dst_a3DDF','dst_c1DDF','dst_c3DDF','dst_a1SFWET','dst_a3SFWET','dst_c1SFWET','dst_c3SFWET', + 'O3','LHFLX', + 'O3_2DTDA_trop','O3_2DTDB_trop','O3_2DTDD_trop','O3_2DTDE_trop','O3_2DTDI_trop','O3_2DTDL_trop', + 'O3_2DTDN_trop','O3_2DTDO_trop','O3_2DTDS_trop','O3_2DTDU_trop','O3_2DTRE_trop','O3_2DTRI_trop', + 'O3_SRF','NO_2DTDS','NO_TDLgt','NO2_2DTDD','NO2_2DTDS','NO2_TDAcf','CO_SRF','TROPE3D_P','TROP_P', + 'CDNUMC','SFDMS','so4_a1_sfgaex1','so4_a2_sfgaex1','so4_a3_sfgaex1','so4_a5_sfgaex1','soa_a1_sfgaex1', + 'soa_a2_sfgaex1','soa_a3_sfgaex1','GS_soa_a1','GS_soa_a2','GS_soa_a3','AQSO4_H2O2','AQSO4_O3', + 'SFSO2','SO2_CLXF','SO2','DF_SO2','AQ_SO2','GS_SO2','WD_SO2','ABURDENSO4_STR','ABURDENSO4_TRO', + 'ABURDENSO4','ABURDENBC','ABURDENDUST','ABURDENMOM','ABURDENPOM','ABURDENSEASALT', + 'ABURDENSOA','AODSO4_STR','AODSO4_TRO', + 'EXTINCT','AODABS','AODABSBC','CLDICE','CLDLIQ','CLD_CAL_TMPLIQ','CLD_CAL_TMPICE','Mass_bc_srf', + 'Mass_dst_srf','Mass_mom_srf','Mass_ncl_srf','Mass_pom_srf','Mass_so4_srf','Mass_soa_srf','Mass_bc_850', + 'Mass_dst_850','Mass_mom_850','Mass_ncl_850','Mass_pom_850','Mass_so4_850','Mass_soa_850','Mass_bc_500', + 'Mass_dst_500','Mass_mom_500','Mass_ncl_500','Mass_pom_500','Mass_so4_500','Mass_soa_500','Mass_bc_330', + 'Mass_dst_330','Mass_mom_330','Mass_ncl_330','Mass_pom_330','Mass_so4_330','Mass_soa_330','Mass_bc_200', + 'Mass_dst_200','Mass_mom_200','Mass_ncl_200','Mass_pom_200','Mass_so4_200','Mass_soa_200', + 'O3_2DTDD','O3_2DCIP','O3_2DCIL','CO_2DTDS','CO_2DTDD','CO_2DCEP','CO_2DCEL','NO_2DTDD', + 'FLNTC','SAODVIS', + 'H2OLNZ', + 'dst_a1SF','dst_a3SF', + 'PHIS','CLOUD','TGCLDIWP','TGCLDCWP','AREL', + 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANPTOP_ISCCP','CLD_CAL', + 'CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN', + 'CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN', + 'CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN', + 'CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN', + 'CLWMODIS','CLIMODIS' + + fincl2 = 'PS', 'FLUT','PRECT','U200','V200','U850','V850', + 'TCO','SCO','TREFHTMN:M','TREFHTMX:X','TREFHT','QREFHT' + fincl3 = 'PS', 'PSL','PRECT','TUQ','TVQ','UBOT','VBOT','TREFHT','FLUT','OMEGA500','TBOT','U850','V850','U200','V200','T200','T500','Z700' + fincl4 = 'PRECT' + fincl5 = 'O3_SRF' + fincl6 = 'CO_2DMSD','NO2_2DMSD','NO_2DMSD','O3_2DMSD','O3_2DMSD_trop' + + ! -- chemUCI settings ------------------ + history_chemdyg_summary = .true. + history_gaschmbudget_2D = .false. + history_gaschmbudget_2D_levels = .false. + history_gaschmbudget_num = 6 !! no impact if history_gaschmbudget_2D = .false. + + ! -- MAM5 settings ------------------ + is_output_interactive_volc = .true. diff --git a/cime_config/testmods_dirs/allactive/wcprodssp/user_nl_elm b/cime_config/testmods_dirs/allactive/wcprodssp/user_nl_elm index ed03b5e2616d..7bfdbac5c84b 100644 --- a/cime_config/testmods_dirs/allactive/wcprodssp/user_nl_elm +++ b/cime_config/testmods_dirs/allactive/wcprodssp/user_nl_elm @@ -1,9 +1,44 @@ -! fsurdat used is not the same file for the reference historical run (as recorded in elm.r's global attribute) +! fsurdat might be set to be not the same file for the reference historical run (as recorded in elm.r's global attribute) CHECK_FINIDAT_FSURDAT_CONSISTENCY = .false. -! Finidat to be updated, The one below not compatible with v3 lnd config (with TOP and BGC mode, new grid) -! finidat = "$DIN_LOC_ROOT/e3sm_init/V2.SSP370_SSP585.ne30pg2_EC30to60E2r2/v2.LR.historical_0101/2015-01-01-00000/v2.LR.historical_0101.elm.r.noNaN.2015-01-01-00000.nc" + + hist_dov2xy = .true.,.true. + hist_fexcl1 = 'AGWDNPP','ALTMAX_LASTYEAR','AVAIL_RETRANSP','AVAILC','BAF_CROP', + 'BAF_PEATF','BIOCHEM_PMIN_TO_PLANT','CH4_SURF_AERE_SAT','CH4_SURF_AERE_UNSAT','CH4_SURF_DIFF_SAT', + 'CH4_SURF_DIFF_UNSAT','CH4_SURF_EBUL_SAT','CH4_SURF_EBUL_UNSAT','CMASS_BALANCE_ERROR','cn_scalar', + 'COL_PTRUNC','CONC_CH4_SAT','CONC_CH4_UNSAT','CONC_O2_SAT','CONC_O2_UNSAT', + 'cp_scalar','CWDC_HR','CWDC_LOSS','CWDC_TO_LITR2C','CWDC_TO_LITR3C', + 'CWDC_vr','CWDN_TO_LITR2N','CWDN_TO_LITR3N','CWDN_vr','CWDP_TO_LITR2P', + 'CWDP_TO_LITR3P','CWDP_vr','DWT_CONV_CFLUX_DRIBBLED','F_CO2_SOIL','F_CO2_SOIL_vr', + 'F_DENIT_vr','F_N2O_DENIT','F_N2O_NIT','F_NIT_vr','FCH4_DFSAT', + 'FINUNDATED_LAG','FPI_P_vr','FPI_vr','FROOTC_LOSS','HR_vr', + 'LABILEP_TO_SECONDP','LABILEP_vr','LAND_UPTAKE','LEAF_MR','leaf_npimbalance', + 'LEAFC_LOSS','LEAFC_TO_LITTER','LFC2','LITR1_HR','LITR1C_TO_SOIL1C', + 'LITR1C_vr','LITR1N_TNDNCY_VERT_TRANS','LITR1N_TO_SOIL1N','LITR1N_vr','LITR1P_TNDNCY_VERT_TRANS', + 'LITR1P_TO_SOIL1P','LITR1P_vr','LITR2_HR','LITR2C_TO_SOIL2C','LITR2C_vr', + 'LITR2N_TNDNCY_VERT_TRANS','LITR2N_TO_SOIL2N','LITR2N_vr','LITR2P_TNDNCY_VERT_TRANS','LITR2P_TO_SOIL2P', + 'LITR2P_vr','LITR3_HR','LITR3C_TO_SOIL3C','LITR3C_vr','LITR3N_TNDNCY_VERT_TRANS', + 'LITR3N_TO_SOIL3N','LITR3N_vr','LITR3P_TNDNCY_VERT_TRANS','LITR3P_TO_SOIL3P','LITR3P_vr', + 'M_LITR1C_TO_LEACHING','M_LITR2C_TO_LEACHING','M_LITR3C_TO_LEACHING','M_SOIL1C_TO_LEACHING','M_SOIL2C_TO_LEACHING', + 'M_SOIL3C_TO_LEACHING','M_SOIL4C_TO_LEACHING','NDEPLOY','NEM','nlim_m', + 'o2_decomp_depth_unsat','OCCLP_vr','PDEPLOY','PLANT_CALLOC','PLANT_NDEMAND', + 'PLANT_NDEMAND_COL','PLANT_PALLOC','PLANT_PDEMAND','PLANT_PDEMAND_COL','plim_m', + 'POT_F_DENIT','POT_F_NIT','POTENTIAL_IMMOB','POTENTIAL_IMMOB_P','PRIMP_TO_LABILEP', + 'PRIMP_vr','PROD1P_LOSS','QOVER_LAG','RETRANSN_TO_NPOOL','RETRANSP_TO_PPOOL', + 'SCALARAVG_vr','SECONDP_TO_LABILEP','SECONDP_TO_OCCLP','SECONDP_vr','SMIN_NH4_vr', + 'SMIN_NO3_vr','SMINN_TO_SOIL1N_L1','SMINN_TO_SOIL2N_L2','SMINN_TO_SOIL2N_S1','SMINN_TO_SOIL3N_L3', + 'SMINN_TO_SOIL3N_S2','SMINN_TO_SOIL4N_S3','SMINP_TO_SOIL1P_L1','SMINP_TO_SOIL2P_L2','SMINP_TO_SOIL2P_S1', + 'SMINP_TO_SOIL3P_L3','SMINP_TO_SOIL3P_S2','SMINP_TO_SOIL4P_S3','SMINP_vr','SOIL1_HR','SOIL1C_TO_SOIL2C','SOIL1C_vr','SOIL1N_TNDNCY_VERT_TRANS','SOIL1N_TO_SOIL2N','SOIL1N_vr', + 'SOIL1P_TNDNCY_VERT_TRANS','SOIL1P_TO_SOIL2P','SOIL1P_vr','SOIL2_HR','SOIL2C_TO_SOIL3C', + 'SOIL2C_vr','SOIL2N_TNDNCY_VERT_TRANS','SOIL2N_TO_SOIL3N','SOIL2N_vr','SOIL2P_TNDNCY_VERT_TRANS', + 'SOIL2P_TO_SOIL3P','SOIL2P_vr','SOIL3_HR','SOIL3C_TO_SOIL4C','SOIL3C_vr', + 'SOIL3N_TNDNCY_VERT_TRANS','SOIL3N_TO_SOIL4N','SOIL3N_vr','SOIL3P_TNDNCY_VERT_TRANS','SOIL3P_TO_SOIL4P', + 'SOIL3P_vr','SOIL4_HR','SOIL4C_vr','SOIL4N_TNDNCY_VERT_TRANS','SOIL4N_TO_SMINN', + 'SOIL4N_vr','SOIL4P_TNDNCY_VERT_TRANS','SOIL4P_TO_SMINP','SOIL4P_vr','SOLUTIONP_vr', + 'TCS_MONTH_BEGIN','TCS_MONTH_END','TOTCOLCH4','water_scalar','WF', + 'wlim_m','WOODC_LOSS','WTGQ' + hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC' hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' hist_mfilt = 1,365 hist_nhtfrq = -24,-24 diff --git a/cime_config/testmods_dirs/bench/noio/user_nl_mpaso b/cime_config/testmods_dirs/bench/noio/user_nl_mpaso index fcaa3656188f..046a22bb8831 100644 --- a/cime_config/testmods_dirs/bench/noio/user_nl_mpaso +++ b/cime_config/testmods_dirs/bench/noio/user_nl_mpaso @@ -6,3 +6,4 @@ config_am_timeseriesstatsmonthlymin_enable=false config_am_timeseriesstatsmonthlymax_enable=false config_am_eddyproductvariables_enable=false config_am_oceanheatcontent_enable=false +config_am_conservationcheck_enable=false diff --git a/cime_config/testmods_dirs/config_pes_tests.xml b/cime_config/testmods_dirs/config_pes_tests.xml index 3301617b9bb4..4ef7663c540a 100644 --- a/cime_config/testmods_dirs/config_pes_tests.xml +++ b/cime_config/testmods_dirs/config_pes_tests.xml @@ -115,34 +115,34 @@ - + - tests+chrysalis: -compset WCYCL* -res ne30pg*EC30to60E2r2 on 4 nodes pure-MPI, ~2.38 sypd + tests+chrysalis: -compset WCYCL* -res ne30pg*IcoswISC30E3r5 on 6 nodes pure-MPI - 192 - 192 - 192 - 192 + 320 + 320 + 320 + 320 64 - 192 + 320 - 192 + 320 - tests+chrysalis: --compset BGC* --res ne30pg2_r05_EC30to60E2r2 on 5 nodes pure-MPI, ~0.9 sypd + tests+chrysalis: --compset BGC* --res ne30pg2_r05_IcoswISC30E3r5 on 9 nodes pure-MPI - 256 - 256 - 256 - 256 - 64 - 256 + 448 + 448 + 448 + 448 + 128 + 448 - 256 + 448 @@ -159,7 +159,7 @@ - tests+anvil: --compset WCYCL* --res ne30pg2_EC30to60E2r2 on 16 nodes pure-MPI, ~2.7 sypd + tests+anvil: --compset WCYCL* --res ne30pg2_IcoswISC30E3r5 on 16 nodes pure-MPI 396 396 @@ -173,7 +173,7 @@ - tests+anvil: --compset BGC* --res ne30pg2_r05_EC30to60E2r2 on 30 nodes pure-MPI, ~3 sypd + tests+anvil: --compset BGC* --res ne30pg2_r05_IcoswISC30E3r5 on 30 nodes pure-MPI 675 684 @@ -284,4 +284,34 @@ + + + + + tests+chrysalis: any compset on oQU240 grid, 1x32x2 NODESxMPIxOMP + 32 + 64 + + 32 + 32 + 32 + 32 + 32 + 32 + 32 + 32 + + + 2 + 2 + 2 + 2 + 2 + 2 + 2 + 2 + + + + diff --git a/cime_config/testmods_dirs/dinloc/case/shell_commands b/cime_config/testmods_dirs/dinloc/case/shell_commands new file mode 100755 index 000000000000..0b9483459f45 --- /dev/null +++ b/cime_config/testmods_dirs/dinloc/case/shell_commands @@ -0,0 +1,4 @@ +#!/bin/bash +# Set DIN to be local to the case +./xmlchange DIN_LOC_ROOT="`./xmlquery --value CASEROOT`/inputdata" +./xmlchange DIN_LOC_ROOT_CLMFORC="`./xmlquery --value CASEROOT`/inputdata/atm/datm7" diff --git a/cime_config/testmods_dirs/dinloc/scratch/shell_commands b/cime_config/testmods_dirs/dinloc/scratch/shell_commands new file mode 100755 index 000000000000..640ac482947f --- /dev/null +++ b/cime_config/testmods_dirs/dinloc/scratch/shell_commands @@ -0,0 +1,4 @@ +#!/bin/bash +# Set DIN to be in SCRATCH space +./xmlchange DIN_LOC_ROOT="${SCRATCH}/inputdata" +./xmlchange DIN_LOC_ROOT_CLMFORC="${SCRATCH}/inputdata/atm/datm7" diff --git a/cime_config/tests.py b/cime_config/tests.py index 5e94f9c30073..b4b54dceeb9c 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -25,7 +25,7 @@ "e3sm_mosart_exenoshare": { "time" : "0:45:00", "tests" : ( - "ERS.ne30pg2_r05_EC30to60E2r2.GPMPAS-JRA.mosart-rof_ocn_2way", + "ERS.ne30pg2_r05_IcoswISC30E3r5.GPMPAS-JRA.mosart-rof_ocn_2way", ) }, @@ -54,9 +54,11 @@ "e3sm_land_exenoshare" : { "time" : "0:45:00", "tests" : ( + "ERS.f19_g16.IERA5ELM", + "ERS.f19_g16.IERA56HRELM", "ERS_Ld20.f45_f45.IELMFATES.elm-fates", - "ERS.hcru_hcru.I20TRGSWCNPRDCTCBC.elm-erosion", "ERS.f09_g16.IELMBC.elm-simple_decomp", + "ERS.hcru_hcru.IELM.elm-multi_inst", ) }, @@ -65,7 +67,7 @@ "tests" : ( "ERS_D.f19_f19.IELM.elm-ic_f19_f19_ielm", "ERS_D.f09_g16.I1850ELMCN", - "ERS_D.ne11_oQU240.I20TRELM", + "ERS_D.ne4pg2_oQU480.I20TRELM.elm-disableDynpftCheck", "SMS_Ly2_P1x1_D.1x1_smallvilleIA.IELMCNCROP.elm-lulcc_sville", "ERS_D.f19_g16.I1850GSWCNPRDCTCBC.elm-ctc_f19_g16_I1850GSWCNPRDCTCBC", "ERS_D.f09_f09.IELM.elm-solar_rad", @@ -91,7 +93,7 @@ "SMS_Ly2_P1x1.1x1_smallvilleIA.IELMCNCROP.elm-fan", "SMS.r05_r05.IELM.elm-topounit", "ERS.ELM_USRDAT.I1850ELM.elm-usrdat", - "ERS.f09_f09.IELM.elm-lnd_rof_2way", + "ERS.r05_r05.IELM.elm-lnd_rof_2way", "ERS.r05_r05.IELM.elm-V2_ELM_MOSART_features", "ERS.ELM_USRDAT.IELM.elm-surface_water_dynamics" ) @@ -100,32 +102,51 @@ "e3sm_atm_developer" : { "inherit" : ("eam_theta_pg2"), "tests" : ( - "ERP_Ln18.ne4_oQU240.F2010", - "SMS_Ln9.ne4_oQU240.F2010.eam-outfrq9s", - "SMS.ne4_oQU240.F2010.eam-cosplite", + "ERP_Ld3.ne4pg2_oQU480.F2010", + "SMS_Ln9.ne4pg2_oQU480.F2010.eam-outfrq9s", + "SMS.ne4pg2_oQU480.F2010.eam-cosplite", "SMS_R_Ld5.ne4_ne4.FSCM-ARM97.eam-scm", - "SMS_D_Ln5.ne4_oQU240.F2010", + "SMS_D_Ln5.ne4pg2_oQU480.F2010", "SMS_Ln5.ne4pg2_oQU480.F2010", - "ERS_D.ne4_oQU240.F2010.eam-hommexx", + "ERS_D.ne4pg2_oQU480.F2010.eam-hommexx", "SMS_Ln9_P24x1.ne4_ne4.FDPSCREAM-ARM97", ) }, "e3sm_ice_developer" : { "tests" : ( - "SMS_D_Ld1.TL319_EC30to60E2r2.DTESTM-JRA1p5.mpassi-jra_1958", + "SMS_D_Ld1.TL319_IcoswISC30E3r5.DTESTM-JRA1p5.mpassi-jra_1958", "ERS_Ld5.T62_oQU240.DTESTM", "PEM_Ln5.T62_oQU240wLI.DTESTM", "PET_Ln5.T62_oQU240.DTESTM", ) }, + "e3sm_cryo_developer" : { + "tests" : ( + "SMS_D_Ld1.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF.mpaso-jra_1958", + "ERS_Ld5.T62_oQU240wLI.GMPAS-DIB-IAF-PISMF", + "PEM_Ln5.T62_oQU240wLI.GMPAS-DIB-IAF-PISMF", + "PET_Ln5.T62_oQU240wLI.GMPAS-DIB-IAF-PISMF", + "ERS_Ld5.T62_oQU240wLI.GMPAS-DIB-IAF-DISMF", + "PEM_Ln5.T62_oQU240wLI.GMPAS-DIB-IAF-DISMF", + "PET_Ln5.T62_oQU240wLI.GMPAS-DIB-IAF-DISMF", + ) + }, + + "e3sm_landice_developer" : { + "tests" : ( + "SMS.ne30pg2_r05_EC30to60E2r2_gis20.IGELM_MLI.elm-gis20kmSMS", + "ERS.ne30pg2_r05_EC30to60E2r2_gis20.IGELM_MLI.elm-gis20kmERS", + ) + }, + "eam_condidiag" : { "tests" : ( "SMS_D_Ln5.ne4pg2_oQU480.F2010", "SMS_D_Ln5.ne4pg2_oQU480.F2010.eam-condidiag_dcape", - "ERP_Ln18.ne4_oQU240.F2010.eam-condidiag_dcape", - "ERP_Ln18.ne4_oQU240.F2010.eam-condidiag_rhi", + "ERP_Ld3.ne4pg2_oQU480.F2010.eam-condidiag_dcape", + "ERP_Ld3.ne4pg2_oQU480.F2010.eam-condidiag_rhi", ) }, @@ -167,38 +188,38 @@ "e3sm_atm_integration" : { "inherit" : ("eam_preqx", "eam_theta"), "tests" : ( - "ERP_Ln9.ne4_ne4.FAQP", - "SMS_Ld1.ne4_ne4.FAQP.eam-clubb_only", - "ERP_Ln9.ne4_ne4.FRCE", - "PET_Ln5.ne4_oQU240.F2010.allactive-mach-pet", - "PEM_Ln5.ne4_oQU240.F2010", - "SMS_D_Ln5.ne4_oQU240.F2010.eam-cosplite_nhtfrq5", - "SMS_Ln1.ne4_oQU240.F2010.eam-chem_pp", - "SMS_Ln5.ne30pg2_r05_EC30to60E2r2.BGCEXP_LNDATM_CNPRDCTC_20TR", - "SMS_Ln5.ne30pg2_r05_EC30to60E2r2.BGCEXP_LNDATM_CNPRDCTC_1850", - "SMS_D_Ln5.ne4_oQU240.F2010.eam-clubb_sp", - "ERS_Ld5.ne4_oQU240.F2010.eam-rrtmgp", - "ERS_Ld5.ne4_oQU240.F2010.eam-rrtmgpxx", - "REP_Ln5.ne4_oQU240.F2010", - "SMS_Ld9.ne4pg2_oQU480.F2010.eam-thetahy_sl_pg2_mass", - "ERP_Ld9.ne4_ne4.FIDEAL.allactive-pioroot1", + "ERP_Ln9.ne4pg2_ne4pg2.FAQP", + "SMS_Ld1.ne4pg2_ne4pg2.FAQP.eam-clubb_only", + "ERP_Ln9.ne4pg2_ne4pg2.FRCE", + "PET_Ln5.ne4pg2_oQU480.F2010.allactive-mach-pet", + "PEM_Ln5.ne4pg2_oQU480.F2010", + "SMS_D_Ln5.ne4pg2_oQU480.F2010.eam-cosplite_nhtfrq5", + "SMS_Ln1.ne4pg2_oQU480.F2010.eam-chem_pp", + "SMS_Ln5.ne30pg2_r05_IcoswISC30E3r5.BGCEXP_LNDATM_CNPRDCTC_20TR", + "SMS_Ln5.ne30pg2_r05_IcoswISC30E3r5.BGCEXP_LNDATM_CNPRDCTC_1850", + "SMS_D_Ln5.ne4pg2_oQU480.F2010.eam-clubb_sp", + "ERS_Ld5.ne4pg2_oQU480.F2010.eam-rrtmgp", + "ERS_Ld5.ne4pg2_oQU480.F2010.eam-rrtmgpxx", + "REP_Ln5.ne4pg2_oQU480.F2010", + "SMS_Ld3.ne4pg2_oQU480.F2010.eam-thetahy_sl_pg2_mass", + "ERP_Ld3.ne4pg2_ne4pg2.FIDEAL.allactive-pioroot1", ) }, #atmopheric tests for extra coverage "e3sm_atm_extra_coverage" : { "tests" : ( - "SMS_Lm1.ne4_oQU240.F2010", - "ERS_Ld31.ne4_oQU240.F2010", - "ERP_Lm3.ne4_oQU240.F2010", - "SMS_D_Ln5.ne30_oECv3.F2010", - "ERP_Ld3.ne30_oECv3.F2010.allactive-pioroot1", - "SMS_Ly1.ne4_oQU240.F2010", + "SMS_Lm1.ne4pg2_oQU480.F2010", + "ERS_Ld31.ne4pg2_oQU480.F2010", + "ERP_Lm3.ne4pg2_oQU480.F2010", + "SMS_D_Ln5.ne30pg2_r05_IcoswISC30E3r5.F2010", + "ERP_Ld3.ne30pg2_r05_IcoswISC30E3r5.F2010.allactive-pioroot1", + "SMS_Ly1.ne4pg2_oQU480.F2010", "SMS_D_Ln5.ne45pg2_ne45pg2.FAQP", - "SMS_D_Ln5.ne4_oQU240.F2010.eam-implicit_stress", - "ERS_Ld5.ne30_oECv3.F2010.eam-implicit_stress", - "ERP_Ln18.ne4_oQU240.F2010.eam-condidiag_dcape", - "ERP_Ln18.ne4_oQU240.F2010.eam-condidiag_rhi", + "SMS_D_Ln5.ne4pg2_oQU480.F2010.eam-implicit_stress", + "ERS_Ld5.ne30pg2_r05_IcoswISC30E3r5.F2010.eam-implicit_stress", + "ERP_Ld3.ne4pg2_oQU480.F2010.eam-condidiag_dcape", + "ERP_Ld3.ne4pg2_oQU480.F2010.eam-condidiag_rhi", ) }, @@ -212,19 +233,30 @@ "e3sm_atm_prod" : { "tests" : ( "SMS_Ln5.ne30pg2_r05_IcoswISC30E3r5.F2010.eam-wcprod_F2010", - "SMS.ne30pg2_r05_IcoswISC30E3r5.F20TR.eam-wcprod_F20TR", + "SMS_Ld1.ne30pg2_r05_IcoswISC30E3r5.F20TR.eam-wcprod_F20TR", ) }, #atmopheric nbfb tests "e3sm_atm_nbfb" : { "tests" : ( - "PGN_P1x1.ne4_oQU240.F2010", - "TSC_PS.ne4_oQU240.F2010", - "MVK_PS.ne4_oQU240.F2010", + "PGN_P1x1.ne4pg2_oQU480.F2010", + "TSC_PS.ne4pg2_oQU480.F2010", + "MVK_PS.ne4pg2_oQU480.F2010", + ) + }, + + #ocean non bit-for-bit test + "e3sm_ocn_nbfb": { + "tests": ( + "MVKO_PS.T62_oQU240.GMPAS-NYF", ) }, + "e3sm_nbfb": { + "inherit": ("e3sm_atm_nbfb", "e3sm_ocn_nbfb") + }, + "e3sm_ocnice_stealth_features" : { "tests" : ( "SMS_D_Ld1.T62_oQU240wLI.GMPAS-IAF-PISMF.mpaso-impl_top_drag", @@ -236,8 +268,8 @@ "e3sm_ocnice_extra_coverage" : { "inherit" : ("e3sm_ocnice_stealth_features"), "tests" : ( - "ERS_P480_Ld5.T62_ECwISC30to60E2r1.GMPAS-DIB-IAF-PISMF", - "PEM_P480_Ld5.T62_ECwISC30to60E2r1.GMPAS-DIB-IAF-PISMF", + "ERS_P480_Ld5.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF.mpaso-jra_1958", + "PEM_P480_Ld5.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF.mpaso-jra_1958", "SMS.ne30_oECv3_gis.IGELM_MLI.elm-extrasnowlayers", ) }, @@ -247,7 +279,7 @@ "tests" : ( "ERP.ne4pg2_oQU480.F2010.eam-v3atm_dustemis", "REP.ne4pg2_oQU480.F2010.eam-v3atm_dustemis", - "SMS.ne30pg2_EC30to60E2r2.F2010.eam-v3atm_dustemis", + "SMS.ne30pg2_IcoswISC30E3r5.F2010.eam-v3atm_dustemis", "SMS_D_Ln5.ne4pg2_oQU480.F2010.eam-v3atm_dustemis", "PET_Ln5.ne4pg2_oQU480.F2010.eam-v3atm_dustemis", "PEM_Ln5.ne4pg2_oQU480.F2010.eam-v3atm_dustemis", @@ -256,7 +288,7 @@ }, "e3sm_developer" : { - "inherit" : ("e3sm_land_developer", "e3sm_atm_developer", "e3sm_ice_developer"), + "inherit" : ("e3sm_land_developer", "e3sm_atm_developer", "e3sm_ice_developer", "e3sm_cryo_developer"), "time" : "0:45:00", "tests" : ( "ERS.f19_g16_rx1.A", @@ -266,11 +298,9 @@ "NCK.f19_g16_rx1.A", "SMS.ne30_f19_g16_rx1.A", "ERS_Ld5.T62_oQU120.CMPASO-NYF", - "SMS_Ld1.T62_oQU240wLI.GMPAS-IAF-DISMF", "ERS.f09_g16_g.MALISIA", "SMS.T62_oQU120_ais20.MPAS_LISIO_TEST", - "SMS.f09_g16_a.IGELM_MLI", - "SMS_P12x2.ne4_oQU240.WCYCL1850NS.allactive-mach_mods", + "SMS_P12x2.ne4pg2_oQU480.WCYCL1850NS.allactive-mach_mods", "ERS_Ln9.ne4pg2_ne4pg2.F2010-MMF1.eam-mmf_crmout", ) }, @@ -284,27 +314,26 @@ }, "e3sm_integration" : { - "inherit" : ("e3sm_developer", "e3sm_atm_integration", "e3sm_mmf_integration"), + "inherit" : ("e3sm_developer", "e3sm_atm_integration", "e3sm_mmf_integration", "e3sm_rrm"), "time" : "03:00:00", "tests" : ( "ERS.ne4pg2_oQU480.WCYCL1850NS", - "SMS_D_Ld1.ne30pg2_EC30to60E2r2.WCYCL1850.allactive-wcprod", - "SMS_D_Ld1.ne30pg2_EC30to60E2r2.WCYCLSSP370.allactive-wcprodssp", - "ERS_Ld3.ne4_oQU240.F2010", - #"ERT_Ld31.ne16_g37.B1850C5",#add this line back in with the new correct compset + "SMS_D_Ld1.ne30pg2_r05_IcoswISC30E3r5.WCYCL1850.allactive-wcprod", + "SMS_D_Ld1.ne30pg2_r05_IcoswISC30E3r5.WCYCLSSP370.allactive-wcprodssp", + "ERS_Ld3.ne4pg2_oQU480.F2010", "NCK.ne4pg2_oQU480.WCYCL1850NS", "PET.f19_g16.X.allactive-mach-pet", "PET.f45_g37_rx1.A.allactive-mach-pet", - "PET_Ln9_PS.ne30pg2_EC30to60E2r2.WCYCL1850.allactive-mach-pet", - "PEM_Ln9.ne30pg2_EC30to60E2r2.WCYCL1850", - "ERP_Ld3.ne30pg2_EC30to60E2r2.WCYCL1850.allactive-pioroot1", - "SMS_D_Ln5.conusx4v1_r05_oECv3.F2010", - "SMS_Ld2.ne30pg2_r05_EC30to60E2r2.BGCEXP_CNTL_CNPECACNT_1850.elm-bgcexp", - "SMS_Ld2.ne30pg2_r05_EC30to60E2r2.BGCEXP_CNTL_CNPRDCTC_1850.elm-bgcexp", + "PET_Ln9_PS.ne30pg2_r05_IcoswISC30E3r5.WCYCL1850.allactive-mach-pet", + "PEM_Ln9.ne30pg2_r05_IcoswISC30E3r5.WCYCL1850", + "ERP_Ld3.ne30pg2_r05_IcoswISC30E3r5.WCYCL1850.allactive-pioroot1", + "SMS_Ld2.ne30pg2_r05_IcoswISC30E3r5.BGCEXP_CNTL_CNPECACNT_1850.elm-bgcexp", + "SMS_Ld2.ne30pg2_r05_IcoswISC30E3r5.BGCEXP_CNTL_CNPRDCTC_1850.elm-bgcexp", "SMS_D_Ld3.T62_oQU120.CMPASO-IAF", - "SMS_D_Ld1.ne30pg2_r05_EC30to60E2r2.WCYCL1850", "SMS_Ln5.ne30pg2_ne30pg2.F2010-SCREAM-LR-DYAMOND2", - "ERS_Ld3.ne30pg2_r05_EC30to60E2r2.WCYCL1850.allactive-nlmaps", + "ERS_Ld3.ne30pg2_r05_IcoswISC30E3r5.WCYCL1850.allactive-nlmaps", + "SMS_D_Ld1.ne30pg2_r05_IcoswISC30E3r5.CRYO1850-DISMF", + "ERS.hcru_hcru.I20TRGSWCNPRDCTCBC.elm-erosion", ) }, @@ -328,7 +357,7 @@ #e3sm tests for RRM grids "e3sm_rrm" : { "tests" : ( - "SMS_D_Ln5.conusx4v1_r05_oECv3.F2010", + "SMS_D_Ln5.conusx4v1pg2_r05_IcoswISC30E3r5.F2010", ) }, @@ -346,13 +375,13 @@ "e3sm_prod" : { "inherit" : "e3sm_atm_prod", "tests" : ( - "SMS_Ld1.ne30pg2_r05_IcoswISC30E3r5.WCYCL1850.allactive-wcprod_1850_r05", "SMS_Ld1.ne30pg2_r05_IcoswISC30E3r5.WCYCL1850-1pctCO2.allactive-wcprod_1850_1pctCO2", "SMS_Ld1.ne30pg2_r05_IcoswISC30E3r5.WCYCL1850-4xCO2.allactive-wcprod_1850_4xCO2", "SMS_Ld1.ne30pg2_r05_IcoswISC30E3r5.WCYCL1850.allactive-wcprod_1850", - "SMS_Ld1.ne30pg2_EC30to60E2r2.WCYCLSSP370.allactive-wcprodssp", - "SMS_Ld1.ne30pg2_EC30to60E2r2.WCYCLSSP585.allactive-wcprodssp", - "SMS_PS.northamericax4v1pg2_WC14to60E2r3.WCYCL1850.allactive-wcprodrrm_1850", + "SMS_Ld1.ne30pg2_r05_IcoswISC30E3r5.WCYCLSSP370.allactive-wcprodssp", + "SMS_Ld1.ne30pg2_r05_IcoswISC30E3r5.WCYCLSSP585.allactive-wcprodssp", + "SMS_Ld1_PS.northamericax4v1pg2_WC14to60E2r3.WCYCL1850.allactive-wcprodrrm_1850", + "SMS_D_Ld1.ne30pg2_r05_IcoswISC30E3r5.CRYO1850", ) }, @@ -369,11 +398,8 @@ #e3sm performance-benching of production-like runs "e3sm_prod_bench" : { "tests" : ( - "PFS.ne30pg2_r05_oECv3.F2010.bench-noio", - "PFS.ne30pg2_r05_oECv3.F20TR.bench-noio", - "PFS.ne30pg2_r05_EC30to60E2r2.WCYCL1850.bench-noio", - "PFS.ne30pg2_EC30to60E2r2.WCYCL1850.bench-noio", - "PFS_PS.northamericax4v1pg2_WC14to60E2r3.WCYCL1850.bench-noio", + "PFS.ne30pg2_r05_IcoswISC30E3r5.F2010.bench-noio", + "PFS.ne30pg2_r05_IcoswISC30E3r5.WCYCL1850.bench-noio", ) }, @@ -415,6 +441,7 @@ "ERP_D_Ld3.f19_g16.IELMFATES.elm-fates_cold", "ERS_D_Ld3_PS.f09_g16.IELMFATES.elm-fates_cold", "ERS_D_Ld5.f45_g37.IELMFATES.elm-fates_cold", + "ERS_D_Ld30.f45_g37.IELMFATES.elm-fates_cold_landuse", "ERS_Ld30.f45_g37.IELMFATES.elm-fates_satphen", "ERS_Ld30.f45_g37.IELMFATES.elm-fates_cold_fixedbiogeo", "ERS_Ld30.f45_g37.IELMFATES.elm-fates_cold_nocomp", @@ -424,6 +451,7 @@ "ERS_Ld60.f45_g37.IELMFATES.elm-fates_cold_nofire", "ERS_Ld60.f45_g37.IELMFATES.elm-fates_cold_st3", "ERS_Ld60.f45_g37.IELMFATES.elm-fates_cold_pphys", + "SMS_D_Ld15.f45_g37.IELMFATES.elm-fates_cold_twostream", ) }, @@ -454,27 +482,27 @@ "share" : True, "time" : "01:00:00", "tests" : ( - "SMS.ne4_oQU240.F2010.eam-preqx_ftype0", - "SMS.ne4_oQU240.F2010.eam-preqx_ftype1", - "SMS.ne4_oQU240.F2010.eam-preqx_ftype4", + "SMS.ne4pg2_oQU480.F2010.eam-preqx_ftype0", + "SMS.ne4pg2_oQU480.F2010.eam-preqx_ftype1", + "SMS.ne4pg2_oQU480.F2010.eam-preqx_ftype4", ) }, "eam_theta" : { "share" : True, "time" : "02:00:00", "tests" : ( - "SMS.ne4_oQU240.F2010.eam-thetahy_ftype0", - "SMS.ne4_oQU240.F2010.eam-thetahy_ftype1", - "SMS.ne4_oQU240.F2010.eam-thetahy_ftype2", - "SMS.ne4_oQU240.F2010.eam-thetahy_ftype2_energy", - "SMS.ne4_oQU240.F2010.eam-thetahy_ftype4", - "SMS.ne4_oQU240.F2010.eam-thetanh_ftype0", - "SMS.ne4_oQU240.F2010.eam-thetanh_ftype1", - "SMS.ne4_oQU240.F2010.eam-thetanh_ftype2", - "SMS.ne4_oQU240.F2010.eam-thetanh_ftype4", - "SMS.ne4_oQU240.F2010.eam-thetahy_sl", - "ERS.ne4_oQU240.F2010.eam-thetahy_ftype2", - "ERS.ne4_oQU240.F2010.eam-thetanh_ftype2", + "SMS.ne4pg2_oQU480.F2010.eam-thetahy_ftype0", + "SMS.ne4pg2_oQU480.F2010.eam-thetahy_ftype1", + "SMS.ne4pg2_oQU480.F2010.eam-thetahy_ftype2", + "SMS.ne4pg2_oQU480.F2010.eam-thetahy_ftype2_energy", + "SMS.ne4pg2_oQU480.F2010.eam-thetahy_ftype4", + "SMS.ne4pg2_oQU480.F2010.eam-thetanh_ftype0", + "SMS.ne4pg2_oQU480.F2010.eam-thetanh_ftype1", + "SMS.ne4pg2_oQU480.F2010.eam-thetanh_ftype2", + "SMS.ne4pg2_oQU480.F2010.eam-thetanh_ftype4", + "SMS.ne4pg2_oQU480.F2010.eam-thetahy_sl", + "ERS.ne4pg2_oQU480.F2010.eam-thetahy_ftype2", + "ERS.ne4pg2_oQU480.F2010.eam-thetanh_ftype2", ) }, "eam_theta_pg2" : { @@ -612,13 +640,21 @@ "e3sm_scream_v1_lowres" : { + "time" : "01:00:00", + "inherit" : ("e3sm_scream_mam4xx_v1_lowres"), + "tests" : ( + "ERP_D_Lh4.ne4_ne4.F2010-SCREAMv1.scream-output-preset-1", + "ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1.scream-output-preset-2", + "SMS_D_Ln9.ne4_ne4.F2010-SCREAMv1-noAero.scream-output-preset-3", + "ERP_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-output-preset-4", + "ERS_D_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-rad_frequency_2--scream-output-preset-5", + ) + }, + + "e3sm_scream_v1_dp-eamxx" : { "time" : "01:00:00", "tests" : ( - "ERP_D_Lh4.ne4_ne4.F2010-SCREAMv1", - "ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1", - "SMS_D_Ln9.ne4_ne4.F2010-SCREAMv1-noAero", - "ERP_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1", - "ERS_D_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-rad_frequency_2", + "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-DYCOMSrf01", # 225 phys cols, roughly size of ne2 ) }, @@ -626,27 +662,26 @@ # should be fast, so we limit it to low res and add some thread tests # specifically for mappy. "e3sm_scream_v1_at" : { - "inherit" : ("e3sm_scream_v1_lowres"), - "tests" : ("PET_Ln9_P32x2.ne4pg2_ne4pg2.F2010-SCREAMv1") + "inherit" : ("e3sm_scream_v1_lowres", "e3sm_scream_v1_dp-eamxx"), + "tests" : ("PET_Ln9_P32x2.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-output-preset-1") }, "e3sm_scream_v1_medres" : { "time" : "02:00:00", "tests" : ( # "SMS_D_Ln2.ne30_ne30.F2000-SCREAMv1-AQP1", # Uncomment once IC file for ne30 is ready - "ERS_Ln22.ne30_ne30.F2010-SCREAMv1.scream-internal_diagnostics_level", - "PEM_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1", - "ERS_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-small_kernels", - "ERP_Ln22.conusx4v1pg2_r05_oECv3.F2010-SCREAMv1-noAero.scream-bfbhash", + "ERS_Ln22.ne30_ne30.F2010-SCREAMv1.scream-internal_diagnostics_level--scream-output-preset-3", + "PEM_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-spa_remap--scream-output-preset-4", + "ERS_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5", + "ERP_Ln22.conusx4v1pg2_r05_oECv3.F2010-SCREAMv1-noAero.scream-bfbhash--scream-output-preset-6", ) }, + # Used to track performance "e3sm_scream_v1_hires" : { - "time" : "03:00:00", + "time" : "01:00:00", "tests" : ( - "SMS_D_Ln12.ne120_r0125_oRRS18to6v3.F2010-SCREAMv1", - "SMS_Ln12.ne120_ne120.F2010-SCREAMv1", -# "SMS_Ln12.ne120_r0125_oRRS18to6v3.F2000-SCREAMv1-AQP1", add when aquap 120 inputs available + "SMS_Ln300.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-perf_test--scream-output-preset-1" ) }, @@ -658,7 +693,7 @@ # Disable the two 111422-commented tests b/c they fail on pm-gpu and # we're not using MPASSI right now. #111422 "ERP_Ln22.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.atmlndactive-rtm_off", - "ERS_D_Ln22.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.atmlndactive-rtm_off", + "ERS_D_Ln22.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.atmlndactive-rtm_off--scream-output-preset-1", # "ERS_Ln22.ne30_oECv3.F2010-SCREAMv1-MPASSI.atmlndactive-rtm_off", #111422 "PEM_Ln90.ne30pg2_EC30to60E2r2.F2010-SCREAMv1-MPASSI", # "ERS_Ln22.ne30pg2_EC30to60E2r2.F2010-SCREAMv1-MPASSI.atmlndactive-rtm_off", @@ -682,6 +717,14 @@ ) }, + "e3sm_scream_mam4xx_v1_lowres" : { + "time" : "01:00:00", + "tests" : ( + "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-optics", + ) + }, + + "e3sm_gpuacc" : { "tests" : ( "SMS_Ld1.T62_oEC60to30v3.CMPASO-NYF", @@ -1003,4 +1046,3 @@ "e3sm_superbfb_atm", "e3sm_superbfb_wcycl"), }, } - diff --git a/components/cmake/build_model.cmake b/components/cmake/build_model.cmake index 16f1f039ebde..535081663a68 100644 --- a/components/cmake/build_model.cmake +++ b/components/cmake/build_model.cmake @@ -249,6 +249,11 @@ macro(build_model COMP_CLASS COMP_NAME) target_link_libraries(${TARGET_NAME} ${ITEM}) endforeach() + if (USE_MOAB) + target_link_libraries(${TARGET_NAME} ${MOAB_LIBRARIES}) + target_include_directories(${TARGET_NAME} PRIVATE ${MOAB_INCLUDE_DIRS}) + endif() + # Make sure we link blas/lapack if (NOT DEFINED ENV{SKIP_BLAS}) target_link_libraries(${TARGET_NAME} BLAS::BLAS LAPACK::LAPACK) diff --git a/components/cmake/modules/FindPIO.cmake b/components/cmake/modules/FindPIO.cmake index 7df8a9ba1062..0277918ac8e0 100644 --- a/components/cmake/modules/FindPIO.cmake +++ b/components/cmake/modules/FindPIO.cmake @@ -18,6 +18,9 @@ endif() if (PIO_VERSION STREQUAL 2) # This is a pio2 library set(PIOLIBS "${PIO_LIBDIR}/libpiof.a;${PIO_LIBDIR}/libpioc.a") + if (DEFINED ENV{ADIOS2_ROOT}) + list(APPEND PIOLIBS "${PIO_LIBDIR}/libadios2pio-nm-lib.a") + endif() else() # This is a pio1 library set(PIOLIBS "${PIO_LIBDIR}/libpio.a") diff --git a/components/data_comps/datm/cime_config/config_component.xml b/components/data_comps/datm/cime_config/config_component.xml index 779b930662ca..cb982f52fcff 100644 --- a/components/data_comps/datm/cime_config/config_component.xml +++ b/components/data_comps/datm/cime_config/config_component.xml @@ -10,12 +10,14 @@ This file may have atm desc entries. --> - Data driven ATM + Data driven ATM QIAN data set QIAN with water isotopes CRUNCEP data set CLM CRU NCEP v7 data set GSWP3v1 data set + Fifth generation ECMWF reanalysis + Fifth generation ECMWF reanalysis,6 hourly data MOSART test data set using older NLDAS data NLDAS2 regional 0.125 degree data set over the U.S. (25-53N, 235-293E). WARNING: Garbage data will be produced for runs extending beyond this regional domain. Coupler hist data set (in this mode, it is strongly recommended that the model domain and the coupler history forcing are on the same domain) @@ -43,13 +45,13 @@ char - CORE2_NYF,CORE2_IAF,CLM_QIAN,CLM_QIAN_WISO,CLM1PT,CLMCRUNCEP,CLMCRUNCEPv7,CLMGSWP3v1,CLMMOSARTTEST,CLMNLDAS2,CPLHIST,CORE_IAF_JRA,IAF_JRA_1p5,CORE_IAF_JRA_1p4_2018,CORE_RYF8485_JRA,CORE_RYF9091_JRA,CORE_RYF0304_JRA,CFSv2,CFSR + CORE2_NYF,CORE2_IAF,CLM_QIAN,CLM_QIAN_WISO,CLM1PT,CLMCRUNCEP,CLMCRUNCEPv7,CLMGSWP3v1,ELMERA5,ERA56HR,CLMMOSARTTEST,CLMNLDAS2,CPLHIST,CORE_IAF_JRA,IAF_JRA_1p5,CORE_IAF_JRA_1p4_2018,CORE_RYF8485_JRA,CORE_RYF9091_JRA,CORE_RYF0304_JRA,CFSv2,CFSR CORE2_NYF run_component_datm env_run.xml Mode for data atmosphere component. CORE2_NYF (CORE2 normal year forcing) are modes used in forcing prognostic ocean/sea-ice components. - CLM_QIAN, CLMCRUNCEP, CLMCRUNCEPv7, CLMGSWP3v1, CLMMOSARTTEST, CLMNLDAS2 and CLM1PT are modes using observational data for forcing prognostic land components. + CLM_QIAN, CLMCRUNCEP, CLMCRUNCEPv7, CLMGSWP3v1, ELMERA5,ERA56HR, CLMMOSARTTEST, CLMNLDAS2 and CLM1PT are modes using observational data for forcing prognostic land components. WARNING for CLMNLDAS2: This is a regional forcing dataset over the U.S. (25-53N, 235-293E). Garbage data will be produced for runs extending beyond this regional domain. WARNING for CLMGSWP3v1: Humidity is identically zero for last time step in Dec/2013 and all of 2014 so you should NOT use 2014 data (see cime issue #3653 -- https://github.com/ESMCI/cime/issues/3653). @@ -68,6 +70,8 @@ data (see cime issue #3653 -- https://github.com/ESMCI/cime/issues/3653). CLMCRUNCEP CLMCRUNCEPv7 CLMGSWP3v1 + ELMERA5 + ERA56HR CLMMOSARTTEST CLMNLDAS2 CLM1PT @@ -240,9 +244,13 @@ data (see cime issue #3653 -- https://github.com/ESMCI/cime/issues/3653). 1 1 $DATM_CLMNCEP_YR_START + 1 + 1 $DATM_CLMNCEP_YR_START $DATM_CLMNCEP_YR_START $DATM_CLMNCEP_YR_START + $DATM_CLMNCEP_YR_START + $DATM_CLMNCEP_YR_START run_component_datm env_run.xml @@ -286,6 +294,8 @@ data (see cime issue #3653 -- https://github.com/ESMCI/cime/issues/3653). 2005 2002 1958 + 1979 + 1979 run_component_datm env_run.xml @@ -330,6 +340,8 @@ data (see cime issue #3653 -- https://github.com/ESMCI/cime/issues/3653). 2003 2016 2020 + 1979 + 1979 run_component_datm env_run.xml diff --git a/components/data_comps/datm/cime_config/namelist_definition_datm.xml b/components/data_comps/datm/cime_config/namelist_definition_datm.xml index e033e9494645..529e79a03f68 100644 --- a/components/data_comps/datm/cime_config/namelist_definition_datm.xml +++ b/components/data_comps/datm/cime_config/namelist_definition_datm.xml @@ -36,6 +36,8 @@ CLMCRUNCEP = Run with the CLM CRU NCEP V4 ( default ) forcing valid from 1900 to 2010 (force CLM) CLMCRUNCEPv7 = Run with the CLM CRU NCEP V7 forcing valid from 1900 to 2010 (force CLM) CLMGSWP3v1 = Run with the CLM GSWP3 V1 forcing (force CLM) + ELMERA5 = Run with the ELM fifth generation ECMWF reanalysis from 1979 to present + ERA56HR = Run with the ELM fifth generation ECMWF reanalysis from 1979 to present CLMMOSARTTEST = Run with the CLM NLDAS data (force CLM) for testing MOSART CLMNLDAS2 = Run with the CLM NLDAS2 regional forcing valid from 1980 to 2018 (force CLM) CLM1PT = Run with supplied single point data (force CLM) @@ -103,6 +105,26 @@ CLMGSWP3v1.Solar CLMGSWP3v1.Precip CLMGSWP3v1.TPQW + + ELMERA5.msdrswrf # mean surface direct shortwave radiation flux + ELMERA5.msdfswrf # mean surface diffuse shortwave radiation flux + ELMERA5.mcpr # mean convective precipitation rate + ELMERA5.mlspr # mean large-scale precipitation rate + ELMERA5.t2m # temperature at 2 m + ELMERA5.sp # surface pressure + ELMERA5.d2m # dew point temperature at 2 m + ELMERA5.w10 # wind speed at 10 m + ELMERA5.msdwlwrf # mean surface downward longwave radiation flux + + ERA56HR.msdrswrf # mean surface direct shortwave radiation flux + ERA56HR.msdfswrf # mean surface diffuse shortwave radiation flux + ERA56HR.mcpr # mean convective precipitation rate + ERA56HR.mlspr # mean large-scale precipitation rate + ERA56HR.t2m # temperature at 2 m + ERA56HR.sp # surface pressure + ERA56HR.d2m # dew point temperature at 2 m + ERA56HR.w10 # wind speed at 10 m + ERA56HR.msdwlwrf # mean surface downward longwave radiation flux CLMMOSARTTEST @@ -202,6 +224,8 @@ CLMCRUNCEP.Solar,CLMCRUNCEP.Precip,CLMCRUNCEP.TPQW CLMCRUNCEPv7.Solar,CLMCRUNCEPv7.Precip,CLMCRUNCEPv7.TPQW CLMGSWP3v1.Solar,CLMGSWP3v1.Precip,CLMGSWP3v1.TPQW + ELMERA5.msdrswrf,ELMERA5.msdfswrf,ELMERA5.mcpr,ELMERA5.mlspr,ELMERA5.t2m,ELMERA5.sp,ELMERA5.d2m,ELMERA5.w10,ELMERA5.msdwlwrf + ERA56HR.msdrswrf,ERA56HR.msdfswrf,ERA56HR.mcpr,ERA56HR.mlspr,ERA56HR.t2m,ERA56HR.sp,ERA56HR.d2m,ERA56HR.w10,ERA56HR.msdwlwrf CLMMOSARTTEST CLMNLDAS2.Solar,CLMNLDAS2.Precip,CLMNLDAS2.TPQW CORE2_NYF.GISS,CORE2_NYF.GXGXS,CORE2_NYF.NCEP @@ -234,6 +258,8 @@ $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.cruncep_qianFill.0.5d.v7.c160715 $DIN_LOC_ROOT/share/domains/domain.clm $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.GSWP3.0.5d.v1.c170516 + $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.ERA.0.25d.v5.c180614 + $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.ERA.6HRLY.0.25d.v5.c180614 $DIN_LOC_ROOT/share/domains/domain.clm $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.cruncep_qianFill.0.5d.V5.c140715 $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.GSWP3.0.5d.v1.c170516 @@ -302,6 +328,8 @@ domain.lnd.360x720.130305.nc domain.lnd.360x720_gswp3.0v1.c170606.nc domain.lnd.360x720_gswp3.0v1.c170606.nc + domain.lnd.era5_721x1440_rdrlat_EC30to60E2r2.221115.nc + domain.lnd.era5_721x1440_rdrlat_EC30to60E2r2.221115.nc domain.lnd.nldas2_0224x0464_c110415.nc domain.lnd.0.125nldas2_0.125nldas2.190410.nc nyf.giss.T62.051007.nc @@ -478,6 +506,24 @@ $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.GSWP3.0.5d.v1.c170516/Solar3Hrly $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.GSWP3.0.5d.v1.c170516/Precip3Hrly $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.GSWP3.0.5d.v1.c170516/TPHWL3Hrly + $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.ERA.0.25d.v5.c180614/swdn + $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.ERA.0.25d.v5.c180614/swdn + $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.ERA.0.25d.v5.c180614/prec + $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.ERA.0.25d.v5.c180614/prec + $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.ERA.0.25d.v5.c180614/tbot + $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.ERA.0.25d.v5.c180614/pbot + $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.ERA.0.25d.v5.c180614/tdew + $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.ERA.0.25d.v5.c180614/wind + $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.ERA.0.25d.v5.c180614/lwdn + $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.ERA.6HRLY.0.25d.v5.c180614/swdn + $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.ERA.6HRLY.0.25d.v5.c180614/swdn + $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.ERA.6HRLY.0.25d.v5.c180614/prec + $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.ERA.6HRLY.0.25d.v5.c180614/prec + $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.ERA.6HRLY.0.25d.v5.c180614/tbot + $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.ERA.6HRLY.0.25d.v5.c180614/pbot + $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.ERA.6HRLY.0.25d.v5.c180614/tdew + $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.ERA.6HRLY.0.25d.v5.c180614/wind + $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.ERA.6HRLY.0.25d.v5.c180614/lwdn $DIN_LOC_ROOT/atm/datm7/NLDAS $DIN_LOC_ROOT/atm/datm7/atm_forcing.datm7.NLDAS2.0.125d.v1/Solar $DIN_LOC_ROOT/atm/datm7/atm_forcing.datm7.NLDAS2.0.125d.v1/Precip @@ -553,6 +599,24 @@ clmforc.GSWP3.c2011.0.5x0.5.Solr.%ym.nc clmforc.GSWP3.c2011.0.5x0.5.Prec.%ym.nc clmforc.GSWP3.c2011.0.5x0.5.TPQWL.%ym.nc + elmforc.ERA5.c2018.0.25d.msdrswrf.%ym.nc + elmforc.ERA5.c2018.0.25d.msdfswrf.%ym.nc + elmforc.ERA5.c2018.0.25d.mcpr.%ym.nc + elmforc.ERA5.c2018.0.25d.mlspr.%ym.nc + elmforc.ERA5.c2018.0.25d.t2m.%ym.nc + elmforc.ERA5.c2018.0.25d.sp.%ym.nc + elmforc.ERA5.c2018.0.25d.d2m.%ym.nc + elmforc.ERA5.c2018.0.25d.w10.%ym.nc + elmforc.ERA5.c2018.0.25d.msdwlwrf.%ym.nc + elmforc.ERA5.c2018.0.25d.msdrswrf.%ym.nc + elmforc.ERA5.c2018.0.25d.msdfswrf.%ym.nc + elmforc.ERA5.c2018.0.25d.mcpr.%ym.nc + elmforc.ERA5.c2018.0.25d.mlspr.%ym.nc + elmforc.ERA5.c2018.0.25d.t2m.%ym.nc + elmforc.ERA5.c2018.0.25d.sp.%ym.nc + elmforc.ERA5.c2018.0.25d.d2m.%ym.nc + elmforc.ERA5.c2018.0.25d.w10.%ym.nc + elmforc.ERA5.c2018.0.25d.msdwlwrf.%ym.nc clmforc.nldas.%ym.nc ctsmforc.NLDAS2.0.125d.v1.Solr.%ym.nc ctsmforc.NLDAS2.0.125d.v1.Prec.%ym.nc @@ -1523,6 +1587,60 @@ PSRF pbot FLDS lwdn + + msdrswrf swdndr + + + msdfswrf swdndf + + + mcpr precc + + + mlspr precl + + + t2m tbot + + + sp pbot + + + d2m tdew + + + w10 wind + + + msdwlwrf lwdn + + + msdrswrf swdndr + + + msdfswrf swdndf + + + mcpr precc + + + mlspr precl + + + t2m tbot + + + sp pbot + + + d2m tdew + + + w10 wind + + + msdwlwrf lwdn + TBOT tbot WIND wind @@ -1793,13 +1911,15 @@ $DATM_CLMNCEP_YR_ALIGN $DATM_CLMNCEP_YR_ALIGN $DATM_CLMNCEP_YR_ALIGN + $DATM_CLMNCEP_YR_ALIGN + $DATM_CLMNCEP_YR_ALIGN $DATM_CLMNCEP_YR_ALIGN $DATM_CLMNCEP_YR_ALIGN 1 1 1 - 1 - 1 + $DATM_CLMNCEP_YR_ALIGN + $DATM_CLMNCEP_YR_ALIGN 1 1850 @@ -1843,6 +1963,8 @@ $DATM_CLMNCEP_YR_START $DATM_CLMNCEP_YR_START $DATM_CLMNCEP_YR_START + $DATM_CLMNCEP_YR_START + $DATM_CLMNCEP_YR_START $DATM_CLMNCEP_YR_START $DATM_CLMNCEP_YR_START 1 @@ -1914,6 +2036,8 @@ $DATM_CLMNCEP_YR_END $DATM_CLMNCEP_YR_END $DATM_CLMNCEP_YR_END + $DATM_CLMNCEP_YR_END + $DATM_CLMNCEP_YR_END $DATM_CLMNCEP_YR_END $DATM_CLMNCEP_YR_END 1 @@ -1988,6 +2112,16 @@ 900 0 0 + -3600 + -3600 + -60 + -60 + -60 + -21600 + -21600 + -60 + -60 + -60 @@ -2054,6 +2188,8 @@ NULL CLMNCEP + CLMNCEP + CLMNCEP CORE2_NYF CORE2_IAF CORE_IAF_JRA @@ -2093,7 +2229,9 @@ valid values: 'copy','spval','nn','nnoni','nnonj' - nn + nn + copy + copy @@ -2248,6 +2386,16 @@ nearest coszen nearest + coszen + coszen + upper + linear + linear + coszen + coszen + upper + linear + linear coszen nearest nearest @@ -2340,6 +2488,25 @@ 3.0 3.0 3.0 + 2.5 + 2.5 + 2.5 + 2.5 + 2.5 + 2.5 + 2.5 + 2.5 + 2.5 + 2.5 + 2.5 + 2.5 + 2.5 + 2.5 + 2.5 + 2.5 + 2.5 + 2.5 + diff --git a/components/data_comps/datm/src/atm_comp_mct.F90 b/components/data_comps/datm/src/atm_comp_mct.F90 index a5f2855ae63e..4970ac726d79 100644 --- a/components/data_comps/datm/src/atm_comp_mct.F90 +++ b/components/data_comps/datm/src/atm_comp_mct.F90 @@ -18,6 +18,11 @@ module atm_comp_mct use datm_shr_mod , only: presaero use seq_flds_mod , only: seq_flds_a2x_fields, seq_flds_x2a_fields +#ifdef HAVE_MOAB + use seq_comm_mct, only : mphaid ! iMOAB app id for phys atm; comp atm is 5, phys 5+200 + use iso_c_binding + use iMOAB , only: iMOAB_RegisterApplication +#endif ! !PUBLIC TYPES: implicit none private ! except @@ -80,6 +85,9 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) real(R8) :: orbObliqr ! orb obliquity (radians) real(R8) :: nextsw_cday ! calendar of next atm sw +#ifdef HAVE_MOAB + integer(IN) :: ATM_PHYS_CID ! used to create a new comp id for phys atm; 200+ compid +#endif !--- formats --- character(*), parameter :: F00 = "('(datm_comp_init) ',8a)" integer(IN) , parameter :: master_task=0 ! task number of master task @@ -164,6 +172,18 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) ! Initialize datm !---------------------------------------------------------------------------- + +#ifdef HAVE_MOAB + ATM_PHYS_CID = 200 + compid + ierr = iMOAB_RegisterApplication(trim("DATM")//C_NULL_CHAR, mpicom, ATM_PHYS_CID, mphaid) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in registering data atm comp' + call shr_sys_abort(subname//' ERROR in registering data atm comp') + endif + ! send path of atm domain file to MOAB coupler. Note that here we may have the land domain in some cases? + call seq_infodata_PutData( infodata, atm_mesh=SDATM%domainFile) +#endif + call datm_comp_init(Eclock, x2a, a2x, & seq_flds_x2a_fields, seq_flds_a2x_fields, & SDATM, gsmap, ggrid, mpicom, compid, my_task, master_task, & diff --git a/components/data_comps/datm/src/datm_comp_mod.F90 b/components/data_comps/datm/src/datm_comp_mod.F90 index 3ad18c4a238b..6f0adeedcd08 100644 --- a/components/data_comps/datm/src/datm_comp_mod.F90 +++ b/components/data_comps/datm/src/datm_comp_mod.F90 @@ -32,6 +32,9 @@ module datm_comp_mod use datm_shr_mod , only: iradsw ! namelist input use datm_shr_mod , only: nullstr +#ifdef HAVE_MOAB + use iso_c_binding +#endif ! !PUBLIC TYPES: implicit none @@ -212,6 +215,13 @@ subroutine datm_comp_init(Eclock, x2a, a2x, & scmMode, scmlat, scmlon, & orbEccen, orbMvelpp, orbLambm0, orbObliqr, phase, nextsw_cday) +#ifdef HAVE_MOAB + use iMOAB, only: iMOAB_DefineTagStorage, iMOAB_GetDoubleTagStorage, & + iMOAB_SetIntTagStorage, iMOAB_SetDoubleTagStorage, & + iMOAB_ResolveSharedEntities, iMOAB_CreateVertices, & + iMOAB_GetMeshInfo, iMOAB_UpdateMeshInfo, iMOAB_WriteMesh + use seq_comm_mct, only : mphaid ! iMOAB app id for phys atm; comp atm is 5, phys 5+200 +#endif ! !DESCRIPTION: initialize data atm model implicit none @@ -258,6 +268,19 @@ subroutine datm_comp_init(Eclock, x2a, a2x, & character(CL) :: calendar ! calendar type character(CL) :: flds_strm +#ifdef HAVE_MOAB + character*400 tagname + real(R8) latv, lonv + integer iv, tagindex, ilat, ilon, ierr !, arrsize, nfields + real(R8), allocatable, target :: data(:) + integer(IN), pointer :: idata(:) ! temporary + real(r8), dimension(:), allocatable :: moab_vert_coords ! temporary + !real(R8), allocatable, target :: vtags_zero(:, :) +#ifdef MOABDEBUG + character*100 outfile, wopts +#endif +#endif + !--- formats --- character(*), parameter :: F00 = "('(datm_comp_init) ',8a)" character(*), parameter :: F0L = "('(datm_comp_init) ',a, l2)" @@ -347,6 +370,110 @@ subroutine datm_comp_init(Eclock, x2a, a2x, & call shr_dmodel_rearrGGrid(SDATM%grid, ggrid, gsmap, rearr, mpicom) call t_stopf('datm_initmctdom') +#ifdef HAVE_MOAB + ilat = mct_aVect_indexRA(ggrid%data,'lat') + ilon = mct_aVect_indexRA(ggrid%data,'lon') + allocate(moab_vert_coords(lsize*3)) + do iv = 1, lsize + lonv = ggrid%data%rAttr(ilon, iv) * SHR_CONST_PI/180. + latv = ggrid%data%rAttr(ilat, iv) * SHR_CONST_PI/180. + moab_vert_coords(3*iv-2)=COS(latv)*COS(lonv) + moab_vert_coords(3*iv-1)=COS(latv)*SIN(lonv) + moab_vert_coords(3*iv )=SIN(latv) + enddo + + ! create the vertices with coordinates from MCT domain + ierr = iMOAB_CreateVertices(mphaid, lsize*3, 3, moab_vert_coords) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to create MOAB vertices in land model') + + tagname='GLOBAL_ID'//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mphaid, tagname, & + 0, & ! dense, integer + 1, & ! number of components + tagindex ) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to retrieve GLOBAL_ID tag ') + + ! get list of global IDs for Dofs + call mct_gsMap_orderedPoints(gsMap, my_task, idata) + + ierr = iMOAB_SetIntTagStorage ( mphaid, tagname, lsize, & + 0, & ! vertex type + idata) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to set GLOBAL_ID tag ') + + ierr = iMOAB_ResolveSharedEntities( mphaid, lsize, idata ); + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to resolve shared entities') + + deallocate(moab_vert_coords) + deallocate(idata) + + ierr = iMOAB_UpdateMeshInfo( mphaid ) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to update mesh info ') + + allocate(data(lsize)) + ierr = iMOAB_DefineTagStorage( mphaid, "area:aream:frac:mask"//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to create tag: area:aream:frac:mask' ) + + data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'area'),:) + tagname='area'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mphaid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to get area tag ') + + ! set the same data for aream (model area) as area + ! data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'aream'),:) + tagname='aream'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mphaid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to set aream tag ') + + data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'mask'),:) + tagname='mask'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mphaid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to set mask tag ') + + data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'frac'),:) + tagname='frac'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mphaid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to set frac tag ') + + deallocate(data) + + ! define tags + ierr = iMOAB_DefineTagStorage( mphaid, trim(seq_flds_x2a_fields)//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to create seq_flds_x2a_fields tags ') + + ierr = iMOAB_DefineTagStorage( mphaid, trim(seq_flds_a2x_fields)//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to create seq_flds_a2x_fields tags ') + +#endif !---------------------------------------------------------------------------- ! Initialize MCT attribute vectors !---------------------------------------------------------------------------- @@ -479,6 +606,24 @@ subroutine datm_comp_init(Eclock, x2a, a2x, & yc(:) = ggrid%data%rAttr(klat,:) call t_stopf('datm_initmctavs') +#ifdef HAVE_MOAB + ierr = iMOAB_DefineTagStorage( mphaid, trim(flds_strm)//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to create flds_strm tags ') +#ifdef MOABDEBUG + ! debug test + outfile = 'AtmDataMesh.h5m'//C_NULL_CHAR + wopts = ';PARALLEL=WRITE_PART'//C_NULL_CHAR ! + ! write out the mesh file to disk + ierr = iMOAB_WriteMesh(mphaid, trim(outfile), trim(wopts)) + if (ierr .ne. 0) then + call shr_sys_abort(subname//' ERROR in writing data mesh atm ') + endif +#endif +#endif !---------------------------------------------------------------------------- ! Read restart @@ -580,7 +725,14 @@ subroutine datm_comp_run(EClock, x2a, a2x, & nextsw_cday, case_name) ! !DESCRIPTION: run method for datm model - +#ifdef HAVE_MOAB + use seq_flds_mod , only: seq_flds_a2x_fields ! this should not be an argument in datm_comp_init + use seq_comm_mct, only : mphaid ! + use seq_flds_mod, only: moab_set_tag_from_av +#ifdef MOABDEBUG + use iMOAB, only: iMOAB_WriteMesh +#endif +#endif implicit none ! !INPUT/OUTPUT PARAMETERS: @@ -627,7 +779,18 @@ subroutine datm_comp_run(EClock, x2a, a2x, & !--- temporaries real(R8) :: uprime,vprime,swndr,swndf,swvdr,swvdf,ratio_rvrf real(R8) :: tbot,pbot,rtmp,vp,ea,e,qsat,frac +#ifdef HAVE_MOAB + real(R8), allocatable, target :: datam(:) + type(mct_list) :: temp_list + integer :: size_list, index_list + type(mct_string) :: mctOStr ! + character*400 tagname, mct_field +#ifdef MOABDEBUG + integer :: cur_datm_stepno, ierr + character*100 outfile, wopts, lnum +#endif +#endif character(*), parameter :: F00 = "('(datm_comp_run) ',8a)" character(*), parameter :: F04 = "('(datm_comp_run) ',2a,2i8,'s')" character(*), parameter :: subName = "(datm_comp_run) " @@ -675,6 +838,8 @@ subroutine datm_comp_run(EClock, x2a, a2x, & allocate(count_av(SDATM%nstreams)) allocate(count_st(SDATM%nstreams)) end if + + do n = 1,SDATM%nstreams if (firstcall) then call shr_dmodel_translate_list(SDATM%avs(n),a2x,& @@ -685,6 +850,7 @@ subroutine datm_comp_run(EClock, x2a, a2x, & ilist_av(n),olist_av(n),rearr) end if enddo + do n = 1,SDATM%nstreams if (firstcall) then call shr_dmodel_translate_list(SDATM%avs(n),avstrm,& @@ -696,7 +862,6 @@ subroutine datm_comp_run(EClock, x2a, a2x, & end if enddo call t_stopf('datm_scatter') - !------------------------------------------------- ! Determine data model behavior based on the mode !------------------------------------------------- @@ -1110,7 +1275,6 @@ subroutine datm_comp_run(EClock, x2a, a2x, & !---------------------------------------------------------- ! bias correction / anomaly forcing ( end block ) !---------------------------------------------------------- - !-------------------- ! Write restart !-------------------- @@ -1145,6 +1309,33 @@ subroutine datm_comp_run(EClock, x2a, a2x, & ! Reset shr logging to original values !---------------------------------------------------------------------------- +#ifdef HAVE_MOAB + lsize = mct_avect_lsize(a2x) ! is it the same as mct_avect_lsize(avstrm) ? + allocate(datam(lsize)) ! + call mct_list_init(temp_list ,seq_flds_a2x_fields) + size_list=mct_list_nitem (temp_list) + do index_list = 1, size_list + call mct_list_get(mctOStr,index_list,temp_list) + mct_field = mct_string_toChar(mctOStr) + tagname= trim(mct_field)//C_NULL_CHAR + call moab_set_tag_from_av(tagname, a2x, index_list, mphaid, datam, lsize) ! loop over all a2x fields, not just a few + enddo + call mct_list_clean(temp_list) + deallocate(datam) ! maybe we should keep it around, deallocate at the final only? + +#ifdef MOABDEBUG + call seq_timemgr_EClockGetData( EClock, stepno=cur_datm_stepno ) + write(lnum,"(I0.2)")cur_datm_stepno + outfile = 'datm_comp_run_'//trim(lnum)//'.h5m'//C_NULL_CHAR + wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR + ierr = iMOAB_WriteMesh(mphaid, outfile, wopts) + if (ierr > 0 ) then + write(logunit,*) 'Failed to write data atm component state ' + endif +#endif + +#endif + call t_startf('datm_run2') if (my_task == master_task) then write(logunit,F04) trim(myModelName),': model date ', CurrentYMD,CurrentTOD diff --git a/components/data_comps/docn/src/docn_comp_mod.F90 b/components/data_comps/docn/src/docn_comp_mod.F90 index beefa9901b0e..e692882c9db1 100644 --- a/components/data_comps/docn/src/docn_comp_mod.F90 +++ b/components/data_comps/docn/src/docn_comp_mod.F90 @@ -31,6 +31,12 @@ module docn_comp_mod use docn_shr_mod , only: sst_constant_value ! namelist input use docn_shr_mod , only: nullstr +#ifdef HAVE_MOAB +! character(1024) :: domain_file ! file containing domain info (set my input) + use seq_comm_mct, only: mpoid ! iMOAB pid for ocean mesh on component pes + + use iso_c_binding +#endif ! !PUBLIC TYPES: implicit none @@ -75,6 +81,10 @@ module docn_comp_mod integer(IN), pointer :: imask(:) real(R8), pointer :: xc(:), yc(:) ! arryas of model latitudes and longitudes +#ifdef HAVE_MOAB + integer :: mdpoid ! data: ocean local component +#endif + !-------------------------------------------------------------------------- integer(IN) , parameter :: ktrans = 8 character(12) , parameter :: avifld(1:ktrans) = & @@ -90,6 +100,18 @@ module docn_comp_mod CONTAINS !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +#ifdef HAVE_MOAB + SUBROUTINE errorout(ierr, message) + integer ierr + character*(*) message + if (ierr.ne.0) then + print *, message + call exit (1) + end if + return + end subroutine +#endif + !=============================================================================== subroutine docn_comp_init(Eclock, x2o, o2x, & seq_flds_x2o_fields, seq_flds_o2x_fields, & @@ -100,6 +122,13 @@ subroutine docn_comp_init(Eclock, x2o, o2x, & ! !DESCRIPTION: initialize docn model use pio , only : iosystem_desc_t use shr_pio_mod, only : shr_pio_getiosys, shr_pio_getiotype +#ifdef HAVE_MOAB +#include "moab/MOABConfig.h" + use iMOAB, only: iMOAB_DefineTagStorage, iMOAB_GetDoubleTagStorage, & + iMOAB_SetIntTagStorage, iMOAB_SetDoubleTagStorage, & + iMOAB_ResolveSharedEntities, iMOAB_CreateVertices, & + iMOAB_GetMeshInfo, iMOAB_UpdateMeshInfo +#endif implicit none ! !INPUT/OUTPUT PARAMETERS: @@ -139,6 +168,15 @@ subroutine docn_comp_init(Eclock, x2o, o2x, & logical :: write_restart=.false. type(iosystem_desc_t), pointer :: ocn_pio_subsystem +#ifdef HAVE_MOAB + character*400 tagname + real(R8) latv, lonv + integer iv, tagindex + real(R8), allocatable, target :: data(:) + integer(IN), pointer :: idata(:) ! temporary + real(r8), dimension(:), allocatable :: moab_vert_coords ! temporary +#endif + !--- formats --- character(*), parameter :: F00 = "('(docn_comp_init) ',8a)" character(*), parameter :: F0L = "('(docn_comp_init) ',a, l2)" @@ -147,7 +185,7 @@ subroutine docn_comp_init(Eclock, x2o, o2x, & character(*), parameter :: F03 = "('(docn_comp_init) ',a,i8,a)" character(*), parameter :: F04 = "('(docn_comp_init) ',2a,2i8,'s')" character(*), parameter :: F05 = "('(docn_comp_init) ',a,2f10.4)" - character(*), parameter :: F06 = "('(docn_comp_init) ',a,f10.4)" + character(*), parameter :: F06 = "('(docn_comp_init) ',a,5i8)" character(*), parameter :: F90 = "('(docn_comp_init) ',73('='))" character(*), parameter :: F91 = "('(docn_comp_init) ',73('-'))" character(*), parameter :: subName = "(docn_comp_init) " @@ -279,13 +317,6 @@ subroutine docn_comp_init(Eclock, x2o, o2x, & kmask = mct_aVect_indexRA(ggrid%data,'mask') imask(:) = nint(ggrid%data%rAttr(kmask,:)) - kfrac = mct_aVect_indexRA(ggrid%data,'frac') - - ksomask = mct_aVect_indexRA(o2x,'So_omask', perrwith='quiet') - if (ksomask /= 0) then - o2x%rAttr(ksomask, :) = ggrid%data%rAttr(kfrac,:) - end if - index_lon = mct_aVect_indexRA(ggrid%data,'lon') xc(:) = ggrid%data%rAttr(index_lon,:) @@ -294,6 +325,115 @@ subroutine docn_comp_init(Eclock, x2o, o2x, & call t_stopf('docn_initmctavs') +#ifdef HAVE_MOAB + + allocate(moab_vert_coords(lsize*3)) + do iv = 1, lsize + lonv = xc(iv) * SHR_CONST_PI/180. + latv = yc(iv) * SHR_CONST_PI/180. + moab_vert_coords(3*iv-2)=COS(latv)*COS(lonv) + moab_vert_coords(3*iv-1)=COS(latv)*SIN(lonv) + moab_vert_coords(3*iv )=SIN(latv) + enddo + + ! create the vertices with coordinates from MCT domain + ierr = iMOAB_CreateVertices(mpoid, lsize*3, 3, moab_vert_coords) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to create MOAB vertices in land model') + + tagname='GLOBAL_ID'//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mpoid, tagname, & + 0, & ! dense, integer + 1, & ! number of components + tagindex ) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to retrieve GLOBAL_ID tag ') + + ! get list of global IDs for Dofs + call mct_gsMap_orderedPoints(gsMap, my_task, idata) + + ierr = iMOAB_SetIntTagStorage ( mpoid, tagname, lsize, & + 0, & ! vertex type + idata) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to set GLOBAL_ID tag ') + + ierr = iMOAB_ResolveSharedEntities( mpoid, lsize, idata ); + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to resolve shared entities') + + deallocate(moab_vert_coords) + deallocate(idata) + + ierr = iMOAB_UpdateMeshInfo( mpoid ) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to update mesh info ') + + allocate(data(lsize)) + ierr = iMOAB_DefineTagStorage( mpoid, "area:aream:frac:mask"//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call errorout(ierr, 'Error: fail to create tag: area:aream:frac:mask' ) + + data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'area'),:) + tagname='area'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mpoid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call errorout(ierr, 'Error: fail to get area tag ') + + ! set the same data for aream (model area) as area + ! data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'aream'),:) + tagname='aream'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mpoid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call errorout(ierr, 'Error: fail to set aream tag ') + + data(:) = ggrid%data%rAttr(kmask,:) + tagname='mask'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mpoid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call errorout(ierr, 'Error: fail to set mask tag ') + + data(:) = ggrid%data%rAttr(kfrac,:) + tagname='frac'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mpoid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call errorout(ierr, 'Error: fail to set frac tag ') + + deallocate(data) + + ! define tags + ierr = iMOAB_DefineTagStorage( mpoid, trim(seq_flds_x2o_fields)//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call errorout(ierr, 'Error: fail to create seq_flds_x2o_fields tags ') + + ierr = iMOAB_DefineTagStorage( mpoid, trim(seq_flds_o2x_fields)//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call errorout(ierr, 'Error: fail to create seq_flds_o2x_fields tags ') + + ierr = iMOAB_DefineTagStorage( mpoid, trim(flds_strm)//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call errorout(ierr, 'Error: fail to create flds_strm tags ') +#endif !---------------------------------------------------------------------------- ! Read restart !---------------------------------------------------------------------------- @@ -379,14 +519,19 @@ subroutine docn_comp_init(Eclock, x2o, o2x, & end subroutine docn_comp_init - !=============================================================================== - subroutine docn_comp_run(EClock, x2o, o2x, & SDOCN, gsmap, ggrid, mpicom, compid, my_task, master_task, & inst_suffix, logunit, read_restart, write_restart, & target_ymd, target_tod, case_name) ! !DESCRIPTION: run method for docn model +#ifdef HAVE_MOAB + use iMOAB, only: iMOAB_GetMeshInfo, & + iMOAB_SetDoubleTagStorage, & + iMOAB_WriteMesh + use seq_flds_mod, only: moab_set_tag_from_av +#endif + implicit none ! !INPUT/OUTPUT PARAMETERS: @@ -423,6 +568,19 @@ subroutine docn_comp_run(EClock, x2o, o2x, & real(R8), parameter :: & swp = 0.67_R8*(exp((-1._R8*shr_const_zsrflyr) /1.0_R8)) + 0.33_R8*exp((-1._R8*shr_const_zsrflyr)/17.0_R8) +#ifdef HAVE_MOAB + integer :: ierr ! error code + integer :: kgg + character*100 tagname + integer tagindex + real(R8), allocatable, target :: data(:) +#ifdef MOABDEBUG + integer :: cur_docn_stepno + character*100 outfile, wopts, lnum +#endif + +#endif + character(*), parameter :: F00 = "('(docn_comp_run) ',8a)" character(*), parameter :: F01 = "('(docn_comp_run) ',a, i7,2x,i5,2x,i5,2x,d21.14)" character(*), parameter :: F04 = "('(docn_comp_run) ',2a,2i8,'s')" @@ -486,6 +644,8 @@ subroutine docn_comp_run(EClock, x2o, o2x, & ! Determine data model behavior based on the mode !------------------------------------------------- + if (my_task .EQ. master_task) & + write(logunit,*) "DOCN datamode case = ", trim(datamode) call t_startf('docn_datamode') select case (trim(datamode)) @@ -647,10 +807,70 @@ subroutine docn_comp_run(EClock, x2o, o2x, & call t_stopf('docn_datamode') +#ifdef HAVE_MOAB + + allocate(data(lsize)) + data(:) = 0.0 + + ! set dense double tags on vertices of the temporary DOCN app + ! first set o2x data + call moab_set_tag_from_av('So_t'//C_NULL_CHAR, o2x, kt, mpoid, data, lsize) + + call moab_set_tag_from_av('So_s'//C_NULL_CHAR, o2x, ks, mpoid, data, lsize) + + call moab_set_tag_from_av( 'So_u'//C_NULL_CHAR, o2x, ku, mpoid, data, lsize) + + call moab_set_tag_from_av( 'So_v'//C_NULL_CHAR, o2x, kv, mpoid, data, lsize) + + call moab_set_tag_from_av( 'So_dhdx'//C_NULL_CHAR, o2x, kdhdx, mpoid, data, lsize) + + call moab_set_tag_from_av( 'So_dhdy'//C_NULL_CHAR, o2x, kdhdy, mpoid, data, lsize) + + call moab_set_tag_from_av( 'Fioo_q'//C_NULL_CHAR, o2x, kq, mpoid, data, lsize) + + if (kswp /= 0) then + call moab_set_tag_from_av( 'So_fswpen'//C_NULL_CHAR, o2x, kswp, mpoid, data, lsize) + endif + + ! next set x2o data + call moab_set_tag_from_av( 'Foxx_swnet'//C_NULL_CHAR, x2o, kswnet, mpoid, data, lsize) + + call moab_set_tag_from_av( 'Foxx_lwup'//C_NULL_CHAR, x2o, klwup, mpoid, data, lsize) + + call moab_set_tag_from_av( 'Foxx_sen'//C_NULL_CHAR, x2o, ksen, mpoid, data, lsize) + + call moab_set_tag_from_av( 'Foxx_lat'//C_NULL_CHAR, x2o, klat, mpoid, data, lsize) + + call moab_set_tag_from_av( 'Foxx_rofi'//C_NULL_CHAR, x2o, krofi, mpoid, data, lsize) + + call moab_set_tag_from_av( 'Faxa_lwdn'//C_NULL_CHAR, x2o, klwdn, mpoid, data, lsize) + + call moab_set_tag_from_av( 'Faxa_snow'//C_NULL_CHAR, x2o, ksnow, mpoid, data, lsize) + + call moab_set_tag_from_av( 'Fioi_melth'//C_NULL_CHAR, x2o, kmelth, mpoid, data, lsize) + + ! next set avstrm data + call moab_set_tag_from_av( 'strm_h'//C_NULL_CHAR, avstrm, kh, mpoid, data, lsize) + + call moab_set_tag_from_av( 'strm_qbot'//C_NULL_CHAR, avstrm, kqbot, mpoid, data, lsize) + + +#ifdef MOABDEBUG + call seq_timemgr_EClockGetData( EClock, stepno=cur_docn_stepno ) + write(lnum,"(I0.2)")cur_docn_stepno + outfile = 'docn_comp_run_'//trim(lnum)//'.h5m'//C_NULL_CHAR + wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR + ierr = iMOAB_WriteMesh(mpoid, outfile, wopts) + if (ierr > 0 ) then + write(logunit,*) 'Failed to write ocean component state ' + endif +#endif + +#endif + !---------------------------------------------------------- ! Debug output !---------------------------------------------------------- - if (dbug > 0 .and. my_task == master_task) then do n = 1,lsize write(logunit,F01)'import: ymd,tod,n,Foxx_swnet = ', target_ymd, target_tod, n, x2o%rattr(kswnet,n) diff --git a/components/data_comps/docn/src/ocn_comp_mct.F90 b/components/data_comps/docn/src/ocn_comp_mct.F90 index 22c095b3b57b..6b029981438f 100644 --- a/components/data_comps/docn/src/ocn_comp_mct.F90 +++ b/components/data_comps/docn/src/ocn_comp_mct.F90 @@ -18,6 +18,11 @@ module ocn_comp_mct use docn_shr_mod , only: docn_shr_read_namelists use seq_flds_mod , only: seq_flds_x2o_fields, seq_flds_o2x_fields +#ifdef HAVE_MOAB + use seq_comm_mct, only: mpoid ! iMOAB pid for ocean mesh on component pes + use iso_c_binding +#endif + ! !PUBLIC TYPES: implicit none private ! except @@ -53,6 +58,10 @@ module ocn_comp_mct !=============================================================================== subroutine ocn_init_mct( EClock, cdata, x2o, o2x, NLFilename ) +#ifdef HAVE_MOAB + use iMOAB, only: iMOAB_RegisterApplication +#endif + ! !DESCRIPTION: initialize docn model implicit none @@ -156,6 +165,17 @@ subroutine ocn_init_mct( EClock, cdata, x2o, o2x, NLFilename ) ! Initialize docn !---------------------------------------------------------------------------- + +#ifdef HAVE_MOAB + ierr = iMOAB_RegisterApplication(trim("DOCN")//C_NULL_CHAR, mpicom, compid, mpoid) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in registering data ocn comp' + call shr_sys_abort(subname//' ERROR in registering data ocn comp') + endif + ! send path of ocean domain file to MOAB coupler. + call seq_infodata_PutData( infodata, ocn_domain=SDOCN%domainFile) +#endif + call docn_comp_init(Eclock, x2o, o2x, & seq_flds_x2o_fields, seq_flds_o2x_fields, & SDOCN, gsmap, ggrid, mpicom, compid, my_task, master_task, & diff --git a/components/data_comps/drof/cime_config/namelist_definition_drof.xml b/components/data_comps/drof/cime_config/namelist_definition_drof.xml index 369e23965189..c4139552c706 100644 --- a/components/data_comps/drof/cime_config/namelist_definition_drof.xml +++ b/components/data_comps/drof/cime_config/namelist_definition_drof.xml @@ -205,10 +205,10 @@ RAF_9091.JRA.v1.3.runoff.180404.nc RAF_0304.JRA.v1.3.runoff.180404.nc - JRA.v1.5.runoff.%y.no_rofi_no_rofl.210505.nc + JRA.v1.5.runoff.%y.no_rofi_no_rofl.240411.nc - JRA.v1.5.runoff.%y.210505.nc + JRA.v1.5.runoff.%y.240411.nc JRA.v1.4.runoff.%y.no_rofi.190214.nc diff --git a/components/eam/bld/namelist_files/namelist_defaults_eam.xml b/components/eam/bld/namelist_files/namelist_defaults_eam.xml index 56c816b1753c..cfd9bf682c8e 100755 --- a/components/eam/bld/namelist_files/namelist_defaults_eam.xml +++ b/components/eam/bld/namelist_files/namelist_defaults_eam.xml @@ -917,6 +917,7 @@ 1.0D0 .false. 100.D6 + 200.D6 0.1D6 -999. -999. diff --git a/components/eam/bld/namelist_files/namelist_definition.xml b/components/eam/bld/namelist_files/namelist_definition.xml index bc9977f67b58..b3ecb5e9290a 100644 --- a/components/eam/bld/namelist_files/namelist_definition.xml +++ b/components/eam/bld/namelist_files/namelist_definition.xml @@ -473,6 +473,26 @@ Turn on additional verbose output for integrated conservation checks. Default: FALSE + + + +Number of zonal mean basis functions (number of m=0 spherical harmonics) used in +Transformed Eulerian Mean (TEM) diagnostics + + + +Number of latitude grid points for zonal average TEM diagnostics history fields + + + + Frequency of TEM diagnostic calculations. + If > 0, frequency is specified as number of timesteps. + If < 0, frequency is specified as number of hours. + + -Turn off microphysics computation. +Turn off microphysics computation for MG2. For P3 this disables the generation of +liquid precipitation via autoconversion only, to be used for idealized simulations of +warm phase boundary layer clouds. Default: FALSE | | diff --git a/components/eam/bld/namelist_files/use_cases/1850_eam_CMIP6_chemUCI-Linoz-mam5-vbs-1pctCO2.xml b/components/eam/bld/namelist_files/use_cases/1850_eam_CMIP6_chemUCI-Linoz-mam5-vbs-1pctCO2.xml index c4f96718134e..41803ef4469f 100644 --- a/components/eam/bld/namelist_files/use_cases/1850_eam_CMIP6_chemUCI-Linoz-mam5-vbs-1pctCO2.xml +++ b/components/eam/bld/namelist_files/use_cases/1850_eam_CMIP6_chemUCI-Linoz-mam5-vbs-1pctCO2.xml @@ -15,7 +15,7 @@ 18500101 FIXED - + 808.249e-9 273.0211e-9 diff --git a/components/eam/bld/namelist_files/use_cases/1850_eam_CMIP6_chemUCI-Linoz-mam5-vbs.xml b/components/eam/bld/namelist_files/use_cases/1850_eam_CMIP6_chemUCI-Linoz-mam5-vbs.xml index 1518954b5332..c8cc7ee0b0fb 100644 --- a/components/eam/bld/namelist_files/use_cases/1850_eam_CMIP6_chemUCI-Linoz-mam5-vbs.xml +++ b/components/eam/bld/namelist_files/use_cases/1850_eam_CMIP6_chemUCI-Linoz-mam5-vbs.xml @@ -15,7 +15,7 @@ 18500101 FIXED - + 808.249e-9 273.0211e-9 diff --git a/components/eam/bld/namelist_files/use_cases/1950_eam_CMIP6_chemUCI-Linoz-mam5-vbs.xml b/components/eam/bld/namelist_files/use_cases/1950_eam_CMIP6_chemUCI-Linoz-mam5-vbs.xml new file mode 100644 index 000000000000..b415cd84b844 --- /dev/null +++ b/components/eam/bld/namelist_files/use_cases/1950_eam_CMIP6_chemUCI-Linoz-mam5-vbs.xml @@ -0,0 +1,136 @@ +----- modified based on 1850 version ---- combined with 1950 use_case for V2 + + + + + +.true. + + +atm/cam/inic/homme/NGD_v3atm.ne30pg2.eam.i.0001-01-01-00000.c20230106.nc + + + +atm/cam/solar/Solar_1950control_input4MIPS_c20171208.nc +19500101 +FIXED + + + +1163.821e-9 +289.739e-9 +62.83147e-12 +6.382257e-12 + + +.true. +.true. +.true. + + +CYCLICAL +1950 +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_NO2_aircraft_vertical_1750-2015_1.9x2.5_c20170608.nc + +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/so2_elev_strat_1950.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/emissions-cmip6_e3sm_SOAG0_elev_1850-2014_1.9x2.5_c20230201.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_elev_1850-2014_c180205.nc + + +CYCLICAL +1950 +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C2H4_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C2H6_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C3H8_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH2O_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH3CHO_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH3COCH3_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CO_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_ISOP_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_ISOP_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_MTERP_surface_1850-2014_1.9x2.5_c20230126.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_NO_surface_1850-2014_1.9x2.5_c20220425.nc +atm/cam/chem/trop_mozart_aero/emis/DMSflux.1950.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20171210.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/emissions-cmip6_e3sm_SOAG0_surf_1850-2014_1.9x2.5_c20230201.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions_E90_surface_1750-2015_1.9x2.5_c20210408.nc + + + + +CYCLICAL +1955 +oxid_1.9x2.5_L26_1850-2015_c20181106.nc +'' +atm/cam/chem/trop_mozart_aero/oxid +'prsd_O3:O3','prsd_NO3:NO3','prsd_OH:OH' + + +ch4_oxid_1.9x2.5_L26_1990-1999clim.c090804.nc +atm/cam/chem/methane +CYCLICAL +1995 +'' +'prsd_ch4:CH4' + + + + 'A:H2OLNZ:H2O', 'N:O2:O2', 'N:CO2:CO2', + 'A:O3:O3', 'A:N2OLNZ:N2O', 'A:CH4LNZ:CH4', + 'N:CFC11:CFC11', 'N:CFC12:CFC12', + 'M:mam5_mode1:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode1_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode2:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode2_rrtmg_c130628.nc', + 'M:mam5_mode3:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode3_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode4:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode4_rrtmg_c130628.nc', + 'M:mam5_mode5:$INPUTDATA_ROOT/atm/cam/physprops/mam5_mode5_rrtmg_sig1.2_dgnl.40_c03072023.nc' + + + +3 +1 +'atm/cam/chem/trop_mam/marine_BGC/' +'CYCLICAL' +'monthly_macromolecules_0.1deg_bilinear_latlon_year01_merge_date.nc' +0 +0 +'chla:CHL1','mpoly:TRUEPOLYC','mprot:TRUEPROTC','mlip:TRUELIPC' + + +atm/cam/chem/trop_mozart/ub/Linoz_Chlorine_Loading_CMIP6_0003-2017_c20171114.nc +19500101 +FIXED +1950 +linv3_1849-2017_CMIP6_Hist_10deg_58km_c20230705.nc +atm/cam/chem/trop_mozart/ub +CYCLICAL + + +'xactive_lnd' +'O3','H2O2','CH2O','CH3OOH','NO','NO2','HNO3','HO2NO2','PAN','CO','CH3COCH3','C2H5OOH','CH3CHO','H2SO4','SO2','NO3','N2O5','SOAG0','SOAG15','SOAG24','SOAG35','SOAG34','SOAG33','SOAG32','SOAG31' +'NEU' +'C2H5OOH','CH2O','CH3CHO','CH3OOH','H2O2','H2SO4','HNO3','HO2NO2','SO2','SOAG0','SOAG15','SOAG24','SOAG35','SOAG34','SOAG33','SOAG32','SOAG31' +'CH2O', 'CH3O2', 'CH3OOH', 'PAN', 'CO', 'C2H6', 'C3H8', 'C2H4', 'ROHO2', 'CH3COCH3', 'C2H5O2', 'C2H5OOH', 'CH3CHO', 'CH3CO3', 'ISOP', 'ISOPO2', 'MVKMACR', 'MVKO2' + +'' + + +1950 + + + + + + diff --git a/components/eam/bld/namelist_files/use_cases/SSP245_eam_CMIP6_chemUCI-Linoz-mam5-vbs.xml b/components/eam/bld/namelist_files/use_cases/SSP245_eam_CMIP6_chemUCI-Linoz-mam5-vbs.xml new file mode 100644 index 000000000000..120fb939165e --- /dev/null +++ b/components/eam/bld/namelist_files/use_cases/SSP245_eam_CMIP6_chemUCI-Linoz-mam5-vbs.xml @@ -0,0 +1,107 @@ + + + + +.true. + + +atm/cam/solar/Solar_1850-2299_input4MIPS_c20181106.nc +SERIAL + + +atm/cam/ggas/GHG_CMIP_SSP245-1-2-1_Annual_Global_2015-2500_c20200807.nc +RAMPED + + +.true. +.true. +.true. + + +INTERP_MISSING_MONTHS +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_NO2_aircraft_vertical_2015-2100_1.9x2.5_c20240219.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_so2_volc_elev_2015-2100_c240331.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_SOAG0_elev_2015-2100_1.9x2.5_c20240219.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_bc_a4_elev_2015-2100_c200716.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_num_a1_elev_2015-2100_c200716.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_num_a2_elev_2015-2100_c200716.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_num_a4_elev_2015-2100_c200716.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_pom_a4_elev_2015-2100_c200716.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_so4_a1_elev_2015-2100_c200716.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_so4_a2_elev_2015-2100_c200716.nc + + +INTERP_MISSING_MONTHS +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_C2H4_surface_2015-2100_1.9x2.5_c20240219.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_C2H6_surface_2015-2100_1.9x2.5_c20240219.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_C3H8_surface_2015-2100_1.9x2.5_c20240219.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_CH2O_surface_2015-2100_1.9x2.5_c20240219.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_CH3CHO_surface_2015-2100_1.9x2.5_c20240219.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_CH3COCH3_surface_2015-2100_1.9x2.5_c20240219.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_CO_surface_2015-2100_1.9x2.5_c20240219.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_ISOP_surface_2015-2100_1.9x2.5_c20240219.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_ISOP_surface_2015-2100_1.9x2.5_c20240219.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_MTERP_surface_2015-2100_1.9x2.5_c20240219.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_NO_surface_2015-2100_1.9x2.5_c20240219.nc +atm/cam/chem/trop_mozart_aero/emis/DMSflux.1850-2100.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20160727.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_so2_surf_2015-2100_c200716.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_SOAG0_surf_2015-2100_1.9x2.5_c20240219.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_bc_a4_surf_2015-2100_c200716.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_num_a1_surf_2015-2100_c200716.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_num_a2_surf_2015-2100_c200716.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_num_a4_surf_2015-2100_c200716.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_pom_a4_surf_2015-2100_c200716.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_so4_a1_surf_2015-2100_c200716.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_so4_a2_surf_2015-2100_c200716.nc +atm/cam/chem/trop_mozart/ub/emissions_E90_surface_1750-2101_1.9x2.5_c20231222.nc + + + + +INTERP_MISSING_MONTHS +atm/cam/chem/trop_mozart_aero/oxid +oxid_SSP245_1.9x2.5_L70_1849-2101_c20240228.nc +'prsd_O3:O3','prsd_NO3:NO3','prsd_OH:OH' +'' + + + + 'A:H2OLNZ:H2O', 'N:O2:O2', 'N:CO2:CO2', + 'A:O3:O3', 'A:N2OLNZ:N2O', 'A:CH4LNZ:CH4', + 'N:CFC11:CFC11', 'N:CFC12:CFC12', + 'M:mam5_mode1:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode1_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode2:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode2_rrtmg_c130628.nc', + 'M:mam5_mode3:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode3_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode4:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode4_rrtmg_c130628.nc', + 'M:mam5_mode5:$INPUTDATA_ROOT/atm/cam/physprops/mam5_mode5_rrtmg_sig1.2_dgnl.40_c03072023.nc' + + + +3 +1 +'atm/cam/chem/trop_mam/marine_BGC/' +'CYCLICAL' +'monthly_macromolecules_0.1deg_bilinear_latlon_year01_merge_date.nc' +0 +0 +'chla:CHL1','mpoly:TRUEPOLYC','mprot:TRUEPROTC','mlip:TRUELIPC' + + +atm/cam/chem/trop_mozart/ub/Linoz_Chlorine_Loading_CMIP6_Hist_SSP245_0003-2503_c20200808.nc +SERIAL +linv3_1849-2101_CMIP6_Hist_SSP245_10deg_58km_c20230705.nc +atm/cam/chem/trop_mozart/ub +INTERP_MISSING_MONTHS + + +'xactive_lnd' +'O3','H2O2','CH2O','CH3OOH','NO','NO2','HNO3','HO2NO2','PAN','CO','CH3COCH3','C2H5OOH','CH3CHO','H2SO4','SO2','NO3','N2O5','SOAG0','SOAG15','SOAG24','SOAG35','SOAG34','SOAG33','SOAG32','SOAG31' +'NEU' +'C2H5OOH','CH2O','CH3CHO','CH3OOH','H2O2','H2SO4','HNO3','HO2NO2','SO2','SOAG0','SOAG15','SOAG24','SOAG35','SOAG34','SOAG33','SOAG32','SOAG31' +'CH2O', 'CH3O2', 'CH3OOH', 'PAN', 'CO', 'C2H6', 'C3H8', 'C2H4', 'ROHO2', 'CH3COCH3', 'C2H5O2', 'C2H5OOH', 'CH3CHO', 'CH3CO3', 'ISOP', 'ISOPO2', 'MVKMACR', 'MVKO2' +'' + + +2015-2100 + + diff --git a/components/eam/bld/namelist_files/use_cases/SSP370_eam_CMIP6.xml b/components/eam/bld/namelist_files/use_cases/SSP370_eam_CMIP6.xml old mode 100755 new mode 100644 diff --git a/components/eam/bld/namelist_files/use_cases/SSP370_eam_CMIP6_chemUCI-Linoz-mam5-vbs.xml b/components/eam/bld/namelist_files/use_cases/SSP370_eam_CMIP6_chemUCI-Linoz-mam5-vbs.xml old mode 100755 new mode 100644 index 60912c544194..e15be0889fdc --- a/components/eam/bld/namelist_files/use_cases/SSP370_eam_CMIP6_chemUCI-Linoz-mam5-vbs.xml +++ b/components/eam/bld/namelist_files/use_cases/SSP370_eam_CMIP6_chemUCI-Linoz-mam5-vbs.xml @@ -12,13 +12,6 @@ atm/cam/ggas/GHG_CMIP_SSP370-1-2-1_Annual_Global_2015-2500_c20210509.nc RAMPED - -atm/cam/volc -CMIP_DOE-ACME_radiation_average_1850-2014_v3_c20171204.nc -VOLC_CMIP6 -CYCLICAL -1 - .true. .true. @@ -26,8 +19,9 @@ INTERP_MISSING_MONTHS -atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_so2_elev_2015-2100_c210216.nc -atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_soag_elev_2015-2100_c210216.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_NO2_aircraft_vertical_2015-2100_1.9x2.5_c20240208.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_so2_volc_elev_2015-2100_c240331.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_SOAG0_elev_2015-2100_1.9x2.5_c20240208.nc atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_bc_a4_elev_2015-2100_c210216.nc atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_num_a1_elev_2015-2100_c210216.nc atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_num_a2_elev_2015-2100_c210216.nc @@ -38,8 +32,20 @@ INTERP_MISSING_MONTHS +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_C2H4_surface_2015-2100_1.9x2.5_c20240208.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_C2H6_surface_2015-2100_1.9x2.5_c20240208.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_C3H8_surface_2015-2100_1.9x2.5_c20240208.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_CH2O_surface_2015-2100_1.9x2.5_c20240208.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_CH3CHO_surface_2015-2100_1.9x2.5_c20240208.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_CH3COCH3_surface_2015-2100_1.9x2.5_c20240208.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_CO_surface_2015-2100_1.9x2.5_c20240208.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_ISOP_surface_2015-2100_1.9x2.5_c20240208.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_ISOP_surface_2015-2100_1.9x2.5_c20240208.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_MTERP_surface_2015-2100_1.9x2.5_c20240208.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_NO_surface_2015-2100_1.9x2.5_c20240208.nc atm/cam/chem/trop_mozart_aero/emis/DMSflux.1850-2100.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20160727.nc atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_so2_surf_2015-2100_c210216.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_SOAG0_surf_2015-2100_1.9x2.5_c20240208.nc atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_bc_a4_surf_2015-2100_c210216.nc atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_num_a1_surf_2015-2100_c210216.nc atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_num_a2_surf_2015-2100_c210216.nc @@ -47,14 +53,28 @@ atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_pom_a4_surf_2015-2100_c210216.nc atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_so4_a1_surf_2015-2100_c210216.nc atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_so4_a2_surf_2015-2100_c210216.nc +atm/cam/chem/trop_mozart/ub/emissions_E90_surface_1750-2101_1.9x2.5_c20231222.nc + + INTERP_MISSING_MONTHS atm/cam/chem/trop_mozart_aero/oxid -oxid_SSP370_1.9x2.5_L70_2015-2100_c20211006.nc +oxid_SSP370_1.9x2.5_L70_1849-2101_c20240228.nc +'prsd_O3:O3','prsd_NO3:NO3','prsd_OH:OH' '' - + + + 'A:H2OLNZ:H2O', 'N:O2:O2', 'N:CO2:CO2', + 'A:O3:O3', 'A:N2OLNZ:N2O', 'A:CH4LNZ:CH4', + 'N:CFC11:CFC11', 'N:CFC12:CFC12', + 'M:mam5_mode1:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode1_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode2:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode2_rrtmg_c130628.nc', + 'M:mam5_mode3:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode3_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode4:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode4_rrtmg_c130628.nc', + 'M:mam5_mode5:$INPUTDATA_ROOT/atm/cam/physprops/mam5_mode5_rrtmg_sig1.2_dgnl.40_c03072023.nc' + 3 @@ -74,7 +94,12 @@ INTERP_MISSING_MONTHS -'H2O2', 'H2SO4', 'SO2' +'xactive_lnd' +'O3','H2O2','CH2O','CH3OOH','NO','NO2','HNO3','HO2NO2','PAN','CO','CH3COCH3','C2H5OOH','CH3CHO','H2SO4','SO2','NO3','N2O5','SOAG0','SOAG15','SOAG24','SOAG35','SOAG34','SOAG33','SOAG32','SOAG31' +'NEU' +'C2H5OOH','CH2O','CH3CHO','CH3OOH','H2O2','H2SO4','HNO3','HO2NO2','SO2','SOAG0','SOAG15','SOAG24','SOAG35','SOAG34','SOAG33','SOAG32','SOAG31' +'CH2O', 'CH3O2', 'CH3OOH', 'PAN', 'CO', 'C2H6', 'C3H8', 'C2H4', 'ROHO2', 'CH3COCH3', 'C2H5O2', 'C2H5OOH', 'CH3CHO', 'CH3CO3', 'ISOP', 'ISOPO2', 'MVKMACR', 'MVKO2' +'' 2015-2100 diff --git a/components/eam/bld/namelist_files/use_cases/SSP585_eam_CMIP6.xml b/components/eam/bld/namelist_files/use_cases/SSP585_eam_CMIP6.xml old mode 100755 new mode 100644 diff --git a/components/eam/bld/namelist_files/use_cases/SSP585_eam_CMIP6_chemUCI-Linoz-mam5-vbs.xml b/components/eam/bld/namelist_files/use_cases/SSP585_eam_CMIP6_chemUCI-Linoz-mam5-vbs.xml old mode 100755 new mode 100644 index 98e2dcf76042..f9c45623d5a9 --- a/components/eam/bld/namelist_files/use_cases/SSP585_eam_CMIP6_chemUCI-Linoz-mam5-vbs.xml +++ b/components/eam/bld/namelist_files/use_cases/SSP585_eam_CMIP6_chemUCI-Linoz-mam5-vbs.xml @@ -12,13 +12,6 @@ atm/cam/ggas/GHG_CMIP_SSP585-1-2-1_Annual_Global_2015-2500_c20190310.nc RAMPED - -atm/cam/volc -CMIP_DOE-ACME_radiation_average_1850-2014_v3_c20171204.nc -VOLC_CMIP6 -CYCLICAL -1 - .true. .true. @@ -26,8 +19,9 @@ INTERP_MISSING_MONTHS -atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_so2_elev_2015-2100_c190828.nc -atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_soag_elev_2015-2100_c190828.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_NO2_aircraft_vertical_2015-2100_1.9x2.5_c20240304.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_so2_volc_elev_2015-2100_c240331.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_SOAG0_elev_2015-2100_1.9x2.5_c20240304.nc atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_bc_a4_elev_2015-2100_c190828.nc atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_num_a1_elev_2015-2100_c190828.nc atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_num_a2_elev_2015-2100_c190828.nc @@ -38,8 +32,20 @@ INTERP_MISSING_MONTHS +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_C2H4_surface_2015-2100_1.9x2.5_c20240304.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_C2H6_surface_2015-2100_1.9x2.5_c20240304.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_C3H8_surface_2015-2100_1.9x2.5_c20240304.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_CH2O_surface_2015-2100_1.9x2.5_c20240304.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_CH3CHO_surface_2015-2100_1.9x2.5_c20240304.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_CH3COCH3_surface_2015-2100_1.9x2.5_c20240304.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_CO_surface_2015-2100_1.9x2.5_c20240304.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_ISOP_surface_2015-2100_1.9x2.5_c20240304.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_ISOP_surface_2015-2100_1.9x2.5_c20240304.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_MTERP_surface_2015-2100_1.9x2.5_c20240304.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_NO_surface_2015-2100_1.9x2.5_c20240304.nc atm/cam/chem/trop_mozart_aero/emis/DMSflux.1850-2100.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20160727.nc atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_so2_surf_2015-2100_c190828.nc +atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_SOAG0_surf_2015-2100_1.9x2.5_c20240304.nc atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_bc_a4_surf_2015-2100_c190828.nc atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_num_a1_surf_2015-2100_c190828.nc atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_num_a2_surf_2015-2100_c190828.nc @@ -47,14 +53,28 @@ atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_pom_a4_surf_2015-2100_c190828.nc atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_so4_a1_surf_2015-2100_c190828.nc atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_so4_a2_surf_2015-2100_c190828.nc +atm/cam/chem/trop_mozart/ub/emissions_E90_surface_1750-2101_1.9x2.5_c20231222.nc + + INTERP_MISSING_MONTHS atm/cam/chem/trop_mozart_aero/oxid -oxid_1.9x2.5_L70_2015-2100_c20190421.nc +oxid_SSP585_1.9x2.5_L70_2014-2101_c20240228.nc +'prsd_O3:O3','prsd_NO3:NO3','prsd_OH:OH' '' - + + + 'A:H2OLNZ:H2O', 'N:O2:O2', 'N:CO2:CO2', + 'A:O3:O3', 'A:N2OLNZ:N2O', 'A:CH4LNZ:CH4', + 'N:CFC11:CFC11', 'N:CFC12:CFC12', + 'M:mam5_mode1:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode1_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode2:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode2_rrtmg_c130628.nc', + 'M:mam5_mode3:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode3_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode4:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode4_rrtmg_c130628.nc', + 'M:mam5_mode5:$INPUTDATA_ROOT/atm/cam/physprops/mam5_mode5_rrtmg_sig1.2_dgnl.40_c03072023.nc' + 3 @@ -74,7 +94,12 @@ INTERP_MISSING_MONTHS -'H2O2', 'H2SO4', 'SO2' +'xactive_lnd' +'O3','H2O2','CH2O','CH3OOH','NO','NO2','HNO3','HO2NO2','PAN','CO','CH3COCH3','C2H5OOH','CH3CHO','H2SO4','SO2','NO3','N2O5','SOAG0','SOAG15','SOAG24','SOAG35','SOAG34','SOAG33','SOAG32','SOAG31' +'NEU' +'C2H5OOH','CH2O','CH3CHO','CH3OOH','H2O2','H2SO4','HNO3','HO2NO2','SO2','SOAG0','SOAG15','SOAG24','SOAG35','SOAG34','SOAG33','SOAG32','SOAG31' +'CH2O', 'CH3O2', 'CH3OOH', 'PAN', 'CO', 'C2H6', 'C3H8', 'C2H4', 'ROHO2', 'CH3COCH3', 'C2H5O2', 'C2H5OOH', 'CH3CHO', 'CH3CO3', 'ISOP', 'ISOPO2', 'MVKMACR', 'MVKO2' +'' 2015-2100 diff --git a/components/eam/bld/namelist_files/use_cases/scam_generic.xml b/components/eam/bld/namelist_files/use_cases/scam_generic.xml index 395d9a6ec860..2b77e3b630ae 100644 --- a/components/eam/bld/namelist_files/use_cases/scam_generic.xml +++ b/components/eam/bld/namelist_files/use_cases/scam_generic.xml @@ -3,7 +3,6 @@ -atm/cam/inic/homme/cami_mam3_Linoz_ne4np4_L72_c160909.nc atm/cam/chem/trop_mozart_aero/emis/aces4bgc_nvsoa_soag_elev_2000_c160427.nc diff --git a/components/eam/cime_config/config_component.xml b/components/eam/cime_config/config_component.xml index c7585072ac94..1a961b84affe 100755 --- a/components/eam/cime_config/config_component.xml +++ b/components/eam/cime_config/config_component.xml @@ -53,8 +53,7 @@ -phys default &eamv3_phys_defaults; &eamv3_chem_defaults; &eamv3_phys_defaults; &eamv3_chem_defaults; - &eamv3_phys_defaults; &eamv3_chem_defaults; - &eamv3_phys_defaults; &eamv2_chem_defaults; + &eamv3_phys_defaults; &eamv3_chem_defaults; -bc_dep_to_snow_updates -co2_cycle @@ -129,6 +128,7 @@ 20TR_eam_CMIP6_chemUCI-Linoz-mam5-vbs SSP585_eam_CMIP6_chemUCI-Linoz-mam5-vbs SSP370_eam_CMIP6_chemUCI-Linoz-mam5-vbs + SSP245_eam_CMIP6_chemUCI-Linoz-mam5-vbs SSP585_cam5_CMIP6_bgc 20TR_E3SMv1_superfast_ar5-emis 20TRS_E3SMv1_superfast_ar5-emis @@ -136,7 +136,7 @@ 20TR_eam_chemUCI-Linoz-mam5 scam_arm95 scm_arm97_chemUCI-Linoz-mam5-vbs - scm_generic_chemUCI-Linoz-mam5-vbs + scam_generic 1850-PD_cam5 aquaplanet_EAMv1 RCEMIP_EAMv1 @@ -194,24 +194,24 @@ - cam5 physics: - EAM with complete set of E3SM atmospheric mods for V3 (72 layers model) with chemUCI, Linozv3, MAM5 and VBS SOA - CMIP6-DECK: - high-res (ne120 w/ 18to6 ocn) 72 layer E3SM v1 atmos model with CMIP6 forcings - low-res (ne30 w/ oECv3 ocn) 72 layer E3SM v1 atmos model with CMIP6 forcings + EAM with + complete set of E3SM atmospheric mods for V3 (80 layers, P3, ZM with convective microphysics, chemUCI, Linozv3, MAM5 and VBS SOA): + CMIP6 forcings: E3SM plus super-fast chemistry with AR5 emissions: EAM super_fast_llnl chemistry: EAM CO2 ramp: - single column EAM ARM95 IOP test case: - single column EAM ARM97 IOP test case: + single column ARM95 IOP test case: + single column ARM97 IOP test case: single column EAM RICO IOP test case: - EAM adiabatic physics: - EAM ideal physics: + adiabatic physics: + ideal physics: Atmospheric Model Intercomparison Project protocol: - EAM prognostic CO2 cycle turned on. + prognostic CO2 cycle turned on. Fortran version of SCREAM with SHOC, P3, RRTMGP, and prescribed aerosol. Initialized for DYAMOND1 (2016-08-01). Fortran version of SCREAM with NH dycore, SHOC, P3, RRTMGP, prescribed aerosol, and no deep convection. Initialized for DYAMOND1 (2016-08-01). Fortran version of SCREAM with SHOC, P3, RRTMGP, and prescribed aerosol. Initialized for DYAMOND2 (2020-01-20). Fortran version of SCREAM with NH dycore, SHOC, P3, RRTMGP, prescribed aerosol, and no deep convection. Initialized for DYAMOND2 (2020-01-20). + Fortran version of SCREAM with SHOC, P3, RRTMGP, and prescribed aerosol: doubly periodic boundary conditions. Fortran version of SCREAM with SHOC, P3, RRTMGP, and prescribed aerosol. Fortran version of SCREAM with NH dycore, SHOC, P3, RRTMGP, prescribed aerosol, and no deep convection. diff --git a/components/eam/cime_config/config_compsets.xml b/components/eam/cime_config/config_compsets.xml index 894ecccea05d..39971fbb0945 100644 --- a/components/eam/cime_config/config_compsets.xml +++ b/components/eam/cime_config/config_compsets.xml @@ -23,12 +23,12 @@ F1950-CICE - 1950_EAM%CMIP6_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV + 1950_EAM%CMIP6_ELM%CNPRDCTCBCTOP_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV F1950 - 1950_EAM%CMIP6_ELM%SPBC_MPASSI%PRES_DOCN%DOM_MOSART_SGLC_SWAV + 1950_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%PRES_DOCN%DOM_MOSART_SGLC_SWAV @@ -51,14 +51,19 @@ 2010_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%PRES_DOCN%DOM_MOSART_SGLC_SWAV + + FSSP245 + SSP245_eam_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%PRES_DOCN%DOM_MOSART_SGLC_SWAV + + FSSP370 - SSP370_eam_EAM%CMIP6_ELM%SPBC_MPASSI%PRES_DOCN%DOM_MOSART_SGLC_SWAV + SSP370_eam_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%PRES_DOCN%DOM_MOSART_SGLC_SWAV FSSP585 - SSP585_eam_EAM%CMIP6_ELM%SPBC_MPASSI%PRES_DOCN%DOM_MOSART_SGLC_SWAV + SSP585_eam_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%PRES_DOCN%DOM_MOSART_SGLC_SWAV diff --git a/components/eam/cime_config/config_pes.xml b/components/eam/cime_config/config_pes.xml index 79f0b1c0dfcd..15a45fe92726 100644 --- a/components/eam/cime_config/config_pes.xml +++ b/components/eam/cime_config/config_pes.xml @@ -293,6 +293,21 @@ + + + pm-cpu: any compset on ne4pg2 grid + + 96 + 96 + 96 + 96 + 96 + 96 + 1 + 1 + + + @@ -756,6 +771,21 @@ + + + improv pelayout for tri-grid BGC tests with EAM+DOCN + + -4 + -4 + -4 + -4 + -4 + -4 + -4 + -4 + + + @@ -774,7 +804,7 @@ - @@ -799,7 +829,7 @@ - @@ -840,6 +870,22 @@ + + + + + gcp12 eam F compset --res ne30pg2_IcoswISC30E3r5 on 4 nodes + + -4 + -4 + -4 + -4 + -4 + -4 + + + + @@ -1062,6 +1108,21 @@ + + + --res conusx4v1_r05_oECv3 --compset F2010 + + -4 + -4 + -4 + -4 + -4 + -4 + -4 + -4 + + + --res conusx4v1_r05_oECv3 --compset F2010 @@ -1075,6 +1136,19 @@ + + + pm-cpu/gcp, eam, 2 nodes: --res conusx4v1_r05_oECv3 --compset F2010 + + -2 + -2 + -2 + -2 + -2 + -2 + + + diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_ftype0/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_ftype0/user_nl_eam index fa2b5daa6400..ae9a48579dfe 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_ftype0/user_nl_eam +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_ftype0/user_nl_eam @@ -1,4 +1,4 @@ se_ftype=0 -cubed_sphere_map=0 +cubed_sphere_map=2 theta_hydrostatic_mode=.true. diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_ftype1/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_ftype1/user_nl_eam index 2f675809c25e..ad90db9711b8 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_ftype1/user_nl_eam +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_ftype1/user_nl_eam @@ -1,5 +1,5 @@ se_ftype=1 -cubed_sphere_map=0 +cubed_sphere_map=2 theta_hydrostatic_mode=.true. diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_ftype2/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_ftype2/user_nl_eam index 601317baa1c8..03cda6cbf372 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_ftype2/user_nl_eam +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_ftype2/user_nl_eam @@ -1,5 +1,5 @@ se_ftype=2 -cubed_sphere_map=0 +cubed_sphere_map=2 theta_hydrostatic_mode=.true. diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_ftype4/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_ftype4/user_nl_eam index 59083a775066..c05cebbabef9 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_ftype4/user_nl_eam +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_ftype4/user_nl_eam @@ -1,5 +1,5 @@ se_ftype=4 -cubed_sphere_map=0 +cubed_sphere_map=2 theta_hydrostatic_mode=.true. diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_sl/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_sl/user_nl_eam index 2810cbd5a285..06a1bab6fca8 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_sl/user_nl_eam +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_sl/user_nl_eam @@ -1,7 +1,7 @@ theta_hydrostatic_mode=.true. tstep_type=5 -cubed_sphere_map=0 +cubed_sphere_map=2 transport_alg=12 semi_lagrange_cdr_alg=20 diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/thetanh_ftype0/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetanh_ftype0/user_nl_eam index ed3e180931d5..ed5fffdd21cb 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/thetanh_ftype0/user_nl_eam +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetanh_ftype0/user_nl_eam @@ -1,5 +1,5 @@ se_ftype=0 -cubed_sphere_map=0 +cubed_sphere_map=2 theta_hydrostatic_mode=.false. tstep_type=7 se_nsplit=10 diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/thetanh_ftype1/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetanh_ftype1/user_nl_eam index f9600e5abde0..d0d6c6b9edca 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/thetanh_ftype1/user_nl_eam +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetanh_ftype1/user_nl_eam @@ -1,5 +1,5 @@ se_ftype=1 -cubed_sphere_map=0 +cubed_sphere_map=2 theta_hydrostatic_mode=.false. tstep_type=7 se_nsplit=10 diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/thetanh_ftype2/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetanh_ftype2/user_nl_eam index e147f00913c9..a380b60b76b3 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/thetanh_ftype2/user_nl_eam +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetanh_ftype2/user_nl_eam @@ -1,5 +1,5 @@ se_ftype=2 -cubed_sphere_map=0 +cubed_sphere_map=2 theta_hydrostatic_mode=.false. tstep_type=7 transport_alg=0 diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/thetanh_ftype4/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetanh_ftype4/user_nl_eam index 455deb86d7f6..f1cc0ef1f330 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/thetanh_ftype4/user_nl_eam +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetanh_ftype4/user_nl_eam @@ -1,5 +1,5 @@ se_ftype=4 -cubed_sphere_map=0 +cubed_sphere_map=2 theta_hydrostatic_mode=.false. tstep_type=7 se_nsplit=10 diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod/shell_commands b/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod/shell_commands index 6e8ae38ac8e9..0067b2f35de5 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod/shell_commands +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod/shell_commands @@ -1,3 +1,3 @@ #!/bin/bash -./xmlchange --append CAM_CONFIG_OPTS='-cosp' +./xmlchange --append CAM_CONFIG_OPTS='-cosp' diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod/user_nl_eam index eeac1d647b1d..095c6946f5de 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod/user_nl_eam +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod/user_nl_eam @@ -1,11 +1,59 @@ - nhtfrq = -24,-24,-6,-6,-3,-24,-24 - mfilt = 1,30,120,120,240,30,1 - avgflag_pertape = 'A','A','I','A','A','A','I' - fexcl1 = 'CFAD_SR532_CAL', 'LINOZ_DO3', 'LINOZ_DO3_PSC', 'LINOZ_O3CLIM', 'LINOZ_O3COL', 'LINOZ_SSO3', 'hstobie_linoz' - fincl1 = 'extinct_sw_inp','extinct_lw_bnd7','extinct_lw_inp','CLD_CAL', 'TREFMNAV', 'TREFMXAV' - fincl2 = 'FLUT','PRECT','U200','V200','U850','V850','Z500','OMEGA500','UBOT','VBOT','TREFHT','TREFHTMN:M','TREFHTMX:X','QREFHT','TS','PS','TMQ','TUQ','TVQ','TOZ', 'FLDS', 'FLNS', 'FSDS', 'FSNS', 'SHFLX', 'LHFLX', 'TGCLDCWP', 'TGCLDIWP', 'TGCLDLWP', 'CLDTOT', 'T250', 'T200', 'T150', 'T100', 'T050', 'T025', 'T010', 'T005', 'T002', 'T001', 'TTOP', 'U250', 'U150', 'U100', 'U050', 'U025', 'U010', 'U005', 'U002', 'U001', 'UTOP', 'FSNT', 'FLNT' - fincl3 = 'PSL','T200','T500','U850','V850','UBOT','VBOT','TREFHT', 'Z700', 'TBOT:M' - fincl4 = 'FLUT','U200','U850','PRECT','OMEGA500' - fincl5 = 'PRECT','PRECC','TUQ','TVQ','QFLX','SHFLX','U90M','V90M' - fincl6 = 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANTAU_ISCCP','MEANPTOP_ISCCP','MEANTB_ISCCP','CLDTOT_CAL','CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN','CLDHGH_CAL','CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN','CLDMED_CAL','CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN','CLDLOW_CAL','CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN' - fincl7 = 'O3', 'PS', 'TROP_P' +cosp_lite = .true. + +empty_htapes = .true. + +avgflag_pertape = 'A','A','A','A','I','I' +nhtfrq = -24,-24,-6,-3,-1,-24 +mfilt = 1,30,120,240,720,1 + +fincl1 = 'AODALL','AODBC','AODDUST','AODPOM','AODSO4','AODSOA','AODSS','AODVIS', + 'CLDLOW','CLDMED','CLDHGH','CLDTOT', + 'CLDHGH_CAL','CLDLOW_CAL','CLDMED_CAL','CLD_MISR','CLDTOT_CAL', + 'CLMODIS','FISCCP1_COSP','FLDS','FLNS','FLNSC','FLNT','FLUT', + 'FLUTC','FSDS','FSDSC','FSNS','FSNSC','FSNT','FSNTOA','FSNTOAC','FSNTC', + 'ICEFRAC','LANDFRAC','LWCF','OCNFRAC','OMEGA','PRECC','PRECL','PRECSC','PRECSL','PS','PSL','Q', + 'QFLX','QREFHT','RELHUM','SCO','SHFLX','SOLIN','SWCF','T','TAUX','TAUY','TCO', + 'TGCLDLWP','TMQ','TREFHT','TREFMNAV','TREFMXAV','TS','U','U10','V','Z3', + 'dst_a1DDF','dst_a3DDF','dst_c1DDF','dst_c3DDF','dst_a1SFWET','dst_a3SFWET','dst_c1SFWET','dst_c3SFWET', + 'O3','LHFLX', + 'O3_2DTDA_trop','O3_2DTDB_trop','O3_2DTDD_trop','O3_2DTDE_trop','O3_2DTDI_trop','O3_2DTDL_trop', + 'O3_2DTDN_trop','O3_2DTDO_trop','O3_2DTDS_trop','O3_2DTDU_trop','O3_2DTRE_trop','O3_2DTRI_trop', + 'O3_SRF','NO_2DTDS','NO_TDLgt','NO2_2DTDD','NO2_2DTDS','NO2_TDAcf','CO_SRF','TROPE3D_P','TROP_P', + 'CDNUMC','SFDMS','so4_a1_sfgaex1','so4_a2_sfgaex1','so4_a3_sfgaex1','so4_a5_sfgaex1','soa_a1_sfgaex1', + 'soa_a2_sfgaex1','soa_a3_sfgaex1','GS_soa_a1','GS_soa_a2','GS_soa_a3','AQSO4_H2O2','AQSO4_O3', + 'SFSO2','SO2_CLXF','SO2','DF_SO2','AQ_SO2','GS_SO2','WD_SO2','ABURDENSO4_STR','ABURDENSO4_TRO', + 'ABURDENSO4','ABURDENBC','ABURDENDUST','ABURDENMOM','ABURDENPOM','ABURDENSEASALT', + 'ABURDENSOA','AODSO4_STR','AODSO4_TRO', + 'EXTINCT','AODABS','AODABSBC','CLDICE','CLDLIQ','CLD_CAL_TMPLIQ','CLD_CAL_TMPICE','Mass_bc_srf', + 'Mass_dst_srf','Mass_mom_srf','Mass_ncl_srf','Mass_pom_srf','Mass_so4_srf','Mass_soa_srf','Mass_bc_850', + 'Mass_dst_850','Mass_mom_850','Mass_ncl_850','Mass_pom_850','Mass_so4_850','Mass_soa_850','Mass_bc_500', + 'Mass_dst_500','Mass_mom_500','Mass_ncl_500','Mass_pom_500','Mass_so4_500','Mass_soa_500','Mass_bc_330', + 'Mass_dst_330','Mass_mom_330','Mass_ncl_330','Mass_pom_330','Mass_so4_330','Mass_soa_330','Mass_bc_200', + 'Mass_dst_200','Mass_mom_200','Mass_ncl_200','Mass_pom_200','Mass_so4_200','Mass_soa_200', + 'O3_2DTDD','O3_2DCIP','O3_2DCIL','CO_2DTDS','CO_2DTDD','CO_2DCEP','CO_2DCEL','NO_2DTDD', + 'FLNTC','SAODVIS', + 'H2OLNZ', + 'dst_a1SF','dst_a3SF', + 'PHIS','CLOUD','TGCLDIWP','TGCLDCWP','AREL', + 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANPTOP_ISCCP','CLD_CAL', + 'CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN', + 'CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN', + 'CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN', + 'CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN', + 'CLWMODIS','CLIMODIS' + +fincl2 = 'PS', 'FLUT','PRECT','U200','V200','U850','V850', + 'TCO','SCO','TREFHTMN','TREFHTMX','TREFHT','QREFHT' +fincl3 = 'PS', 'PSL','PRECT','TUQ','TVQ','UBOT','VBOT','TREFHT','FLUT','OMEGA500','TBOT','U850','V850','U200','V200','T200','T500','Z700' +fincl4 = 'PRECT' +fincl5 = 'O3_SRF' +fincl6 = 'CO_2DMSD','NO2_2DMSD','NO_2DMSD','O3_2DMSD','O3_2DMSD_trop' + +! -- chemUCI settings ------------------ +history_chemdyg_summary = .true. +history_gaschmbudget_2D = .false. +history_gaschmbudget_2D_levels = .false. +history_gaschmbudget_num = 6 !! no impact if history_gaschmbudget_2D = .false. + +! -- MAM5 settings ------------------ +is_output_interactive_volc = .true. diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod/user_nl_elm b/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod/user_nl_elm index a93edc10b690..cd1adab77404 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod/user_nl_elm +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod/user_nl_elm @@ -1,6 +1,40 @@ - finidat = ' ' - hist_dov2xy = .true.,.true. - hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' - hist_mfilt = 1,365 - hist_nhtfrq = -24,-24 - hist_avgflag_pertape = 'A','A' +hist_dov2xy = .true.,.true. +hist_fexcl1 = 'AGWDNPP','ALTMAX_LASTYEAR','AVAIL_RETRANSP','AVAILC','BAF_CROP', + 'BAF_PEATF','BIOCHEM_PMIN_TO_PLANT','CH4_SURF_AERE_SAT','CH4_SURF_AERE_UNSAT','CH4_SURF_DIFF_SAT', + 'CH4_SURF_DIFF_UNSAT','CH4_SURF_EBUL_SAT','CH4_SURF_EBUL_UNSAT','CMASS_BALANCE_ERROR','cn_scalar', + 'COL_PTRUNC','CONC_CH4_SAT','CONC_CH4_UNSAT','CONC_O2_SAT','CONC_O2_UNSAT', + 'cp_scalar','CWDC_HR','CWDC_LOSS','CWDC_TO_LITR2C','CWDC_TO_LITR3C', + 'CWDC_vr','CWDN_TO_LITR2N','CWDN_TO_LITR3N','CWDN_vr','CWDP_TO_LITR2P', + 'CWDP_TO_LITR3P','CWDP_vr','DWT_CONV_CFLUX_DRIBBLED','F_CO2_SOIL','F_CO2_SOIL_vr', + 'F_DENIT_vr','F_N2O_DENIT','F_N2O_NIT','F_NIT_vr','FCH4_DFSAT', + 'FINUNDATED_LAG','FPI_P_vr','FPI_vr','FROOTC_LOSS','HR_vr', + 'LABILEP_TO_SECONDP','LABILEP_vr','LAND_UPTAKE','LEAF_MR','leaf_npimbalance', + 'LEAFC_LOSS','LEAFC_TO_LITTER','LFC2','LITR1_HR','LITR1C_TO_SOIL1C', + 'LITR1C_vr','LITR1N_TNDNCY_VERT_TRANS','LITR1N_TO_SOIL1N','LITR1N_vr','LITR1P_TNDNCY_VERT_TRANS', + 'LITR1P_TO_SOIL1P','LITR1P_vr','LITR2_HR','LITR2C_TO_SOIL2C','LITR2C_vr', + 'LITR2N_TNDNCY_VERT_TRANS','LITR2N_TO_SOIL2N','LITR2N_vr','LITR2P_TNDNCY_VERT_TRANS','LITR2P_TO_SOIL2P', + 'LITR2P_vr','LITR3_HR','LITR3C_TO_SOIL3C','LITR3C_vr','LITR3N_TNDNCY_VERT_TRANS', + 'LITR3N_TO_SOIL3N','LITR3N_vr','LITR3P_TNDNCY_VERT_TRANS','LITR3P_TO_SOIL3P','LITR3P_vr', + 'M_LITR1C_TO_LEACHING','M_LITR2C_TO_LEACHING','M_LITR3C_TO_LEACHING','M_SOIL1C_TO_LEACHING','M_SOIL2C_TO_LEACHING', + 'M_SOIL3C_TO_LEACHING','M_SOIL4C_TO_LEACHING','NDEPLOY','NEM','nlim_m', + 'o2_decomp_depth_unsat','OCCLP_vr','PDEPLOY','PLANT_CALLOC','PLANT_NDEMAND', + 'PLANT_NDEMAND_COL','PLANT_PALLOC','PLANT_PDEMAND','PLANT_PDEMAND_COL','plim_m', + 'POT_F_DENIT','POT_F_NIT','POTENTIAL_IMMOB','POTENTIAL_IMMOB_P','PRIMP_TO_LABILEP', + 'PRIMP_vr','PROD1P_LOSS','QOVER_LAG','RETRANSN_TO_NPOOL','RETRANSP_TO_PPOOL', + 'SCALARAVG_vr','SECONDP_TO_LABILEP','SECONDP_TO_OCCLP','SECONDP_vr','SMIN_NH4_vr', + 'SMIN_NO3_vr','SMINN_TO_SOIL1N_L1','SMINN_TO_SOIL2N_L2','SMINN_TO_SOIL2N_S1','SMINN_TO_SOIL3N_L3', + 'SMINN_TO_SOIL3N_S2','SMINN_TO_SOIL4N_S3','SMINP_TO_SOIL1P_L1','SMINP_TO_SOIL2P_L2','SMINP_TO_SOIL2P_S1', + 'SMINP_TO_SOIL3P_L3','SMINP_TO_SOIL3P_S2','SMINP_TO_SOIL4P_S3','SMINP_vr','SOIL1_HR','SOIL1C_TO_SOIL2C','SOIL1C_vr','SOIL1N_TNDNCY_VERT_TRANS','SOIL1N_TO_SOIL2N','SOIL1N_vr', + 'SOIL1P_TNDNCY_VERT_TRANS','SOIL1P_TO_SOIL2P','SOIL1P_vr','SOIL2_HR','SOIL2C_TO_SOIL3C', + 'SOIL2C_vr','SOIL2N_TNDNCY_VERT_TRANS','SOIL2N_TO_SOIL3N','SOIL2N_vr','SOIL2P_TNDNCY_VERT_TRANS', + 'SOIL2P_TO_SOIL3P','SOIL2P_vr','SOIL3_HR','SOIL3C_TO_SOIL4C','SOIL3C_vr', + 'SOIL3N_TNDNCY_VERT_TRANS','SOIL3N_TO_SOIL4N','SOIL3N_vr','SOIL3P_TNDNCY_VERT_TRANS','SOIL3P_TO_SOIL4P', + 'SOIL3P_vr','SOIL4_HR','SOIL4C_vr','SOIL4N_TNDNCY_VERT_TRANS','SOIL4N_TO_SMINN', + 'SOIL4N_vr','SOIL4P_TNDNCY_VERT_TRANS','SOIL4P_TO_SMINP','SOIL4P_vr','SOLUTIONP_vr', + 'TCS_MONTH_BEGIN','TCS_MONTH_END','TOTCOLCH4','water_scalar','WF', + 'wlim_m','WOODC_LOSS','WTGQ' +hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC' +hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' +hist_mfilt = 1,365 +hist_nhtfrq = -24,-24 +hist_avgflag_pertape = 'A','A' diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F2010/shell_commands b/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F2010/shell_commands index 6e8ae38ac8e9..0067b2f35de5 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F2010/shell_commands +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F2010/shell_commands @@ -1,3 +1,3 @@ #!/bin/bash -./xmlchange --append CAM_CONFIG_OPTS='-cosp' +./xmlchange --append CAM_CONFIG_OPTS='-cosp' diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F2010/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F2010/user_nl_eam index eeac1d647b1d..095c6946f5de 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F2010/user_nl_eam +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F2010/user_nl_eam @@ -1,11 +1,59 @@ - nhtfrq = -24,-24,-6,-6,-3,-24,-24 - mfilt = 1,30,120,120,240,30,1 - avgflag_pertape = 'A','A','I','A','A','A','I' - fexcl1 = 'CFAD_SR532_CAL', 'LINOZ_DO3', 'LINOZ_DO3_PSC', 'LINOZ_O3CLIM', 'LINOZ_O3COL', 'LINOZ_SSO3', 'hstobie_linoz' - fincl1 = 'extinct_sw_inp','extinct_lw_bnd7','extinct_lw_inp','CLD_CAL', 'TREFMNAV', 'TREFMXAV' - fincl2 = 'FLUT','PRECT','U200','V200','U850','V850','Z500','OMEGA500','UBOT','VBOT','TREFHT','TREFHTMN:M','TREFHTMX:X','QREFHT','TS','PS','TMQ','TUQ','TVQ','TOZ', 'FLDS', 'FLNS', 'FSDS', 'FSNS', 'SHFLX', 'LHFLX', 'TGCLDCWP', 'TGCLDIWP', 'TGCLDLWP', 'CLDTOT', 'T250', 'T200', 'T150', 'T100', 'T050', 'T025', 'T010', 'T005', 'T002', 'T001', 'TTOP', 'U250', 'U150', 'U100', 'U050', 'U025', 'U010', 'U005', 'U002', 'U001', 'UTOP', 'FSNT', 'FLNT' - fincl3 = 'PSL','T200','T500','U850','V850','UBOT','VBOT','TREFHT', 'Z700', 'TBOT:M' - fincl4 = 'FLUT','U200','U850','PRECT','OMEGA500' - fincl5 = 'PRECT','PRECC','TUQ','TVQ','QFLX','SHFLX','U90M','V90M' - fincl6 = 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANTAU_ISCCP','MEANPTOP_ISCCP','MEANTB_ISCCP','CLDTOT_CAL','CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN','CLDHGH_CAL','CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN','CLDMED_CAL','CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN','CLDLOW_CAL','CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN' - fincl7 = 'O3', 'PS', 'TROP_P' +cosp_lite = .true. + +empty_htapes = .true. + +avgflag_pertape = 'A','A','A','A','I','I' +nhtfrq = -24,-24,-6,-3,-1,-24 +mfilt = 1,30,120,240,720,1 + +fincl1 = 'AODALL','AODBC','AODDUST','AODPOM','AODSO4','AODSOA','AODSS','AODVIS', + 'CLDLOW','CLDMED','CLDHGH','CLDTOT', + 'CLDHGH_CAL','CLDLOW_CAL','CLDMED_CAL','CLD_MISR','CLDTOT_CAL', + 'CLMODIS','FISCCP1_COSP','FLDS','FLNS','FLNSC','FLNT','FLUT', + 'FLUTC','FSDS','FSDSC','FSNS','FSNSC','FSNT','FSNTOA','FSNTOAC','FSNTC', + 'ICEFRAC','LANDFRAC','LWCF','OCNFRAC','OMEGA','PRECC','PRECL','PRECSC','PRECSL','PS','PSL','Q', + 'QFLX','QREFHT','RELHUM','SCO','SHFLX','SOLIN','SWCF','T','TAUX','TAUY','TCO', + 'TGCLDLWP','TMQ','TREFHT','TREFMNAV','TREFMXAV','TS','U','U10','V','Z3', + 'dst_a1DDF','dst_a3DDF','dst_c1DDF','dst_c3DDF','dst_a1SFWET','dst_a3SFWET','dst_c1SFWET','dst_c3SFWET', + 'O3','LHFLX', + 'O3_2DTDA_trop','O3_2DTDB_trop','O3_2DTDD_trop','O3_2DTDE_trop','O3_2DTDI_trop','O3_2DTDL_trop', + 'O3_2DTDN_trop','O3_2DTDO_trop','O3_2DTDS_trop','O3_2DTDU_trop','O3_2DTRE_trop','O3_2DTRI_trop', + 'O3_SRF','NO_2DTDS','NO_TDLgt','NO2_2DTDD','NO2_2DTDS','NO2_TDAcf','CO_SRF','TROPE3D_P','TROP_P', + 'CDNUMC','SFDMS','so4_a1_sfgaex1','so4_a2_sfgaex1','so4_a3_sfgaex1','so4_a5_sfgaex1','soa_a1_sfgaex1', + 'soa_a2_sfgaex1','soa_a3_sfgaex1','GS_soa_a1','GS_soa_a2','GS_soa_a3','AQSO4_H2O2','AQSO4_O3', + 'SFSO2','SO2_CLXF','SO2','DF_SO2','AQ_SO2','GS_SO2','WD_SO2','ABURDENSO4_STR','ABURDENSO4_TRO', + 'ABURDENSO4','ABURDENBC','ABURDENDUST','ABURDENMOM','ABURDENPOM','ABURDENSEASALT', + 'ABURDENSOA','AODSO4_STR','AODSO4_TRO', + 'EXTINCT','AODABS','AODABSBC','CLDICE','CLDLIQ','CLD_CAL_TMPLIQ','CLD_CAL_TMPICE','Mass_bc_srf', + 'Mass_dst_srf','Mass_mom_srf','Mass_ncl_srf','Mass_pom_srf','Mass_so4_srf','Mass_soa_srf','Mass_bc_850', + 'Mass_dst_850','Mass_mom_850','Mass_ncl_850','Mass_pom_850','Mass_so4_850','Mass_soa_850','Mass_bc_500', + 'Mass_dst_500','Mass_mom_500','Mass_ncl_500','Mass_pom_500','Mass_so4_500','Mass_soa_500','Mass_bc_330', + 'Mass_dst_330','Mass_mom_330','Mass_ncl_330','Mass_pom_330','Mass_so4_330','Mass_soa_330','Mass_bc_200', + 'Mass_dst_200','Mass_mom_200','Mass_ncl_200','Mass_pom_200','Mass_so4_200','Mass_soa_200', + 'O3_2DTDD','O3_2DCIP','O3_2DCIL','CO_2DTDS','CO_2DTDD','CO_2DCEP','CO_2DCEL','NO_2DTDD', + 'FLNTC','SAODVIS', + 'H2OLNZ', + 'dst_a1SF','dst_a3SF', + 'PHIS','CLOUD','TGCLDIWP','TGCLDCWP','AREL', + 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANPTOP_ISCCP','CLD_CAL', + 'CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN', + 'CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN', + 'CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN', + 'CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN', + 'CLWMODIS','CLIMODIS' + +fincl2 = 'PS', 'FLUT','PRECT','U200','V200','U850','V850', + 'TCO','SCO','TREFHTMN','TREFHTMX','TREFHT','QREFHT' +fincl3 = 'PS', 'PSL','PRECT','TUQ','TVQ','UBOT','VBOT','TREFHT','FLUT','OMEGA500','TBOT','U850','V850','U200','V200','T200','T500','Z700' +fincl4 = 'PRECT' +fincl5 = 'O3_SRF' +fincl6 = 'CO_2DMSD','NO2_2DMSD','NO_2DMSD','O3_2DMSD','O3_2DMSD_trop' + +! -- chemUCI settings ------------------ +history_chemdyg_summary = .true. +history_gaschmbudget_2D = .false. +history_gaschmbudget_2D_levels = .false. +history_gaschmbudget_num = 6 !! no impact if history_gaschmbudget_2D = .false. + +! -- MAM5 settings ------------------ +is_output_interactive_volc = .true. diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F2010/user_nl_elm b/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F2010/user_nl_elm index b2a7d247ac2b..cd1adab77404 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F2010/user_nl_elm +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F2010/user_nl_elm @@ -1,7 +1,40 @@ -! finidat below not compatitle with new t05 fsurdat with _TOP scheme. This test to be replaced later -! finidat = '${DIN_LOC_ROOT}/lnd/clm2/initdata_map/clmi.F2010.ne30pg2_r05_oECv3.SMS_Ln5.c20230213.nc' - hist_dov2xy = .true.,.true. - hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' - hist_mfilt = 1,365 - hist_nhtfrq = -24,-24 - hist_avgflag_pertape = 'A','A' +hist_dov2xy = .true.,.true. +hist_fexcl1 = 'AGWDNPP','ALTMAX_LASTYEAR','AVAIL_RETRANSP','AVAILC','BAF_CROP', + 'BAF_PEATF','BIOCHEM_PMIN_TO_PLANT','CH4_SURF_AERE_SAT','CH4_SURF_AERE_UNSAT','CH4_SURF_DIFF_SAT', + 'CH4_SURF_DIFF_UNSAT','CH4_SURF_EBUL_SAT','CH4_SURF_EBUL_UNSAT','CMASS_BALANCE_ERROR','cn_scalar', + 'COL_PTRUNC','CONC_CH4_SAT','CONC_CH4_UNSAT','CONC_O2_SAT','CONC_O2_UNSAT', + 'cp_scalar','CWDC_HR','CWDC_LOSS','CWDC_TO_LITR2C','CWDC_TO_LITR3C', + 'CWDC_vr','CWDN_TO_LITR2N','CWDN_TO_LITR3N','CWDN_vr','CWDP_TO_LITR2P', + 'CWDP_TO_LITR3P','CWDP_vr','DWT_CONV_CFLUX_DRIBBLED','F_CO2_SOIL','F_CO2_SOIL_vr', + 'F_DENIT_vr','F_N2O_DENIT','F_N2O_NIT','F_NIT_vr','FCH4_DFSAT', + 'FINUNDATED_LAG','FPI_P_vr','FPI_vr','FROOTC_LOSS','HR_vr', + 'LABILEP_TO_SECONDP','LABILEP_vr','LAND_UPTAKE','LEAF_MR','leaf_npimbalance', + 'LEAFC_LOSS','LEAFC_TO_LITTER','LFC2','LITR1_HR','LITR1C_TO_SOIL1C', + 'LITR1C_vr','LITR1N_TNDNCY_VERT_TRANS','LITR1N_TO_SOIL1N','LITR1N_vr','LITR1P_TNDNCY_VERT_TRANS', + 'LITR1P_TO_SOIL1P','LITR1P_vr','LITR2_HR','LITR2C_TO_SOIL2C','LITR2C_vr', + 'LITR2N_TNDNCY_VERT_TRANS','LITR2N_TO_SOIL2N','LITR2N_vr','LITR2P_TNDNCY_VERT_TRANS','LITR2P_TO_SOIL2P', + 'LITR2P_vr','LITR3_HR','LITR3C_TO_SOIL3C','LITR3C_vr','LITR3N_TNDNCY_VERT_TRANS', + 'LITR3N_TO_SOIL3N','LITR3N_vr','LITR3P_TNDNCY_VERT_TRANS','LITR3P_TO_SOIL3P','LITR3P_vr', + 'M_LITR1C_TO_LEACHING','M_LITR2C_TO_LEACHING','M_LITR3C_TO_LEACHING','M_SOIL1C_TO_LEACHING','M_SOIL2C_TO_LEACHING', + 'M_SOIL3C_TO_LEACHING','M_SOIL4C_TO_LEACHING','NDEPLOY','NEM','nlim_m', + 'o2_decomp_depth_unsat','OCCLP_vr','PDEPLOY','PLANT_CALLOC','PLANT_NDEMAND', + 'PLANT_NDEMAND_COL','PLANT_PALLOC','PLANT_PDEMAND','PLANT_PDEMAND_COL','plim_m', + 'POT_F_DENIT','POT_F_NIT','POTENTIAL_IMMOB','POTENTIAL_IMMOB_P','PRIMP_TO_LABILEP', + 'PRIMP_vr','PROD1P_LOSS','QOVER_LAG','RETRANSN_TO_NPOOL','RETRANSP_TO_PPOOL', + 'SCALARAVG_vr','SECONDP_TO_LABILEP','SECONDP_TO_OCCLP','SECONDP_vr','SMIN_NH4_vr', + 'SMIN_NO3_vr','SMINN_TO_SOIL1N_L1','SMINN_TO_SOIL2N_L2','SMINN_TO_SOIL2N_S1','SMINN_TO_SOIL3N_L3', + 'SMINN_TO_SOIL3N_S2','SMINN_TO_SOIL4N_S3','SMINP_TO_SOIL1P_L1','SMINP_TO_SOIL2P_L2','SMINP_TO_SOIL2P_S1', + 'SMINP_TO_SOIL3P_L3','SMINP_TO_SOIL3P_S2','SMINP_TO_SOIL4P_S3','SMINP_vr','SOIL1_HR','SOIL1C_TO_SOIL2C','SOIL1C_vr','SOIL1N_TNDNCY_VERT_TRANS','SOIL1N_TO_SOIL2N','SOIL1N_vr', + 'SOIL1P_TNDNCY_VERT_TRANS','SOIL1P_TO_SOIL2P','SOIL1P_vr','SOIL2_HR','SOIL2C_TO_SOIL3C', + 'SOIL2C_vr','SOIL2N_TNDNCY_VERT_TRANS','SOIL2N_TO_SOIL3N','SOIL2N_vr','SOIL2P_TNDNCY_VERT_TRANS', + 'SOIL2P_TO_SOIL3P','SOIL2P_vr','SOIL3_HR','SOIL3C_TO_SOIL4C','SOIL3C_vr', + 'SOIL3N_TNDNCY_VERT_TRANS','SOIL3N_TO_SOIL4N','SOIL3N_vr','SOIL3P_TNDNCY_VERT_TRANS','SOIL3P_TO_SOIL4P', + 'SOIL3P_vr','SOIL4_HR','SOIL4C_vr','SOIL4N_TNDNCY_VERT_TRANS','SOIL4N_TO_SMINN', + 'SOIL4N_vr','SOIL4P_TNDNCY_VERT_TRANS','SOIL4P_TO_SMINP','SOIL4P_vr','SOLUTIONP_vr', + 'TCS_MONTH_BEGIN','TCS_MONTH_END','TOTCOLCH4','water_scalar','WF', + 'wlim_m','WOODC_LOSS','WTGQ' +hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC' +hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' +hist_mfilt = 1,365 +hist_nhtfrq = -24,-24 +hist_avgflag_pertape = 'A','A' diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F20TR/shell_commands b/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F20TR/shell_commands index 6e8ae38ac8e9..0067b2f35de5 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F20TR/shell_commands +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F20TR/shell_commands @@ -1,3 +1,3 @@ #!/bin/bash -./xmlchange --append CAM_CONFIG_OPTS='-cosp' +./xmlchange --append CAM_CONFIG_OPTS='-cosp' diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F20TR/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F20TR/user_nl_eam index eeac1d647b1d..095c6946f5de 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F20TR/user_nl_eam +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F20TR/user_nl_eam @@ -1,11 +1,59 @@ - nhtfrq = -24,-24,-6,-6,-3,-24,-24 - mfilt = 1,30,120,120,240,30,1 - avgflag_pertape = 'A','A','I','A','A','A','I' - fexcl1 = 'CFAD_SR532_CAL', 'LINOZ_DO3', 'LINOZ_DO3_PSC', 'LINOZ_O3CLIM', 'LINOZ_O3COL', 'LINOZ_SSO3', 'hstobie_linoz' - fincl1 = 'extinct_sw_inp','extinct_lw_bnd7','extinct_lw_inp','CLD_CAL', 'TREFMNAV', 'TREFMXAV' - fincl2 = 'FLUT','PRECT','U200','V200','U850','V850','Z500','OMEGA500','UBOT','VBOT','TREFHT','TREFHTMN:M','TREFHTMX:X','QREFHT','TS','PS','TMQ','TUQ','TVQ','TOZ', 'FLDS', 'FLNS', 'FSDS', 'FSNS', 'SHFLX', 'LHFLX', 'TGCLDCWP', 'TGCLDIWP', 'TGCLDLWP', 'CLDTOT', 'T250', 'T200', 'T150', 'T100', 'T050', 'T025', 'T010', 'T005', 'T002', 'T001', 'TTOP', 'U250', 'U150', 'U100', 'U050', 'U025', 'U010', 'U005', 'U002', 'U001', 'UTOP', 'FSNT', 'FLNT' - fincl3 = 'PSL','T200','T500','U850','V850','UBOT','VBOT','TREFHT', 'Z700', 'TBOT:M' - fincl4 = 'FLUT','U200','U850','PRECT','OMEGA500' - fincl5 = 'PRECT','PRECC','TUQ','TVQ','QFLX','SHFLX','U90M','V90M' - fincl6 = 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANTAU_ISCCP','MEANPTOP_ISCCP','MEANTB_ISCCP','CLDTOT_CAL','CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN','CLDHGH_CAL','CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN','CLDMED_CAL','CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN','CLDLOW_CAL','CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN' - fincl7 = 'O3', 'PS', 'TROP_P' +cosp_lite = .true. + +empty_htapes = .true. + +avgflag_pertape = 'A','A','A','A','I','I' +nhtfrq = -24,-24,-6,-3,-1,-24 +mfilt = 1,30,120,240,720,1 + +fincl1 = 'AODALL','AODBC','AODDUST','AODPOM','AODSO4','AODSOA','AODSS','AODVIS', + 'CLDLOW','CLDMED','CLDHGH','CLDTOT', + 'CLDHGH_CAL','CLDLOW_CAL','CLDMED_CAL','CLD_MISR','CLDTOT_CAL', + 'CLMODIS','FISCCP1_COSP','FLDS','FLNS','FLNSC','FLNT','FLUT', + 'FLUTC','FSDS','FSDSC','FSNS','FSNSC','FSNT','FSNTOA','FSNTOAC','FSNTC', + 'ICEFRAC','LANDFRAC','LWCF','OCNFRAC','OMEGA','PRECC','PRECL','PRECSC','PRECSL','PS','PSL','Q', + 'QFLX','QREFHT','RELHUM','SCO','SHFLX','SOLIN','SWCF','T','TAUX','TAUY','TCO', + 'TGCLDLWP','TMQ','TREFHT','TREFMNAV','TREFMXAV','TS','U','U10','V','Z3', + 'dst_a1DDF','dst_a3DDF','dst_c1DDF','dst_c3DDF','dst_a1SFWET','dst_a3SFWET','dst_c1SFWET','dst_c3SFWET', + 'O3','LHFLX', + 'O3_2DTDA_trop','O3_2DTDB_trop','O3_2DTDD_trop','O3_2DTDE_trop','O3_2DTDI_trop','O3_2DTDL_trop', + 'O3_2DTDN_trop','O3_2DTDO_trop','O3_2DTDS_trop','O3_2DTDU_trop','O3_2DTRE_trop','O3_2DTRI_trop', + 'O3_SRF','NO_2DTDS','NO_TDLgt','NO2_2DTDD','NO2_2DTDS','NO2_TDAcf','CO_SRF','TROPE3D_P','TROP_P', + 'CDNUMC','SFDMS','so4_a1_sfgaex1','so4_a2_sfgaex1','so4_a3_sfgaex1','so4_a5_sfgaex1','soa_a1_sfgaex1', + 'soa_a2_sfgaex1','soa_a3_sfgaex1','GS_soa_a1','GS_soa_a2','GS_soa_a3','AQSO4_H2O2','AQSO4_O3', + 'SFSO2','SO2_CLXF','SO2','DF_SO2','AQ_SO2','GS_SO2','WD_SO2','ABURDENSO4_STR','ABURDENSO4_TRO', + 'ABURDENSO4','ABURDENBC','ABURDENDUST','ABURDENMOM','ABURDENPOM','ABURDENSEASALT', + 'ABURDENSOA','AODSO4_STR','AODSO4_TRO', + 'EXTINCT','AODABS','AODABSBC','CLDICE','CLDLIQ','CLD_CAL_TMPLIQ','CLD_CAL_TMPICE','Mass_bc_srf', + 'Mass_dst_srf','Mass_mom_srf','Mass_ncl_srf','Mass_pom_srf','Mass_so4_srf','Mass_soa_srf','Mass_bc_850', + 'Mass_dst_850','Mass_mom_850','Mass_ncl_850','Mass_pom_850','Mass_so4_850','Mass_soa_850','Mass_bc_500', + 'Mass_dst_500','Mass_mom_500','Mass_ncl_500','Mass_pom_500','Mass_so4_500','Mass_soa_500','Mass_bc_330', + 'Mass_dst_330','Mass_mom_330','Mass_ncl_330','Mass_pom_330','Mass_so4_330','Mass_soa_330','Mass_bc_200', + 'Mass_dst_200','Mass_mom_200','Mass_ncl_200','Mass_pom_200','Mass_so4_200','Mass_soa_200', + 'O3_2DTDD','O3_2DCIP','O3_2DCIL','CO_2DTDS','CO_2DTDD','CO_2DCEP','CO_2DCEL','NO_2DTDD', + 'FLNTC','SAODVIS', + 'H2OLNZ', + 'dst_a1SF','dst_a3SF', + 'PHIS','CLOUD','TGCLDIWP','TGCLDCWP','AREL', + 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANPTOP_ISCCP','CLD_CAL', + 'CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN', + 'CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN', + 'CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN', + 'CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN', + 'CLWMODIS','CLIMODIS' + +fincl2 = 'PS', 'FLUT','PRECT','U200','V200','U850','V850', + 'TCO','SCO','TREFHTMN','TREFHTMX','TREFHT','QREFHT' +fincl3 = 'PS', 'PSL','PRECT','TUQ','TVQ','UBOT','VBOT','TREFHT','FLUT','OMEGA500','TBOT','U850','V850','U200','V200','T200','T500','Z700' +fincl4 = 'PRECT' +fincl5 = 'O3_SRF' +fincl6 = 'CO_2DMSD','NO2_2DMSD','NO_2DMSD','O3_2DMSD','O3_2DMSD_trop' + +! -- chemUCI settings ------------------ +history_chemdyg_summary = .true. +history_gaschmbudget_2D = .false. +history_gaschmbudget_2D_levels = .false. +history_gaschmbudget_num = 6 !! no impact if history_gaschmbudget_2D = .false. + +! -- MAM5 settings ------------------ +is_output_interactive_volc = .true. diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F20TR/user_nl_elm b/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F20TR/user_nl_elm index b2a7d247ac2b..cd1adab77404 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F20TR/user_nl_elm +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F20TR/user_nl_elm @@ -1,7 +1,40 @@ -! finidat below not compatitle with new t05 fsurdat with _TOP scheme. This test to be replaced later -! finidat = '${DIN_LOC_ROOT}/lnd/clm2/initdata_map/clmi.F2010.ne30pg2_r05_oECv3.SMS_Ln5.c20230213.nc' - hist_dov2xy = .true.,.true. - hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' - hist_mfilt = 1,365 - hist_nhtfrq = -24,-24 - hist_avgflag_pertape = 'A','A' +hist_dov2xy = .true.,.true. +hist_fexcl1 = 'AGWDNPP','ALTMAX_LASTYEAR','AVAIL_RETRANSP','AVAILC','BAF_CROP', + 'BAF_PEATF','BIOCHEM_PMIN_TO_PLANT','CH4_SURF_AERE_SAT','CH4_SURF_AERE_UNSAT','CH4_SURF_DIFF_SAT', + 'CH4_SURF_DIFF_UNSAT','CH4_SURF_EBUL_SAT','CH4_SURF_EBUL_UNSAT','CMASS_BALANCE_ERROR','cn_scalar', + 'COL_PTRUNC','CONC_CH4_SAT','CONC_CH4_UNSAT','CONC_O2_SAT','CONC_O2_UNSAT', + 'cp_scalar','CWDC_HR','CWDC_LOSS','CWDC_TO_LITR2C','CWDC_TO_LITR3C', + 'CWDC_vr','CWDN_TO_LITR2N','CWDN_TO_LITR3N','CWDN_vr','CWDP_TO_LITR2P', + 'CWDP_TO_LITR3P','CWDP_vr','DWT_CONV_CFLUX_DRIBBLED','F_CO2_SOIL','F_CO2_SOIL_vr', + 'F_DENIT_vr','F_N2O_DENIT','F_N2O_NIT','F_NIT_vr','FCH4_DFSAT', + 'FINUNDATED_LAG','FPI_P_vr','FPI_vr','FROOTC_LOSS','HR_vr', + 'LABILEP_TO_SECONDP','LABILEP_vr','LAND_UPTAKE','LEAF_MR','leaf_npimbalance', + 'LEAFC_LOSS','LEAFC_TO_LITTER','LFC2','LITR1_HR','LITR1C_TO_SOIL1C', + 'LITR1C_vr','LITR1N_TNDNCY_VERT_TRANS','LITR1N_TO_SOIL1N','LITR1N_vr','LITR1P_TNDNCY_VERT_TRANS', + 'LITR1P_TO_SOIL1P','LITR1P_vr','LITR2_HR','LITR2C_TO_SOIL2C','LITR2C_vr', + 'LITR2N_TNDNCY_VERT_TRANS','LITR2N_TO_SOIL2N','LITR2N_vr','LITR2P_TNDNCY_VERT_TRANS','LITR2P_TO_SOIL2P', + 'LITR2P_vr','LITR3_HR','LITR3C_TO_SOIL3C','LITR3C_vr','LITR3N_TNDNCY_VERT_TRANS', + 'LITR3N_TO_SOIL3N','LITR3N_vr','LITR3P_TNDNCY_VERT_TRANS','LITR3P_TO_SOIL3P','LITR3P_vr', + 'M_LITR1C_TO_LEACHING','M_LITR2C_TO_LEACHING','M_LITR3C_TO_LEACHING','M_SOIL1C_TO_LEACHING','M_SOIL2C_TO_LEACHING', + 'M_SOIL3C_TO_LEACHING','M_SOIL4C_TO_LEACHING','NDEPLOY','NEM','nlim_m', + 'o2_decomp_depth_unsat','OCCLP_vr','PDEPLOY','PLANT_CALLOC','PLANT_NDEMAND', + 'PLANT_NDEMAND_COL','PLANT_PALLOC','PLANT_PDEMAND','PLANT_PDEMAND_COL','plim_m', + 'POT_F_DENIT','POT_F_NIT','POTENTIAL_IMMOB','POTENTIAL_IMMOB_P','PRIMP_TO_LABILEP', + 'PRIMP_vr','PROD1P_LOSS','QOVER_LAG','RETRANSN_TO_NPOOL','RETRANSP_TO_PPOOL', + 'SCALARAVG_vr','SECONDP_TO_LABILEP','SECONDP_TO_OCCLP','SECONDP_vr','SMIN_NH4_vr', + 'SMIN_NO3_vr','SMINN_TO_SOIL1N_L1','SMINN_TO_SOIL2N_L2','SMINN_TO_SOIL2N_S1','SMINN_TO_SOIL3N_L3', + 'SMINN_TO_SOIL3N_S2','SMINN_TO_SOIL4N_S3','SMINP_TO_SOIL1P_L1','SMINP_TO_SOIL2P_L2','SMINP_TO_SOIL2P_S1', + 'SMINP_TO_SOIL3P_L3','SMINP_TO_SOIL3P_S2','SMINP_TO_SOIL4P_S3','SMINP_vr','SOIL1_HR','SOIL1C_TO_SOIL2C','SOIL1C_vr','SOIL1N_TNDNCY_VERT_TRANS','SOIL1N_TO_SOIL2N','SOIL1N_vr', + 'SOIL1P_TNDNCY_VERT_TRANS','SOIL1P_TO_SOIL2P','SOIL1P_vr','SOIL2_HR','SOIL2C_TO_SOIL3C', + 'SOIL2C_vr','SOIL2N_TNDNCY_VERT_TRANS','SOIL2N_TO_SOIL3N','SOIL2N_vr','SOIL2P_TNDNCY_VERT_TRANS', + 'SOIL2P_TO_SOIL3P','SOIL2P_vr','SOIL3_HR','SOIL3C_TO_SOIL4C','SOIL3C_vr', + 'SOIL3N_TNDNCY_VERT_TRANS','SOIL3N_TO_SOIL4N','SOIL3N_vr','SOIL3P_TNDNCY_VERT_TRANS','SOIL3P_TO_SOIL4P', + 'SOIL3P_vr','SOIL4_HR','SOIL4C_vr','SOIL4N_TNDNCY_VERT_TRANS','SOIL4N_TO_SMINN', + 'SOIL4N_vr','SOIL4P_TNDNCY_VERT_TRANS','SOIL4P_TO_SMINP','SOIL4P_vr','SOLUTIONP_vr', + 'TCS_MONTH_BEGIN','TCS_MONTH_END','TOTCOLCH4','water_scalar','WF', + 'wlim_m','WOODC_LOSS','WTGQ' +hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC' +hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' +hist_mfilt = 1,365 +hist_nhtfrq = -24,-24 +hist_avgflag_pertape = 'A','A' diff --git a/components/eam/docs/dev-guide/index.md b/components/eam/docs/dev-guide/index.md index 553bf5f36975..6879c32cbe3c 100644 --- a/components/eam/docs/dev-guide/index.md +++ b/components/eam/docs/dev-guide/index.md @@ -1 +1,3 @@ -start of the EAM Developer's Guide +# EAM Developer Guide + +coming soon. diff --git a/components/eam/docs/index.md b/components/eam/docs/index.md index 6632c22ca62d..55870e3778d0 100644 --- a/components/eam/docs/index.md +++ b/components/eam/docs/index.md @@ -1,7 +1,7 @@ # The E3SM Atmosphere Model (EAM) -Some introductory text here +EAM is the state-of-the-art atmospheric component of the E3SM model that uses a Spectral Element Dynamical Core and a suite of parameterizations to represent a range of atmospheric processes, which are described in the Technical Guide (see below). Its nominal resolution is roughly 100km in the horizontal, with a model time step of 1800 seconds, and runs with 80 layers in the vertical (model top of 60km). -* The [EAM User's Guide](user-guide/index.md) explains how to control EAM when its running within E3SM. -* The [EAM Developer's Guide](dev-guide/index.md) explains EAM data structures and how to write new code. -* The [EAM Techincal Guide](tech-guide/index.md) explains the science behind EAM's code +* The [EAM User Guide](user-guide/index.md) explains how to control EAM when its running within E3SM. +* The [EAM Developer Guide](dev-guide/index.md) explains EAM data structures and how to write new code. +* The [EAM Technical Guide](tech-guide/index.md) explains the science behind EAM's code diff --git a/components/eam/docs/tech-guide/armdiags.md b/components/eam/docs/tech-guide/armdiags.md new file mode 100644 index 000000000000..47266db48f2a --- /dev/null +++ b/components/eam/docs/tech-guide/armdiags.md @@ -0,0 +1,56 @@ +# ARM diagnostics + +## Overview + +The ARM data-oriented metrics and diagnostics package (ARM Diags) was developed to facilitate the use of ARM data in climate model evaluation and model intercomparison (Zhang et al., 2020)[@zhang_arm_2020]. It includes ARM data sets, compiled from multiple ARM data products, and a Python-based analysis toolkit for computation ad visualization. It also includes simulation data from models participating the CMIP, which allows climate-modeling groups to compare a new, candidate version of their model to existing CMIP models. The ARM Diags has been applied in several model evaluation studies to help address a range of issues in climate models (Zheng et al., 2023;[@zheng_assessment_2023] Emmenegger et al., 2022;[@emmenegger_evaluating_2022] Zhang et al., 2018[@zhang_causes_2018]). The Majority of ARM Diags sets are ported into E3SM Diags (Zhang et al., 2022)[@zhang_e3sm_2022] for routine evaluation of the model. + +## To enable the use of ARM Diags + +To enable using ARM Diags for a simulation, often, a new tape that output at high-frequency over limited-area (nearest grid box to supported ARM site) needs to be included in the namelist file, an example as follows: + +```fortran +fincl7 = 'PS','Q','T','Z3','CLOUD','CONCLD','CLDICE', + 'CLDLIQ','FREQR','REI','REL','PRECT','TMQ','PRECC', + 'TREFHT','QREFHT','OMEGA','CLDTOT','LHFLX','SHFLX', + 'FLDS','FSDS','FLNS','FSNS','FLNSC','FSDSC','FSNSC', + 'AODVIS','AODABS','LS_FLXPRC','LS_FLXSNW', + 'LS_REFFRAIN','ZMFLXPRC','ZMFLXSNW','CCN1','CCN2', + 'CCN3','CCN4','CCN5','num_a1','num_a2','num_a3', + 'num_a4','so4_a1','so4_a2','so4_a3','AREL','TGCLDLWP', + 'AQRAIN','ANRAIN','FREQR','PRECL','RELHUM' +fincl7lonlat='262.5e_36.6n','203.4e_71.3n','147.4e_2.0s', + '166.9e_0.5s','130.9e_12.4s','331.97e_39.09n' +``` + +Note that in this example fincl7 should set to write output at hourly (`nhtfrq = -1`). And here additional variables are included for ARM simulator analysis. The ARM site information is shown below: + +```fortran + "sgpc1": ["97.5W 36.4N Oklahoma ARM"], + + "nsac1": ["156.6W 71.3N Barrow ARM"], + + "twpc1": ["147.4E 2.1S Manus ARM"], + + "twpc2": ["166.9E 0.5S Nauru ARM"], + + "twpc3": ["130.9E 12.4S Darwin ARM"], + + "enac1": ["28.0E 39.1N Graciosa Island ARM"], +``` + +## Diagnostics and metrics currently implemented in the ARM Diags + +| Statistical Metrics | Variables | Time sampling | +| ------------------------- | --------------------------------------------------------------- | ----------------- | +| Line plots and Taylor diagrams for annual cycle variability of each variable | Precipitation, column water vapor, surface energy budget components, near-surface temperature and specific humidity, surface pressure, total cloud fraction, and aerosol optical depth. | Monthly mean | +| Contour and vertical profiles of annual cycle and diurnal cycle of cloud fraction | Vertical profiles of cloud fraction | Hourly mean | +| Line and harmonic dial plots of diurnal cycle of precipitation | Surface precipitation rate | Hourly mean | +| Probability density function (PDF) plots of precipitation rate | Surface precipitation rate | Hourly mean | +| CCN Annual Cycles | CCN number concentrations at 0.1%, 0.2%, 0.5% and 1.0% supersaturation levels | Hourly mean | +| Aerosol Annual Cycles | Total aerosol number concentration | Hourly mean | +| Aerosol Chemical Annual Cycles | Organic, sulfate, nitrate, ammonium, chloride mass concentration | Hourly mean | + +| Process-oriented metrics | Variables | Time sampling | +| ------------------------- | ------------------------------------------------------------- | ----------------- | +| Convection Onset | 1. Surface precipitation rate
2. Column Precipitable Water Vapor | Hourly mean | +| Aerosol-CCN Activation | 1. Total aerosol number concentration
2. CCN number concentrations at different supersaturation levels (0.1%, 0.2%, 0.5% and 1.0) | Hourly mean | diff --git a/components/eam/docs/tech-guide/chemUCIlinozv3.md b/components/eam/docs/tech-guide/chemUCIlinozv3.md new file mode 100644 index 000000000000..b6b202c00f8c --- /dev/null +++ b/components/eam/docs/tech-guide/chemUCIlinozv3.md @@ -0,0 +1,9 @@ +# chemUCI and Linoz + +## Overview + +Atmospheric interactive chemistry is handled by chemUCI (in the troposphere) and Linoz v3 (in the stratosphere). chemUCI consists of 28 advected tracers for the O3-CH4-HOx-NOx-NMVOCs chemistry. Compared to E3SMv2, the E3SMv3 linearized stratospheric chemistry scheme (Linoz v3) expends the interactive species to include O3, N2O, NOy, and CH4. The boundary between stratosphere and troposphere adopts the e90 tropopause algorithm. + +## Namelist parameters + +[chemUCI and Linoz Namelist Parameters](../user-guide/namelist_parameters.md#chemuci-and-linoz-v3) diff --git a/components/eam/docs/tech-guide/clubb.md b/components/eam/docs/tech-guide/clubb.md new file mode 100644 index 000000000000..2737f753850e --- /dev/null +++ b/components/eam/docs/tech-guide/clubb.md @@ -0,0 +1,9 @@ +# Cloud Layers Unified By Binormals + +## Overview + +The Cloud Layers Unified By Binormals (CLUBB) parameterization is a parameterization of subgrid-scale turbulence and clouds.[@larson_clubb-silhs_2022,@bogenschutz_path_2018,@larson_using_2005,@golaz_pdf-based_2002] It prognoses turbulent fluxes of heat, moisture, and momentum, and it diagnoses the liquid cloud fraction and liquid water mixing ratio. To do so, it prognoses higher-order turbulence moments and closes those prognostic equations by use of an assumed double-Gaussian shape of the subgrid probability density function. CLUBB operates throughout the troposphere, but it contributes especially to the planetary boundary layer and low-cloud regimes, including stratocumulus and shallow cumulus regimes. + +## Namelist parameters + +[CLUBB Namelist Parameters](../user-guide/namelist_parameters.md#cloud-layers-unified-by-binormals) diff --git a/components/eam/docs/tech-guide/cosp.md b/components/eam/docs/tech-guide/cosp.md new file mode 100644 index 000000000000..ef724b12a9f5 --- /dev/null +++ b/components/eam/docs/tech-guide/cosp.md @@ -0,0 +1,27 @@ +# Cloud Feedback Model Intercomparison Project (CFMIP) Observation Simulator Package + +## Overview + +The Cloud Feedback Model Intercomparison Project (CFMIP) Observation Simulator Package (COSP; Bodas-Salcedo et al., 2011;[@bodas-salcedo_cosp_2011] Swales et al., 2018;[@swales_cloud_2018] Zhang et al., 2019;[@zhang_evaluation_2019] Zhang et al., 2024[@zhang_understanding_2024]) was developed to improve the consistency between model clouds and satellite observations. COSP contains several independent satellite simulators for better comparing model clouds with satellite measurements collected by the International Satellite Cloud Climatology Project (ISCCP), the Moderate Resolution Imaging Spectroradiometer (MODIS), the Multi-angle Imaging SpectroRadiometer (MISR), Cloud-Aerosol Lidar and Infrared Pathfinder Satellite Observation (CALIPSO), and CloudSat. The use of satellite simulators will not only make a fairer comparison between model clouds and satellite data but also allow a more in-depth analysis of clouds. For example, clouds can be assessed in terms of their optical properties and vertical location, which dictate their radiative effects. + +## Namelist parameters + +[COSP Namelist Parameters](../user-guide/namelist_parameters.md#cloud-feedback-model-intercomparison-project-cfmip-observation-simulator-package) + +## To turn on COSP outputs + +Run (add to the run script) the following command before running `./case.setup` + +`./xmlchange --id CAM_CONFIG_OPTS --append --val='-cosp'` + +| Related Outputs | Description | +| ------------------------- | ----------------------------------------------------------------- | +| `FISCCP1_COSP` | Grid-box fraction covered by each ISCCP D level cloud type | +| `CLMODIS` | MODIS Cloud Area Fraction | +| `CLD_MISR` | Cloud Fraction from MISR Simulator | +| `CLDTOT_CAL` | Calipso Total Cloud Fraction | +| `CLDHGH_CAL` | Calipso High-level Cloud Fraction | +| `CLDMED_CAL` | Calipso Mid-level Cloud Fraction | +| `CLDLOW_CAL` | Calipso Low-level Cloud Fraction | +| `CLD_CAL_TMPLIQ` | Calipso Liquid Cloud Fraction as a function of temperature | +| `CLD_CAL_TMPICE` | Calipso Ice Cloud Fraction as a function of temperature | diff --git a/components/eam/docs/tech-guide/dust.md b/components/eam/docs/tech-guide/dust.md new file mode 100644 index 000000000000..59f7cc3a966b --- /dev/null +++ b/components/eam/docs/tech-guide/dust.md @@ -0,0 +1,9 @@ +# Dust aerosol + +## Overview + +Dust-related processes are represented in the E3SM atmosphere and land model components. In E3SMv3, dust deposition will also be coupled with the sea ice and ocean biogeochemistry in the ocean model. Total emission fluxes of dust particles are calculated at each model time step. A new dust emission scheme following Kok et al. (2014)[@kok_improved_2014] is implemented to E3SMv3, replacing the Zender scheme (Zender et al., 2003)[@zender_mineral_2003] in E3SMv1 and v2 as the default option. The new dust emission scheme includes a time-varying soil erodibility factor for dust mobilization, and includes dust sources in high latitudes (e.g., >60 degree N). A manuscript by Feng et al. is in prep to document the performance of the new emission scheme on dust life cycle and radiative effects in E3SMv3. Dust aerosol is represented in the accumulation and coarse aerosol modes of the MAM4 module following emission. Other dust properties such as optical properties and size distribution at emission are documented in Feng et al. (2022).[@feng_global_2022] + +## Namelist parameters + +[Dust Namelist Parameters](../user-guide/namelist_parameters.md#dust-aerosol) diff --git a/components/eam/docs/tech-guide/homme.md b/components/eam/docs/tech-guide/homme.md new file mode 100644 index 000000000000..42561abde98b --- /dev/null +++ b/components/eam/docs/tech-guide/homme.md @@ -0,0 +1,11 @@ +# High-Order Methods Modeling Environment + +## Overview + +EAM uses the a dynamical core (dycore) from the High Order Method Modeling Environment (HOMME).[@taylor_compatible_2010,@guba_spectral_2014,@taylor_energy_2020] The EAM dycore solves the atmospheric primitive equations governing the evolution of velocity, density, pressure and temperature, as well as the transport of water species and related hydrometers, aerosols and other atmospheric constituents. The governing equations are written in a vertically lagrangian terrain following mass coordinate. They are discretized with second order finite differences in the radial direction and spectral finite elements in the horizontal (surface of the sphere) directions, and advanced in time with a 3rd order accurate 5 stage Runge-Kutta method. Dissipation is added through the use of monotoncity contraints on some advectiion terms, explicitly added hyperviscosity, and a Laplacian-based sponge layer in the first few layers at the model top. The transported species makes use of an efficient interpolatory semi-Lagrangian method. EAMv3 uses 80 layers in the vertical. The use of the spectral finite element method allows EAMv3 to run on fully unstructured grids, including the cubed-sphere grid ([SE Atmosphere Grid Overview (EAM & CAM)](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/34113147)) which provides quasi-uniform resolution over the globe, and regionally refined meshes (RRM) which enhance horizontal resolution in regions of interest ([Library of Regionally-Refined Model (RRM) Grids](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/3690397775)). + +## Namelist parameters + +Many dynamical core parameters can not be changed independently. For example, increasing the hyperviscosity coefficient may require reducing the hyperviscosity timestep. Dycore timesteps are tuned for each resolution and the defaults are close to the CFL stability limit. For complete details, as well as their interactions, see [EAM's HOMME dycore](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/1044644202/EAM+s+HOMME+dycore). + +[HOMME Namelist Parameters](../user-guide/namelist_parameters.md#homme) diff --git a/components/eam/docs/tech-guide/index.md b/components/eam/docs/tech-guide/index.md index 44a4f1921e25..437f5636edfe 100644 --- a/components/eam/docs/tech-guide/index.md +++ b/components/eam/docs/tech-guide/index.md @@ -1 +1,31 @@ -start of the EAM Technical Guide +# EAM Technical Guide + +This Technical Guide describes the physics of version 3 of the E3SM Atmospheric Model + +## Dynamics and Physics + +- [HOMME](homme.md): Dynamical core. + +- [P3](p3.md): Stratiform cloud microphysics scheme. + +- [CLUBB](clubb.md): Parameterization of subgrid-scale turbulence and clouds. + +- [Zhang-McFarlane](zm.md): Deep convection parameterization. + +- [RRTMG](rrtmg.md): Parameterization of radiation. + +- [MAM](mam.md): Primary parameterization schemes used to represent aerosols. + +- [VBS](vbs.md): Parameterization of secondary organic aerosols. + +- [Dust](dust.md): Parameterization of dust emissions. + +- [OCEANFILMS](oceanfilms.md): Parameterization of sea soray irganic aerosol emissions. + +- [chemUCI + Linoz v3](chemUCIlinozv3.md): Interactive atmospheric chemistry packages. + +## Diagnostic outputs + +- [COSP](cosp.md): Scheme that allows the model to output satellite simulator output. + +- [ARM Diags](armdiags.md): Diagnostic package that allows the model output to be compared against ARM measurements. diff --git a/components/eam/docs/tech-guide/mam.md b/components/eam/docs/tech-guide/mam.md new file mode 100644 index 000000000000..99886682f463 --- /dev/null +++ b/components/eam/docs/tech-guide/mam.md @@ -0,0 +1,17 @@ +# Modal Aerosol Module + +## MAM5 Overview + +The Five-mode Modal Aerosol Model (MAM5) supersedes the MAM4 utilized in previous iterations of E3SM (E3SM-V1 and -V2). MAM5 introduces a fifth mode, specifically designed to represent stratospheric coarse mode aerosols, primarily originating from volcanic eruptions and DMS decomposition. This mode exclusively comprises sulfate aerosols, characterized by a smaller standard deviation (STD) value of 1.2. The STD value denotes the width of the aerosol mode, where a higher STD implies a greater gravitational settling effect (Wang et al., 2020;[@wang_aerosols_2020] Liu et al., 2012[@liu_toward_2012]). By setting the STD to 1.2, the simulated properties of volcanic aerosols align closely with observational findings (Mills et al., 2016).[@mills_global_2016] MAM5 represents a pioneering aerosol model, effectively segregating tropospheric and stratospheric aerosols (Ke et al., in preparation), thereby mitigating the risk of overestimating dust and sea salt aerosols within the stratosphere in previous MAM4 (Visioni et al., 2021).[@visioni_limitations_2022] Volcanic eruptions derived from Neely and Schmidt (2016).[@neely_iii_volcaneesm_2016] + +## MAM4 Overview + +The representation of atmospheric aerosols and their roles in the Earth system by EAMv1/v2/v3 was inherited from the global aerosol-climate model EAMv0 and its four-mode modal aerosol module (MAM4), including Aitken, primary-carbon, accumulation, and coarse modes (Liu et al., 2016).[@liu_description_2016] It treats a combination of processes, controlling the evolution of aerosols that are either directly emitted or converted from precursor gases from a variety of natural and anthropogenic sources. The processes include transport (by grid-scale wind, subgrid turbulence, convection, and sedimentation), aerosol microphysics (i.e., particle nucleation, condensation/evaporation of trace gases, aging, and coagulation), cloud processing (i.e., aqueous chemistry, scavenging by hydrometeors, resuspension from evaporating hydrometeors, and wet deposition), and dry deposition. Aerosol species in the original MAM4 (Liu et al., 2016)[@liu_description_2016] include sulfate, primary organic aerosol (POA) or particulate organic matter (POM), secondary organic aerosol (SOA), black carbon (BC), sea salt, and mineral dust. As described by Wang et al. (2020),[@wang_aerosols_2020] the enhanced MAM4 in EAMv1/v2 added marine organic aerosol (MOA) to all four modes (Burrows et al., 2022).[@burrows_oceanfilms_2022] In MAM4 of EAMv3, the Aitken mode has sulfate, sea salt, SOA and MOA; the primary-carbon mode has BC, POA and MOA; the accumulation and coarse modes include all seven species. Ammonium (NH4) and nitrate (NO3) aerosols are also explicitly treated in EAMv3 (Wu et al., 2022),[@wu_development_2022] as an optional feature for research, in which new species (NH4, NO3, Ca, CO3, Na, Cl) are introduced to the Aitken, accumulation and coarse modes. All aerosol species within each of the four individual modes the MAM4 is assumed to be internally mixed and represented by a single number concentration, while particles are externally mixed among the different modes. + +### Sea salt + +In MAM4, sea salt aerosol is represented in the Aitken, accumulation, and coarse mode with particle emission size (diameter) ranges of 0.02-0.08, 0.08-1.0, and 1.0-10.0 μm, respectively. The emission flux of natural sea salt is first divided into 31 size bins, following the parameterization of Mårtensson et al. (2003)[@martensson_laboratory_2003] and Monahan et al. (1986),[@monahan_model_1986] and then redistributed to the three MAM4 size modes. + +## Namelist parameters + +[MAM Namelist Parameters](../user-guide/namelist_parameters.md#modal-aerosol-module) diff --git a/components/eam/docs/tech-guide/oceanfilms.md b/components/eam/docs/tech-guide/oceanfilms.md new file mode 100644 index 000000000000..1ac45cfd7999 --- /dev/null +++ b/components/eam/docs/tech-guide/oceanfilms.md @@ -0,0 +1,9 @@ +# OCEANFILMS + +## Overview + +E3SM (v1-v3) uses the OCEANFILMS (Organic Compounds from Ecosystems to Aerosols: Natural Films and Interfaces via Langmuir Molecular Surfactants) parameterization to represent sea spray organic aerosol emissions. OCEANFILMS is a physically based model that links sea spray chemistry with ocean biogeochemistry using a Langmuir partitioning approach. The underlying physical assumptions and parameterization are described in Burrows et al. (2014);[@burrows_physically_2014] the implementation in E3SM and impact on clouds and climate are documented in Burrows et al. (2022).[@burrows_oceanfilms_2022] + +## Namelist parameters + +[OCEANFILMS Namelist Parameters](../user-guide/namelist_parameters.md#oceanfilms) diff --git a/components/eam/docs/tech-guide/p3.md b/components/eam/docs/tech-guide/p3.md new file mode 100644 index 000000000000..af06aaa6aab0 --- /dev/null +++ b/components/eam/docs/tech-guide/p3.md @@ -0,0 +1,9 @@ +# Predicted Particle Properties + +## Overview + +The stratiform cloud microphysics scheme in v3 is Predicted Particle Properties (P3; Morrison & Milbrandt, 2015;[@morrison_parameterization_2015] Milbrandt & Morrison, 2016[@milbrandt_parameterization_2016]). P3 offers a new approach to representing the evolution of ice particles that is more physical than the traditional approach of using predefined ice categories. It has been implemented in E3SM (Wang et al., 2021)[@wang_impact_2021] to address the limitations in the original microphysics scheme- the lack of riming particles and the artificial conversion between ice crystals and snow particles. The current version in E3SM is a two-moment scheme with a single ice category (Morrison & Milbrandt, 2015).[@morrison_parameterization_2015] In addition to the total ice number and mass mixing ratios, P3 prognoses the rimed mass and rimed volume mixing ratios, which allows for the prediction of the continuous evolution of the rime fraction and particle density. It is worth noting that the ice nucleation parameterizations are changed to be aerosol-dependent in this study, with the heterogenous ice nucleation parameterizations from the Classical Nucleation Theory (Liu et al., 2012)[@liu_toward_2012] and the homogenous in-situ cirrus formation based on Liu and Penner (2005).[@liu_ice_2005] This differs from the P3 used in WRF and that used in the E3SM v1 in Wang et al. (2021)[@wang_impact_2021] where the heterogeneous ice nucleation parameterizations are temperature dependent only. + +## Namelist parameters + +[P3 Namelist Parameters](../user-guide/namelist_parameters.md#predicted-particle-properties) diff --git a/components/eam/docs/tech-guide/rrtmg.md b/components/eam/docs/tech-guide/rrtmg.md new file mode 100644 index 000000000000..7afa8df61224 --- /dev/null +++ b/components/eam/docs/tech-guide/rrtmg.md @@ -0,0 +1,9 @@ +# Rapid Radiative Transfer Model for GCMs + +## Overview + +The calculation of radiative energy flux through the atmosphere is done using the RRTMG radiation package (Iacono et al., 2008;[@iacono_radiative_2008] Mlawer et al., 1997[@mlawer_radiative_1997]). The details are consistent with the implementation in CAM5 described in Neale et al. (2012).[@neale_description_2012] Radiative fluxes are broadly split into shortwave and longwave and computed by separate codes. The shortwave solver uses the 2-stream approximation, while the longwave is an absorption/emission code. Both shortwave and longwave use the correlated k-distribution method for integration of fluxes. Subgrid cloud overlap is accounted for using the Monte Carlo Independent Column Approximation (MCICA; Pincus and Morcrette, 2003),[@pincus_fast_2003] assuming the cloudy portions of the column are maximally overlapped in vertically contiguous layers and randomly overlapped when two layers are separated by a completely clear layer. Cloud optics are parameterized as described in Neale et al.(2010).[@neale_description_2012] + +## Namelist parameters + +[RRTMG Namelist Parameters](../user-guide/namelist_parameters.md#rapid-radiative-transfer-model-for-gcms) diff --git a/components/eam/docs/tech-guide/vbs.md b/components/eam/docs/tech-guide/vbs.md new file mode 100644 index 000000000000..26535a89308d --- /dev/null +++ b/components/eam/docs/tech-guide/vbs.md @@ -0,0 +1,5 @@ +# Volatility Basis Set (VBS) approach for SOA + +## Overview + +A modified volatility basis set (VBS) approach is used for both SOA precursor gases and particulate SOA that includes gas‐phase functionalization/fragmentation and particle‐phase oligomerization similar to FragNVSOA configuration of Shrivastava et al. (2015).[@shrivastava_global_2015] It includes a detailed treatment of SOA precursor gas chemistry including multigenerational aging via fragmentation and functionalization reactions, particle‐phase oligomerization that generates low “effective volatility” SOA, and particle‐phase loss by photolysis. The sources of SOA include natural biogenic, anthropogenic and biomass burning organic gases that are oxidized to form lower volatility species and undergo dynamic gas-particle partitioning to form SOA. Biogenic SOA is formed by oxidation of isoprene (ISOP_VBS) and monoterpene (C10H16) volatile organic compounds (VOCs). Emissions of anthropogenic and biomass burning organic gases in the range of intermediate volatility organics (IVOCs, referred to as SOAG0) are estimated as total primary organic matter (POM) emitted from these sources multiplied by specified tunable factors. IVOCs undergo multigenerational aging with OH radicals forming SOA corresponding to anthropogenic and biomass burning sources. Photolysis of SOA is included as a chemical sink in the particle phase, in addition to dry and wet removal sinks. The photolysis rate constant of particle-phase SOA is assumed to be 0.04% of typical NO2 photolysis frequencies following Hodzic et al. (2016).[@hodzic_rethinking_2016] The emissions of VBS SOA related gas species and oxidants (prescribed) read from a file are documented in the [User Guide](../user-guide/index.md). diff --git a/components/eam/docs/tech-guide/zm.md b/components/eam/docs/tech-guide/zm.md new file mode 100644 index 000000000000..2e3c5b9b6070 --- /dev/null +++ b/components/eam/docs/tech-guide/zm.md @@ -0,0 +1,27 @@ +# Zhang and McFarlane deep convection scheme + +## Overview + +The ZM scheme (Zhang and McFarlane, 1995)[@zhang_sensitivity_1995] used in E3SMv3 is a bulk mass flux-type scheme; it has three components: a trigger for convection initiation, a cloud model including both updrafts and downdrafts, and a closure. The original CAPE-based trigger for convection was replaced by a trigger function based on dynamic CAPE generation by Xie et al. (2019)[@xie_improved_2019] (see dCAPE-ULL description below for more details). The closure predicts cloud base mass flux using dilute CAPE (Neale et al., 2008).[@neale_impact_2008] The updraft model is a bulk entraining plume model. Both updrafts and downdrafts are assumed saturated, with downdraft mass flux at the downdraft initiation level set proportional to the updraft cloud base mass flux. The microphysical processes inside the updrafts are represented by a convective microphysics scheme (see ZM convective microphysics description below). An additional adjustment is made to cloud base mass flux to incorporate the effect of large-scale circulation (see mass flux adjustment description below). + +### dCAPE-ULL + +A notable update related to clouds and precipitation in E3SMv2 is the use of a new convective trigger function described by Xie et al. (2019)[@xie_improved_2019] in ZM to improve the simulation of precipitation and its diurnal cycle. The new convective trigger named as dCAPE-ULL uses the dynamic CAPE (dCAPE) trigger developed by Xie and Zhang (2000)[@xie_impact_2000] with an unrestricted air parcel launch level (ULL) approach used by Wang et al. (2015).[@wang_impacts_2015] It was designed to address the unrealistically strong coupling of convection to the surface heating in ZM that often results in unrealistically too active model convection during the day in summer season over lands and improve the model capability to capture mid-level convection for nocturnal precipitation. + +### ZM convective microphysics + +The convective microphysics scheme is based on the work of Song and Zhang (2011)[@song_microphysics_2011] to improve the representation of microphysical processes in convective clouds and their interaction with aerosol and stratiform clouds in GCMs. It explicitly treats the mass mixing ratio and number concentration of five hydrometeor species (cloud water, cloud ice, rain, snow, and graupel). The scheme is linked to stratiform cloud microphysics parameterization through convective detrainment of cloud liquid/ice water content and droplet/crystal number concentration, and to aerosols through cloud droplet activation and ice nucleation processes. Previous evaluations of the scheme showed that it improved the simulation of convective cloud properties and cloud hydrological cycle (Song et al., 2012;[@song_evaluation_2012] Storer et al., 2015[@storer_effects_2015]). The assessment demonstrates that the convective microphysics scheme not only significantly improves the simulation of tropical variability across multiple scales but also enhances the simulation of climatological mean states. + +### Mass flux adjustment + +The convective mass flux adjustment (MAdj) is designed to represent the dynamical effects of large-scale vertical motion on convection. With MAdj, convection is enhanced (suppressed) when there is large-scale ascending (descending) motion at the planetary boundary layer top. The coupling of convection with the large-scale circulation significantly improves the simulation of climate variability across multiple scales from diurnal cycle, convectively coupled equatorial waves, to Madden-Julian oscillations (Song et al., 2023).[@song_incorporating_2023] + +### MCSP + +Due to inadequate model resolution, organized mesoscale convection cannot be resolved in general circulation models and thus needs to be parameterized. The Multiscale Coherent Structure Parameterization (MCSP) aims at representing the dynamical and physical effects of organized mesoscale convection. + +MCSP applies a sinusoidal baroclinic profile in the temperature, moisture, and momentum fields to represent the impact. Moncrieff et al. (2017)[@moncrieff_simulation_2017] and Chen et al. (2021)[@chen_effects_2021] have found that by adding MCSP, the both the representation of large-scale precipitation systems and the modes of variability from Tropical waves are improved. + +## Namelist parameters + +[ZM Namelist Parameters](../user-guide/namelist_parameters.md#zhang-and-mcfarlane-deep-convection-scheme) diff --git a/components/eam/docs/user-guide/aerosol_phys_prop.md b/components/eam/docs/user-guide/aerosol_phys_prop.md new file mode 100644 index 000000000000..6d80565d9718 --- /dev/null +++ b/components/eam/docs/user-guide/aerosol_phys_prop.md @@ -0,0 +1,111 @@ + +# Aerosol physical properties + +## Description + +Key information + +- Original physical properties of aerosols are documented in Liu et al. (2012).[@liu_toward_2012] Detailed information is included in the supplement. + +- Physical properties of aerosols used in E3SMv1 are documented in Wang et al. (2020).[@wang_aerosols_2020] + +- Marine organic aerosol properties are defined in Burrows et al. (2022).[@burrows_oceanfilms_2022] + +- Dust refractive index and longwave absorption coefficients are updated by Feng et al. (2022).[@feng_global_2022] + +- BC and POM hygroscopicity values are updated by Shan et al. (2024). + +- Physical properties of the fifth mode aerosols are documented by Ke et al. (2024). + +## Included fields + +### Mode properties + +- Geometric standard deviation of each lognormal mode + +- Nominal geometric mean diameter and its lower/upper bound of values for each mode + +- Coefficients of polynomial expression (lookup-table) for extinction, absorption, and asymmetry parameter calculations + +- Aerosol refractive index table needed for interpolation (lookup-table) calculation (for different wavelengths) + +- Crystalization and deliquesence relative humidity thresholds for aerosol wateruptake calculations + +### Species properties + +- Aerosol refractive index for each species + +- Density for each species + +- Aerosol hygroscopicity for each species + +- Note that some of variables in the species property file are for bulk aerosols, so we ignore the description for them. + +## Files + +Species properties + +```fortran +/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/sulfate_rrtmg_c080918.nc +/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/ocpho_rrtmg_c130709_kPOM0.04.nc +/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/ocphi_rrtmg_c100508.nc +/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/bcpho_rrtmg_c100508.nc +/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/dust_aeronet_rrtmg_c141106.nc +/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/ssam_rrtmg_c100508.nc +/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/poly_rrtmg_c130816.nc +``` + +Mode properties + +```fortran +/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/mam4_mode1_rrtmg_aeronetdust_c141106.nc', +/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/mam4_mode2_rrtmg_c130628.nc', +/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/mam4_mode3_rrtmg_aeronetdust_c141106.nc', +/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/mam4_mode4_rrtmg_c130628.nc', +/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/mam5_mode5_rrtmg_sig1.2_dgnl.40_c03072023.nc' +``` + +## Namelist + +```fortran + mode_defs = 'mam5_mode1:accum:=', + 'A:num_a1:N:num_c1:num_mr:+', + 'A:so4_a1:N:so4_c1:sulfate:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/sulfate_rrtmg_c080918.nc:+', + 'A:pom_a1:N:pom_c1:p-organic:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/ocpho_rrtmg_c130709_kPOM0.04.nc:+', + 'A:soa_a1:N:soa_c1:s-organic:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/ocphi_rrtmg_c100508.nc:+', + 'A:bc_a1:N:bc_c1:black-c:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/bcpho_rrtmg_c100508.nc:+', + 'A:dst_a1:N:dst_c1:dust:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/dust_aeronet_rrtmg_c141106.nc:+', + 'A:ncl_a1:N:ncl_c1:seasalt:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/ssam_rrtmg_c100508.nc:+', + 'A:mom_a1:N:mom_c1:m-organic:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/poly_rrtmg_c130816.nc', + 'mam5_mode2:aitken:=', + 'A:num_a2:N:num_c2:num_mr:+', + 'A:so4_a2:N:so4_c2:sulfate:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/sulfate_rrtmg_c080918.nc:+', + 'A:soa_a2:N:soa_c2:s-organic:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/ocphi_rrtmg_c100508.nc:+', + 'A:ncl_a2:N:ncl_c2:seasalt:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/ssam_rrtmg_c100508.nc:+', + 'A:mom_a2:N:mom_c2:m-organic:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/poly_rrtmg_c130816.nc', + 'mam5_mode3:coarse:=', + 'A:num_a3:N:num_c3:num_mr:+', + 'A:dst_a3:N:dst_c3:dust:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/dust_aeronet_rrtmg_c141106.nc:+', + 'A:ncl_a3:N:ncl_c3:seasalt:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/ssam_rrtmg_c100508.nc:+', + 'A:so4_a3:N:so4_c3:sulfate:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/sulfate_rrtmg_c080918.nc:+', + 'A:bc_a3:N:bc_c3:black-c:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/bcpho_rrtmg_c100508.nc:+', + 'A:pom_a3:N:pom_c3:p-organic:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/ocpho_rrtmg_c130709_kPOM0.04.nc:+', + 'A:soa_a3:N:soa_c3:s-organic:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/ocphi_rrtmg_c100508.nc:+', + 'A:mom_a3:N:mom_c3:m-organic:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/poly_rrtmg_c130816.nc', + 'mam5_mode4:primary_carbon:=', + 'A:num_a4:N:num_c4:num_mr:+', + 'A:pom_a4:N:pom_c4:p-organic:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/ocpho_rrtmg_c130709_kPOM0.04.nc:+', + 'A:bc_a4:N:bc_c4:black-c:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/bcpho_rrtmg_c100508.nc:+', + 'A:mom_a4:N:mom_c4:m-organic:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/poly_rrtmg_c130816.nc', + 'mam5_mode5:strat_coarse:=', + 'A:num_a5:N:num_c5:num_mr:+', + 'A:so4_a5:N:so4_c5:sulfate:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/sulfate_rrtmg_c080918.nc' + rad_climate = 'A:H2OLNZ:H2O', 'N:O2:O2','N:CO2:CO2', 'A:O3:O3', + 'A:N2OLNZ:N2O', 'A:CH4LNZ:CH4','N:CFC11:CFC11', 'N:CFC12:CFC12', + 'M:mam5_mode1:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/mam4_mode1_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode2:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/mam4_mode2_rrtmg_c130628.nc', + 'M:mam5_mode3:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/mam4_mode3_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode4:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/mam4_mode4_rrtmg_c130628.nc', + 'M:mam5_mode5:/lcrc/group/e3sm/data/inputdata/atm/cam/physprops/mam5_mode5_rrtmg_sig1.2_dgnl.40_c03072023.nc' + +``` diff --git a/components/eam/docs/user-guide/emission_oxidant_files.md b/components/eam/docs/user-guide/emission_oxidant_files.md new file mode 100644 index 000000000000..72868b6b8458 --- /dev/null +++ b/components/eam/docs/user-guide/emission_oxidant_files.md @@ -0,0 +1,196 @@ + +# Emission files for EAMv3 gas and aerosol species and oxidant file for VBS SOA and stratosphere sulfate formation + +## Overview + +This page documents emissions data for all required aerosols and precursor gases as well as oxidants input data for running EAMv3, mostly for the MAM4 aerosol scheme, from anthropogenic (i.e., industrial, energy, transportation, domestic, and agriculture activity sectors) and natural (i.e., sea spray, vegetation, fire smoke, volcano) sources. Sulfur from agriculture, domestic, transportation, waste, and shipping sectors is emitted at the surface while sulfur from energy and industry sectors is emitted at 100-300 m above the surface, and sulfur from forest fire and grass fire is emitted at higher elevations (0-6 km). POM and BC from forest fire and grass fire are emitted at 0-6 km, while those from other sources (domestic, energy, industry, transportation, waste, and shipping) are emitted at the surface. Injection height profiles for fire emissions are derived from the corresponding AeroCom profiles (Dentener et al., 2006)[@dentener_emissions_2006], which give emissions in 6 altitude ranges (0-0.1, 0.1-0.5, 0.5-1, 1-2, 2-3, and 3-6 km). Otherwise, species emissions are assumed to be at the surface (bottom layer). Number emission fluxes each mode are calculated from mass emission fluxes based on AeroCom prescribed lognormal size distributions. + +## Aerosols and gas precursors (common for EAMv1/v2/v3) + +* Species: SO2, SOAG0, DMS, bc_a4, pom_a4, so4_a1, so4_a2, num_a1, num_a2, num_a4 +* Data sources + * Most of the original data are directly from input4MIPs with the following exceptions (E3SM specific treatments) + * SO2 takes 97.5% from the input4MIPs data (all SO2-em-anthro_input4MIPs sectors) + * SO4_a1 surf takes 2.5% from the corresponding surface sectors of input4MIPs data (SO2-em-anthro_input4MIPs sectors: AGR, SLV, WST, SHP) + * SO4_a2 surf takes 2.5% from the corresponding surface sectors of input4MIPs data (SO2-em-anthro_input4MIPs sectors: TRA, RCO) + * SO4_a1 elev takes 2.5% from the corresponding elevated sectors of input4MIPs data (SO2-em-anthro_input4MIPs sectors: ENE, IND) + * SO2 (97.5%) and SO4_a1 (2.5%) also take emissions from AR5 input file for sector contvolc (constant volcanic degassing) + * SOAG0 emissions are obtained by scaling OC (POM) emissions with a tunable factor. + * num_a1, num_a2 and num_a4 are determined by mass concentration of aerosols species in the corresponding sectors and modes. + +## Marine organic sea spray + +Marine organic sea spray aerosol contributions are parameterized following the OCEANFILMS parameterization (E3SMv1; Burrows et al., 2014;[@burrows_physically_2014] 2022[@burrows_oceanfilms_2022]). The input file for this parameterization provides a climatology of the ocean surface concentrations of several groups of organic macromolecules. Briefly, the Parallel Ocean Program (POP; Maltrud et al., 1998)[@maltrud_global_1998] and its biogeochemical elemental cycling (BEC) routines (Moore et al., 2004)[@moore_upper_2004] were used to simulate marine biogeochemistry fields, including particulate organic matter (POC), chlorophyll, and zooplankton concentrations; these fields were used to generate maps of the estimated surface distributions of classes of macromolecules following the methods described in Burrows et al. (2014).[@burrows_physically_2014] The scripts used to accomplish this translation are available [here](https://github.com/E3SM-Project/PreAndPostProcessingScripts/blob/devel/prepare_model_inputfiles/emis/marine_organic_aerosol/JAN_1850_MASTERLANG.jnl). + +The file used as an input to E3SM is available here: +[https://web.lcrc.anl.gov/public/e3sm/inputdata/atm/cam/chem/trop_mam/marine_BGC/monthly_macromolecules_0.1deg_bilinear_latlon_year01_merge_date.nc](https://web.lcrc.anl.gov/public/e3sm/inputdata/atm/cam/chem/trop_mam/marine_BGC/monthly_macromolecules_0.1deg_bilinear_latlon_year01_merge_date.nc) + +And is also published as a citeable dataset on Zenodo: + +Elliott, S. M., Maltrud, M., & Burrows, S. M. (2015). Macromolecule distributions input file for the OCEANFILMS parameterization (Version v1) [Data set]. [Zenodo](https://doi.org/10.5281/zenodo.6320812). + +## Oceanic dimethyl sulfide concentrations + +Dimethyl sulfide (DMS) fluxes to the atmosphere are calculated in E3SM as a function of prescribed surface oceanic DMS concentrations, and an air-sea flux piston velocity that is a function of wind speed. + +E3SM uses a DMS surface concentration dataset developed from a dynamic ocean biogeochemistry simulation; the methods and underlying assumptions used to produce this dataset are documented in Wang, et al. (2015). The resolution in the DMS dataset is 1.9x2.5 degrees. + +## New gas species for chemUCI in EAMv3 + +* Species: C2H4, C2H6, C3H8, CH2O, CH3CHO, CH3COCH3, CO, E90, ISOP, NO, NO2 +* Data sources + * anthropogenic, biomass burning, and aircraft emissions (NO2) are regridded from NCAR CESM2 emission files. They are time-dependent during historical period and in the future scenarios. + * biogenic emissions (C2H4, C2H6, C3H8, CH2O, CH3CHO, CH3COCH3, CO, ISOP) are from MEGAN-MACC offline data + * 1850-1979: monthly input cycled yearly from 30-year mean (1980-2009) + * 1980-2014: time-varying MEGAN-MACC data (historical) + * 2015-2100: monthly input cycled yearly from 30-year mean (1980-2009) + * natural emissions from oceans (C2H4, C2H6, C3H8, CO) and soil (NO) are regridded from NCAR CESM2 emission files. Just cycled yearly during the historical period and in the future scenarios. + * E90 emissions? + +## Oxidants file needed for VBS SOA and stratosphere sulfate formation + +* Species: prsd_O3, prsd_NO3, prsd_OH +* Data sources + * the file for historical simulation is the same as v1 and v2, inherited from CESM + * files for SSPs are regridded from NCAR CESM2 tracer files + +## Namelist setting for emissions input + +### Historical (WCYCL20TR)/AMIP (F20TR) + +```fortran + ext_frc_specifier = 'NO2 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_NO2_aircraft_vertical_1750-2015_1.9x2.5_c20170608.nc', + 'SO2 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_elev_1850-2014_c180205_kzm_1850_2014_volcano.nc', + 'SOAG0 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/emissions-cmip6_e3sm_SOAG0_elev_1850-2014_1.9x2.5_c20230201.nc', + 'bc_a4 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_elev_1850-2014_c180205.nc', + 'num_a1 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_elev_1850-2014_c180205.nc', + 'num_a2 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_elev_1850-2014_c180205.nc', + 'num_a4 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_elev_1850-2014_c180205.nc', + 'pom_a4 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_elev_1850-2014_c180205.nc', + 'so4_a1 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_elev_1850-2014_c180205.nc', + 'so4_a2 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_elev_1850-2014_c180205.nc' + + srf_emis_specifier = 'C10H16 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_MTERP_surface_1850-2014_1.9x2.5_c20230126.nc', + 'C2H4 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C2H4_surface_1850-2014_1.9x2.5_c20210323.nc', + 'C2H6 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C2H6_surface_1850-2014_1.9x2.5_c20210323.nc', + 'C3H8 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C3H8_surface_1850-2014_1.9x2.5_c20210323.nc', + 'CH2O -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH2O_surface_1850-2014_1.9x2.5_c20210323.nc', + 'CH3CHO -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH3CHO_surface_1850-2014_1.9x2.5_c20210323.nc', + 'CH3COCH3 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH3COCH3_surface_1850-2014_1.9x2.5_c20210323.nc', + 'CO -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CO_surface_1850-2014_1.9x2.5_c20210323.nc', + 'DMS -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DMSflux.1850-2100.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20160727.nc', + 'E90 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions_E90_surface_1750-2015_1.9x2.5_c20210408.nc', + 'ISOP -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_ISOP_surface_1850-2014_1.9x2.5_c20210323.nc', + 'ISOP_VBS -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_ISOP_surface_1850-2014_1.9x2.5_c20210323.nc', + 'NO -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_NO_surface_1850-2014_1.9x2.5_c20220425.nc', + 'SO2 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_surf_1850-2014_c180205.nc', + 'SOAG0 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/emissions-cmip6_e3sm_SOAG0_surf_1850-2014_1.9x2.5_c20230201.nc', + 'bc_a4 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_surf_1850-2014_c180205.nc', + 'num_a1 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_surf_1850-2014_c180205.nc', + 'num_a2 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_surf_1850-2014_c180205.nc', + 'num_a4 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_surf_1850-2014_c180205.nc', + 'pom_a4 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_surf_1850-2014_c180205.nc', + 'so4_a1 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_surf_1850-2014_c180205.nc', + 'so4_a2 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_surf_1850-2014_c180205.nc' + + tracer_cnst_datapath = '\$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/oxid' + tracer_cnst_file = 'oxid_1.9x2.5_L26_1850-2015_c20181106.nc' + tracer_cnst_filelist = '' + tracer_cnst_specifier = 'prsd_O3:O3','prsd_NO3:NO3','prsd_OH:OH' + tracer_cnst_type = 'INTERP_MISSING_MONTHS' +``` + +### F2010 + +```fortran + ext_frc_cycle_yr = 2010 + ext_frc_specifier = 'NO2 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_NO2_aircraft_vertical_2010_clim_1.9x2.5_c20230213.nc', + 'SO2 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_elev_1x1_2010_clim_c20190821.nc', + 'SOAG0 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/emissions-cmip6_e3sm_SOAG0_elev_2010_clim_1.9x2.5_c20230213.nc', + 'bc_a4 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_elev_1x1_2010_clim_c20190821.nc', + 'num_a1 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_elev_1x1_2010_clim_c20190821.nc', + 'num_a2 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_elev_1x1_2010_clim_c20190821.nc', + 'num_a4 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_elev_1x1_2010_clim_c20190821.nc', + 'pom_a4 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_elev_1x1_2010_clim_c20190821.nc', + 'so4_a1 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_elev_1x1_2010_clim_c20190821.nc', + 'so4_a2 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_elev_1x1_2010_clim_c20190821.nc' + ext_frc_type = 'CYCLICAL' + + srf_emis_cycle_yr = 2010 + srf_emis_specifier = 'C10H16 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_MTERP_surface_2010_clim_1.9x2.5_c20230213.nc', + 'C2H4 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C2H4_surface_2010_clim_1.9x2.5_c20230213.nc', + 'C2H6 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C2H6_surface_2010_clim_1.9x2.5_c20230213.nc', + 'C3H8 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C3H8_surface_2010_clim_1.9x2.5_c20230213.nc', + 'CH2O -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH2O_surface_2010_clim_1.9x2.5_c20230213.nc', + 'CH3CHO -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH3CHO_surface_2010_clim_1.9x2.5_c20230213.nc', + 'CH3COCH3 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH3COCH3_surface_2010_clim_1.9x2.5_c20230213.nc', + 'CO -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CO_surface_2010_clim_1.9x2.5_c20230213.nc', + 'DMS -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DMSflux.2010.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20190220.nc', + 'E90 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions_E90_surface_2010_clim_1.9x2.5_c20230213.nc', + 'ISOP -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_ISOP_surface_2010_clim_1.9x2.5_c20230213.nc', + 'ISOP_VBS -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_ISOP_surface_2010_clim_1.9x2.5_c20230213.nc', + 'NO -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_NO_surface_2010_clim_1.9x2.5_c20230213.nc', + 'SO2 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_surf_1x1_2010_clim_c20190821.nc', + 'SOAG0 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/emissions-cmip6_e3sm_SOAG0_surf_2010_clim_1.9x2.5_c20230213.nc', + 'bc_a4 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_surf_1x1_2010_clim_c20190821.nc', + 'num_a1 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_surf_1x1_2010_clim_c20190821.nc', + 'num_a2 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_surf_1x1_2010_clim_c20190821.nc', + 'num_a4 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_surf_1x1_2010_clim_c20190821.nc', + 'pom_a4 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_surf_1x1_2010_clim_c20190821.nc', + 'so4_a1 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_surf_1x1_2010_clim_c20190821.nc', + 'so4_a2 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_surf_1x1_2010_clim_c20190821.nc' + srf_emis_type = 'CYCLICAL' + + tracer_cnst_cycle_yr = 2015 + tracer_cnst_datapath = '\$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/oxid' + tracer_cnst_file = 'oxid_1.9x2.5_L26_1850-2015_c20181106.nc' + tracer_cnst_filelist = '' + tracer_cnst_specifier = 'prsd_O3:O3','prsd_NO3:NO3','prsd_OH:OH' + tracer_cnst_type = 'CYCLICAL' + ``` + +### Future Scenarios + +#### SSP370 + +```fortran +ext_frc_specifier = 'NO2 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_NO2_aircraft_vertical_2015-2100_1.9x2.5_c20240208.nc', + 'SO2 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_so2_elev_2015-2100_c210216.nc', + 'SOAG0 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_SOAG0_elev_2015-2100_1.9x2.5_c20240208.nc', + 'bc_a4 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_bc_a4_elev_2015-2100_c210216.nc', + 'num_a1 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_num_a1_elev_2015-2100_c210216.nc', + 'num_a2 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_num_a2_elev_2015-2100_c210216.nc', + 'num_a4 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_num_a4_elev_2015-2100_c210216.nc', + 'pom_a4 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_pom_a4_elev_2015-2100_c210216.nc', + 'so4_a1 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_so4_a1_elev_2015-2100_c210216.nc', + 'so4_a2 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_so4_a2_elev_2015-2100_c210216.nc' + +srf_emis_specifier = 'C10H16 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_MTERP_surface_2015-2100_1.9x2.5_c20240208.nc', + 'C2H4 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_C2H4_surface_2015-2100_1.9x2.5_c20240208.nc', + 'C2H6 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_C2H6_surface_2015-2100_1.9x2.5_c20240208.nc', + 'C3H8 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_C3H8_surface_2015-2100_1.9x2.5_c20240208.nc', + 'CH2O -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_CH2O_surface_2015-2100_1.9x2.5_c20240208.nc', + 'CH3CHO -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_CH3CHO_surface_2015-2100_1.9x2.5_c20240208.nc', + 'CH3COCH3 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_CH3COCH3_surface_2015-2100_1.9x2.5_c20240208.nc', + 'CO -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_CO_surface_2015-2100_1.9x2.5_c20240208.nc ', + 'DMS -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DMSflux.1850-2100.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20160727.nc', + 'E90 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart/ub/emissions_E90_surface_1750-2101_1.9x2.5_c20231222.nc', + 'ISOP -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_ISOP_surface_2015-2100_1.9x2.5_c20240208.nc', + 'ISOP_VBS -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_ISOP_surface_2015-2100_1.9x2.5_c20240208.nc', + 'NO -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_NO_surface_2015-2100_1.9x2.5_c20240208.nc', + 'SO2 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_so2_surf_2015-2100_c210216.nc', + 'SOAG0 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_SOAG0_surf_2015-2100_1.9x2.5_c20240208.nc', + 'bc_a4 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_bc_a4_surf_2015-2100_c210216.nc', + 'num_a1 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_num_a1_surf_2015-2100_c210216.nc', + 'num_a2 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_num_a2_surf_2015-2100_c210216.nc', + 'num_a4 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_num_a4_surf_2015-2100_c210216.nc', + 'pom_a4 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_pom_a4_surf_2015-2100_c210216.nc', + 'so4_a1 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_so4_a1_surf_2015-2100_c210216.nc', + 'so4_a2 -> \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_so4_a2_surf_2015-2100_c210216.nc' + + tracer_cnst_datapath = '\$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/oxid' + tracer_cnst_file = 'oxid_SSP370_1.9x2.5_L70_1849-2101_c20240228.nc' + tracer_cnst_filelist = '' + tracer_cnst_specifier = 'prsd_O3:O3','prsd_NO3:NO3','prsd_OH:OH' + tracer_cnst_type = 'INTERP_MISSING_MONTHS' +``` diff --git a/components/eam/docs/user-guide/index.md b/components/eam/docs/user-guide/index.md index 81084e869cde..c2bd7f065383 100644 --- a/components/eam/docs/user-guide/index.md +++ b/components/eam/docs/user-guide/index.md @@ -1 +1,175 @@ -start of the EAM User's Guide + +# EAM User Guide + +This User Guide describes how to set up and run EAM. + +## Steps to build and run EAM + +EAM is not available as a standalone model. Instead, EAM can be run in an atmosphere-only configuration. +The difference when running in atmosphere-only mode (without an interactive ocean or sea-ice) would be to +change the *compset* and *grid*. +See the +[Case Control System Basic Usage](https://esmci.github.io/cime/versions/master/html/users_guide/index.html#case-control-system-part-1-basic-usage) for general descriptions of compsets and grids. + +Certain namelist paramaters, input data files, and output file specifcations can also be modified. +These are described below as ways to customize runs. + +Step-by-step instructions on how to run and analyze E3SM with a script can be found at +[E3SM step-by-step guide](https://docs.e3sm.org/running-e3sm-guide/) + +## Scientifically supported compsets and grids + +### Compsets + +All of the compsets below run with the complete set of E3SM atmospheric configuration of EAMV3. For more information on the schemes in EAMv3, see the [Technical Guide](../tech-guide/index.md) + +`F2010` - Climatological present day climate (year 2010) + +`F1850` - Climatological pre-industrial-day climate (year 1850) + +`F20TR` - Historical EAM simulation with time varying sea-surface temperatures, aerosol emissions, and greenhouse gas forcings (year 1850-2014) + +### Grids + +Only one grid combination is currently supported for the above compsets: + +`ne30pg2_r05_IcoswISC30E3r5` - ne30pg2 atmosphere, 0.5deg x 0.5deg land grid, and Icosahedral 30 km mesh with ice shelves cavities (wISC), E3SMv3 (E3) revision r5 + +## Customizing runs + +### Compile-time options + +Some customizations require making changes before the model is built. + +### Run-time options + +Run-time customization is enabled by a Fortran namelist. + +Namelist parameters can be changed from default values by putting them in the `user_nl_eam` file in the case directory +with the desired new value. + +This [Table of Namelist Parameters](namelist_parameters.md) includes many of the paramaters that control +physics schemes described in the [tech-guide](../tech-guide/index.md) + +#### History File Namelist Parameters + +By default, EAM will output a set of monthly-averaged variables. Additional output files can be specified using the following flags in the `user_nl_eam` file: + +`finclX` - List of variables (in single quotes and separated by commas) that are added to tape X. + +`fexclX` - List of variables (in single quotes and separated by commas) that will be excluded in tape X. + +`nhtfrq` - List of write frequencies for each of the history files. A value of 0 denotes a monthly frequency. Negative values denote hourly frequencies (e.g., `-3` will write an output every 3 hours). Positive values denotes the frequency in model timesteps (e.g., `4` will write an output every 4 timesteps). + +`mfilt` - List that sets the number of timesteps to write in a single file before starting a new file. + +`avgflag_pertape` - List that sets the type of output to write. Choices are `'A'` for time-averaged output, `'A'` for instantaneous output, `'M'` for time-minimum output, and `'X'` for time-maximum output. + +#### Example output specification + +```fortran +nhtfrq = 0,-24,-6,-3 +mfilt = 1,30,120,24 +avgflag_pertape = 'A','A','A','I' + +fexcl1 = 'U10' # Removes U10 output from monthly files +fincl2 = 'PS','FLUT','PRECT','U200','V200','U850', + 'V850','TCO','SCO','TREFHT','QREFHT' # Output files of daily-averaged output, which includes 30 days of output in each file +fincl3 = 'PS', 'PSL','PRECT','TUQ','TVQ','UBOT', + 'VBOT','TREFHT','FLUT','OMEGA500','TBOT', + 'U850','V850','U200','V200','T200','T500', + 'Z700' # Output files of 6-hour-averaged output, which includes 30 days of output in each file +fincl4 = 'PRECT' # Output files of 3-hourly output with 3 days of output in every file +``` + +## Input datasets + +Many properties of the simulated atmosphere are controlled by data sets read in by the model during initialization. +Changing the content of these files is another way to customize a run. + +### Greenhouse gases (non-reacting) + +Greenhouse gas concentration inputs of non-reacting species are taken from CMIP6 Forcing Datasets provided from the input4MIPs data collection. In addition to what is provided by the input4MIPS, 2015 and 2016 have been added by extrapolating from 2013 and 2014. + +```fortran +inputdata/atm/cam/ggas/GHG_CMIP-1-2-0_Annual_Global_0000-2014_c20180105.nc +``` + +### [Aerosol physical properties](aerosol_phys_prop.md) + +The aerosol properties files provide aerosol refractive index, density, and aerosol hygroscopicty information for +each aerosol species, as well as information about lognormal mode definition and lookup tables of polynomial +expression coefficients for aerosol optics calculation for each mode. These aerosol physical and chemical properties +are used by the radiation, aerosol microphysics and other related source and sink processes, and droplet +activation/ice nucleation schemes. + +### [Aerosol and gas emission and oxidant files](emission_oxidant_files.md) + +Details of the aerosol and gas emission and oxidant files used in various historical, present-day, and future scenarios. + +### Linoz v3 input files + +Linozv3 uses the ozone tendency, (net production minus loss) calculated from its climatological mean state (function of month, latitude, altitude) and a first-order Taylor series expansion about the local ozone, temperature, and overhead ozone column. For historical simulation, Linozv3 uses the linozv3 data files with monthly resolution, spanning the dates 1849-01 -- 2014-12. + +#### Historical files + +```fortran + linoz_data_file = ‘linv3_1849-2101_CMIP6_Hist_10deg_58km_c20231207.nc’ + linoz_data_path = '/lcrc/group/e3sm/data/inputdata/atm/cam/chem/trop_mozart/ub' + linoz_data_type = 'INTERP_MISSING_MONTHS' +``` + +Refer to [this page](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/3764486280/Production+of+the+Linoz+v3+data) for more details on Linoz v3 input files. + +### Topography + +The global elevation on the atmosphere grid is a key input dataset. The dataset is grid dependent. It contains the geopotential data (on both the GLL dynamics and PG2 physics grids) and two surface roughness quantities, `SGH` and `SGH30`. + +EAMv3 NE30 data: + +```fortran +inputdata/atm/cam/topo/USGS-gtopo30_ne30np4pg2_x6t-SGH.c20210614.nc' +``` + +This file is computed via a complex procedure that starts with high resolution dataset (for EAMv3, we use the GTOPO30, a 30 arc-second resolution data set on a lat-lon grid) that is then downsampled to a 3km cubed-sphere grid (cube3000) and then downsampled to the atmosphere grid on the (GLL nodes), and then smoothed with the same viscosity operator used by the dycore. The smoothed GLL topography is then mapped to the PG2 grid. Finally, two different surface roughness fields are computed: + +- `SGH30`: the variance between GTOPO30 and GTOPO30-downsampled-to-PG2 (independent of any dycore specific smoothing). (used by CLUBB, TMS, vertical_diffusion) +- `SGH`: the variance between the cube3000 data and the smoothed PG2 data. (used by GWD parameterizations) + +### Land-use / land cover change + +Info needed on land-use land cover change / land surface data + +Refer to [ELM documentation](https://docs.e3sm.org/E3SM/ELM/). + +### Ocean/sea ice + +The sea surface temperature and sea-ice coverage data used in F-case simulations are based on CMIP6 Forcing Datasets provided from the input4MIPs data collection. The file for the `F2010` compset is created from taking a monthly climatology of years 2005-2014. The file for the `F1850` compset is created from taking a monthly climatology of years 1870-1879.* + +*[Provenenance for F1850 file](https://acme-climate.atlassian.net/wiki/spaces/ATM/pages/201525378/Provenance+for+CMIP6+DECK+SST+Sea-Ice+input+data+for+E3SM) + +`F20TR` + +```fortran +inputdata/ocn/docn7/SSTDATA/sst_ice_CMIP6_DECK_E3SM_1x1_c20180213.nc +``` + +`F2010` + +```fortran +inputdata/ocn/docn7/SSTDATA/sst_ice_CMIP6_DECK_E3SM_1x1_2010_clim_c20190821.nc +``` + +`F1850` + +```fortran +inputdata/ocn/docn7/SSTDATA/sst_ice_CMIP6_DECK_E3SM_1x1_1850_clim_c20190125.nc +``` + +### Solar input + +As with greenhouse gas emissions, solar input files are taken from the input4MIPs data collection that were prepared for CMIP6 Forcing Datasets. + +```fortran +inputdata/atm/cam/solar/Solar_1850-2299_input4MIPS_c20181106.nc +``` diff --git a/components/eam/docs/user-guide/namelist_parameters.md b/components/eam/docs/user-guide/namelist_parameters.md new file mode 100644 index 000000000000..b09e8c44ffde --- /dev/null +++ b/components/eam/docs/user-guide/namelist_parameters.md @@ -0,0 +1,158 @@ + +# Namelist parameters associated with atmosphere schemes + +## chemUCI and Linoz v3 + +| Parameter | Description | Default value* | +| ---------------------------- | ------------------------------------------------------------------------ | ---------------------- | +| `airpl_emis_file` | Aviation emission | | +| `chlorine_loading_file` | Chlorine loading | | +| `chlorine_loading_fixed_ymd` | | | +| `chlorine_loading_type` | | | +| `ext_frc_specifier` | 3-D emissions | | +| `ext_frc_cycle_yr` | | | +| `ext_frc_type` | | | +| `srf_emis_specifier` | Surface emissions | | +| `srf_emis_cycle_yr` | | | +| `srf_emis_type` | Upper bound of mean raindrop diameter | | +| `linoz_data_file` | Linoz data file | | +| `linoz_data_cycle_yr` | | | +| `linoz_data_path` | | | +| `linoz_data_type` | | | +| `lght_no_prd_factor` | Lightning NOx emission factor | `5.0` | +| `fstrat_efold_list` | Tracer (from troposphere) list with e-folding decay in the stratosphere | | + +* Many of these namelist parameters specify input data files. Check the `atm_in` file for examples or refer to the [Users' Guide](../user-guide/index.md). + +## Cloud Layers Unified By Binormals + +| Parameter | Description | Default value | +| -------------- | ------------------------------------------------------------------------------------------- | -------------- | +| `gamma_coef` | Width of vertical velocity within a Gaussian PDF component at low skewness | `0.12` | +| `gamma_coefb` | Width of vertical velocity within a Gaussian PDF component at high skewness | `0.28` | +| `C8` | Coefficient of damping of third moment of vertical velocity, w’3 | `5.2` | +| `C1` | Coefficient of damping of second vertical moment of vertical velocity, w’2, at low skewness | `2.4` | +| `C14` | Coefficient of damping of second horizontal moments of vertical velocity, u’2 and v’2 | `2.0` | +| `c_k10` | Ratio of diffusivity of momentum to heat | `0.35` | + +## Dust aerosol + +| Parameter | Description | Default value | +| ------------------------- | ---------------------------------------------- | ------------------------------------------------- | +| `dust_emis_scheme`* | The v3 dust emission scheme (Kok et al., 2014) | `2`
(set to 1 to switch to the v1/v2 scheme) | + +*This parameter is set in `user_nl_drv` + +## HOMME + +| Parameter | Description | Default value | +| ---------------- | ------------------------------------------------------------------------------------------- | -------------- | +| `se_tstep` | Main dycore timestep. Additional parameters control the hyper viscsosity, trancer and vertical remap timesteps, which are derived from se_tstep.
units = seconds | Scales linearly with horizontal resolution.
NE30 default: `300` | +| `nu` | Tensor hyperviscosity coefficient, independent of spatial resolution.
units = 1/s | `3.4e-8` | +| `nu_top` | Scalar viscosity at model top.
units = m^2/s | Horizontal resolution dependent
NE30 default: `2.5e5` | +| `transport_alg` | Select between semi-lagrangian and Eulerian based transport schemes | `12` = semi-lagranian method with monotinicity and mass preservation | +| `statefreq` | print a varieity of dycore metrics to the atm.log file every “statefreq” timesteps | `480` | +| `vert_remap_alg` | Algorithm used to remap the vertically lagrangian levels back to the reference levels | `10` = strict monotonicity applied on top of a 2nd order accurate PPM method | +| `se_ftype` | Controls how physics tendencies are applied. 0=”dribbled” in during dynamics timesteps. 1=”hard adjustment” after each physics timestep. 2=hybrid approach: hard adjustment for tracers, dribbled for remaining tendencies | `2` | + +## Modal Aerosol Module + +| Parameter | Description | Default value | +| ------------------------ | ----------------------------------------------------------------------------------- | --------------------------- | +| `is_output_interactive_volc` | Switch for diagnostic output of the stratospheric aerosol optics | `.false.` | +| `mam_amicphys_optaa` | Recommended option of the new time-splitting treatment of H2SO4 production and loss | `1`
(0 to turn it off) | +| `n_so4_monolayers_pcage` | Number of monolayers required to age primary-carbon mode particles | `3` | +| `seasalt_emis_scale` | Tuning parameter for sea salt emission | `0.55` | + +## OCEANFILMS + +| Parameter | Description | Default value | +| ------------------------- | ----------------------------------------------------------------- | ---------------------- | +| `mam_mom_cycle_yr` | | `1` | +| `mam_mom_datapath` | Full pathname of the directory that contains the files specified in mam_mom_filelist | `'atm/cam/chem/trop_mam/marine_BGC/'` | +| `mam_mom_filename` | Filename of file that contains a sequence of filenames for prescribed marine organic matter ocean concentrations. The filenames in this file are relative to the directory specified by mam_mom_datapath.| `'monthly_macromolecules_0.1deg_bilinear_latlon_year01_merge_date.nc'` | +| `mam_mom_rmfile` | Remove the file containing prescribed aerosol deposition fluxes from local disk when no longer needed. | `FALSE` | +| `mam_mom_specifier` | Names of variables containing aerosol data in the prescribed aerosol datasets. | `'chla:CHL1','mpoly:TRUEPOLYC','mprot:TRUEPROTC','mlip:TRUELIPC'` | +| `mam_mom_datatype` | Type of time interpolation for data in mam_mom files. Can be set to `'CYCLICAL'`, `'SERIAL'`, `'INTERP_MISSING_MONTHS'`, or `'FIXED'`. | `'CYCLICAL'` | +| `mam_mom_cycle_yr` | The cycle year of the prescribed aerosol flux data if mam_mom_type is `'CYCLICAL'`. Format: YYYY | `1` | +| `mam_mom_fixed_ymd` | The date at which the prescribed aerosol flux data is fixed if mam_mom_type is `'FIXED'`. Format: YYYYMMDD | `0` | +| `mam_mom_fixed_tod` | The time of day (seconds) corresponding to mam_mom_fixed_ymd at which the prescribed aerosol flux data is fixed if mam_mom_type is 'FIXED'. | `0` | +| `mam_mom_bubble_thickness` | Bubble film thickness (in m) for marine organic aerosol emission mechanism. The physically reasonable range is approximately (0.1 - 1) x 10^ -6. | `0.1e-6` | +| `mam_mom_mixing_state` | Switch to select mixing state assumption in marine organic aerosol code. Currently implemented options: 0 : total external mixture, add to mass; 1 : total external mixture, replace mass; 2 : total internal mixture, add to mass; 3 : total internal mixture, replace mass. | `0` [Note: set to 3 in the atm_in namelist] | +| `mam_mom_parameterization` | Selection of alternate parameterizations for marine organic matter emissions. Set fmoa=1 for Burrows et al. (2014) [@burrows_physically_2014] parameterization; fmoa=2 for Gantt et al. (2011) [@gantt_wind_2011] parameterization; fmoa=3 for simple parameterization based on Quinn et al., 2014; [@quinn_contribution_2014] fmoa=4 for Rinaldi et al. (JGR, 2013).* [@rinaldi_is_2013] | `1` | + +*Note: non-default values have not been carefully tested and may not work as expected. + +## Predicted Particle Properties + +| Parameter | Description | Default value | +| ------------------------- | ----------------------------------------------------------------- | ---------------------- | +| `do_prescribed_ccn` | Turn on the prescribed CCN if true | `false` | +| `micro_aerosolactivation` | Turn on aerosol activation if true | `true` | +| `micro_p3_lookup_dir` | Directory of P3 look-up tables | `inputdata/atm/cam/physprops` | +| `micro_p3_tableversion` | P3 look-up table Version | `4.1.2` | +| `micro_subgrid_cloud` | Sub-grid cloud properties | `true` | +| `micro_tend_output` | Output of P3 microphysical process rates | `false` | +| `p3_accret_coeff` | Tunable parameter for adjusting rain accretion efficiency | `117.25` | +| `p3_autocon_coeff` | Tunable parameter for adjusting droplet autoconversion efficiency | `30500` | +| `p3_embryonic_rain_size` | Radius of embryomic raindrops from auto-conversion | `0.000025` (m) | +| `p3_max_mean_rain_size` | Upper bound of mean raindrop diameter | `0.005` (m) | +| `p3_mincdnc` | Lower bound of droplet number concentration | `20.d6` (# m-3) | +| `p3_nc_autocon_expon` | Nc exponent in droplet auto-conversion | `-1.1` | +| `p3_qc_accret_expon` | Qc exponent in rain accretion | `1.15` | +| `p3_qc_autocon_expon` | Qc exponeent in droplet autoconversion | `3.19` | +| `p3_wbf_coeff` | Tunable parameter for adjusting WBF efficiency | `1.0` | +| `do_cooper_inp3` | Turn on Cooper ice nucleation scheme if true | `false` | + +## Rapid Radiative Transfer Model for GCMs + +| Parameter | Description | Default value | +| ------------------------- | ----------------------------------------------------------------- | ---------------------- | +| `iradsw` | Frequency for updating shortwave fluxes and heating rate; iradsw > 0 interpreted as number of timesteps, iradsw < 0 interpreted as hours; iradsw = 0 disables shortwave radiation entirely | `-1` | +| `iradlw` | Frequency for updating longwave fluxes and heating rate; iradlw > 0 interpreted as number of timesteps, iradlw < 0 interpreted as hours; iradlw = 0 disables longwave radiation entirely | `-1` | +| `irad_always` | Length of time in timesteps (irad_always > 0) or in hours (irad_always < 0) SW/LW radiation will be run continuously from the start of an initial or restart run | `0` | +| `use_rad_dt_cosz` | If true, use the radiation dt for all cosz calculations; calculates solar zenith angle averaged over a time step. In default model solar zenith angle is held constant over time | `.true.`
(set by namelist_defaults_eam.xml for default physics) | +| `spectralflux` | Calculate fluxes (up and down) per band | `.false.` | +| `liqcldoptics` | Choice of cloud optical property parameterization for liquid clouds. Valid options are ‘slingo’ or ‘gammadist’ | `gammadist` | +| `icecldoptics` | Choice of cloud optical property parameterization for ice clouds. Valid options are ‘ebertcurry’ or ‘mitchell’ | `mitchell` | + +## Zhang and McFarlane deep convection scheme + +| ZM Parameters | Description | Default value | +| ------------------------- | ----------------------------------------------------------------- | ---------------------- | +| `zmconv_ke` | Tunable evaporation efficiency in ZM deep convection scheme | `2.5E-6` | +| `zmconv_tau` | Relaxation time in ZM deep convection scheme | `3600` | +| `zmconv_dmpdz` | Parcel fractional mass entrainment rate | `-0.7E-3` | +| `zmconv_alfa` | Initial downdraft mass flux fraction | `0.14D0` | +| `zmconv_tiedke_add` | Temperature perturbation of an air parcel | `0.8D0` | +| `zmconv_cape_cin` | Number of negative buoyancy regions that are allowed | `1` | + +| dCAPE-ULL Parameters | Description | Default value | +| ------------------------- | ----------------------------------------------------------------- | ---------------------- | +| `zmconv_trigdcape_ull` | DCAPE trigger along with unrestricted launching level for ZM deep convection scheme | `.true.` | +| `zmconv_trig_dcape_only` | DCAPE only trigger for ZM deep convection scheme | `.false.`
If true, zmconv_trigdcape_ull must be false to use the dcape only trigger. | +| `zmconv_trig_ull_only` | Use unrestricted launching level (ULL) only trigger for ZM deep convection scheme | `.false.`
If true, zmconv_trigdcape_ull must be false to use the ull only trigger. | + +| Conv. micro. Parameters | Description | Default value | +| ------------------------- | ----------------------------------------------------------------- | ---------------------- | +| `zmconv_microp` | Convective microphysics option in ZM convection scheme | `true` | +| `zmconv_auto_fac` | Cloud droplet-rain autoconversion enhancement factor in the convective microphysics scheme | `7.0` | +| `zmconv_accr_fac` | Cloud droplet-rain accretion enhancement factor in the convective microphysics scheme | `1.5` | +| `zmconv_micro_dcs` | Autoconversion size threshold for cloud ice to snow (m) | `150.E-6` | + +| Mass flux adj. Parameters | Description | Default value | +| ------------------------- | ----------------------------------------------------------------- | ---------------------- | +| `zmconv_clos_dyn_adj` | Apply mass flux adjustment to ZM convection scheme | `true` | + +| MCSP Parameters | Description | Default value | +| ---------------------------- | ----------------------------------------------------------------- | ---------------------- | +| `zmconv_mcsp_heat_coeff` | MCSP heating coefficient | `0.3` | +| `zmconv_mcsp_moisture_coeff` | MCSP moisture coefficient | `0.0` | +| `zmconv_mcsp_uwind_coeff` | MCSP zonal wind coefficient | `0.0` | +| `zmconv_mcsp_vwind_coeff` | MCSP meridional wind coefficient | `0.0` | + +## Cloud Feedback Model Intercomparison Project (CFMIP) Observation Simulator Package + +| Parameter | Description | Default value | +| ------------------------- | ----------------------------------------------------------------- | ---------------------- | +| `cosp_lite` | This namelist sets cosp_ncolumns=10 and cosp_nradsteps=3 (appropriate for COSP statistics derived from seasonal averages), and runs MISR, ISCCP, MODIS, and CALIPSO lidar simulators (cosp_lmisr_sim=.true.,cosp_lisccp_sim=.true., cosp_lmodis_sim=.true.,cosp_llidar_sim=.true.). | `false` | diff --git a/components/eam/mkdocs.yml b/components/eam/mkdocs.yml index f64ab33620fa..c310e410285b 100644 --- a/components/eam/mkdocs.yml +++ b/components/eam/mkdocs.yml @@ -2,6 +2,23 @@ site_name: EAM nav: - Introduction: 'index.md' - - Users's Guide: user-guide/index.md - - Developers's Guide: dev-guide/index.md - - Technical Guide: tech-guide/index.md + - User Guide: + - user-guide/index.md + - Namelist Parameters: user-guide/namelist_parameters.md + - Aerosol Properties: user-guide/aerosol_phys_prop.md + - Emission Oxidants: user-guide/emission_oxidant_files.md + - Developer Guide: dev-guide/index.md + - Technical Guide: + - tech-guide/index.md + - tech-guide/homme.md + - tech-guide/p3.md + - tech-guide/clubb.md + - tech-guide/zm.md + - RRTMG: tech-guide/rrtmg.md + - tech-guide/mam.md + - tech-guide/vbs.md + - tech-guide/dust.md + - tech-guide/oceanfilms.md + - tech-guide/chemUCIlinozv3.md + - COSP: tech-guide/cosp.md + - tech-guide/armdiags.md diff --git a/components/eam/src/chemistry/mozart/chemistry.F90 b/components/eam/src/chemistry/mozart/chemistry.F90 index 57412d8f8189..f78ceda1dea7 100644 --- a/components/eam/src/chemistry/mozart/chemistry.F90 +++ b/components/eam/src/chemistry/mozart/chemistry.F90 @@ -1574,44 +1574,43 @@ subroutine chem_timestep_tend( state, ptend, cam_in, cam_out, dt, pbuf, fh2o, f end if ! HHLEE 20210923 if (history_gaschmbudget_2D_levels .or. history_chemdyg_summary) then - ftem_layers = 0.0_r8 - gas_ac_layers = 0.0_r8 - - gas_ac_idx = pbuf_get_index(gas_ac_name(n)) - call pbuf_get_field(pbuf, gas_ac_idx, gas_ac ) + ftem_layers = 0.0_r8 + gas_ac_layers = 0.0_r8 + if( nstep /= 0 ) then + gas_ac_idx = pbuf_get_index(gas_ac_name(n)) + call pbuf_get_field(pbuf, gas_ac_idx, gas_ac ) - if (gaschmbudget_2D_L1_e .lt. gaschmbudget_2D_L1_s .or. gaschmbudget_2D_L2_e .lt. gaschmbudget_2D_L2_s .or. & - gaschmbudget_2D_L3_e .lt. gaschmbudget_2D_L3_s .or. gaschmbudget_2D_L4_e .lt. gaschmbudget_2D_L4_s ) then - call endrun('chem_readnl: ERROR 2D chem diags layers, layer ending index is less than starting index') - end if + if (gaschmbudget_2D_L1_e .lt. gaschmbudget_2D_L1_s .or. gaschmbudget_2D_L2_e .lt. gaschmbudget_2D_L2_s .or. & + gaschmbudget_2D_L3_e .lt. gaschmbudget_2D_L3_s .or. gaschmbudget_2D_L4_e .lt. gaschmbudget_2D_L4_s ) then + call endrun('chem_readnl: ERROR 2D chem diags layers, layer ending index is less than starting index') + end if - do k= gaschmbudget_2D_L1_s, gaschmbudget_2D_L1_e ! 0-90 hPa - ftem_layers(:ncol,1) = ftem_layers(:ncol,1) + ftem(:ncol,k) - gas_ac_layers(:ncol,1) = gas_ac_layers(:ncol,1) + gas_ac(:ncol,k) - end do - do k= gaschmbudget_2D_L2_s, gaschmbudget_2D_L2_e ! 90-300 hPa - ftem_layers(:ncol,2) = ftem_layers(:ncol,2) + ftem(:ncol,k) - gas_ac_layers(:ncol,2) = gas_ac_layers(:ncol,2) + gas_ac(:ncol,k) - end do - do k= gaschmbudget_2D_L3_s, gaschmbudget_2D_L3_e ! 300-850 hPa - ftem_layers(:ncol,3) = ftem_layers(:ncol,3) + ftem(:ncol,k) - gas_ac_layers(:ncol,3) = gas_ac_layers(:ncol,3) + gas_ac(:ncol,k) - end do - do k= gaschmbudget_2D_L4_s, gaschmbudget_2D_L4_e ! 850 hPa - surface - ftem_layers(:ncol,4) = ftem_layers(:ncol,4) + ftem(:ncol,k) - gas_ac_layers(:ncol,4) = gas_ac_layers(:ncol,4) + gas_ac(:ncol,k) - end do + do k= gaschmbudget_2D_L1_s, gaschmbudget_2D_L1_e ! 0-90 hPa + ftem_layers(:ncol,1) = ftem_layers(:ncol,1) + ftem(:ncol,k) + gas_ac_layers(:ncol,1) = gas_ac_layers(:ncol,1) + gas_ac(:ncol,k) + end do + do k= gaschmbudget_2D_L2_s, gaschmbudget_2D_L2_e ! 90-300 hPa + ftem_layers(:ncol,2) = ftem_layers(:ncol,2) + ftem(:ncol,k) + gas_ac_layers(:ncol,2) = gas_ac_layers(:ncol,2) + gas_ac(:ncol,k) + end do + do k= gaschmbudget_2D_L3_s, gaschmbudget_2D_L3_e ! 300-850 hPa + ftem_layers(:ncol,3) = ftem_layers(:ncol,3) + ftem(:ncol,k) + gas_ac_layers(:ncol,3) = gas_ac_layers(:ncol,3) + gas_ac(:ncol,k) + end do + do k= gaschmbudget_2D_L4_s, gaschmbudget_2D_L4_e ! 850 hPa - surface + ftem_layers(:ncol,4) = ftem_layers(:ncol,4) + ftem(:ncol,k) + gas_ac_layers(:ncol,4) = gas_ac_layers(:ncol,4) + gas_ac(:ncol,k) + end do + endif if (history_gaschmbudget_2D_levels ) then - call outfld(trim(solsym(n))//'_2DMSB_L1', ftem_layers(:ncol,1), pcols, lchnk) - call outfld(trim(solsym(n))//'_2DMSB_L2', ftem_layers(:ncol,2), pcols, lchnk) - call outfld(trim(solsym(n))//'_2DMSB_L3', ftem_layers(:ncol,3), pcols, lchnk) - call outfld(trim(solsym(n))//'_2DMSB_L4', ftem_layers(:ncol,4), pcols, lchnk) + call outfld(trim(solsym(n))//'_2DMSB_L1', ftem_layers(:ncol,1), pcols, lchnk) + call outfld(trim(solsym(n))//'_2DMSB_L2', ftem_layers(:ncol,2), pcols, lchnk) + call outfld(trim(solsym(n))//'_2DMSB_L3', ftem_layers(:ncol,3), pcols, lchnk) + call outfld(trim(solsym(n))//'_2DMSB_L4', ftem_layers(:ncol,4), pcols, lchnk) endif - if( nstep == 0 ) then - Diff_layers(:ncol,:) = 0.0_r8 - else - Diff_layers(:ncol,:) = 0.0_r8 + Diff_layers(:ncol,:) = 0.0_r8 + if( nstep /= 0 ) then Diff_layers(:ncol,1) = (ftem_layers(:ncol,1) - gas_ac_layers(:ncol,1))/dt Diff_layers(:ncol,2) = (ftem_layers(:ncol,2) - gas_ac_layers(:ncol,2))/dt Diff_layers(:ncol,3) = (ftem_layers(:ncol,3) - gas_ac_layers(:ncol,3))/dt @@ -1629,18 +1628,18 @@ subroutine chem_timestep_tend( state, ptend, cam_in, cam_out, dt, pbuf, fh2o, f trim(solsym(n))=='N2OLNZ' .or. trim(solsym(n))=='CH4LNZ') then ftem_layers = 0.0_r8 gas_ac_layers = 0.0_r8 - do k=1,pver - ftem_layers(:ncol,1) = ftem_layers(:ncol,1) + ftem(:ncol,k) * tropFlagInt(:ncol,k) - gas_ac_layers(:ncol,1) = gas_ac_layers(:ncol,1) + gas_ac(:ncol,k) * tropFlagInt(:ncol,k) - end do + if( nstep /= 0 ) then + do k=1,pver + ftem_layers(:ncol,1) = ftem_layers(:ncol,1) + ftem(:ncol,k) * tropFlagInt(:ncol,k) + gas_ac_layers(:ncol,1) = gas_ac_layers(:ncol,1) + gas_ac(:ncol,k) * tropFlagInt(:ncol,k) + end do + endif if (history_gaschmbudget_2D_levels ) then - call outfld(trim(solsym(n))//'_2DMSB_trop', ftem_layers(:ncol,1), pcols, lchnk ) + call outfld(trim(solsym(n))//'_2DMSB_trop', ftem_layers(:ncol,1), pcols, lchnk ) endif - - if( nstep == 0 ) then - Diff_layers(:ncol,:) = 0.0_r8 - else - Diff_layers(:ncol,:) = 0.0_r8 + + Diff_layers(:ncol,:) = 0.0_r8 + if( nstep /= 0 ) then Diff_layers(:ncol,1) = (ftem_layers(:ncol,1) - gas_ac_layers(:ncol,1))/dt end if diff --git a/components/eam/src/chemistry/mozart/mo_chm_diags.F90 b/components/eam/src/chemistry/mozart/mo_chm_diags.F90 index 8154d4ff3e90..6ae4cfe2f656 100644 --- a/components/eam/src/chemistry/mozart/mo_chm_diags.F90 +++ b/components/eam/src/chemistry/mozart/mo_chm_diags.F90 @@ -1488,16 +1488,16 @@ subroutine chm_diags( lchnk, ncol, vmr, mmr, rxt_rates, invariants, depvel, depf call outfld( 'Mass_'//trim(aerosol_name(aerosol_idx))//'_srf', mass_3d_tmp(:ncol,pver), ncol, lchnk ) - call vertinterp(ncol, pcols, pver, pmid, 85000._r8, mass_3d_tmp, mass_at_pressure) + call vertinterp(ncol, ncol, pver, pmid(:ncol,:), 85000._r8, mass_3d_tmp, mass_at_pressure) call outfld( 'Mass_'//trim(aerosol_name(aerosol_idx))//'_850', mass_at_pressure, ncol, lchnk ) - call vertinterp(ncol, pcols, pver, pmid, 50000._r8, mass_3d_tmp, mass_at_pressure) + call vertinterp(ncol, ncol, pver, pmid(:ncol,:), 50000._r8, mass_3d_tmp, mass_at_pressure) call outfld( 'Mass_'//trim(aerosol_name(aerosol_idx))//'_500', mass_at_pressure, ncol, lchnk ) - call vertinterp(ncol, pcols, pver, pmid, 33000._r8, mass_3d_tmp, mass_at_pressure) + call vertinterp(ncol, ncol, pver, pmid(:ncol,:), 33000._r8, mass_3d_tmp, mass_at_pressure) call outfld( 'Mass_'//trim(aerosol_name(aerosol_idx))//'_330', mass_at_pressure, ncol, lchnk ) - call vertinterp(ncol, pcols, pver, pmid, 20000._r8, mass_3d_tmp, mass_at_pressure) + call vertinterp(ncol, ncol, pver, pmid(:ncol,:), 20000._r8, mass_3d_tmp, mass_at_pressure) call outfld( 'Mass_'//trim(aerosol_name(aerosol_idx))//'_200', mass_at_pressure, ncol, lchnk ) end do diff --git a/components/eam/src/control/cam_history.F90 b/components/eam/src/control/cam_history.F90 index c1b66c734d4e..1d7d84245c77 100644 --- a/components/eam/src/control/cam_history.F90 +++ b/components/eam/src/control/cam_history.F90 @@ -2727,7 +2727,7 @@ subroutine patch_init(t) end if do i = 1, size(tape(t)%grid_ids) call cam_grid_compute_patch(tape(t)%grid_ids(i), patchptr%patches(i),& - beglon, endlon, beglat, endlat) + beglon, endlon, beglat, endlat, collect_column_output(t)) end do nullify(patchptr) end do diff --git a/components/eam/src/control/runtime_opts.F90 b/components/eam/src/control/runtime_opts.F90 index 27dd2931a097..c52c02a5c238 100644 --- a/components/eam/src/control/runtime_opts.F90 +++ b/components/eam/src/control/runtime_opts.F90 @@ -275,6 +275,7 @@ subroutine read_namelist(single_column_in, scmlon_in, scmlat_in, scm_multcols_in use cam_diagnostics, only: diag_readnl use nudging, only: nudging_readnl use radheat, only: radheat_readnl + use phys_grid_ctem, only: phys_grid_ctem_readnl #if ( defined OFFLINE_DYN ) use metdata, only: metdata_readnl #endif @@ -540,6 +541,7 @@ subroutine read_namelist(single_column_in, scmlon_in, scmlat_in, scm_multcols_in call nudging_readnl(nlfilename) call radheat_readnl(nlfilename) call vd_readnl(nlfilename) + call phys_grid_ctem_readnl(nlfilename,dtime) #if ( defined OFFLINE_DYN ) call metdata_readnl(nlfilename) #endif diff --git a/components/eam/src/dynamics/se/inital.F90 b/components/eam/src/dynamics/se/inital.F90 index 7d4c7b946bd8..552568a070ba 100644 --- a/components/eam/src/dynamics/se/inital.F90 +++ b/components/eam/src/dynamics/se/inital.F90 @@ -25,6 +25,7 @@ subroutine cam_initial(dyn_in, dyn_out, NLFileName) use startup_initialconds, only: initial_conds use cam_logfile, only: iulog use perf_mod, only: t_startf, t_stopf + use phys_grid_ctem, only: phys_grid_ctem_reg ! modules from SE use parallel_mod, only : par @@ -42,6 +43,9 @@ subroutine cam_initial(dyn_in, dyn_out, NLFileName) if(par%masterproc ) write(iulog,*) 'Running phys_grid_init()' call phys_grid_init() + ! Register zonal average grid for phys TEM diagnostics + call phys_grid_ctem_reg() + ! Initialize index values for advected and non-advected tracers call phys_register() diff --git a/components/eam/src/dynamics/se/restart_dynamics.F90 b/components/eam/src/dynamics/se/restart_dynamics.F90 index 8412da32d335..5fce98b2d4b2 100644 --- a/components/eam/src/dynamics/se/restart_dynamics.F90 +++ b/components/eam/src/dynamics/se/restart_dynamics.F90 @@ -379,6 +379,7 @@ subroutine read_restart_dynamics (File, dyn_in, dyn_out, NLFileName) use spmd_dyn, only: spmd_readnl use control_mod, only: qsplit use time_mod, only: TimeLevel_Qdp + use phys_grid_ctem, only: phys_grid_ctem_reg ! ! Input arguments @@ -413,6 +414,9 @@ subroutine read_restart_dynamics (File, dyn_in, dyn_out, NLFileName) if(par%masterproc ) write(iulog,*) 'Running phys_grid_init()' call phys_grid_init() + ! Register zonal average grid for phys TEM diagnostics + call phys_grid_ctem_reg() + ! Initialize index values for advected and non-advected tracers call phys_register() diff --git a/components/eam/src/dynamics/se/se_iop_intr_mod.F90 b/components/eam/src/dynamics/se/se_iop_intr_mod.F90 index 02862980053c..b77ef1540d4c 100644 --- a/components/eam/src/dynamics/se/se_iop_intr_mod.F90 +++ b/components/eam/src/dynamics/se/se_iop_intr_mod.F90 @@ -56,7 +56,7 @@ subroutine iop_setinitial(elem) integer i, j, k, cix, ie, thelev integer inumliq, inumice, icldliq, icldice - if (.not. use_replay .and. get_nstep() .eq. 0 .and. par%dynproc) then + if (get_nstep() .eq. 0 .and. par%dynproc) then call cnst_get_ind('NUMLIQ', inumliq, abrtf=.false.) call cnst_get_ind('NUMICE', inumice, abrtf=.false.) call cnst_get_ind('CLDLIQ', icldliq) @@ -215,8 +215,7 @@ subroutine iop_setfield(elem,iop_update_phase1) integer i, j, k, ie do ie=1,nelemd - if (have_ps .and. use_replay .and. .not. iop_update_phase1) elem(ie)%state%ps_v(:,:,:) = psobs - if (have_ps .and. .not. use_replay) elem(ie)%state%ps_v(:,:,:) = psobs + if (have_ps) elem(ie)%state%ps_v(:,:,:) = psobs do i=1, PLEV ! If DP CRM mode do NOT write over dycore vertical velocity if ((have_omega .and. iop_update_phase1) .and. .not. dp_crm) elem(ie)%derived%omega_p(:,:,i)=wfld(i) ! set t to tobs at first diff --git a/components/eam/src/dynamics/se/stepon.F90 b/components/eam/src/dynamics/se/stepon.F90 index 831fd6d87603..783122d31473 100644 --- a/components/eam/src/dynamics/se/stepon.F90 +++ b/components/eam/src/dynamics/se/stepon.F90 @@ -526,9 +526,10 @@ subroutine stepon_run3(dtime, cam_out, phys_state, dyn_in, dyn_out) use hycoef, only: hyam, hybm use dimensions_mod, only: nlev, nelemd, np, npsq use se_iop_intr_mod, only: iop_setfield, iop_setinitial - use dyn_comp, only: TimeLevel + use dyn_comp, only: TimeLevel, hvcoord use cam_history, only: outfld use cam_logfile, only: iulog + use element_ops, only: get_temperature use mpishorthand real(r8), intent(in) :: dtime ! Time-step real(r8) :: ftmp_temp(np,np,nlev,nelemd), ftmp_q(np,np,nlev,pcnst,nelemd) @@ -542,6 +543,7 @@ subroutine stepon_run3(dtime, cam_out, phys_state, dyn_in, dyn_out) type (dyn_export_t), intent(inout) :: dyn_out ! Dynamics export container type (element_t), pointer :: elem(:) integer :: rc, i, j, k, p, ie, tl_f + real(r8) :: temperature(np,np,nlev) ! Temperature from dynamics #if defined (E3SM_SCM_REPLAY) real(r8) :: forcing_temp(npsq,nlev), forcing_q(npsq,nlev,pcnst) #endif @@ -554,7 +556,10 @@ subroutine stepon_run3(dtime, cam_out, phys_state, dyn_in, dyn_out) ! Save ftmp stuff to get state before dynamics is called do ie=1,nelemd - ftmp_temp(:,:,:,ie) = dyn_in%elem(ie)%state%T(:,:,:,tl_f) + + call get_temperature(dyn_in%elem(ie),temperature,hvcoord,tl_f) + + ftmp_temp(:,:,:,ie) = temperature(:,:,:) ftmp_q(:,:,:,:,ie) = dyn_in%elem(ie)%state%Q(:,:,:,:) enddo @@ -585,8 +590,8 @@ subroutine stepon_run3(dtime, cam_out, phys_state, dyn_in, dyn_out) if (dp_crm) then do ie=1,nelemd - out_gridx(:,:) = dyn_in%elem(ie)%spherep(:,:)%lat - out_gridy(:,:) = dyn_in%elem(ie)%spherep(:,:)%lon + out_gridx(:,:) = dyn_in%elem(ie)%spherep(:,:)%lon + out_gridy(:,:) = dyn_in%elem(ie)%spherep(:,:)%lat call outfld('crm_grid_x', out_gridx, npsq, ie) call outfld('crm_grid_y', out_gridy, npsq, ie) enddo @@ -599,6 +604,10 @@ subroutine stepon_run3(dtime, cam_out, phys_state, dyn_in, dyn_out) tl_f = TimeLevel%n0 do ie=1,nelemd + + ! Get temperature from dynamics state + call get_temperature(dyn_in%elem(ie),temperature,hvcoord,tl_f) + do k=1,nlev do j=1,np do i=1,np @@ -606,9 +615,9 @@ subroutine stepon_run3(dtime, cam_out, phys_state, dyn_in, dyn_out) ! Note that this calculation will not provide b4b results with ! an E3SM because the dynamics tendency is not computed in the exact ! same way as an E3SM run, introducing error with roundoff - forcing_temp(i+(j-1)*np,k) = (dyn_in%elem(ie)%state%T(i,j,k,tl_f) - & + forcing_temp(i+(j-1)*np,k) = (temperature(i,j,k) - & ftmp_temp(i,j,k,ie))/dtime - dyn_in%elem(ie)%derived%FT(i,j,k) - out_temp(i+(j-1)*np,k) = dyn_in%elem(ie)%state%T(i,j,k,tl_f) + out_temp(i+(j-1)*np,k) = temperature(i,j,k) out_u(i+(j-1)*np,k) = dyn_in%elem(ie)%state%v(i,j,1,k,tl_f) out_v(i+(j-1)*np,k) = dyn_in%elem(ie)%state%v(i,j,2,k,tl_f) out_q(i+(j-1)*np,k) = dyn_in%elem(ie)%state%Q(i,j,k,1) diff --git a/components/eam/src/physics/cam/co2_diagnostics.F90 b/components/eam/src/physics/cam/co2_diagnostics.F90 index c24c908a5246..81fdb3e9d76b 100644 --- a/components/eam/src/physics/cam/co2_diagnostics.F90 +++ b/components/eam/src/physics/cam/co2_diagnostics.F90 @@ -433,16 +433,24 @@ subroutine print_global_carbon_diags(state, dtime, nstep) ( co2_print_diags_monthly .and. is_end_curr_month() ) .or. & ( co2_print_diags_total .and. is_last_step() ) ) then call gmean(tc, tc_glob, c_num_var) + else + tc_glob(:) = 1.e36 end if if ( co2_print_diags_timestep) then call gmean(flux_ts, flux_ts_glob, f_ts_num_var) + else + flux_ts_glob(:) = 0._r8 end if if ( co2_print_diags_monthly .and. is_end_curr_month() ) then call gmean(flux_mon, flux_mon_glob, f_mon_num_var) + else + flux_mon_glob(:) = 0._r8 end if if ( co2_print_diags_total .and. is_last_step() ) then call gmean(flux_run, flux_run_glob, f_run_num_var) + else + flux_run_glob(:) = 0._r8 end if ! assign global means to readable variables diff --git a/components/eam/src/physics/cam/ndrop.F90 b/components/eam/src/physics/cam/ndrop.F90 index b4dc999309bf..6ffe89fa730b 100644 --- a/components/eam/src/physics/cam/ndrop.F90 +++ b/components/eam/src/physics/cam/ndrop.F90 @@ -58,6 +58,10 @@ module ndrop (/ 0.02_r8, 0.05_r8, 0.1_r8, 0.2_r8, 0.5_r8, 1.0_r8 /) character(len=8) :: ccn_name(psat)= & (/'CCN1','CCN2','CCN3','CCN4','CCN5','CCN6'/) +! The following additional fields output CCN in units of 1/kg, +! which could be handy for appplications such as SCREAM-SPA. +character(len=8) :: ccnmair_name(psat)= & + (/'CCN1MAIR','CCN2MAIR','CCN3MAIR','CCN4MAIR','CCN5MAIR','CCN6MAIR'/) ! indices in state and pbuf structures integer :: numliq_idx = -1 @@ -274,6 +278,12 @@ subroutine ndrop_init call addfld('CCN5',(/ 'lev' /), 'A','1/cm3','CCN concentration at S=0.5%') call addfld('CCN6',(/ 'lev' /), 'A','1/cm3','CCN concentration at S=1.0%') + call addfld('CCN1MAIR',(/ 'lev' /), 'A','1/kg','CCN concentration at S=0.02%') + call addfld('CCN2MAIR',(/ 'lev' /), 'A','1/kg','CCN concentration at S=0.05%') + call addfld('CCN3MAIR',(/ 'lev' /), 'A','1/kg','CCN concentration at S=0.1%') + call addfld('CCN4MAIR',(/ 'lev' /), 'A','1/kg','CCN concentration at S=0.2%') + call addfld('CCN5MAIR',(/ 'lev' /), 'A','1/kg','CCN concentration at S=0.5%') + call addfld('CCN6MAIR',(/ 'lev' /), 'A','1/kg','CCN concentration at S=1.0%') call addfld('WTKE', (/ 'lev' /), 'A', 'm/s', 'Standard deviation of updraft velocity') call addfld('NDROPMIX', (/ 'lev' /), 'A', '1/kg/s', 'Droplet number mixing') @@ -435,7 +445,8 @@ subroutine dropmixnuc( & real(r8), allocatable :: coltend(:,:) ! column tendency for diagnostic output real(r8), allocatable :: coltend_cw(:,:) ! column tendency - real(r8) :: ccn(pcols,pver,psat) ! number conc of aerosols activated at supersat + real(r8) :: ccn(pcols,pver,psat) ! number conc [1/m3] of aerosols activated at supersat + real(r8) :: ccnmair(pcols,pver,psat) ! number conc [1/kg] of aerosols activated at supersat integer :: ccn3d_idx real(r8), pointer :: ccn3d(:, :) @@ -1105,9 +1116,10 @@ subroutine dropmixnuc( & call outfld('NDROPMIX', ndropmix, pcols, lchnk) call outfld('WTKE ', wtke, pcols, lchnk) - call ccncalc(state, pbuf, cs, ccn) + call ccncalc(state, pbuf, cs, ccn, ccnmair) do l = 1, psat call outfld(ccn_name(l), ccn(1,1,l), pcols, lchnk) + call outfld(ccnmair_name(l), ccnmair(1,1,l), pcols, lchnk) enddo if(do_aerocom_ind3) then @@ -1704,7 +1716,7 @@ end subroutine maxsat !=============================================================================== -subroutine ccncalc(state, pbuf, cs, ccn) +subroutine ccncalc(state, pbuf, cs, ccn, ccnmair) ! calculates number concentration of aerosols activated as CCN at ! supersaturation supersat. @@ -1721,6 +1733,7 @@ subroutine ccncalc(state, pbuf, cs, ccn) real(r8), intent(in) :: cs(pcols,pver) ! air density (kg/m3) real(r8), intent(out) :: ccn(pcols,pver,psat) ! number conc of aerosols activated at supersat (#/m3) + real(r8), intent(out) :: ccnmair(pcols,pver,psat) ! number conc of aerosols activated at supersat (#/kg) ! local @@ -1770,6 +1783,7 @@ subroutine ccncalc(state, pbuf, cs, ccn) end do ccn = 0._r8 + ccnmair = 0._r8 do k=top_lev,pver do i=1,ncol @@ -1796,6 +1810,7 @@ subroutine ccncalc(state, pbuf, cs, ccn) do i=1,ncol arg(i)=argfactor(m)*log(sm(i)/super(l)) ccn(i,k,l)=ccn(i,k,l)+naerosol(i)*0.5_r8*(1._r8-erf(arg(i))) + ccnmair(i,k,l)=ccn(i,k,l)/cs(i,k) ! convert from #/m3 to #/kg enddo enddo enddo diff --git a/components/eam/src/physics/cam/phys_grid_ctem.F90 b/components/eam/src/physics/cam/phys_grid_ctem.F90 new file mode 100644 index 000000000000..470264a66819 --- /dev/null +++ b/components/eam/src/physics/cam/phys_grid_ctem.F90 @@ -0,0 +1,405 @@ +!---------------------------------------------------------------------------------- +! circulation diagnostics -- terms of the Transformed Eulerian Mean (TEM) equation +! +! This module makes use of the zonal_mean_mod methods to produce terms needed +! for the TEM budget from the physics grid state. Without this online +! calculation of terms, high-frequency 3D history output would be required to +! calculate the terms offline. The resulting zonal mean quantities and eddy +! covariance terms take up much less disk space and can be output monthly +! +! History output variables: +! Uzm Zonal-Mean zonal wind +! Vzm Zonal-Mean meridional wind +! Wzm Zonal-Mean vertical wind +! THzm Zonal-Mean potential temp +! VTHzm Meridional Heat Flux +! WTHzm Vertical Heat Flux +! UVzm Meridional Flux of Zonal Momentum +! UWzm Vertical Flux of Zonal Momentum +! THphys Potential temp +!---------------------------------------------------------------------------------- +module phys_grid_ctem +use shr_kind_mod, only: r8 => shr_kind_r8 +use ppgrid, only: begchunk, endchunk, pcols, pver +use physics_types, only: physics_state +use cam_history, only: addfld, outfld +use zonal_mean_mod,only: ZonalAverage_t, ZonalMean_t +use physconst, only: pi +use cam_logfile, only: iulog +use cam_abortutils,only: endrun, handle_allocate_error +use namelist_utils,only: find_group_name +use spmd_utils, only: masterproc, mpi_integer, masterprocid, mpicom +use time_manager, only: get_nstep + +use shr_const_mod, only: rgas => shr_const_rgas ! J/K/kmole +use shr_const_mod, only: grav => shr_const_g ! m/s2 +use string_utils, only: int2str + +implicit none + +private +public :: phys_grid_ctem_readnl +public :: phys_grid_ctem_reg +public :: phys_grid_ctem_init +public :: phys_grid_ctem_diags +public :: phys_grid_ctem_final + +type(ZonalMean_t) :: ZMobj +type(ZonalAverage_t) :: ZAobj + +integer :: nzalat = -huge(1) +integer :: nzmbas = -huge(1) + +integer :: ntimesteps = -huge(1) ! number of time steps bewteen TEM calculations + +logical :: do_tem_diags = .false. + +contains + +!------------------------------------------------------------------------------ +!------------------------------------------------------------------------------ +subroutine phys_grid_ctem_readnl(nlfile,dtime) + character(len=*), intent(in) :: nlfile + integer, intent(in) :: dtime ! Step time in seconds + integer :: ierr, unitn + + character(len=*), parameter :: prefix = 'phys_grid_ctem_readnl: ' + + integer :: phys_grid_ctem_zm_nbas + integer :: phys_grid_ctem_za_nlat + integer :: phys_grid_ctem_nfreq + + namelist /phys_grid_ctem_opts/ phys_grid_ctem_zm_nbas, phys_grid_ctem_za_nlat, phys_grid_ctem_nfreq + + phys_grid_ctem_zm_nbas = 0 + phys_grid_ctem_za_nlat = 0 + phys_grid_ctem_nfreq = 0 + + ! Read in namelist values + !------------------------ + if(masterproc) then + open(newunit=unitn, file=trim(nlfile), status='old') + call find_group_name(unitn, 'phys_grid_ctem_opts', status=ierr) + if(ierr == 0) then + read(unitn,phys_grid_ctem_opts,iostat=ierr) + if(ierr /= 0) then + call endrun(prefix//'ERROR reading namelist') + end if + end if + close(unitn) + end if + + call MPI_bcast(phys_grid_ctem_zm_nbas, 1, mpi_integer, masterprocid, mpicom, ierr) + if (ierr /= 0) call endrun(prefix//'FATAL: mpi_bcast: phys_grid_ctem_zm_nbas') + call MPI_bcast(phys_grid_ctem_za_nlat, 1, mpi_integer, masterprocid, mpicom, ierr) + if (ierr /= 0) call endrun(prefix//'FATAL: mpi_bcast: phys_grid_ctem_za_nlat') + call MPI_bcast(phys_grid_ctem_nfreq, 1, mpi_integer, masterprocid, mpicom, ierr) + if (ierr /= 0) call endrun(prefix//'FATAL: mpi_bcast: phys_grid_ctem_nfreq') + + do_tem_diags = .false. + if (phys_grid_ctem_nfreq/=0) then + if (.not.(phys_grid_ctem_zm_nbas>0 .and. phys_grid_ctem_za_nlat>0)) then + call endrun(prefix//'inconsistent phys_grid_ctem namelist settings -- phys_grid_ctem_zm_nbas=' & + //int2str(phys_grid_ctem_zm_nbas)//', phys_grid_ctem_za_nlat='//int2str(phys_grid_ctem_za_nlat)) + end if + if (phys_grid_ctem_nfreq>0) then + ntimesteps = phys_grid_ctem_nfreq + else + ntimesteps = nint( -phys_grid_ctem_nfreq*3600._r8/dtime ) + end if + if (ntimesteps<1) then + call endrun(prefix//'invalid ntimesteps -- phys_grid_ctem_nfreq needs to be a larger negative value ' & + //'or the model time step needs to be shorter') + end if + do_tem_diags = .true. + end if + + if (masterproc) then + if (do_tem_diags) then + write(iulog,*) 'TEM diagnostics will be calculated every ',ntimesteps,' time steps' + write(iulog,*) ' phys_grid_ctem_zm_nbas = ', phys_grid_ctem_zm_nbas + write(iulog,*) ' phys_grid_ctem_za_nlat = ', phys_grid_ctem_za_nlat + write(iulog,*) ' phys_grid_ctem_nfreq = ', phys_grid_ctem_nfreq + else + write(iulog,*) 'TEM diagnostics will not be performed' + end if + endif + + if (do_tem_diags) then + nzalat = phys_grid_ctem_za_nlat + nzmbas = phys_grid_ctem_zm_nbas + end if + +end subroutine phys_grid_ctem_readnl + +!----------------------------------------------------------------------------- +!----------------------------------------------------------------------------- +subroutine phys_grid_ctem_reg + + use cam_grid_support, only: horiz_coord_t, horiz_coord_create, iMap, cam_grid_register + + type(horiz_coord_t), pointer :: zalon_coord + type(horiz_coord_t), pointer :: zalat_coord + integer(iMap), pointer :: grid_map(:,:) + + real(r8) :: zalats(nzalat) + real(r8) :: area(nzalat) + real(r8) :: zalons(1) + real(r8) :: dlatrad, dlatdeg, lat1, lat2 + real(r8) :: total_area + real(r8) :: total_wght + integer :: j, astat + + real(r8), parameter :: latdeg0 = -90._r8 + real(r8), parameter :: latrad0 = -pi*0.5_r8 + real(r8), parameter :: fourpi = pi*4._r8 + + integer, parameter :: ctem_zavg_phys_decomp = 333 ! Must be unique within CAM + + if (.not.do_tem_diags) return + + nullify(zalat_coord) + nullify(zalon_coord) + nullify(grid_map) + + zalons(1) = 0._r8 + + dlatrad = pi/real(nzalat,kind=r8) + dlatdeg = 180._r8/real(nzalat,kind=r8) + total_area = 0._r8 + total_wght = 0._r8 + + ! calculate latitudes and areas of zonal average grid boxes + do j = 1,nzalat + zalats(j) = latdeg0 + (real(j,kind=r8)-0.5_r8)*dlatdeg + lat1 = latrad0 + real(j-1,kind=r8)*dlatrad + lat2 = latrad0 + real(j ,kind=r8)*dlatrad + area(j) = 2._r8*pi*(sin(lat2)-sin(lat1)) + total_area = total_area + area(j) + total_wght = total_wght + 0.5_r8*(sin(lat2)-sin(lat1)) + end do + + ! sanity check + if ( abs(1._r8-total_wght)>1.e-12_r8 .or. abs(fourpi-total_area)>1.e-12_r8 ) then + call endrun('phys_grid_ctem_reg: problem with area/wght calc') + end if + + ! initialize zonal-average and zonal-mean utility objects + call ZAobj%init(zalats,area,nzalat,GEN_GAUSSLATS=.false.) + call ZMobj%init(nzmbas) + + ! Zonal average grid for history fields + + zalat_coord => horiz_coord_create('zalat', '', nzalat, 'latitude', 'degrees_north', 1, nzalat, zalats) + zalon_coord => horiz_coord_create('zalon', '', 1, 'longitude', 'degrees_east', 1, 1, zalons) + + ! grid decomposition map + allocate(grid_map(4,nzalat), stat=astat) + call handle_allocate_error(astat, 'phys_grid_ctem_reg', 'grid_map') + + do j = 1,nzalat + grid_map(1,j) = 1 + grid_map(2,j) = j + if (masterproc) then + grid_map(3,j) = 1 + grid_map(4,j) = j + else + grid_map(3,j) = 0 + grid_map(4,j) = 0 + end if + end do + + ! register the zonal average grid + call cam_grid_register('ctem_zavg_phys', ctem_zavg_phys_decomp, zalat_coord, zalon_coord, grid_map, & + unstruct=.false., zonal_grid=.true.) + +end subroutine phys_grid_ctem_reg + +!----------------------------------------------------------------------------- +!----------------------------------------------------------------------------- +subroutine phys_grid_ctem_init + + if (.not.do_tem_diags) return + + call addfld ('Uzm', (/'lev'/), 'A','m s-1', 'Zonal-Mean zonal wind', gridname='ctem_zavg_phys' ) + call addfld ('Vzm', (/'lev'/), 'A','m s-1', 'Zonal-Mean meridional wind', gridname='ctem_zavg_phys' ) + call addfld ('Wzm', (/'lev'/), 'A','m s-1', 'Zonal-Mean vertical wind', gridname='ctem_zavg_phys' ) + call addfld ('THzm', (/'lev'/), 'A','K', 'Zonal-Mean potential temp', gridname='ctem_zavg_phys' ) + call addfld ('VTHzm',(/'lev'/), 'A','K m s-1','Meridional Heat Flux:', gridname='ctem_zavg_phys') + call addfld ('WTHzm',(/'lev'/), 'A','K m s-1','Vertical Heat Flux:', gridname='ctem_zavg_phys') + call addfld ('UVzm', (/'lev'/), 'A','m2 s-2', 'Meridional Flux of Zonal Momentum', gridname='ctem_zavg_phys') + call addfld ('UWzm', (/'lev'/), 'A','m2 s-2', 'Vertical Flux of Zonal Momentum', gridname='ctem_zavg_phys') + call addfld ('THphys',(/'lev'/), 'A', 'K', 'Potential temp', gridname='physgrid' ) + +end subroutine phys_grid_ctem_init + +!----------------------------------------------------------------------------- +!----------------------------------------------------------------------------- +subroutine phys_grid_ctem_diags(phys_state) + type(physics_state), intent(in) :: phys_state(begchunk:endchunk) + + character(len=*), parameter :: prefix = 'phys_grid_ctem_diags: ' + + real(r8) :: u(pcols,pver,begchunk:endchunk) + real(r8) :: v(pcols,pver,begchunk:endchunk) + real(r8) :: w(pcols,pver,begchunk:endchunk) + + real(r8) :: uzm(pcols,pver,begchunk:endchunk) + real(r8) :: vzm(pcols,pver,begchunk:endchunk) + real(r8) :: wzm(pcols,pver,begchunk:endchunk) + + real(r8) :: ud(pcols,pver,begchunk:endchunk) + real(r8) :: vd(pcols,pver,begchunk:endchunk) + real(r8) :: wd(pcols,pver,begchunk:endchunk) + real(r8) :: thd(pcols,pver,begchunk:endchunk) + + real(r8) :: uvp(pcols,pver,begchunk:endchunk) + real(r8) :: uwp(pcols,pver,begchunk:endchunk) + real(r8) :: vthp(pcols,pver,begchunk:endchunk) + real(r8) :: wthp(pcols,pver,begchunk:endchunk) + + integer :: lchnk, ncol, j, k + + ! potential temperature + real(r8) :: theta(pcols,pver,begchunk:endchunk) + real(r8) :: thzm(pcols,pver,begchunk:endchunk) + + real(r8) :: uvza(nzalat,pver) + real(r8) :: uwza(nzalat,pver) + real(r8) :: vthza(nzalat,pver) + real(r8) :: wthza(nzalat,pver) + + real(r8) :: uza(nzalat,pver) + real(r8) :: vza(nzalat,pver) + real(r8) :: wza(nzalat,pver) + real(r8) :: thza(nzalat,pver) + + real(r8) :: mbarv ! molecular weight of dry air (g/mol) + real(r8) :: sheight(pcols,pver) ! pressure scale height (m) + + if (.not.do_calc()) return + + ! In CESM/WACCM the variable mbarv is provided by the "air_composition" + ! module, which is not in E3SM, so we just use a rough approximation + mbarv = 28.97 + + do lchnk = begchunk,endchunk + + ncol = phys_state(lchnk)%ncol + + ! scale height + sheight(:ncol,:) = phys_state(lchnk)%t(:ncol,:) * rgas / ( mbarv * grav ) ! meters + + ! potential temperature + theta(:ncol,:,lchnk) = phys_state(lchnk)%t(:ncol,:) * phys_state(lchnk)%exner(:ncol,:) + + ! vertical velocity + w(:ncol,:,lchnk) = -sheight(:ncol,:) * phys_state(lchnk)%omega(:ncol,:) / phys_state(lchnk)%pmid(:ncol,:) + + u(:ncol,:,lchnk) = phys_state(lchnk)%u(:ncol,:) + v(:ncol,:,lchnk) = phys_state(lchnk)%v(:ncol,:) + + end do + + ! zonal means evaluated on the physics grid (3D) to be used in the deviations calculation below + uzm(:,:,:) = zmean_fld(u(:,:,:)) + vzm(:,:,:) = zmean_fld(v(:,:,:)) + wzm(:,:,:) = zmean_fld(w(:,:,:)) + thzm(:,:,:) = zmean_fld(theta(:,:,:)) + + + ! diagnostic output + do lchnk = begchunk, endchunk + call outfld( 'THphys', theta(:,:,lchnk), pcols, lchnk) + end do + + do lchnk = begchunk,endchunk + ncol = phys_state(lchnk)%ncol + do k = 1,pver + ! zonal deviations + thd(:ncol,k,lchnk) = theta(:ncol,k,lchnk) - thzm(:ncol,k,lchnk) + ud(:ncol,k,lchnk) = u(:ncol,k,lchnk) - uzm(:ncol,k,lchnk) + vd(:ncol,k,lchnk) = v(:ncol,k,lchnk) - vzm(:ncol,k,lchnk) + wd(:ncol,k,lchnk) = w(:ncol,k,lchnk) - wzm(:ncol,k,lchnk) + ! fluxes + uvp(:ncol,k,lchnk) = ud(:ncol,k,lchnk) * vd(:ncol,k,lchnk) + uwp(:ncol,k,lchnk) = ud(:ncol,k,lchnk) * wd(:ncol,k,lchnk) + vthp(:ncol,k,lchnk) = vd(:ncol,k,lchnk) * thd(:ncol,k,lchnk) + wthp(:ncol,k,lchnk) = wd(:ncol,k,lchnk) * thd(:ncol,k,lchnk) + end do + end do + + ! evaluate and output fluxes on the zonal-average grid + call ZAobj%binAvg(uvp, uvza) + call ZAobj%binAvg(uwp, uwza) + call ZAobj%binAvg(vthp, vthza) + call ZAobj%binAvg(wthp, wthza) + + + if (any(abs(uvza)>1.e20_r8)) call endrun(prefix//'bad values in uvza') + if (any(abs(uwza)>1.e20_r8)) call endrun(prefix//'bad values in uwza') + if (any(abs(vthza)>1.e20_r8)) call endrun(prefix//'bad values in vthza') + if (any(abs(wthza)>1.e20_r8)) call endrun(prefix//'bad values in wthza') + + call ZAobj%binAvg(uzm, uza) + call ZAobj%binAvg(vzm, vza) + call ZAobj%binAvg(wzm, wza) + call ZAobj%binAvg(thzm, thza) + + + if (any(abs(uza)>1.e20_r8)) call endrun(prefix//'bad values in uza') + if (any(abs(vza)>1.e20_r8)) call endrun(prefix//'bad values in vza') + if (any(abs(wza)>1.e20_r8)) call endrun(prefix//'bad values in wza') + if (any(abs(thza)>1.e20_r8)) call endrun(prefix//'bad values in thza') + + ! diagnostic output + do j = 1,nzalat + call outfld('Uzm',uza(j,:),1,j) + call outfld('Vzm',vza(j,:),1,j) + call outfld('Wzm',wza(j,:),1,j) + call outfld('THzm',thza(j,:),1,j) + call outfld('UVzm',uvza(j,:),1,j) + call outfld('UWzm',uwza(j,:),1,j) + call outfld('VTHzm',vthza(j,:),1,j) + call outfld('WTHzm',wthza(j,:),1,j) + end do + + contains + + !------------------------------------------------------------------------------ + ! utility function for evaluating 3D zonal mean fields + !------------------------------------------------------------------------------ + function zmean_fld( fld ) result(fldzm) + + real(r8), intent(in) :: fld(pcols,pver,begchunk:endchunk) + + real(r8) :: fldzm(pcols,pver,begchunk:endchunk) + + real(r8) :: Zonal_Bamp3d(nzmbas,pver) + + call ZMobj%calc_amps(fld,Zonal_Bamp3d) + call ZMobj%eval_grid(Zonal_Bamp3d,fldzm) + + end function zmean_fld + + !------------------------------------------------------------------------------ + ! utility function returns TRUE when time to update TEM diags + !------------------------------------------------------------------------------ + logical function do_calc() + + integer :: nstep + nstep = get_nstep() + do_calc = do_tem_diags .and. mod(nstep,ntimesteps) == 0 + + end function do_calc + +end subroutine phys_grid_ctem_diags + +!----------------------------------------------------------------------------- +!----------------------------------------------------------------------------- +subroutine phys_grid_ctem_final + call ZAobj%final() + call ZMobj%final() +end subroutine phys_grid_ctem_final + +end module phys_grid_ctem diff --git a/components/eam/src/physics/cam/physpkg.F90 b/components/eam/src/physics/cam/physpkg.F90 index 5f942152a20f..5ece67252791 100644 --- a/components/eam/src/physics/cam/physpkg.F90 +++ b/components/eam/src/physics/cam/physpkg.F90 @@ -526,10 +526,6 @@ subroutine phys_inidat( cam_out, pbuf2d ) end do deallocate(tptr) - do lchnk=begchunk,endchunk - cam_out(lchnk)%tbot(:) = posinf - end do - ! ! 3-D fields ! @@ -788,6 +784,7 @@ subroutine phys_init( phys_state, phys_tend, pbuf2d, cam_out ) use output_aerocom_aie, only: output_aerocom_aie_init, do_aerocom_ind3 use misc_diagnostics, only: dcape_diags_init use conditional_diag_output_utils, only: cnd_diag_output_init + use phys_grid_ctem, only: phys_grid_ctem_init ! Input/output arguments type(physics_state), pointer :: phys_state(:) @@ -1013,6 +1010,9 @@ subroutine phys_init( phys_state, phys_tend, pbuf2d, cam_out ) !-------------------------------- if(Nudge_Model) call nudging_init + ! Initialize Transformed Eularian Mean (TEM) diagnostics + call phys_grid_ctem_init() + !BSINGH - addfld and adddefault calls for perturb growth testing if(pergro_test_active)call add_fld_default_calls() @@ -1495,6 +1495,7 @@ subroutine phys_final( phys_state, phys_tend, pbuf2d, phys_diag ) use chemistry, only : chem_final use wv_saturation, only : wv_sat_final use radiation, only: radiation_final + use phys_grid_ctem, only : phys_grid_ctem_final !----------------------------------------------------------------------- ! ! Purpose: @@ -1531,6 +1532,10 @@ subroutine phys_final( phys_state, phys_tend, pbuf2d, phys_diag ) call print_cost_p call t_stopf ('print_cost_p') + call t_startf ('phys_grid_ctem_final') + call phys_grid_ctem_final() + call t_stopf ('phys_grid_ctem_final') + end subroutine phys_final @@ -3147,6 +3152,7 @@ subroutine phys_timestep_init(phys_state, cam_out, pbuf2d) use nudging, only: Nudge_Model,nudging_timestep_init use seasalt_model, only: advance_ocean_data, has_mam_mom + use phys_grid_ctem, only: phys_grid_ctem_diags implicit none @@ -3225,6 +3231,9 @@ subroutine phys_timestep_init(phys_state, cam_out, pbuf2d) !---------------------------------- if(Nudge_Model) call nudging_timestep_init(phys_state) + ! Update Transformed Eularian Mean (TEM) diagnostics + call phys_grid_ctem_diags(phys_state) + end subroutine phys_timestep_init diff --git a/components/eam/src/physics/cam/zonal_mean_mod.F90 b/components/eam/src/physics/cam/zonal_mean_mod.F90 new file mode 100644 index 000000000000..e3e52d0bd6dd --- /dev/null +++ b/components/eam/src/physics/cam/zonal_mean_mod.F90 @@ -0,0 +1,2000 @@ +module zonal_mean_mod +!====================================================================== +! +! Purpose: Compute and make use of Zonal Mean values on physgrid +! +! This module implements 3 data structures for the spectral analysis +! and synthesis of zonal mean values based on m=0 spherical harmonics. +! +! ZonalMean_t: For the analysis/synthesis of zonal mean values +! on a 2D grid of points distributed over the +! surface of a sphere. +! ZonalProfile_t: For the analysis/synthesis of zonal mean values +! on a meridional grid that spans the latitudes +! from SP to NP +! ZonalAverage_t: To calculate zonal mean values via a simple +! area weighted bin-averaging of 2D grid points +! assigned to each latitude band. +! +! NOTE: The weighting of the Zonal Profiles values is scaled such +! that ZonalMean_t amplitudes can be used to evaluate values +! on the ZonalProfile_t grid and vice-versa. +! +! The ZonalMean_t computes global integrals to compute basis +! amplitudes. For distributed environments the cost of these +! can be reduced using the The ZonalAverage_t data structures. +! +! USAGE: +! +! (1) Compute Zonal mean amplitudes and synthesize values on 2D/3D physgrid +! +! Usage: type(ZonalMean_t):: ZM +! ========================================= +! call ZM%init(nbas) +! ------------------ +! - Initialize the data structure with 'nbas' basis functions +! for the given physgrid latitudes and areas. +! +! Arguments: +! integer ,intent(in):: nbas -Number of m=0 spherical harmonics +! +! call ZM%calc_amps(Gdata,Bamp) +! ----------------------------- +! - For the initialized ZonalMean_t; Given Gdata() values on the physgrid, +! compute the zonal mean basis amplitudes Bamp(). +! +! Interface: 2D data on the physgrid +! real(r8),intent(in ):: Gdata(pcols,begchunk:endchunk) +! real(r8),intent(out):: Bamp (nbas) +! +! Interface: 3D data on the physgrid +! real(r8),intent(in ):: Gdata(pcols,pver,begchunk:endchunk) +! real(r8),intent(out):: Bamp (nbas,pver) +! +! call ZM%eval_grid(Bamp,Gdata) +! ----------------------------- +! - For the initialized ZonalMean_t; Given Bamp() zonal mean basis +! amplitudes, compute the Gdata() values on the physgrid. +! +! Interface: 2D data on the physgrid +! real(r8),intent(in ):: Bamp (nbas) +! real(r8),intent(out):: Gdata(pcols,begchunk:endchunk) +! +! Interface: 3D data on the physgrid +! real(r8),intent(in ):: Bamp (nbas,pver) +! real(r8),intent(out):: Gdata(pcols,pver,begchunk:endchunk) +! +! +! (2) Compute Zonal mean amplitudes and synthesize values on Zonal profile grid +! +! Usage: type(ZonalProfile_t):: ZP +! ========================================= +! call ZP%init(lats,area,nlat,nbas,GEN_GAUSSLATS=.true.) +! ------------------------------------------------------ +! - Initialize the data structure for the given number of +! latitudes. Either use the given Latitudes and weights, +! or OPTIONALLY create profile gridpoints and associated +! area weights from SP to NP. Then initialize 'nbas' basis +! functions for the profile gridpoints. +! If the user supplies the lats/area values, the area values must +! be correctly scaled such that the global area adds up to 4PI. +! Otherwise, the ampitudes between ZonalProfile_t and ZonalMean_t +! are not interchangable. +! +! Arguments: +! real(r8),intent(inout):: lats(:) - Latitudes of meridional grid. +! real(r8),intent(inout):: area(:) - Area of each meridional gridpoint. +! integer ,intent(in) :: nlat - Number of meridional gridpoints. +! integer ,intent(in) :: nbas - Number of m=0 spherical harmonics +! logical ,intent(in),optional:: GEN_GAUSLATS - Flag to generate +! lats/areas values. +! +! call ZP%calc_amps(Zdata,Bamp) +! ----------------------------- +! - Given Zdata() on the Zonal profile grid, compute the +! zonal basis amplitudes Bamp(). +! +! Interface: 1D data on (nlat) grid +! real(r8),intent(in ):: Zdata(nlat) - Meridional Profile data +! real(r8),intent(out):: Bamp (nbas) - Zonal Basis Amplitudes +! +! Interface: 2D data on (nlat,pver) grid +! real(r8),intent(in ):: Zdata(nlat,pver) - Meridional Profile data +! real(r8),intent(out):: Bamp (nbas,pver) - Zonal Basis Amplitudes +! +! call ZP%eval_grid(Bamp,Zdata) +! ----------------------------- +! - Given Bamp() zonal basis amplitudes, evaluate the Zdata() +! values on the Zonal profile grid. +! +! Interface: 1D data on (nlat) grid +! real(r8),intent(in ):: Bamp (nbas) - Zonal Basis Amplitudes +! real(r8),intent(out):: Zdata(nlat) - Meridional Profile data +! +! Interface: 2D data on (nlat,pver) grid +! real(r8),intent(in ):: Bamp (nbas,pver) - Zonal Basis Amplitudes +! real(r8),intent(out):: Zdata(nlat,pver) - Meridional Profile data +! +! (3) Compute Zonal mean averages (FASTER/LESS-ACCURATE) on Zonal profile grid +! (For the created zonal profile, just bin average area weighted +! 2D/3D physgrid grid values) +! +! Usage: type(ZonalAverage_t):: ZA +! ========================================= +! call ZA%init(lats,area,nlat,GEN_GAUSSLATS=.true.) +! -------------------------------------------------- +! - Given the latitude/area for the nlat meridional gridpoints, initialize +! the ZonalAverage data structure for computing bin-averaging of physgrid +! values. It is assumed that the domain of these gridpoints of the +! profile span latitudes from SP to NP. +! The optional GEN_GAUSSLATS flag allows for the generation of Gaussian +! latitude gridpoints. The generated grid over-writes the given values +! lats and area passed by the user. +! +! Arguments: +! real(r8),intent(inout):: lats(nlat) - Latitudes of meridional grid. +! real(r8),intent(inout):: area(nlat) - Area of meridional gridpoints. +! integer ,intent(in):: nlat - Number of meridional gridpoints +! logical,intent(in),optional:: GEN_GAUSLATS - Flag to generate +! lats/areas values. +! +! call ZA%binAvg(Gdata,Zdata) +! --------------------------- +! - For the initialized ZonalAverage_t; Given Gdata() on the physgrid, +! compute bin averages and return Zdata() on the Zonal profile grid. +! +! Interface: 2D data on the physgrid +! real(r8),intent(out):: Gdata(pcols,begchunk:endchunk) +! real(r8),intent(out):: Zdata(nlat) +! +! Interface: 3D data on the physgrid +! real(r8),intent(out):: Gdata(pcols,pver,begchunk:endchunk) +! real(r8),intent(out):: Zdata(nlat,pver) +! +!====================================================================== + +use shr_kind_mod, only: r8=>SHR_KIND_R8 +use phys_grid, only: get_ncols_p, get_rlat_p, get_wght_all_p, get_nlcols_p +use ppgrid, only: begchunk, endchunk, pcols +use shr_reprosum_mod,only: shr_reprosum_calc +use cam_abortutils, only: endrun, handle_allocate_error +use spmd_utils, only: mpicom +use physconst, only: pi +use phys_grid, only: ngcols_p +use cam_logfile, only: iulog + +implicit none +private + +public :: ZonalMean_t +public :: ZonalProfile_t +public :: ZonalAverage_t + +! Type definitions +!------------------- +type ZonalMean_t + private + integer :: nbas + real(r8),allocatable:: area (:,:) + real(r8),allocatable:: basis(:,:,:) + real(r8),allocatable:: map (:,:) + contains + procedure,pass:: init => init_ZonalMean + generic,public:: calc_amps => calc_ZonalMean_2Damps, & + calc_ZonalMean_3Damps + generic,public:: eval_grid => eval_ZonalMean_2Dgrid, & + eval_ZonalMean_3Dgrid + procedure,private,pass:: calc_ZonalMean_2Damps + procedure,private,pass:: calc_ZonalMean_3Damps + procedure,private,pass:: eval_ZonalMean_2Dgrid + procedure,private,pass:: eval_ZonalMean_3Dgrid + procedure, pass :: final => final_ZonalMean +end type ZonalMean_t + +type ZonalProfile_t + private + integer :: nlat + integer :: nbas + real(r8),allocatable:: area (:) + real(r8),allocatable:: basis(:,:) + real(r8),allocatable:: map (:,:) + contains + procedure,pass:: init => init_ZonalProfile + generic,public:: calc_amps => calc_ZonalProfile_1Damps, & + calc_ZonalProfile_2Damps + generic,public:: eval_grid => eval_ZonalProfile_1Dgrid, & + eval_ZonalProfile_2Dgrid + procedure,private,pass:: calc_ZonalProfile_1Damps + procedure,private,pass:: calc_ZonalProfile_2Damps + procedure,private,pass:: eval_ZonalProfile_1Dgrid + procedure,private,pass:: eval_ZonalProfile_2Dgrid + procedure, pass :: final => final_ZonalProfile +end type ZonalProfile_t + +type ZonalAverage_t + private + integer :: nlat + real(r8),allocatable:: area (:) + real(r8),allocatable:: a_norm (:) + real(r8),allocatable:: area_g (:,:) + integer ,allocatable:: idx_map(:,:) + contains + procedure,pass:: init => init_ZonalAverage + generic,public:: binAvg => calc_ZonalAverage_2DbinAvg, & + calc_ZonalAverage_3DbinAvg + procedure,private,pass:: calc_ZonalAverage_2DbinAvg + procedure,private,pass:: calc_ZonalAverage_3DbinAvg + procedure, pass :: final => final_ZonalAverage +end type ZonalAverage_t + +real(r8), parameter :: halfPI = 0.5_r8*pi +real(r8), parameter :: twoPI = 2.0_r8*pi +real(r8), parameter :: fourPI = 4.0_r8*pi +real(r8), parameter :: qrtrPI = 0.25_r8*pi +real(r8), parameter :: invSqrt4pi = 1._r8/sqrt(fourPI) + +contains +!======================================================================= +subroutine init_ZonalMean(this,I_nbas) + ! + ! init_ZonalMean: Initialize the ZonalMean data structures for the + ! physics grid. It is assumed that the domain + ! of these gridpoints spans the surface of the sphere. + ! The representation of basis functions is + ! normalized w.r.t integration over the sphere. + !===================================================================== + ! + ! Passed Variables + !------------------ + class(ZonalMean_t) :: this + integer ,intent(in):: I_nbas + ! + ! Local Values + !-------------- + real(r8),allocatable:: Clats(:,:) + real(r8),allocatable:: Bcoef(:) + real(r8),allocatable:: Csum (:,:) + real(r8),allocatable:: Cvec (:) + real(r8),allocatable:: Bsum (:,:) + real(r8),allocatable:: Bnorm(:) + real(r8),allocatable:: Bcov (:,:) + real(r8):: area(pcols),rlat + + integer :: nn,n2,nb,lchnk,ncols,cc + integer :: cnum,Cvec_len + + integer :: nlcols, count, astat + character(len=*), parameter :: subname = 'init_ZonalMean' + + if (I_nbas<1) then + call endrun('ZonalMean%init: ERROR I_nbas must be greater than 0') + end if + + ! Allocate space + !----------------- + if(allocated(this%area )) deallocate(this%area) + if(allocated(this%basis)) deallocate(this%basis) + if(allocated(this%map )) deallocate(this%map) + + this%nbas = I_nbas + allocate(this%area (pcols,begchunk:endchunk), stat=astat) + call handle_allocate_error(astat, subname, 'this%area') + allocate(this%basis(pcols,begchunk:endchunk,I_nbas), stat=astat) + call handle_allocate_error(astat, subname, 'this%basis') + allocate(this%map (I_nbas,I_nbas), stat=astat) + call handle_allocate_error(astat, subname, 'this%map') + this%area (:,:) = 0._r8 + this%basis(:,:,:) = 0._r8 + this%map (:,:) = 0._r8 + + Cvec_len = 0 + do nn= 1,this%nbas + do n2=nn,this%nbas + Cvec_len = Cvec_len + 1 + end do + end do + + nlcols = get_nlcols_p() + + allocate(Clats(pcols,begchunk:endchunk), stat=astat) + call handle_allocate_error(astat, subname, 'Clats') + allocate(Bcoef(I_nbas/2+1), stat=astat) + call handle_allocate_error(astat, subname, 'Bcoef') + allocate(Csum (nlcols, Cvec_len), stat=astat) + call handle_allocate_error(astat, subname, 'Csum') + allocate(Cvec (Cvec_len), stat=astat) + call handle_allocate_error(astat, subname, 'Cvec') + allocate(Bsum (nlcols, I_nbas), stat=astat) + call handle_allocate_error(astat, subname, 'Bsum') + allocate(Bnorm(I_nbas), stat=astat) + call handle_allocate_error(astat, subname, 'Bnorm') + allocate(Bcov (I_nbas,I_nbas), stat=astat) + call handle_allocate_error(astat, subname, 'Bcov') + + Bsum(:,:) = 0._r8 + Csum(:,:) = 0._r8 + + ! Save a copy of the area weights for each ncol gridpoint + ! and convert Latitudes to SP->NP colatitudes in radians + !------------------------------------------------------- + do lchnk=begchunk,endchunk + ncols = get_ncols_p(lchnk) + call get_wght_all_p(lchnk, ncols, area) + do cc = 1,ncols + rlat=get_rlat_p(lchnk,cc) + this%area(cc,lchnk) = area(cc) + Clats (cc,lchnk) = rlat + halfPI + end do + end do + + ! Add first basis for the mean values. + !------------------------------------------ + this%basis(:,begchunk:endchunk,1) = invSqrt4pi + + ! Loop over the remaining basis functions + !--------------------------------------- + do nn=2,this%nbas + nb = nn-1 + + ! Generate coefs for the basis + !------------------------------ + call sh_gen_basis_coefs(nb,0,Bcoef) + + ! Create basis for the coefs at each ncol gridpoint + !--------------------------------------------------- + do lchnk=begchunk,endchunk + ncols = get_ncols_p(lchnk) + do cc = 1,ncols + call sh_create_basis(nb,0,Clats(cc,lchnk),Bcoef,this%basis(cc,lchnk,nn)) + end do + end do + end do ! nn=2,this%nbas + + ! Numerically normalize the basis funnctions + !-------------------------------------------------------------- + do nn=1,this%nbas + count = 0 + do lchnk=begchunk,endchunk + ncols = get_ncols_p(lchnk) + do cc = 1,ncols + count=count+1 + Bsum(count,nn) = this%basis(cc,lchnk,nn)*this%basis(cc,lchnk,nn)*this%area(cc,lchnk) + end do + end do + end do ! nn=1,this%nbas + + call shr_reprosum_calc(Bsum, Bnorm, count, nlcols, this%nbas, gbl_count=ngcols_p, commid=mpicom) + + do nn=1,this%nbas + do lchnk=begchunk,endchunk + ncols = get_ncols_p(lchnk) + this%basis(:ncols,lchnk,nn) = this%basis(:ncols,lchnk,nn)/sqrt(Bnorm(nn)) + end do + end do ! nn=1,this%nbas + + ! Compute covariance matrix for basis functions + ! (Yes, they are theoretically orthonormal, but lets make sure) + !--------------------------------------------------------------- + cnum = 0 + do nn= 1,this%nbas + do n2=nn,this%nbas + cnum = cnum + 1 + count = 0 + do lchnk=begchunk,endchunk + ncols = get_ncols_p(lchnk) + do cc = 1,ncols + count=count+1 + Csum(count,cnum) = this%basis(cc,lchnk,nn)*this%basis(cc,lchnk,n2)*this%area(cc,lchnk) + end do + end do + + end do + end do + + call shr_reprosum_calc(Csum, Cvec, count, nlcols, Cvec_len, gbl_count=ngcols_p, commid=mpicom) + + cnum = 0 + do nn= 1,this%nbas + do n2=nn,this%nbas + cnum = cnum + 1 + Bcov(nn,n2) = Cvec(cnum) + Bcov(n2,nn) = Cvec(cnum) + end do + end do + + ! Invert to get the basis amplitude map + !-------------------------------------- + call Invert_Matrix(Bcov,this%nbas,this%map) + + ! End Routine + !------------ + deallocate(Clats) + deallocate(Bcoef) + deallocate(Csum ) + deallocate(Cvec ) + deallocate(Bsum ) + deallocate(Bnorm) + deallocate(Bcov ) + +end subroutine init_ZonalMean +!======================================================================= + + +!======================================================================= +subroutine calc_ZonalMean_2Damps(this,I_Gdata,O_Bamp) + ! + ! calc_ZonalMean_2Damps: Given 2D data values for the ncol gridpoints, + ! compute the zonal mean basis amplitudes. + !===================================================================== + ! + ! Passed Variables + !------------------ + class(ZonalMean_t) :: this + real(r8),intent(in ) :: I_Gdata(pcols,begchunk:endchunk) + real(r8),intent(out) :: O_Bamp(:) + ! + ! Local Values + !-------------- + real(r8),allocatable :: Csum(:,:) + real(r8),allocatable :: Gcov(:) + integer :: nn,n2,ncols,lchnk,cc + integer :: nlcols, count, astat + + character(len=*), parameter :: subname = 'calc_ZonalMean_2Damps' + + nlcols = get_nlcols_p() + + allocate(Gcov(this%nbas), stat=astat) + call handle_allocate_error(astat, subname, 'Gcov') + allocate(Csum(nlcols, this%nbas), stat=astat) + call handle_allocate_error(astat, subname, 'Csum') + Csum(:,:) = 0._r8 + + ! Compute Covariance with input data and basis functions + !-------------------------------------------------------- + do nn= 1,this%nbas + count = 0 + do lchnk=begchunk,endchunk + ncols = get_ncols_p(lchnk) + do cc = 1,ncols + count=count+1 + Csum(count,nn) = I_Gdata(cc,lchnk)*this%basis(cc,lchnk,nn)*this%area(cc,lchnk) + end do + end do + end do + + call shr_reprosum_calc(Csum, Gcov, count, nlcols, this%nbas, gbl_count=ngcols_p, commid=mpicom) + + ! Multiply by map to get the amplitudes + !------------------------------------------- + do nn=1,this%nbas + O_Bamp(nn) = 0._r8 + do n2=1,this%nbas + O_Bamp(nn) = O_Bamp(nn) + this%map(n2,nn)*Gcov(n2) + end do + end do + + ! End Routine + !------------ + deallocate(Csum) + deallocate(Gcov) + +end subroutine calc_ZonalMean_2Damps +!======================================================================= + + +!======================================================================= +subroutine calc_ZonalMean_3Damps(this,I_Gdata,O_Bamp) + ! + ! calc_ZonalMean_3Damps: Given 3D data values for the ncol,nlev gridpoints, + ! compute the zonal mean basis amplitudes. + !===================================================================== + ! + ! Passed Variables + !------------------ + class(ZonalMean_t) :: this + real(r8),intent(in ):: I_Gdata(:,:,begchunk:) + real(r8),intent(out):: O_Bamp (:,:) + ! + ! Local Values + !-------------- + real(r8),allocatable:: Csum (:,:) + real(r8),allocatable:: Gcov (:) + integer:: nn,n2,ncols,lchnk,cc + integer:: Nsum,ns,ll + integer :: nlcols, count, astat + + integer :: nlev + character(len=*), parameter :: subname = 'calc_ZonalMean_3Damps' + + nlev = size(I_Gdata,dim=2) + + nlcols = get_nlcols_p() + allocate(Gcov(this%nbas), stat=astat) + call handle_allocate_error(astat, subname, 'Gcov') + allocate(Csum(nlcols, this%nbas), stat=astat) + call handle_allocate_error(astat, subname, 'Csum') + + Csum(:,:) = 0._r8 + O_Bamp(:,:) = 0._r8 + + ! Compute Covariance with input data and basis functions + !-------------------------------------------------------- + do ll= 1,nlev + + Csum(:,:) = 0._r8 + Gcov(:) = 0._r8 + + do nn= 1,this%nbas + count = 0 + do lchnk=begchunk,endchunk + ncols = get_ncols_p(lchnk) + do cc = 1,ncols + count=count+1 + Csum(count,nn) = I_Gdata(cc,ll,lchnk)*this%basis(cc,lchnk,nn)*this%area(cc,lchnk) + end do + end do + end do + + call shr_reprosum_calc(Csum, Gcov, count, nlcols, this%nbas, gbl_count=ngcols_p, commid=mpicom) + + ! Multiply by map to get the amplitudes + !------------------------------------------- + do nn=1,this%nbas + O_Bamp(nn,ll) = 0._r8 + do n2=1,this%nbas + O_Bamp(nn,ll) = O_Bamp(nn,ll) + this%map(n2,nn)*Gcov(n2) + end do + end do + + end do + + ! End Routine + !------------ + deallocate(Csum) + deallocate(Gcov) + +end subroutine calc_ZonalMean_3Damps +!======================================================================= + + +!======================================================================= +subroutine eval_ZonalMean_2Dgrid(this,I_Bamp,O_Gdata) + ! + ! eval_ZonalMean_2Dgrid: Given the zonal mean basis amplitudes, + ! compute 2D data values for the ncol gridpoints. + !===================================================================== + ! + ! Passed Variables + !------------------ + class(ZonalMean_t) :: this + real(r8),intent(in ):: I_Bamp (:) + real(r8),intent(out):: O_Gdata(pcols,begchunk:endchunk) + ! + ! Local Values + !-------------- + integer:: nn,ncols,lchnk,cc + + O_Gdata(:,:) = 0._r8 + + ! Construct grid values from basis amplitudes. + !-------------------------------------------------- + + do nn=1,this%nbas + do lchnk=begchunk,endchunk + ncols = get_ncols_p(lchnk) + do cc = 1,ncols + O_Gdata(cc,lchnk) = O_Gdata(cc,lchnk) + (I_Bamp(nn)*this%basis(cc,lchnk,nn)) + end do + end do + end do + +end subroutine eval_ZonalMean_2Dgrid +!======================================================================= + + +!======================================================================= +subroutine eval_ZonalMean_3Dgrid(this,I_Bamp,O_Gdata) + ! + ! eval_ZonalMean_3Dgrid: Given the zonal mean basis amplitudes, + ! compute 3D data values for the ncol,nlev gridpoints. + !===================================================================== + ! + ! Passed Variables + !------------------ + class(ZonalMean_t) :: this + real(r8),intent(in ):: I_Bamp (:,:) + real(r8),intent(out):: O_Gdata(:,:,begchunk:) + ! + ! Local Values + !-------------- + integer:: nn,ncols,lchnk,cc + integer:: ll + + integer :: nlev + nlev = size(O_Gdata,dim=2) + + O_Gdata(:,:,:) = 0._r8 + + ! Construct grid values from basis amplitudes. + !-------------------------------------------------- + + do ll = 1,nlev + do nn=1,this%nbas + do lchnk=begchunk,endchunk + ncols = get_ncols_p(lchnk) + do cc = 1,ncols + O_Gdata(cc,ll,lchnk) = O_Gdata(cc,ll,lchnk) + (I_Bamp(nn,ll)*this%basis(cc,lchnk,nn)) + end do + end do + end do + end do + +end subroutine eval_ZonalMean_3Dgrid +!======================================================================= + +!======================================================================= +subroutine final_ZonalMean(this) + class(ZonalMean_t) :: this + + if(allocated(this%area )) deallocate(this%area) + if(allocated(this%basis)) deallocate(this%basis) + if(allocated(this%map )) deallocate(this%map) + +end subroutine final_ZonalMean +!======================================================================= + +!======================================================================= +subroutine init_ZonalProfile(this,IO_lats,IO_area,I_nlat,I_nbas,GEN_GAUSSLATS) + ! + ! init_ZonalProfile: Initialize the ZonalProfile data structure for the + ! given nlat gridpoints. It is assumed that the domain + ! of these gridpoints of the profile span latitudes + ! from SP to NP. + ! The representation of basis functions functions is + ! normalized w.r.t integration over the sphere so that + ! when configured for tha same number of basis functions, + ! the calculated amplitudes are interchangable with + ! those for the ZonalMean_t class. + ! + ! The optional GEN_GAUSSLATS flag allows for the + ! generation of Gaussian latitudes. The generated grid + ! over-writes the values of IO_lats/IO_area passed by + ! the user. + !===================================================================== + ! + ! Passed Variables + !------------------ + class(ZonalProfile_t) :: this + real(r8) ,intent(inout):: IO_lats(:) + real(r8) ,intent(inout):: IO_area(:) + integer ,intent(in):: I_nlat + integer ,intent(in):: I_nbas + logical,optional,intent(in):: GEN_GAUSSLATS + ! + ! Local Values + !-------------- + real(r8),allocatable:: Clats(:) + real(r8),allocatable:: Bcoef(:) + real(r8),allocatable:: Bcov (:,:) + real(r8):: Bnorm + integer :: ii,nn,n2,nb,ierr, astat + logical :: generate_lats + + character(len=*), parameter :: subname = 'init_ZonalProfile' + + generate_lats = .false. + + if (present(GEN_GAUSSLATS)) then + generate_lats = GEN_GAUSSLATS + end if + + ! Allocate space + !----------------- + if(allocated(this%area )) deallocate(this%area) + if(allocated(this%basis)) deallocate(this%basis) + if(allocated(this%map )) deallocate(this%map) + + this%nlat = I_nlat + this%nbas = I_nbas + allocate(this%area (I_nlat), stat=astat) + call handle_allocate_error(astat, subname, 'this%area') + allocate(this%basis(I_nlat,I_nbas), stat=astat) + call handle_allocate_error(astat, subname, 'this%basis') + allocate(this%map (I_nbas,I_nbas), stat=astat) + call handle_allocate_error(astat, subname, 'this%map') + + allocate(Clats(I_nlat), stat=astat) + call handle_allocate_error(astat, subname, 'Clats') + allocate(Bcoef(I_nbas/2+1), stat=astat) + call handle_allocate_error(astat, subname, 'Bcoef') + allocate(Bcov (I_nbas,I_nbas), stat=astat) + call handle_allocate_error(astat, subname, 'Bcov') + + ! Optionally create the Latitude Gridpoints + ! and their associated area weights. Otherwise + ! they need to be supplied by the user. + !----------------------------------------------- + if(generate_lats) then + + ! Create a Gaussian grid from SP to NP + !-------------------------------------- + call sh_create_gaus_grid(I_nlat,Clats,IO_area,ierr) + if (ierr/=0) then + call endrun('init_ZonalProfile: Error creating Gaussian grid') + end if + + ! Convert generated colatitudes SP->NP to Lats and convert + ! to degrees and scale the area for global 2D integrals + !----------------------------------------------------------- + do nn=1,I_nlat + IO_lats(nn) = (45._r8*Clats(nn)/qrtrPI) - 90._r8 + IO_area(nn) = IO_area(nn)*twoPI + end do + else + ! Convert Latitudes to SP->NP colatitudes in radians + !---------------------------------------------------- + do nn=1,I_nlat + Clats(nn) = (IO_lats(nn) + 90._r8)*qrtrPI/45._r8 + end do + endif + + ! Copy the area weights for each nlat + ! gridpoint to the data structure + !--------------------------------------- + this%area(1:I_nlat) = IO_area(1:I_nlat) + + ! Add first basis for the mean values. + !------------------------------------------ + this%basis(:,1) = invSqrt4pi + Bnorm = 0._r8 + do ii=1,I_nlat + Bnorm = Bnorm + (this%basis(ii,1)*this%basis(ii,1)*this%area(ii)) + end do + this%basis(:,1) = this%basis(:,1)/sqrt(Bnorm) + + ! Loop over the remaining basis functions + !--------------------------------------- + do nn=2,I_nbas + nb = nn-1 + + ! Generate coefs for the basis + !------------------------------ + call sh_gen_basis_coefs(nb,0,Bcoef) + + ! Create an un-normalized basis for the + ! coefs at each nlat gridpoint + !--------------------------------------- + do ii=1,I_nlat + call sh_create_basis(nb,0,Clats(ii),Bcoef,this%basis(ii,nn)) + end do + + ! Numerically normalize the basis funnction + !-------------------------------------------------------------- + Bnorm = 0._r8 + do ii=1,I_nlat + Bnorm = Bnorm + (this%basis(ii,nn)*this%basis(ii,nn)*this%area(ii)) + end do + this%basis(:,nn) = this%basis(:,nn)/sqrt(Bnorm) + + end do ! nn=1,I_nbas + + ! Compute covariance matrix for basis functions + ! (Yes, they are theoretically orthonormal, but lets make sure) + !-------------------------------------------------------------- + do nn=1,I_nbas + do n2=1,I_nbas + Bcov(nn,n2) = 0._r8 + do ii=1,I_nlat + Bcov(nn,n2) = Bcov(nn,n2) + (this%basis(ii,nn)*this%basis(ii,n2)*this%area(ii)) + end do + end do + end do + + ! Invert to get the basis amplitude map + !-------------------------------------- + call Invert_Matrix(Bcov,I_nbas,this%map) + + ! End Routine + !------------ + deallocate(Clats) + deallocate(Bcoef) + deallocate(Bcov ) + +end subroutine init_ZonalProfile +!======================================================================= + + +!======================================================================= +subroutine calc_ZonalProfile_1Damps(this,I_Zdata,O_Bamp) + ! + ! calc_ZonalProfile_1Damps: Given 1D data values for the nlat zonal + ! profiles gridpoints, compute the zonal + ! profile basis amplitudes. + !===================================================================== + ! + ! Passed Variables + !------------------ + class(ZonalProfile_t):: this + real(r8),intent(in ):: I_Zdata(:) + real(r8),intent(out):: O_Bamp (:) + ! + ! Local Values + !-------------- + real(r8),allocatable:: Gcov(:) + integer:: ii,nn,n2, astat + character(len=*), parameter :: subname = 'calc_ZonalProfile_1Damps' + + ! Compute Covariance with input data and basis functions + !-------------------------------------------------------- + allocate(Gcov(this%nbas), stat=astat) + call handle_allocate_error(astat, subname, 'Gcov') + do nn=1,this%nbas + Gcov(nn) = 0._r8 + do ii=1,this%nlat + Gcov(nn) = Gcov(nn) + (I_Zdata(ii)*this%basis(ii,nn)*this%area(ii)) + end do + end do + + ! Multiply by map to get the amplitudes + !------------------------------------------- + do nn=1,this%nbas + O_Bamp(nn) = 0._r8 + do n2=1,this%nbas + O_Bamp(nn) = O_Bamp(nn) + this%map(n2,nn)*Gcov(n2) + end do + end do + + deallocate(Gcov) + +end subroutine calc_ZonalProfile_1Damps +!======================================================================= + + +!======================================================================= +subroutine calc_ZonalProfile_2Damps(this,I_Zdata,O_Bamp) + ! + ! calc_ZonalProfile_2Damps: Given 2D data values for the nlat,nlev zonal + ! profiles gridpoints, compute the zonal + ! profile basis amplitudes. + !===================================================================== + ! + ! Passed Variables + !------------------ + class(ZonalProfile_t):: this + real(r8),intent(in ):: I_Zdata(:,:) + real(r8),intent(out):: O_Bamp (:,:) + ! + ! Local Values + !-------------- + real(r8),allocatable:: Gcov(:,:) + integer:: ii,nn,n2,ilev + + integer :: nlev, astat + character(len=*), parameter :: subname = 'calc_ZonalProfile_2Damps' + + nlev = size(I_Zdata,dim=2) + + ! Compute Covariance with input data and basis functions + !-------------------------------------------------------- + allocate(Gcov(this%nbas,nlev), stat=astat) + call handle_allocate_error(astat, subname, 'Gcov') + do ilev=1,nlev + do nn=1,this%nbas + Gcov(nn,ilev) = 0._r8 + do ii=1,this%nlat + Gcov(nn,ilev) = Gcov(nn,ilev) + (I_Zdata(ii,ilev)*this%basis(ii,nn)*this%area(ii)) + end do + end do + end do + + ! Multiply by map to get the amplitudes + !------------------------------------------- + do ilev=1,nlev + do nn=1,this%nbas + O_Bamp(nn,ilev) = 0._r8 + do n2=1,this%nbas + O_Bamp(nn,ilev) = O_Bamp(nn,ilev) + this%map(n2,nn)*Gcov(n2,ilev) + end do + end do + end do + deallocate(Gcov) + +end subroutine calc_ZonalProfile_2Damps +!======================================================================= + + +!======================================================================= +subroutine eval_ZonalProfile_1Dgrid(this,I_Bamp,O_Zdata) + ! + ! eval_ZonalProfile_1Dgrid: Given the zonal profile basis amplitudes, + ! compute 1D data values for the nlat gridpoints. + !===================================================================== + ! + ! Passed Variables + !------------------ + class(ZonalProfile_t):: this + real(r8),intent(in ):: I_Bamp (:) + real(r8),intent(out):: O_Zdata(:) + ! + ! Local Values + !-------------- + integer:: ii,nn + + ! Construct grid values from basis amplitudes. + !-------------------------------------------------- + O_Zdata(1:this%nlat) = 0._r8 + do nn=1,this%nbas + do ii=1,this%nlat + O_Zdata(ii) = O_Zdata(ii) + (I_Bamp(nn)*this%basis(ii,nn)) + end do + end do + +end subroutine eval_ZonalProfile_1Dgrid +!======================================================================= + + +!======================================================================= +subroutine eval_ZonalProfile_2Dgrid(this,I_Bamp,O_Zdata) + ! + ! eval_ZonalProfile_2Dgrid: Given the zonal profile basis amplitudes, + ! compute 2D data values for the nlat,nlev gridpoints. + !===================================================================== + ! + ! Passed Variables + !------------------ + class(ZonalProfile_t):: this + real(r8),intent(in ):: I_Bamp (:,:) + real(r8),intent(out):: O_Zdata(:,:) + ! + ! Local Values + !-------------- + integer:: ii,nn,ilev + + integer :: nlev + + nlev = size(I_Bamp,dim=2) + + ! Construct grid values from basis amplitudes. + !-------------------------------------------------- + O_Zdata(1:this%nlat,1:nlev) = 0._r8 + do nn=1,this%nbas + do ilev=1,nlev + do ii=1,this%nlat + O_Zdata(ii,ilev) = O_Zdata(ii,ilev) + (I_Bamp(nn,ilev)*this%basis(ii,nn)) + end do + end do + end do + +end subroutine eval_ZonalProfile_2Dgrid +!======================================================================= + +!======================================================================= +subroutine final_ZonalProfile(this) + class(ZonalProfile_t) :: this + + if(allocated(this%area )) deallocate(this%area) + if(allocated(this%basis)) deallocate(this%basis) + if(allocated(this%map )) deallocate(this%map) + +end subroutine final_ZonalProfile +!======================================================================= + +!======================================================================= +subroutine init_ZonalAverage(this,IO_lats,IO_area,I_nlat,GEN_GAUSSLATS) + ! + ! init_ZonalAverage: Initialize the ZonalAverage data structure for the + ! given nlat gridpoints. It is assumed that the domain + ! of these gridpoints of the profile span latitudes + ! from SP to NP. + ! + ! The optional GEN_GAUSSLATS flag allows for the + ! generation of Gaussian latitudes. The generated grid + ! over-writes the values of IO_lats/IO_area passed by + ! the user. + !===================================================================== + ! + ! Passed Variables + !------------------ + class(ZonalAverage_t) :: this + real(r8) ,intent(inout):: IO_lats(:) + real(r8) ,intent(inout):: IO_area(:) + integer ,intent(in):: I_nlat + logical,optional,intent(in):: GEN_GAUSSLATS + ! + ! Local Values + !-------------- + real(r8),allocatable:: Clats (:) + real(r8),allocatable:: Glats (:,:) + real(r8),allocatable:: BinLat(:) + real(r8),allocatable:: Asum (:,:) + real(r8),allocatable:: Anorm (:) + real(r8):: area(pcols),rlat + integer :: nn,jj,ierr, astat + integer :: ncols,lchnk,cc,jlat + integer :: nlcols, count + logical :: generate_lats + character(len=*), parameter :: subname = 'init_ZonalAverage' + + generate_lats = .false. + + if (present(GEN_GAUSSLATS)) then + generate_lats = GEN_GAUSSLATS + end if + + nlcols = get_nlcols_p() + + ! Allocate space + !----------------- + if(allocated(this%area )) deallocate(this%area) + if(allocated(this%a_norm )) deallocate(this%a_norm) + if(allocated(this%area_g )) deallocate(this%area_g) + if(allocated(this%idx_map)) deallocate(this%idx_map) + + this%nlat = I_nlat + allocate(this%area (I_nlat), stat=astat) + call handle_allocate_error(astat, subname, 'this%area') + allocate(this%a_norm (I_nlat), stat=astat) + call handle_allocate_error(astat, subname, 'this%a_norm') + allocate(this%area_g (pcols,begchunk:endchunk), stat=astat) + call handle_allocate_error(astat, subname, 'this%area_g') + allocate(this%idx_map(pcols,begchunk:endchunk), stat=astat) + call handle_allocate_error(astat, subname, 'this%idx_map') + + allocate(Clats (I_nlat), stat=astat) + call handle_allocate_error(astat, subname, 'Clats') + allocate(BinLat(I_nlat+1), stat=astat) + call handle_allocate_error(astat, subname, 'BinLat') + allocate(Glats (pcols,begchunk:endchunk), stat=astat) + call handle_allocate_error(astat, subname, 'Glats') + allocate(Asum (nlcols,I_nlat), stat=astat) + call handle_allocate_error(astat, subname, 'Asum') + allocate(Anorm (I_nlat), stat=astat) + call handle_allocate_error(astat, subname, 'Anorm') + + ! Optionally create the Latitude Gridpoints + ! and their associated area weights. Otherwise + ! they need to be supplied by the user. + !----------------------------------------------- + if(generate_lats) then + + ! Create a Gaussin grid from SP to NP + !-------------------------------------- + call sh_create_gaus_grid(this%nlat,Clats,IO_area,ierr) + if (ierr/=0) then + call endrun('init_ZonalAverage: Error creating Gaussian grid') + end if + + ! Convert generated colatitudes SP->NP to Lats and convert + ! to degrees and scale the area for global 2D integrals + !----------------------------------------------------------- + do nn=1,this%nlat + IO_lats(nn) = (45._r8*Clats(nn)/qrtrPI) - 90._r8 + IO_area(nn) = IO_area(nn)*twoPI + end do + else + ! Convert Latitudes to SP->NP colatitudes in radians + !---------------------------------------------------- + do nn=1,this%nlat + Clats(nn) = (IO_lats(nn) + 90._r8)*qrtrPI/45._r8 + end do + endif + + ! Copy the Lat grid area weights to the data structure + !----------------------------------------------------- + this%area(1:this%nlat) = IO_area(1:this%nlat) + + ! Save a copy of the area weights for each 2D gridpoint + ! and convert Latitudes to SP->NP colatitudes in radians + !------------------------------------------------------- + do lchnk=begchunk,endchunk + ncols = get_ncols_p(lchnk) + call get_wght_all_p(lchnk, ncols, area) + do cc = 1,ncols + rlat=get_rlat_p(lchnk,cc) + this%area_g(cc,lchnk) = area(cc) + Glats (cc,lchnk) = rlat + halfPI + end do + end do + + ! Set boundaries for Latitude bins + !----------------------------------- + BinLat(1) = 0._r8 + BinLat(this%nlat+1) = pi + do nn=2,this%nlat + BinLat(nn) = (Clats(nn-1)+Clats(nn))/2._r8 + end do + + ! Loop over 2D gridpoints and determine its lat bin index + !--------------------------------------------------------- + do lchnk=begchunk,endchunk + ncols = get_ncols_p(lchnk) + do cc = 1,ncols + jlat = -1 + if((Glats(cc,lchnk)<=BinLat(2)).and. & + (Glats(cc,lchnk)>=BinLat(1)) ) then + jlat = 1 + elseif((Glats(cc,lchnk)>=BinLat(this%nlat) ).and. & + (Glats(cc,lchnk)<=BinLat(this%nlat+1)) ) then + jlat = this%nlat + else + do jj=2,(this%nlat-1) + if((Glats(cc,lchnk)>BinLat(jj )).and. & + (Glats(cc,lchnk)<=BinLat(jj+1)) ) then + jlat = jj + exit + endif + end do + endif + if (jlat<1) then + call endrun('ZonalAverage init ERROR: jlat not in range') + endif + this%idx_map(cc,lchnk) = jlat + end do + end do + + ! Initialize 2D Area sums for each bin + !-------------------------------------- + Asum(:,:) = 0._r8 + Anorm(:) = 0._r8 + count = 0 + do lchnk=begchunk,endchunk + ncols = get_ncols_p(lchnk) + do cc = 1,ncols + jlat = this%idx_map(cc,lchnk) + count=count+1 + Asum(count,jlat) = this%area_g(cc,lchnk) + end do + end do + + call shr_reprosum_calc(Asum, Anorm, count, nlcols, I_nlat, gbl_count=ngcols_p, commid=mpicom) + + this%a_norm = Anorm + + if (.not.all(Anorm(:)>0._r8)) then + write(iulog,*) 'init_ZonalAverage -- ERROR in Anorm values: ' + do jlat = 1,I_nlat + if (.not.Anorm(jlat)>0._r8) then + write(iulog,*) ' Anorm(',jlat,'): ', Anorm(jlat) + endif + end do + call endrun('init_ZonalAverage -- ERROR in Anorm values') + end if + + ! End Routine + !------------ + deallocate(Clats) + deallocate(BinLat) + deallocate(Glats) + deallocate(Asum) + deallocate(Anorm) + +end subroutine init_ZonalAverage +!======================================================================= + + +!======================================================================= +subroutine calc_ZonalAverage_2DbinAvg(this,I_Gdata,O_Zdata) + ! + ! calc_ZonalAverage_2DbinAvg: Given 2D data values for ncol gridpoints, + ! compute the nlat area weighted binAvg profile + !===================================================================== + ! + ! Passed Variables + !------------------ + class(ZonalAverage_t):: this + real(r8),intent(in ):: I_Gdata(pcols,begchunk:endchunk) + real(r8),intent(out):: O_Zdata(:) + ! + ! Local Values + !-------------- + real(r8),allocatable:: Asum (:,:) + integer:: nn,ncols,lchnk,cc,jlat + integer :: nlcols, count, astat + character(len=*), parameter :: subname = 'calc_ZonalAverage_2DbinAvg' + + nlcols = get_nlcols_p() + + + ! Initialize Zonal profile + !--------------------------- + allocate(Asum(nlcols,this%nlat), stat=astat) + call handle_allocate_error(astat, subname, 'Asum') + Asum(:,:) = 0._r8 + + O_Zdata(1:this%nlat) = 0._r8 + + ! Compute area-weighted sums + !----------------------------- + count = 0 + do lchnk=begchunk,endchunk + ncols = get_ncols_p(lchnk) + do cc = 1,ncols + jlat = this%idx_map(cc,lchnk) + count=count+1 + Asum(count,jlat) = I_Gdata(cc,lchnk)*this%area_g(cc,lchnk) + end do + end do + + call shr_reprosum_calc(Asum,O_Zdata,count, nlcols, this%nlat,gbl_count=ngcols_p, commid=mpicom) + + ! Divide by area norm to get the averages + !----------------------------------------- + do nn=1,this%nlat + O_Zdata(nn) = O_Zdata(nn)/this%a_norm(nn) + end do + + deallocate(Asum) + +end subroutine calc_ZonalAverage_2DbinAvg +!======================================================================= + + +!======================================================================= +subroutine calc_ZonalAverage_3DbinAvg(this,I_Gdata,O_Zdata) + ! + ! calc_ZonalAverage_3DbinAvg: Given 3D data values for ncol,nlev gridpoints, + ! compute the nlat,nlev area weighted binAvg profile + !===================================================================== + ! + ! Passed Variables + !------------------ + class(ZonalAverage_t):: this + real(r8),intent(in ):: I_Gdata(:,:,begchunk:) + real(r8),intent(out):: O_Zdata(:,:) + ! + ! Local Values + !-------------- + real(r8),allocatable:: Gsum(:) + real(r8),allocatable:: Asum(:,:) + integer:: nn,ncols,lchnk,cc,jlat + integer:: Nsum,ilev,ns + + integer :: nlev + integer :: nlcols, count, astat + character(len=*), parameter :: subname = 'calc_ZonalAverage_3DbinAvg' + + nlev = size(I_Gdata,dim=2) + nlcols = get_nlcols_p() + + ! Initialize Zonal profile + !--------------------------- + Nsum = this%nlat*nlev + allocate(Gsum(Nsum), stat=astat) + call handle_allocate_error(astat, subname, 'Gsum') + allocate(Asum(nlcols,Nsum), stat=astat) + call handle_allocate_error(astat, subname, 'Asum') + Asum(:,:) = 0._r8 + + O_Zdata(1:this%nlat,1:nlev) = 0._r8 + + ! Compute area-weighted sums + !----------------------------- + do ilev = 1,nlev + count = 0 + do lchnk=begchunk,endchunk + ncols = get_ncols_p(lchnk) + do cc = 1,ncols + jlat = this%idx_map(cc,lchnk) + ns = jlat + (ilev-1)*this%nlat + count=count+1 + Asum(count,ns) = I_Gdata(cc,ilev,lchnk)*this%area_g(cc,lchnk) + end do + end do + end do + + call shr_reprosum_calc(Asum,Gsum, count, nlcols, Nsum, gbl_count=ngcols_p, commid=mpicom) + + ! Divide by area norm to get the averages + !----------------------------------------- + do ilev = 1,nlev + do nn = 1,this%nlat + ns = nn + (ilev-1)*this%nlat + O_Zdata(nn,ilev) = Gsum(ns)/this%a_norm(nn) + end do + end do + + deallocate(Gsum) + deallocate(Asum) + +end subroutine calc_ZonalAverage_3DbinAvg +!======================================================================= + +!======================================================================= +subroutine final_ZonalAverage(this) + class(ZonalAverage_t) :: this + + if(allocated(this%area )) deallocate(this%area) + if(allocated(this%a_norm )) deallocate(this%a_norm) + if(allocated(this%area_g )) deallocate(this%area_g) + if(allocated(this%idx_map)) deallocate(this%idx_map) + +end subroutine final_ZonalAverage +!======================================================================= + + +!======================================================================= +subroutine Invert_Matrix(I_Mat,Nbas,O_InvMat) + ! + ! Invert_Matrix: Given the NbasxNbas matrix, calculate and return + ! the inverse of the matrix. + ! + ! Implemented with the LAPACK DGESV routine. + ! + !==================================================================== + ! + ! Passed Variables + !------------------ + real(r8), intent(inout) :: I_Mat(:,:) ! input matrix contains P*L*U + ! decomposition on output + integer, intent(in) :: Nbas + real(r8), intent(out) :: O_InvMat(:,:) + ! + ! Local Values + !------------- + integer, allocatable :: Indx(:) ! pivot indices + integer :: astat, ii + character(len=*), parameter :: subname = 'Invert_Matrix' + character(len=80) :: msg + + external DGESV + + ! Allocate work space + !--------------------- + allocate(Indx(Nbas), stat=astat) + call handle_allocate_error(astat, subname, 'Indx') + + ! Initialize the inverse array with the identity matrix + !------------------------------------------------------- + O_InvMat(:,:) = 0._r8 + do ii=1,Nbas + O_InvMat(ii,ii) = 1._r8 + end do + + call DGESV(Nbas, Nbas, I_Mat, Nbas, Indx, O_InvMat, Nbas, astat) + + if (astat < 0) then + write(msg, '(a, i1, a)') 'argument # ', abs(astat), ' has an illegal value' + call endrun(subname//': DGESV error return: '//msg) + else if (astat > 0) then + call endrun(subname//': DGESV error return: matrix is singular') + end if + + deallocate(Indx) + +end subroutine Invert_Matrix +!======================================================================= + +!======================================================================= +! legacy spherepack routines +!======================================================================= +subroutine sh_gen_basis_coefs(nn,mm,cp) + ! + ! spherepack alfk + ! + ! dimension of real cp(nn/2 + 1) + ! arguments + ! + ! purpose computes fourier coefficients in the trigonometric series + ! representation of the normalized associated + ! legendre function pbar(nn,mm,theta) for use by + ! sh_gen_basis_coefs in calculating pbar(nn,mm,theta). + ! + ! first define the normalized associated + ! legendre functions + ! + ! pbar(mm,nn,theta) = sqrt((2*nn+1)*factorial(nn-mm) + ! /(2*factorial(nn+mm)))*sin(theta)**mm/(2**nn* + ! factorial(nn)) times the (nn+mm)th derivative of + ! (x**2-1)**nn with respect to x=cos(theta) + ! + ! where theta is colatitude. + ! + ! then subroutine sh_gen_basis_coefs computes the coefficients + ! cp(k) in the following trigonometric + ! expansion of pbar(m,n,theta). + ! + ! 1) for n even and m even, pbar(mm,nn,theta) = + ! .5*cp(1) plus the sum from k=1 to k=nn/2 + ! of cp(k+1)*cos(2*k*th) + ! + ! 2) for nn even and mm odd, pbar(mm,nn,theta) = + ! the sum from k=1 to k=nn/2 of + ! cp(k)*sin(2*k*th) + ! + ! 3) for n odd and m even, pbar(mm,nn,theta) = + ! the sum from k=1 to k=(nn+1)/2 of + ! cp(k)*cos((2*k-1)*th) + ! + ! 4) for nn odd and mm odd, pbar(mm,nn,theta) = + ! the sum from k=1 to k=(nn+1)/2 of + ! cp(k)*sin((2*k-1)*th) + ! + ! arguments + ! + ! on input nn + ! nonnegative integer specifying the degree of + ! pbar(nn,mm,theta) + ! + ! mm + ! is the order of pbar(nn,mm,theta). mm can be + ! any integer however cp is computed such that + ! pbar(nn,mm,theta) = 0 if abs(m) is greater + ! than nn and pbar(nn,mm,theta) = (-1)**mm* + ! pbar(nn,-mm,theta) for negative mm. + ! + ! on output cp + ! array of length (nn/2)+1 + ! which contains the fourier coefficients in + ! the trigonometric series representation of + ! pbar(nn,mm,theta) + ! + ! special conditions none + ! + ! algorithm the highest order coefficient is determined in + ! closed form and the remainig coefficients are + ! determined as the solution of a backward + ! recurrence relation. + ! + !===================================================================== + ! + ! Passed Variables + !------------------ + integer ,intent(in ):: nn + integer ,intent(in ):: mm + real(r8),intent(out):: cp(nn/2+1) + ! + ! Local Values + !---------------- + real(r8):: fnum,fnmh + real(r8):: pm1 + real(r8):: t1,t2 + real(r8):: fden + real(r8):: cp2 + real(r8):: fnnp1 + real(r8):: fnmsq + real(r8):: fk + real(r8):: a1,b1,C1 + integer :: ma,nmms2,nex + integer :: ii,jj + + real(r8),parameter:: SC10=1024._r8 + real(r8),parameter:: SC20=SC10*SC10 + real(r8),parameter:: SC40=SC20*SC20 + + cp(1) = 0._r8 + ma = abs(mm) + if(ma>nn) return + + if((nn-1)<0) then + cp(1) = sqrt(2._r8) + return + elseif((nn-1)==0) then + if(ma/=0) then + cp(1) = sqrt(.75_r8) + if(mm==-1) cp(1) = -cp(1) + else + cp(1) = sqrt(1.5_r8) + endif + return + else + if(mod(nn+ma,2)/=0) then + nmms2 = (nn-ma-1)/2 + fnum = nn + ma + 2 + fnmh = nn - ma + 2 + pm1 = -1._r8 + else + nmms2 = (nn-ma)/2 + fnum = nn + ma + 1 + fnmh = nn - ma + 1 + pm1 = 1._r8 + endif + endif + + t1 = 1._r8/SC20 + nex = 20 + fden = 2._r8 + if(nmms2>=1) then + do ii = 1,nmms2 + t1 = fnum*t1/fden + if (t1>SC20) then + t1 = t1/SC40 + nex = nex + 40 + endif + fnum = fnum + 2._r8 + fden = fden + 2._r8 + end do + endif + + if(mod(ma/2,2)/=0) then + t1 = -t1/2._r8**(nn-1-nex) + else + t1 = t1/2._r8**(nn-1-nex) + endif + t2 = 1._r8 + if(ma/=0) then + do ii = 1,ma + t2 = fnmh*t2/ (fnmh+pm1) + fnmh = fnmh + 2._r8 + end do + endif + + cp2 = t1*sqrt((nn+.5_r8)*t2) + fnnp1 = nn*(nn+1) + fnmsq = fnnp1 - 2._r8*ma*ma + + if((mod(nn,2)==0).and.(mod(ma,2)==0)) then + jj = 1+(nn+1)/2 + else + jj = (nn+1)/2 + endif + + cp(jj) = cp2 + if(mm<0) then + if(mod(ma,2)/=0) cp(jj) = -cp(jj) + endif + if(jj<=1) return + + fk = nn + a1 = (fk-2._r8)*(fk-1._r8) - fnnp1 + b1 = 2._r8* (fk*fk-fnmsq) + cp(jj-1) = b1*cp(jj)/a1 + + jj = jj - 1 + do while(jj>1) + fk = fk - 2._r8 + a1 = (fk-2._r8)*(fk-1._r8) - fnnp1 + b1 = -2._r8*(fk*fk-fnmsq) + c1 = (fk+1._r8)*(fk+2._r8) - fnnp1 + cp(jj-1) = -(b1*cp(jj)+c1*cp(jj+1))/a1 + jj = jj - 1 + end do + +end subroutine sh_gen_basis_coefs +!======================================================================= + +!======================================================================= +subroutine sh_create_basis(nn,mm,theta,cp,pb) + ! + ! spherepack lfpt + ! + ! dimension of + ! arguments + ! cp((nn/2)+1) + ! + ! purpose routine sh_create_basis uses coefficients computed by + ! routine sh_gen_basis_coefs to compute the + ! normalized associated legendre function pbar(nn,mm,theta) + ! at colatitude theta. + ! + ! arguments + ! + ! on input nn + ! nonnegative integer specifying the degree of + ! pbar(nn,mm,theta) + ! mm + ! is the order of pbar(nn,mm,theta). mm can be + ! any integer however pbar(nn,mm,theta) = 0 + ! if abs(mm) is greater than nn and + ! pbar(nn,mm,theta) = (-1)**mm*pbar(nn,-mm,theta) + ! for negative mm. + ! + ! theta + ! colatitude in radians + ! + ! cp + ! array of length (nn/2)+1 + ! containing coefficients computed by routine + ! sh_gen_basis_coefs + ! + ! on output pb + ! variable containing pbar(n,m,theta) + ! + ! special conditions calls to routine sh_create_basis must be preceded by an + ! appropriate call to routine sh_gen_basis_coefs. + ! + ! algorithm the trigonometric series formula used by + ! routine sh_create_basis to calculate pbar(nn,mm,theta) at + ! colatitude theta depends on mm and nn as follows: + ! + ! 1) for nn even and mm even, the formula is + ! .5*cp(1) plus the sum from k=1 to k=n/2 + ! of cp(k)*cos(2*k*theta) + ! 2) for nn even and mm odd. the formula is + ! the sum from k=1 to k=nn/2 of + ! cp(k)*sin(2*k*theta) + ! 3) for nn odd and mm even, the formula is + ! the sum from k=1 to k=(nn+1)/2 of + ! cp(k)*cos((2*k-1)*theta) + ! 4) for nn odd and mm odd, the formula is + ! the sum from k=1 to k=(nn+1)/2 of + ! cp(k)*sin((2*k-1)*theta) + ! + !===================================================================== + integer, intent(in) :: nn,mm + real(r8), intent(in) :: theta + real(r8), intent(in) :: cp(:) + real(r8), intent(out) :: pb + + real(r8) :: cdt + real(r8) :: sdt + real(r8) :: ct + real(r8) :: st + real(r8) :: summ + real(r8) :: cth + + integer:: ma,nmod,mmod,kdo + integer:: kp1,kk + + pb = 0._r8 + ma = abs(mm) + if(ma>nn) return + + if(nn<=0) then + if(ma<=0) then + pb = sqrt(.5_r8) + return + endif + endif + + nmod = mod(nn,2) + mmod = mod(ma,2) + + if(nmod<=0) then + if(mmod<=0) then + kdo = nn/2 + 1 + cdt = cos(theta+theta) + sdt = sin(theta+theta) + ct = 1._r8 + st = 0._r8 + summ = .5_r8*cp(1) + do kp1 = 2,kdo + cth = cdt*ct - sdt*st + st = sdt*ct + cdt*st + ct = cth + summ = summ + cp(kp1)*ct + end do + pb = summ + return + endif + kdo = nn/2 + cdt = cos(theta+theta) + sdt = sin(theta+theta) + ct = 1._r8 + st = 0._r8 + summ = 0._r8 + do kk = 1,kdo + cth = cdt*ct - sdt*st + st = sdt*ct + cdt*st + ct = cth + summ = summ + cp(kk)*st + end do + pb = summ + return + endif + + kdo = (nn+1)/2 + if(mmod<=0) then + cdt = cos(theta+theta) + sdt = sin(theta+theta) + ct = cos(theta) + st = -sin(theta) + summ = 0._r8 + do kk = 1,kdo + cth = cdt*ct - sdt*st + st = sdt*ct + cdt*st + ct = cth + summ = summ + cp(kk)*ct + end do + pb = summ + return + endif + + cdt = cos(theta+theta) + sdt = sin(theta+theta) + ct = cos(theta) + st = -sin(theta) + summ = 0._r8 + do kk = 1,kdo + cth = cdt*ct - sdt*st + st = sdt*ct + cdt*st + ct = cth + summ = summ + cp(kk)*st + end do + pb = summ + +end subroutine sh_create_basis +!======================================================================= + +!======================================================================= +subroutine sh_create_gaus_grid(nlat,theta,wts,ierr) + ! + ! spherepack gaqd + ! . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + ! . . + ! . copyright (c) 2001 by ucar . + ! . . + ! . university corporation for atmospheric research . + ! . . + ! . all rights reserved . + ! . . + ! . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + ! + ! February 2002 + ! + ! gauss points and weights are computed using the fourier-newton + ! described in "on computing the points and weights for + ! gauss-legendre quadrature", paul n. swarztrauber, siam journal + ! on scientific computing (DOI 10.1137/S1064827500379690). + ! This routine is faster and more accurate than older program + ! with the same name. + ! + ! computes the nlat gaussian colatitudes and weights + ! in double precision. the colatitudes are in radians and lie in the + ! in the interval (0,pi). + ! + ! input parameters + ! + ! nlat the number of gaussian colatitudes in the interval (0,pi) + ! (between the two poles). nlat must be greater than zero. + ! + ! output parameters + ! + ! theta a double precision array with length nlat + ! containing the gaussian colatitudes in + ! increasing radians on the interval (0,pi). + ! + ! wts a double precision array with lenght nlat + ! containing the gaussian weights. + ! + ! ierror = 0 no errors + ! = 1 if nlat<=0 + ! + !=================================================================== + ! + ! Passed variables + !----------------- + integer ,intent(in ) :: nlat + real(r8),intent(out) :: theta(nlat) + real(r8),intent(out) :: wts(nlat) + integer ,intent(out) :: ierr + ! + ! Local Values + !------------- + real(r8):: sgnd + real(r8):: xx,dtheta,dthalf + real(r8):: cmax,zprev,zlast,zero,zhold,pb,dpb,dcor,summ,cz + integer :: mnlat,ns2,nhalf,nix,it,ii + + real(r8), parameter :: eps = epsilon(1._r8) + + ! check work space length + !------------------------ + if(nlat<=0) then + ierr = 1 + return + endif + ierr = 0 + + ! compute weights and points analytically when nlat=1,2 + !------------------------------------------------------- + if(nlat==1) then + theta(1) = acos(0._r8) + wts (1) = 2._r8 + return + elseif(nlat==2) then + xx = sqrt(1._r8/3._r8) + theta(1) = acos( xx) + theta(2) = acos(-xx) + wts (1) = 1._r8 + wts (2) = 1._r8 + return + endif + + ! Proceed for nlat > 2 + !---------------------- + mnlat = mod(nlat,2) + ns2 = nlat/2 + nhalf = (nlat+1)/2 + + call sh_fourier_coefs_dp(nlat,cz,theta(ns2+1),wts(ns2+1)) + + dtheta = halfPI/nhalf + dthalf = dtheta/2._r8 + cmax = .2_r8*dtheta + + ! estimate first point next to theta = pi/2 + !------------------------------------------- + if(mnlat/=0) then + zero = halfPI - dtheta + zprev = halfPI + nix = nhalf - 1 + else + zero = halfPI - dthalf + nix = nhalf + endif + + do while(nix/=0) + dcor = huge(1._r8) + it = 0 + do while (abs(dcor) > eps*abs(zero)) + it = it + 1 + ! newton iterations + !----------------------- + call sh_legp_dlegp_theta(nlat,zero,cz,theta(ns2+1),wts(ns2+1),pb,dpb) + dcor = pb/dpb + if(dcor.ne.0._r8) then + sgnd = dcor/abs(dcor) + else + sgnd = 1._r8 + endif + dcor = sgnd*min(abs(dcor),cmax) + zero = zero - dcor + end do + + theta(nix) = zero + zhold = zero + + ! wts(nix) = (nlat+nlat+1)/(dpb*dpb) + ! yakimiw's formula permits using old pb and dpb + !-------------------------------------------------- + wts(nix) = (nlat+nlat+1)/ (dpb+pb*dcos(zlast)/dsin(zlast))**2 + nix = nix - 1 + if(nix==nhalf-1) zero = 3._r8*zero - pi + if(nix0) then + cth = cdt + sth = sdt + do kk = 1,kdo + pb = pb + cp(kk)*cth + dpb = dpb - dcp(kk)*sth + chh = cdt*cth - sdt*sth + sth = sdt*cth + cdt*sth + cth = chh + end do + endif + else + ! n odd + !----------- + kdo = (nn+1)/2 + pb = 0._r8 + dpb = 0._r8 + cth = dcos(theta) + sth = dsin(theta) + do kk = 1,kdo + pb = pb + cp(kk)*cth + dpb = dpb - dcp(kk)*sth + chh = cdt*cth - sdt*sth + sth = sdt*cth + cdt*sth + cth = chh + end do + endif + +end subroutine sh_legp_dlegp_theta +!======================================================================= + +end module zonal_mean_mod diff --git a/components/eam/src/physics/crm/crm_physics.F90 b/components/eam/src/physics/crm/crm_physics.F90 index cc65d559bb88..61ef03f8ca65 100644 --- a/components/eam/src/physics/crm/crm_physics.F90 +++ b/components/eam/src/physics/crm/crm_physics.F90 @@ -1302,14 +1302,23 @@ subroutine crm_physics_tend(ztodt, state, tend, ptend, pbuf2d, cam_in, cam_out, call pam_mirror_array_readonly( 'input_tau00', crm_input%tau00 ) ! call pam_mirror_array_readonly( 'input_ul_esmt', crm_input%ul_esmt ) ! call pam_mirror_array_readonly( 'input_vl_esmt', crm_input%vl_esmt ) - ! call pam_mirror_array_readonly( 'input_t_vt', crm_input%t_vt ) - ! call pam_mirror_array_readonly( 'input_q_vt', crm_input%q_vt ) - ! call pam_mirror_array_readonly( 'input_u_vt', crm_input%u_vt ) - call pam_mirror_array_readonly( 'input_nccn_prescribed',crm_input%nccn_prescribed ) call pam_mirror_array_readonly( 'input_nc_nuceat_tend', crm_input%nc_nuceat_tend ) call pam_mirror_array_readonly( 'input_ni_activated', crm_input%ni_activated ) + ! Variance transport inputs and outputs + if (use_MMF_VT) then + call pam_mirror_array_readonly( 'input_vt_t', crm_input%t_vt ) + call pam_mirror_array_readonly( 'input_vt_q', crm_input%q_vt ) + call pam_mirror_array_readonly( 'input_vt_u', crm_input%u_vt ) + call pam_mirror_array_readwrite( 'output_t_vt_tend', crm_output%t_vt_tend ) + call pam_mirror_array_readwrite( 'output_q_vt_tend', crm_output%q_vt_tend ) + call pam_mirror_array_readwrite( 'output_u_vt_tend', crm_output%u_vt_tend ) + call pam_mirror_array_readwrite( 'output_t_vt_ls', crm_output%t_vt_ls ) + call pam_mirror_array_readwrite( 'output_q_vt_ls', crm_output%q_vt_ls ) + call pam_mirror_array_readwrite( 'output_u_vt_ls', crm_output%u_vt_ls ) + end if + call pam_mirror_array_readwrite( 'state_u_wind', crm_state%u_wind ) call pam_mirror_array_readwrite( 'state_v_wind', crm_state%v_wind ) call pam_mirror_array_readwrite( 'state_w_wind', crm_state%w_wind ) @@ -1389,12 +1398,6 @@ subroutine crm_physics_tend(ztodt, state, tend, ptend, pbuf2d, cam_in, cam_out, call pam_mirror_array_readwrite( 'output_qltend', crm_output%qltend, '' ) call pam_mirror_array_readwrite( 'output_qcltend', crm_output%qcltend, '' ) call pam_mirror_array_readwrite( 'output_qiltend', crm_output%qiltend, '' ) - ! call pam_mirror_array_readwrite( 'output_t_vt_tend', crm_output%t_vt_tend, '' ) - ! call pam_mirror_array_readwrite( 'output_q_vt_tend', crm_output%q_vt_tend, '' ) - ! call pam_mirror_array_readwrite( 'output_u_vt_tend', crm_output%u_vt_tend, '' ) - ! call pam_mirror_array_readwrite( 'output_t_vt_ls', crm_output%t_vt_ls, '' ) - ! call pam_mirror_array_readwrite( 'output_q_vt_ls', crm_output%q_vt_ls, '' ) - ! call pam_mirror_array_readwrite( 'output_u_vt_ls', crm_output%u_vt_ls, '' ) call pam_mirror_array_readwrite( 'output_ultend', crm_output%ultend, '' ) call pam_mirror_array_readwrite( 'output_vltend', crm_output%vltend, '' ) ! call pam_mirror_array_readwrite( 'output_tk', crm_output%tk, '' ) @@ -1450,7 +1453,6 @@ subroutine crm_physics_tend(ztodt, state, tend, ptend, pbuf2d, cam_in, cam_out, call pam_register_dimension('gcm_lev',pver) call pam_set_option('use_MMF_VT', use_MMF_VT_tmp ) - call pam_set_option('MMF_VT_wn_max', MMF_VT_wn_max ) call pam_set_option('use_MMF_ESMT', use_MMF_ESMT_tmp ) call pam_set_option('use_crm_accel', use_crm_accel_tmp ) diff --git a/components/eam/src/physics/crm/pam/pam_driver.cpp b/components/eam/src/physics/crm/pam/pam_driver.cpp index f131bbd44f09..bfba64696163 100644 --- a/components/eam/src/physics/crm/pam/pam_driver.cpp +++ b/components/eam/src/physics/crm/pam/pam_driver.cpp @@ -13,6 +13,7 @@ #include "pam_statistics.h" #include "pam_output.h" #include "pam_accelerate.h" +#include "pam_variance_transport.h" #include "sponge_layer.h" #include "surface_friction.h" #include "scream_cxx_interface_finalize.h" @@ -46,6 +47,7 @@ extern "C" void pam_driver() { auto is_first_step = coupler.get_option("is_first_step"); auto is_restart = coupler.get_option("is_restart"); bool use_crm_accel = coupler.get_option("use_crm_accel"); + bool use_MMF_VT = coupler.get_option("use_MMF_VT"); bool enable_physics_tend_stats = coupler.get_option("enable_physics_tend_stats"); //------------------------------------------------------------------------------------------------ // set various coupler options @@ -111,6 +113,11 @@ extern "C" void pam_driver() { // initialize variables for CRM mean-state acceleration if (use_crm_accel) { pam_accelerate_init(coupler); } + if (use_MMF_VT) { + pam_variance_transport_init(coupler); + pam_variance_transport_compute_forcing(coupler); + } + // initilize surface "psuedo-friction" (psuedo => doesn't match "real" GCM friction) auto input_tau = dm_host.get("input_tau00").createDeviceCopy(); auto input_bflx = dm_host.get("input_bflxls").createDeviceCopy(); @@ -166,7 +173,8 @@ extern "C" void pam_driver() { if (enable_check_state) { pam_debug_check_state(coupler, 1, nstep); } - // run a PAM time step + // Apply forcing tendencies + if (use_MMF_VT) { pam_variance_transport_apply_forcing(coupler); } coupler.run_module( "apply_gcm_forcing_tendencies" , modules::apply_gcm_forcing_tendencies ); coupler.run_module( "radiation" , [&] (pam::PamCoupler &coupler) {rad .timeStep(coupler);} ); if (enable_check_state) { pam_debug_check_state(coupler, 2, nstep); } @@ -237,6 +245,11 @@ extern "C" void pam_driver() { pam_statistics_compute_means(coupler); pam_statistics_copy_to_host(coupler); + if (use_MMF_VT) { + pam_variance_transport_compute_feedback(coupler); + pam_variance_transport_copy_to_host(coupler); + } + //------------------------------------------------------------------------------------------------ // Finalize and clean up micro .finalize(coupler); diff --git a/components/eam/src/physics/crm/pam/pam_variance_transport.h b/components/eam/src/physics/crm/pam/pam_variance_transport.h new file mode 100644 index 000000000000..b7067631203f --- /dev/null +++ b/components/eam/src/physics/crm/pam/pam_variance_transport.h @@ -0,0 +1,267 @@ +#pragma once + +#include "pam_coupler.h" + +inline void pam_variance_transport_init( pam::PamCoupler &coupler ) { + using yakl::c::parallel_for; + using yakl::c::SimpleBounds; + using yakl::atomicAdd; + auto &dm_device = coupler.get_data_manager_device_readwrite(); + auto nens = coupler.get_option("ncrms"); + auto nz = coupler.get_option("crm_nz"); + auto ny = coupler.get_option("crm_ny"); + auto nx = coupler.get_option("crm_nx"); + //------------------------------------------------------------------------------------------------ + dm_device.register_and_allocate("vt_temp", "temperature variance", {nz,nens}, {"z","nens"} ); + dm_device.register_and_allocate("vt_rhov", "water vapor variance", {nz,nens}, {"z","nens"} ); + dm_device.register_and_allocate("vt_uvel", "u momentum variance", {nz,nens}, {"z","nens"} ); + dm_device.register_and_allocate("vt_temp_pert", "temperature perturbation from horz mean", {nz,ny,nx,nens}, {"z","y","x","nens"} ); + dm_device.register_and_allocate("vt_rhov_pert", "water vapor perturbation from horz mean", {nz,ny,nx,nens}, {"z","y","x","nens"} ); + dm_device.register_and_allocate("vt_uvel_pert", "u momentum perturbation from horz mean", {nz,ny,nx,nens}, {"z","y","x","nens"} ); + dm_device.register_and_allocate("vt_temp_forcing_tend", "temperature variance forcing tendency", {nz,nens}, {"z","nens"} ); + dm_device.register_and_allocate("vt_rhov_forcing_tend", "water vapor variance forcing tendency", {nz,nens}, {"z","nens"} ); + dm_device.register_and_allocate("vt_uvel_forcing_tend", "u momentum variance forcing tendency", {nz,nens}, {"z","nens"} ); + //------------------------------------------------------------------------------------------------ +} + +inline void pam_variance_transport_diagnose( pam::PamCoupler &coupler ) { + using yakl::c::parallel_for; + using yakl::c::SimpleBounds; + using yakl::atomicAdd; + auto &dm_device = coupler.get_data_manager_device_readwrite(); + auto nens = coupler.get_option("ncrms"); + auto nz = coupler.get_option("crm_nz"); + auto ny = coupler.get_option("crm_ny"); + auto nx = coupler.get_option("crm_nx"); + //------------------------------------------------------------------------------------------------ + auto temp = dm_device.get("temp" ); + auto rhov = dm_device.get("water_vapor"); + auto uvel = dm_device.get("uvel" ); + auto vt_temp = dm_device.get("vt_temp" ); + auto vt_rhov = dm_device.get("vt_rhov" ); + auto vt_uvel = dm_device.get("vt_uvel" ); + auto vt_temp_pert = dm_device.get("vt_temp_pert" ); + auto vt_rhov_pert = dm_device.get("vt_rhov_pert" ); + auto vt_uvel_pert = dm_device.get("vt_uvel_pert" ); + //------------------------------------------------------------------------------------------------ + // local variables + real2d temp_mean("temp_mean", nz, nens); + real2d rhov_mean("rhov_mean", nz, nens); + real2d uvel_mean("uvel_mean", nz, nens); + //------------------------------------------------------------------------------------------------ + // initialize variance and horizontal mean + parallel_for( SimpleBounds<2>(nz,nens) , YAKL_LAMBDA (int k, int n) { + temp_mean(k,n) = 0.0; + rhov_mean(k,n) = 0.0; + uvel_mean(k,n) = 0.0; + vt_temp(k,n) = 0.0; + vt_rhov(k,n) = 0.0; + vt_uvel(k,n) = 0.0; + }); + //------------------------------------------------------------------------------------------------ + // calculate horizontal mean + real r_nx_ny = 1._fp/(nx*ny); // precompute reciprocal to avoid costly divisions + parallel_for(SimpleBounds<4>(nz,ny,nx,nens), YAKL_LAMBDA (int k, int j, int i, int n) { + yakl::atomicAdd( temp_mean(k,n), temp(k,j,i,n)*r_nx_ny ); + yakl::atomicAdd( rhov_mean(k,n), rhov(k,j,i,n)*r_nx_ny ); + yakl::atomicAdd( uvel_mean(k,n), uvel(k,j,i,n)*r_nx_ny ); + }); + //------------------------------------------------------------------------------------------------ + // calculate fluctuations from horz mean + parallel_for(SimpleBounds<4>(nz,ny,nx,nens), YAKL_LAMBDA (int k, int j, int i, int n) { + vt_temp_pert(k,j,i,n) = temp(k,j,i,n) - temp_mean(k,n); + vt_rhov_pert(k,j,i,n) = rhov(k,j,i,n) - rhov_mean(k,n); + vt_uvel_pert(k,j,i,n) = uvel(k,j,i,n) - uvel_mean(k,n); + }); + //------------------------------------------------------------------------------------------------ + // calculate variance + parallel_for(SimpleBounds<4>(nz,ny,nx,nens), YAKL_LAMBDA (int k, int j, int i, int n) { + yakl::atomicAdd( vt_temp(k,n), ( vt_temp_pert(k,j,i,n) * vt_temp_pert(k,j,i,n) ) * r_nx_ny ); + yakl::atomicAdd( vt_rhov(k,n), ( vt_rhov_pert(k,j,i,n) * vt_rhov_pert(k,j,i,n) ) * r_nx_ny ); + yakl::atomicAdd( vt_uvel(k,n), ( vt_uvel_pert(k,j,i,n) * vt_uvel_pert(k,j,i,n) ) * r_nx_ny ); + }); + //------------------------------------------------------------------------------------------------ +} + +inline void pam_variance_transport_compute_forcing( pam::PamCoupler &coupler ) { + using yakl::c::parallel_for; + using yakl::c::SimpleBounds; + auto &dm_device = coupler.get_data_manager_device_readwrite(); + auto &dm_host = coupler.get_data_manager_host_readwrite(); + auto nens = coupler.get_option("ncrms"); + auto nz = coupler.get_option("crm_nz"); + auto ny = coupler.get_option("crm_ny"); + auto nx = coupler.get_option("crm_nx"); + auto gcm_nlev = coupler.get_option("gcm_nlev"); + auto gcm_dt = coupler.get_option("gcm_dt"); + //------------------------------------------------------------------------------------------------ + // update CRM variance values + pam_variance_transport_diagnose(coupler); + //------------------------------------------------------------------------------------------------ + auto temp = dm_device.get("temp" ); + auto rhov = dm_device.get("water_vapor" ); + auto uvel = dm_device.get("uvel" ); + auto vt_temp = dm_device.get("vt_temp" ); + auto vt_rhov = dm_device.get("vt_rhov" ); + auto vt_uvel = dm_device.get("vt_uvel" ); + auto vt_temp_forcing_tend = dm_device.get("vt_temp_forcing_tend"); + auto vt_rhov_forcing_tend = dm_device.get("vt_rhov_forcing_tend"); + auto vt_uvel_forcing_tend = dm_device.get("vt_uvel_forcing_tend"); + auto gcm_vt_temp = dm_host.get("input_vt_t").createDeviceCopy(); + auto gcm_vt_rhov = dm_host.get("input_vt_q").createDeviceCopy(); + auto gcm_vt_uvel = dm_host.get("input_vt_u").createDeviceCopy(); + //------------------------------------------------------------------------------------------------ + // calculate variance transport forcing + parallel_for( SimpleBounds<2>(nz,nens) , YAKL_LAMBDA (int k_crm, int n) { + int k_gcm = gcm_nlev-1-k_crm; + vt_temp_forcing_tend(k_crm,n) = ( gcm_vt_temp(k_gcm,n) - vt_temp(k_crm,n) ) * gcm_dt ; + vt_rhov_forcing_tend(k_crm,n) = ( gcm_vt_rhov(k_gcm,n) - vt_rhov(k_crm,n) ) * gcm_dt ; + vt_uvel_forcing_tend(k_crm,n) = ( gcm_vt_uvel(k_gcm,n) - vt_uvel(k_crm,n) ) * gcm_dt ; + }); + //------------------------------------------------------------------------------------------------ +} + +inline void pam_variance_transport_apply_forcing( pam::PamCoupler &coupler ) { + using yakl::c::parallel_for; + using yakl::c::SimpleBounds; + auto &dm_device = coupler.get_data_manager_device_readwrite(); + auto nens = coupler.get_option("ncrms"); + auto nz = coupler.get_option("crm_nz"); + auto ny = coupler.get_option("crm_ny"); + auto nx = coupler.get_option("crm_nx"); + auto crm_dt = coupler.get_option("crm_dt"); + //------------------------------------------------------------------------------------------------ + // min and max perturbation scaling values are used to limit the + // large-scale forcing from variance transport. This is meant to + // protect against creating unstable situations, although + // problematic scenarios were extremely rare in testing. + // A scaling limit of +/- 10% was found to be adequate. + real constexpr pert_scale_min = 1.0 - 0.005; + real constexpr pert_scale_max = 1.0 + 0.005; + //------------------------------------------------------------------------------------------------ + auto temp = dm_device.get("temp" ); + auto rhov = dm_device.get("water_vapor" ); + auto uvel = dm_device.get("uvel" ); + auto vt_temp = dm_device.get("vt_temp" ); + auto vt_rhov = dm_device.get("vt_rhov" ); + auto vt_uvel = dm_device.get("vt_uvel" ); + auto vt_temp_pert = dm_device.get("vt_temp_pert" ); + auto vt_rhov_pert = dm_device.get("vt_rhov_pert" ); + auto vt_uvel_pert = dm_device.get("vt_uvel_pert" ); + auto vt_temp_forcing_tend = dm_device.get("vt_temp_forcing_tend"); + auto vt_rhov_forcing_tend = dm_device.get("vt_rhov_forcing_tend"); + auto vt_uvel_forcing_tend = dm_device.get("vt_uvel_forcing_tend"); + //------------------------------------------------------------------------------------------------ + // local variables + real2d temp_pert_scale("temp_pert_scale", nz, nens); + real2d rhov_pert_scale("rhov_pert_scale", nz, nens); + real2d uvel_pert_scale("uvel_pert_scale", nz, nens); + //------------------------------------------------------------------------------------------------ + // update CRM variance values + pam_variance_transport_diagnose(coupler); + //------------------------------------------------------------------------------------------------ + // calculate scaling factor for local perturbations + parallel_for( SimpleBounds<2>(nz,nens) , YAKL_LAMBDA (int k, int n) { + // initialize scaling factors to 1.0 + temp_pert_scale(k,n) = 1.0; + rhov_pert_scale(k,n) = 1.0; + uvel_pert_scale(k,n) = 1.0; + // set scaling factors as long as there are perturbations to scale + if (vt_temp(k,n)>0.0) { temp_pert_scale(k,n) = sqrt( 1.0 + crm_dt * vt_temp_forcing_tend(k,n) / vt_temp(k,n) ); } + if (vt_rhov(k,n)>0.0) { rhov_pert_scale(k,n) = sqrt( 1.0 + crm_dt * vt_rhov_forcing_tend(k,n) / vt_rhov(k,n) ); } + if (vt_uvel(k,n)>0.0) { uvel_pert_scale(k,n) = sqrt( 1.0 + crm_dt * vt_uvel_forcing_tend(k,n) / vt_uvel(k,n) ); } + // enforce minimum scaling + temp_pert_scale(k,n) = std::max( temp_pert_scale(k,n), pert_scale_min ); + rhov_pert_scale(k,n) = std::max( rhov_pert_scale(k,n), pert_scale_min ); + uvel_pert_scale(k,n) = std::max( uvel_pert_scale(k,n), pert_scale_min ); + // enforce maximum scaling + temp_pert_scale(k,n) = std::min( temp_pert_scale(k,n), pert_scale_max ); + rhov_pert_scale(k,n) = std::min( rhov_pert_scale(k,n), pert_scale_max ); + uvel_pert_scale(k,n) = std::min( uvel_pert_scale(k,n), pert_scale_max ); + }); + //------------------------------------------------------------------------------------------------ + // apply variance forcing tendency + parallel_for(SimpleBounds<4>(nz,ny,nx,nens), YAKL_LAMBDA (int k, int j, int i, int n) { + real temp_tend_loc = ( temp_pert_scale(k,n) * vt_temp_pert(k,j,i,n) - vt_temp_pert(k,j,i,n) ); + real rhov_tend_loc = ( rhov_pert_scale(k,n) * vt_rhov_pert(k,j,i,n) - vt_rhov_pert(k,j,i,n) ); + real uvel_tend_loc = ( uvel_pert_scale(k,n) * vt_uvel_pert(k,j,i,n) - vt_uvel_pert(k,j,i,n) ); + temp(k,j,i,n) = temp(k,j,i,n) + temp_tend_loc; + rhov(k,j,i,n) = rhov(k,j,i,n) + rhov_tend_loc; + uvel(k,j,i,n) = uvel(k,j,i,n) + uvel_tend_loc; + }); + //------------------------------------------------------------------------------------------------ +} + +inline void pam_variance_transport_compute_feedback( pam::PamCoupler &coupler ) { + using yakl::c::parallel_for; + using yakl::c::SimpleBounds; + using yakl::atomicAdd; + auto &dm_device = coupler.get_data_manager_device_readwrite(); + auto &dm_host = coupler.get_data_manager_host_readwrite(); + auto nens = coupler.get_option("ncrms"); + auto nz = coupler.get_option("crm_nz"); + auto ny = coupler.get_option("crm_ny"); + auto nx = coupler.get_option("crm_nx"); + auto gcm_nlev = coupler.get_option("gcm_nlev"); + auto gcm_dt = coupler.get_option("gcm_dt"); + //------------------------------------------------------------------------------------------------ + // update CRM variance values + pam_variance_transport_diagnose(coupler); + //------------------------------------------------------------------------------------------------ + auto temp = dm_device.get("temp" ); + auto rhov = dm_device.get("water_vapor"); + auto uvel = dm_device.get("uvel" ); + auto vt_temp = dm_device.get("vt_temp" ); + auto vt_rhov = dm_device.get("vt_rhov" ); + auto vt_uvel = dm_device.get("vt_uvel" ); + auto gcm_vt_temp = dm_host.get("input_vt_t").createDeviceCopy(); + auto gcm_vt_rhov = dm_host.get("input_vt_q").createDeviceCopy(); + auto gcm_vt_uvel = dm_host.get("input_vt_u").createDeviceCopy(); + //------------------------------------------------------------------------------------------------ + dm_device.register_and_allocate("vt_temp_feedback_tend", "feedback tend of temp variance", {gcm_nlev,nens},{"gcm_lev","nens"}); + dm_device.register_and_allocate("vt_rhov_feedback_tend", "feedback tend of rhov variance", {gcm_nlev,nens},{"gcm_lev","nens"}); + dm_device.register_and_allocate("vt_uvel_feedback_tend", "feedback tend of uvel variance", {gcm_nlev,nens},{"gcm_lev","nens"}); + auto vt_temp_feedback_tend = dm_device.get("vt_temp_feedback_tend" ); + auto vt_rhov_feedback_tend = dm_device.get("vt_rhov_feedback_tend" ); + auto vt_uvel_feedback_tend = dm_device.get("vt_uvel_feedback_tend" ); + //------------------------------------------------------------------------------------------------ + // calculate variance transport forcing + parallel_for( SimpleBounds<2>(nz,nens) , YAKL_LAMBDA (int k_crm, int n) { + int k_gcm = gcm_nlev-1-k_crm; + if (k_crm("vt_temp_feedback_tend" ); + auto vt_rhov_feedback_tend = dm_device.get("vt_rhov_feedback_tend" ); + auto vt_uvel_feedback_tend = dm_device.get("vt_uvel_feedback_tend" ); + //------------------------------------------------------------------------------------------------ + auto output_vt_temp_tend_host = dm_host.get("output_t_vt_tend" ); + auto output_vt_rhov_tend_host = dm_host.get("output_q_vt_tend" ); + auto output_vt_uvel_tend_host = dm_host.get("output_u_vt_tend" ); + //------------------------------------------------------------------------------------------------ + // Copy the data to host + vt_temp_feedback_tend.deep_copy_to(output_vt_temp_tend_host); + vt_rhov_feedback_tend.deep_copy_to(output_vt_rhov_tend_host); + vt_uvel_feedback_tend.deep_copy_to(output_vt_uvel_tend_host); + yakl::fence(); + //------------------------------------------------------------------------------------------------ +} + diff --git a/components/eam/src/physics/p3/eam/micro_p3.F90 b/components/eam/src/physics/p3/eam/micro_p3.F90 index 730736a2dbf2..e668cbaa494c 100644 --- a/components/eam/src/physics/p3/eam/micro_p3.F90 +++ b/components/eam/src/physics/p3/eam/micro_p3.F90 @@ -56,7 +56,7 @@ module micro_p3 use phys_control, only: use_hetfrz_classnuc ! physical and mathematical constants - use micro_p3_utils, only: rho_1000mb,rho_600mb,ar,br,f1r,f2r,rho_h2o,kr,kc,aimm,mi0,nccnst, & + use micro_p3_utils, only: rho_1000mb,rho_600mb,ar,br,f1r,f2r,rho_h2o,kr,kc,aimm,mi0, & eci,eri,bcn,cpw,cons1,cons3,cons4,cons5,cons6,cons7, & inv_rho_h2o,inv_dropmass,qsmall,nsmall,cp,g,rd,rv,ep_2,inv_cp, & thrd,sxth,piov6,rho_rimeMin, & @@ -441,7 +441,7 @@ SUBROUTINE p3_main_part1(kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribe t_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, qv, th_atm, & qc, nc, qr, nr, & qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, & - nc_incld, nr_incld, ni_incld, bm_incld, is_nucleat_possible, is_hydromet_present) + nc_incld, nr_incld, ni_incld, bm_incld, is_nucleat_possible, is_hydromet_present, nccnst) implicit none @@ -449,7 +449,7 @@ SUBROUTINE p3_main_part1(kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribe integer, intent(in) :: kts, kte, kbot, ktop, kdir logical(btype), intent(in) :: do_predict_nc - real(rtype), intent(in) :: dt + real(rtype), intent(in) :: dt, nccnst real(rtype), intent(in), dimension(kts:kte) :: pres, dpres, dz, nc_nuceat_tend, exner, inv_exner, & inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, nccn_prescribed @@ -561,15 +561,15 @@ SUBROUTINE p3_main_part2(kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribe qm, bm, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, & ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, mu_r, lamr, logn0r, qv2qi_depos_tend, precip_total_tend, & nevapr, qr_evap_tend, vap_liq_exchange, vap_ice_exchange, liq_ice_exchange, pratot, & - prctot, frzimm, frzcnt, frzdep, p3_tend_out, is_hydromet_present) + prctot, frzimm, frzcnt, frzdep, p3_tend_out, is_hydromet_present, do_precip_off, nccnst) implicit none ! args integer, intent(in) :: kts, kte, kbot, ktop, kdir - logical(btype), intent(in) :: do_predict_nc, do_prescribed_CCN - real(rtype), intent(in) :: dt, inv_dt + logical(btype), intent(in) :: do_predict_nc, do_prescribed_CCN, do_precip_off + real(rtype), intent(in) :: dt, inv_dt, nccnst real(rtype), intent(in) :: p3_autocon_coeff, p3_accret_coeff, p3_qc_autocon_expon, p3_nc_autocon_expon, p3_qc_accret_expon, & p3_wbf_coeff, p3_embryonic_rain_size, p3_max_mean_rain_size @@ -875,7 +875,7 @@ SUBROUTINE p3_main_part2(kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribe ! NOTE: cloud_water_autoconversion must be called before droplet_self_collection call cloud_water_autoconversion(rho(k),qc_incld(k),nc_incld(k),inv_qc_relvar(k),& p3_autocon_coeff,p3_qc_autocon_expon,p3_nc_autocon_expon,p3_embryonic_rain_size,& - qc2qr_autoconv_tend,nc2nr_autoconv_tend,ncautr) + do_precip_off,qc2qr_autoconv_tend,nc2nr_autoconv_tend,ncautr) !............................ ! self-collection of droplets @@ -978,7 +978,7 @@ SUBROUTINE p3_main_part2(kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribe !-- warm-phase only processes: call update_prognostic_liquid(qc2qr_accret_tend, nc_accret_tend, qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr, & nc_selfcollect_tend, qr2qv_evap_tend, nr_evap_tend, nr_selfcollect_tend, & - do_predict_nc, do_prescribed_CCN, inv_rho(k), exner(k), latent_heat_vapor(k), dt, & + do_predict_nc, nccnst, do_prescribed_CCN, inv_rho(k), exner(k), latent_heat_vapor(k), dt, & th_atm(k), qv(k), qc(k), nc(k), qr(k), nr(k)) !== @@ -1259,7 +1259,7 @@ SUBROUTINE p3_main(qc,nc,qr,nr,th_atm,qv,dt,qi,qm,ni,bm, p3_wbf_coeff,p3_mincdnc,p3_max_mean_rain_size,p3_embryonic_rain_size, & dpres,exner,qv2qi_depos_tend,precip_total_tend,nevapr,qr_evap_tend,precip_liq_flux,precip_ice_flux,rflx,sflx,cflx,cld_frac_r,cld_frac_l,cld_frac_i, & p3_tend_out,mu_c,lamc,liq_ice_exchange,vap_liq_exchange, & - vap_ice_exchange,qv_prev,t_prev,col_location,diag_equiv_reflectivity,diag_ze_rain,diag_ze_ice & + vap_ice_exchange,qv_prev,t_prev,col_location,do_precip_off,nccnst,diag_equiv_reflectivity,diag_ze_rain,diag_ze_ice & #ifdef SCREAM_CONFIG_IS_CMAKE ,elapsed_s & #endif @@ -1337,6 +1337,10 @@ SUBROUTINE p3_main(qc,nc,qr,nr,th_atm,qv,dt,qi,qm,ni,bm, ! INPUT for prescribed CCN option logical(btype), intent(in) :: do_prescribed_CCN + ! INPUT for idealization options + logical(btype), intent(in) :: do_precip_off + real(rtype), intent(in) :: nccnst + ! INPUT for p3 tuning parameters real(rtype), intent(in) :: p3_autocon_coeff ! autconversion coefficient real(rtype), intent(in) :: p3_accret_coeff ! accretion coefficient @@ -1517,7 +1521,7 @@ SUBROUTINE p3_main(qc,nc,qr,nr,th_atm,qv,dt,qi,qm,ni,bm, rhofaci(i,:), acn(i,:), qv(i,:), th_atm(i,:), qc(i,:), nc(i,:), qr(i,:), nr(i,:), & qi(i,:), ni(i,:), qm(i,:), bm(i,:), qc_incld(i,:), qr_incld(i,:), & qi_incld(i,:), qm_incld(i,:), nc_incld(i,:), nr_incld(i,:), & - ni_incld(i,:), bm_incld(i,:), is_nucleat_possible, is_hydromet_present) + ni_incld(i,:), bm_incld(i,:), is_nucleat_possible, is_hydromet_present, nccnst) if (debug_ON) then tmparr1(i,:) = th_atm(i,:)*inv_exner(i,:)!(pres(i,:)*1.e-5)**(rd*inv_cp) @@ -1541,7 +1545,8 @@ SUBROUTINE p3_main(qc,nc,qr,nr,th_atm,qv,dt,qi,qm,ni,bm, bm_incld(i,:), mu_c(i,:), nu(i,:), lamc(i,:), cdist(i,:), cdist1(i,:), & cdistr(i,:), mu_r(i,:), lamr(i,:), logn0r(i,:), qv2qi_depos_tend(i,:), precip_total_tend(i,:), & nevapr(i,:), qr_evap_tend(i,:), vap_liq_exchange(i,:), vap_ice_exchange(i,:), & - liq_ice_exchange(i,:), pratot(i,:), prctot(i,:), frzimm(i,:), frzcnt(i,:), frzdep(i,:), p3_tend_out(i,:,:), is_hydromet_present) + liq_ice_exchange(i,:), pratot(i,:), prctot(i,:), frzimm(i,:), frzcnt(i,:), frzdep(i,:), p3_tend_out(i,:,:), is_hydromet_present, & + do_precip_off, nccnst) ! measure microphysics processes tendency output @@ -2950,7 +2955,7 @@ end subroutine rain_self_collection subroutine cloud_water_autoconversion(rho,qc_incld,nc_incld,inv_qc_relvar, & p3_autocon_coeff,p3_qc_autocon_expon,p3_nc_autocon_expon,p3_embryonic_rain_size, & - qc2qr_autoconv_tend,nc2nr_autoconv_tend,ncautr) + do_precip_off,qc2qr_autoconv_tend,nc2nr_autoconv_tend,ncautr) implicit none @@ -2963,6 +2968,8 @@ subroutine cloud_water_autoconversion(rho,qc_incld,nc_incld,inv_qc_relvar, real(rtype), intent(in) :: p3_nc_autocon_expon real(rtype), intent(in) :: p3_embryonic_rain_size + logical(btype), intent(in) :: do_precip_off + real(rtype), intent(out) :: qc2qr_autoconv_tend real(rtype), intent(out) :: nc2nr_autoconv_tend real(rtype), intent(out) :: ncautr @@ -2981,8 +2988,9 @@ subroutine cloud_water_autoconversion(rho,qc_incld,nc_incld,inv_qc_relvar, ncautr = qc2qr_autoconv_tend*cons3*(1._rtype/bfb_pow(p3_embryonic_rain_size,3._rtype)) nc2nr_autoconv_tend = qc2qr_autoconv_tend*nc_incld/qc_incld - if (qc2qr_autoconv_tend .eq.0._rtype) nc2nr_autoconv_tend = 0._rtype - if (nc2nr_autoconv_tend.eq.0._rtype) qc2qr_autoconv_tend = 0._rtype + if (qc2qr_autoconv_tend .eq.0._rtype .or. do_precip_off) nc2nr_autoconv_tend = 0._rtype + if (nc2nr_autoconv_tend.eq.0._rtype .or. do_precip_off) qc2qr_autoconv_tend = 0._rtype + if (do_precip_off) ncautr = 0._rtype endif qc_not_small @@ -3502,7 +3510,7 @@ end subroutine update_prognostic_ice subroutine update_prognostic_liquid(qc2qr_accret_tend,nc_accret_tend,qc2qr_autoconv_tend,nc2nr_autoconv_tend, & ncautr,nc_selfcollect_tend, qr2qv_evap_tend,nr_evap_tend,nr_selfcollect_tend, & - do_predict_nc, do_prescribed_CCN, inv_rho,exner,latent_heat_vapor,dt, & + do_predict_nc, nccnst, do_prescribed_CCN, inv_rho,exner,latent_heat_vapor,dt, & th_atm,qv,qc,nc,qr,nr) !-- warm-phase only processes: @@ -3520,6 +3528,7 @@ subroutine update_prognostic_liquid(qc2qr_accret_tend,nc_accret_tend,qc2qr_autoc logical(btype), intent(in) :: do_predict_nc, do_prescribed_CCN + real(rtype), intent(in) :: nccnst real(rtype), intent(in) :: inv_rho real(rtype), intent(in) :: exner real(rtype), intent(in) :: latent_heat_vapor diff --git a/components/eam/src/physics/p3/eam/micro_p3_interface.F90 b/components/eam/src/physics/p3/eam/micro_p3_interface.F90 index 9d53e95fc17f..bb6e164b4a14 100644 --- a/components/eam/src/physics/p3/eam/micro_p3_interface.F90 +++ b/components/eam/src/physics/p3/eam/micro_p3_interface.F90 @@ -42,6 +42,7 @@ module micro_p3_interface use ncdio_atm, only: infld use ppgrid, only: begchunk, endchunk, pcols, pver, pverp,psubcols use cam_history_support, only: add_hist_coord + use iop_data_mod, only: precip_off implicit none save @@ -131,8 +132,8 @@ module micro_p3_interface p3_wbf_coeff = huge(1.0_rtype), & p3_mincdnc = huge(1.0_rtype), & p3_max_mean_rain_size = huge(1.0_rtype), & - p3_embryonic_rain_size = huge(1.0_rtype) - + p3_embryonic_rain_size = huge(1.0_rtype), & + micro_nccons = huge(1.0_rtype) integer :: ncnst @@ -165,7 +166,7 @@ subroutine micro_p3_readnl(nlfile) micro_p3_tableversion, micro_p3_lookup_dir, micro_aerosolactivation, micro_subgrid_cloud, & micro_tend_output, p3_autocon_coeff, p3_qc_autocon_expon, p3_nc_autocon_expon, p3_accret_coeff, & p3_qc_accret_expon, p3_wbf_coeff, p3_max_mean_rain_size, p3_embryonic_rain_size, & - do_prescribed_CCN, do_Cooper_inP3, p3_mincdnc + do_prescribed_CCN, do_Cooper_inP3, p3_mincdnc, micro_nccons !----------------------------------------------------------------------------- @@ -220,6 +221,7 @@ subroutine micro_p3_readnl(nlfile) call mpibcast(p3_embryonic_rain_size, 1 , mpir8, 0, mpicom) call mpibcast(do_prescribed_CCN, 1, mpilog, 0, mpicom) call mpibcast(do_Cooper_inP3, 1, mpilog, 0, mpicom) + call mpibcast(micro_nccons, 1, mpir8, 0, mpicom) #endif @@ -1363,6 +1365,8 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) qv_prev(its:ite,kts:kte), & ! IN qv at end of prev p3_main call kg kg-1 t_prev(its:ite,kts:kte), & ! IN t at end of prev p3_main call K col_location(its:ite,:3), & ! IN column locations + precip_off, & ! IN Option to turn precip (liquid) off + micro_nccons, & ! IN Option for constant droplet concentration diag_equiv_reflectivity(its:ite,kts:kte), & !OUT equivalent reflectivity (rain + ice) [dBz] diag_ze_rain(its:ite,kts:kte),diag_ze_ice(its:ite,kts:kte)) !OUT equivalent reflectivity for rain and ice [dBz] diff --git a/components/eam/src/physics/p3/eam/micro_p3_utils.F90 b/components/eam/src/physics/p3/eam/micro_p3_utils.F90 index dab1c4751323..c2182a14206c 100644 --- a/components/eam/src/physics/p3/eam/micro_p3_utils.F90 +++ b/components/eam/src/physics/p3/eam/micro_p3_utils.F90 @@ -33,9 +33,6 @@ module micro_p3_utils ! maximum total ice concentration (sum of all categories) real(rtype), public, parameter :: max_total_ni = 500.e+3_rtype ! (m) - ! droplet concentration (m-3) - real(rtype), public, parameter :: nccnst = 200.e+6_rtype - ! parameters for Seifert and Beheng (2001) autoconversion/accretion real(rtype), public, parameter :: kc = 9.44e+9_rtype real(rtype), public, parameter :: kr = 5.78e+3_rtype diff --git a/components/eam/src/physics/p3/scream/micro_p3.F90 b/components/eam/src/physics/p3/scream/micro_p3.F90 index e59bcc08239c..f345e5abcf70 100644 --- a/components/eam/src/physics/p3/scream/micro_p3.F90 +++ b/components/eam/src/physics/p3/scream/micro_p3.F90 @@ -1846,7 +1846,8 @@ subroutine get_rain_dsd2(qr,nr,mu_r,lamr,cdistr,logn0r) real(rtype), intent(out) :: lamr,mu_r,cdistr,logn0r !local variables: - real(rtype) :: inv_dum,lammax,lammin + real(rtype) :: lammax,lammin + real(rtype) :: mass_to_d3_factor !-------------------------------------------------------------------------- @@ -1858,25 +1859,25 @@ subroutine get_rain_dsd2(qr,nr,mu_r,lamr,cdistr,logn0r) ! find spot in lookup table ! (scaled N/q for lookup table parameter space_ nr = max(nr,nsmall) - inv_dum = bfb_cbrt(qr/(cons1*nr*6._rtype)) ! Apply constant mu_r: Recall the switch to v4 tables means constant mu_r mu_r = mu_r_constant - lamr = bfb_cbrt(cons1*nr*(mu_r+3._rtype)*(mu_r+2._rtype)*(mu_r+1._rtype)/(qr)) ! recalculate slope based on mu_r + mass_to_d3_factor = cons1*(mu_r+3._rtype)*(mu_r+2._rtype)*(mu_r+1._rtype) + lamr = bfb_cbrt(mass_to_d3_factor*nr/qr) ! recalculate slope based on mu_r lammax = (mu_r+1._rtype)*1.e+5_rtype ! check for slope lammin = (mu_r+1._rtype)*500._rtype !500=1/(2mm) is inverse of max allowed number-weighted mean raindrop diameter ! apply lambda limiters for rain if (lamr.lt.lammin) then lamr = lammin - nr = bfb_exp(3._rtype*bfb_log(lamr)+bfb_log(qr)+bfb_log(bfb_gamma(mu_r+1._rtype))-bfb_log(bfb_gamma(mu_r+4._rtype)))/(cons1) + nr = lamr * lamr * lamr * qr / mass_to_d3_factor elseif (lamr.gt.lammax) then lamr = lammax - nr = bfb_exp(3._rtype*bfb_log(lamr)+bfb_log(qr)+bfb_log(bfb_gamma(mu_r+1._rtype))-bfb_log(bfb_gamma(mu_r+4._rtype)))/(cons1) + nr = lamr * lamr * lamr * qr / mass_to_d3_factor endif cdistr = nr/bfb_gamma(mu_r+1._rtype) - logn0r = bfb_log10(nr)+(mu_r+1._rtype)*bfb_log10(lamr)-bfb_log10(bfb_gamma(mu_r+1._rtype)) !note: logn0r is calculated as log10(n0r) + logn0r = bfb_log10(cdistr)+(mu_r+1._rtype)*bfb_log10(lamr) !note: logn0r is calculated as log10(n0r) else diff --git a/components/eam/src/utils/cam_abortutils.F90 b/components/eam/src/utils/cam_abortutils.F90 index 59e5fece3f2e..6783a5753b3b 100644 --- a/components/eam/src/utils/cam_abortutils.F90 +++ b/components/eam/src/utils/cam_abortutils.F90 @@ -10,7 +10,7 @@ module cam_abortutils !----------------------------------------------------------------------- !- use statements ------------------------------------------------------ !----------------------------------------------------------------------- - use shr_kind_mod, only: shr_kind_in + use shr_kind_mod, only: shr_kind_in, SHR_KIND_CL use shr_sys_mod, only: shr_sys_abort !----------------------------------------------------------------------- @@ -24,6 +24,7 @@ module cam_abortutils ! Public interfaces ---------------------------------------------------- !----------------------------------------------------------------------- public endrun + public handle_allocate_error !----------------------------------------------------------------------- ! Subroutines and functions -------------------------------------------- @@ -63,4 +64,20 @@ subroutine endrun(string) end subroutine + subroutine handle_allocate_error(retval, subname, fieldname) + ! if is not zero, generate an error message and abort + ! Dummy arguments + integer, intent(in) :: retval + character(len=*), intent(in) :: subname + character(len=*), intent(in) :: fieldname + ! Local variable + character(len=SHR_KIND_CL) :: errmsg + + if (retval /= 0) then + write(errmsg, '(4a,i0)') trim(subname), ' error allocating ', & + trim(fieldname), ', error = ', retval + call endrun(errmsg) + end if + end subroutine handle_allocate_error + end module cam_abortutils diff --git a/components/eam/src/utils/cam_grid_support.F90 b/components/eam/src/utils/cam_grid_support.F90 index 1a69ef07f9ce..bc147321b325 100644 --- a/components/eam/src/utils/cam_grid_support.F90 +++ b/components/eam/src/utils/cam_grid_support.F90 @@ -161,12 +161,14 @@ module cam_grid_support logical :: unstructured ! Is this needed? logical :: block_indexed ! .false. for lon/lat logical :: attrs_defined = .false. + logical :: zonal_grid = .false. type(cam_filemap_t), pointer :: map => null() ! global dim map (dof) type(cam_grid_attr_ptr_t), pointer :: attributes => NULL() contains procedure :: print_cam_grid procedure :: is_unstructured => cam_grid_unstructured procedure :: is_block_indexed => cam_grid_block_indexed + procedure :: is_zonal_grid => cam_grid_zonal_grid procedure :: coord_lengths => cam_grid_get_dims procedure :: coord_names => cam_grid_coord_names procedure :: dim_names => cam_grid_dim_names @@ -205,9 +207,12 @@ module cam_grid_support integer :: global_lon_size = 0 ! lon patch dim size real(r8) :: lon_range(2) real(r8) :: lat_range(2) + logical :: collected_columns ! Output unstructured type(cam_filemap_t), pointer :: mask => null() ! map for active pts integer(iMap), pointer :: latmap(:) => null() ! map for patch coords integer(iMap), pointer :: lonmap(:) => null() ! map for patch coords + real(r8), pointer :: lonvals(:) => null() ! For collected output + real(r8), pointer :: latvals(:) => null() ! For collected output contains procedure :: gridid => cam_grid_patch_get_id procedure :: get_axis_names => cam_grid_patch_get_axis_names @@ -307,6 +312,7 @@ end subroutine print_attr_spec public :: cam_grid_has_blocksize, cam_grid_get_block_count public :: cam_grid_get_latvals, cam_grid_get_lonvals public :: cam_grid_is_unstructured, cam_grid_is_block_indexed + public :: cam_grid_is_zonal ! Functions for dealing with patch masks public :: cam_grid_compute_patch @@ -322,6 +328,11 @@ end subroutine print_attr_spec module procedure cam_grid_dimensions_name end interface + interface cam_grid_get_dim_names + module procedure cam_grid_get_dim_names_id + module procedure cam_grid_get_dim_names_name + end interface + interface cam_grid_read_dist_array module procedure cam_grid_read_dist_array_2d_int module procedure cam_grid_read_dist_array_3d_int @@ -739,7 +750,7 @@ integer function num_cam_grid_attrs(gridind) end function num_cam_grid_attrs subroutine cam_grid_register(name, id, lat_coord, lon_coord, map, & - unstruct, block_indexed, src_in, dest_in) + unstruct, block_indexed, zonal_grid, src_in, dest_in) ! Dummy arguments character(len=*), intent(in) :: name integer, intent(in) :: id @@ -748,6 +759,7 @@ subroutine cam_grid_register(name, id, lat_coord, lon_coord, map, & integer(iMap), pointer, intent(in) :: map(:,:) logical, optional, intent(in) :: unstruct logical, optional, intent(in) :: block_indexed + logical, optional, intent(in) :: zonal_grid integer, optional, intent(in) :: src_in(2) integer, optional, intent(in) :: dest_in(2) @@ -798,6 +810,16 @@ subroutine cam_grid_register(name, id, lat_coord, lon_coord, map, & else cam_grids(registeredhgrids)%block_indexed = cam_grids(registeredhgrids)%unstructured end if + if (present(zonal_grid)) then + ! Check the size of the longitude coordinate + call lon_coord%get_coord_len(i) + if (i /= 1) then + call endrun(subname//': lon_coord is not of size 1 for a zonal grid') + end if + cam_grids(registeredhgrids)%zonal_grid = zonal_grid + else + cam_grids(registeredhgrids)%zonal_grid = .false. + end if if (associated(cam_grids(registeredhgrids)%map)) then write(errormsg, *) call endrun(trim(subname)//": new grid map should not be associated") @@ -837,7 +859,8 @@ subroutine print_cam_grid(this) ', lat coord = ', trim(this%lat_coord%name), & ', lon coord = ', trim(this%lon_coord%name), & ', unstruct = ', this%unstructured, & - ', block_ind = ', this%block_indexed + ', block_ind = ', this%block_indexed, & + ', zonal_grid = ', this%zonal_grid attrPtr => this%attributes do while (associated(attrPtr)) !!XXgoldyXX: Is this not working in PGI? @@ -1406,23 +1429,50 @@ subroutine cam_grid_get_coord_names(id, name1, name2) end subroutine cam_grid_get_coord_names - subroutine cam_grid_get_dim_names(id, name1, name2) + !--------------------------------------------------------------------------- + ! + ! cam_grid_get_dim_names: Return the names of the grid axes dimensions. + ! Note that these may be the same + ! + !--------------------------------------------------------------------------- + subroutine cam_grid_get_dim_names_id(id, name1, name2) ! Dummy arguments integer, intent(in) :: id character(len=*), intent(out) :: name1 character(len=*), intent(out) :: name2 - + ! Local variables integer :: gridid gridid = get_cam_grid_index(id) if (gridid > 0) then call cam_grids(gridid)%dim_names(name1, name2) else - call endrun('cam_grid_get_dim_names: Bad grid ID') + call endrun('cam_grid_get_dim_names_id: Bad grid ID') end if - end subroutine cam_grid_get_dim_names + end subroutine cam_grid_get_dim_names_id + + subroutine cam_grid_get_dim_names_name(gridname, name1, name2) + + ! Dummy arguments + character(len=*), intent(in) :: gridname + character(len=*), intent(out) :: name1 + character(len=*), intent(out) :: name2 + + ! Local variables + integer :: gridind + character(len=120) :: errormsg + + gridind = get_cam_grid_index(trim(gridname)) + if (gridind < 0) then + write(errormsg, *) 'No CAM grid with name = ', trim(gridname) + call endrun('cam_grid_get_dim_names_name: '//errormsg) + else + call cam_grids(gridind)%dim_names(name1, name2) + end if + + end subroutine cam_grid_get_dim_names_name logical function cam_grid_has_blocksize(id) @@ -1534,8 +1584,23 @@ logical function cam_grid_is_block_indexed(id) result(block_indexed) end if end function cam_grid_is_block_indexed + logical function cam_grid_is_zonal(id) result(zonal) + + ! Dummy arguments + integer, intent(in) :: id + + ! Local variables + integer :: gridid + gridid = get_cam_grid_index(id) + if (gridid > 0) then + zonal = cam_grids(gridid)%is_zonal_grid() + else + call endrun('s: Bad grid ID') + end if + end function cam_grid_is_zonal + ! Compute or update a grid patch mask - subroutine cam_grid_compute_patch(id, patch, lonl, lonu, latl, latu) + subroutine cam_grid_compute_patch(id, patch, lonl, lonu, latl, latu, cco) ! Dummy arguments integer, intent(in) :: id @@ -1544,13 +1609,14 @@ subroutine cam_grid_compute_patch(id, patch, lonl, lonu, latl, latu) real(r8), intent(in) :: lonu real(r8), intent(in) :: latl real(r8), intent(in) :: latu + logical, intent(in) :: cco ! Collect columns? ! Local variables integer :: gridid gridid = get_cam_grid_index(id) if (gridid > 0) then - call cam_grids(gridid)%get_patch_mask(lonl, lonu, latl, latu, patch) + call cam_grids(gridid)%get_patch_mask(lonl, lonu, latl, latu, patch, cco) else call endrun('cam_grid_compute_patch: Bad grid ID') end if @@ -2409,6 +2475,12 @@ logical function cam_grid_block_indexed(this) cam_grid_block_indexed = this%block_indexed end function cam_grid_block_indexed + logical function cam_grid_zonal_grid(this) + class(cam_grid_t) :: this + + cam_grid_zonal_grid = this%zonal_grid + end function cam_grid_zonal_grid + logical function cam_grid_unstructured(this) class(cam_grid_t) :: this @@ -3197,7 +3269,7 @@ end subroutine cam_grid_write_darray_3d_real ! within the input patch. ! !--------------------------------------------------------------------------- - subroutine cam_grid_get_patch_mask(this, lonl, lonu, latl, latu, patch) + subroutine cam_grid_get_patch_mask(this, lonl, lonu, latl, latu, patch, cco) use spmd_utils, only: mpi_min, mpi_real8, mpicom use physconst, only: pi @@ -3206,6 +3278,7 @@ subroutine cam_grid_get_patch_mask(this, lonl, lonu, latl, latu, patch) real(r8), intent(in) :: lonl, lonu ! Longitude bounds real(r8), intent(in) :: latl, latu ! Latitude bounds type(cam_grid_patch_t), intent(inout) :: patch + logical, intent(in) :: cco ! Collect columns? ! Local arguments real(r8) :: mindist @@ -3228,6 +3301,7 @@ subroutine cam_grid_get_patch_mask(this, lonl, lonu, latl, latu, patch) real(r8), parameter :: maxangle = pi / 4.0_r8 real(r8), parameter :: deg2rad = pi / 180.0_r8 real(r8), parameter :: maxlat = 89.5_r8 + character(len=*), parameter :: subname = 'cam_grid_get_patch_mask' if (.not. associated(this%map)) then call endrun('cam_grid_get_patch_mask: Grid, '//trim(this%name)//', has no map') @@ -3243,6 +3317,9 @@ subroutine cam_grid_get_patch_mask(this, lonl, lonu, latl, latu, patch) ! NB: Compacting the mask must be done after all calls (for a ! particular mask) to this function. end if + if (patch%collected_columns .neqv. cco) then + call endrun(subname//': collected_column mismatch') + end if else if (associated(patch%latmap)) then call endrun('cam_grid_get_patch_mask: unallocated patch has latmap') @@ -3257,17 +3334,29 @@ subroutine cam_grid_get_patch_mask(this, lonl, lonu, latl, latu, patch) end if call patch%mask%clear() ! Set up the lat/lon maps - if (associated(this%lat_coord%values)) then - allocate(patch%latmap(LBOUND(this%lat_coord%values, 1):UBOUND(this%lat_coord%values, 1))) + if (cco) then + ! For collected column output, we need to collect coordinates and values + allocate(patch%latmap(patch%mask%num_elem())) patch%latmap = 0 - else - nullify(patch%latmap) - end if - if (associated(this%lon_coord%values)) then - allocate(patch%lonmap(LBOUND(this%lon_coord%values, 1):UBOUND(this%lon_coord%values, 1))) + allocate(patch%latvals(patch%mask%num_elem())) + patch%latvals = 91.0_r8 + allocate(patch%lonmap(patch%mask%num_elem())) patch%lonmap = 0 + allocate(patch%lonvals(patch%mask%num_elem())) + patch%lonvals = 361.0_r8 else - nullify(patch%lonmap) + if (associated(this%lat_coord%values)) then + allocate(patch%latmap(LBOUND(this%lat_coord%values, 1):UBOUND(this%lat_coord%values, 1))) + patch%latmap = 0 + else + nullify(patch%latmap) + end if + if (associated(this%lon_coord%values)) then + allocate(patch%lonmap(LBOUND(this%lon_coord%values, 1):UBOUND(this%lon_coord%values, 1))) + patch%lonmap = 0 + else + nullify(patch%lonmap) + end if end if end if diff --git a/components/eam/src/utils/string_utils.F90 b/components/eam/src/utils/string_utils.F90 index 3a1bafe5a6c2..476e6f2319ae 100644 --- a/components/eam/src/utils/string_utils.F90 +++ b/components/eam/src/utils/string_utils.F90 @@ -10,7 +10,8 @@ module string_utils to_upper, & ! Convert character string to upper case to_lower, & ! Convert character string to lower case INCSTR, & ! increments a string - GLC ! Position of last significant character in string + GLC, & ! Position of last significant character in string + int2str ! convert integer to left justified string contains @@ -240,4 +241,11 @@ integer function GLC( cs ) end function GLC +character(len=10) function int2str(n) + ! return default integer as a left justified string + integer, intent(in) :: n + !----------------------------------------------------------------------- + write(int2str,'(i0)') n +end function int2str + end module string_utils diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index e2aae3e97b32..453b2d4be8a3 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -1,7 +1,18 @@ +#################################################################### +# Basic setup: version, cime debug, cmake paths,... # +#################################################################### if (NOT DEFINED PROJECT_NAME) - cmake_minimum_required(VERSION 3.3) + cmake_minimum_required(VERSION 3.14) cmake_policy(SET CMP0057 NEW) set(SCREAM_CIME_BUILD FALSE) + + # Print the sha of the last commit (useful to double check which version was tested on CDash) + execute_process (COMMAND git rev-parse HEAD + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + OUTPUT_VARIABLE LAST_GIT_COMMIT_SHA + OUTPUT_STRIP_TRAILING_WHITESPACE) + set(EAMXX_GIT_VERSION ${LAST_GIT_COMMIT_SHA} CACHE STRING "The sha of the last git commit." FORCE) + message(STATUS "The sha of the last commit is ${EAMXX_GIT_VERSION}") else() set(SCREAM_CIME_BUILD TRUE) endif() @@ -20,6 +31,13 @@ if ($ENV{SCREAM_FORCE_CONFIG_FAIL}) message(FATAL_ERROR "Failed, as instructed by environment") endif() +string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_ci) +if (CMAKE_BUILD_TYPE_ci STREQUAL "debug") + set (SCREAM_DEBUG TRUE) +else () + set (SCREAM_DEBUG FALSE) +endif() + # Add the ./cmake folder to cmake path. Also add EKAT's cmake folder set (EKAT_CMAKE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../externals/ekat/cmake) list(APPEND CMAKE_MODULE_PATH @@ -33,6 +51,26 @@ if (SCREAM_CIME_BUILD) ${CMAKE_CURRENT_SOURCE_DIR}/cmake/cime) endif () +# We want to use C++17 in EAMxx +set(CMAKE_CXX_STANDARD 17) + +if (NOT SCREAM_CIME_BUILD) + project(SCREAM CXX C Fortran) + + if (SCREAM_CORI_HACK) + list(APPEND CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "ifcore") + list(REMOVE_ITEM CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "ifport") + endif() + +else() + # Ensure our languages are all enabled + enable_language(C CXX Fortran) +endif() + +#################################################################### +# Kokkos/YAKL-related settings # +#################################################################### + if (Kokkos_ENABLE_CUDA) if (Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE) if (Kokkos_ENABLE_DEBUG_BOUNDS_CHECK) @@ -56,68 +94,6 @@ endif() # to be on. For now, simply ensure Kokkos Serial is enabled option (Kokkos_ENABLE_SERIAL "" ON) -# We want to use C++17 in EAMxx -set(CMAKE_CXX_STANDARD 17) - -if (NOT SCREAM_CIME_BUILD) - project(SCREAM CXX C Fortran) - - if (SCREAM_CORI_HACK) - list(APPEND CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "ifcore") - list(REMOVE_ITEM CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "ifport") - endif() - -else() - # Ensure our languages are all enabled - enable_language(C CXX Fortran) -endif() - -# Print the sha of the last commit (useful to double check which version was tested on CDash) -execute_process (COMMAND git rev-parse HEAD - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} - OUTPUT_VARIABLE LAST_GIT_COMMIT_SHA - OUTPUT_STRIP_TRAILING_WHITESPACE) -set(EAMXX_GIT_VERSION ${LAST_GIT_COMMIT_SHA} CACHE STRING "The sha of the last git commit.") -message(STATUS "The sha of the last commit is ${EAMXX_GIT_VERSION}") - -set(SCREAM_DOUBLE_PRECISION TRUE CACHE BOOL "Set to double precision (default True)") - -# Set the scream base and src directory, to be used across subfolders -set(SCREAM_BASE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -set(SCREAM_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src) -set(SCREAM_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}) - -# Shortcut function, to print a variable -function (print_var var) - message ("${var}: ${${var}}") -endfunction () - -function (check_pack_size master_pack_size pack_size name) - math (EXPR PACK_MODULO "${master_pack_size} % ${pack_size}") - if ((pack_size GREATER master_pack_size) OR (NOT PACK_MODULO EQUAL 0)) - message (FATAL_ERROR "Invalid '${name}' size of ${pack_size}. Needs to be <= ${master_pack_size} and be a factor of it") - endif() -endfunction () - -# Compute reasonable defaults. This needs to happen before the CACHE variables -# are set. -string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_ci) -if (CMAKE_BUILD_TYPE_ci STREQUAL "debug") - set (SCREAM_DEBUG TRUE) -else () - set (SCREAM_DEBUG FALSE) -endif() - -# Add RRTMGP debug checks. Note, we might consider also adding RRTMGP_EXPENSIVE_CHECKS -# to turn on the RRTMGP internal checks here as well, via -# option (RRTMGP_EXPENSIVE_CHECKS "Turn on internal RRTMGP error checking" ${SCREAM_DEBUG}) -# and then adding to scream_config.h: -# #cmakedefine RRTMGP_EXPENSIVE_CHECKS -option (SCREAM_RRTMGP_DEBUG "Turn on extra debug checks in RRTMGP" ${SCREAM_DEBUG}) - -enable_testing() -include(CTest) - set (EAMXX_ENABLE_GPU FALSE CACHE BOOL "") set (CUDA_BUILD FALSE CACHE BOOL "") #needed for yakl if kokkos vars are not visible there? set (HIP_BUILD FALSE CACHE BOOL "") #needed for yakl if kokkos vars are not visible there? @@ -142,13 +118,16 @@ if( NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "[Cc]lang" ) set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -cpp") endif() -### Scream default configuration options +#################################################################### +# EAMxx main configuration options # +#################################################################### + +# First, Compute reasonable defaults. This needs to happen before the CACHE variables are set set(DEFAULT_MAX_RANKS 4) set(DEFAULT_MAX_THREADS 16) set(DEFAULT_MIMIC_GPU FALSE) set(DEFAULT_FPE FALSE) set(DEFAULT_PACK_SIZE 16) -set(DEFAULT_POSSIBLY_NO_PACK FALSE) if (EAMXX_ENABLE_GPU) # On the GPU, the pack size must be 1 set(DEFAULT_PACK_SIZE 1) @@ -203,24 +182,122 @@ if (Kokkos_ENABLE_HIP) set(DEFAULT_SMALL_KERNELS TRUE) endif() -### Set CACHE vars +if (SCREAM_DEBUG) + set(DEFAULT_FPMODEL "strict") + if (SCREAM_PACK_SIZE EQUAL 1 AND NOT EAMXX_ENABLE_GPU) + set(DEFAULT_FPE TRUE) + endif () +endif() + +### Now that reasonable defaults have been computed, set CACHE vars set(SCREAM_MIMIC_GPU ${DEFAULT_MIMIC_GPU} CACHE BOOL "Mimic GPU to correctness-test inter-column parallelism on non-GPU platform") set(SCREAM_PACK_CHECK_BOUNDS FALSE CACHE BOOL "If defined, scream::pack objects check indices against bounds") -set(SCREAM_TEST_DATA_DIR ${CMAKE_CURRENT_BINARY_DIR}/data CACHE PATH "Location of data files generated by tests") set(SCREAM_LIB_ONLY ${DEFAULT_LIB_ONLY} CACHE BOOL "Only build libraries, no exes") set(NetCDF_Fortran_PATH ${DEFAULT_NetCDF_Fortran_PATH} CACHE FILEPATH "Path to netcdf fortran installation") set(NetCDF_C_PATH ${DEFAULT_NetCDF_C_PATH} CACHE FILEPATH "Path to netcdf C installation") set(SCREAM_MACHINE ${DEFAULT_SCREAM_MACHINE} CACHE STRING "The CIME/SCREAM name for the current machine") option(SCREAM_MPI_ON_DEVICE "Whether to use device pointers for MPI calls" ON) -option(SCREAM_ENABLE_MAM "Whether to enable MAM aerosol support" OFF) +option(SCREAM_ENABLE_MAM "Whether to enable MAM aerosol support" ON) set(SCREAM_SMALL_KERNELS ${DEFAULT_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels") if (NOT SCREAM_SMALL_KERNELS) set(EKAT_DISABLE_WORKSPACE_SHARING TRUE CACHE STRING "") endif() +# Add RRTMGP debug checks. Note, we might consider also adding RRTMGP_EXPENSIVE_CHECKS +# to turn on the RRTMGP internal checks here as well, via +# option (RRTMGP_EXPENSIVE_CHECKS "Turn on internal RRTMGP error checking" ${SCREAM_DEBUG}) +# and then adding to scream_config.h: +# #cmakedefine RRTMGP_EXPENSIVE_CHECKS +option (SCREAM_RRTMGP_DEBUG "Turn on extra debug checks in RRTMGP" ${SCREAM_DEBUG}) + +set(SCREAM_DOUBLE_PRECISION TRUE CACHE BOOL "Set to double precision (default True)") + # For now, only used in share/grid/remap/refining_remapper_rma.*pp option (EAMXX_ENABLE_EXPERIMENTAL_CODE "Compile one-sided MPI for refining remappers" OFF) +option (SCREAM_ENABLE_ML_CORRECTION "Whether to enable ML correction parametrization" OFF) + +# Set number of vertical levels +set(SCREAM_NUM_VERTICAL_LEV ${DEFAULT_NUM_VERTICAL_LEV} CACHE STRING + "The number of levels used in the vertical grid." +) +option(SCREAM_HAS_LEAP_YEAR "Whether scream uses leap years or not" ON) + +set(SCREAM_FPMODEL ${DEFAULT_FPMODEL} CACHE STRING "Compiler floating point model") +set(SCREAM_FPE ${DEFAULT_FPE} CACHE BOOL "Enable floating point error exception") + +# Whether to use XYZ as a method to detect memory usage. +option (SCREAM_ENABLE_GETRUSAGE "Whether getrusage can be used to get memory usage." OFF) +option (SCREAM_ENABLE_STATM "Whether /proc/self/statm can be used to get memory usage." OFF) + +# Whether to disable warnings from tpls. +set (SCREAM_DISABLE_TPL_WARNINGS ON CACHE BOOL "") + +# Dycore settings +set(DEFAULT_SCREAM_DYNAMICS_DYCORE "NONE") +if (SCREAM_CIME_BUILD AND SCREAM_DYN_TARGET STREQUAL "theta-l_kokkos") + set (DEFAULT_SCREAM_DYNAMICS_DYCORE "Homme") +endif() + +set(SCREAM_DYNAMICS_DYCORE ${DEFAULT_SCREAM_DYNAMICS_DYCORE} CACHE STRING + "The name of the dycore to be used for dynamics. If NONE, then any code/test requiring dynamics is disabled.") + +string(TOUPPER "${SCREAM_DYNAMICS_DYCORE}" SCREAM_DYNAMICS_DYCORE) +if (NOT ${SCREAM_DOUBLE_PRECISION}) + # Homme cannot handle single precision, for now. This causes tests to fail. + # Fixing this requires adding a config parameter to homme, to switch between + # single and double. That must be done in the upstream repo (E3SM), before + # we can support it here. + # So, for now, if Homme is the requested dyn dycore AND single precision is + # requested, we disable dynamics, printing a warning. + if ("${SCREAM_DYNAMICS_DYCORE}" STREQUAL "HOMME") + message("WARNING! Homme dycore cannot be used in a Single Precision build. Turning Homme off.") + set(SCREAM_DYNAMICS_DYCORE "NONE") + endif() +endif() + +# Set the scream base and src directory, to be used across subfolders +set(SCREAM_BASE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +set(SCREAM_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src) +set(SCREAM_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}) + +#################################################################### +# Packs-related settings # +#################################################################### + +# Determine the main pack size. +set(SCREAM_PACK_SIZE ${DEFAULT_PACK_SIZE} CACHE STRING + "The number of scalars in a scream::pack::Pack and Mask. Larger packs have good performance on conditional-free loops due to improved caching.") + +# Besides the the main pack size, we have a couple of other pack sizes used across EAMxx +# For some routines, SKX may have better performance with pack_size=1 +set(SCREAM_SMALL_PACK_SIZE ${SCREAM_PACK_SIZE} CACHE STRING + "The number of scalars in a scream::pack::SmallPack and SmallMask. Smaller packs can have better performance in loops with conditionals since more of the packs will have masks with uniform value.") +set(SCREAM_POSSIBLY_NO_PACK "${Kokkos_ARCH_SKX}" CACHE BOOL + "Set possibly-no-pack to this value. You can set it to something else to restore packs on SKX for testing.") + +if (SCREAM_POSSIBLY_NO_PACK) + set (SCREAM_POSSIBLY_NO_PACK_SIZE 1) +else() + set (SCREAM_POSSIBLY_NO_PACK_SIZE ${SCREAM_PACK_SIZE}) +endif () + +function (check_pack_size master_pack_size pack_size name) + math (EXPR PACK_MODULO "${master_pack_size} % ${pack_size}") + if ((pack_size GREATER master_pack_size) OR (NOT PACK_MODULO EQUAL 0)) + message (FATAL_ERROR "Invalid '${name}' size of ${pack_size}. Needs to be <= ${master_pack_size} and be a factor of it") + endif() +endfunction () + +# Checks on pack sizes relative to the master one: +check_pack_size(${SCREAM_PACK_SIZE} ${SCREAM_SMALL_PACK_SIZE} "small pack") +# This one is an internal check, as the user cannot set SCREAM_POSSIBLY_NO_PACK_SIZE now. +check_pack_size(${SCREAM_PACK_SIZE} ${SCREAM_POSSIBLY_NO_PACK_SIZE} "possibly no pack") + +#################################################################### +# Input-data locations # +#################################################################### + # Handle input root if (SCREAM_MACHINE AND NOT SCREAM_INPUT_ROOT) execute_process(COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/scripts/query-cime ${SCREAM_MACHINE} DIN_LOC_ROOT @@ -274,140 +351,172 @@ set (SCREAM_DATA_DIR ${SCREAM_INPUT_ROOT}/atm/scream CACHE PATH "" FORCE) set (TOPO_DATA_DIR ${SCREAM_INPUT_ROOT}/atm/cam/topo CACHE PATH "" FORCE) set (IOP_DATA_DIR ${SCREAM_INPUT_ROOT}/atm/cam/scam/iop CACHE PATH "" FORCE) -# -# Handle test level -# +#################################################################### +# Tests-related settings # +#################################################################### + +if (NOT SCREAM_LIB_ONLY) + + # Assuming SCREAM_LIB_ONLY is FALSE (else, no exec is built at all), we need to decide + # wether to build baseline-related execs, and wether we are generating baselines or + # comparing against them. These options can help reducing a bit the code that is built + # when generating baselines or when running memory-check tests (no baselines needed there) + option(SCREAM_ONLY_GENERATE_BASELINES "Whether building only baselines-related executables" OFF) + option(SCREAM_ENABLE_BASELINE_TESTS "Whether to run baselines-related tests" ON) + if (SCREAM_ONLY_GENERATE_BASELINES AND NOT SCREAM_ENABLE_BASELINE_TESTS) + message (FATAL_ERROR + "Makes no sense to set SCREAM_ONLY_GENERATE_BASELINES=ON,\n" + "but set SCREAM_ENABLE_BASELINE_TESTS=OFF.") + endif() -# Constants -set(SCREAM_TEST_LEVEL_AT "0") -set(SCREAM_TEST_LEVEL_NIGHTLY "1") -set(SCREAM_TEST_LEVEL_EXPERIMENTAL "2") + set(SCREAM_BASELINES_DIR "UNSET" CACHE PATH "Folder containing baselines data") + if (SCREAM_ENABLE_BASELINE_TESTS) + if (NOT EXISTS ${SCREAM_BASELINES_DIR}/data OR NOT IS_DIRECTORY ${SCREAM_BASELINES_DIR}/data) + string (CONCAT msg + "Error! Baselines tests enabled, but baseline dir is invalid.\n" + " SCREAM_BASELINES_DIR: ${SCREAM_BASELINES_DIR}") + message ("${msg}") + message (FATAL_ERROR "Aborting...") + endif() + endif() -set(SCREAM_TEST_LEVEL "AT" CACHE STRING "The test level to run. Default is AT. NIGHTLY will run additional tests but is not guaranteed to PASS. EXPERIMENTAL will run even more tests with failures being more likely") + # All baselines tests will add a line to the baseline_list file, + # specifyiing the full name of the baseline they generated. + # When test-all-scream has to generate new baselines, it will run + # ctest -L baseline_gen, and then read this file to find out all the + # baseline files to copy into the baseline directory + set (SCREAM_TEST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/data) + file(MAKE_DIRECTORY ${SCREAM_TEST_OUTPUT_DIR}) + file(TOUCH ${SCREAM_TEST_OUTPUT_DIR}/baseline_list) + + set(SCREAM_TEST_LEVEL_AT "0") + set(SCREAM_TEST_LEVEL_NIGHTLY "1") + set(SCREAM_TEST_LEVEL_EXPERIMENTAL "2") + + set(SCREAM_TEST_LEVEL "AT" CACHE STRING "The test level to run. Default is AT. NIGHTLY will run additional tests but is not guaranteed to PASS. EXPERIMENTAL will run even more tests with failures being more likely") + + if (SCREAM_TEST_LEVEL STREQUAL "AT") + set(SCREAM_TEST_LEVEL ${SCREAM_TEST_LEVEL_AT}) + elseif (SCREAM_TEST_LEVEL STREQUAL "NIGHTLY") + set(SCREAM_TEST_LEVEL ${SCREAM_TEST_LEVEL_NIGHTLY}) + elseif (SCREAM_TEST_LEVEL STREQUAL "EXPERIMENTAL") + set(SCREAM_TEST_LEVEL ${SCREAM_TEST_LEVEL_EXPERIMENTAL}) + else() + message(FATAL_ERROR "Unknown SCREAM_TEST_LEVEL '${SCREAM_TEST_LEVEL}'") + endif() -if (SCREAM_TEST_LEVEL STREQUAL "AT") - set(SCREAM_TEST_LEVEL ${SCREAM_TEST_LEVEL_AT}) -elseif (SCREAM_TEST_LEVEL STREQUAL "NIGHTLY") - set(SCREAM_TEST_LEVEL ${SCREAM_TEST_LEVEL_NIGHTLY}) -elseif (SCREAM_TEST_LEVEL STREQUAL "EXPERIMENTAL") - set(SCREAM_TEST_LEVEL ${SCREAM_TEST_LEVEL_EXPERIMENTAL}) -else() - message(FATAL_ERROR "Unknown SCREAM_TEST_LEVEL '${SCREAM_TEST_LEVEL}'") -endif() + set(SCREAM_TEST_MAX_THREADS ${DEFAULT_MAX_THREADS} CACHE STRING "Upper limit on threads per rank for threaded tests") + set(SCREAM_TEST_THREAD_INC 1 CACHE STRING "Thread count increment for threaded tests") + set(SCREAM_TEST_MAX_RANKS ${DEFAULT_MAX_RANKS} CACHE STRING "Upper limit on ranks for mpi tests") + math(EXPR DEFAULT_MAX_TOTAL_THREADS "${SCREAM_TEST_MAX_RANKS}*${SCREAM_TEST_MAX_THREADS}") + set(SCREAM_TEST_MAX_TOTAL_THREADS ${DEFAULT_MAX_TOTAL_THREADS} CACHE STRING "Upper limit on nranks*threads for threaded tests") + + # Make sure SCREAM_TEST_MAX_RANKS and SCREAM_TEST_MAX_THREADS do not individually exceed SCREAM_TEST_MAX_TOTAL_THREADS + if (SCREAM_TEST_MAX_THREADS GREATER ${SCREAM_TEST_MAX_TOTAL_THREADS}) + string(CONCAT msg + "The requested number of max threads/rank (${SCREAM_TEST_MAX_THREADS}) is larger " + "than the max total threads (${SCREAM_TEST_MAX_TOTAL_THREADS}). Setting " + "SCREAM_TEST_MAX_THREADS=${SCREAM_TEST_MAX_TOTAL_THREADS}") + message(STATUS "${msg}") + set (SCREAM_TEST_MAX_THREADS ${SCREAM_TEST_MAX_TOTAL_THREADS}) + endif() + if (SCREAM_TEST_MAX_RANKS GREATER ${SCREAM_TEST_MAX_TOTAL_THREADS}) + string(CONCAT msg + "The requested number of max ranks (${SCREAM_TEST_MAX_RANKS}) is larger " + "than the max total threads (${SCREAM_TEST_MAX_TOTAL_THREADS}). Setting " + "SCREAM_TEST_MAX_RANKS=${SCREAM_TEST_MAX_TOTAL_THREADS}") + message(STATUS "${msg}") + set (SCREAM_TEST_MAX_RANKS ${SCREAM_TEST_MAX_TOTAL_THREADS}) + endif() + # This is a meta-variable, which individual tests can use to set *different* degrees + # of testing, in terms of resolutions. E.g., for SHORT use 3 timesteps, for MEDIUM use 10, + # for LONG use 100. It is *completely* up to the test to decide what short, medium, and long mean. + if (EKAT_ENABLE_COVERAGE OR EKAT_ENABLE_CUDA_MEMCHECK OR EKAT_ENABLE_VALGRIND OR EKAT_ENABLE_COMPUTE_SANITIZER) + set (SCREAM_TEST_SIZE_DEFAULT SHORT) + # also set thread_ing=$max_thread - 1, so we test at most 2 threading configurations + if (SCREAM_TEST_MAX_THREADS GREATER 1) + math (EXPR SCREAM_TEST_THREAD_INC ${SCREAM_TEST_MAX_THREADS}-1) + endif() + else() + set (SCREAM_TEST_SIZE_DEFAULT MEDIUM) + endif() -# Assuming SCREAM_LIB_ONLY is FALSE (else, no exec is built at all), we provide the option -# of building only baseline-related execs. By default, this option is off (menaing "build everything"). -# However, when generating baselines, this can be useful to reduce the amount of stuff compiled. -set(SCREAM_BASELINES_ONLY FALSE CACHE BOOL "Whether building only baselines-related executables") + set(SCREAM_TEST_SIZE ${SCREAM_TEST_SIZE_DEFAULT} CACHE STRING "The kind of testing to perform: SHORT, MEDIUM, LONG. Only applies to certain tests and is generally used to reduce test length when valgrind/cuda-memcheck are on.") + set(SCREAM_TEST_VALID_SIZES "SHORT;MEDIUM;LONG" CACHE INTERNAL "List of valid values for SCREAM_TEST_SIZE") -# Certain builds are not meant to compare against baselines. For instance, a valgrind build -# has the sole purpose of detecting memory errors. For these builds, we can disable baselines tests -set(SCREAM_ENABLE_BASELINE_TESTS TRUE CACHE BOOL "Whether to run baselines-related tests") + if (SCREAM_TEST_SIZE STREQUAL "SHORT") + add_definitions(-DSCREAM_SHORT_TESTS) + endif() -if (SCREAM_BASELINES_ONLY AND NOT SCREAM_ENABLE_BASELINE_TESTS) - message (FATAL_ERROR - "Makes no sense to set SCREAM_BASELINES_ONLY=ON,\n" - "but set SCREAM_ENABLE_BASELINE_TESTS=OFF.") + enable_testing() + include(CTest) endif() -# Set number of vertical levels -set(SCREAM_NUM_VERTICAL_LEV ${DEFAULT_NUM_VERTICAL_LEV} CACHE STRING - "The number of levels used in the vertical grid." -) -option(SCREAM_HAS_LEAP_YEAR "Whether scream uses leap years or not" ON) +#################################################################### +# Configure all tpls and subfolders # +#################################################################### + +if (DEFINED ENV{SCREAM_FAKE_ONLY}) + # We don't really need to build ekat, but we do need to configure the test-launcher + + # Declare some vars that Ekat would have declared, and may be used later + option (EKAT_ENABLE_MPI "Whether EKAT requires MPI." ON) + option (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES "Whether test-launcher should try to manage thread distribution. Requires a ctest resource file to be effective." OFF) + option (EKAT_ENABLE_VALGRIND "Whether to run tests with valgrind" OFF) + set(EKAT_VALGRIND_SUPPRESSION_FILE "" CACHE FILEPATH "Use this valgrind suppression file if valgrind is enabled.") + set (EKAT_ENABLE_GPU False) + if (Kokkos_ENABLE_CUDA OR Kokkos_ENABLE_HIP OR Kokkos_ENABLE_SYCL) + set (EKAT_ENABLE_GPU True) + endif () -## Work out pack sizes. -# Determine the master pack size. -set(SCREAM_PACK_SIZE ${DEFAULT_PACK_SIZE} CACHE STRING - "The number of scalars in a scream::pack::Pack and Mask. Larger packs have good performance on conditional-free loops due to improved caching.") -# With the master pack size determined, we have constraints on the others. -set(DEFAULT_SMALL_PACK_SIZE ${SCREAM_PACK_SIZE}) -# For some routines, SKX may have better performance with pksize=1 -if (Kokkos_ARCH_SKX) - set(DEFAULT_POSSIBLY_NO_PACK TRUE) -endif () -set(SCREAM_SMALL_PACK_SIZE ${DEFAULT_SMALL_PACK_SIZE} CACHE STRING - "The number of scalars in a scream::pack::SmallPack and SmallMask. Smaller packs can have better performance in loops with conditionals since more of the packs will have masks with uniform value.") -set(SCREAM_POSSIBLY_NO_PACK ${DEFAULT_POSSIBLY_NO_PACK} CACHE BOOL - "Set possibly-no-pack to this value. You can set it to something else to restore packs on SKX for testing.") -set (DEFAULT_POSSIBLY_NO_PACK_SIZE ${SCREAM_PACK_SIZE}) + if (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES) + set (TEST_LAUNCHER_MANAGE_RESOURCES True) + else() + set (TEST_LAUNCHER_MANAGE_RESOURCES False) + endif() + if (EKAT_ENABLE_GPU) + set (TEST_LAUNCHER_ON_GPU True) + else() + set (TEST_LAUNCHER_ON_GPU False) + endif() -if (SCREAM_POSSIBLY_NO_PACK) - set (DEFAULT_POSSIBLY_NO_PACK_SIZE 1) -endif () -set (SCREAM_POSSIBLY_NO_PACK_SIZE ${DEFAULT_POSSIBLY_NO_PACK_SIZE}) -# Checks on pack sizes relative to the master one: -check_pack_size(${SCREAM_PACK_SIZE} ${SCREAM_SMALL_PACK_SIZE} "small pack") -# This one is an internal check, as the user cannot set SCREAM_POSSIBLY_NO_PACK_SIZE now. -check_pack_size(${SCREAM_PACK_SIZE} ${SCREAM_POSSIBLY_NO_PACK_SIZE} "possibly no pack") + set (EKAT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../externals/ekat) -## Now we have pack sizes. Proceed with other config options that depend on -## these. + if (EKAT_ENABLE_MPI) + find_package(MPI REQUIRED COMPONENTS C) -if (SCREAM_DEBUG) - set(DEFAULT_FPMODEL "strict") - if (${SCREAM_PACK_SIZE} EQUAL 1 AND NOT ${EAMXX_ENABLE_GPU}) - set(DEFAULT_FPE TRUE) - endif () -endif() -set(SCREAM_FPMODEL ${DEFAULT_FPMODEL} CACHE STRING "Compiler floating point model") -set(SCREAM_FPE ${DEFAULT_FPE} CACHE BOOL "Enable floating point error exception") -### Experimental, under development + + F20TR-SCREAMv1 + 20TR_SCREAM_ELM%SPBC_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV + + + + F2010-SCREAMv1-DP-DYCOMSrf01 + 2010_SCREAM_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx%DYCOMSrf01 + Experimental, under development + + 2016-08-01 2020-01-20 + 1999-07-10 + + + + + + + 864 + + + + + + TRUE + + + + + + TRUE + + + + + + 31.5 + + + + + + 238.5 + + + + + + 225 + + + + + + 1 + + + + + + 225 + + + + + + 1 diff --git a/components/eamxx/cime_config/eamxx_buildnml.py b/components/eamxx/cime_config/eamxx_buildnml.py index db05bd2e20e1..0c7ed8e52aac 100644 --- a/components/eamxx/cime_config/eamxx_buildnml.py +++ b/components/eamxx/cime_config/eamxx_buildnml.py @@ -21,7 +21,7 @@ from utils import ensure_yaml # pylint: disable=no-name-in-module ensure_yaml() import yaml -from yaml_utils import Bools,Ints,Floats,Strings,array_representer +from yaml_utils import Bools,Ints,Floats,Strings,array_representer,array_constructor _CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","cime") sys.path.append(os.path.join(_CIMEROOT, "CIME", "Tools")) @@ -465,7 +465,18 @@ def expand_cime_vars(element, case): child.text = do_cime_vars(child.text, case) ############################################################################### -def _create_raw_xml_file_impl(case, xml): +def write_pretty_xml(filepath, xml): +############################################################################### + with open(filepath, "w") as fd: + # dom has better pretty printing than ET in older python versions < 3.9 + dom = md.parseString(ET.tostring(xml, encoding="unicode")) + pretty_xml = dom.toprettyxml(indent=" ") + pretty_xml = os.linesep.join([s for s in pretty_xml.splitlines() + if s.strip()]) + fd.write(pretty_xml) + +############################################################################### +def _create_raw_xml_file_impl(case, xml, filepath=None): ############################################################################### """ On input, xml contains the parsed content of namelist_defaults_scream.xml. @@ -610,32 +621,40 @@ def _create_raw_xml_file_impl(case, xml): selectors = get_valid_selectors(xml) # 1. Evaluate all selectors - evaluate_selectors(xml, case, selectors) + try: + evaluate_selectors(xml, case, selectors) - # 2. Apply all changes in the SCREAM_ATMCHANGE_BUFFER that may alter - # which atm processes are used - apply_atm_procs_list_changes_from_buffer (case,xml) + # 2. Apply all changes in the SCREAM_ATMCHANGE_BUFFER that may alter + # which atm processes are used + apply_atm_procs_list_changes_from_buffer (case,xml) - # 3. Resolve all inheritances - resolve_all_inheritances(xml) + # 3. Resolve all inheritances + resolve_all_inheritances(xml) - # 4. Expand any CIME var that appears inside XML nodes text - expand_cime_vars(xml,case) + # 4. Expand any CIME var that appears inside XML nodes text + expand_cime_vars(xml,case) - # 5. Grab the atmosphere_processes macro list, with all the defaults - atm_procs_defaults = get_child(xml,"atmosphere_processes_defaults",remove=True) + # 5. Grab the atmosphere_processes macro list, with all the defaults + atm_procs_defaults = get_child(xml,"atmosphere_processes_defaults",remove=True) - # 6. Get atm procs list - atm_procs_list = get_child(atm_procs_defaults,"atm_procs_list",remove=True) + # 6. Get atm procs list + atm_procs_list = get_child(atm_procs_defaults,"atm_procs_list",remove=True) - # 7. Form the nested list of atm procs needed, append to atmosphere_driver section - atm_procs = gen_atm_proc_group(atm_procs_list.text, atm_procs_defaults) - atm_procs.tag = "atmosphere_processes" - xml.append(atm_procs) + # 7. Form the nested list of atm procs needed, append to atmosphere_driver section + atm_procs = gen_atm_proc_group(atm_procs_list.text, atm_procs_defaults) + atm_procs.tag = "atmosphere_processes" + xml.append(atm_procs) - # 8. Apply all changes in the SCREAM_ATMCHANGE_BUFFER that do not alter - # which atm processes are used - apply_non_atm_procs_list_changes_from_buffer (case,xml) + # 8. Apply all changes in the SCREAM_ATMCHANGE_BUFFER that do not alter + # which atm processes are used + apply_non_atm_procs_list_changes_from_buffer (case,xml) + except BaseException as e: + if filepath is not None: + dbg_xml_path = filepath.replace(".xml", ".dbg.xml") + write_pretty_xml(dbg_xml_path, xml) + print(f"Error during XML creation, writing {dbg_xml_path}") + + raise e perform_consistency_checks (case, xml) @@ -666,17 +685,11 @@ def create_raw_xml_file(case, caseroot): # be processed early by treating them as if they were made to the defaults file. with open(src, "r") as fd: defaults = ET.parse(fd).getroot() - raw_xml = _create_raw_xml_file_impl(case, defaults) + raw_xml = _create_raw_xml_file_impl(case, defaults, filepath=raw_xml_file) check_all_values(raw_xml) - with open(raw_xml_file, "w") as fd: - # dom has better pretty printing than ET in older python versions < 3.9 - dom = md.parseString(ET.tostring(raw_xml, encoding="unicode")) - pretty_xml = dom.toprettyxml(indent=" ") - pretty_xml = os.linesep.join([s for s in pretty_xml.splitlines() - if s.strip()]) - fd.write(pretty_xml) + write_pretty_xml(raw_xml_file, raw_xml) ############################################################################### def convert_to_dict(element): @@ -877,6 +890,11 @@ def get_file_parameters(caseroot): result = [] for item in raw_xml.findall('.//*[@type="file"]'): + # Certain configurations may not need a file (e.g., a remap + # file for SPA may not be needed if the model resolution + # matches the data file resolution + if item.text is None or item.text=="": + continue result.append(item.text.strip()) for item in raw_xml.findall('.//*[@type="array(file)"]'): @@ -886,7 +904,7 @@ def get_file_parameters(caseroot): return list(OrderedDict.fromkeys(result)) ############################################################################### -def create_input_data_list_file(caseroot): +def create_input_data_list_file(case,caseroot): ############################################################################### """ Create the scream.input_data_list file for this case. This will tell CIME @@ -894,18 +912,51 @@ def create_input_data_list_file(caseroot): """ files_to_download = get_file_parameters(caseroot) + # Add array parsing knowledge to yaml loader + loader = yaml.SafeLoader + loader.add_constructor("!bools",array_constructor) + loader.add_constructor("!ints",array_constructor) + loader.add_constructor("!floats",array_constructor) + loader.add_constructor("!strings",array_constructor) + + # Grab all the output yaml files, open them, and check if horiz_remap_file or vertical_remap_file is used + rundir = case.get_value("RUNDIR") + eamxx_xml_file = os.path.join(caseroot, "namelist_scream.xml") + with open(eamxx_xml_file, "r") as fd: + eamxx_xml = ET.parse(fd).getroot() + + scorpio = get_child(eamxx_xml,'Scorpio') + out_files_xml = get_child(scorpio,"output_yaml_files",must_exist=False) + # out_files = out_files_xml.text.split(",") if (out_files_xml is not None and out_files_xml.text is not None) else [] + # for fn in out_files: + if (out_files_xml is not None and out_files_xml.text is not None): + for fn in out_files_xml.text.split(","): + # Get full name + src_yaml = os.path.expanduser(os.path.join(fn.strip())) + dst_yaml = os.path.expanduser(os.path.join(rundir,'data',os.path.basename(src_yaml))) + + # Load file, and look for the remap file entries + content = yaml.load(open(dst_yaml,"r"),Loader=loader) + if 'horiz_remap_file' in content.keys(): + files_to_download += [content['horiz_remap_file']] + if 'vertical_remap_file' in content.keys(): + files_to_download += [content['vertical_remap_file']] + input_data_list_file = "{}/Buildconf/scream.input_data_list".format(caseroot) if os.path.exists(input_data_list_file): os.remove(input_data_list_file) + din_loc_root = case.get_value("DIN_LOC_ROOT") with open(input_data_list_file, "w") as fd: - for idx, file_path in enumerate(files_to_download): - fd.write("scream_dl_input_{} = {}\n".format(idx, file_path)) + for idx, file_path in enumerate(list(set(files_to_download))): + # Only add files whose full path starts with the CIME's input data location + if file_path.startswith(din_loc_root): + fd.write("scream_dl_input_{} = {}\n".format(idx, file_path)) + ############################################################################### def do_cime_vars_on_yaml_output_files(case, caseroot): ############################################################################### - from yaml_utils import array_constructor rundir = case.get_value("RUNDIR") eamxx_xml_file = os.path.join(caseroot, "namelist_scream.xml") @@ -956,14 +1007,15 @@ def do_cime_vars_on_yaml_output_files(case, caseroot): # produces an output at t=0, which is not present in the restarted run, and # which also causes different timestamp in the file name. # Hence, change default output settings to perform a single AVERAGE step at the end of the run - if case.get_value("TESTCASE") in ["ERP", "ERS"]: - test_env = case.get_env('test') - stop_n = int(test_env.get_value("STOP_N")) - stop_opt = test_env.get_value("STOP_OPTION") - content['output_control']['Frequency'] = stop_n - content['output_control']['frequency_units'] = stop_opt - content['Averaging Type'] = 'AVERAGE' - print ("WARNING: ERS/ERP tests hard code output to consist of a single AVERAGE output step at the end of the run.") + if case.get_value("TESTCASE") in ["ERP", "ERS"] and content['Averaging Type'].upper()=="INSTANT": + hist_n = int(case.get_value("HIST_N",resolved=True)) + hist_opt = case.get_value("HIST_OPTION",resolved=True) + content['output_control']['Frequency'] = hist_n + content['output_control']['frequency_units'] = hist_opt + content['output_control']['skip_t0_output'] = True + print ("ERS/ERP test with INSTANT output detected. Adjusting output control specs:\n") + print (" - setting skip_t0_output=true\n") + print (" - setting freq and freq_units to HIST_N and HIST_OPTION respectively\n") ordered_dump(content, open(dst_yaml, "w")) diff --git a/components/eamxx/cime_config/eamxx_buildnml_impl.py b/components/eamxx/cime_config/eamxx_buildnml_impl.py index 2fa00b7f4a4b..e0ecab146246 100644 --- a/components/eamxx/cime_config/eamxx_buildnml_impl.py +++ b/components/eamxx/cime_config/eamxx_buildnml_impl.py @@ -162,11 +162,12 @@ def refine_type(entry, force_type=None): >>> e = '1.0' >>> refine_type(e,force_type='my_type') Traceback (most recent call last): - NameError: ERROR: Invalid/unsupported force type 'my_type' + CIME.utils.CIMEError: ERROR: Invalid/unsupported force type 'my_type' >>> e = 'true,falsE' >>> refine_type(e,'logical') Traceback (most recent call last): - ValueError: Could not refine 'true,falsE' as type 'logical' + CIME.utils.CIMEError: ERROR: Could not refine 'true,falsE' as type 'logical': + ERROR: For entry of type 'logical', expected 'true' or 'false', got 'true,falsE' >>> refine_type(e,'array(logical)') [True, False] >>> refine_type('', 'array(string)') @@ -176,15 +177,14 @@ def refine_type(entry, force_type=None): >>> refine_type(None, 'array(real)') [] """ - - # If force type is unspecified, try to deduce it + # If force type is unspecified, try to deduce it if force_type is None: expect (entry is not None, "If an entry is None, you must specify the force_type") else: elem_valid = ["logical","integer","real","string","file"] valid = elem_valid + ["array("+e+")" for e in elem_valid] - expect (force_type in valid, exc_type=NameError, + expect (force_type in valid, error_msg=f"Invalid/unsupported force type '{force_type}'") if is_array_type(force_type): @@ -208,7 +208,8 @@ def refine_type(entry, force_type=None): elif entry.upper() == "FALSE": return False else: - return bool(int(entry)) + expect(False, f"For entry of type 'logical', expected 'true' or 'false', got '{entry}'", + exc_type=ValueError) elif elem_type == "integer": tmp = float(entry) @@ -220,7 +221,7 @@ def refine_type(entry, force_type=None): return str(entry) except ValueError as e: - raise ValueError (f"Could not refine '{entry}' as type '{force_type}'") from e + expect(False, f"Could not refine '{entry}' as type '{force_type}':\n{e}") # No force type provided. Try to infer from value if entry.upper() == "TRUE": @@ -273,7 +274,7 @@ def derive_type(entry): elif isinstance(elem_value, str): elem_type = "string" else: - raise RuntimeError("Couldn't derive type of '{}'".format(entry)) + expect(False, "Couldn't derive type of '{}'".format(entry)) if isinstance(refined_value,list): return "array(" + elem_type + ")" @@ -293,7 +294,8 @@ def check_value(elem, value): >>> root = ET.fromstring(xml) >>> check_value(root,'1.5') Traceback (most recent call last): - ValueError: Could not refine '1.5' as type 'integer' + CIME.utils.CIMEError: ERROR: Could not refine '1.5' as type 'integer': + ERROR: Cannot interpret 1.5 as int >>> check_value(root,'3') Traceback (most recent call last): CIME.utils.CIMEError: ERROR: Invalid value '3' for element 'a'. Value not in the valid list ('[1, 2]') diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 71c1bfd4f828..07ef2793f38c 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -48,7 +48,7 @@ be lost if SCREAM_HACK_XML is not enabled. ctl_nl - driver_options,atmosphere_processes,grids_manager,initial_conditions,Scorpio,e3sm_parameters + driver_options,iop_options,atmosphere_processes,grids_manager,initial_conditions,Scorpio,e3sm_parameters @@ -119,9 +119,8 @@ be lost if SCREAM_HACK_XML is not enabled. for the atmosphere processes section(s). 11) The attribute 'locked="true"' is to be used for entries that cannot be changed - via atmchange (see scripts/atmchange). For instance, the overall list of atm procs cannot be changed, - since it would require to re-parse the defaults, to re-generate the correct - defaults for the (possibly) new atm procs. + via atmchange (see scripts/atmchange). If an element is locked, then all children + will be locked as well. 12) The attribute 'constraints' allows to specify constraints on values. Valid constraints are lt, le, ne, gt, ge, and mod. Except the latter (which has slightly different syntax, @@ -203,6 +202,18 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/tables/vn_table_vals.dat8, ${DIN_LOC_ROOT}/atm/scream/tables/vm_table_vals.dat8 + 1350.0 + 1.0 + 1.0 + 67.0 + 0.5 + 1.0 + 50.0 + 900.0 + 0.65 + 0.304 + 1.0 + 0.00028 @@ -226,20 +237,38 @@ be lost if SCREAM_HACK_XML is not enabled. + + + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/mam4_mode1_rrtmg_aeronetdust_c20240206.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/mam4_mode2_rrtmg_c20240206.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/mam4_mode3_rrtmg_aeronetdust_c20240206.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/mam4_mode4_rrtmg_c20240206.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/water_refindex_rrtmg_c20240206.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/ocphi_rrtmg_c20240206.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/dust_aeronet_rrtmg_c20240206.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/ssam_rrtmg_c20240206.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/sulfate_rrtmg_c20240206.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/ocpho_rrtmg_c20240206.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/bcpho_rrtmg_c20240206.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/poly_rrtmg_c20240206.nc + + - + 0 false - false + TIME_DEPENDENT_3D_PROFILE - + - UNSET - ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30_to_ne4_mono_20220502.nc - ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30np4_to_ne4pg2_mono.20220714.nc - none - ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30np4_to_ne30pg2_mono.20220714.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30np4_to_ne120np4_mono_20220502.nc - ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30np4_to_ne512np4_mono_20220506.nc - ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30np4_to_ne120pg2_intbilin_20221012.nc - ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30np4_to_ne256pg2_intbilin_20221011.nc - ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30np4_to_ne512pg2_intbilin_20221012.nc - ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30np4_to_ne1024pg2_intbilin_20221012.nc - - ${DIN_LOC_ROOT}/atm/scream/init/spa_file_unified_and_complete_ne30_20220428.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne120pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne256pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne512pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne1024pg2_20231201.nc + + + + ${DIN_LOC_ROOT}/atm/scream/init/spa_file_unified_and_complete_ne30_20220428.nc + ${DIN_LOC_ROOT}/atm/scream/init/spa_file_unified_and_complete_ne30pg2_20240111.nc + ${DIN_LOC_ROOT}/atm/scream/init/spa_file_unified_and_complete_ne4_20220428.nc + ${DIN_LOC_ROOT}/atm/scream/init/spa_file_unified_and_complete_ne4pg2_20231222.nc @@ -327,6 +356,7 @@ be lost if SCREAM_HACK_XML is not enabled. 3 3 3 + 3 4 true false @@ -343,6 +373,9 @@ be lost if SCREAM_HACK_XML is not enabled. > false + + true + @@ -356,11 +389,15 @@ be lost if SCREAM_HACK_XML is not enabled. 6 2 1 + 1 5 - 10 + 10 + 1 + 1 + 1 1 hours @@ -397,10 +434,8 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/init/screami_ne256np4L128_ifs-20200120_20220914.nc ${DIN_LOC_ROOT}/atm/scream/init/screami_ne512np4L128_20220823.nc ${DIN_LOC_ROOT}/atm/scream/init/screami_ne1024np4L128_era5-20131001-topoadj-16x_20220914.nc - ${DIN_LOC_ROOT}/atm/scream/init/screami_ne1024np4L128_ifs-20160801-topoadjx6t_20221011.nc ${DIN_LOC_ROOT}/atm/scream/init/screami_ne1024np4L128_ifs-20200120-topoadjx6t_20221011.nc ${DIN_LOC_ROOT}/atm/scream/init/screami_aquaplanet_ne4np4L72_20220823.nc - ${DIN_LOC_ROOT}/atm/scream/init/screami_aquaplanet_ne30np4L128_20220823.nc ${DIN_LOC_ROOT}/atm/scream/init/screami_conusx4v1np4L72-topo12x_013023.nc @@ -453,11 +488,19 @@ be lost if SCREAM_HACK_XML is not enabled. 0.0 0.0 0.0 + + + + T_mid + false + 0 + 0.001 + 900.0 - ${SRCROOT}/components/eamxx/data/scream_default_output.yaml + ./${CASE}.scream @@ -475,12 +518,19 @@ be lost if SCREAM_HACK_XML is not enabled. doc="Verbosity level for the atm logger"> info + + warn + false 1e-10 1e-14 Warning true phis,landfrac + false + true @@ -488,11 +538,28 @@ be lost if SCREAM_HACK_XML is not enabled. ${CASE} + + + true + UNSET + ${DIN_LOC_ROOT}/atm/cam/scam/iop/DYCOMSrf01_iopfile_4scam.nc + -999 + 31.5 + -999 + 238.5 + false + true + false + true + + 0 + 2 False 2 + 1 1 6 0 @@ -506,6 +573,7 @@ be lost if SCREAM_HACK_XML is not enabled. 1 1 3.4e-08 + 0.216784 UNSET 250000.0 250000.0 @@ -513,6 +581,7 @@ be lost if SCREAM_HACK_XML is not enabled. 4.0e4 2.0e4 1.0e4 + 1.0e4 100000.0 1 0 @@ -522,6 +591,7 @@ be lost if SCREAM_HACK_XML is not enabled. 0 0 sphere + plane 9 UNSET 4 @@ -530,19 +600,28 @@ be lost if SCREAM_HACK_XML is not enabled. 256 512 1024 + 1024 0 0 + 5 0 + 5 + 0 + 50000 + 0 + 50000 -1 4 cube - UNSET + plane + UNSET 600 300 75 33.33333333333 16.6666666666666 8.3333333333333 + 8.3333333333333 75 9999 1 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands index d3a4a39b668a..cf3ca97f6dd3 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands @@ -1,2 +1,2 @@ $CIMEROOT/../components/eamxx/scripts/atmchange --all internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b -./xmlchange POSTRUN_SCRIPT="$CIMEROOT/../components/eamxx/tests/postrun/check_hashes_ers.py" +./xmlchange POSTRUN_SCRIPT="$CIMEROOT/../components/eamxx/scripts/check-hashes-ers" diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/optics/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/optics/shell_commands new file mode 100644 index 000000000000..1c22bd9ee454 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/optics/shell_commands @@ -0,0 +1,9 @@ + +#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer +#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options +./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" + +$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b +$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,mam4_optics,rrtmgp" -b + + diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/README b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/README new file mode 100644 index 000000000000..5d33c71fed37 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/README @@ -0,0 +1,9 @@ +The testmods in this folder contain different configuration +of output to request to eamxx. All the presets create a single +output file, all with a different name, which means one can +use 2+ presets at the same time. + + +Each preset does basically two things: + 1. create a yaml file in the case folder + 2. add the yaml file to the yaml_output_files xml setting of eamxx diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/diags/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/diags/shell_commands new file mode 100644 index 000000000000..904f46b580f2 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/diags/shell_commands @@ -0,0 +1,37 @@ +# This script generates a (single) yaml file for EAMxx output. +# The output will be INSTANT, with only some diags fields as ouput + +CASEROOT=$(./xmlquery --value CASEROOT) +CASE=$(./xmlquery --value CASE) + +# Scripts location +YAML_EDIT_SCRIPT=$CIMEROOT/../components/eamxx/scripts/edit-output-stream +ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange +YAML_FILE=$CASEROOT/eamxx_diags_output.yaml + +# Figure out the suffix for the physics grid +ATM_GRID=$(./xmlquery --value ATM_GRID) +if [[ $ATM_GRID == *"pg2"* ]]; then + PGTYPE="PG2" +else + PGTYPE="GLL" +fi + +# List of output fields +FIELDS='Exner LiqWaterPath dz geopotential_int PotentialTemperature' +FIELDS+=' precip_liq_surf_mass_flux wind_speed ShortwaveCloudForcing' +FIELDS+=' T_mid_at_model_bot T_mid_at_900hPa' +FIELDS+=' horiz_winds_at_100m_above_surface horiz_winds_at_100m_above_sealevel' + +# Generate the file +$YAML_EDIT_SCRIPT -g \ + -f $YAML_FILE \ + --avg-type INSTANT \ + --freq HIST_N \ + --freq-units HIST_OPTION \ + --prefix ${CASE}.scream.diags.hi \ + --grid "Physics ${PGTYPE}" \ + --fields ${FIELDS} + +# Add this output yaml file to the list of eamxx output streams +$ATMCHANGE output_yaml_files+=$YAML_FILE -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/hremap_to_ne4/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/hremap_to_ne4/shell_commands new file mode 100644 index 000000000000..99289d5e411b --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/hremap_to_ne4/shell_commands @@ -0,0 +1,15 @@ +# Look for all the eamxx_***_output.yaml files in the case folder and +# sets horiz remap if atm grid is ne30pg2 + +CASEROOT=$(./xmlquery --value CASEROOT) +YAML_EDIT_SCRIPT=$CIMEROOT/../components/eamxx/scripts/edit-output-stream +ATM_GRID=$(./xmlquery --value ATM_GRID) + +if [[ $ATM_GRID = "ne30np4.pg2" ]];then + YAML_FILES=$(ls -1 | grep 'eamxx_.*_output.yaml') + for fname in ${YAML_FILES}; do + $YAML_EDIT_SCRIPT -f $fname --horiz-remap-file \${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne4pg2_20231201.nc + done +else + echo "Note: testmod 'hremap_to_ne4' only works for ne30pg2 atm grid" +fi diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys/shell_commands new file mode 100644 index 000000000000..116cdf111b43 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys/shell_commands @@ -0,0 +1,34 @@ +# This script generates a (single) yaml file for EAMxx output. +# The output will be INSTANT, with only a few state vars + +CASEROOT=$(./xmlquery --value CASEROOT) +CASE=$(./xmlquery --value CASE) + +# Scripts location +YAML_EDIT_SCRIPT=$CIMEROOT/../components/eamxx/scripts/edit-output-stream +ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange +YAML_FILE=$CASEROOT/eamxx_phys_output.yaml + +# Figure out the suffix for the physics grid +ATM_GRID=$(./xmlquery --value ATM_GRID) +if [[ $ATM_GRID == *"pg2"* ]]; then + PGTYPE="PG2" +else + PGTYPE="GLL" +fi + +# List of output fields +FIELDS='horiz_winds T_mid tracers pseudo_density p_mid p_int' + +# Generate the file +$YAML_EDIT_SCRIPT -g \ + -f $YAML_FILE \ + --avg-type INSTANT \ + --freq HIST_N \ + --freq-units HIST_OPTION \ + --prefix ${CASE}.scream.phys.hi \ + --grid "Physics ${PGTYPE}" \ + --fields ${FIELDS} + +# Add this output yaml file to the list of eamxx output streams +$ATMCHANGE output_yaml_files+=$YAML_FILE -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys_dyn/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys_dyn/shell_commands new file mode 100644 index 000000000000..0952b4afe3f9 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys_dyn/shell_commands @@ -0,0 +1,41 @@ +# This script generates a (single) yaml file for EAMxx output. +# The output will be INSTANT, with only dyn state vars + +CASEROOT=$(./xmlquery --value CASEROOT) +CASE=$(./xmlquery --value CASE) + +# Scripts location +YAML_EDIT_SCRIPT=$CIMEROOT/../components/eamxx/scripts/edit-output-stream +ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange +YAML_FILE=$CASEROOT/eamxx_dyn_output.yaml + +# Figure out the suffix for the physics grid +ATM_GRID=$(./xmlquery --value ATM_GRID) +if [[ $ATM_GRID == *"pg2"* ]]; then + PGTYPE="PG2" +else + PGTYPE="GLL" +fi + +# List of output fields +FIELDS='v_dyn vtheta_dp_dyn dp3d_dyn w_int_dyn phis_dyn phi_int_dyn ps_dyn omega_dyn Qdp_dyn' + +# Generate the file +$YAML_EDIT_SCRIPT -g \ + -f $YAML_FILE \ + --avg-type INSTANT \ + --freq HIST_N \ + --freq-units HIST_OPTION \ + --prefix ${CASE}.scream.phys_dyn.hi \ + --grid Dynamics \ + --io-grid 'Physics GLL' \ + --fields ${FIELDS} + +# Add also a couple of fields on the phys grid, to trigger 2-grid in same stream +$YAML_EDIT_SCRIPT \ + -f $YAML_FILE \ + --grid "Physics ${PGTYPE}" \ + --fields T_mid horiz_winds + +# Add this output yaml file to the list of eamxx output streams +$ATMCHANGE output_yaml_files+=$YAML_FILE -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/1/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/1/shell_commands new file mode 100644 index 000000000000..74bd2a4d0213 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/1/shell_commands @@ -0,0 +1,9 @@ +# This preset uses the three output streams (phys_dyn, phys, and diags) +# It does not add remap, and uses INSTANT output + +SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/output + +# Add the three streams +. $SCRIPTS_DIR/phys/shell_commands +. $SCRIPTS_DIR/phys_dyn/shell_commands +. $SCRIPTS_DIR/diags/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/2/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/2/shell_commands new file mode 100644 index 000000000000..aea7feb5bbd6 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/2/shell_commands @@ -0,0 +1,12 @@ +# This preset uses the three output streams (phys_dyn, phys, and diags) +# It does not add remap, and uses AVERAGE output + +SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/output + +# Add the three streams +. $SCRIPTS_DIR/phys/shell_commands +. $SCRIPTS_DIR/phys_dyn/shell_commands +. $SCRIPTS_DIR/diags/shell_commands + +# Change avg-type to AVERAGE for all streams +. $SCRIPTS_DIR/set_avg/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/3/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/3/shell_commands new file mode 100644 index 000000000000..da1d02dc5de1 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/3/shell_commands @@ -0,0 +1,11 @@ +# This preset uses the three output streams (phys, and diags) +# It adds horiz remap, and uses INSTANT output + +SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/output + +# Add the phys/diags streams (cannot add phys_dyn, b/c we use horiz remap) +. $SCRIPTS_DIR/phys/shell_commands +. $SCRIPTS_DIR/diags/shell_commands + +# Add horiz remap +. $SCRIPTS_DIR/hremap_to_ne4/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/4/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/4/shell_commands new file mode 100644 index 000000000000..521ada7f5082 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/4/shell_commands @@ -0,0 +1,14 @@ +# This preset uses the three output streams (phys, and diags) +# It adds horizontal remap, and uses AVERAGE output + +SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/output + +# Add the phys/diags streams (cannot add phys_dyn, b/c we use horiz remap) +. $SCRIPTS_DIR/phys/shell_commands +. $SCRIPTS_DIR/diags/shell_commands + +# Add horiz remap +. $SCRIPTS_DIR/hremap_to_ne4/shell_commands + +# Use AVERAGE +. $SCRIPTS_DIR/set_avg/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/5/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/5/shell_commands new file mode 100644 index 000000000000..bd60a0472a87 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/5/shell_commands @@ -0,0 +1,15 @@ +# This preset uses the three output streams (phys_dyn, phys, and diags) +# It adds vertical remap, and uses AVERAGE output + +SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/output + +# Add the phys/dyn/diags streams +. $SCRIPTS_DIR/phys/shell_commands +. $SCRIPTS_DIR/phys_dyn/shell_commands +. $SCRIPTS_DIR/diags/shell_commands + +# Add vertical remap +. $SCRIPTS_DIR/vremap/shell_commands + +# Use AVERAGE +. $SCRIPTS_DIR/set_avg/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/6/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/6/shell_commands new file mode 100644 index 000000000000..937d16468516 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/6/shell_commands @@ -0,0 +1,17 @@ +# This preset uses the three output streams (phys, and diags) +# It adds horizontal and remap, and uses AVERAGE output + +SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/output + +# Add the phys/diags streams (cannot add phys_dyn, b/c we use horiz remap) +. $SCRIPTS_DIR/phys/shell_commands +. $SCRIPTS_DIR/diags/shell_commands + +# Add horiz remap +. $SCRIPTS_DIR/hremap_to_ne4/shell_commands + +# Add vert remap +. $SCRIPTS_DIR/vremap/shell_commands + +# Use AVERAGE +. $SCRIPTS_DIR/set_avg/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/set_avg/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/set_avg/shell_commands new file mode 100644 index 000000000000..dc58bc6301a7 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/set_avg/shell_commands @@ -0,0 +1,10 @@ +# Look for all the eamxx_***_output.yaml files in the case folder and +# change the avg type to average. + +CASEROOT=$(./xmlquery --value CASEROOT) +YAML_EDIT_SCRIPT=$CIMEROOT/../components/eamxx/scripts/edit-output-stream + +YAML_FILES=$(ls -1 | grep 'eamxx_.*_output.yaml') +for fname in ${YAML_FILES}; do + $YAML_EDIT_SCRIPT -f $fname --avg-type average +done diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/vremap/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/vremap/shell_commands new file mode 100644 index 000000000000..151928669d1a --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/vremap/shell_commands @@ -0,0 +1,10 @@ +# Look for all the eamxx_***_output.yaml files in the case folder and +# sets vertical remap + +CASEROOT=$(./xmlquery --value CASEROOT) +YAML_EDIT_SCRIPT=$CIMEROOT/../components/eamxx/scripts/edit-output-stream + +YAML_FILES=$(ls -1 | grep 'eamxx_.*_output.yaml') +for fname in ${YAML_FILES}; do + $YAML_EDIT_SCRIPT -f $fname --vertical-remap-file \${DIN_LOC_ROOT}/atm/scream/maps/vrt_remapping_p_levs_20230926.nc +done diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/perf_test/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/perf_test/shell_commands new file mode 100644 index 000000000000..d4e7c2a95377 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/perf_test/shell_commands @@ -0,0 +1,5 @@ + +# Force us to use 1 node to eliminate network noise +if [ `./xmlquery --value MACH` == frontier-scream-gpu ]; then + ./xmlchange NTASKS=8 +fi diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands index 04989a22796a..e6773dce4199 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands @@ -1,7 +1,2 @@ ./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_SMALL_KERNELS On' $CIMEROOT/../components/eamxx/scripts/atmchange --all internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b - -f=$(./xmlquery --value MACH) -if [ $f == chrysalis ]; then - ./xmlchange BATCH_COMMAND_FLAGS="--time 00:30:00 -p debug --account e3sm --exclude=chr-0512" -fi diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/spa_remap/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/spa_remap/shell_commands new file mode 100644 index 000000000000..5a07fb378464 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/spa_remap/shell_commands @@ -0,0 +1,2 @@ +$CIMEROOT/../components/eamxx/scripts/atmchange -b spa_data_file='${DIN_LOC_ROOT}'/atm/scream/init/spa_file_unified_and_complete_ne4pg2_20231222.nc +$CIMEROOT/../components/eamxx/scripts/atmchange -b spa_remap_file='${DIN_LOC_ROOT}'/atm/scream/maps/map_ne4pg2_to_ne30pg2_20231201.nc diff --git a/components/eamxx/cime_config/tests/eamxx_default_files.py b/components/eamxx/cime_config/tests/eamxx_default_files.py new file mode 100644 index 000000000000..b39d2d5c1553 --- /dev/null +++ b/components/eamxx/cime_config/tests/eamxx_default_files.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 + +import os +import http +import pathlib +import unittest +import urllib.request +import xml.etree.ElementTree as ET + + +class testNamelistDefaultsScream(unittest.TestCase): + def setUp(self): + """ + Set up the environment for the test by setting the DIN_LOC_ROOT + environment variable. Parse the 'namelist_defaults_scream.xml' + file and extract the files of interest based on the DIN_LOC_ROOT + variable or the array(file) type. Assign the extracted files + to the 'my_files' attribute of the test instance. + """ + + os.environ["DIN_LOC_ROOT"] = "https://web.lcrc.anl.gov/public/e3sm/inputdata/" + + scream_defaults_path = pathlib.Path(__file__) + tree = ET.parse(f"{scream_defaults_path.parent.parent}/namelist_defaults_scream.xml") + root = tree.getroot() + + files_of_interest = [ + child.text for child in root.findall(".//") + if child.text and child.text.startswith("${DIN_LOC_ROOT}") + ] + + more_files_of_interest = [ + child.text for child in root.findall(".//") + if child.text and "type" in child.attrib.keys() and child.attrib["type"]=="array(file)" + ] + + files_of_interest.extend( + text.strip() for text_list in more_files_of_interest for text in text_list.split(",") + if text.strip().startswith("${DIN_LOC_ROOT}") + ) + + self.my_files = [ + file.replace("${DIN_LOC_ROOT}/", "") + for file in files_of_interest + ] + + self.my_lines = [] + with open( + f"{scream_defaults_path.parent.parent}/namelist_defaults_scream.xml", + "r" + ) as the_file: + for a_line in the_file: + self.my_lines.append(a_line) + + def test_ascii_lines(self): + """ + Test that all lines are ASCII + """ + + for i_line, a_line in enumerate(self.my_lines): + with self.subTest(i_line=i_line): + self.assertTrue( + a_line.isascii(), + msg=f"\nERROR! This line is not ASCII!\n{a_line}" + ) + + def test_opening_files(self): + """ + Test the opening of files from the inputdata server. + """ + + for i_file in range(len(self.my_files)): + with self.subTest(i_file=i_file): + try: + request_return = urllib.request.urlopen( + f"{os.environ['DIN_LOC_ROOT']}{self.my_files[i_file]}" + ) + self.assertIsInstance(request_return, http.client.HTTPResponse) + except urllib.error.HTTPError: + file_name = f"{os.environ['DIN_LOC_ROOT']}{self.my_files[i_file]}" + self.assertTrue( + False, + msg=f"\nERROR! This file doesn't exist!\n{file_name}" + ) + + def test_expected_fail(self): + """ + Test an expected failure by manipulating the file name. + """ + + with self.assertRaises(urllib.error.HTTPError): + some_phony_file = f"{self.my_files[5][:-5]}some_phony_file.nc" + urllib.request.urlopen( + f"{os.environ['DIN_LOC_ROOT']}{some_phony_file}" + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/components/eamxx/cmake/machine-files/lassen.cmake b/components/eamxx/cmake/machine-files/lassen.cmake deleted file mode 100644 index 36b69c7f0253..000000000000 --- a/components/eamxx/cmake/machine-files/lassen.cmake +++ /dev/null @@ -1,9 +0,0 @@ -include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) -common_setup() - -set(NetCDF_PATH /usr/gdata/climdat/netcdf CACHE STRING "") -set(NetCDF_Fortran_PATH /usr/gdata/climdat/netcdf CACHE STRING "") -set(LAPACK_LIBRARIES /usr/lib64/liblapack.so CACHE STRING "") -set(CMAKE_CXX_FLAGS "-DTHRUST_IGNORE_CUB_VERSION_CHECK" CACHE STRING "" FORCE) - -set(SCREAM_INPUT_ROOT "/usr/gdata/climdat/ccsm3data/inputdata/" CACHE STRING "") diff --git a/components/eamxx/cmake/machine-files/mappy.cmake b/components/eamxx/cmake/machine-files/mappy.cmake index 7c1fc8cf25ea..29fb2e74b8a1 100644 --- a/components/eamxx/cmake/machine-files/mappy.cmake +++ b/components/eamxx/cmake/machine-files/mappy.cmake @@ -1,3 +1,5 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() -set(PYTHON_EXECUTABLE "/ascldap/users/jgfouca/packages/Python-3.8.5/bin/python3.8" CACHE STRING "" FORCE) \ No newline at end of file +set(PYTHON_EXECUTABLE "/ascldap/users/jgfouca/packages/Python-3.8.5/bin/python3.8" CACHE STRING "" FORCE) + +set(CMAKE_Fortran_FLAGS "-fallow-argument-mismatch" CACHE STRING "" FORCE) diff --git a/components/eamxx/cmake/machine-files/quartz-intel.cmake b/components/eamxx/cmake/machine-files/quartz-intel.cmake index 753c782702db..defd8cbb2d33 100644 --- a/components/eamxx/cmake/machine-files/quartz-intel.cmake +++ b/components/eamxx/cmake/machine-files/quartz-intel.cmake @@ -4,4 +4,4 @@ set(PYTHON_EXECUTABLE "/usr/tce/packages/python/python-3.9.12/bin/python3" CACHE set(PYTHON_LIBRARIES "/usr/lib64/libpython3.9.so.1.0" CACHE STRING "" FORCE) option (SCREAM_ENABLE_ML_CORRECTION "Whether to enable ML correction parametrization" ON) set(HDF5_DISABLE_VERSION_CHECK 1 CACHE STRING "" FORCE) -execute_process(COMMAND source /usr/WS1/climdat/python_venv/3.9.2/screamML/bin/activate) +execute_process(COMMAND source /usr/WS1/e3sm/python_venv/3.9.2/screamML/bin/activate) diff --git a/components/eamxx/cmake/machine-files/quartz.cmake b/components/eamxx/cmake/machine-files/quartz.cmake index ee9a3dcbffd3..e4b4fcbd8a57 100644 --- a/components/eamxx/cmake/machine-files/quartz.cmake +++ b/components/eamxx/cmake/machine-files/quartz.cmake @@ -16,4 +16,4 @@ elseif ("${COMPILER}" STREQUAL "gnu") set(CMAKE_EXE_LINKER_FLAGS "-L/usr/tce/packages/gcc/gcc-8.3.1/rh/lib/gcc/x86_64-redhat-linux/8/" CACHE STRING "" FORCE) endif() -set(SCREAM_INPUT_ROOT "/usr/gdata/climdat/ccsm3data/inputdata" CACHE STRING "") +set(SCREAM_INPUT_ROOT "/usr/gdata/e3sm/ccsm3data/inputdata" CACHE STRING "") diff --git a/components/eamxx/cmake/machine-files/ruby-intel.cmake b/components/eamxx/cmake/machine-files/ruby-intel.cmake index 63fff478fdaf..9c6318da4952 100644 --- a/components/eamxx/cmake/machine-files/ruby-intel.cmake +++ b/components/eamxx/cmake/machine-files/ruby-intel.cmake @@ -4,4 +4,4 @@ set(PYTHON_EXECUTABLE "/usr/tce/packages/python/python-3.9.12/bin/python3" CACHE set(PYTHON_LIBRARIES "/usr/lib64/libpython3.9.so.1.0" CACHE STRING "" FORCE) option (SCREAM_ENABLE_ML_CORRECTION "Whether to enable ML correction parametrization" ON) set(HDF5_DISABLE_VERSION_CHECK 1 CACHE STRING "" FORCE) -execute_process(COMMAND source /usr/WS1/climdat/python_venv/3.9.2/screamML/bin/activate) +execute_process(COMMAND source /usr/WS1/e3sm/python_venv/3.9.2/screamML/bin/activate) diff --git a/components/eamxx/cmake/machine-files/ruby.cmake b/components/eamxx/cmake/machine-files/ruby.cmake index d0a9de4baf4b..77d6e6618c71 100644 --- a/components/eamxx/cmake/machine-files/ruby.cmake +++ b/components/eamxx/cmake/machine-files/ruby.cmake @@ -12,4 +12,4 @@ include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) -set(SCREAM_INPUT_ROOT "/usr/gdata/climdat/ccsm3data/inputdata" CACHE STRING "") +set(SCREAM_INPUT_ROOT "/usr/gdata/e3sm/ccsm3data/inputdata" CACHE STRING "") diff --git a/components/eamxx/cmake/machine-files/syrah.cmake b/components/eamxx/cmake/machine-files/syrah.cmake deleted file mode 100644 index f03fc1e9d469..000000000000 --- a/components/eamxx/cmake/machine-files/syrah.cmake +++ /dev/null @@ -1,13 +0,0 @@ -include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) -common_setup() - -include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) -include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) - -# Enable Sandy Bridge arch in Kokkos -option(Kokkos_ARCH_SNB "" ON) - -set(CMAKE_CXX_FLAGS "-w -cxxlib=/usr/tce/packages/gcc/gcc-8.3.1/rh" CACHE STRING "" FORCE) -set(CMAKE_EXE_LINKER_FLAGS "-L/usr/tce/packages/gcc/gcc-8.3.1/rh/lib/gcc/x86_64-redhat-linux/8/ -mkl" CACHE STRING "" FORCE) - -set(SCREAM_INPUT_ROOT "/usr/gdata/climdat/ccsm3data/inputdata/" CACHE STRING "") diff --git a/components/eamxx/data/SCREAM_YAML_README b/components/eamxx/data/SCREAM_YAML_README deleted file mode 100644 index daf425845e31..000000000000 --- a/components/eamxx/data/SCREAM_YAML_README +++ /dev/null @@ -1,52 +0,0 @@ - -INTRO: - -The scream_input.yaml is the key file for configuring a SCREAM run. This file will be -processed and copied to $case/run/scream_input.yaml by scream's buidnml script, which -is called during case.setup. Note, this is for runtime coniguration -only. Cmake/build-time configuration should be done through SCREAM_CMAKE_OPTIONS. - -For inline comments, see the version of scream_input.yaml that lives in the repo -(components/eamxx/data/scream_input.yaml) - -Note, the $case/run/scream_input.yaml will NEVER be overwritten by subsequent -calls to case.setup/buildnml in order to avoid blowing away potential local -modifications. To force a regeneration of this file, it should be removed from the -case and `./case.setup --reset` should be called. - -SECTIONS: - - Atmosphere Driver: Contains settings for the AD. Can turn off processes by editing "Number of Entries" and - changing the Process N list. - - SCREAM: For general SCREAM settings - - HOMME: For HOMME settings. These settings will be translated into data/namelist.nl - -SYNTAX: - -This file supports some special syntax in addition to basic YAML: -'${VAR}' will be used to refer to env variables in the CIME case - -' val1 : key2 => val2 : elseval>' will be used to express conditional -statements. If switch_val matches key1, then the expression evaluates to val1; if switch_val -matches key2, then the expression evaluates to val2; if it matches neither, then -the expression evaluates to elseval. The elseval component of this expression is optional. -You can have any number (N>=1) of key => val sections. - -Example, if you wanted tstep to depend on atm grid resolution: - - tstep: "<${ATM_GRID} : ne4np4 => 300 : 30>" - -This would give all ne4 cases a timestep of 300, otherwise it would be 30. - -You could specify multiple grid->timestep relationships this way: - - tstep: "<${ATM_GRID} : ne4np4 => 300 : ne30np4 => 100 : 30>" - -Regex matching is supported: - - tstep: "<${ATM_GRID} : .*ne4.* => 300 : .*ne30.* => 100 : 30>" - -Note: none of this special syntax will be automatically reprocessed if the case XML values -are changed. Regenerating this file is necessary if relevant case XML values are modified. diff --git a/components/eamxx/data/scream_default_output.yaml b/components/eamxx/data/scream_default_output.yaml deleted file mode 100644 index 7e0a45f12d6a..000000000000 --- a/components/eamxx/data/scream_default_output.yaml +++ /dev/null @@ -1,75 +0,0 @@ -%YAML 1.1 ---- -filename_prefix: ${CASE}.scream.hi -# WARNING: ERS/ERP tets will override this with AVERAGE -Averaging Type: Instant -# One output every 31 days if output frequency is set to once per hour -Max Snapshots Per File: 744 -Fields: - Physics ${PHYSICS_GRID_TYPE}: - Field Names: - # HOMME - - ps - - pseudo_density - - omega - - p_int - - p_mid - # SHOC + HOMME - - horiz_winds - # SHOC - - cldfrac_liq - - eddy_diff_mom - - sgs_buoy_flux - - tke - - pbl_height - # CLD - - cldfrac_ice_for_analysis - - cldfrac_tot_for_analysis - # P3 - - bm - - nc - - ni - - nr - - qi - - qm - - qr - - eff_radius_qc - - eff_radius_qi - - eff_radius_qr - - precip_ice_surf_mass - - precip_liq_surf_mass - - rainfrac - # SHOC + P3 - - qc - - qv - # SHOC + P3 + RRTMGP + HOMME - - T_mid - # RRTMGP - - sfc_alb_dir_vis - - LW_flux_dn - - LW_flux_up - - SW_flux_dn - - SW_flux_up - - sfc_flux_lw_dn - - sfc_flux_sw_net - - cldtot - - cldlow - - cldmed - - cldhgh - # Surface Fluxes - - surf_evap - - surf_sens_flux - # Diagnostics - - PotentialTemperature - # GLL output for homme states. - Dynamics: - Field Names: - - ps_dyn - - dp3d_dyn - - omega_dyn - IO Grid Name: Physics GLL -output_control: -# WARNING: ERS/ERP tets will override this with STOP_N/STOP_OPTION - Frequency: ${HIST_N} - frequency_units: ${HIST_OPTION} -... diff --git a/components/eamxx/data/scream_default_remap.yaml b/components/eamxx/data/scream_default_remap.yaml deleted file mode 100644 index 8bf47386c76d..000000000000 --- a/components/eamxx/data/scream_default_remap.yaml +++ /dev/null @@ -1,67 +0,0 @@ -%YAML 1.1 ---- -filename_prefix: ${CASE}.scream.arm_sites.hi -Averaging Type: Instant -Max Snapshots Per File: 744 # One output every 31 days -#remap_file: /g/g17/donahue5/Code/e3sm/scream-docs/regional_output_sites/20221123_ARM_sites_map.nc -remap_file: /usr/gdata/climdat/ccsm3data/inputdata/atm/scream/maps/map_ne30np4_to_ne4pg2_mono.20220714.nc -Fields: - Physics ${PHYSICS_GRID_TYPE}: - Field Names: - # HOMME - - ps - - pseudo_density - - omega - - p_int - - p_mid - # SHOC + HOMME - - horiz_winds - # SHOC - - cldfrac_liq - - eddy_diff_mom - - sgs_buoy_flux - - tke - - pbl_height - # CLD - - cldfrac_ice - - cldfrac_tot - # P3 - - bm - - nc - - ni - - nr - - qi - - qm - - qr - - eff_radius_qc - - eff_radius_qi - - eff_radius_qr - - precip_ice_surf_mass - - precip_liq_surf_mass - - rainfrac - # SHOC + P3 - - qc - - qv - # SHOC + P3 + RRTMGP + HOMME - - T_mid - # RRTMGP - - sfc_alb_dir_vis - - LW_flux_dn - - LW_flux_up - - SW_flux_dn - - SW_flux_up - - sfc_flux_lw_dn - - sfc_flux_sw_net - - cldtot - - cldlow - - cldmed - - cldhgh - # Surface Fluxes - - surf_evap - - surf_sens_flux - # Diagnostics -# - PotentialTemperature -output_control: - Frequency: ${HIST_N} - frequency_units: ${HIST_OPTION} -... diff --git a/components/eamxx/docs/developer/standalone_testing.md b/components/eamxx/docs/developer/standalone_testing.md index aedb7b2cbaad..63ea01fc612e 100644 --- a/components/eamxx/docs/developer/standalone_testing.md +++ b/components/eamxx/docs/developer/standalone_testing.md @@ -33,9 +33,9 @@ make baseline ``` The tests will run, automatically using the baseline file, which is located in -the CMake-configurable path `${SCREAM_TEST_DATA_DIR}`. By default, this path is -set to `data/` within your build directory (which is `$RUN_ROOT_DIR`, in -our case). +the CMake-configurable path `${SCREAM_BASELINES_DIR}`. By default, this path is +set to an invalid string. If baselines tests are enabled, we check that a valid +path has been provided. To run all of SCREAM's tests, make sure you're in `$RUN_ROOT_DIR` and type diff --git a/components/eamxx/docs/user/coarse_nudging.md b/components/eamxx/docs/user/coarse_nudging.md index c52ce9a0eb24..3d7309c42aaa 100644 --- a/components/eamxx/docs/user/coarse_nudging.md +++ b/components/eamxx/docs/user/coarse_nudging.md @@ -20,6 +20,6 @@ In other words, the following options are needed: ```shell ./atmchange atm_procs_list=(sc_import,nudging,homme,physics,sc_export) ./atmchange nudging_fields=U,V -./atmchange nudging_filename=/path/to/nudging_data_ne4pg2_L72.nc +./atmchange nudging_filenames_patterns=/path/to/nudging_data_ne4pg2_L72.nc ./atmchange nudging_refine_remap_mapfile=/another/path/to/mapping_file_ne4pg2_to_ne120pg2.nc ``` diff --git a/components/eamxx/mkdocs.yml b/components/eamxx/mkdocs.yaml similarity index 100% rename from components/eamxx/mkdocs.yml rename to components/eamxx/mkdocs.yaml diff --git a/components/eamxx/scripts/atm_manip.py b/components/eamxx/scripts/atm_manip.py index 2a763bc1990d..fbd8381e1727 100755 --- a/components/eamxx/scripts/atm_manip.py +++ b/components/eamxx/scripts/atm_manip.py @@ -170,7 +170,7 @@ def modify_ap_list(xml_root, group, ap_list_str, append_this): 'p1,p2,p1' >>> modify_ap_list(tree,node,"p1,p3",False) Traceback (most recent call last): - ValueError: ERROR: Unrecognized atm proc name 'p3'. To declare a new group, prepend and append '_' to the name. + SystemExit: ERROR: Unrecognized atm proc name 'p3'. To declare a new group, prepend and append '_' to the name. >>> modify_ap_list(tree,node,"p1,_my_group_",False) True >>> get_child(node,"atm_procs_list").text @@ -203,11 +203,11 @@ def modify_ap_list(xml_root, group, ap_list_str, append_this): new_aps = [n for n in add_aps if find_node(ap_defaults,n) is None] for ap in new_aps: - expect (ap[0]=="_" and ap[-1]=="_" and len(ap)>2, exc_type=ValueError, - error_msg=f"Unrecognized atm proc name '{ap}'. To declare a new group, prepend and append '_' to the name.") + expect (ap[0]=="_" and ap[-1]=="_" and len(ap)>2, + f"Unrecognized atm proc name '{ap}'. To declare a new group, prepend and append '_' to the name.") group = gen_atm_proc_group("", ap_defaults) group.tag = ap - + ap_defaults.append(group) # Update the 'atm_procs_list' in this node @@ -217,6 +217,25 @@ def modify_ap_list(xml_root, group, ap_list_str, append_this): curr_apl.text = ','.join(ap_list) return True +############################################################################### +def is_locked_impl(node): +############################################################################### + return "locked" in node.attrib.keys() and str(node.attrib["locked"]).upper() == "TRUE" + +############################################################################### +def is_locked(xml_root, node): +############################################################################### + if is_locked_impl(node): + return True + else: + parent_map = create_parent_map(xml_root) + parents = get_parents(node, parent_map) + for parent in parents: + if is_locked_impl(parent): + return True + + return False + ############################################################################### def apply_change(xml_root, node, new_value, append_this): ############################################################################### @@ -231,6 +250,7 @@ def apply_change(xml_root, node, new_value, append_this): if append_this: + expect (not is_locked(xml_root, node), f"Cannot change {node.tag}, it is locked") expect ("type" in node.attrib.keys(), f"Error! Missing type information for {node.tag}") type_ = node.attrib["type"] @@ -238,7 +258,11 @@ def apply_change(xml_root, node, new_value, append_this): "Error! Can only append with array and string types.\n" f" - name: {node.tag}\n" f" - type: {type_}") - if is_array_type(type_): + + if node.text is None: + node.text = "" + + if is_array_type(type_) and node.text!="": node.text += ", " + new_value else: node.text += new_value @@ -246,6 +270,7 @@ def apply_change(xml_root, node, new_value, append_this): any_change = True elif node.text != new_value: + expect (not is_locked(xml_root, node), f"Cannot change {node.tag}, it is locked") check_value(node,new_value) node.text = new_value any_change = True @@ -283,16 +308,37 @@ def atm_config_chg_impl(xml_root, change, all_matches=False): """ >>> xml = ''' ... - ... 1,2,3 - ... 1 - ... 1 - ... one - ... one - ... one - ... - ... two - ... 2 - ... + ... 1,2,3 + ... 1 + ... 1 + ... one + ... one + ... one + ... + ... two + ... 2 + ... + ... + ... + ... + ... hi + ... + ... + ... + ... + ... + ... + ... hi + ... + ... + ... + ... + ... + ... + ... hi + ... + ... + ... ... ... ''' >>> import xml.etree.ElementTree as ET @@ -306,7 +352,8 @@ def atm_config_chg_impl(xml_root, change, all_matches=False): >>> ################ INVALID TYPE ####################### >>> atm_config_chg_impl(tree,'prop2=two') Traceback (most recent call last): - ValueError: Could not refine 'two' as type 'integer' + CIME.utils.CIMEError: ERROR: Could not refine 'two' as type 'integer': + could not convert string to float: 'two' >>> ################ INVALID VALUE ####################### >>> atm_config_chg_impl(tree,'prop2=3') Traceback (most recent call last): @@ -350,6 +397,16 @@ def atm_config_chg_impl(xml_root, change, all_matches=False): True >>> get_xml_nodes(tree,'e')[0].text 'one, two' + >>> ################ Test locked ################## + >>> atm_config_chg_impl(tree, 'lprop2=yo') + Traceback (most recent call last): + SystemExit: ERROR: Cannot change lprop2, it is locked + >>> atm_config_chg_impl(tree, 'lprop3=yo') + Traceback (most recent call last): + SystemExit: ERROR: Cannot change lprop3, it is locked + >>> atm_config_chg_impl(tree, 'lprop4=yo') + Traceback (most recent call last): + SystemExit: ERROR: Cannot change lprop4, it is locked """ node_name, new_value, append_this = parse_change(change) matches = get_xml_nodes(xml_root, node_name) diff --git a/components/eamxx/scripts/change-param-pattern b/components/eamxx/scripts/change-param-pattern index 5ab26a64df3b..a54c7292d246 100755 --- a/components/eamxx/scripts/change-param-pattern +++ b/components/eamxx/scripts/change-param-pattern @@ -35,4 +35,4 @@ for bad_name_under in bad_name_unders: run_cmd_no_fail(f"sed -i -e 's/{bad_name_ws}/{good_name}/g' $(git grep -l '{bad_name_ws}')") run_cmd_no_fail(f"git commit -a -m '{bad_name_ws} -> {good_name}'") print(" Testing") - run_cmd_no_fail("./create_test ERS_D_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1 --compiler=gnu9", from_dir="../../cime/scripts") + run_cmd_no_fail("./create_test ERS_D_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1", from_dir="../../cime/scripts") diff --git a/components/eamxx/scripts/check-hashes-ers b/components/eamxx/scripts/check-hashes-ers new file mode 100755 index 000000000000..6d9da4b2f98e --- /dev/null +++ b/components/eamxx/scripts/check-hashes-ers @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 + +""" +See https://acme-climate.atlassian.net/wiki/spaces/NGDNA/pages/3831923056/EAMxx+BFB+hashing +for full explanation. + +This script is used by the scream-internal_diagnostics_level testmod to check +hash output after a test has run. +""" + +import sys, re, glob, pathlib, argparse + +from utils import run_cmd_no_fail, expect + +############################################################################### +def parse_command_line(args, description): +############################################################################### + parser = argparse.ArgumentParser( + usage="""\n{0} [=] ... +OR +{0} --help + +\033[1mEXAMPLES:\033[0m + \033[1;32m# Run hash checker on /my/case/dir \033[0m + > {0} /my/case/dir +""".format(pathlib.Path(args[0]).name), + description=description, + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "case_dir", + help="The test case you want to check" + ) + + return parser.parse_args(args[1:]) + +############################################################################### +def readall(fn): +############################################################################### + with open(fn,'r') as f: + txt = f.read() + return txt + +############################################################################### +def greptxt(pattern, txt): +############################################################################### + return re.findall('(?:' + pattern + ').*', txt, flags=re.MULTILINE) + +############################################################################### +def grep(pattern, fn): +############################################################################### + txt = readall(fn) + return greptxt(pattern, txt) + +############################################################################### +def get_log_glob_from_atm_modelio(case_dir): +############################################################################### + filename = case_dir / 'CaseDocs' / 'atm_modelio.nml' + ln = grep('diro = ', filename)[0] + run_dir = pathlib.Path(ln.split()[2].split('"')[1]) + ln = grep('logfile = ', filename)[0] + atm_log_fn = ln.split()[2].split('"')[1] + id_ = atm_log_fn.split('.')[2] + return str(run_dir / '**' / f'e3sm.log.{id_}*') + +############################################################################### +def get_hash_lines(fn): +############################################################################### + rlns = run_cmd_no_fail(f'zgrep exxhash {fn}').splitlines() + lns = [] + if len(rlns) == 0: return lns + for rln in rlns: + pos = rln.find('exxhash') + lns.append(rln[pos:]) + return lns + +############################################################################### +def parse_time(hash_ln): +############################################################################### + return hash_ln.split()[1:3] + +############################################################################### +def all_equal(t1, t2): +############################################################################### + if len(t1) != len(t2): return False + for i in range(len(t1)): + if t1[i] != t2[i]: return False + return True + +############################################################################### +def find_first_index_at_time(lns, time): +############################################################################### + for i, ln in enumerate(lns): + t = parse_time(ln) + if all_equal(time, t): return i + return None + +############################################################################### +def diff(l1, l2): +############################################################################### + diffs = [] + for i in range(len(l1)): + if l1[i] != l2[i]: + diffs.append((l1[i], l2[i])) + return diffs + +############################################################################### +def check_hashes_ers(case_dir): +############################################################################### + case_dir_p = pathlib.Path(case_dir) + expect(case_dir_p.is_dir(), f"{case_dir} is not a dir") + + # Look for the two e3sm.log files. + glob_pat = get_log_glob_from_atm_modelio(case_dir_p) + e3sm_fns = glob.glob(glob_pat, recursive=True) + if len(e3sm_fns) == 0: + print('Could not find e3sm.log files with glob string {}'.format(glob_pat)) + return False + e3sm_fns.sort() + if len(e3sm_fns) == 1: + # This is the first run. Exit and wait for the second + # run. (POSTRUN_SCRIPT is called after each of the two runs.) + print('Exiting on first run.') + return True + print('Diffing base {} and restart {}'.format(e3sm_fns[0], e3sm_fns[1])) + + # Because of the prefixed 1: and 2: on some systems, we can't just use + # zdiff. + lns = [] + for f in e3sm_fns: + lns.append(get_hash_lines(f)) + time = parse_time(lns[1][0]) + time_idx = find_first_index_at_time(lns[0], time) + if time_idx is None: + print('Could not find a start time.') + return False + lns[0] = lns[0][time_idx:] + if len(lns[0]) != len(lns[1]): + print('Number of hash lines starting at restart time do not agree.') + return False + diffs = diff(lns[0], lns[1]) + + # Flushed prints to e3sm.log can sometimes conflict with other + # output. Permit up to 'thr' diffs so we don't fail due to badly printed + # lines. This isn't a big loss in checking because an ERS_Ln22 second run + # writes > 1000 hash lines, and a true loss of BFBness is nearly certain to + # propagate to a large number of subsequent hashes. + thr = 5 + if len(lns[0]) < 100: thr = 0 + + ok = True + if len(diffs) > thr: + print('DIFF') + print(diffs[-10:]) + ok = False + else: + print('OK') + + return ok + +############################################################################### +def _main_func(description): +############################################################################### + success = check_hashes_ers(**vars(parse_command_line(sys.argv, description))) + sys.exit(0 if success else 1) + +############################################################################### + +if (__name__ == "__main__"): + _main_func(__doc__) diff --git a/components/eamxx/scripts/cime-nml-tests b/components/eamxx/scripts/cime-nml-tests index 0a60376bd45f..b128121cf95f 100755 --- a/components/eamxx/scripts/cime-nml-tests +++ b/components/eamxx/scripts/cime-nml-tests @@ -222,9 +222,10 @@ class TestBuildnml(unittest.TestCase): # Append to an existing entry name = 'output_yaml_files' out = run_cmd_no_fail(f"./atmchange {name}+=a.yaml", from_dir=case) + out = run_cmd_no_fail(f"./atmchange {name}+=b.yaml", from_dir=case) # Get the yaml files - expected =f'{EAMXX_DIR / "data/scream_default_output.yaml"}, a.yaml' + expected =f'a.yaml, b.yaml' self._get_values(case, name, value=expected, expect_equal=True) ########################################################################### diff --git a/components/eamxx/scripts/eamxx-params-docs-autogen b/components/eamxx/scripts/eamxx-params-docs-autogen index 3024677d96b0..9bea5242d5a9 100755 --- a/components/eamxx/scripts/eamxx-params-docs-autogen +++ b/components/eamxx/scripts/eamxx-params-docs-autogen @@ -20,6 +20,7 @@ from mdutils import Html sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "cime_config")) from eamxx_buildnml_impl import resolve_all_inheritances, get_valid_selectors +from atm_manip import is_locked_impl ############################################################################### def parse_command_line(args, description): @@ -34,14 +35,15 @@ def parse_command_line(args, description): return parser.parse_args(args[1:]) ########################################################################### -def add_param(docs,scope,item): +def add_param(docs, scope, item): ########################################################################### # Locked parameters are not to be configured at runtime, so don't even bother # E.g, a locked param is something we need to get in the input file, like # the restart write frequency, but we don't want the user to modify it # via atmchange - if "locked" in item.attrib.keys(): + if is_locked_impl(item): return + docs.new_line(f"* {scope}{item.tag}:") pdoc = item.attrib['doc'] if 'doc' in item.attrib.keys() else "**MISSING**" @@ -53,21 +55,23 @@ def add_param(docs,scope,item): pvalid = item.attrib['valid_values'] if 'valid_values' in item.attrib.keys() else None if pvalid is not None: docs.new_line(f" - valid values: {pvalid}") + pconstr = item.attrib['constraints'] if 'constraints' in item.attrib.keys() else None if pconstr is not None: docs.new_line(f" - constraints: {pconstr}") ########################################################################### -def add_children(docs,xml,scope=""): +def add_children(docs, elem, scope=""): ########################################################################### done = [] # Locked parameters are not to be configured at runtime, so don't even bother # E.g, a locked param is something we need to get in the input file, like # the restart write frequency, but we don't want the user to modify it # via atmchange - if "locked" in xml.attrib.keys(): + if is_locked_impl(elem): return - for item in xml: + + for item in elem: # The same entry may appear multiple times in the XML defaults file, # each time with different selectors. We don't want to generate the # same documentation twice. @@ -75,9 +79,10 @@ def add_children(docs,xml,scope=""): continue done.append(item.tag) if len(item)>0: - add_children (docs,item,f"{scope}{xml.tag}::") + add_children (docs,item,f"{scope}{elem.tag}::") else: - add_param(docs,f"{scope}{xml.tag}::",item) + add_param(docs,f"{scope}{elem.tag}::",item) + docs.new_line() ########################################################################### @@ -107,7 +112,7 @@ def generate_params_docs(): continue docs.new_header(level=2,title=ap.tag) add_children(docs,ap) - + ic = xml_defaults.find('initial_conditions') docs.new_header(level=1,title="Initial Conditions Parameters") add_children(docs,ic) @@ -123,6 +128,7 @@ def generate_params_docs(): homme = xml_defaults.find('ctl_nl') docs.new_header(level=1,title='Homme namelist') add_children(docs,homme) + docs.create_md_file() print("Generating eamxx params documentation ... SUCCESS!") diff --git a/components/eamxx/scripts/edit-output-stream b/components/eamxx/scripts/edit-output-stream new file mode 100755 index 000000000000..76c88e69ca39 --- /dev/null +++ b/components/eamxx/scripts/edit-output-stream @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 + +""" +Edit (or create) an output stream yaml file +""" + +import argparse, sys, pathlib + +from edit_output_stream import edit_output_stream_impl + +############################################################################### +def parse_command_line(args, description): +############################################################################### + parser = argparse.ArgumentParser( + usage="""\n{0} = [=] ... +OR +{0} --help + +\033[1mEXAMPLES:\033[0m + \033[1;32m# Generate empty file (with invalid options)'\033[0m + > {0} -g -f my_output.yaml + + \033[1;32m# Generate empty file (with invalid options), overwrite if existing'\033[0m + > {0} -g -O -f my_output.yaml + + \033[1;32m# Change avg type to Instant, output frequency to 1 day\033[0m + > {0} -f my_output.yaml --avg-type instant --freq 1 --freq-units ndays + + \033[1;32m# Set horiz remaping\033[0m + > {0} -f my_output.yaml --horiz-map-file /path/to/map.nc +""".format(pathlib.Path(args[0]).name), + description=description, + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "-f","--filename", + required=True, + type=str, + help="The name of the yaml file to be configured", + ) + + parser.add_argument( + "--prefix", + type=str, + help="The prefix of the output files", + ) + + parser.add_argument( + "-g", "--generate", + default=False, + dest="generate", + action="store_true", + help="Generate a new file", + ) + parser.add_argument( + "-O", "--overwrite", + default=False, + action="store_true", + help="When generating a new file, overwrite existing file (if any)", + ) + + parser.add_argument( + "-r","--reset", + default=None, + nargs="+", + type=str, + help="List of options to remove (or reset to default values) BEFORE doing any other edit" + ) + + parser.add_argument( + "--avg-type", + default=None, + type=str.lower, + help="Set the averaging type", + choices=['instant','average','max','min'], + ) + + parser.add_argument( + "--skip-t0-output", + action="store_true", + help="Skip t=case_t0 output (only relevant for INSTANT avg)" + ) + + parser.add_argument( + "--freq-units", + default=None, + type=str.lower, + help="Set the output frequency units", + ) + + parser.add_argument( + "--freq", + type=str.lower, + default=None, + help="Set the output frequency", + ) + + parser.add_argument( + "--grid", + default=None, + type=str, + help="Specify grid for which --fields/--io-grid options apply to", + ) + + parser.add_argument( + "--fields", + default=[], + nargs="+", + help="Fields to add to output", + ) + + parser.add_argument( + "--io-grid", + default=None, + type=str, + help="Name of grid onto which to remap fields before outputing them", + ) + + parser.add_argument( + "--horiz-remap-file", + default=None, + help="Map file to use for horizontal remap", + ) + + parser.add_argument( + "--vertical-remap-file", + default=None, + help="Map file to use for horizontal remap", + ) + + return parser.parse_args(args[1:]) + +############################################################################### +def _main_func(description): +############################################################################### + if "--test" in sys.argv: + from doctest import testmod + import edit_output_stream + testmod() + testmod(m=edit_output_stream) + else: + edit_output_stream_impl(**vars(parse_command_line(sys.argv, description))) + sys.exit(0) + +############################################################################### + +if (__name__ == "__main__"): + _main_func(__doc__) diff --git a/components/eamxx/scripts/edit_output_stream.py b/components/eamxx/scripts/edit_output_stream.py new file mode 100644 index 000000000000..8ca194a47f98 --- /dev/null +++ b/components/eamxx/scripts/edit_output_stream.py @@ -0,0 +1,223 @@ +import pathlib + +from utils import expect, ensure_yaml + +ensure_yaml() +import yaml + +############################################################################### +def generate_empty_yaml(filename,overwrite): +############################################################################### + """ + Generate a yaml file with basic fields set, but containing empty or invalid values + >>> fname = "__test__.yaml" + >>> generate_empty_yaml(fname,False) + >>> generate_empty_yaml(fname,False) + Traceback (most recent call last): + SystemExit: ERROR: YAML file already exist. Re-run with -O/--overwrite to overwrite existing file + >>> generate_empty_yaml(fname,True) + >>> data = yaml.load(open(fname,'r'),Loader=yaml.SafeLoader) + >>> len(data) + 4 + >>> data["filename_prefix"] + 'UNSET' + >>> data["Averaging Type"] + 'INVALID' + >>> len(data["Fields"]) + 0 + >>> oc = data["output_control"] + >>> len(oc) + 2 + >>> oc["Frequency"] + -1 + >>> oc["frequency_units"] + 'never' + >>> # Clean up the file + >>> pathlib.Path(fname).unlink() + """ + file = pathlib.Path(filename).resolve() + + expect (overwrite or not file.exists(), + "YAML file already exist. Re-run with -O/--overwrite to overwrite existing file") + + if file.exists(): + file.unlink() + + data = {} + data["filename_prefix"] = "UNSET" + data["Averaging Type"] = "INVALID" + data["Fields"] = {} + data["output_control"] = {} + data["output_control"]["skip_t0_output"] = "false" + data["output_control"]["Frequency"] = -1 + data["output_control"]["frequency_units"] = "never" + + with open(file,'w') as fd: + yaml.dump(data,fd,Dumper=yaml.SafeDumper,explicit_start=True,explicit_end=True,version=(1,2)) + +############################################################################### +def edit_output_stream_impl(filename,prefix=None,generate=False,overwrite=False, + avg_type=None,skip_t0_output=False,freq_units=None,freq=None, + grid=None,fields=[],reset=None,io_grid=None, + horiz_remap_file=None,vertical_remap_file=None): +############################################################################### + """ + Apply the requested changes to the output stream yaml file + >>> fname = '__test__.yaml' + >>> # Create the file + >>> edit_output_stream_impl(fname,generate=True,prefix='foo') + >>> # Set some basic options, and then check + >>> edit_output_stream_impl(fname,avg_type='max',freq_units='ndays',freq=10) + >>> data = yaml.load(open(fname,'r'),Loader=yaml.SafeLoader) + >>> data['filename_prefix'] + 'foo' + >>> data['Averaging Type'] + 'max' + >>> data['output_control']['Frequency'] + 10 + >>> data['output_control']['frequency_units'] + 'ndays' + >>> # Set fields options, and then check + >>> edit_output_stream_impl(fname,fields=['a','b'],grid='my_grid',io_grid='other_grid') + >>> data = yaml.load(open(fname,'r'),Loader=yaml.SafeLoader) + >>> f = data['Fields']['my_grid']['Field Names'] + >>> f.sort() + >>> f + ['a', 'b'] + >>> data['Fields']['my_grid']['IO Grid Name'] + 'other_grid' + >>> # No remap if online remap (IO Grid Name) is set + >>> edit_output_stream_impl(fname,horiz_remap_file='blah') + Traceback (most recent call last): + SystemExit: ERROR: Cannot use online remap and horiz/vert remap at the same time. + >>> edit_output_stream_impl(fname,vertical_remap_file='blah') + Traceback (most recent call last): + SystemExit: ERROR: Cannot use online remap and horiz/vert remap at the same time. + >>> # Remove io grid and fields + >>> edit_output_stream_impl(fname,reset=['fields','io-grid']) + Traceback (most recent call last): + SystemExit: ERROR: Fields reset requested, but no grid name provided. Re-run with --grid GRID_NAME + >>> edit_output_stream_impl(fname,reset=['fields','io-grid'],grid='my_grid') + >>> data = yaml.load(open(fname,'r'),Loader=yaml.SafeLoader) + >>> 'my_grid' in data['Fields'].keys() + False + >>> # Set remap options, and then check + >>> edit_output_stream_impl(fname,horiz_remap_file='blah1',vertical_remap_file='blah2') + >>> data = yaml.load(open(fname,'r'),Loader=yaml.SafeLoader) + >>> data['horiz_remap_file'] + 'blah1' + >>> data['vertical_remap_file'] + 'blah2' + >>> # Clean up the file + >>> pathlib.Path(fname).unlink() + """ + + if generate: + generate_empty_yaml(filename,overwrite) + + file = pathlib.Path(filename).resolve() + + expect (file.exists(), + "YAML file does not exist. Re-run with -g/--generate to create") + + data = yaml.load(open(file,"r"),Loader=yaml.SafeLoader) + + # Before adding new options, process all the reset requests + if reset is not None: + for s in reset: + if s=="avg-type": + data["Averaging Type"] = "INVALID" + elif s=="skip_t0_output": + data["skip_t0_output"] = "false" + elif s=="preifx": + data["filename_prefix"] = "UNSET" + elif s=="freq": + data["output_control"]["Frequency"] = -1 + elif s=="freq_units": + data["output_control"]["frequency_units"] = "never" + elif s=="horiz_remap_file": + del data["horiz_remap_file"] + elif s=="vert_remap_file": + del data["vert_remap_file"] + elif s=="fields": + expect (grid is not None, + "Fields reset requested, but no grid name provided. Re-run with --grid GRID_NAME") + if grid in data["Fields"].keys(): + data["Fields"][grid]["Field Names"] = [] + + # Remove this grid if there are no other options set + if len(data["Fields"][grid])==1: + del data["Fields"][grid] + elif s=="io-grid": + expect (grid is not None, + "IO grid reset requested, but no grid name provided. Re-run with --grid GRID_NAME") + if grid in data["Fields"].keys(): + del data["Fields"][grid]["IO Grid Name"] + + # Remove this grid if there's not other options set other than + # fields names, and field names is an empty list + if len(data["Fields"][grid])==1 and len(data["Fields"][grid]["Field Names"])==0: + del data["Fields"][grid] + + if prefix is not None: + data["filename_prefix"] = prefix + + if avg_type is not None: + data["Averaging Type"] = avg_type + + if skip_t0_output is not None: + data["skip_t0_output"] = skip_t0_output + + if freq is not None: + expect (freq.lstrip('-+').isnumeric() or freq=='hist_n', + f"Invalid value '{freq}' for --freq. Valid options are\n" + " - an integer\n" + " - HIST_N\n") + data["output_control"]["Frequency"] = int(freq) if freq.lstrip('-+').isnumeric() else f"${{{freq.upper()}}}" + + if freq_units is not None: + explicit = ['nsteps','nsecs','nmins','nhours','ndays','nmonths','nyears'] + expect (freq_units in explicit or freq_units=='hist_option', + f"Invalid value '{freq_units}' for --freq-units. Valid options are (case insensitive)\n" + " - explicit values: 'nsteps','nsecs','nmins','nhours','ndays','nmonths','nyears'\n" + " - CIME variables : 'HIST_OPTION'\n") + + data["output_control"]["frequency_units"] = freq_units if freq_units in explicit else f"${{{freq_units.upper()}}}" + + if horiz_remap_file is not None: + data["horiz_remap_file"] = horiz_remap_file + + if vertical_remap_file is not None: + data["vertical_remap_file"] = vertical_remap_file + + if len(fields)>0 or io_grid is not None: + expect (grid is not None, + "Fields list specified, but no grid name provided. Re-run with --grid GRID_NAME") + + section = data["Fields"].setdefault(grid,{}) + if "Field Names" not in section.keys(): + section["Field Names"] = [] + + fnames = section["Field Names"] + fnames += fields + fnames = list(set(fnames)) + section["Field Names"] = fnames + + if io_grid is not None: + section["IO Grid Name"] = io_grid + # If not already present, add an empty list of field names + section.setdefault("Field Names",[]) + + data["Fields"][grid] = section + + # We cannot do online remap (typically dyn->physGLL) if horiz or vert remap is used + has_online_remap = False + for k,v in data["Fields"].items(): + has_online_remap = has_online_remap or "IO Grid Name" in v.keys(); + has_vert_remap = "vertical_remap_file" in data.keys() + has_horiz_remap = "horiz_remap_file" in data.keys() + expect (not has_online_remap or (not has_vert_remap and not has_horiz_remap), + "Cannot use online remap and horiz/vert remap at the same time.") + + with open(file,'w') as fd: + yaml.dump(dict(data),fd,Dumper=yaml.SafeDumper,explicit_start=True,explicit_end=True,version=(1,2)) diff --git a/components/eamxx/scripts/git_utils.py b/components/eamxx/scripts/git_utils.py index e3aa70025398..a7202bfbfaba 100644 --- a/components/eamxx/scripts/git_utils.py +++ b/components/eamxx/scripts/git_utils.py @@ -55,7 +55,7 @@ def get_current_head(repo=None): return branch ############################################################################### -def git_refs_difference (cmp_ref, head="HEAD", repo=None): +def git_refs_difference(cmp_ref, head="HEAD", repo=None): ############################################################################### """ Return the difference in commits between cmp_ref and head. @@ -136,6 +136,17 @@ def merge_git_ref(git_ref, repo=None, verbose=False, dry_run=False): print ("git ref {} successfully merged.".format(git_ref)) print_last_commit() +############################################################################### +def create_backup_commit (repo=None, dry_run=False): +############################################################################### + + bkp_cmd = "git add -A && git commit -m 'WARNING: test-all-scream backup commit'" + if dry_run: + print (f"Would run: {bkp_cmd}") + else: + run_cmd_no_fail(bkp_cmd, from_dir=repo) + expect(is_repo_clean(repo=repo), "Something went wrong while performing the backup commit") + ############################################################################### def print_last_commit(git_ref=None, repo=None, dry_run=False): ############################################################################### @@ -182,7 +193,7 @@ def get_git_toplevel_dir(repo=None): return output if stat == 0 else None ############################################################################### -def cleanup_repo(orig_branch, orig_commit, repo=None, dry_run=False): +def cleanup_repo(orig_branch, orig_commit, has_backup_commit=False, repo=None, dry_run=False): ############################################################################### """ Discards all unstaged changes, as well as untracked files @@ -206,4 +217,10 @@ def cleanup_repo(orig_branch, orig_commit, repo=None, dry_run=False): # NOTE: if you reset the branch, don't forget to re-update the modules!! if curr_commit != orig_commit and not dry_run: run_cmd_no_fail("git reset --hard {}".format(orig_commit), from_dir=repo) + if has_backup_commit: + # This can happen if we ran an integration test with a dirty repo. + # test_all_scream will create a temporary backup commit, which we + # need to undo, but leaving the changed files in the workspace. + # So DON'T add --hard to this call! + run_cmd_no_fail("git reset {HEAD~1}", from_dir=repo) update_submodules(repo=repo) diff --git a/components/eamxx/scripts/jenkins/jenkins_cleanup_impl.sh b/components/eamxx/scripts/jenkins/jenkins_cleanup_impl.sh index 942be64d8df6..3282f86e7158 100755 --- a/components/eamxx/scripts/jenkins/jenkins_cleanup_impl.sh +++ b/components/eamxx/scripts/jenkins/jenkins_cleanup_impl.sh @@ -1,11 +1,10 @@ #!/bin/bash -xe -# Adjust this number to keep more/less builds -echo "WORKSPACE: ${WORKSPACE}, BUILD_ID: ${BUILD_ID}" +echo "RUNNING CLEANUP FOR WORKSPACE: ${WORKSPACE}, BUILD_ID: ${BUILD_ID}" cd ${WORKSPACE} -NUM_KEEP=30 +NUM_KEEP=12 # Adjust this number to keep more/fewer builds KEEP_LAST=${BUILD_ID} KEEP_FIRST=$((${BUILD_ID}-${NUM_KEEP})) KEEP="$(seq ${KEEP_FIRST} 1 ${KEEP_LAST})" @@ -15,3 +14,11 @@ REMOVE_THESE="$(ls -1 | grep -vF "${KEEP}")" echo "Purging old builds: ${REMOVE_THESE}." /bin/rm -rf $REMOVE_THESE + +# Now clean up the scratch area +if [[ "$NODE_NAME" == "mappy" ]]; then + # Ensure we have a newer python + source $JENKINS_SCRIPT_DIR/${NODE_NAME}_setup + + $JENKINS_SCRIPT_DIR/scratch_cleanup.py +fi diff --git a/components/eamxx/scripts/jenkins/jenkins_common_impl.sh b/components/eamxx/scripts/jenkins/jenkins_common_impl.sh index fe293debe446..664c911585e2 100755 --- a/components/eamxx/scripts/jenkins/jenkins_common_impl.sh +++ b/components/eamxx/scripts/jenkins/jenkins_common_impl.sh @@ -72,7 +72,12 @@ if [ $skip_testing -eq 0 ]; then # IF such dir is not found, then the default (ctest-build/baselines) is used BASELINES_DIR=AUTO - TAS_ARGS="--baseline-dir $BASELINES_DIR \$compiler -c EKAT_DISABLE_TPL_WARNINGS=ON -p -i -m \$machine" + TAS_ARGS="--baseline-dir $BASELINES_DIR \$compiler -p -c EKAT_DISABLE_TPL_WARNINGS=ON -i -m \$machine" + # pm-gpu needs to do work in scratch area in order not to fill home quota + if [[ "$SCREAM_MACHINE" == "pm-gpu" ]]; then + TAS_ARGS="${TAS_ARGS} -w /pscratch/sd/e/e3smtest/e3sm_scratch/pm-gpu/ctest-build" + fi + # Now that we are starting to run things that we expect could fail, we # do not want the script to exit on any fail since this will prevent # later tests from running. @@ -120,7 +125,9 @@ if [ $skip_testing -eq 0 ]; then fi fi - if [[ "$SCREAM_MACHINE" == "weaver" ]]; then + if [[ -z "$SCREAM_FAKE_ONLY" && "$SCREAM_MACHINE" == "weaver" ]]; then + # The fake-only tests don't launch any kernels which will cause all + # the compute-sanitizer runs to fail. ./scripts/gather-all-data "./scripts/test-all-scream -t csm -t csr -t csi -t css ${TAS_ARGS}" -l -m $SCREAM_MACHINE if [[ $? != 0 ]]; then fails=$fails+1; @@ -138,20 +145,6 @@ if [ $skip_testing -eq 0 ]; then # Run scripts-tests if [[ $test_scripts == 1 ]]; then - # JGF: I'm not sure there's much value in these dry-run comparisons - # since we aren't changing HEADs - ./scripts/scripts-tests -g -m $SCREAM_MACHINE - if [[ $? != 0 ]]; then - fails=$fails+1; - scripts_fail=1 - fi - - ./scripts/scripts-tests -c -m $SCREAM_MACHINE - if [[ $? != 0 ]]; then - fails=$fails+1; - scripts_fail=1 - fi - ./scripts/scripts-tests -f -m $SCREAM_MACHINE if [[ $? != 0 ]]; then fails=$fails+1; @@ -195,7 +188,7 @@ if [ $skip_testing -eq 0 ]; then if [[ $test_v1 == 1 ]]; then # AT runs should be fast. => run only low resolution - this_output=$(../../cime/scripts/create_test e3sm_scream_v1_at --compiler=gnu9 -c -b master --wait) + this_output=$(../../cime/scripts/create_test e3sm_scream_v1_at -c -b master --wait) if [[ $? != 0 ]]; then fails=$fails+1; v1_fail=1 diff --git a/components/eamxx/scripts/jenkins/pm-gpu_setup b/components/eamxx/scripts/jenkins/pm-gpu_setup new file mode 100644 index 000000000000..7bc04f72f9da --- /dev/null +++ b/components/eamxx/scripts/jenkins/pm-gpu_setup @@ -0,0 +1,2 @@ +source /global/common/software/e3sm/anaconda_envs/load_latest_cime_env.sh +SCREAM_MACHINE=pm-gpu diff --git a/components/eamxx/scripts/jenkins/scratch_cleanup.py b/components/eamxx/scripts/jenkins/scratch_cleanup.py new file mode 100755 index 000000000000..0195195a157e --- /dev/null +++ b/components/eamxx/scripts/jenkins/scratch_cleanup.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 + +""" +Clean up old files in the scratch area for mappy. +""" + +from pathlib import Path +import re, time, shutil, sys, argparse + +############################################################################### +def parse_command_line(args, description): +############################################################################### + parser = argparse.ArgumentParser( + usage="""\n{0} [-c HOURS] +OR +{0} --help + +\033[1mEXAMPLES:\033[0m + \033[1;32m# Purge files older than 20 hours \033[0m + > {0} -c 20 +""".format(Path(args[0]).name), + description=description, + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument("-c", "--cutoff", type=int, default=30, help="The cutoff age for purging in hours") + + parser.add_argument("-d", "--dry-run", action="store_true", help="Do a dry run, don't actually remove files") + + args = parser.parse_args(args[1:]) + + return args + +############################################################################### +def scratch_cleanup(cutoff, dry_run): +############################################################################### + scratch = Path('/ascldap/users/e3sm-jenkins/acme/scratch') + + timestamp_re = re.compile(r'.*(20[0-9]{6}_[0-9]{6}).*') + timestamps = set() + for item in scratch.iterdir(): + basename = item.name + re_match = timestamp_re.match(basename) + if re_match: + timestamps.add(re_match.groups()[0]) + + tformat = "%Y%m%d_%H%M%S" + curr_time = time.time() + + for timestamp in timestamps: + timestamp_time = time.mktime(time.strptime(timestamp, tformat)) + age_in_hours = (curr_time - timestamp_time) / 3600 + if age_in_hours > cutoff: + print(f"Timestamp {timestamp} is {age_in_hours} hours old and corresponding files will be removed") + files_to_remove = scratch.glob(f"*{timestamp}*") + for file_to_remove in files_to_remove: + print(f" Removing {file_to_remove}") + if not dry_run: + if file_to_remove.is_dir(): + shutil.rmtree(file_to_remove) + else: + file_to_remove.unlink() + + return True + +############################################################################### +def _main_func(description): +############################################################################### + success = scratch_cleanup(**vars(parse_command_line(sys.argv, description))) + + sys.exit(0 if success else 1) + +############################################################################### + +if __name__ == "__main__": + _main_func(__doc__) diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index cd717cba6b97..df89ae7b5add 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -4,6 +4,8 @@ ensure_psutil() import psutil +CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","cime") + # MACHINE -> (env_setup, # list of shell commands to set up scream-approved env # compilers, # list of compilers [CXX, F90, C] # batch submit prefix, # string shell commmand prefix @@ -26,22 +28,22 @@ ["mpicxx","mpifort","mpicc"], "bsub -I -q rhel8 -n 4 -gpu num=4", "/home/projects/e3sm/scream/pr-autotester/master-baselines/weaver/"), - "mappy" : (["module purge", "module load sems-archive-env acme-env acme-cmake/3.26.3 sems-archive-gcc/9.2.0 sems-archive-git/2.10.1 acme-openmpi/4.0.7 acme-netcdf/4.7.4/acme"], + "mappy" : (["module purge", "module load sems-archive-env acme-env acme-cmake/3.26.3 acme-gcc/11.2.0 sems-archive-git/2.10.1 acme-openmpi/4.1.4 acme-netcdf/4.7.4/acme", "export GATOR_INITIAL_MB=4000MB"], ["mpicxx","mpifort","mpicc"], "", "/sems-data-store/ACME/baselines/scream/master-baselines"), "lassen" : (["module --force purge", "module load git gcc/8.3.1 cuda/11.8.0 cmake/3.16.8 spectrum-mpi python/3.7.2", "export LLNL_USE_OMPI_VARS='y'", - "export PATH=/usr/gdata/climdat/netcdf/bin:$PATH", - "export LD_LIBRARY_PATH=/usr/gdata/climdat/netcdf/lib:$LD_LIBRARY_PATH", + "export PATH=/usr/gdata/e3sm/netcdf/bin:$PATH", + "export LD_LIBRARY_PATH=/usr/gdata/e3sm/netcdf/lib:$LD_LIBRARY_PATH", ], ["mpicxx","mpifort","mpicc"], "bsub -Ip -qpdebug", ""), - "ruby-intel" : (["module --force purge", "module use --append /usr/gdata/climdat/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], + "ruby-intel" : (["module --force purge", "module use --append /usr/gdata/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], ["mpicxx","mpifort","mpicc"], "salloc --partition=pdebug", ""), - "quartz-intel" : (["module --force purge", "module use --append /usr/gdata/climdat/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], + "quartz-intel" : (["module --force purge", "module use --append /usr/gdata/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], ["mpicxx","mpifort","mpicc"], "salloc --partition=pdebug", ""), @@ -57,15 +59,19 @@ ["mpicxx","mpifort","mpicc"], "bsub -I -q batch -W 0:30 -P cli115 -nnodes 1", "/gpfs/alpine/cli115/proj-shared/scream/master-baselines"), - "pm-gpu" : (["module load PrgEnv-gnu gcc/10.3.0 cudatoolkit craype-accel-nvidia80 cray-libsci craype cray-mpich cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf cmake evp-patch","module unload craype-accel-host perftools-base perftools darshan", "export NVCC_WRAPPER_DEFAULT_COMPILER=CC", "export NVCC_WRAPPER_DEFAULT_ARCH=sm_80"], + "pm-cpu" : ([f"eval $({CIMEROOT}/CIME/Tools/get_case_env -c SMS.ne4pg2_ne4pg2.F2010-SCREAMv1.pm-cpu_gnu)"], + ["CC","ftn","cc"], + "salloc --time 00:30:00 --nodes=1 --constraint=cpu -q debug --account e3sm_g", + "/global/cfs/cdirs/e3sm/baselines/gnu/scream/pm-cpu"), + "pm-gpu" : ([f"eval $({CIMEROOT}/CIME/Tools/get_case_env -c SMS.ne4pg2_ne4pg2.F2010-SCREAMv1.pm-gpu_gnugpu)", "echo cuda=true"], ["CC","ftn","cc"], - "srun --time 00:30:00 --nodes=1 --constraint=gpu --exclusive -q regular --account e3sm_g", - ""), + "salloc --time 02:00:00 --nodes=4 --constraint=gpu --gpus-per-node=4 --gpu-bind=none --exclusive -q regular --account e3sm_g", + "/global/cfs/cdirs/e3sm/baselines/gnugpu/scream/pm-gpu"), "compy" : (["module purge", "module load cmake/3.19.6 gcc/8.1.0 mvapich2/2.3.1 python/3.7.3"], ["mpicxx","mpifort","mpicc"], "srun --time 02:00:00 --nodes=1 -p short --exclusive --account e3sm", ""), - "chrysalis" : (["eval $(../../cime/CIME/Tools/get_case_env)", "export OMP_NUM_THREADS=1"], + "chrysalis" : ([f"eval $({CIMEROOT}/CIME/Tools/get_case_env)", "export OMP_NUM_THREADS=1"], ["mpic++","mpif90","mpicc"], "srun --mpi=pmi2 -l -N 1 --kill-on-bad-exit --cpu_bind=cores", "/lcrc/group/e3sm/baselines/chrys/intel/scream"), @@ -193,10 +199,20 @@ def get_mach_testing_resources(machine): of jobs across cores. """ if is_cuda_machine(machine): - return int(run_cmd_no_fail("nvidia-smi -L | wc -l")) + prefix = "srun " if is_salloc(machine) else "" + return int(run_cmd_no_fail(f"{prefix}nvidia-smi -L | wc -l")) else: return get_available_cpu_count() +############################################################################### +def is_salloc(machine): +############################################################################### + """ + Return true if we are running on an salloc'd job. + """ + bcmd = get_mach_batch_command(machine) + return "salloc" in bcmd and "srun" not in bcmd + ############################################################################### def is_cuda_machine(machine): ############################################################################### diff --git a/components/eamxx/scripts/scripts-tests b/components/eamxx/scripts/scripts-tests index b70a697a7d55..b5af87320243 100755 --- a/components/eamxx/scripts/scripts-tests +++ b/components/eamxx/scripts/scripts-tests @@ -3,14 +3,7 @@ """ Script containing python test suite for SCREAM test infrastructure. This suite should be run to confirm overall -correctness. You should run this test once in generation mode to -generate baseline results using your reference commit (common -ancestor) and once in comparison mode to compare against these -baselines using your development commit. Baseline and compare runs -will use dry-run modes so we are only comparing hypothetical shell -commands, not actually running them. - -You can also do a full run which will actually execute the commands. +correctness. If you are on a batch machine, it is expected that you are on a compute node. @@ -26,43 +19,17 @@ from machines_specs import is_machine_supported, is_cuda_machine from git_utils import get_current_branch, get_current_commit, get_current_head, git_refs_difference, \ is_repo_clean, get_common_ancestor, checkout_git_ref, get_git_toplevel_dir -import unittest, argparse, sys, difflib, shutil, os +import unittest, argparse, sys, shutil, os from pathlib import Path # Globals TEST_DIR = Path(__file__).resolve().parent CONFIG = { "machine" : None, - "compare" : False, - "generate" : False, "full" : False, "jenkins" : False } -############################################################################### -def run_cmd_store_output(test_obj, cmd, output_file): -############################################################################### - output_file.parent.mkdir(parents=True, exist_ok=True) - output = run_cmd_assert_result(test_obj, cmd, from_dir=TEST_DIR) - head = get_current_head() - output = output.replace(head, "CURRENT_HEAD_NORMALIZED") - output_file.write_text(output) - -############################################################################### -def run_cmd_check_baseline(test_obj, cmd, baseline_path): -############################################################################### - test_obj.assertTrue(baseline_path.is_file(), msg="Missing baseline {}".format(baseline_path)) - output = run_cmd_assert_result(test_obj, cmd, from_dir=TEST_DIR) - head = get_current_head() - output = output.replace(head, "CURRENT_HEAD_NORMALIZED") - diff = difflib.unified_diff( - baseline_path.read_text().splitlines(), - output.splitlines(), - fromfile=str(baseline_path), - tofile=cmd) - diff_output = "\n".join(diff) - test_obj.assertEqual("", diff_output, msg=diff_output) - ############################################################################### def test_cmake_cache_contents(test_obj, build_name, cache_var, expected_value): ############################################################################### @@ -150,8 +117,6 @@ class TestBaseOuter: # Hides the TestBase class from test scanner self._source_file = source_file self._cmds = list(cmds) self._machine = CONFIG["machine"] - self._compare = CONFIG["compare"] - self._generate = CONFIG["generate"] self._full = CONFIG["full"] self._jenkins = CONFIG["jenkins"] @@ -160,42 +125,20 @@ class TestBaseOuter: # Hides the TestBase class from test scanner self._results = TEST_DIR.joinpath("results") self._results.mkdir(parents=True, exist_ok=True) # pylint: disable=no-member - def get_baseline(self, cmd, machine): - return self._results.joinpath(self._source_file).with_suffix("").\ - joinpath(machine, self.get_cmd(cmd, machine, dry_run=False).translate(str.maketrans(" /='", "____"))) - - def get_cmd(self, cmd, machine, dry_run=True): - return "{}{}".format(cmd.replace("$machine", machine).replace("$results", str(self._results)), - " --dry-run" if (dry_run and "--dry-run" not in cmd) else "") + def get_cmd(self, cmd, machine): + return cmd.replace("$machine", machine).replace("$results", str(self._results)) def test_doctests(self): run_cmd_assert_result(self, "python3 -m doctest {}".format(self._source_file), from_dir=TEST_DIR) def test_pylint(self): ensure_pylint() - run_cmd_assert_result(self, "python3 -m pylint --disable C --disable R {}".format(self._source_file), from_dir=TEST_DIR, verbose=True) - - def test_gen_baseline(self): - if self._generate: - for cmd in self._cmds: - run_cmd_store_output(self, self.get_cmd(cmd, self._machine), self.get_baseline(cmd, self._machine)) - else: - self.skipTest("Skipping dry run baseline generation") - - def test_cmp_baseline(self): - if self._compare: - for cmd in self._cmds: - if "-p" in cmd: - # Parallel builds can generate scrambled output. Skip them. - continue - run_cmd_check_baseline(self, self.get_cmd(cmd, self._machine), self.get_baseline(cmd, self._machine)) - else: - self.skipTest("Skipping dry run baseline comparison") + run_cmd_assert_result(self, "python3 -m pylint --disable C --disable R {}".format(self._source_file), from_dir=TEST_DIR) def test_full(self): if self._full: for cmd in self._cmds: - run_cmd_assert_result(self, self.get_cmd(cmd, self._machine, dry_run=False), from_dir=TEST_DIR) + run_cmd_assert_result(self, self.get_cmd(cmd, self._machine), from_dir=TEST_DIR) else: self.skipTest("Skipping full run") @@ -271,9 +214,8 @@ class TestTestAllScream(TestBaseOuter.TestBase): ############################################################################### CMDS_TO_TEST = [ - "./test-all-scream -m $machine -b HEAD -k -p", - "./test-all-scream -m $machine -b HEAD -k -t dbg", - "./test-all-scream --baseline-dir $results -p -c EKAT_DISABLE_TPL_WARNINGS=ON -i -m $machine --submit --dry-run", # always dry run + "./test-all-scream -m $machine -p -i -c EKAT_DISABLE_TPL_WARNINGS=ON", + "./test-all-scream -m $machine -t dbg", ] def __init__(self, *internal_args): @@ -288,9 +230,9 @@ class TestTestAllScream(TestBaseOuter.TestBase): Test the 'dbg' test in test-all-scream. It should set certain CMake values """ if not self._jenkins: - options = "-b HEAD -k -t dbg --config-only" + options = "-t dbg --config-only" cmd = self.get_cmd("./test-all-scream -m $machine {}".format(options), - self._machine, dry_run=False) + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_cmake_cache_contents(self, "full_debug", "CMAKE_BUILD_TYPE", "Debug") test_cmake_cache_contents(self, "full_debug", "SCREAM_DOUBLE_PRECISION", "TRUE") @@ -307,9 +249,9 @@ class TestTestAllScream(TestBaseOuter.TestBase): Test the 'sp' test in test-all-scream. It should set certain CMake values """ if not self._jenkins: - options = "-b HEAD -k -t sp --config-only" + options = "-t sp --config-only" cmd = self.get_cmd("./test-all-scream -m $machine {}".format(options), - self._machine, dry_run=False) + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_cmake_cache_contents(self, "full_sp_debug", "CMAKE_BUILD_TYPE", "Debug") test_cmake_cache_contents(self, "full_sp_debug", "SCREAM_DOUBLE_PRECISION", "FALSE") @@ -329,9 +271,9 @@ class TestTestAllScream(TestBaseOuter.TestBase): if is_cuda_machine(self._machine): self.skipTest("Skipping FPE check on cuda") else: - options = "-b HEAD -k -t fpe --config-only" + options = "-t fpe --config-only" cmd = self.get_cmd("./test-all-scream -m $machine {}".format(options), - self._machine, dry_run=False) + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_cmake_cache_contents(self, "debug_nopack_fpe", "CMAKE_BUILD_TYPE", "Debug") test_cmake_cache_contents(self, "debug_nopack_fpe", "SCREAM_DOUBLE_PRECISION", "TRUE") @@ -345,9 +287,9 @@ class TestTestAllScream(TestBaseOuter.TestBase): Test the mem (default mem-check build) in test-all-scream. It should set certain CMake values """ if not self._jenkins: - options = "-b HEAD -k -t mem --config-only" + options = "-t mem --config-only" cmd = self.get_cmd("./test-all-scream -m $machine {}".format(options), - self._machine, dry_run=False) + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) builddir = "compute_sanitizer_memcheck" if is_cuda_machine(self._machine) else "valgrind" test_cmake_cache_contents(self, builddir, "CMAKE_BUILD_TYPE", "Debug") @@ -365,9 +307,9 @@ class TestTestAllScream(TestBaseOuter.TestBase): Test the 'opt' test in test-all-scream. It should set certain CMake values """ if not self._jenkins: - options = "-b HEAD -k -t opt --config-only" + options = "-t opt --config-only" cmd = self.get_cmd("./test-all-scream -m $machine {}".format(options), - self._machine, dry_run=False) + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_cmake_cache_contents(self, "release", "CMAKE_BUILD_TYPE", "Release") else: @@ -379,7 +321,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): Test that the 'dbg' test in test-all-scream detects and returns non-zero if there's a cmake configure error """ if self._full: - cmd = self.get_cmd("./test-all-scream -e SCREAM_FORCE_CONFIG_FAIL=True -m $machine -b HEAD -k -t dbg", self._machine, dry_run=False) + cmd = self.get_cmd("./test-all-scream -e SCREAM_FORCE_CONFIG_FAIL=True -m $machine -t dbg", self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR, expect_works=False) else: self.skipTest("Skipping full run") @@ -389,7 +331,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): Test that the 'dbg' test in test-all-scream detects and returns non-zero if there's a build error """ if self._full: - cmd = self.get_cmd("./test-all-scream -e SCREAM_FORCE_BUILD_FAIL=True -m $machine -b HEAD -k -t dbg", self._machine, dry_run=False) + cmd = self.get_cmd("./test-all-scream -e SCREAM_FORCE_BUILD_FAIL=True -m $machine -t dbg", self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR, expect_works=False) else: self.skipTest("Skipping full run") @@ -399,7 +341,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): Test that a failure from ctest in the testing phase is caught by test-all-scream """ if self._full: - cmd = self.get_cmd("./test-all-scream -e SCREAM_FORCE_RUN_FAIL=True -m $machine -b HEAD -k -t dbg", self._machine, dry_run=False) + cmd = self.get_cmd("./test-all-scream -e SCREAM_FORCE_RUN_FAIL=True -m $machine -t dbg", self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR, expect_works=False) else: self.skipTest("Skipping full run") @@ -409,7 +351,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): Test that the at test level works """ if self._full: - cmd = self.get_cmd("./test-all-scream -x -m $machine -b HEAD -k -t dbg", self._machine, dry_run=False) + cmd = self.get_cmd("./test-all-scream -x -m $machine -t dbg", self._machine) output = run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_test_levels_were_run(self, output, ["AT"], ["NIGHTLY", "EXPERIMENTAL"]) else: @@ -420,7 +362,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): Test that the nightly test level works """ if self._full: - cmd = self.get_cmd("./test-all-scream -x --test-level=nightly -m $machine -b HEAD -k -t dbg", self._machine, dry_run=False) + cmd = self.get_cmd("./test-all-scream -x --test-level=nightly -m $machine -t dbg", self._machine) output = run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_test_levels_were_run(self, output, ["AT", "NIGHTLY"], ["EXPERIMENTAL"]) else: @@ -431,7 +373,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): Test that the experimental test level works """ if self._full: - cmd = self.get_cmd("./test-all-scream -x --test-level=experimental -m $machine -b HEAD -k -t dbg", self._machine, dry_run=False) + cmd = self.get_cmd("./test-all-scream -x --test-level=experimental -m $machine -t dbg", self._machine) output = run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_test_levels_were_run(self, output, ["AT", "NIGHTLY", "EXPERIMENTAL"], []) else: @@ -445,13 +387,12 @@ class TestTestAllScream(TestBaseOuter.TestBase): So set SCREAM_FAKE_ONLY=ON, to lower the build time """ if self._full: - baseline_dir = TEST_DIR.parent/"ctest-build"/"baselines" - # Start a couple new tests, baselines will be generated + # Start a couple new tests, baselines will be generated in ctest-build/baselines env = "SCREAM_FAKE_ONLY=ON SCREAM_FAKE_GIT_HEAD=FAKE1" - opts = "-b HEAD -k -t dbg -t sp --no-tests" - cmd = self.get_cmd("{} ./test-all-scream -m $machine {}".format(env,opts), - self._machine, dry_run=False) + opts = " -g -t dbg -t sp --no-tests" + cmd = self.get_cmd(f"{env} ./test-all-scream -m $machine {opts}", + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_baseline_has_sha(self, TEST_DIR, "full_debug", "FAKE1") @@ -459,39 +400,42 @@ class TestTestAllScream(TestBaseOuter.TestBase): # Re-run reusing baselines from above env = "SCREAM_FAKE_ONLY=ON SCREAM_FAKE_GIT_HEAD=FAKE2" - opts = "--baseline-dir={} -b HEAD -k -t dbg -t sp --no-tests".format(baseline_dir) - cmd = self.get_cmd("{} ./test-all-scream -m $machine {}".format(env,opts), - self._machine, dry_run=False) + opts = "--baseline-dir LOCAL -t dbg -t sp --no-tests" + cmd = self.get_cmd(f"{env} ./test-all-scream -m $machine {opts}", + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_baseline_has_sha(self, TEST_DIR, "full_debug", "FAKE1") test_baseline_has_sha(self, TEST_DIR, "full_sp_debug", "FAKE1") # Re-run dbg reusing baselines from above with a fake commit that's not ahead + # The flag -u implies -g, but nothing should happen, since SCREAM_FAKE_AHEAD=0 env = "SCREAM_FAKE_ONLY=ON SCREAM_FAKE_AHEAD=0 SCREAM_FAKE_GIT_HEAD=FAKE2" - opts = "--baseline-dir={} -b HEAD -k -t dbg -u --no-tests".format(baseline_dir) - cmd = self.get_cmd("{} ./test-all-scream -m $machine {}".format(env,opts), - self._machine, dry_run=False) + opts = "--baseline-dir LOCAL -t dbg -u --no-tests" + cmd = self.get_cmd(f"{env} ./test-all-scream -m $machine {opts}", + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_baseline_has_sha(self, TEST_DIR, "full_debug", "FAKE1") test_baseline_has_sha(self, TEST_DIR, "full_sp_debug", "FAKE1") # Re-run dbg reusing baselines from above but expire them + # The flag -u implies -g, and since SCREAM_FAKE_AHEAD=1, baseline should be regenerated env = "SCREAM_FAKE_ONLY=ON SCREAM_FAKE_AHEAD=1 SCREAM_FAKE_GIT_HEAD=FAKE2" - opts = "--baseline-dir={} -b HEAD -k -t dbg -u --no-tests".format(baseline_dir) - cmd = self.get_cmd("{} ./test-all-scream -m $machine {}".format(env,opts), - self._machine, dry_run=False) + opts = "--baseline-dir LOCAL -t dbg -u --no-tests" + cmd = self.get_cmd(f"{env} ./test-all-scream -m $machine {opts}", + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_baseline_has_sha(self, TEST_DIR, "full_debug", "FAKE2") test_baseline_has_sha(self, TEST_DIR, "full_sp_debug", "FAKE1") # Re-run reusing some baselines and expiring others + # The dbg baselines were generated in the prev step, so this should only gen the sp baselines env = "SCREAM_FAKE_ONLY=ON SCREAM_FAKE_AHEAD=1 SCREAM_FAKE_GIT_HEAD=FAKE2" - opts = "--baseline-dir={} -b HEAD -k -t dbg -t sp -u --no-tests".format(baseline_dir) - cmd = self.get_cmd("{} ./test-all-scream -m $machine {}".format(env,opts), - self._machine, dry_run=False) + opts = "--baseline-dir LOCAL -t dbg -t sp -u --no-tests" + cmd = self.get_cmd(f"{env} ./test-all-scream -m $machine {opts}", + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_baseline_has_sha(self, TEST_DIR, "full_debug", "FAKE2") @@ -499,9 +443,9 @@ class TestTestAllScream(TestBaseOuter.TestBase): # Re-run without reusing baselines, should force regeneration env = "SCREAM_FAKE_ONLY=ON SCREAM_FAKE_GIT_HEAD=FAKE3" - opts = "-b HEAD -k -t dbg -t sp --no-tests" - cmd = self.get_cmd("{} ./test-all-scream -m $machine {}".format(env,opts), - self._machine, dry_run=False) + opts = "-g -t dbg -t sp --no-tests" + cmd = self.get_cmd(f"{env} ./test-all-scream -m $machine {opts}", + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_baseline_has_sha(self, TEST_DIR, "full_debug", "FAKE3") @@ -518,21 +462,13 @@ class TestTestAllScream(TestBaseOuter.TestBase): can manage resources correctly. """ if self._full and self._machine == "mappy": - spread_test_opts = "-x -e SCREAM_TEST_THREAD_SPREAD=True -c EKAT_TEST_LAUNCHER_MANAGE_RESOURCES=True -c EKAT_MPIRUN_EXE=mpiexec -c EKAT_MPI_EXTRA_ARGS='-bind-to core' -c EKAT_MPI_NP_FLAG='--map-by' -c EKAT_MPI_THREAD_FLAG='' -m $machine -b HEAD -k -t dbg" + spread_test_opts = "-x -e SCREAM_TEST_THREAD_SPREAD=True -e SCREAM_TEST_RANK_SPREAD=True -c EKAT_TEST_LAUNCHER_MANAGE_RESOURCES=True -c EKAT_MPIRUN_EXE=mpiexec -c EKAT_MPI_EXTRA_ARGS='-bind-to core' -c EKAT_MPI_NP_FLAG='--map-by' -c EKAT_MPI_THREAD_FLAG='' -m $machine -t dbg" - cmd = self.get_cmd(f"./test-all-scream {spread_test_opts} --ctest-parallel-level=40 ", self._machine, dry_run=False) + cmd = self.get_cmd(f"./test-all-scream {spread_test_opts} --ctest-parallel-level=40 ", self._machine) output = run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_omp_spread(self, output, 40) - cmd = self.get_cmd(f"./test-all-scream {spread_test_opts} --ctest-parallel-level=40 ", self._machine, dry_run=False) - output = run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) - test_omp_spread(self, output, 40) - - cmd = self.get_cmd(f"taskset -c 8-47 ./test-all-scream {spread_test_opts} --ctest-parallel-level=40 ", self._machine, dry_run=False) - output = run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) - test_omp_spread(self, output, 48, begin=8) - - cmd = self.get_cmd(f"taskset -c 8-47 ./test-all-scream {spread_test_opts} --ctest-parallel-level=40 ", self._machine, dry_run=False) + cmd = self.get_cmd(f"taskset -c 8-47 ./test-all-scream {spread_test_opts} --ctest-parallel-level=40 ", self._machine) output = run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_omp_spread(self, output, 48, begin=8) @@ -547,7 +483,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): # We set PULLREQUESTNUM to block dashboard submission # We set SCREAM_FAKE_AUTO to not interere with real baselines cmd = self.get_cmd("PR_LABELS= NODE_NAME={} SCREAM_FAKE_AUTO=TRUE PULLREQUESTNUM=42 ./jenkins/jenkins_common.sh".format(self._machine), - self._machine, dry_run=False) + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) else: @@ -561,7 +497,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): # We set PULLREQUESTNUM to block dashboard submission # We set SCREAM_FAKE_AUTO to not interere with real baselines cmd = self.get_cmd("PR_LABELS= NODE_NAME={} SCREAM_FAKE_AUTO=TRUE PULLREQUESTNUM= ./jenkins/jenkins_common.sh".format(self._machine), - self._machine, dry_run=False) + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) else: @@ -574,7 +510,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): if self._jenkins: # Any fail will do, we already checked test-all-scream captures all the fail types cmd = self.get_cmd("PR_LABELS= SCREAM_FORCE_CONFIG_FAIL=True NODE_NAME={} SCREAM_FAKE_AUTO=TRUE PULLREQUESTNUM=42 ./jenkins/jenkins_common.sh".format(self._machine), - self._machine, dry_run=False) + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR, expect_works=False) else: @@ -587,7 +523,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): if self._jenkins: # Any fail will do, we already checked test-all-scream captures all the fail types cmd = self.get_cmd("PR_LABELS= SCREAM_FORCE_CONFIG_FAIL=True NODE_NAME={} SCREAM_FAKE_AUTO=TRUE PULLREQUESTNUM= ./jenkins/jenkins_common.sh".format(self._machine), - self._machine, dry_run=False) + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR, expect_works=False) else: @@ -598,7 +534,7 @@ class TestGatherAllData(TestBaseOuter.TestBase): ############################################################################### CMDS_TO_TEST = [ - "./gather-all-data './scripts/test-all-scream -m $machine -b HEAD -k' -l -m $machine", + "./gather-all-data 'echo -m $machine' -l -m $machine", ] def __init__(self, *internal_args): @@ -637,27 +573,10 @@ OR \033[1;32m# Run pylint tests for test_all_scream \033[0m > {0} TestTestAllScream.test_pylint - \033[1;32m# Do a dry-run generation for test_all_scream \033[0m - > {0} -g TestTestAllScream -m $machine - - \033[1;32m# Do a dry-run comparison for test_all_scream \033[0m - > {0} -c TestTestAllScream -m $machine - \033[1;32m# Do a full test run of test_all_scream \033[0m > {0} -f -m $machine TestTestAllScream - \033[1;32m# Do a full test run of everything \033[0m - > {0} -f -m $machine - - \033[1;32m# Do a dry-run generation for everything \033[0m - > {0} -g -m $machine - - \033[1;32m# Do a dry-run comparison for comparison \033[0m - > {0} -c -m $machine - \033[1;32m# Run every possible test. This should be done before a PR is issued \033[0m - > {0} -g -m $machine # You likely want to do this for a reference commit - > {0} -c -m $machine > {0} -f -m $machine \033[1;32m# Test Jenkins script \033[0m @@ -673,12 +592,6 @@ OR parser.add_argument("-m", "--machine", help="Provide machine name. This is required for full (not dry) runs") - parser.add_argument("-g", "--generate", action="store_true", - help="Do a dry run with baseline generation") - - parser.add_argument("-c", "--compare", action="store_true", - help="Do a dry run with baseline comparison") - parser.add_argument("-f", "--full", action="store_true", help="Do a full (not dry) run") @@ -691,7 +604,7 @@ OR return args ############################################################################### -def scripts_tests(machine=None, generate=False, compare=False, full=False, jenkins=False): +def scripts_tests(machine=None, full=False, jenkins=False): ############################################################################### os.environ["SCREAM_FAKE_ONLY"] = "True" @@ -700,9 +613,6 @@ def scripts_tests(machine=None, generate=False, compare=False, full=False, jenki expect(is_machine_supported(machine), "Machine {} is not supported".format(machine)) CONFIG["machine"] = machine - expect(not (generate and compare), "Cannot do generate and compare in the same run") - CONFIG["compare"] = compare - CONFIG["generate"] = generate CONFIG["jenkins"] = jenkins if full: diff --git a/components/eamxx/scripts/test-all-scream b/components/eamxx/scripts/test-all-scream index 4a6d0710b4e3..bb55ad23c216 100755 --- a/components/eamxx/scripts/test-all-scream +++ b/components/eamxx/scripts/test-all-scream @@ -16,6 +16,7 @@ check_minimum_python_version(3, 4) import argparse, sys, pathlib +from test_factory import get_test_name_dict from test_all_scream import TestAllScream ############################################################################### @@ -35,10 +36,6 @@ OR > cd $scream_repo/components/eamxx > ./scripts/{0} --preserve-env -m melvin - \033[1;32m# Run all tests on current machine with default behavior except using a custom ref for baseline generation\033[0m - > cd $scream_repo/components/eamxx - > ./scripts/{0} -m melvin -b BASELINE_REF - \033[1;32m# Run all tests on current machine with default behavior except using pre-existing baselines (skips baseline generation) \033[0m > cd $scream_repo/components/eamxx > ./scripts/{0} -m melvin --baseline-dir=PATH_TO_BASELINES @@ -49,7 +46,7 @@ OR \033[1;32m# Run all tests on current machine with default behavior on a repo with uncommitted changes\033[0m > cd $scream_repo/components/eamxx - > ./scripts/{0} -m melvin -k -b HEAD + > ./scripts/{0} -m melvin -k -b AUTO """.format(pathlib.Path(args[0]).name), description=description, formatter_class=argparse.ArgumentDefaultsHelpFormatter @@ -63,17 +60,14 @@ OR parser.add_argument("-p", "--parallel", action="store_true", help="Launch the different build types stacks in parallel") - parser.add_argument("-f", "--fast-fail", action="store_true", - help="Stop testing when the first failure is detected") - - parser.add_argument("-b", "--baseline-ref", default=None, # default will be computed later - help="What commit to use to generate baselines. Default is merge-base of current commit and origin/master (or HEAD if --keep-tree)") + parser.add_argument("-g", "--generate", action="store_true", + help="Instruct test-all-scream to generate baselines from current commit. Skips tests") - parser.add_argument("--baseline-dir", default=None, - help="Use baselines from the given directory, skip baseline creation.") + parser.add_argument("-b", "--baseline-dir", default=None, + help="Directory where baselines should be read from (or written to, if -g/-i is used)") parser.add_argument("-u", "--update-expired-baselines", action="store_true", - help="Update baselines that appear to be expired.") + help="Update baselines that appear to be expired (only used with -g)") parser.add_argument("-m", "--machine", help="Provide machine name. This is *always* required. It can, but does not" @@ -86,9 +80,6 @@ OR parser.add_argument("--config-only", action="store_true", help="In the testing phase, only run config step, skip build and tests") - parser.add_argument("-k", "--keep-tree", action="store_true", - help="Allow to keep the current work tree when testing against HEAD (only valid with `-b HEAD`)") - parser.add_argument("-c", "--custom-cmake-opts", action="append", default=[], help="Extra custom options to pass to cmake. Can use multiple times for multiple cmake options. The -D is added for you") @@ -100,12 +91,12 @@ OR parser.add_argument("--preserve-env", action="store_true", help="Whether to skip machine env setup, and preserve the current user env (useful to manually test new modules)") - choices_doc = ", ".join(["'{}' ({})".format(k, v) for k, v in TestAllScream.get_test_name_desc().items()]) + choices_doc = ", ".join(["'{}' ({})".format(k, v) for k, v in get_test_name_dict().items()]) parser.add_argument("-t", "--test", dest="tests", action="append", default=[], help=f"Only run specific test configurations, choices={choices_doc}") parser.add_argument("-i", "--integration-test", action="store_true", - help="Merge origin/master into this branch before testing.") + help="Merge origin/master into this branch before testing (implies -u).") parser.add_argument("-l", "--local", action="store_true", help="Allow to not specify a machine name, and have test-all-scream to look" @@ -123,9 +114,6 @@ OR parser.add_argument("--quick-rerun-failed", action="store_true", help="Do not clean the build dir, and do not reconfigure. Just (incremental) build and retest failed tests only.") - parser.add_argument("-d", "--dry-run", action="store_true", - help="Do a dry run, commands will be printed but not executed") - parser.add_argument("--make-parallel-level", action="store", type=int, default=0, help="Max number of jobs to be created during compilation. If not provided, use default for given machine.") diff --git a/components/eamxx/scripts/test_all_scream.py b/components/eamxx/scripts/test_all_scream.py index dfade8beda0d..ffbe70e94f68 100644 --- a/components/eamxx/scripts/test_all_scream.py +++ b/components/eamxx/scripts/test_all_scream.py @@ -1,6 +1,10 @@ -from utils import run_cmd, run_cmd_no_fail, expect, check_minimum_python_version, ensure_psutil +from utils import run_cmd, run_cmd_no_fail, expect, check_minimum_python_version, ensure_psutil, \ + SharedArea, safe_copy from git_utils import get_current_head, get_current_commit, get_current_branch, is_repo_clean, \ - cleanup_repo, merge_git_ref, checkout_git_ref, git_refs_difference, print_last_commit + cleanup_repo, merge_git_ref, git_refs_difference, print_last_commit, \ + create_backup_commit, checkout_git_ref + +from test_factory import create_tests, COV from machines_specs import get_mach_compilation_resources, get_mach_testing_resources, \ get_mach_baseline_root_dir, setup_mach_env, is_cuda_machine, \ @@ -18,270 +22,19 @@ import psutil import re -from collections import OrderedDict from pathlib import Path -############################################################################### -class TestProperty(object): -############################################################################### - - """ - Parent class of predefined test types for SCREAM standalone. test-all-scream - offers a number of customization points, but you may need to just use - cmake if you need maximal customization. You can run test-all-scream --dry-run - to get the corresponding cmake command which can then be used as a starting - point for making your own cmake command. - """ - - def __init__(self, longname, description, cmake_args, - uses_baselines=True, on_by_default=True, default_test_len=None): - # What the user uses to select tests via test-all-scream CLI. - # Should also match the class name when converted to caps - self.shortname = type(self).__name__.lower() - - # A longer name used to name baseline and test directories for a test. - # Also used in output/error messages to refer to the test - self.longname = longname - - # A longer decription of the test - self.description = description - - # Cmake config args for this test. Check that quoting is done with - # single quotes. - self.cmake_args = cmake_args - for name, arg in self.cmake_args: - expect('"' not in arg, - f"In test definition for {longname}, found cmake args with double quotes {name}='{arg}'" - "Please use single quotes if quotes are needed.") - - # Does the test do baseline testing - self.uses_baselines = uses_baselines - - # Should this test be run if the user did not specify tests at all? - self.on_by_default = on_by_default - - # Should this test have a default test size - self.default_test_len = default_test_len - - # - # Properties not set by constructor (Set by the main TestAllScream object) - # - - # Resources used by this test. - self.compile_res_count = None - self.testing_res_count = None - - # Does this test need baselines - self.missing_baselines = False - - # - # Common - # - - if not self.uses_baselines: - self.cmake_args += [("SCREAM_ENABLE_BASELINE_TESTS", "False")] - - def disable_baselines(self): - if self.uses_baselines: - self.uses_baselines = False - self.cmake_args += [("SCREAM_ENABLE_BASELINE_TESTS", "False")] - - # Tests will generally be referred to via their longname - def __str__(self): - return self.longname - -############################################################################### -class DBG(TestProperty): -############################################################################### - - CMAKE_ARGS = [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_DEFAULT_BFB", "True")] - - def __init__(self, _): - TestProperty.__init__( - self, - "full_debug", - "debug", - self.CMAKE_ARGS, - ) - -############################################################################### -class SP(TestProperty): -############################################################################### - - def __init__(self, _): - TestProperty.__init__( - self, - "full_sp_debug", - "debug single precision", - DBG.CMAKE_ARGS + [("SCREAM_DOUBLE_PRECISION", "False")], - ) - -############################################################################### -class FPE(TestProperty): -############################################################################### - - def __init__(self, tas): - TestProperty.__init__( - self, - "debug_nopack_fpe", - "debug pksize=1 floating point exceptions on", - DBG.CMAKE_ARGS + [("SCREAM_PACK_SIZE", "1"), ("SCREAM_FPE","True")], - uses_baselines=False, - on_by_default=(tas is not None and not tas.on_cuda()) - ) - -############################################################################### -class OPT(TestProperty): -############################################################################### - - def __init__(self, _): - TestProperty.__init__( - self, - "release", - "release", - [("CMAKE_BUILD_TYPE", "Release")], - ) - -############################################################################### -class COV(TestProperty): -############################################################################### - - def __init__(self, _): - TestProperty.__init__( - self, - "coverage", - "debug coverage", - [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_ENABLE_COVERAGE", "True")], - uses_baselines=False, - on_by_default=False, - default_test_len="short" - ) - -############################################################################### -class VALG(TestProperty): -############################################################################### - - def __init__(self, tas): - TestProperty.__init__( - self, - "valgrind", - "debug with valgrind", - [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_ENABLE_VALGRIND", "True")], - uses_baselines=False, - on_by_default=False, - default_test_len="short" - ) - if tas is not None: - # If a stored suppression file exists for this machine, use it - persistent_supp_file = tas.get_root_dir() / "scripts" / "jenkins" / "valgrind" / f"{tas.get_machine()}.supp" - if persistent_supp_file.exists(): - self.cmake_args.append( ("EKAT_VALGRIND_SUPPRESSION_FILE", str(persistent_supp_file)) ) - -############################################################################### -class CSM(TestProperty): -############################################################################### - - def __init__(self, _): - TestProperty.__init__( - self, - "compute_sanitizer_memcheck", - "debug with compute sanitizer memcheck", - [("CMAKE_BUILD_TYPE", "Debug"), - ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), - ("EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=memcheck")], - uses_baselines=False, - on_by_default=False, - default_test_len="short" - ) - -############################################################################### -class CSR(TestProperty): -############################################################################### - - def __init__(self, _): - TestProperty.__init__( - self, - "compute_sanitizer_racecheck", - "debug with compute sanitizer racecheck", - [("CMAKE_BUILD_TYPE", "Debug"), - ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), - ("EKAT_COMPUTE_SANITIZER_OPTIONS", "'--tool=racecheck --racecheck-detect-level=error'")], - uses_baselines=False, - on_by_default=False, - default_test_len="short" - ) - -############################################################################### -class CSI(TestProperty): -############################################################################### - - def __init__(self, _): - TestProperty.__init__( - self, - "compute_sanitizer_initcheck", - "debug with compute sanitizer initcheck", - [("CMAKE_BUILD_TYPE", "Debug"), - ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), - ("EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=initcheck")], - uses_baselines=False, - on_by_default=False, - default_test_len="short" - ) - -############################################################################### -class CSS(TestProperty): -############################################################################### - - def __init__(self, _): - TestProperty.__init__( - self, - "compute_sanitizer_synccheck", - "debug with compute sanitizer synccheck", - [("CMAKE_BUILD_TYPE", "Debug"), - ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), - ("EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=synccheck")], - uses_baselines=False, - on_by_default=False, - default_test_len="short" - ) - -############################################################################### -def test_factory(user_req_tests, tas): -############################################################################### - testclasses = TestProperty.__subclasses__() - if not user_req_tests: - result = [testclass(tas) for testclass in testclasses - if testclass(tas).on_by_default] - else: - valid_names = [testclass(tas).shortname for testclass in testclasses] - for user_req_test in user_req_tests: - expect(user_req_test in valid_names, f"'{user_req_test}' is not a known test") - - result = [testclass(tas) for testclass in testclasses if testclass(tas).shortname in user_req_tests] - - return result - ############################################################################### class TestAllScream(object): ############################################################################### - ########################################################################### - @classmethod - def get_test_name_desc(cls): - ########################################################################### - """ - Returns a dict mapping short test names to full names - """ - testclasses = TestProperty.__subclasses__() - return OrderedDict([(testc(None).shortname, testc(None).description) for testc in testclasses]) - ########################################################################### def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, - submit=False, parallel=False, fast_fail=False, - baseline_ref=None, baseline_dir=None, machine=None, no_tests=False, config_only=False, keep_tree=False, + submit=False, parallel=False, generate=False, no_tests=False, + baseline_dir=None, machine=None, config_only=False, custom_cmake_opts=(), custom_env_vars=(), preserve_env=False, tests=(), integration_test=False, local=False, root_dir=None, work_dir=None, - quick_rerun=False,quick_rerun_failed=False,dry_run=False, + quick_rerun=False,quick_rerun_failed=False, make_parallel_level=0, ctest_parallel_level=0, update_expired_baselines=False, extra_verbose=False, limit_test_regex=None, test_level="at", test_size=None, force_baseline_regen=False): @@ -299,13 +52,10 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, self._c_compiler = c_compiler self._submit = submit self._parallel = parallel - self._fast_fail = fast_fail - self._baseline_ref = baseline_ref self._machine = machine self._local = local - self._perform_tests = not no_tests + self._run_tests = not no_tests self._config_only = config_only - self._keep_tree = keep_tree self._baseline_dir = baseline_dir self._custom_cmake_opts = custom_cmake_opts self._custom_env_vars = custom_env_vars @@ -315,15 +65,16 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, self._integration_test = integration_test self._quick_rerun = quick_rerun self._quick_rerun_failed = quick_rerun_failed - self._dry_run = dry_run - self._update_expired_baselines= update_expired_baselines self._extra_verbose = extra_verbose self._limit_test_regex = limit_test_regex self._test_level = test_level self._test_size = test_size self._force_baseline_regen = force_baseline_regen - - # Not all builds are ment to perform comparisons against pre-built baselines + # Integration test always updates expired baselines + self._update_expired_baselines= update_expired_baselines or self._integration_test or self._force_baseline_regen + # If we are to update expired baselines, then we must run the generate phase + # NOTE: the gen phase will do nothing if baselines are present and not expired + self._generate = generate or self._update_expired_baselines if self._quick_rerun_failed: self._quick_rerun = True @@ -355,30 +106,22 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, self._root_dir = Path(__file__).resolve().parent.parent else: self._root_dir = Path(self._root_dir).resolve() - expect(self._root_dir.is_dir() and self._root_dir.parts()[-2:] == ("scream", "components"), - f"Bad root-dir '{self._root_dir}', should be: $scream_repo/components/eamxx") + expect(self._root_dir.is_dir() and list(self._root_dir.parts)[-2:] == ["components","eamxx"], + f"Bad root-dir '{self._root_dir}', should end with: /components/eamxx") # Make our test objects! Change mem to default mem-check test for current platform if "mem" in tests: tests[tests.index("mem")] = "csm" if self.on_cuda() else "valg" - self._tests = test_factory(tests, self) + self._tests = create_tests(tests, self) if self._work_dir is not None: self._work_dir = Path(self._work_dir).absolute() - expect(self._work_dir.is_dir(), - f"Error! Work directory '{self._work_dir}' does not exist.") else: self._work_dir = self._root_dir.absolute().joinpath("ctest-build") - self._work_dir.mkdir(exist_ok=True) - os.chdir(str(self._root_dir)) # needed, or else every git command will need repo=root_dir - expect(get_current_commit(), f"Root dir: {self._root_dir}, does not appear to be a git repo") - - # Print some info on the branch - self._original_branch = get_current_branch() - self._original_commit = get_current_commit() + self._work_dir.mkdir(parents=True, exist_ok=True) - print_last_commit(git_ref=self._original_branch, dry_run=self._dry_run) + os.chdir(str(self._root_dir)) # needed, or else every git command will need repo=root_dir ################################### # Compilation/testing resources # @@ -445,80 +188,84 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, # Setup the env on this machine setup_mach_env(self._machine, ctest_j=ctest_max_jobs) + ############################################ + # Check repo status # + ############################################ + + expect(get_current_commit(), f"Root dir: {self._root_dir}, does not appear to be a git repo") + + # Get git status info. Besides printing this info, we will need it to restore the repo initial + # configuration if we are running an integration test (where baselines need to be created + # from the origin/master commit) + self._original_branch = get_current_branch() + self._original_commit = get_current_commit() + + print_last_commit(git_ref=self._original_branch) + + # If we have an integration test, we need to merge master. Hence, do two things: + # 1) create bkp commit for all uncommitted/unstaged changes + # 2) save commit, so we can undo the merge after testing + self._has_backup_commit = False + if self._integration_test: + if not is_repo_clean(): + # Back up work in a temporary commit + create_backup_commit() + self._has_backup_commit = True + + self._original_commit = get_current_commit() + ################################### # Compute baseline info # ################################### expect (not self._baseline_dir or self._work_dir != self._baseline_dir, - f"Error! For your safety, do NOT use '{self._work_dir}' to store baselines. Move them to a different directory (even a subdirectory if that works).") - - # If no baseline ref/dir was provided, use default master baseline dir for this machine - # NOTE: if user specifies baseline ref, baseline dir will be set later to a path within work dir - if self._baseline_dir is None and self._baseline_ref is None: - self._baseline_dir = "AUTO" - print ("No '--baseline-dir XYZ' nor '-b XYZ' provided. Testing against default baselines dir for this machine.") - - # If -k was used, make sure it's allowed - if self._keep_tree: - expect(not self._integration_test, "Should not be doing keep-tree with integration testing") - print("WARNING! You have uncommitted changes in your repo.", - " The PASS/FAIL status may depend on these changes", - " so if you want to keep them, don't forget to create a commit.",sep="\n") - if self._baseline_dir is None: - # Make sure the baseline ref is HEAD - expect(self._baseline_ref == "HEAD", - "The option --keep-tree is only available when testing against pre-built baselines " - "(--baseline-dir) or HEAD (-b HEAD)") + f"Error! For your safety, do NOT use '{self._work_dir}' (the work_dir) to store baselines. Move them to a different directory (even a subdirectory if that works).") + + # These two dir are special dir for "on-the-fly baselines" and "machine's official baselines" + local_baseline_dir = self._work_dir/"baselines" + auto_dir = Path(get_mach_baseline_root_dir(self._machine)).absolute() + # Handle the "fake" auto case, used in scripts tests + if "SCREAM_FAKE_AUTO" in os.environ: + auto_dir = auto_dir / "fake" + + if self._baseline_dir == "LOCAL": + self._baseline_dir = local_baseline_dir + elif self._baseline_dir == "AUTO": + self._baseline_dir = auto_dir + elif self._baseline_dir is None: + if self._generate and not self._integration_test: + print ("No '--baseline-dir XYZ' provided. Baselines will be generated in {local_baseline_dir}.") + print ("NOTE: test-all-scream will proceed as if --force-baseline-regen was passed") + self._baseline_dir = local_baseline_dir + self._force_baseline_regen = True + self._update_expired_baselines = True else: - # Make sure the baseline ref is unset (or HEAD) - expect(self._baseline_ref is None or self._baseline_ref == "HEAD", - "The option --keep-tree is only available when testing against pre-built baselines " - "(--baseline-dir) or HEAD (-b HEAD)") - else: - expect(self._dry_run or is_repo_clean(), - "Repo must be clean before running. If testing against HEAD or pre-built baselines, " - "you can pass `--keep-tree` to allow non-clean repo.") + print ("No '--baseline-dir XYZ' provided. Testing against default baselines dir for this machine.") + self._baseline_dir = auto_dir - # For integration test, enforce baseline_ref==origin/master, and proceed to merge origin/master - if self._integration_test: - expect (self._baseline_ref is None or self._baseline_ref=="origin/master", - "Error! Integration tests cannot be done against an arbitrary baseline ref.") + self._baseline_dir = Path(self._baseline_dir).absolute() - # Set baseline ref and merge it - self._baseline_ref = "origin/master" - merge_git_ref(git_ref=self._baseline_ref, verbose=True, dry_run=self._dry_run) + # Only integration tests can overwrite the mach-specific baselines + if self._baseline_dir==auto_dir: + expect (not self._generate or self._integration_test or self._force_baseline_regen, + "You are not allowed to overwrite baselines in AUTO dir folder. Only -i and --force-baseline-regen can do that\n" + f" AUTO dir: {auto_dir}") - # Always update expired baselines if this is an integration test - self._update_expired_baselines = True + # Make the baseline dir, if not already existing. + if self._generate: + self.create_tests_dirs(self._baseline_dir, clean=False) - # By now, we should have at least one between baseline_dir and baseline_ref set (possibly both) - default_baselines_root_dir = self._work_dir/"baselines" - if self._baseline_dir is None: - # Use default baseline dir, and create it if necessary - self._baseline_dir = Path(default_baselines_root_dir).absolute() - self.create_tests_dirs(self._baseline_dir, True) # Wipe out previous baselines + # For now, assume baselines are generated from HEAD. If -i was used, we'll change this + self._baseline_ref = "origin/master" if self._integration_test else self._original_commit - else: - if self._baseline_dir == "AUTO": - expect (self._baseline_ref is None or self._baseline_ref == "origin/master", - "Do not specify `-b XYZ` when using `--baseline-dir AUTO`. The AUTO baseline dir should be used for the master baselines only.\n" - " `-b XYZ` needs to probably build baselines for ref XYZ. However, no baselines will be built if the dir already contains baselines.\n") - # We treat the "AUTO" string as a request for automatic baseline dir. - auto_dir = get_mach_baseline_root_dir(self._machine) - self._baseline_dir = Path(auto_dir) if auto_dir else default_baselines_root_dir - if "SCREAM_FAKE_AUTO" in os.environ: - self._baseline_dir = self._baseline_dir/"fake" - else: - self._baseline_dir = Path(self._baseline_dir).absolute() - - # Make sure the baseline folders exist (but do not purge content if they exist) - self.create_tests_dirs(self._baseline_dir, False) - - # Do not do baseline operations if mem checking is on + # Check baselines status print (f"Checking baselines directory: {self._baseline_dir}") - self.baselines_are_present() + missing_baselines = self.check_baselines_are_present() + expect (len(missing_baselines)==0 or self._generate, + f"Missing baselines for builds {missing_baselines}. Re-run with -g to generate them") + if self._update_expired_baselines: - self.baselines_are_expired() + self.check_baselines_are_expired() ############################################ # Deduce compilers if needed/possible # @@ -531,16 +278,11 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, if self._c_compiler is None: self._c_compiler = get_mach_c_compiler(self._machine) - if not self._dry_run: - self._f90_compiler = run_cmd_no_fail(f"which {self._f90_compiler}") - self._cxx_compiler = run_cmd_no_fail(f"which {self._cxx_compiler}") - self._c_compiler = run_cmd_no_fail(f"which {self._c_compiler}") - ############################################################################### def create_tests_dirs(self, root, clean): ############################################################################### - # Make sure the baseline root directory exists + # Make sure the tests root directory exists root.mkdir(parents=True,exist_ok=True) # Create build directories (one per test) @@ -553,9 +295,11 @@ def create_tests_dirs(self, root, clean): # TypeError: lstat: illegal type for path parameter shutil.rmtree(str(test_dir)) - # Create this baseline's build dir - if not test_dir.exists(): - test_dir.mkdir(parents=True) + # Create this built type's build dir (if not already existing) + test_dir.mkdir(parents=True,exist_ok=True) + + # Create the 'data' subdir (if not already existing) + (test_dir / "data").mkdir(parents=False,exist_ok=True) ############################################################################### def get_baseline_file_sha(self, test): @@ -568,8 +312,9 @@ def get_baseline_file_sha(self, test): return None ############################################################################### - def set_baseline_file_sha(self, test, sha): + def set_baseline_file_sha(self, test): ############################################################################### + sha = get_current_commit() baseline_file = (self.get_preexisting_baseline(test).parent)/"baseline_git_sha" with baseline_file.open("w", encoding="utf-8") as fd: return fd.write(sha) @@ -601,7 +346,7 @@ def get_preexisting_baseline(self, test): return self._baseline_dir/str(test)/"data" ############################################################################### - def baselines_are_present(self): + def check_baselines_are_present(self): ############################################################################### """ Check that all baselines are present (one subdir for all values of self._tests) @@ -611,19 +356,23 @@ def baselines_are_present(self): expect(self._baseline_dir is not None, "Error! Baseline directory not correctly set.") + missing = [] for test in self._tests: if test.uses_baselines: data_dir = self.get_preexisting_baseline(test) if not data_dir.is_dir(): - test.missing_baselines = True + test.baselines_missing = True + missing += [test.longname] print(f" -> Test {test} is missing baselines") else: print(f" -> Test {test} appears to have baselines") else: print(f" -> Test {test} does not use baselines") + return missing + ############################################################################### - def baselines_are_expired(self): + def check_baselines_are_expired(self): ############################################################################### """ Baselines are expired if either: @@ -632,40 +381,50 @@ def baselines_are_expired(self): """ baseline_ref_sha = get_current_commit(commit=self._baseline_ref) - # Sanity check - expect(self._baseline_dir is not None, "Error! This routine should only be called when testing against pre-existing baselines.") - for test in self._tests: - if test.uses_baselines and not test.missing_baselines: - # this test is not missing a baseline, but it may be expired. - - baseline_file_sha = self.get_baseline_file_sha(test) - if baseline_file_sha is None: - test.missing_baselines = True - print(f" -> Test {test} has no stored sha so must be considered expired") - else: - num_ref_is_behind_file, num_ref_is_ahead_file = git_refs_difference(baseline_file_sha, baseline_ref_sha) - - # If the copy in our repo is behind, then we need to update the repo - expect (num_ref_is_behind_file==0 or not self._integration_test, + if not test.uses_baselines or test.baselines_missing: + continue + + if self._force_baseline_regen: + test.baselines_expired = True + print(f" -> Test {test} baselines are expired because self._force_baseline_regen=True") + continue + + # this test is not missing a baseline, but it may be expired. + baseline_file_sha = self.get_baseline_file_sha(test) + if baseline_file_sha is None: + test.baselines_missing = True + print(f" -> Test {test} has no stored sha so must be considered expired") + continue + + # There is a sha file, so check how it compares with self._baseline_ref + try: + num_ref_is_behind_file, num_ref_is_ahead_file = git_refs_difference(baseline_file_sha, baseline_ref_sha) + except SystemExit as e: + test.baselines_expired = True + reason = f"Failed to get refs difference between {baseline_file_sha} and {baseline_ref_sha} because: {e}" + print(f" -> Test {test} baselines are expired because {reason}") + continue + + # If the copy in our repo is behind, then we need to update the repo + expect (num_ref_is_behind_file==0 or not self._integration_test, f"""Error! Your repo seems stale, since the baseline sha in your repo is behind the one last used to generated them. We do *not* allow an integration test to replace baselines with older ones, for security reasons. If this is a legitimate case where baselines need to be 'rewound', e.g. b/c of a (hopefully VERY RARE) force push to master, then remove existing baselines first. Otherwise, please run 'git fetch $remote'. - - baseline_ref: {self._baseline_ref} - - repo baseline sha: {baseline_ref_sha} - - last used baseline sha: {baseline_file_sha}""") - - # If the copy in our repo is not ahead, then baselines are not expired - if num_ref_is_ahead_file > 0 or self._force_baseline_regen: - test.missing_baselines = True - reason = "forcing baseline regen" if self._force_baseline_regen \ - else f"{self._baseline_ref} is ahead of the baseline commit by {num_ref_is_ahead_file}" - print(f" -> Test {test} baselines are expired because {reason}") - else: - print(f" -> Test {test} baselines are valid and do not need to be regenerated") +- baseline_ref: {self._baseline_ref} +- repo baseline sha: {baseline_ref_sha} +- last used baseline sha: {baseline_file_sha}""") + + # If the copy in our repo is ahead, then baselines are expired + if num_ref_is_ahead_file > 0: + test.baselines_expired = True + reason = f"{self._baseline_ref} is ahead of the existing baseline commit {baseline_file_sha} by {num_ref_is_ahead_file}" + print(f" -> Test {test} baselines are expired because {reason}") + else: + print(f" -> Test {test} baselines are valid and do not need to be regenerated") ############################################################################### def get_machine_file(self): @@ -691,6 +450,9 @@ def generate_cmake_config(self, test, for_ctest=False): stat, c_path, _ = run_cmd("nc-config --prefix") if stat == 0: result += f" -DNetCDF_C_PATH={c_path}" + stat, pc_path, _ = run_cmd("pnetcdf-config --prefix") + if stat == 0: + result += f" -DPnetCDF_C_PATH={pc_path}" # Test-specific cmake options for key, value in test.cmake_args: @@ -815,7 +577,7 @@ def generate_ctest_config(self, cmake_config, extra_configs, test): result += f"--resource-spec-file {test_dir}/ctest_resource_file.json " if self._baseline_dir is not None and test.uses_baselines: - cmake_config += f" -DSCREAM_TEST_DATA_DIR={self.get_preexisting_baseline(test)}" + cmake_config += f" -DSCREAM_BASELINES_DIR={self.get_preexisting_baseline(test).parent}" if not self._submit: result += "-DNO_SUBMIT=True " @@ -847,91 +609,111 @@ def generate_ctest_config(self, cmake_config, extra_configs, test): return result ############################################################################### - def generate_baselines(self, test, commit): + def generate_baselines(self, test): ############################################################################### expect(test.uses_baselines, f"Something is off. generate_baseline should have not be called for test {test}") - test_dir = self.get_test_dir(self._baseline_dir, test) + baseline_dir = self.get_test_dir(self._baseline_dir, test) + test_dir = self.get_test_dir(self._work_dir / "tas_baseline_build", test) + if test_dir.exists(): + shutil.rmtree(test_dir) + test_dir.mkdir() + num_test_res = self.create_ctest_resource_file(test,test_dir) cmake_config = self.generate_cmake_config(test) - cmake_config += " -DSCREAM_BASELINES_ONLY=ON" - cmake_config += f" -DSCREAM_TEST_DATA_DIR={test_dir}/data" + cmake_config += " -DSCREAM_ONLY_GENERATE_BASELINES=ON" + cmake_config += f" -DSCREAM_BASELINES_DIR={baseline_dir}" + cmake_config += f" -DSCREAM_TEST_MAX_TOTAL_THREADS={num_test_res}" print("===============================================================================") print(f"Generating baseline for test {test} with config '{cmake_config}'") print("===============================================================================") - success = True + # We cannot just crash if we fail to generate baselines, since we would + # not get a dashboard report if we did that. Instead, just ensure there is + # no baseline file to compare against if there's a problem. + stat, _, err = run_cmd(f"{cmake_config} {self._root_dir}", + from_dir=test_dir, verbose=True) + if stat != 0: + print (f"WARNING: Failed to create baselines (config phase):\n{err}") + return False - try: - # We cannot just crash if we fail to generate baselines, since we would - # not get a dashboard report if we did that. Instead, just ensure there is - # no baseline file to compare against if there's a problem. - stat, _, err = run_cmd(f"{cmake_config} {self._root_dir}", - from_dir=test_dir, verbose=True, dry_run=self._dry_run) - if stat != 0: - print (f"WARNING: Failed to configure baselines:\n{err}") - success = False + cmd = f"make -j{test.compile_res_count}" + if self._parallel: + start, end = self.get_taskset_range(test) + cmd = f"taskset -c {start}-{end} sh -c '{cmd}'" - else: - cmd = f"make -j{test.compile_res_count} && make -j{test.testing_res_count} baseline" - if self._parallel: - start, end = self.get_taskset_range(test) - cmd = f"taskset -c {start}-{end} sh -c '{cmd}'" + stat, _, err = run_cmd(cmd, from_dir=test_dir, verbose=True) - stat, _, err = run_cmd(cmd, from_dir=test_dir, verbose=True, dry_run=self._dry_run) + if stat != 0: + print (f"WARNING: Failed to create baselines (build phase):\n{err}") + return False - if stat != 0: - print(f"WARNING: Failed to create baselines:\n{err}") - success = False + cmd = f"ctest -j{test.testing_res_count}" + cmd += " -L baseline_gen" + cmd += f" --resource-spec-file {test_dir}/ctest_resource_file.json" + stat, _, err = run_cmd(cmd, from_dir=test_dir, verbose=True) - finally: - # Clean up the directory, by removing everything but the 'data' subfolder. This must - # happen unconditionally or else subsequent runs could be corrupted - run_cmd_no_fail(r"find -maxdepth 1 -not -name data ! -path . -exec rm -rf {} \;", - from_dir=test_dir, verbose=True, dry_run=self._dry_run) + if stat != 0: + print (f"WARNING: Failed to create baselines (run phase):\n{err}") + return False - if success: - # Store the sha used for baselines generation - self.set_baseline_file_sha(test, commit) - test.missing_baselines = False + # Read list of nc files to copy to baseline dir + with open(test_dir/"data/baseline_list","r",encoding="utf-8") as fd: + files = fd.read().splitlines() - return success + with SharedArea(): + for fn in files: + # In case appending to the file leaves an empty line at the end + if fn != "": + src = Path(fn) + dst = baseline_dir / "data" / src.name + safe_copy(src, dst) + + # Store the sha used for baselines generation + self.set_baseline_file_sha(test) + test.baselines_missing = False + + # Clean up the directory by removing everything + shutil.rmtree(test_dir) + + return True ############################################################################### def generate_all_baselines(self): ############################################################################### - git_head_ref = get_current_head() + + tests_needing_baselines = self.baselines_to_be_generated() + if len(tests_needing_baselines)==0: + return True + + # Switch to baseline ref + checkout_git_ref (self._baseline_ref) print("###############################################################################") - print(f"Generating baselines for ref {self._baseline_ref}") + print(f"Generating baselines from git ref {self._baseline_ref}") print("###############################################################################") - commit = get_current_commit(commit=self._baseline_ref) - - # Switch to the baseline commit - checkout_git_ref(self._baseline_ref, verbose=True, dry_run=self._dry_run) + tas_baseline_bld = self._work_dir / "tas_baseline_build" + if tas_baseline_bld.exists(): + shutil.rmtree(tas_baseline_bld) + tas_baseline_bld.mkdir() success = True - tests_needing_baselines = [test for test in self._tests if test.missing_baselines] num_workers = len(tests_needing_baselines) if self._parallel else 1 with threading3.ProcessPoolExecutor(max_workers=num_workers) as executor: future_to_test = { - executor.submit(self.generate_baselines, test, commit) : test + executor.submit(self.generate_baselines, test) : test for test in tests_needing_baselines} for future in threading3.as_completed(future_to_test): test = future_to_test[future] success &= future.result() - if not success and self._fast_fail: - print(f"Generation of baselines for test {test} failed") - return False - - # Switch back to the branch commit - checkout_git_ref(git_head_ref, verbose=True, dry_run=self._dry_run) + # Restore original commit + checkout_git_ref (self._original_commit) return success @@ -957,12 +739,13 @@ def run_test(self, test): if self._quick_rerun_failed: ctest_config += "--rerun-failed " else: - # This directory might have been used also to build the model to generate baselines. + # This directory might have been used before during another test-all-scream run. # Although it's ok to build in the same dir, we MUST make sure to erase cmake's cache - # and internal files from the previous build (CMakeCache.txt and CMakeFiles folder) - run_cmd_no_fail("rm -rf CMake*", from_dir=test_dir, dry_run=self._dry_run) + # and internal files from the previous build (CMakeCache.txt and CMakeFiles folder), + # Otherwise, we may not pick up changes in certain cmake vars that are already cached. + run_cmd_no_fail("rm -rf CMake*", from_dir=test_dir) - success = run_cmd(ctest_config, from_dir=test_dir, arg_stdout=None, arg_stderr=None, verbose=True, dry_run=self._dry_run)[0] == 0 + success = run_cmd(ctest_config, from_dir=test_dir, arg_stdout=None, arg_stderr=None, verbose=True)[0] == 0 return success @@ -973,9 +756,6 @@ def run_all_tests(self): print("Running tests!") print("###############################################################################") - # First, create build directories (one per test). If existing, nuke the content - self.create_tests_dirs(self._work_dir, not self._quick_rerun) - success = True tests_success = { test : False @@ -991,10 +771,6 @@ def run_all_tests(self): test = future_to_test[future] tests_success[test] = future.result() success &= tests_success[test] - # If failed, and fast fail is requested, return immediately - # Note: this is effective only if num_worksers=1 - if not success and self._fast_fail: - break for t,s in tests_success.items(): if not s: @@ -1049,6 +825,23 @@ def get_last_ctest_file(self,test,phase): else: return None + ############################################################################### + def baselines_to_be_generated(self): + ############################################################################### + """ + Return list of baselines to generate. Baselines need to be generated if + - they are missing + - they are expired and we asked to update expired baselines + """ + ret = [] + for test in self._tests: + if test.baselines_missing: + ret.append(test) + elif self._update_expired_baselines and test.baselines_expired: + ret.append(test) + + return ret + ############################################################################### def test_all_scream(self): ############################################################################### @@ -1060,25 +853,30 @@ def test_all_scream(self): success = True try: - # If needed, generate baselines first - tests_needing_baselines = [test for test in self._tests if test.missing_baselines] - if tests_needing_baselines: - expect(self._baseline_ref is not None, "Missing baseline ref") + if self._integration_test: + # Merge origin/master + merge_git_ref(git_ref=self._baseline_ref, verbose=True) + + if self._generate: success = self.generate_all_baselines() if not success: print ("Error(s) occurred during baselines generation phase") + + # Do not continue testing, as you may be testing against old/invalid baselines return False - # If requested, run tests - if self._perform_tests: + if self._run_tests: + # First, create build directories (one per test). If existing, nuke the content + self.create_tests_dirs(self._work_dir, not self._quick_rerun) + success &= self.run_all_tests() if not success: print ("Error(s) occurred during test phase") finally: - if not self._keep_tree: - # Cleanup the repo if needed - cleanup_repo(self._original_branch, self._original_commit, dry_run=self._dry_run) + # Cleanup the repo if needed + if self._original_commit!=get_current_commit(): + cleanup_repo(self._original_branch, self._original_commit, self._has_backup_commit) return success diff --git a/components/eamxx/scripts/test_factory.py b/components/eamxx/scripts/test_factory.py new file mode 100644 index 000000000000..3c480e3c6d8b --- /dev/null +++ b/components/eamxx/scripts/test_factory.py @@ -0,0 +1,258 @@ +# This module contains classes that describe a test type for test-all-scream +# Each test type (here represented by a TestProperty object) can have different +# flags, build type, profiling, cmake options, etc. +# The function "test_factory" can be used to get a list of test types from +# their string representation. + +from collections import OrderedDict +from utils import expect + +############################################################################### +class TestProperty(object): +############################################################################### + + """ + Parent class of predefined test types for SCREAM standalone. test-all-scream + offers a number of customization points, but you may need to just use + cmake if you need maximal customization. You can run test-all-scream --dry-run + to get the corresponding cmake command which can then be used as a starting + point for making your own cmake command. + """ + + def __init__(self, longname, description, cmake_args, + uses_baselines=True, on_by_default=True, default_test_len=None): + # What the user uses to select tests via test-all-scream CLI. + # Should also match the class name when converted to caps + self.shortname = type(self).__name__.lower() + + # A longer name used to name baseline and test directories for a test. + # Also used in output/error messages to refer to the test + self.longname = longname + + # A longer decription of the test + self.description = description + + # Cmake config args for this test. Check that quoting is done with + # single quotes. + self.cmake_args = cmake_args + for name, arg in self.cmake_args: + expect('"' not in arg, + f"In test definition for {longname}, found cmake args with double quotes {name}='{arg}'" + "Please use single quotes if quotes are needed.") + + # Does the test do baseline testing + self.uses_baselines = uses_baselines + + # Should this test be run if the user did not specify tests at all? + self.on_by_default = on_by_default + + # Should this test have a default test size + self.default_test_len = default_test_len + + # + # Properties not set by constructor (Set by the main TestAllScream object) + # + + # Resources used by this test. + self.compile_res_count = None + self.testing_res_count = None + + # Does this test need baselines + self.baselines_missing = False + self.baselines_expired = False + + # + # Common + # + + if not self.uses_baselines: + self.cmake_args += [("SCREAM_ENABLE_BASELINE_TESTS", "False")] + + def disable_baselines(self): + if self.uses_baselines: + self.uses_baselines = False + self.cmake_args += [("SCREAM_ENABLE_BASELINE_TESTS", "False")] + + # Tests will generally be referred to via their longname + def __str__(self): + return self.longname + +############################################################################### +class DBG(TestProperty): +############################################################################### + + CMAKE_ARGS = [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_DEFAULT_BFB", "True")] + + def __init__(self, _): + TestProperty.__init__( + self, + "full_debug", + "debug", + self.CMAKE_ARGS, + ) + +############################################################################### +class SP(TestProperty): +############################################################################### + + def __init__(self, _): + TestProperty.__init__( + self, + "full_sp_debug", + "debug single precision", + DBG.CMAKE_ARGS + [("SCREAM_DOUBLE_PRECISION", "False")], + ) + +############################################################################### +class FPE(TestProperty): +############################################################################### + + def __init__(self, tas): + TestProperty.__init__( + self, + "debug_nopack_fpe", + "debug pksize=1 floating point exceptions on", + DBG.CMAKE_ARGS + [("SCREAM_PACK_SIZE", "1"), ("SCREAM_FPE","True")], + uses_baselines=False, + on_by_default=(tas is not None and not tas.on_cuda()) + ) + +############################################################################### +class OPT(TestProperty): +############################################################################### + + def __init__(self, _): + TestProperty.__init__( + self, + "release", + "release", + [("CMAKE_BUILD_TYPE", "Release")], + ) + +############################################################################### +class COV(TestProperty): +############################################################################### + + def __init__(self, _): + TestProperty.__init__( + self, + "coverage", + "debug coverage", + [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_ENABLE_COVERAGE", "True")], + uses_baselines=False, + on_by_default=False, + default_test_len="short" + ) + +############################################################################### +class VALG(TestProperty): +############################################################################### + + def __init__(self, tas): + TestProperty.__init__( + self, + "valgrind", + "debug with valgrind", + [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_ENABLE_VALGRIND", "True")], + uses_baselines=False, + on_by_default=False, + default_test_len="short" + ) + if tas is not None: + # If a stored suppression file exists for this machine, use it + persistent_supp_file = tas.get_root_dir() / "scripts" / "jenkins" / "valgrind" / f"{tas.get_machine()}.supp" + if persistent_supp_file.exists(): + self.cmake_args.append( ("EKAT_VALGRIND_SUPPRESSION_FILE", str(persistent_supp_file)) ) + +############################################################################### +class CSM(TestProperty): +############################################################################### + + def __init__(self, _): + TestProperty.__init__( + self, + "compute_sanitizer_memcheck", + "debug with compute sanitizer memcheck", + [("CMAKE_BUILD_TYPE", "Debug"), + ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), + ("EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=memcheck")], + uses_baselines=False, + on_by_default=False, + default_test_len="short" + ) + +############################################################################### +class CSR(TestProperty): +############################################################################### + + def __init__(self, _): + TestProperty.__init__( + self, + "compute_sanitizer_racecheck", + "debug with compute sanitizer racecheck", + [("CMAKE_BUILD_TYPE", "Debug"), + ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), + ("EKAT_COMPUTE_SANITIZER_OPTIONS", "'--tool=racecheck --racecheck-detect-level=error'")], + uses_baselines=False, + on_by_default=False, + default_test_len="short" + ) + +############################################################################### +class CSI(TestProperty): +############################################################################### + + def __init__(self, _): + TestProperty.__init__( + self, + "compute_sanitizer_initcheck", + "debug with compute sanitizer initcheck", + [("CMAKE_BUILD_TYPE", "Debug"), + ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), + ("EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=initcheck")], + uses_baselines=False, + on_by_default=False, + default_test_len="short" + ) + +############################################################################### +class CSS(TestProperty): +############################################################################### + + def __init__(self, _): + TestProperty.__init__( + self, + "compute_sanitizer_synccheck", + "debug with compute sanitizer synccheck", + [("CMAKE_BUILD_TYPE", "Debug"), + ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), + ("EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=synccheck")], + uses_baselines=False, + on_by_default=False, + default_test_len="short" + ) + +############################################################################### +def create_tests(user_req_tests, tas): +############################################################################### + testclasses = TestProperty.__subclasses__() + if not user_req_tests: + result = [testclass(tas) for testclass in testclasses + if testclass(tas).on_by_default] + else: + valid_names = [testclass(tas).shortname for testclass in testclasses] + for user_req_test in user_req_tests: + expect(user_req_test in valid_names, f"'{user_req_test}' is not a known test") + + result = [testclass(tas) for testclass in testclasses if testclass(tas).shortname in user_req_tests] + + return result + +########################################################################### +def get_test_name_dict(): +########################################################################### + """ + Returns a dict mapping short test names to full names + """ + testclasses = TestProperty.__subclasses__() + return OrderedDict([(testc(None).shortname, testc(None).description) for testc in testclasses]) diff --git a/components/eamxx/scripts/utils.py b/components/eamxx/scripts/utils.py index d08075a5e34b..9aafd09ae8ae 100644 --- a/components/eamxx/scripts/utils.py +++ b/components/eamxx/scripts/utils.py @@ -2,9 +2,11 @@ Utilities """ -import os, sys, re, signal, subprocess, site, time +import os, sys, re, signal, subprocess, site, time, shutil from importlib import import_module +import stat as statlib from pathlib import Path +from distutils import file_util # pylint: disable=deprecated-module ############################################################################### def expect(condition, error_msg, exc_type=SystemExit, error_prefix="ERROR:"): @@ -415,3 +417,81 @@ def ensure_yaml(): _ensure_pylib_impl("yaml", pip_libname="pyyaml",min_version def ensure_pylint(): _ensure_pylib_impl("pylint") def ensure_psutil(): _ensure_pylib_impl("psutil") def ensure_netcdf4(): _ensure_pylib_impl("netCDF4") + +############################################################################### +def safe_copy(src_path, tgt_path, preserve_meta=True): +############################################################################### + """ + A flexbile and safe copy routine. Will try to copy file and metadata, but this + can fail if the current user doesn't own the tgt file. A fallback data-only copy is + attempted in this case. Works even if overwriting a read-only file. + + tgt_path can be a directory, src_path must be a file + + most of the complexity here is handling the case where the tgt_path file already + exists. This problem does not exist for the tree operations so we don't need to wrap those. + + preserve_meta toggles if file meta-data, like permissions, should be preserved. If you are + copying baseline files, you should be within a SharedArea context manager and preserve_meta + should be false so that the umask set up by SharedArea can take affect regardless of the + permissions of the src files. + """ + + # Only works for str paths for now + src_path = str(src_path) + tgt_path = str(tgt_path) + + tgt_path = ( + os.path.join(tgt_path, os.path.basename(src_path)) + if os.path.isdir(tgt_path) + else tgt_path + ) + + # Handle pre-existing file + if os.path.isfile(tgt_path): + st = os.stat(tgt_path) + owner_uid = st.st_uid + + # Handle read-only files if possible + if not os.access(tgt_path, os.W_OK): + if owner_uid == os.getuid(): + # I am the owner, make writeable + os.chmod(tgt_path, st.st_mode | statlib.S_IWRITE) + else: + # I won't be able to copy this file + raise OSError( + "Cannot copy over file {}, it is readonly and you are not the owner".format( + tgt_path + ) + ) + + if owner_uid == os.getuid(): + # I am the owner, copy file contents, permissions, and metadata + file_util.copy_file( + src_path, + tgt_path, + preserve_mode=preserve_meta, + preserve_times=preserve_meta, + verbose=0, + ) + else: + # I am not the owner, just copy file contents + shutil.copyfile(src_path, tgt_path) + + else: + # We are making a new file, copy file contents, permissions, and metadata. + # This can fail if the underlying directory is not writable by current user. + file_util.copy_file( + src_path, + tgt_path, + preserve_mode=preserve_meta, + preserve_times=preserve_meta, + verbose=0, + ) + + # If src file was executable, then the tgt file should be too + st = os.stat(tgt_path) + if os.access(src_path, os.X_OK) and st.st_uid == os.getuid(): + os.chmod( + tgt_path, st.st_mode | statlib.S_IXUSR | statlib.S_IXGRP | statlib.S_IXOTH + ) diff --git a/components/eamxx/src/CMakeLists.txt b/components/eamxx/src/CMakeLists.txt index 9568d3cf85ce..59a5d6646443 100644 --- a/components/eamxx/src/CMakeLists.txt +++ b/components/eamxx/src/CMakeLists.txt @@ -4,9 +4,6 @@ add_subdirectory(dynamics) add_subdirectory(physics) add_subdirectory(diagnostics) add_subdirectory(control) -if ("${SCREAM_DYNAMICS_DYCORE}" STREQUAL "HOMME") - add_subdirectory(doubly-periodic) -endif() if (PROJECT_NAME STREQUAL "E3SM") add_subdirectory(mct_coupling) endif() diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index 96adc56127d4..a5dacd856d75 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -167,44 +167,44 @@ init_time_stamps (const util::TimeStamp& run_t0, const util::TimeStamp& case_t0) void AtmosphereDriver:: -setup_intensive_observation_period () +setup_iop () { - // At this point, must have comm, params, initialized timestamps, and grids created. - check_ad_status(s_comm_set | s_params_set | s_ts_inited | s_grids_created); + // At this point, must have comm, params, initialized timestamps created. + check_ad_status(s_comm_set | s_params_set | s_ts_inited); // Check to make sure iop is not already initialized - EKAT_REQUIRE_MSG(not m_intensive_observation_period, "Error! setup_intensive_observation_period() is " - "called, but IOP already set up.\n"); + EKAT_REQUIRE_MSG(not m_iop, "Error! setup_iop() is called, but IOP already set up.\n"); // This function should only be called if we are enabling IOP const bool enable_iop = - m_atm_params.sublist("driver_options").get("enable_intensive_observation_period", false); - EKAT_REQUIRE_MSG(enable_iop, "Error! setup_intensive_observation_period() is called, but " - "enable_intensive_observation_period=false " + m_atm_params.sublist("driver_options").get("enable_iop", false); + EKAT_REQUIRE_MSG(enable_iop, "Error! setup_iop() is called, but enable_iop=false " "in driver_options parameters.\n"); - // Params must include intensive_observation_period_options sublist. - const auto iop_sublist_exists = m_atm_params.isSublist("intensive_observation_period_options"); + // Params must include iop_options sublist. + const auto iop_sublist_exists = m_atm_params.isSublist("iop_options"); EKAT_REQUIRE_MSG(iop_sublist_exists, - "Error! setup_intensive_observation_period() is called, but no intensive_observation_period_options " + "Error! setup_iop() is called, but no iop_options " "defined in parameters.\n"); - const auto iop_params = m_atm_params.sublist("intensive_observation_period_options"); + const auto iop_params = m_atm_params.sublist("iop_options"); const auto phys_grid = m_grids_manager->get_grid("Physics"); const auto nlevs = phys_grid->get_num_vertical_levels(); const auto hyam = phys_grid->get_geometry_data("hyam"); const auto hybm = phys_grid->get_geometry_data("hybm"); - m_intensive_observation_period = - std::make_shared(m_atm_comm, - iop_params, - m_run_t0, - nlevs, - hyam, - hybm); + m_iop = std::make_shared(m_atm_comm, + iop_params, + m_run_t0, + nlevs, + hyam, + hybm); auto dx_short_f = phys_grid->get_geometry_data("dx_short"); - m_intensive_observation_period->set_grid_spacing(dx_short_f.get_view()()); + m_iop->set_grid_spacing(dx_short_f.get_view()()); + + // Set IOP object in atm processes + m_atm_process_group->set_iop(m_iop); } void AtmosphereDriver::create_atm_processes() @@ -279,6 +279,14 @@ void AtmosphereDriver::create_grids() setup_shoc_tms_links(); } + // IOP object needs the grids_manager to have been created, but is then needed in set_grids() + // implementation of some processes, so setup here. + const bool enable_iop = + m_atm_params.sublist("driver_options").get("enable_iop", false); + if (enable_iop) { + setup_iop (); + } + // Set the grids in the processes. Do this by passing the grids manager. // Each process will grab what they need m_atm_process_group->set_grids(m_grids_manager); @@ -338,10 +346,6 @@ void AtmosphereDriver::setup_surface_coupling_processes () const std::shared_ptr importer = std::dynamic_pointer_cast(atm_proc); importer->setup_surface_coupling_data(*m_surface_coupling_import_data_manager); - - if (m_intensive_observation_period) { - importer->set_intensive_observation_period(m_intensive_observation_period); - } } if (atm_proc->type() == AtmosphereProcessType::SurfaceCouplingExporter) { exporter_found = true; @@ -691,9 +695,9 @@ void AtmosphereDriver::initialize_output_managers () { checkpoint_params.set("Frequency",-1); if (io_params.isSublist("model_restart")) { auto restart_pl = io_params.sublist("model_restart"); - m_output_managers.emplace_back(); + restart_pl.set("Averaging Type","Instant"); restart_pl.sublist("provenance") = m_atm_params.sublist("provenance"); - auto& om = m_output_managers.back(); + auto& om = m_output_managers.emplace_back(); if (fvphyshack) { // Don't save CGLL fields from ICs to the restart file. std::map fms; @@ -839,7 +843,7 @@ initialize_fields () auto hw = fm->get_field("horiz_winds"); const auto& fid = hw.get_header().get_identifier(); const auto& layout = fid.get_layout(); - const int vec_dim = layout.get_vector_dim(); + const int vec_dim = layout.get_vector_component_idx(); const auto& units = fid.get_units(); auto U = hw.subfield("U",units,vec_dim,0); auto V = hw.subfield("V",units,vec_dim,1); @@ -855,7 +859,7 @@ initialize_fields () auto hw = fm->get_field("surf_mom_flux"); const auto& fid = hw.get_header().get_identifier(); const auto& layout = fid.get_layout(); - const int vec_dim = layout.get_vector_dim(); + const int vec_dim = layout.get_vector_component_idx(); const auto& units = fid.get_units(); auto surf_mom_flux_U = hw.subfield("surf_mom_flux_U",units,vec_dim,0); auto surf_mom_flux_V = hw.subfield("surf_mom_flux_V",units,vec_dim,1); @@ -937,29 +941,34 @@ void AtmosphereDriver::create_logger () { auto& driver_options_pl = m_atm_params.sublist("driver_options"); ci_string log_fname = driver_options_pl.get("Atm Log File","atm.log"); - ci_string log_level_str = driver_options_pl.get("atm_log_level","info"); + ci_string log_level = driver_options_pl.get("atm_log_level","info"); + ci_string flush_level = driver_options_pl.get("atm_flush_level","warn"); EKAT_REQUIRE_MSG (log_fname!="", "Invalid string for 'Atm Log File': '" + log_fname + "'.\n"); - LogLevel log_level; - if (log_level_str=="trace") { - log_level = LogLevel::trace; - } else if (log_level_str=="debug") { - log_level = LogLevel::debug; - } else if (log_level_str=="info") { - log_level = LogLevel::info; - } else if (log_level_str=="warn") { - log_level = LogLevel::warn; - } else if (log_level_str=="err") { - log_level = LogLevel::err; - } else if (log_level_str=="off") { - log_level = LogLevel::off; - } else { - EKAT_ERROR_MSG ("Invalid choice for 'atm_log_level': " + log_level_str + "\n"); - } + auto str2lev = [](const std::string& s, const std::string& name) { + LogLevel lev; + if (s=="trace") { + lev = LogLevel::trace; + } else if (s=="debug") { + lev = LogLevel::debug; + } else if (s=="info") { + lev = LogLevel::info; + } else if (s=="warn") { + lev = LogLevel::warn; + } else if (s=="err") { + lev = LogLevel::err; + } else if (s=="off") { + lev = LogLevel::off; + } else { + EKAT_ERROR_MSG ("Invalid choice for '" + name + "': " + s + "\n"); + } + return lev; + }; using logger_t = Logger; - m_atm_logger = std::make_shared(log_fname,log_level,m_atm_comm,""); + m_atm_logger = std::make_shared(log_fname,str2lev(log_level,"atm_log_level"),m_atm_comm,""); + m_atm_logger->flush_on(str2lev(flush_level,"atm_flush_level")); m_atm_logger->set_no_format(); // In CIME runs, this is already set to false, so atm log does not pollute e3sm.loc. @@ -1129,7 +1138,7 @@ void AtmosphereDriver::set_initial_conditions () } } - if (m_intensive_observation_period) { + if (m_iop) { // For runs with IOP, call to setup io grids and lat // lon information needed for reading from file for (const auto& it : m_field_mgrs) { @@ -1140,7 +1149,7 @@ void AtmosphereDriver::set_initial_conditions () ic_pl.get("Filename") : ic_pl.get("topography_filename"); - m_intensive_observation_period->setup_io_info(file_name, it.second->get_grid()); + m_iop->setup_io_info(file_name, it.second->get_grid()); } } } @@ -1152,15 +1161,15 @@ void AtmosphereDriver::set_initial_conditions () m_atm_logger->info(" [EAMxx] IC filename: " + file_name); for (const auto& it : m_field_mgrs) { const auto& grid_name = it.first; - if (not m_intensive_observation_period) { + if (not m_iop) { read_fields_from_file (ic_fields_names[grid_name],it.second->get_grid(),file_name,m_current_ts); } else { // For IOP enabled, we load from file and copy data from the closest // lat/lon column to every other column - m_intensive_observation_period->read_fields_from_file_for_iop(file_name, - ic_fields_names[grid_name], - m_current_ts, - it.second); + m_iop->read_fields_from_file_for_iop(file_name, + ic_fields_names[grid_name], + m_current_ts, + it.second); } } } @@ -1227,7 +1236,7 @@ void AtmosphereDriver::set_initial_conditions () m_atm_logger->info(" filename: " + file_name); for (const auto& it : m_field_mgrs) { const auto& grid_name = it.first; - if (not m_intensive_observation_period) { + if (not m_iop) { // Topography files always use "ncol_d" for the GLL grid value of ncol. // To ensure we read in the correct value, we must change the name for that dimension auto io_grid = it.second->get_grid(); @@ -1243,11 +1252,11 @@ void AtmosphereDriver::set_initial_conditions () } else { // For IOP enabled, we load from file and copy data from the closest // lat/lon column to every other column - m_intensive_observation_period->read_fields_from_file_for_iop(file_name, - topography_file_fields_names[grid_name], - topography_eamxx_fields_names[grid_name], - m_current_ts, - it.second); + m_iop->read_fields_from_file_for_iop(file_name, + topography_file_fields_names[grid_name], + topography_eamxx_fields_names[grid_name], + m_current_ts, + it.second); } } // Store in provenance list, for later usage in output file metadata @@ -1268,16 +1277,16 @@ void AtmosphereDriver::set_initial_conditions () m_atm_params.sublist("provenance").set("topography_file","NONE"); } - if (m_intensive_observation_period) { + if (m_iop) { // Load IOP data file data for initial time stamp - m_intensive_observation_period->read_iop_file_data(m_current_ts); + m_iop->read_iop_file_data(m_current_ts); // Now that ICs are processed, set appropriate fields using IOP file data. // Since ICs are loaded on GLL grid, we set those fields only and dynamics // will take care of the rest (for PG2 case). if (m_field_mgrs.count("Physics GLL") > 0) { const auto& fm = m_field_mgrs.at("Physics GLL"); - m_intensive_observation_period->set_fields_from_iop_data(fm); + m_iop->set_fields_from_iop_data(fm); } } @@ -1441,12 +1450,15 @@ initialize_constant_field(const FieldIdentifier& fid, // The user provided a constant value for this field. Simply use that. const auto& layout = f.get_header().get_identifier().get_layout(); - // For vector fields, we expect something like "fname: [val0,...,valN], - // where the field dim is N+1. For scalars, "fname: val". So check the - // field layout first, so we know what to get from the parameter list. - if (layout.is_vector_layout()) { - const auto idim = layout.get_vector_dim(); - const auto vec_dim = layout.dim(idim); + // For vector fields, we allow either single value init or vector value init. + // That is, both these are ok + // fname: val + // fname: [val1,...,valN] + // In the first case, all entries of the field are inited to val, while in the latter, + // each component is inited to the corresponding entry of the array. + if (layout.is_vector_layout() and ic_pl.isType>(name)) { + const auto idim = layout.get_vector_component_idx(); + const auto vec_dim = layout.get_vector_dim(); const auto& values = ic_pl.get>(name); EKAT_REQUIRE_MSG (values.size()==static_cast(vec_dim), "Error! Initial condition values array for '" + name + "' has the wrong dimension.\n" @@ -1554,12 +1566,6 @@ initialize (const ekat::Comm& atm_comm, create_grids (); - const bool enable_iop = - m_atm_params.sublist("driver_options").get("enable_intensive_observation_period", false); - if (enable_iop) { - setup_intensive_observation_period (); - } - create_fields (); initialize_fields (); diff --git a/components/eamxx/src/control/atmosphere_driver.hpp b/components/eamxx/src/control/atmosphere_driver.hpp index bf694da3fb3e..1cd8d4db08e7 100644 --- a/components/eamxx/src/control/atmosphere_driver.hpp +++ b/components/eamxx/src/control/atmosphere_driver.hpp @@ -72,7 +72,7 @@ class AtmosphereDriver void init_scorpio (const int atm_id = 0); // Setup IntensiveObservationPeriod - void setup_intensive_observation_period (); + void setup_iop (); // Create atm processes, without initializing them void create_atm_processes (); @@ -207,7 +207,7 @@ class AtmosphereDriver std::shared_ptr m_surface_coupling_import_data_manager; std::shared_ptr m_surface_coupling_export_data_manager; - std::shared_ptr m_intensive_observation_period; + std::shared_ptr m_iop; // This is the time stamp at the beginning of the time step. util::TimeStamp m_current_ts; diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp index 1bf32b924215..e816801fa618 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp @@ -170,9 +170,11 @@ void SurfaceCouplingImporter::do_import(const bool called_during_initialization) } }); - // If IOP is defined, potentially overwrite imports with data from IOP file - if (m_intensive_observation_period) { - overwrite_iop_imports(called_during_initialization); + if (m_iop) { + if (m_iop->get_params().get("iop_srf_prop")) { + // Overwrite imports with data from IOP file + overwrite_iop_imports(called_during_initialization); + } } } // ========================================================================================= @@ -181,11 +183,9 @@ void SurfaceCouplingImporter::overwrite_iop_imports (const bool called_during_in using policy_type = KokkosTypes::RangePolicy; using C = physics::Constants; - const auto& iop = m_intensive_observation_period; - - const auto has_lhflx = iop->has_iop_field("lhflx"); - const auto has_shflx = iop->has_iop_field("shflx"); - const auto has_Tg = iop->has_iop_field("Tg"); + const auto has_lhflx = m_iop->has_iop_field("lhflx"); + const auto has_shflx = m_iop->has_iop_field("shflx"); + const auto has_Tg = m_iop->has_iop_field("Tg"); static constexpr Real latvap = C::LatVap; static constexpr Real stebol = C::stebol; @@ -205,19 +205,19 @@ void SurfaceCouplingImporter::overwrite_iop_imports (const bool called_during_in // Store IOP surf data into col_val Real col_val(std::nan("")); if (fname == "surf_evap" && has_lhflx) { - const auto f = iop->get_iop_field("lhflx"); + const auto f = m_iop->get_iop_field("lhflx"); f.sync_to_host(); col_val = f.get_view()()/latvap; } else if (fname == "surf_sens_flux" && has_shflx) { - const auto f = iop->get_iop_field("shflx"); + const auto f = m_iop->get_iop_field("shflx"); f.sync_to_host(); col_val = f.get_view()(); } else if (fname == "surf_radiative_T" && has_Tg) { - const auto f = iop->get_iop_field("Tg"); + const auto f = m_iop->get_iop_field("Tg"); f.sync_to_host(); col_val = f.get_view()(); } else if (fname == "surf_lw_flux_up" && has_Tg) { - const auto f = iop->get_iop_field("Tg"); + const auto f = m_iop->get_iop_field("Tg"); f.sync_to_host(); col_val = stebol*std::pow(f.get_view()(), 4); } else { diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp index 5884c9d40af2..3a34a8b2951e 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp @@ -5,8 +5,6 @@ #include "ekat/ekat_parameter_list.hpp" #include "share/atm_process/SCDataManager.hpp" -#include "control/intensive_observation_period.hpp" - #include "surface_coupling_utils.hpp" #include @@ -36,7 +34,6 @@ class SurfaceCouplingImporter : public AtmosphereProcess using uview_2d = Unmanaged>; using name_t = char[32]; - using iop_ptr = std::shared_ptr; // Constructors SurfaceCouplingImporter (const ekat::Comm& comm, const ekat::ParameterList& params); @@ -64,9 +61,6 @@ class SurfaceCouplingImporter : public AtmosphereProcess // Overwrite imports for IOP cases with IOP file surface data void overwrite_iop_imports (const bool called_during_initialization); - void set_intensive_observation_period (const iop_ptr& iop) { - m_intensive_observation_period = iop; - } protected: // The three main overrides for the subcomponent @@ -102,9 +96,6 @@ class SurfaceCouplingImporter : public AtmosphereProcess view_1d m_column_info_d; decltype(m_column_info_d)::HostMirror m_column_info_h; - // Intensive observation period object. - iop_ptr m_intensive_observation_period; - // The grid is needed for property checks std::shared_ptr m_grid; }; // class SurfaceCouplingImporter diff --git a/components/eamxx/src/control/intensive_observation_period.cpp b/components/eamxx/src/control/intensive_observation_period.cpp index f4ab466c17ba..d925b21bfaa3 100644 --- a/components/eamxx/src/control/intensive_observation_period.cpp +++ b/components/eamxx/src/control/intensive_observation_period.cpp @@ -116,7 +116,7 @@ IntensiveObservationPeriod(const ekat::Comm& comm, EKAT_REQUIRE_MSG(m_params.isParameter("target_latitude") && m_params.isParameter("target_longitude"), "Error! Using intensive observation period files requires " "target_latitude and target_longitude be gives as parameters in " - "\"intensive_observation_period_options\" in the input yaml file.\n"); + "\"iop_options\" in the input yaml file.\n"); const auto target_lat = m_params.get("target_latitude"); const auto target_lon = m_params.get("target_longitude"); EKAT_REQUIRE_MSG(-90 <= target_lat and target_lat <= 90, @@ -135,16 +135,18 @@ IntensiveObservationPeriod(const ekat::Comm& comm, if (not m_params.isParameter("iop_nudge_tscale")) m_params.set("iop_nudge_tscale", 10800); if (not m_params.isParameter("zero_non_iop_tracers")) m_params.set("zero_non_iop_tracers", false); + // Store hybrid coords in helper fields + m_helper_fields.insert({"hyam", hyam}); + m_helper_fields.insert({"hybm", hybm}); + // Use IOP file to initialize parameters // and timestepping information - initialize_iop_file(run_t0, model_nlevs, hyam, hybm); + initialize_iop_file(run_t0, model_nlevs); } void IntensiveObservationPeriod:: initialize_iop_file(const util::TimeStamp& run_t0, - int model_nlevs, - const Field& hyam, - const Field& hybm) + int model_nlevs) { EKAT_REQUIRE_MSG(m_params.isParameter("iop_file"), "Error! Using IOP requires defining an iop_file parameter.\n"); @@ -184,6 +186,8 @@ initialize_iop_file(const util::TimeStamp& run_t0, if (scorpio::has_variable(iop_file, srf_varname)) { m_iop_field_surface_varnames.insert({iop_varname, srf_varname}); } + // Store that the IOP variable is found in the IOP file + m_iop_field_type.insert({iop_varname, IOPFieldType::FromFile}); // Allocate field for variable FieldIdentifier fid(iop_varname, fl, ekat::units::Units::nondimensional(), ""); @@ -191,6 +195,10 @@ initialize_iop_file(const util::TimeStamp& run_t0, EKAT_REQUIRE_MSG(field_rank <= 1, "Error! Unexpected field rank "+std::to_string(field_rank)+" for iop file fields.\n"); Field field(fid); + if (fl.has_tag(FieldTag::LevelMidPoint) or fl.has_tag(FieldTag::LevelInterface)) { + // Request packsize allocation for level layout + field.get_header().get_alloc_properties().request_allocation(Pack::n); + } field.allocate_view(); m_iop_fields.insert({iop_varname, field}); } @@ -230,13 +238,42 @@ initialize_iop_file(const util::TimeStamp& run_t0, setup_iop_field({"Q2"}, fl_vector); setup_iop_field({"omega"}, fl_vector, "Ptend"); - // Make sure Ps, T, and q are defined in the iop file + // Require Ps, T, q, divT, divq are all defined in the iop file EKAT_REQUIRE_MSG(has_iop_field("Ps"), - "Error! Using IOP file requires variable \"Ps\".\n"); + "Error! IOP file required to contain variable \"Ps\".\n"); EKAT_REQUIRE_MSG(has_iop_field("T"), - "Error! Using IOP file requires variable \"T\".\n"); + "Error! IOP file required to contain variable \"T\".\n"); EKAT_REQUIRE_MSG(has_iop_field("q"), - "Error! Using IOP file requires variable \"q\".\n"); + "Error! IOP file required to contain variable \"q\".\n"); + EKAT_REQUIRE_MSG(has_iop_field("divT"), + "Error! IOP file required to contain variable \"divT\".\n"); + EKAT_REQUIRE_MSG(has_iop_field("divq"), + "Error! IOP file required to contain variable \"divq\".\n"); + + // If we have the vertical component of T/Q forcing, define 3d forcing as a computed field. + if (has_iop_field("vertdivT")) { + FieldIdentifier fid("divT3d", fl_vector, ekat::units::Units::nondimensional(), ""); + Field field(fid); + field.get_header().get_alloc_properties().request_allocation(Pack::n); + field.allocate_view(); + m_iop_fields.insert({"divT3d", field}); + m_iop_field_type.insert({"divT3d", IOPFieldType::Computed}); + } + if (has_iop_field("vertdivq")) { + FieldIdentifier fid("divq3d", fl_vector, ekat::units::Units::nondimensional(), ""); + Field field(fid); + field.get_header().get_alloc_properties().request_allocation(Pack::n); + field.allocate_view(); + m_iop_fields.insert({"divq3d", field}); + m_iop_field_type.insert({"divq3d", IOPFieldType::Computed}); + } + + // Enforce that 3D forcing is all-or-nothing. + const bool both = (has_iop_field("divT3d") and has_iop_field("divq3d")); + const bool neither = (not (has_iop_field("divT3d") or has_iop_field("divq3d"))); + EKAT_REQUIRE_MSG(both or neither, + "Error! Either T and q both have 3d forcing, or neither have 3d forcing.\n"); + m_params.set("use_3d_forcing", both); // Initialize time information int bdate; @@ -286,6 +323,7 @@ initialize_iop_file(const util::TimeStamp& run_t0, ekat::units::Units::nondimensional(), ""); Field iop_file_pressure(fid); + iop_file_pressure.get_header().get_alloc_properties().request_allocation(Pack::n); iop_file_pressure.allocate_view(); auto data = iop_file_pressure.get_view().data(); read_variable_from_file(iop_file, "lev", "real", {"lev"}, -1, data); @@ -300,12 +338,9 @@ initialize_iop_file(const util::TimeStamp& run_t0, fl_vector, ekat::units::Units::nondimensional(), ""); Field model_pressure(model_pres_fid); + model_pressure.get_header().get_alloc_properties().request_allocation(Pack::n); model_pressure.allocate_view(); m_helper_fields.insert({"model_pressure", model_pressure}); - - // Store hyam and hybm in helper fields - m_helper_fields.insert({"hyam", hyam}); - m_helper_fields.insert({"hybm", hybm}); } void IntensiveObservationPeriod:: @@ -400,7 +435,8 @@ read_fields_from_file_for_iop (const std::string& file_name, const vos& field_names_nc, const vos& field_names_eamxx, const util::TimeStamp& initial_ts, - const field_mgr_ptr field_mgr) + const field_mgr_ptr field_mgr, + const int time_index) { const auto dummy_units = ekat::units::Units::nondimensional(); @@ -456,7 +492,7 @@ read_fields_from_file_for_iop (const std::string& file_name, // Read data from file AtmosphereInput file_reader(file_name,io_grid,io_fields); - file_reader.read_variables(); + file_reader.read_variables(time_index); file_reader.finalize(); // For each field, broadcast data from closest lat/lon column to all processors @@ -502,27 +538,27 @@ read_fields_from_file_for_iop (const std::string& file_name, void IntensiveObservationPeriod:: read_iop_file_data (const util::TimeStamp& current_ts) { - const auto iop_file = m_params.get("iop_file"); + // Query to see if we need to load data from IOP file. + // If we are still in the time interval as the previous + // read from iop file, there is no need to reload data. const auto iop_file_time_idx = m_time_info.get_iop_file_time_idx(current_ts); - - // Sanity check EKAT_REQUIRE_MSG(iop_file_time_idx >= m_time_info.time_idx_of_current_data, "Error! Attempting to read previous iop file data time index.\n"); - - // If we are still in the time interval as the previous read from iop file, - // there is no need to reload data. Return early if (iop_file_time_idx == m_time_info.time_idx_of_current_data) return; + const auto iop_file = m_params.get("iop_file"); const auto file_levs = scorpio::get_dimlen(iop_file, "lev"); const auto iop_file_pressure = m_helper_fields["iop_file_pressure"]; const auto model_pressure = m_helper_fields["model_pressure"]; const auto surface_pressure = m_iop_fields["Ps"]; - // Loop through iop fields, if rank 1 fields exist we need to - // gather information for vertically interpolating views + // Loop through iop fields, if any rank 1 fields are loaded from file, + // we need to gather information for vertical interpolation bool has_level_data = false; for (auto& it : m_iop_fields) { - if (it.second.rank() == 1) { + if (it.second.rank() == 1 + and + m_iop_field_type.at(it.first)==IOPFieldType::FromFile) { has_level_data = true; break; } @@ -589,6 +625,10 @@ read_iop_file_data (const util::TimeStamp& current_ts) Kokkos::Max(iop_file_start), Kokkos::Min(iop_file_end)); + // If no file pressures are found outide the reference pressure range, set to file level endpoints + if (iop_file_start == Kokkos::reduction_identity::max()) iop_file_start = 0; + if (iop_file_end == Kokkos::reduction_identity::min()) iop_file_end = adjusted_file_levs; + // Find model pressure levels just inside range of file pressure levels Kokkos::parallel_reduce(model_nlevs, KOKKOS_LAMBDA (const int& ilev, int& lmin, int& lmax) { if (model_pres_v(ilev) >= iop_file_pres_v(iop_file_start) && ilev < lmin) { @@ -600,6 +640,10 @@ read_iop_file_data (const util::TimeStamp& current_ts) }, Kokkos::Min(model_start), Kokkos::Max(model_end)); + + // If not reference pressures are found inside file pressures, set to model level endpoints + if (model_start == Kokkos::reduction_identity::min()) model_start = model_nlevs-1; + if (model_end == Kokkos::reduction_identity::max()) model_end = 1; } // Loop through fields and store data from file @@ -607,6 +651,9 @@ read_iop_file_data (const util::TimeStamp& current_ts) auto fname = it.first; auto field = it.second; + // If this is a computed field, do not attempt to load from file + if (m_iop_field_type.at(fname)==IOPFieldType::Computed) continue; + // File may use different varname than IOP class auto file_varname = (m_iop_file_varnames.count(fname) > 0) ? m_iop_file_varnames[fname] : fname; @@ -625,6 +672,7 @@ read_iop_file_data (const util::TimeStamp& current_ts) ekat::units::Units::nondimensional(), ""); Field iop_file_field(fid); + iop_file_field.get_header().get_alloc_properties().request_allocation(Pack::n); iop_file_field.allocate_view(); // Read data from iop file. @@ -657,8 +705,8 @@ read_iop_file_data (const util::TimeStamp& current_ts) iop_file_field.sync_to_dev(); // Vertically interpolate iop file data to iop fields. - // Note: ekat lininterp requires packs. Use 1d packs here. - // TODO: allow for nontrivial packsize. + // Note: ekat lininterp requires packs. Use 1d packs here + // to easily mask out levels which we do not want to interpolate. const auto iop_file_pres_v = iop_file_pressure.get_view(); const auto model_pres_v = model_pressure.get_view(); const auto iop_file_v = iop_file_field.get_view(); @@ -685,22 +733,42 @@ read_iop_file_data (const util::TimeStamp& current_ts) // the interpolated region with the value at model_start/model_end if (fname == "T" || fname == "q" || fname == "u" || fname == "u_ls" || fname == "v" || fname == "v_ls") { - if (model_start > 0) { - Kokkos::parallel_for(Kokkos::RangePolicy<>(0, model_start), - KOKKOS_LAMBDA (const int ilev) { - iop_field_v(ilev) = iop_field_v(model_start); - }); - } - if (model_end < total_nlevs) { - Kokkos::parallel_for(Kokkos::RangePolicy<>(model_end, total_nlevs), - KOKKOS_LAMBDA (const int ilev) { - iop_field_v(ilev) = iop_field_v(model_end-1); - }); - } + Kokkos::parallel_for(Kokkos::RangePolicy<>(0, model_start+1), + KOKKOS_LAMBDA (const int ilev) { + iop_field_v(ilev) = iop_file_v(0); + }); + Kokkos::parallel_for(Kokkos::RangePolicy<>(model_end-1, total_nlevs), + KOKKOS_LAMBDA (const int ilev) { + iop_field_v(ilev) = iop_file_v(adjusted_file_levs-1); + }); } } } + // Calculate 3d forcing (if applicable). + if (has_iop_field("divT3d")) { + if (m_iop_field_type.at("divT3d")==IOPFieldType::Computed) { + const auto divT = get_iop_field("divT").get_view(); + const auto vertdivT = get_iop_field("vertdivT").get_view(); + const auto divT3d = get_iop_field("divT3d").get_view(); + const auto nlevs = get_iop_field("divT3d").get_header().get_identifier().get_layout().dim(0); + Kokkos::parallel_for(nlevs, KOKKOS_LAMBDA (const int ilev) { + divT3d(ilev) = divT(ilev) + vertdivT(ilev); + }); + } + } + if (has_iop_field("divq3d")) { + if (m_iop_field_type.at("divq3d")==IOPFieldType::Computed) { + const auto divq = get_iop_field("divq").get_view(); + const auto vertdivq = get_iop_field("vertdivq").get_view(); + const auto divq3d = get_iop_field("divq3d").get_view(); + const auto nlevs = get_iop_field("divq3d").get_header().get_identifier().get_layout().dim(0); + Kokkos::parallel_for(nlevs, KOKKOS_LAMBDA (const int ilev) { + divq3d(ilev) = divq(ilev) + vertdivq(ilev); + }); + } + } + // Now that data is loaded, reset the index of the currently loaded data. m_time_info.time_idx_of_current_data = iop_file_time_idx; } diff --git a/components/eamxx/src/control/intensive_observation_period.hpp b/components/eamxx/src/control/intensive_observation_period.hpp index 6e70f44a0f56..6c0b3640ffb2 100644 --- a/components/eamxx/src/control/intensive_observation_period.hpp +++ b/components/eamxx/src/control/intensive_observation_period.hpp @@ -26,6 +26,8 @@ class IntensiveObservationPeriod using KT = ekat::KokkosTypes; using ESU = ekat::ExeSpaceUtils; + using Pack = ekat::Pack; + using Pack1d = ekat::Pack; template using view_1d = KT::template view_1d; @@ -35,14 +37,13 @@ class IntensiveObservationPeriod using view_3d = KT::template view_3d; template using view_1d_host = typename view_1d::HostMirror; - using Pack1d = ekat::Pack; public: // Constructor // Input: // - comm: MPI communicator - // - params: Input yaml file needs intensive_observation_period_options sublist + // - params: Input yaml file needs iop_options sublist // - run_t0: Initial timestamp for the simulation // - model_nlevs: Number of vertical levels in the simulation. Needed since // the iop file contains a (potentially) different number of levels @@ -69,31 +70,35 @@ class IntensiveObservationPeriod void setup_io_info (const std::string& file_name, const grid_ptr& grid); - // Read ICs from file for IOP cases. We set all columns in the - // given fields to the values of the column in the file with the - // closest lat,lon pair to the target lat,lon in the parameters. - // The setup_io_info must be called for the correct grids before - // this function can be called. - // Input: + // Read ICs and SPA data from file and remap to fields in field_mgr. + // The remap is defined by setting all columns in the given fields to the + // values of the column in the file with the closest lat,lon pair to + // the target lat,lon in the parameters. + // The function setup_io_info() must be called for the grids corresponding + // to the file data before this function can be called. + // Fields in the field_mgr must have the same number of levels as the file. + // Inputs and outputs: // - file_name: Name of the file used to load field data (IC or topo file) // - field_names_nc: Field names used by the input file // - field_names_eamxx: Field names used by eamxx - // - initial_ts: Inital timestamp - // Input/output + // - initial_ts: Inital timestamp. // - field_mgr: Field manager containing fields that need data read from files + // - time_index: Time index of read. time_index=-1 will read the latest time in file. void read_fields_from_file_for_iop(const std::string& file_name, const vos& field_names_nc, const vos& field_names_eamxx, const util::TimeStamp& initial_ts, - const field_mgr_ptr field_mgr); + const field_mgr_ptr field_mgr, + const int time_index = -1); // Version of above, but where nc and eamxx field names are identical void read_fields_from_file_for_iop(const std::string& file_name, const vos& field_names, const util::TimeStamp& initial_ts, - const field_mgr_ptr field_mgr) + const field_mgr_ptr field_mgr, + const int time_index = -1) { - read_fields_from_file_for_iop(file_name, field_names, field_names, initial_ts, field_mgr); + read_fields_from_file_for_iop(file_name, field_names, field_names, initial_ts, field_mgr, time_index); } // Set fields using data loaded from the iop file @@ -175,10 +180,13 @@ class IntensiveObservationPeriod } }; + enum IOPFieldType { + FromFile, + Computed + }; + void initialize_iop_file(const util::TimeStamp& run_t0, - int model_nlevs, - const Field& hyam, - const Field& hybm); + int model_nlevs); ekat::Comm m_comm; ekat::ParameterList m_params; @@ -195,6 +203,7 @@ class IntensiveObservationPeriod std::map m_iop_file_varnames; std::map m_iop_field_surface_varnames; + std::map m_iop_field_type; }; // class IntensiveObservationPeriod } // namespace control diff --git a/components/eamxx/src/control/tests/CMakeLists.txt b/components/eamxx/src/control/tests/CMakeLists.txt index 43aa5cebab1b..401c0f576268 100644 --- a/components/eamxx/src/control/tests/CMakeLists.txt +++ b/components/eamxx/src/control/tests/CMakeLists.txt @@ -1,5 +1,5 @@ # NOTE: if you have baseline-type tests, add the subdirectory OUTSIDE the following if statement -if (NOT ${SCREAM_BASELINES_ONLY}) +if (NOT ${SCREAM_ONLY_GENERATE_BASELINES}) include (ScreamUtils) # Unit test the ad diff --git a/components/eamxx/src/diagnostics/CMakeLists.txt b/components/eamxx/src/diagnostics/CMakeLists.txt index 5818c4d836a8..f34d5b99638f 100644 --- a/components/eamxx/src/diagnostics/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/CMakeLists.txt @@ -16,6 +16,7 @@ set(DIAGNOSTIC_SRCS vertical_layer.cpp virtual_temperature.cpp water_path.cpp + wind_speed.cpp ) add_library(diagnostics ${DIAGNOSTIC_SRCS}) diff --git a/components/eamxx/src/diagnostics/exner.hpp b/components/eamxx/src/diagnostics/exner.hpp index 5e029b716bf0..dead0a5cbf64 100644 --- a/components/eamxx/src/diagnostics/exner.hpp +++ b/components/eamxx/src/diagnostics/exner.hpp @@ -16,9 +16,6 @@ class ExnerDiagnostic : public AtmosphereDiagnostic // Constructors ExnerDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - // The name of the diagnostic std::string name () const { return "Exner"; } diff --git a/components/eamxx/src/diagnostics/field_at_height.cpp b/components/eamxx/src/diagnostics/field_at_height.cpp index 7759dfe5811d..38ae5e3e5503 100644 --- a/components/eamxx/src/diagnostics/field_at_height.cpp +++ b/components/eamxx/src/diagnostics/field_at_height.cpp @@ -38,6 +38,13 @@ FieldAtHeight (const ekat::Comm& comm, const ekat::ParameterList& params) : AtmosphereDiagnostic(comm,params) { m_field_name = m_params.get("field_name"); + auto surf_ref = m_params.get("surface_reference"); + EKAT_REQUIRE_MSG(surf_ref == "sealevel" or surf_ref == "surface", + "Error! Invalid surface reference for FieldAtHeight.\n" + " - field name: " + m_field_name + "\n" + " - surface reference: " + surf_ref + "\n" + " - valid options: sealevel, surface\n"); + m_z_name = (surf_ref == "sealevel") ? "z" : "geopotential"; const auto& location = m_params.get("vertical_location"); auto chars_start = location.find_first_not_of("0123456789."); EKAT_REQUIRE_MSG (chars_start!=0 && chars_start!=std::string::npos, @@ -52,7 +59,7 @@ FieldAtHeight (const ekat::Comm& comm, const ekat::ParameterList& params) "Error! Invalid string for height value for FieldAtHeight.\n" " - input string : " + location + "\n" " - expected format: Nm, with N integer\n"); - m_diag_name = m_field_name + "_at_" + m_params.get("vertical_location"); + m_diag_name = m_field_name + "_at_" + m_params.get("vertical_location") + "_above_" + surf_ref; } void FieldAtHeight:: @@ -62,8 +69,8 @@ set_grids (const std::shared_ptr grids_manager) add_field(m_field_name,gname); // We don't know yet which one we need - add_field("z_mid",gname); - add_field("z_int",gname); + add_field(m_z_name+"_mid",gname); + add_field(m_z_name+"_int",gname); } void FieldAtHeight:: @@ -90,7 +97,7 @@ initialize_impl (const RunType /*run_type*/) " - field layout: " + to_string(layout) + "\n"); // Figure out the z value - m_z_name = tag==LEV ? "z_mid" : "z_int"; + m_z_suffix = tag==LEV ? "_mid" : "_int"; // All good, create the diag output FieldIdentifier d_fid (m_diag_name,layout.strip_dim(tag),fid.get_units(),fid.get_grid_name()); @@ -111,7 +118,7 @@ initialize_impl (const RunType /*run_type*/) // ========================================================================================= void FieldAtHeight::compute_diagnostic_impl() { - const auto z_view = get_field_in(m_z_name).get_view(); + const auto z_view = get_field_in(m_z_name + m_z_suffix).get_view(); const Field& f = get_field_in(m_field_name); const auto& fl = f.get_header().get_identifier().get_layout(); diff --git a/components/eamxx/src/diagnostics/field_at_height.hpp b/components/eamxx/src/diagnostics/field_at_height.hpp index 54843b4f1a0e..e6198153f940 100644 --- a/components/eamxx/src/diagnostics/field_at_height.hpp +++ b/components/eamxx/src/diagnostics/field_at_height.hpp @@ -33,6 +33,7 @@ class FieldAtHeight : public AtmosphereDiagnostic std::string m_diag_name; std::string m_z_name; + std::string m_z_suffix; std::string m_field_name; Real m_z; diff --git a/components/eamxx/src/diagnostics/longwave_cloud_forcing.cpp b/components/eamxx/src/diagnostics/longwave_cloud_forcing.cpp index 23ef59891e9f..4e44dff9dc76 100644 --- a/components/eamxx/src/diagnostics/longwave_cloud_forcing.cpp +++ b/components/eamxx/src/diagnostics/longwave_cloud_forcing.cpp @@ -19,6 +19,8 @@ void LongwaveCloudForcingDiagnostic::set_grids(const std::shared_ptrget_grid("Physics"); const auto& grid_name = grid->name(); @@ -33,7 +35,7 @@ void LongwaveCloudForcingDiagnostic::set_grids(const std::shared_ptr("LW_clrsky_flux_up", scalar3d_layout_mid, W/m2, grid_name); // Construct and allocate the diagnostic field - FieldIdentifier fid (name(), scalar2d_layout_col, W/m2, grid_name); + FieldIdentifier fid (name(), scalar2d_layout_col, radflux_units, grid_name); m_diagnostic_output = Field(fid); auto& C_ap = m_diagnostic_output.get_header().get_alloc_properties(); C_ap.request_allocation(); diff --git a/components/eamxx/src/diagnostics/longwave_cloud_forcing.hpp b/components/eamxx/src/diagnostics/longwave_cloud_forcing.hpp index 9703330ce0cb..efe0e7c04760 100644 --- a/components/eamxx/src/diagnostics/longwave_cloud_forcing.hpp +++ b/components/eamxx/src/diagnostics/longwave_cloud_forcing.hpp @@ -16,9 +16,6 @@ class LongwaveCloudForcingDiagnostic : public AtmosphereDiagnostic // Constructors LongwaveCloudForcingDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - // The name of the diagnostic std::string name () const { return "LongwaveCloudForcing"; } diff --git a/components/eamxx/src/diagnostics/potential_temperature.hpp b/components/eamxx/src/diagnostics/potential_temperature.hpp index 1e0af49fb597..933a6935005d 100644 --- a/components/eamxx/src/diagnostics/potential_temperature.hpp +++ b/components/eamxx/src/diagnostics/potential_temperature.hpp @@ -20,9 +20,6 @@ class PotentialTemperatureDiagnostic : public AtmosphereDiagnostic // Constructors PotentialTemperatureDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - // The name of the diagnostic std::string name () const { return "PotentialTemperature"; } diff --git a/components/eamxx/src/diagnostics/register_diagnostics.hpp b/components/eamxx/src/diagnostics/register_diagnostics.hpp index 181a531a7c87..1f0c3bd63e3f 100644 --- a/components/eamxx/src/diagnostics/register_diagnostics.hpp +++ b/components/eamxx/src/diagnostics/register_diagnostics.hpp @@ -19,6 +19,7 @@ #include "diagnostics/field_at_pressure_level.hpp" #include "diagnostics/precip_surf_mass_flux.hpp" #include "diagnostics/surf_upward_latent_heat_flux.hpp" +#include "diagnostics/wind_speed.hpp" namespace scream { @@ -45,6 +46,7 @@ inline void register_diagnostics () { diag_factory.register_product("VaporFlux",&create_atmosphere_diagnostic); diag_factory.register_product("precip_surf_mass_flux",&create_atmosphere_diagnostic); diag_factory.register_product("surface_upward_latent_heat_flux",&create_atmosphere_diagnostic); + diag_factory.register_product("wind_speed",&create_atmosphere_diagnostic); } } // namespace scream diff --git a/components/eamxx/src/diagnostics/sea_level_pressure.hpp b/components/eamxx/src/diagnostics/sea_level_pressure.hpp index 8c522a2e75fe..0ae793658cd1 100644 --- a/components/eamxx/src/diagnostics/sea_level_pressure.hpp +++ b/components/eamxx/src/diagnostics/sea_level_pressure.hpp @@ -25,9 +25,6 @@ class SeaLevelPressureDiagnostic : public AtmosphereDiagnostic // Constructors SeaLevelPressureDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - // The name of the diagnostic std::string name () const { return "SeaLevelPressure"; } diff --git a/components/eamxx/src/diagnostics/shortwave_cloud_forcing.cpp b/components/eamxx/src/diagnostics/shortwave_cloud_forcing.cpp index e0e815549c8a..9ca5ea4a8a14 100644 --- a/components/eamxx/src/diagnostics/shortwave_cloud_forcing.cpp +++ b/components/eamxx/src/diagnostics/shortwave_cloud_forcing.cpp @@ -17,7 +17,8 @@ void ShortwaveCloudForcingDiagnostic::set_grids(const std::shared_ptrget_grid("Physics"); const auto& grid_name = grid->name(); @@ -28,13 +29,13 @@ void ShortwaveCloudForcingDiagnostic::set_grids(const std::shared_ptr("SW_flux_dn", scalar3d_layout_mid, W/m2, grid_name); - add_field("SW_flux_up", scalar3d_layout_mid, W/m2, grid_name); - add_field("SW_clrsky_flux_dn", scalar3d_layout_mid, W/m2, grid_name); - add_field("SW_clrsky_flux_up", scalar3d_layout_mid, W/m2, grid_name); + add_field("SW_flux_dn", scalar3d_layout_mid, radflux_units, grid_name); + add_field("SW_flux_up", scalar3d_layout_mid, radflux_units, grid_name); + add_field("SW_clrsky_flux_dn", scalar3d_layout_mid, radflux_units, grid_name); + add_field("SW_clrsky_flux_up", scalar3d_layout_mid, radflux_units, grid_name); // Construct and allocate the diagnostic field - FieldIdentifier fid (name(), scalar2d_layout_col, W/m2, grid_name); + FieldIdentifier fid (name(), scalar2d_layout_col, radflux_units, grid_name); m_diagnostic_output = Field(fid); auto& C_ap = m_diagnostic_output.get_header().get_alloc_properties(); C_ap.request_allocation(); diff --git a/components/eamxx/src/diagnostics/shortwave_cloud_forcing.hpp b/components/eamxx/src/diagnostics/shortwave_cloud_forcing.hpp index 9d676338a765..421d06d3fe07 100644 --- a/components/eamxx/src/diagnostics/shortwave_cloud_forcing.hpp +++ b/components/eamxx/src/diagnostics/shortwave_cloud_forcing.hpp @@ -16,9 +16,6 @@ class ShortwaveCloudForcingDiagnostic : public AtmosphereDiagnostic // Constructors ShortwaveCloudForcingDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - // The name of the diagnostic std::string name () const { return "ShortwaveCloudForcing"; } diff --git a/components/eamxx/src/diagnostics/surf_upward_latent_heat_flux.cpp b/components/eamxx/src/diagnostics/surf_upward_latent_heat_flux.cpp index 19a49ad595c5..009f8e6dd77f 100644 --- a/components/eamxx/src/diagnostics/surf_upward_latent_heat_flux.cpp +++ b/components/eamxx/src/diagnostics/surf_upward_latent_heat_flux.cpp @@ -22,6 +22,9 @@ set_grids (const std::shared_ptr grids_manager) { const auto m2 = ekat::units::m * ekat::units::m; const auto W = ekat::units::W; + auto radflux_units = W/(m2); + radflux_units.set_string("W/m2"); + const auto surf_evap_units = ekat::units::kg / m2 / ekat::units::s; auto grid = grids_manager->get_grid("Physics"); @@ -35,7 +38,7 @@ set_grids (const std::shared_ptr grids_manager) add_field("surf_evap", scalar2d_layout_mid, surf_evap_units, grid_name); // Construct and allocate the diagnostic field - FieldIdentifier fid(name(), scalar2d_layout_mid, W/m2, grid_name); + FieldIdentifier fid(name(), scalar2d_layout_mid, radflux_units, grid_name); // handle parent class member variables m_diagnostic_output = Field(fid); m_diagnostic_output.get_header().get_alloc_properties().request_allocation(); diff --git a/components/eamxx/src/diagnostics/tests/CMakeLists.txt b/components/eamxx/src/diagnostics/tests/CMakeLists.txt index 0882d3f6119d..d413b013f444 100644 --- a/components/eamxx/src/diagnostics/tests/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/tests/CMakeLists.txt @@ -6,7 +6,7 @@ function (createDiagTest test_name test_srcs) LABELS diagnostics) endfunction () -if (NOT SCREAM_BASELINES_ONLY) +if (NOT SCREAM_ONLY_GENERATE_BASELINES) include(ScreamUtils) # Test extracting a single level of a field @@ -59,4 +59,6 @@ if (NOT SCREAM_BASELINES_ONLY) # Test surface latent heat flux CreateDiagTest(surface_upward_latent_heat_flux "surf_upward_latent_heat_flux_tests.cpp") + # Test wind speed diagnostic + CreateDiagTest(wind_speed "wind_speed_tests.cpp") endif() diff --git a/components/eamxx/src/diagnostics/tests/field_at_height_tests.cpp b/components/eamxx/src/diagnostics/tests/field_at_height_tests.cpp index 0d45eb62e879..57a057116102 100644 --- a/components/eamxx/src/diagnostics/tests/field_at_height_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/field_at_height_tests.cpp @@ -9,6 +9,10 @@ namespace scream { +void f_z_src(const Real y0, const Real m, const Field& z_data, Field& out_data); +void f_z_tgt(const Real y0, const Real m, const Real z_target, const Field& z_data, Field& out_data); +bool views_are_approx_equal(const Field& f0, const Field& f1, const Real tol, const bool msg = true); + TEST_CASE("field_at_height") { using namespace ShortFieldTagsNames; @@ -18,11 +22,12 @@ TEST_CASE("field_at_height") // Get an MPI comm group for test ekat::Comm comm(MPI_COMM_WORLD); - constexpr int nruns = 10; + constexpr int nruns = 100; util::TimeStamp t0 ({2022,1,1},{0,0,0}); // Create a grids manager w/ a point grid + constexpr Real tol = std::numeric_limits::epsilon()*1e5; int ncols = 3; int ndims = 4; int nlevs = 10; @@ -31,35 +36,57 @@ TEST_CASE("field_at_height") gm->build_grids(); auto grid = gm->get_grid("Point Grid"); - // Create input test fields, as well as z_mid/int fields const auto m = ekat::units::m; + // Create input data test fields FieldIdentifier s_mid_fid ("s_mid",FieldLayout({COL, LEV},{ncols, nlevs }),m,grid->name()); FieldIdentifier s_int_fid ("s_int",FieldLayout({COL, ILEV},{ncols, nlevs+1}),m,grid->name()); FieldIdentifier v_mid_fid ("v_mid",FieldLayout({COL,CMP, LEV},{ncols,ndims,nlevs }),m,grid->name()); FieldIdentifier v_int_fid ("v_int",FieldLayout({COL,CMP,ILEV},{ncols,ndims,nlevs+1}),m,grid->name()); - FieldIdentifier z_mid_fid ("z_mid",FieldLayout({COL, LEV},{ncols, nlevs }),m,grid->name()); - FieldIdentifier z_int_fid ("z_int",FieldLayout({COL, ILEV},{ncols, nlevs+1}),m,grid->name()); + // Create vertical fields z and geo on both midpoints and interfaces + FieldIdentifier z_surf_fid ("z_surf", FieldLayout({COL },{ncols }),m,grid->name()); + FieldIdentifier z_mid_fid ("z_mid", FieldLayout({COL, LEV},{ncols, nlevs }),m,grid->name()); + FieldIdentifier z_int_fid ("z_int", FieldLayout({COL, ILEV},{ncols, nlevs+1}),m,grid->name()); + FieldIdentifier geo_mid_fid ("geopotential_mid",FieldLayout({COL, LEV},{ncols, nlevs }),m,grid->name()); + FieldIdentifier geo_int_fid ("geopotential_int",FieldLayout({COL, ILEV},{ncols, nlevs+1}),m,grid->name()); + // Keep track of reference fields for comparison + FieldIdentifier s_tgt_fid ("scalar_target",FieldLayout({COL },{ncols }),m,grid->name()); + FieldIdentifier v_tgt_fid ("vector_target",FieldLayout({COL,CMP},{ncols,ndims}),m,grid->name()); - Field s_mid (s_mid_fid); - Field s_int (s_int_fid); - Field v_mid (v_mid_fid); - Field v_int (v_int_fid); - Field z_mid (z_mid_fid); - Field z_int (z_int_fid); + Field s_mid (s_mid_fid); + Field s_int (s_int_fid); + Field v_mid (v_mid_fid); + Field v_int (v_int_fid); + Field z_surf (z_surf_fid); + Field z_mid (z_mid_fid); + Field z_int (z_int_fid); + Field geo_mid (geo_mid_fid); + Field geo_int (geo_int_fid); + Field s_tgt (s_tgt_fid); + Field v_tgt (v_tgt_fid); s_mid.allocate_view(); s_int.allocate_view(); v_mid.allocate_view(); v_int.allocate_view(); + z_surf.allocate_view(); z_mid.allocate_view(); z_int.allocate_view(); + geo_mid.allocate_view(); + geo_int.allocate_view(); + s_tgt.allocate_view(); + v_tgt.allocate_view(); s_mid.get_header().get_tracking().update_time_stamp(t0); s_int.get_header().get_tracking().update_time_stamp(t0); v_mid.get_header().get_tracking().update_time_stamp(t0); v_int.get_header().get_tracking().update_time_stamp(t0); + z_surf.get_header().get_tracking().update_time_stamp(t0); z_mid.get_header().get_tracking().update_time_stamp(t0); z_int.get_header().get_tracking().update_time_stamp(t0); + geo_mid.get_header().get_tracking().update_time_stamp(t0); + geo_int.get_header().get_tracking().update_time_stamp(t0); + s_tgt.get_header().get_tracking().update_time_stamp(t0); + v_tgt.get_header().get_tracking().update_time_stamp(t0); auto print = [&](const std::string& msg) { if (comm.am_i_root()) { @@ -69,16 +96,19 @@ TEST_CASE("field_at_height") auto engine = scream::setup_random_test(&comm); using IPDF = std::uniform_int_distribution; + using RPDF = std::uniform_real_distribution; IPDF pdf_fields (0,1000); - IPDF pdf_levs (1,nlevs-1); + RPDF pdf_m (1,10); + RPDF pdf_y0 (0,5); // Lambda to create and run a diag, and return output auto run_diag = [&](const Field& f, const Field& z, - const std::string& loc) { + const std::string& loc, const std::string& surf_ref) { util::TimeStamp t0 ({2022,1,1},{0,0,0}); auto& factory = AtmosphereDiagnosticFactory::instance(); ekat::ParameterList pl; + pl.set("surface_reference",surf_ref); pl.set("vertical_location",loc); pl.set("field_name",f.name()); pl.set("grid_name",grid->name()); @@ -92,135 +122,231 @@ TEST_CASE("field_at_height") return diag->get_diagnostic(); }; - // Create z(i,j)=nlevs-j, which makes testing easier - for (auto f : {z_mid, z_int}) { - auto v = f.get_view(); - const auto& dims = f.get_header().get_identifier().get_layout().dims(); - for (int i=0; i(); + const auto& zmid_v = z_mid.get_view(); + const auto& zsurf_v = z_surf.get_view(); + const auto& geoint_v = geo_int.get_view(); + const auto& geomid_v = geo_mid.get_view(); + int min_col_thickness = z_top; + int max_surf = 0; + for (int ii=0; ii max_surf ? zsurf_v(ii) : max_surf; + const Real col_thickness = z_top - zsurf_v(ii); + min_col_thickness = min_col_thickness < col_thickness ? col_thickness : min_col_thickness; + const Real dz = (z_top - zsurf_v(ii))/nlevs; + zint_v(ii,0) = z_top; + geoint_v(ii,0) = z_top - zsurf_v(ii); // Note, the distance above surface needs to consider the surface height. + for (int jj=0; jj Testing throws error with unsupported reference height...\n"); + { + REQUIRE_THROWS(run_diag (s_mid,geo_mid,"1m","foobar")); } + print(" -> Testing throws error with unsupported reference height... OK\n"); // Run many times - Real z_tgt,lev_tgt; + int z_tgt; std::string loc; - for (int irun=0; irun(); - const auto& size = f.get_header().get_identifier().get_layout().size(); - for (int i=0; i Testing for a reference height above %s...\n",surf_ref.c_str()); + const auto mid_src = surf_ref == "sealevel" ? z_mid : geo_mid; + const auto int_src = surf_ref == "sealevel" ? z_int : geo_int; + const int max_surf_4test = surf_ref == "sealevel" ? max_surf : 0; + for (int irun=0; irun Testing with z_tgt coinciding with a z level\n"); - { - print(" -> scalar midpoint field...............\n"); - auto d = run_diag (s_mid,z_mid,loc); - auto tgt = s_mid.subfield(1,static_cast(lev_tgt)); - REQUIRE (views_are_equal(d,tgt,&comm)); - print(" -> scalar midpoint field............... OK!\n"); - } - { - print(" -> scalar interface field...............\n"); - auto d = run_diag (s_int,z_int,loc); - // z_mid = nlevs+1-ilev, so the tgt slice is nlevs+1-z_tgt - auto tgt = s_int.subfield(1,static_cast(lev_tgt)); - REQUIRE (views_are_equal(d,tgt,&comm)); - print(" -> scalar interface field............... OK!\n"); - } - { - print(" -> vector midpoint field...............\n"); - auto d = run_diag (v_mid,z_mid,loc); - // We can't subview over 3rd index and keep layout right, - // so do all cols separately - for (int i=0; i(lev_tgt)); - REQUIRE (views_are_equal(di,tgt,&comm)); + // Set target z-slice for testing to a random value. + z_tgt = pdf_levs(engine)+max_surf_4test; + loc = std::to_string(z_tgt) + "m"; + printf(" -> test at height of %s.............\n",loc.c_str()); + { + print(" -> scalar midpoint field...............\n"); + auto d = run_diag(s_mid,mid_src,loc,surf_ref); + f_z_tgt(inter,slope,z_tgt,mid_src,s_tgt); + REQUIRE (views_are_approx_equal(d,s_tgt,tol)); + print(" -> scalar midpoint field............... OK!\n"); + } + { + print(" -> scalar interface field...............\n"); + auto d = run_diag (s_int,int_src,loc,surf_ref); + f_z_tgt(inter,slope,z_tgt,int_src,s_tgt); + REQUIRE (views_are_approx_equal(d,s_tgt,tol)); + print(" -> scalar interface field............... OK!\n"); + } + { + print(" -> vector midpoint field...............\n"); + auto d = run_diag (v_mid,mid_src,loc,surf_ref); + f_z_tgt(inter,slope,z_tgt,mid_src,v_tgt); + REQUIRE (views_are_approx_equal(d,v_tgt,tol)); + print(" -> vector midpoint field............... OK!\n"); + } + { + print(" -> vector interface field...............\n"); + auto d = run_diag (v_int,int_src,loc,surf_ref); + f_z_tgt(inter,slope,z_tgt,int_src,v_tgt); + REQUIRE (views_are_approx_equal(d,v_tgt,tol)); + print(" -> vector interface field............... OK!\n"); + } + { + print(" -> Forced fail, give incorrect location...............\n"); + const int z_tgt_adj = (z_tgt+max_surf_4test)/2; + std::string loc_err = std::to_string(z_tgt_adj) + "m"; + auto d = run_diag(s_int,int_src,loc_err,surf_ref); + f_z_tgt(inter,slope,z_tgt,int_src,s_tgt); + REQUIRE (!views_are_approx_equal(d,s_tgt,tol,false)); + print(" -> Forced fail, give incorrect location............... OK!\n"); } - print(" -> vector midpoint field............... OK!\n"); } { - print(" -> vector interface field...............\n"); - auto d = run_diag (v_int,z_int,loc); - // We can't subview over 3rd index and keep layout right, - // so do all cols separately - for (int i=0; i(lev_tgt)); - REQUIRE (views_are_equal(di,tgt,&comm)); - } - print(" -> vector interface field............... OK!\n"); + print(" -> Forced extrapolation ...............\n"); + auto slope = pdf_m(engine); + auto inter = pdf_y0(engine); + f_z_src(inter, slope, int_src, s_int); + print(" -> at top...............\n"); + z_tgt = 2*z_top; + std::string loc = std::to_string(z_tgt) + "m"; + auto dtop = run_diag(s_int,int_src,loc,surf_ref); + f_z_tgt(inter,slope,z_tgt,int_src,s_tgt); + REQUIRE (views_are_approx_equal(dtop,s_tgt,tol)); + print(" -> at bot...............\n"); + z_tgt = 0; + loc = std::to_string(z_tgt) + "m"; + auto dbot = run_diag(s_int,int_src,loc,surf_ref); + f_z_tgt(inter,slope,z_tgt,int_src,s_tgt); + REQUIRE (views_are_approx_equal(dbot,s_tgt,tol)); + print(" -> Forced extrapolation............... OK!\n"); } + printf(" -> Testing for a reference height above %s... OK!\n",surf_ref.c_str()); + } +} - z_tgt = pdf_levs(engine) + 0.5; - lev_tgt = nlevs-z_tgt; - loc = std::to_string(z_tgt) + "m"; - - auto zp1 = static_cast(std::round(lev_tgt+0.5)); - auto zm1 = static_cast(std::round(lev_tgt-0.5)); - - print(" -> Testing with z_tgt between levels\n"); - { - print(" -> scalar midpoint field...............\n"); - auto d = run_diag (s_mid,z_mid,loc); - auto tgt = s_mid.subfield(1,zp1).clone(); - tgt.update(s_mid.subfield(1,zm1),0.5,0.5); - REQUIRE (views_are_equal(d,tgt,&comm)); - print(" -> scalar midpoint field............... OK!\n"); +//------------------------------- +// Set up the inpute data. To make the test simple we assume a linear distribution of the data +// with height. That way we can exactly calculate what a linear interpolation to a random +// height would be. +void f_z_src(const Real y0, const Real m, const Field& z_data, Field& out_data) { + using namespace ShortFieldTagsNames; + const auto layout = out_data.get_header().get_identifier().get_layout(); + if (layout.has_tag(CMP)) { // Is a vector layout, meaning different dims than z_data. + const auto& dims = layout.dims(); + const auto& z_view = z_data.get_view(); + const auto& out_view = out_data.get_view(); + for (int ii=0; ii scalar interface field...............\n"); - auto d = run_diag (s_int,z_int,loc); - auto tgt = s_int.subfield(1,zp1).clone(); - tgt.update(s_int.subfield(1,zm1),0.5,0.5); - REQUIRE (views_are_equal(d,tgt,&comm)); - print(" -> scalar interface field............... OK!\n"); + } else { // Not a vector output, easier to deal with + const auto z_view = z_data.get_internal_view_data(); + const auto& size = z_data.get_header().get_identifier().get_layout().size(); + auto out_view = out_data.get_internal_view_data(); + for (int ii=0; ii vector midpoint field...............\n"); - auto d = run_diag (v_mid,z_mid,loc); - // We can't subview over 3rd index and keep layout right, - // so do all cols separately - for (int i=0; i(); + const auto& zdims = z_data.get_header().get_identifier().get_layout().dims(); + if (layout.has_tag(CMP)) { // Is a vector layout, meaning different dims than z_target. + const auto& dims = layout.dims(); + const auto& out_view = out_data.get_view(); + for (int ii=0; ii z_view(ii,0)) { + out_view(ii,nd) = y0 + m*(nd+1)*z_view(ii,0); + } else if ( z_target < z_view(ii,zdims[1]-1)) { + out_view(ii,nd) = y0 + m*(nd+1)*z_view(ii,zdims[1]-1); + } else { + out_view(ii,nd) = y0 + m*(nd+1)*z_target; + } } - print(" -> vector midpoint field............... OK!\n"); } - { - print(" -> vector interface field...............\n"); - auto d = run_diag (v_int,z_int,loc); - // We can't subview over 3rd index and keep layout right, - // so do all cols separately - for (int i=0; i(); + for (int ii=0; ii z_view(ii,0)) { + out_view(ii) = y0 + m*z_view(ii,0); + } else if ( z_target < z_view(ii,zdims[1]-1)) { + out_view(ii) = y0 + m*z_view(ii,zdims[1]-1); + } else { + out_view(ii) = y0 + m*z_target; } - print(" -> vector interface field............... OK!\n"); } } + out_data.sync_to_dev(); +} +/*-----------------------------------------------------------------------------------------------*/ +bool views_are_approx_equal(const Field& f0, const Field& f1, const Real tol, const bool msg) +{ + const auto& l0 = f0.get_header().get_identifier().get_layout(); + const auto& l1 = f1.get_header().get_identifier().get_layout(); + EKAT_REQUIRE_MSG(l0==l1,"Error! views_are_approx_equal - the two fields don't have matching layouts."); + // Take advantage of field utils update, min and max to assess the max difference between the two fields + // simply. + auto ft = f0.clone(); + ft.update(f1,1.0,-1.0); + auto d_min = field_min(ft); + auto d_max = field_max(ft); + if (std::abs(d_min) > tol or std::abs(d_max) > tol) { + if (msg) { + printf("The two copies of (%16s) are NOT approx equal within a tolerance of %e.\n The min and max errors are %e and %e respectively.\n",f0.name().c_str(),tol,d_min,d_max); + } + return false; + } else { + return true; + } + } } // namespace scream diff --git a/components/eamxx/src/diagnostics/tests/vapor_flux_tests.cpp b/components/eamxx/src/diagnostics/tests/vapor_flux_tests.cpp index aceb705a61c7..bcd7efc8bb87 100644 --- a/components/eamxx/src/diagnostics/tests/vapor_flux_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/vapor_flux_tests.cpp @@ -83,7 +83,7 @@ void run(std::mt19937_64& engine) REQUIRE_THROWS (diag_factory.create("VaporFlux",comm,params)); // No 'Wind Component' params.set("Wind Component","foo"); REQUIRE_THROWS (diag_factory.create("VaporFlux",comm,params)); // Invalid 'Wind Component' - for (const std::string& which_comp : {"Zonal", "Meridional"}) { + for (const std::string which_comp : {"Zonal", "Meridional"}) { // Construct the Diagnostic params.set("Wind Component",which_comp); auto diag = diag_factory.create("VaporFlux",comm,params); diff --git a/components/eamxx/src/diagnostics/tests/wind_speed_tests.cpp b/components/eamxx/src/diagnostics/tests/wind_speed_tests.cpp new file mode 100644 index 000000000000..7e3affedacb2 --- /dev/null +++ b/components/eamxx/src/diagnostics/tests/wind_speed_tests.cpp @@ -0,0 +1,96 @@ +#include "catch2/catch.hpp" + +#include "diagnostics/register_diagnostics.hpp" +#include "share/grid/mesh_free_grids_manager.hpp" +#include "share/util/scream_setup_random_test.hpp" +#include "share/field/field_utils.hpp" + +namespace scream { + +std::shared_ptr +create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { + + const int num_global_cols = ncols*comm.size(); + + using vos_t = std::vector; + ekat::ParameterList gm_params; + gm_params.set("grids_names",vos_t{"Point Grid"}); + auto& pl = gm_params.sublist("Point Grid"); + pl.set("type","point_grid"); + pl.set("aliases",vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", nlevs); + + auto gm = create_mesh_free_grids_manager(comm,gm_params); + gm->build_grids(); + + return gm; +} + +TEST_CASE("wind_speed") +{ + using namespace ShortFieldTagsNames; + using namespace ekat::units; + + // A world comm + ekat::Comm comm(MPI_COMM_WORLD); + + // A time stamp + util::TimeStamp t0 ({2022,1,1},{0,0,0}); + + // Create a grids manager - single column for these tests + constexpr int nlevs = 33; + const int ngcols = 2*comm.size();; + auto gm = create_gm(comm,ngcols,nlevs); + auto grid = gm->get_grid("Physics"); + + // Input (randomized) velocity + auto vector3d = grid->get_3d_vector_layout(true,CMP,2); + FieldIdentifier uv_fid ("horiz_winds",vector3d,m/s,grid->name()); + Field uv(uv_fid); + uv.allocate_view(); + uv.get_header().get_tracking().update_time_stamp(t0); + + // Construct random number generator stuff + using RPDF = std::uniform_real_distribution; + RPDF pdf(-1,1); + auto engine = scream::setup_random_test(); + + // Construct the Diagnostics + std::map> diags; + auto& diag_factory = AtmosphereDiagnosticFactory::instance(); + register_diagnostics(); + + constexpr int ntests = 5; + for (int itest=0; itestset_grids(gm); + diag->set_required_field(uv); + diag->initialize(t0,RunType::Initial); + + // Run diag + diag->compute_diagnostic(); + + // Check result + uv.sync_to_host(); + diag->get_diagnostic().sync_to_host(); + + auto uv_h = uv.get_view(); + auto ws_h = diag->get_diagnostic().get_view(); + + for (int icol=0; icolget_num_local_dofs(); ++icol) { + for (int ilev=0; ilev + +namespace scream +{ + +WindSpeed:: +WindSpeed (const ekat::Comm& comm, const ekat::ParameterList& params) + : AtmosphereDiagnostic(comm,params) +{ + // Nothing to do here +} + +void WindSpeed:: +set_grids(const std::shared_ptr grids_manager) +{ + using namespace ekat::units; + using namespace ShortFieldTagsNames; + + auto grid = grids_manager->get_grid("Physics"); + const auto& grid_name = grid->name(); + + m_ncols = grid->get_num_local_dofs(); + m_nlevs = grid->get_num_vertical_levels(); + + auto scalar3d = grid->get_3d_scalar_layout(true); + auto vector3d = grid->get_3d_vector_layout(true,CMP,2); + + // The fields required for this diagnostic to be computed + add_field("horiz_winds", vector3d, Pa, grid_name); + + // Construct and allocate the 3d wind_speed field + FieldIdentifier fid ("wind_speed", scalar3d, m/s, grid_name); + m_diagnostic_output = Field(fid); + m_diagnostic_output.allocate_view(); +} + +void WindSpeed::compute_diagnostic_impl() +{ + using KT = KokkosTypes; + using RP = typename KT::RangePolicy; + + const auto uv = get_field_in("horiz_winds").get_view(); + const auto ws = m_diagnostic_output.get_view(); + + const int nlevs = m_nlevs; + Kokkos::parallel_for("Compute " + name(), RP(0,m_nlevs*m_ncols), + KOKKOS_LAMBDA(const int& idx) { + const int icol = idx / nlevs; + const int ilev = idx % nlevs; + const auto& u = uv(icol,0,ilev); + const auto& v = uv(icol,1,ilev); + ws (icol,ilev) = sqrt(u*u + v*v); + }); +} + +} //namespace scream diff --git a/components/eamxx/src/diagnostics/wind_speed.hpp b/components/eamxx/src/diagnostics/wind_speed.hpp new file mode 100644 index 000000000000..91ac551a1b76 --- /dev/null +++ b/components/eamxx/src/diagnostics/wind_speed.hpp @@ -0,0 +1,37 @@ +#ifndef EAMXX_WIND_SPEED_HPP +#define EAMXX_WIND_SPEED_HPP + +#include "share/atm_process/atmosphere_diagnostic.hpp" + +namespace scream +{ + +/* + * This diagnostic will compute the magnitute of the horiz_winds vector + */ + +class WindSpeed : public AtmosphereDiagnostic +{ +public: + // Constructors + WindSpeed (const ekat::Comm& comm, const ekat::ParameterList& params); + + // The name of the diagnostic + std::string name () const override { return "wind_speed"; } + + // Set the grid + void set_grids (const std::shared_ptr grids_manager) override; + +protected: +#ifdef KOKKOS_ENABLE_CUDA +public: +#endif + void compute_diagnostic_impl () override; + + int m_ncols; + int m_nlevs; +}; + +} //namespace scream + +#endif // EAMXX_WIND_SPEED_HPP diff --git a/components/eamxx/src/doubly-periodic/CMakeLists.txt b/components/eamxx/src/doubly-periodic/CMakeLists.txt deleted file mode 100644 index 09dab68a8e28..000000000000 --- a/components/eamxx/src/doubly-periodic/CMakeLists.txt +++ /dev/null @@ -1,60 +0,0 @@ -include (ScreamUtils) - -set(DP_SRCS - dp_f90.cpp - dp_iso_c.f90 - #${SCREAM_BASE_DIR}/../eam/src/control/apply_iop_forcing.F90 - #${SCREAM_BASE_DIR}/../eam/src/dynamics/se/se_iop_intr_mod.F90", - #${SCREAM_BASE_DIR}/../eam/src/control/iop_data_mod.F90", - #${SCREAM_BASE_DIR}/../eam/src/control/history_iop.F90" -) - -# Set cmake config options for Homme -if (NOT "${SCREAM_DYNAMICS_DYCORE}" STREQUAL "HOMME") - message(FATAL_ERROR "Requires homme") -endif() - -# Get or create the dynamics lib -# HOMME_TARGET NP PLEV QSIZE_D -CreateDynamicsLib("theta-l_kokkos" 4 72 10) - -if (NOT SCREAM_LIB_ONLY) - list(APPEND DP_SRCS - dp_functions_f90.cpp - ) # Add f90 bridges needed for testing -endif() - -# Add ETI source files if not on CUDA/HIP -if (NOT EAMXX_ENABLE_GPU OR Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE OR Kokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE) - list(APPEND DP_SRCS - eti/dp_advance_iop_forcing.cpp - eti/dp_advance_iop_nudging.cpp - eti/dp_advance_iop_subsidence.cpp - eti/dp_iop_setinitial.cpp - eti/dp_iop_broadcast.cpp - eti/dp_apply_iop_forcing.cpp - eti/dp_iop_domain_relaxation.cpp - eti/dp_crm_resolved_turb.cpp - eti/dp_iop_default_opts.cpp - eti/dp_iop_setopts.cpp - eti/dp_setiopupdate_init.cpp - eti/dp_setiopupdate.cpp - eti/dp_readiopdata.cpp - eti/dp_iop_intht.cpp - ) # DP ETI SRCS -endif() - -add_library(dp ${DP_SRCS}) -set_target_properties(dp PROPERTIES - Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/modules -) -target_include_directories(dp PUBLIC - ${CMAKE_CURRENT_BINARY_DIR}/modules - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/impl -) -target_link_libraries(dp PUBLIC physics_share scream_share ${dynLibName}) - -#if (NOT SCREAM_LIB_ONLY) -# add_subdirectory(tests) -#endif() diff --git a/components/eamxx/src/doubly-periodic/dp_constants.hpp b/components/eamxx/src/doubly-periodic/dp_constants.hpp deleted file mode 100644 index c974106f83ec..000000000000 --- a/components/eamxx/src/doubly-periodic/dp_constants.hpp +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef DP_CONSTANTS_HPP -#define DP_CONSTANTS_HPP - -namespace scream { -namespace dp { - -/* - * Mathematical constants used by dp. - */ - -template -struct Constants -{ - static constexpr Scalar iop_nudge_tq_low = 1050; - static constexpr Scalar iop_nudge_tq_high = 0; - static constexpr Scalar iop_nudge_tscale = 10800; -}; - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/dp_f90.cpp b/components/eamxx/src/doubly-periodic/dp_f90.cpp deleted file mode 100644 index 2802e5266fd5..000000000000 --- a/components/eamxx/src/doubly-periodic/dp_f90.cpp +++ /dev/null @@ -1,27 +0,0 @@ -#include "dp_f90.hpp" -#include "physics_constants.hpp" - -#include "ekat/ekat_assert.hpp" - -using scream::Int; - -extern "C" { - -void init_time_level_c (const int& nm1, const int& n0, const int& np1, - const int& nstep, const int& nstep0); - -} - -namespace scream { -namespace dp { - -void dp_init(const bool force_reinit) { - static bool is_init = false; - if (!is_init || force_reinit) { - init_time_level_c(10, 3, 11, 5, 4); - is_init = true; - } -} - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/dp_f90.hpp b/components/eamxx/src/doubly-periodic/dp_f90.hpp deleted file mode 100644 index 338a583f777a..000000000000 --- a/components/eamxx/src/doubly-periodic/dp_f90.hpp +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef SCREAM_DP_F90_HPP -#define SCREAM_DP_F90_HPP - -#include "share/scream_types.hpp" - -#include -#include - -namespace scream { -namespace dp { - -// Initialize DP. This is only for standalone DP testing. -void dp_init(const bool force_reinit=false); - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/dp_functions.hpp b/components/eamxx/src/doubly-periodic/dp_functions.hpp deleted file mode 100644 index 18e6ec58ba9e..000000000000 --- a/components/eamxx/src/doubly-periodic/dp_functions.hpp +++ /dev/null @@ -1,325 +0,0 @@ -#ifndef DP_FUNCTIONS_HPP -#define DP_FUNCTIONS_HPP - -#include "physics/share/physics_constants.hpp" -#include "dp_constants.hpp" - -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack_kokkos.hpp" -#include "ekat/ekat_workspace.hpp" - -#include "Elements.hpp" -#include "Tracers.hpp" - -namespace scream { -namespace dp { - -/* - * Functions is a stateless struct used to encapsulate a - * number of functions for DP. We use the ETI pattern for - * these functions. - * - * DP assumptions: - * - Kokkos team policies have a vector length of 1 - */ - -using element_t = Homme::Elements; -using tracer_t = Homme::Tracers; -struct hvcoord_t{}; -struct timelevel_t{}; -struct hybrid_t{}; - -template -struct Functions -{ - // - // ------- Types -------- - // - - using Scalar = ScalarT; - using Device = DeviceT; - - template - using BigPack = ekat::Pack; - template - using SmallPack = ekat::Pack; - - using IntSmallPack = SmallPack; - using Pack = BigPack; - using Spack = SmallPack; - - using Mask = ekat::Mask; - using Smask = ekat::Mask; - - using KT = ekat::KokkosTypes; - using ExeSpace = typename KT::ExeSpace; - - using C = physics::Constants; - using DPC = dp::Constants; - - template - using view_1d = typename KT::template view_1d; - template - using view_2d = typename KT::template view_2d; - template - using view_3d = typename KT::template view_3d; - - template - using view_1d_ptr_array = typename KT::template view_1d_ptr_carray; - - template - using uview_1d = typename ekat::template Unmanaged >; - - template - using uview_2d = typename ekat::template Unmanaged >; - - using MemberType = typename KT::MemberType; - - using WorkspaceMgr = typename ekat::WorkspaceManager; - using Workspace = typename WorkspaceMgr::Workspace; - - // - // --------- Functions --------- - // - - // --------------------------------------------------------------------- - // Define the pressures of the interfaces and midpoints from the - // coordinate definitions and the surface pressure. - // --------------------------------------------------------------------- - KOKKOS_FUNCTION - static void plevs0( - // Input arguments - const Int& nver, // vertical dimension - const Scalar& ps, // Surface pressure (pascals) - const uview_1d& hyai, // ps0 component of hybrid coordinate - interfaces - const uview_1d& hyam, // ps0 component of hybrid coordinate - midpoints - const uview_1d& hybi, // ps component of hybrid coordinate - interfaces - const uview_1d& hybm, // ps component of hybrid coordinate - midpoints - // Kokkos stuff - const MemberType& team, - // Output arguments - const uview_1d& pint, // Pressure at model interfaces - const uview_1d& pmid, // Pressure at model levels - const uview_1d& pdel); // Layer thickness (pint(k+1) - pint(k)) - - //----------------------------------------------------------------------- - // advance_iop_forcing - // Purpose: - // Apply large scale forcing for t, q, u, and v as provided by the - // case IOP forcing file. - // - // Author: - // Original version: Adopted from CAM3.5/CAM5 - // Updated version for E3SM: Peter Bogenschutz (bogenschutz1@llnl.gov) - // and replaces the forecast.F90 routine in CAM3.5/CAM5/CAM6/E3SMv1/E3SMv2 - // CXX version: James Foucar (jgfouca@sandia.gov) - // - //----------------------------------------------------------------------- - KOKKOS_FUNCTION - static void advance_iop_forcing( - // Input arguments - const Int& plev, // number of vertical levels - const Int& pcnst, // number of advected constituents including cloud water - const bool& have_u, // dataset contains u - const bool& have_v, // dataset contains v - const bool& dp_crm, // use 3d forcing - const bool& use_3dfrc, // use 3d forcing - const Scalar& scm_dt, // model time step [s] - const Scalar& ps_in, // surface pressure [Pa] - const uview_1d& u_in, // zonal wind [m/s] - const uview_1d& v_in, // meridional wind [m/s] - const uview_1d& t_in, // temperature [K] - const uview_2d& q_in, // q tracer array [units vary] - const uview_1d& t_phys_frc, // temperature forcing from physics [K/s] - const uview_1d& divt3d, // 3D T advection - const uview_2d& divq3d, // 3D q advection - const uview_1d& divt, // Divergence of temperature - const uview_2d& divq, // Divergence of moisture - const uview_1d& wfld, // Vertical motion (slt) - const uview_1d& uobs, // actual u wind - const uview_1d& vobs, // actual v wind - const uview_1d& hyai, // ps0 component of hybrid coordinate - interfaces - const uview_1d& hyam, // ps0 component of hybrid coordinate - midpoints - const uview_1d& hybi, // ps component of hybrid coordinate - interfaces - const uview_1d& hybm, // ps component of hybrid coordinate - midpoints - // Kokkos stuff - const MemberType& team, - const Workspace& workspace, - // Output arguments - const uview_1d& u_update, // updated temperature [K] - const uview_1d& v_update, // updated q tracer array [units vary] - const uview_1d& t_update, // updated zonal wind [m/s] - const uview_2d& q_update); // updated meridional wind [m/s] - - //----------------------------------------------------------------------- - // advance_iop_nudging - // Purpose: - // Option to nudge t and q to observations as specified by the IOP file - // - // Author: - // Original version: Adopted from CAM3.5/CAM5 - // Updated version for E3SM: Peter Bogenschutz (bogenschutz1@llnl.gov) - // CXX version: Conrad Clevenger (tccleve@sandia.gov) - // - //----------------------------------------------------------------------- - KOKKOS_FUNCTION - static void advance_iop_nudging( - // Input arguments - const Int& plev, // number of vertical levels - const Scalar& scm_dt, // model time step [s] - const Scalar& ps_in, // surface pressure [Pa] - const uview_1d& t_in, // temperature [K] - const uview_1d& q_in, // water vapor mixing ratio [kg/kg] - const uview_1d& tobs, // observed temperature [K] - const uview_1d& qobs, // observed vapor mixing ratio [kg/kg] - const uview_1d& hyai, // ps0 component of hybrid coordinate - interfaces - const uview_1d& hyam, // ps0 component of hybrid coordinate - midpoints - const uview_1d& hybi, // ps component of hybrid coordinate - interfaces - const uview_1d& hybm, // ps component of hybrid coordinate - midpoints - // Kokkos stuff - const MemberType& team, - const Workspace& workspace, - // Output arguments - const uview_1d& t_update, // updated temperature [K] - const uview_1d& q_update, // updated water vapor [kg/kg] - const uview_1d& relaxt, // relaxation of temperature [K/s] - const uview_1d& relaxq); // relaxation of vapor [kg/kg/s] - - KOKKOS_INLINE_FUNCTION - static void do_advance_iop_subsidence_update( - const Int& k, - const Int& plev, - const Spack& fac, - const Spack& swfldint, - const Spack& swfldint_p1, - const uview_1d& in, - const uview_1d& in_s, - const uview_1d& update); - - //----------------------------------------------------------------------- - // - // Purpose: - // Option to compute effects of large scale subsidence on T, q, u, and v. - // Code originated from CAM3.5/CAM5 Eulerian subsidence computation for SCM - // in the old forecast.f90 routine. - //----------------------------------------------------------------------- - KOKKOS_FUNCTION - static void advance_iop_subsidence( - // Input arguments - const Int& plev, // number of vertical levels - const Int& pcnst, // number of advected constituents including cloud water - const Scalar& scm_dt, // model time step [s] - const Scalar& ps_in, // surface pressure [Pa] - const uview_1d& u_in, // zonal wind [m/s] - const uview_1d& v_in, // meridional wind [m/s] - const uview_1d& t_in, // temperature [K] - const uview_2d& q_in, // tracer [vary] - const uview_1d& hyai, // ps0 component of hybrid coordinate - interfaces - const uview_1d& hyam, // ps0 component of hybrid coordinate - midpoints - const uview_1d& hybi, // ps component of hybrid coordinate - interfaces - const uview_1d& hybm, // ps component of hybrid coordinate - midpoints - const uview_1d& wfld, // Vertical motion (slt) - // Kokkos stuff - const MemberType& team, - const Workspace& workspace, - // Output arguments - const uview_1d& u_update, // zonal wind [m/s] - const uview_1d& v_update, // meridional wind [m/s] - const uview_1d& t_update, // temperature [m/s] - const uview_2d& q_update); // tracer [vary] - - //--------------------------------------------------------- - // Purpose: Set initial values from IOP files (where available) - // when running SCM or DP-CRM. - //---------------------------------------------------------- - static void iop_setinitial( - // Input arguments - const Int& plev, // number of vertical levels - const Int& pcnst, // number of advected constituents including cloud water - const Int& nelemd, // number of elements per MPI task - const Int& np, // NP - const Int& nstep, // the timestep number - const bool& use_replay, // use e3sm generated forcing - const bool& dynproc, // Designation of a dynamics processor - AaronDonahue - const bool& have_t, // dataset contains t - const bool& have_q, // dataset contains q - const bool& have_ps, // dataset contains ps - const bool& have_u, // dataset contains u - const bool& have_v, // dataset contains v - const bool& have_numliq, // dataset contains numliq - const bool& have_cldliq, // dataset contains cldliq - const bool& have_numice, // dataset contains numice - const bool& have_cldice, // dataset contains cldice - const bool& scm_zero_non_iop_tracers, // Ignore all tracers from initial conditions file, and default all tracers not specified in IOP to minimum value (usually zero) - const bool& is_first_restart_step, // is first restart step - const uview_1d& qmin, // minimum permitted constituent concentration (kg/kg) (pcnst) - const uview_1d& uobs, // actual u wind - const uview_1d& vobs, // actual v wind - const uview_1d& numliqobs, // actual ??? - const uview_1d& numiceobs, // actual ??? - const uview_1d& cldliqobs, // actual ??? - const uview_1d& cldiceobs, // actual ??? - const Scalar& psobs, // ??? - const uview_1d& dx_short, // short length scale in km (nelemd) - // Input/Output arguments - Scalar& dyn_dx_size, // for use in doubly periodic CRM mode - tracer_t& tracers, // tracers - element_t& elem, // elements - const uview_1d& tobs, // actual temperature, dims=(plev) - const uview_1d& qobs); // actual W.V. Mixing ratio - - KOKKOS_FUNCTION - static void iop_broadcast(); - - KOKKOS_FUNCTION - static void apply_iop_forcing(const Int& nelemd, const uview_1d& elem, hvcoord_t& hvcoord, const hybrid_t& hybrid, const timelevel_t& tl, const Int& n, const bool& t_before_advance, const Int& nets, const Int& nete); - - KOKKOS_FUNCTION - static void iop_domain_relaxation(const Int& nelemd, const Int& np, const Int& nlev, const uview_1d& elem, const hvcoord_t& hvcoord, const hybrid_t& hybrid, const Int& t1, const uview_1d& dp, const Int& nelemd_todo, const Int& np_todo, const Spack& dt); - - KOKKOS_FUNCTION - static void crm_resolved_turb(const Int& nelemd, const uview_1d& elem, const hvcoord_t& hvcoord, const hybrid_t& hybrid, const Int& t1, const Int& nelemd_todo, const Int& np_todo); - - static void iop_default_opts(Spack& scmlat_out, Spack& scmlon_out, std::string& iopfile_out, bool& single_column_out, bool& scm_iop_srf_prop_out, bool& iop_nudge_tq_out, bool& iop_nudge_uv_out, Spack& iop_nudge_tq_low_out, Spack& iop_nudge_tq_high_out, Spack& iop_nudge_tscale_out, bool& scm_observed_aero_out, bool& iop_dosubsidence_out, bool& scm_multcols_out, bool& dp_crm_out, Spack& iop_perturb_high_out, bool& precip_off_out, bool& scm_zero_non_iop_tracers_out); - - static void iop_setopts(const Spack& scmlat_in, const Spack& scmlon_in, const std::string& iopfile_in, const bool& single_column_in, const bool& scm_iop_srf_prop_in, const bool& iop_nudge_tq_in, const bool& iop_nudge_uv_in, const Spack& iop_nudge_tq_low_in, const Spack& iop_nudge_tq_high_in, const Spack& iop_nudge_tscale_in, const bool& scm_observed_aero_in, const bool& iop_dosubsidence_in, const bool& scm_multcols_in, const bool& dp_crm_in, const Spack& iop_perturb_high_in, const bool& precip_off_in, const bool& scm_zero_non_iop_tracers_in); - - KOKKOS_FUNCTION - static void setiopupdate_init(); - - KOKKOS_FUNCTION - static void setiopupdate(); - - KOKKOS_FUNCTION - static void readiopdata(const Int& plev, const bool& iop_update_phase1, const uview_1d& hyam, const uview_1d& hybm); - - KOKKOS_FUNCTION - static void iop_intht(); -}; // struct Functions - -} // namespace dp -} // namespace scream - -// If a GPU build, without relocatable device code enabled, make all code available -// to the translation unit; otherwise, ETI is used. -#if defined(EAMXX_ENABLE_GPU) && !defined(KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE) \ - && !defined(KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE) - -# include "impl/dp_advance_iop_forcing_impl.hpp" -# include "impl/dp_advance_iop_nudging_impl.hpp" -# include "impl/dp_advance_iop_subsidence_impl.hpp" -# include "impl/dp_iop_setinitial_impl.hpp" -# include "impl/dp_iop_broadcast_impl.hpp" -# include "impl/dp_apply_iop_forcing_impl.hpp" -# include "impl/dp_iop_domain_relaxation_impl.hpp" -# include "impl/dp_crm_resolved_turb_impl.hpp" -# include "impl/dp_iop_default_opts_impl.hpp" -# include "impl/dp_iop_setopts_impl.hpp" -# include "impl/dp_setiopupdate_init_impl.hpp" -# include "impl/dp_setiopupdate_impl.hpp" -# include "impl/dp_readiopdata_impl.hpp" -# include "impl/dp_iop_intht_impl.hpp" -#endif // GPU && !KOKKOS_ENABLE_*_RELOCATABLE_DEVICE_CODE - -#endif // DP_FUNCTIONS_HPP diff --git a/components/eamxx/src/doubly-periodic/dp_functions_f90.cpp b/components/eamxx/src/doubly-periodic/dp_functions_f90.cpp deleted file mode 100644 index bad3c68dcb2d..000000000000 --- a/components/eamxx/src/doubly-periodic/dp_functions_f90.cpp +++ /dev/null @@ -1,526 +0,0 @@ -#include "dp_functions_f90.hpp" - -#include "dp_f90.hpp" - -#include "ekat/ekat_assert.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "ekat/ekat_pack_kokkos.hpp" -#include "ekat/kokkos/ekat_subview_utils.hpp" - -#include "share/util/scream_deep_copy.hpp" - -#include - -using scream::Real; -using scream::Int; - -using scream::dp::element_t; -using scream::dp::hvcoord_t; -using scream::dp::hybrid_t; -using scream::dp::timelevel_t; - -// -// A C interface to DP fortran calls. The stubs below will link to fortran definitions in dp_iso_c.f90 -// - -extern "C" { - -void advance_iop_forcing_c(Int plev, Int pcnst, Real scm_dt, Real ps_in, Real* u_in, Real* v_in, Real* t_in, Real* q_in, Real* t_phys_frc, Real* u_update, Real* v_update, Real* t_update, Real* q_update); -void advance_iop_nudging_c(Int plev, Real scm_dt, Real ps_in, Real* t_in, Real* q_in, Real* t_update, Real* q_update, Real* relaxt, Real* relaxq); -void advance_iop_subsidence_c(Int plev, Int pcnst, Real scm_dt, Real ps_in, Real* u_in, Real* v_in, Real* t_in, Real* q_in, Real* u_update, Real* v_update, Real* t_update, Real* q_update); -void iop_setinitial_c(Int nelemd, element_t* elem); -void iop_broadcast_c(); -void apply_iop_forcing_c(Int nelemd, element_t* elem, hvcoord_t* hvcoord, hybrid_t* hybrid, timelevel_t* tl, Int n, bool t_before_advance, Int nets, Int nete); -void iop_domain_relaxation_c(Int nelemd, Int np, Int nlev, element_t* elem, hvcoord_t hvcoord, hybrid_t hybrid, Int t1, Real* dp, Int nelemd_todo, Int np_todo, Real dt); -void crm_resolved_turb_c(Int nelemd, element_t* elem, hvcoord_t hvcoord, hybrid_t hybrid, Int t1, Int nelemd_todo, Int np_todo); -void iop_default_opts_c(Real* scmlat_out, Real* scmlon_out, char** iopfile_out, bool* single_column_out, bool* scm_iop_srf_prop_out, bool* iop_nudge_tq_out, bool* iop_nudge_uv_out, Real* iop_nudge_tq_low_out, Real* iop_nudge_tq_high_out, Real* iop_nudge_tscale_out, bool* scm_observed_aero_out, bool* iop_dosubsidence_out, bool* scm_multcols_out, bool* dp_crm_out, Real* iop_perturb_high_out, bool* precip_off_out, bool* scm_zero_non_iop_tracers_out); -void iop_setopts_c(Real scmlat_in, Real scmlon_in, const char** iopfile_in, bool single_column_in, bool scm_iop_srf_prop_in, bool iop_nudge_tq_in, bool iop_nudge_uv_in, Real iop_nudge_tq_low_in, Real iop_nudge_tq_high_in, Real iop_nudge_tscale_in, bool scm_observed_aero_in, bool iop_dosubsidence_in, bool scm_multcols_in, bool dp_crm_in, Real iop_perturb_high_in, bool precip_off_in, bool scm_zero_non_iop_tracers_in); -void setiopupdate_init_c(); -void setiopupdate_c(); -void readiopdata_c(Int plev, bool iop_update_phase1, Real* hyam, Real* hybm); -void iop_intht_c(); -} // extern "C" : end _c decls - -namespace scream { -namespace dp { - -// -// Glue functions to call fortran from from C++ with the Data struct -// - -void advance_iop_forcing(AdvanceIopForcingData& d) -{ - dp_init(); - d.transpose(); - advance_iop_forcing_c(d.plev, d.pcnst, d.scm_dt, d.ps_in, d.u_in, d.v_in, d.t_in, d.q_in, d.t_phys_frc, d.u_update, d.v_update, d.t_update, d.q_update); - d.transpose(); -} - - -void advance_iop_nudging(AdvanceIopNudgingData& d) -{ - dp_init(); - advance_iop_nudging_c(d.plev, d.scm_dt, d.ps_in, d.t_in, d.q_in, d.t_update, d.q_update, d.relaxt, d.relaxq); -} - -void advance_iop_subsidence(AdvanceIopSubsidenceData& d) -{ - dp_init(); - d.transpose(); - advance_iop_subsidence_c(d.plev, d.pcnst, d.scm_dt, d.ps_in, d.u_in, d.v_in, d.t_in, d.q_in, d.u_update, d.v_update, d.t_update, d.q_update); - d.transpose(); -} - -void iop_setinitial(IopSetinitialData& d) -{ - dp_init(); - //iop_setinitial_c(d.nelemd, d.elem); -} - -void iop_broadcast(IopBroadcastData& d) -{ - dp_init(); - iop_broadcast_c(); -} - -void apply_iop_forcing(ApplyIopForcingData& d) -{ - dp_init(); - apply_iop_forcing_c(d.nelemd, d.elem, &d.hvcoord, &d.hybrid, &d.tl, d.n, d.t_before_advance, d.nets, d.nete); -} - -void iop_domain_relaxation(IopDomainRelaxationData& d) -{ - dp_init(); - d.transpose(); - iop_domain_relaxation_c(d.nelemd, d.np, d.nlev, d.elem, d.hvcoord, d.hybrid, d.t1, d.dp, d.nelemd_todo, d.np_todo, d.dt); - d.transpose(); -} - -void crm_resolved_turb(CrmResolvedTurbData& d) -{ - dp_init(); - crm_resolved_turb_c(d.nelemd, d.elem, d.hvcoord, d.hybrid, d.t1, d.nelemd_todo, d.np_todo); -} - -void iop_default_opts(IopDefaultOptsData& d) -{ - dp_init(); - char cbuff[512] = ""; - char* buffptr = cbuff; - iop_default_opts_c(&d.scmlat_out, &d.scmlon_out, &buffptr, &d.single_column_out, &d.scm_iop_srf_prop_out, &d.iop_nudge_tq_out, &d.iop_nudge_uv_out, &d.iop_nudge_tq_low_out, &d.iop_nudge_tq_high_out, &d.iop_nudge_tscale_out, &d.scm_observed_aero_out, &d.iop_dosubsidence_out, &d.scm_multcols_out, &d.dp_crm_out, &d.iop_perturb_high_out, &d.precip_off_out, &d.scm_zero_non_iop_tracers_out); - d.iopfile_out = std::string(buffptr); -} - -void iop_setopts(IopSetoptsData& d) -{ - dp_init(); - const char* cptr = d.iopfile_in.c_str(); - iop_setopts_c(d.scmlat_in, d.scmlon_in, &cptr, d.single_column_in, d.scm_iop_srf_prop_in, d.iop_nudge_tq_in, d.iop_nudge_uv_in, d.iop_nudge_tq_low_in, d.iop_nudge_tq_high_in, d.iop_nudge_tscale_in, d.scm_observed_aero_in, d.iop_dosubsidence_in, d.scm_multcols_in, d.dp_crm_in, d.iop_perturb_high_in, d.precip_off_in, d.scm_zero_non_iop_tracers_in); -} - -void setiopupdate_init(SetiopupdateInitData& d) -{ - dp_init(); - setiopupdate_init_c(); -} - -void setiopupdate(SetiopupdateData& d) -{ - dp_init(); - setiopupdate_c(); -} - -void readiopdata(ReadiopdataData& d) -{ - dp_init(); - readiopdata_c(d.plev, d.iop_update_phase1, d.hyam, d.hybm); -} - -void iop_intht(IopInthtData& d) -{ - dp_init(); - iop_intht_c(); -} - -// end _c impls - -// -// _f function definitions. These expect data in C layout -// - -void advance_iop_forcing_f(Int plev, Int pcnst, Real scm_dt, Real ps_in, bool have_u, bool have_v, bool dp_crm, bool use_3dfrc, Real* u_in, Real* v_in, Real* t_in, Real* q_in, Real* t_phys_frc, Real* divt3d, Real* divq3d, Real* divt, Real* divq, Real* wfld, Real* uobs, Real* vobs, Real* hyai, Real* hyam, Real* hybi, Real* hybm, Real* u_update, Real* v_update, Real* t_update, Real* q_update) -{ - using DPF = Functions; - - using Spack = typename DPF::Spack; - using view_1d = typename DPF::view_1d; - using view_2d = typename DPF::view_2d; - using KT = typename DPF::KT; - using ExeSpace = typename KT::ExeSpace; - using MemberType = typename DPF::MemberType; - - // Some of the workspaces need plev+1 items - const Int plev_pack = ekat::npack(plev); - const Int plevp_pack = ekat::npack(plev+1); - - // Set up views - std::vector temp_d(AdvanceIopForcingData::NUM_ARRAYS-4); - std::vector temp_2d_d(4); - - ekat::host_to_device({u_in, v_in, t_in, t_phys_frc, divt3d, divt, wfld, uobs, vobs, hyai, hyam, hybi, hybm, u_update, v_update, t_update}, - plev, temp_d); - - ekat::host_to_device({ q_in, divq3d, divq, q_update }, - pcnst, plev, temp_2d_d, true); - - view_1d - u_in_d (temp_d[0]), - v_in_d (temp_d[1]), - t_in_d (temp_d[2]), - t_phys_frc_d (temp_d[3]), - divt3d_d (temp_d[4]), - divt_d (temp_d[5]), - wfld_d (temp_d[6]), - uobs_d (temp_d[7]), - vobs_d (temp_d[8]), - hyai_d (temp_d[9]), - hyam_d (temp_d[10]), - hybi_d (temp_d[11]), - hybm_d (temp_d[12]), - u_update_d (temp_d[13]), - v_update_d (temp_d[14]), - t_update_d (temp_d[15]); - - view_2d - q_in_d (temp_2d_d[0]), - divq3d_d (temp_2d_d[1]), - divq_d (temp_2d_d[2]), - q_update_d(temp_2d_d[3]); - - // Call core function from kernel - auto policy = ekat::ExeSpaceUtils::get_default_team_policy(1, plev_pack); - ekat::WorkspaceManager wsm(plevp_pack, 3, policy); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { - - DPF::advance_iop_forcing( - plev, pcnst, have_u, have_v, dp_crm, use_3dfrc, scm_dt, ps_in, - u_in_d, v_in_d, t_in_d, q_in_d, t_phys_frc_d, divt3d_d, divq3d_d, divt_d, divq_d, wfld_d, uobs_d, vobs_d, hyai_d, hyam_d, hybi_d, hybm_d, - team, wsm.get_workspace(team), - u_update_d, v_update_d, t_update_d, q_update_d); - }); - - // Sync back to host - std::vector inout_views = {t_update_d, u_update_d, v_update_d}; - std::vector inout_views_2d = {q_update_d}; - - ekat::device_to_host({t_update, u_update, v_update}, plev, inout_views); - ekat::device_to_host({q_update}, pcnst, plev, inout_views_2d, true); -} - -void advance_iop_nudging_f(Int plev, Real scm_dt, Real ps_in, Real* t_in, Real* q_in, Real* tobs, Real* qobs, - Real* hyai, Real* hyam, Real* hybi, Real* hybm, - Real* t_update, Real* q_update, Real* relaxt, Real* relaxq) -{ - using DPF = Functions; - - using Spack = typename DPF::Spack; - using view_1d = typename DPF::view_1d; - using KT = typename DPF::KT; - using ExeSpace = typename KT::ExeSpace; - using MemberType = typename DPF::MemberType; - - const Int plev_pack = ekat::npack(plev); - const Int plevp_pack = ekat::npack(plev+1); - - // Set up views - std::vector temp_d(AdvanceIopNudgingData::NUM_ARRAYS); - - ekat::host_to_device({t_in, q_in, tobs, qobs, hyai, hyam, hybi, hybm, t_update, q_update, relaxt, relaxq}, - plev, temp_d); - - int counter=0; - view_1d - t_in_d (temp_d[counter++]), - q_in_d (temp_d[counter++]), - tobs_d (temp_d[counter++]), - qobs_d (temp_d[counter++]), - hyai_d (temp_d[counter++]), - hyam_d (temp_d[counter++]), - hybi_d (temp_d[counter++]), - hybm_d (temp_d[counter++]), - t_update_d(temp_d[counter++]), - q_update_d(temp_d[counter++]), - relaxt_d (temp_d[counter++]), - relaxq_d (temp_d[counter++]); - - // Call core function from kernel - auto policy = ekat::ExeSpaceUtils::get_default_team_policy(1, plev_pack); - ekat::WorkspaceManager wsm(plevp_pack, 4, policy); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { - - DPF::advance_iop_nudging( - plev, scm_dt, ps_in, t_in_d, q_in_d, tobs_d, qobs_d, - hyai_d, hyam_d, hybi_d, hybm_d, - team, wsm.get_workspace(team), - t_update_d, q_update_d, relaxt_d, relaxq_d); - }); - - // Sync back to host - std::vector out_views = {t_update_d, q_update_d, relaxt_d, relaxq_d}; - - ekat::device_to_host({t_update, q_update, relaxt, relaxq}, plev, out_views); -} - -void advance_iop_subsidence_f(Int plev, Int pcnst, Real scm_dt, Real ps_in, Real* u_in, Real* v_in, Real* t_in, Real* q_in, Real* hyai, Real* hyam, Real* hybi, Real* hybm, Real* wfld, Real* u_update, Real* v_update, Real* t_update, Real* q_update) -{ - using DPF = Functions; - - using Spack = typename DPF::Spack; - using view_1d = typename DPF::view_1d; - using view_2d = typename DPF::view_2d; - using KT = typename DPF::KT; - using ExeSpace = typename KT::ExeSpace; - using MemberType = typename DPF::MemberType; - - // Some of the workspaces need plev+1 items - const Int plev_pack = ekat::npack(plev); - const Int plevp_pack = ekat::npack(plev+1); - - // Set up views - std::vector temp_d(AdvanceIopSubsidenceData::NUM_ARRAYS-2); - std::vector temp_2d_d(2); - - ekat::host_to_device({u_in, v_in, t_in, hyai, hyam, hybi, hybm, wfld, u_update, v_update, t_update}, - plev, temp_d); - - ekat::host_to_device({ q_in, q_update }, - pcnst, plev, temp_2d_d, true); - - view_1d - u_in_d (temp_d[0]), - v_in_d (temp_d[1]), - t_in_d (temp_d[2]), - hyai_d (temp_d[3]), - hyam_d (temp_d[4]), - hybi_d (temp_d[5]), - hybm_d (temp_d[6]), - wfld_d (temp_d[7]), - u_update_d (temp_d[8]), - v_update_d (temp_d[9]), - t_update_d (temp_d[10]); - - view_2d - q_in_d (temp_2d_d[0]), - q_update_d(temp_2d_d[1]); - - // Call core function from kernel - auto policy = ekat::ExeSpaceUtils::get_default_team_policy(1, plev_pack); - ekat::WorkspaceManager wsm(plevp_pack, 4, policy); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { - - DPF::advance_iop_subsidence( - plev, pcnst, scm_dt, ps_in, - u_in_d, v_in_d, t_in_d, q_in_d, hyai_d, hyam_d, hybi_d, hybm_d, wfld_d, - team, wsm.get_workspace(team), - u_update_d, v_update_d, t_update_d, q_update_d); - }); - - // Sync back to host - std::vector inout_views = {t_update_d, u_update_d, v_update_d}; - std::vector inout_views_2d = {q_update_d}; - - ekat::device_to_host({t_update, u_update, v_update}, plev, inout_views); - ekat::device_to_host({q_update}, pcnst, plev, inout_views_2d, true); -} - -void iop_setinitial_f(Int plev, Int pcnst, Int nelemd, Int np, Int nstep, Real psobs, bool use_replay, bool dynproc, bool have_t, bool have_q, bool have_ps, bool have_u, bool have_v, bool have_numliq, bool have_cldliq, bool have_numice, bool have_cldice, bool scm_zero_non_iop_tracers, bool is_first_restart_step, Real* qmin, Real* uobs, Real* vobs, Real* numliqobs, Real* numiceobs, Real* cldliqobs, Real* cldiceobs, Real* dx_short, tracer_t* tracers, element_t* elem, Real* dyn_dx_size, Real* tobs, Real* qobs) -{ - using DPF = Functions; - - using Spack = typename DPF::Spack; - using Scalarp = ekat::Pack; - using view_1d = typename DPF::view_1d; - using view_1ds = typename DPF::view_1d; - using sview_1ds = typename DPF::view_1d; - using KT = typename DPF::KT; - using ExeSpace = typename KT::ExeSpace; - using MemberType = typename DPF::MemberType; - - // Some of the workspaces need plev+1 items - const Int plev_pack = ekat::npack(plev); - - // Set up views - std::vector temp_d(8); - std::vector temp2_d(2); - - ekat::host_to_device({uobs, vobs, numliqobs, numiceobs, cldliqobs, cldiceobs, tobs, qobs}, - plev, temp_d); - - std::vector scalar_sizes = {nelemd, pcnst}; - ekat::host_to_device({dx_short, qmin}, scalar_sizes, temp2_d); - - view_1d - uobs_d (temp_d[0]), - vobs_d (temp_d[1]), - numliqobs_d (temp_d[2]), - numiceobs_d (temp_d[3]), - cldliqobs_d (temp_d[4]), - cldiceobs_d (temp_d[5]), - tobs_d (temp_d[6]), - qobs_d (temp_d[7]); - - sview_1ds - dx_short_d(reinterpret_cast(temp2_d[0].data()), scalar_sizes[0]), - qmin_d (reinterpret_cast(temp2_d[1].data()), scalar_sizes[1]); - - // Call core function - DPF::iop_setinitial(plev, pcnst, nelemd, np, nstep, use_replay, dynproc, have_t, have_ps, have_q, have_u, have_v, have_numliq, have_cldliq, have_numice, have_cldice, scm_zero_non_iop_tracers, is_first_restart_step, qmin_d, uobs_d, vobs_d, numliqobs_d, numiceobs_d, cldliqobs_d, cldiceobs_d, psobs, dx_short_d, *dyn_dx_size, *tracers, *elem, tobs_d, qobs_d); - - // Sync back to host - std::vector inout_views = {tobs_d, qobs_d}; - - ekat::device_to_host({tobs, qobs}, plev, inout_views); -} - -void iop_broadcast_f() -{ -#if 0 - using PF = Functions; - - using Spack = typename PF::Spack; - - Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { - PF::iop_broadcast(); - }); -#endif - -} -void apply_iop_forcing_f(Int nelemd, element_t* elem, hvcoord_t* hvcoord, hybrid_t hybrid, timelevel_t tl, Int n, bool t_before_advance, Int nets, Int nete) -{ - // TODO -} -void iop_domain_relaxation_f(Int nelemd, Int np, Int nlev, element_t* elem, hvcoord_t hvcoord, hybrid_t hybrid, Int t1, Real* dp, Int nelemd_todo, Int np_todo, Real dt) -{ - // TODO -} -void crm_resolved_turb_f(Int nelemd, element_t* elem, hvcoord_t hvcoord, hybrid_t hybrid, Int t1, Int nelemd_todo, Int np_todo) -{ - // TODO -} -void iop_default_opts_f(Real* scmlat_out, Real* scmlon_out, char** iopfile_out, bool* single_column_out, bool* scm_iop_srf_prop_out, bool* iop_nudge_tq_out, bool* iop_nudge_uv_out, Real* iop_nudge_tq_low_out, Real* iop_nudge_tq_high_out, Real* iop_nudge_tscale_out, bool* scm_observed_aero_out, bool* iop_dosubsidence_out, bool* scm_multcols_out, bool* dp_crm_out, Real* iop_perturb_high_out, bool* precip_off_out, bool* scm_zero_non_iop_tracers_out) -{ -#if 0 - using PF = Functions; - - using Spack = typename PF::Spack; - using view_1d = typename PF::view_1d; - using bview_1d = typename PF::view_1d; - - view_1d t_d("t_d", 6); - const auto t_h = Kokkos::create_mirror_view(t_d); - - bview_1d bt_d("bt_d", 10); - const auto bt_h = Kokkos::create_mirror_view(bt_d); - - Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { - Spack iop_nudge_tq_high_out_(), iop_nudge_tq_low_out_(), iop_nudge_tscale_out_(), iop_perturb_high_out_(), scmlat_out_(), scmlon_out_(); - bool dp_crm_out_(), iop_dosubsidence_out_(), iop_nudge_tq_out_(), iop_nudge_uv_out_(), precip_off_out_(), scm_iop_srf_prop_out_(), scm_multcols_out_(), scm_observed_aero_out_(), scm_zero_non_iop_tracers_out_(), single_column_out_(); - PF::iop_default_opts(scmlat_out_, scmlon_out_, iopfile_out_, single_column_out_, scm_iop_srf_prop_out_, iop_nudge_tq_out_, iop_nudge_uv_out_, iop_nudge_tq_low_out_, iop_nudge_tq_high_out_, iop_nudge_tscale_out_, scm_observed_aero_out_, iop_dosubsidence_out_, scm_multcols_out_, dp_crm_out_, iop_perturb_high_out_, precip_off_out_, scm_zero_non_iop_tracers_out_); - t_d(0) = iop_nudge_tq_high_out_[0]; - t_d(1) = iop_nudge_tq_low_out_[0]; - t_d(2) = iop_nudge_tscale_out_[0]; - t_d(3) = iop_perturb_high_out_[0]; - t_d(4) = scmlat_out_[0]; - t_d(5) = scmlon_out_[0]; - bt_d(0) = dp_crm_out_; - bt_d(1) = iop_dosubsidence_out_; - bt_d(2) = iop_nudge_tq_out_; - bt_d(3) = iop_nudge_uv_out_; - bt_d(4) = precip_off_out_; - bt_d(5) = scm_iop_srf_prop_out_; - bt_d(6) = scm_multcols_out_; - bt_d(7) = scm_observed_aero_out_; - bt_d(8) = scm_zero_non_iop_tracers_out_; - bt_d(9) = single_column_out_; - }); - Kokkos::deep_copy(t_h, t_d); - Kokkos::deep_copy(bt_h, bt_d); - *iop_nudge_tq_high_out = t_h(0); - *iop_nudge_tq_low_out = t_h(1); - *iop_nudge_tscale_out = t_h(2); - *iop_perturb_high_out = t_h(3); - *scmlat_out = t_h(4); - *scmlon_out = t_h(5); - *dp_crm_out = bt_h(0); - *iop_dosubsidence_out = bt_h(1); - *iop_nudge_tq_out = bt_h(2); - *iop_nudge_uv_out = bt_h(3); - *precip_off_out = bt_h(4); - *scm_iop_srf_prop_out = bt_h(5); - *scm_multcols_out = bt_h(6); - *scm_observed_aero_out = bt_h(7); - *scm_zero_non_iop_tracers_out = bt_h(8); - *single_column_out = bt_h(9); -#endif - -} -void iop_setopts_f(Real scmlat_in, Real scmlon_in, char** iopfile_in, bool single_column_in, bool scm_iop_srf_prop_in, bool iop_nudge_tq_in, bool iop_nudge_uv_in, Real iop_nudge_tq_low_in, Real iop_nudge_tq_high_in, Real iop_nudge_tscale_in, bool scm_observed_aero_in, bool iop_dosubsidence_in, bool scm_multcols_in, bool dp_crm_in, Real iop_perturb_high_in, bool precip_off_in, bool scm_zero_non_iop_tracers_in) -{ -#if 0 - using PF = Functions; - - using Spack = typename PF::Spack; - - Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { - Spack iop_nudge_tq_high_in_(iop_nudge_tq_high_in), iop_nudge_tq_low_in_(iop_nudge_tq_low_in), iop_nudge_tscale_in_(iop_nudge_tscale_in), iop_perturb_high_in_(iop_perturb_high_in), scmlat_in_(scmlat_in), scmlon_in_(scmlon_in); - PF::iop_setopts(scmlat_in_, scmlon_in_, iopfile_in, single_column_in, scm_iop_srf_prop_in, iop_nudge_tq_in, iop_nudge_uv_in, iop_nudge_tq_low_in_, iop_nudge_tq_high_in_, iop_nudge_tscale_in_, scm_observed_aero_in, iop_dosubsidence_in, scm_multcols_in, dp_crm_in, iop_perturb_high_in_, precip_off_in, scm_zero_non_iop_tracers_in); - }); -#endif - -} -void setiopupdate_init_f() -{ -#if 0 - using PF = Functions; - - using Spack = typename PF::Spack; - - Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { - PF::setiopupdate_init(); - }); -#endif - -} -void setiopupdate_f() -{ -#if 0 - using PF = Functions; - - using Spack = typename PF::Spack; - - Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { - PF::setiopupdate(); - }); -#endif - -} -void readiopdata_f(Int plev, bool iop_update_phase1, Real* hyam, Real* hybm) -{ - // TODO -} -void iop_intht_f() -{ -#if 0 - using PF = Functions; - - using Spack = typename PF::Spack; - - Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { - PF::iop_intht(); - }); -#endif - -} -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/dp_functions_f90.hpp b/components/eamxx/src/doubly-periodic/dp_functions_f90.hpp deleted file mode 100644 index 055bfe4ad683..000000000000 --- a/components/eamxx/src/doubly-periodic/dp_functions_f90.hpp +++ /dev/null @@ -1,306 +0,0 @@ -#ifndef SCREAM_DP_FUNCTIONS_F90_HPP -#define SCREAM_DP_FUNCTIONS_F90_HPP - -#include "share/scream_types.hpp" -#include "physics/share/physics_test_data.hpp" - -#include "dp_functions.hpp" -#include "physics_constants.hpp" - -#include -#include -#include - -// -// Bridge functions to call fortran version of dp functions from C++ -// - -namespace scream { -namespace dp { - -struct AdvanceIopForcingData : public PhysicsTestData { - static constexpr size_t NUM_ARRAYS = 20; - - // Inputs - Int plev, pcnst; - Real scm_dt, ps_in; - bool have_u, have_v, dp_crm, use_3dfrc; - Real *u_in, *v_in, *t_in, *q_in, *t_phys_frc, *divt3d, *divq3d, *divt, - *divq, *wfld, *uobs, *vobs, *hyai, *hyam, *hybi, *hybm; - - // Outputs - Real *u_update, *v_update, *t_update, *q_update; - - AdvanceIopForcingData(Int plev_, Int pcnst_, Real scm_dt_, Real ps_in_, bool have_u_, bool have_v_, bool dp_crm_, bool use_3dfrc_) : - PhysicsTestData( - {{ plev_ }, { pcnst_, plev_ }}, - { - { &u_in, &v_in, &t_in, &t_phys_frc, &divt3d, &divt, &divq, &wfld, &uobs, &vobs, &hyai, &hyam, &hybi, &hybm, &u_update, &v_update, &t_update }, - { &q_in, &divq3d, &divq, &q_update } - }), - plev(plev_), pcnst(pcnst_), scm_dt(scm_dt_), ps_in(ps_in_), have_u(have_u_), have_v(have_v_), dp_crm(dp_crm_), use_3dfrc(use_3dfrc_) {} - - PTD_STD_DEF(AdvanceIopForcingData, 8, plev, pcnst, scm_dt, ps_in, have_u, have_v, dp_crm, use_3dfrc); -}; - - -struct AdvanceIopNudgingData : public PhysicsTestData { - static constexpr size_t NUM_ARRAYS = 12; - - // Inputs - Int plev; - Real scm_dt, ps_in; - Real *t_in, *q_in, *tobs, *qobs, *hyai, *hyam, *hybi, *hybm; - - // Outputs - Real *t_update, *q_update, *relaxt, *relaxq; - - AdvanceIopNudgingData(Int plev_, Real scm_dt_, Real ps_in_) : - PhysicsTestData({{ plev_ }}, {{ &t_in, &q_in, &tobs, &qobs, &hyai, &hyam, &hybi, &hybm, &t_update, &q_update, &relaxt, &relaxq }}), - plev(plev_), scm_dt(scm_dt_), ps_in(ps_in_) {} - - PTD_STD_DEF(AdvanceIopNudgingData, 3, plev, scm_dt, ps_in); -}; - -struct AdvanceIopSubsidenceData : public PhysicsTestData { - static constexpr size_t NUM_ARRAYS = 13; - - // Inputs - Int plev, pcnst; - Real scm_dt, ps_in; - Real *u_in, *v_in, *t_in, *q_in, *hyai, *hyam, *hybi, *hybm, *wfld; - - // Outputs - Real *u_update, *v_update, *t_update, *q_update; - - AdvanceIopSubsidenceData(Int plev_, Int pcnst_, Real scm_dt_, Real ps_in_) : - PhysicsTestData( - {{ plev_ }, { plev_, pcnst_ }}, - { - { &u_in, &v_in, &t_in, &hyai, &hyam, &hybi, &hybm, &wfld, &u_update, &v_update, &t_update }, - { &q_in, &q_update } - }), - plev(plev_), pcnst(pcnst_), scm_dt(scm_dt_), ps_in(ps_in_) {} - - PTD_STD_DEF(AdvanceIopSubsidenceData, 4, plev, pcnst, scm_dt, ps_in); -}; - -struct IopSetinitialData : public PhysicsTestData { - // Inputs - Int plev, pcnst, nelemd, np, nstep; - - bool use_replay, dynproc, have_t, have_q, have_ps, have_u, have_v, have_numliq, have_cldliq, have_numice, have_cldice, scm_zero_non_iop_tracers, is_first_restart_step; - - Real psobs; - - Real* qmin, *uobs, *vobs, *numliqobs, *numiceobs, *cldliqobs, *cldiceobs, *dx_short; - - // Inputs/Outputs - tracer_t tracers; - element_t elem; - - Real dyn_dx_size; - - Real* tobs, *qobs; - - IopSetinitialData( - Int plev_, Int pcnst_, Int nelemd_, Int np_, Int nstep_, Real psobs_, - bool use_replay_, bool dynproc_, bool have_t_, bool have_q_, bool have_ps_, bool have_u_, bool have_v_, bool have_numliq_, bool have_cldliq_, bool have_numice_, bool have_cldice_, bool scm_zero_non_iop_tracers_, bool is_first_restart_step_) : - PhysicsTestData( - {{nelemd_}, {pcnst_}, {plev_}}, - { - {&dx_short}, - {&qmin}, - {&uobs, &vobs, &numliqobs, &numiceobs, &cldliqobs, &cldiceobs, &tobs, &qobs} - }), - plev(plev_), pcnst(pcnst_), nelemd(nelemd_), np(np_), nstep(nstep_), psobs(psobs_), - use_replay(use_replay_), dynproc(dynproc_), have_t(have_t_), have_q(have_q_), have_ps(have_ps_), have_u(have_u_), have_v(have_v_), have_numliq(have_numliq_), have_cldliq(have_cldliq_), have_numice(have_numice_), have_cldice(have_cldice_), scm_zero_non_iop_tracers(scm_zero_non_iop_tracers_), is_first_restart_step(is_first_restart_step_) { } - - PTD_STD_DEF(IopSetinitialData, 19, plev, pcnst, nelemd, np, nstep, psobs, use_replay, dynproc, have_t, have_q, have_ps, have_u, have_v, have_numliq, have_cldliq, have_numice, have_cldice, scm_zero_non_iop_tracers, is_first_restart_step); - - void init() - { - tracers.init(nelemd, QSIZE_D); - elem.init(nelemd, true, true, 2); - } - - void randomize(std::mt19937_64& engine) - { - PhysicsTestData::randomize(engine); - init(); - - tracers.randomize(engine(), 0, 1); - elem.randomize(engine()); - - // Where tobs starts being non-zero is important to the algorithm - std::uniform_int_distribution default_int_dist(0, plev); - Int start_nz = default_int_dist(engine); - - for (Int k = 0; k < start_nz; ++k) { - tobs[k] = 0; - } - } -}; - -struct IopBroadcastData : public PhysicsTestData { - // Inputs - Int plev; - - IopBroadcastData(Int plev_=0) : - PhysicsTestData({}, {}), plev(plev_) {} - - PTD_STD_DEF(IopBroadcastData, 1, plev); -}; - -struct ApplyIopForcingData : public PhysicsTestData { - // Inputs - Int plev, nelemd, n, nets, nete; - hybrid_t hybrid; - timelevel_t tl; - bool t_before_advance; - - // Inputs/Outputs - element_t *elem; - hvcoord_t hvcoord; - - ApplyIopForcingData(Int plev_, Int nelemd_, Int n_, Int nets_, Int nete_, bool t_before_advance_) : - PhysicsTestData({}, {}), plev(plev_), nelemd(nelemd_), n(n_), nets(nets_), nete(nete_), t_before_advance(t_before_advance_) {} - - PTD_STD_DEF(ApplyIopForcingData, 6, plev, nelemd, n, nets, nete, t_before_advance); -}; - -struct IopDomainRelaxationData : public PhysicsTestData { - // Inputs - Int nelemd, np, nlev, t1, nelemd_todo, np_todo; - hvcoord_t hvcoord; - hybrid_t hybrid; - Real dt; - - // Inputs/Outputs - element_t *elem; - Real *dp; - - IopDomainRelaxationData(Int nelemd_, Int np_, Int nlev_, Int t1_, Int nelemd_todo_, Int np_todo_, Real dt_) : - PhysicsTestData({{ np_, np_, nlev_ }}, {{ &dp }}), nelemd(nelemd_), np(np_), nlev(nlev_), t1(t1_), nelemd_todo(nelemd_todo_), np_todo(np_todo_), dt(dt_) {} - - PTD_STD_DEF(IopDomainRelaxationData, 7, nelemd, np, nlev, t1, nelemd_todo, np_todo, dt); -}; - -struct CrmResolvedTurbData : public PhysicsTestData { - // Inputs - Int plev, nelemd, t1, nelemd_todo, np_todo; - hvcoord_t hvcoord; - hybrid_t hybrid; - - // Inputs/Outputs - element_t *elem; - - CrmResolvedTurbData(Int plev_, Int nelemd_, Int t1_, Int nelemd_todo_, Int np_todo_) : - PhysicsTestData({}, {}), plev(plev_), nelemd(nelemd_), t1(t1_), nelemd_todo(nelemd_todo_), np_todo(np_todo_) {} - - PTD_STD_DEF(CrmResolvedTurbData, 5, plev, nelemd, t1, nelemd_todo, np_todo); -}; - -struct IopDefaultOptsData { - // Inputs - Int plev; - - // Outputs - Real scmlat_out, scmlon_out, iop_nudge_tq_low_out, iop_nudge_tq_high_out, iop_nudge_tscale_out, iop_perturb_high_out; - std::string iopfile_out; - bool single_column_out, scm_iop_srf_prop_out, iop_nudge_tq_out, iop_nudge_uv_out, scm_observed_aero_out, iop_dosubsidence_out, scm_multcols_out, dp_crm_out, precip_off_out, scm_zero_non_iop_tracers_out; - - void randomize(std::mt19937_64& engine) {} - - IopDefaultOptsData() = default; -}; - -struct IopSetoptsData { - // Inputs - Int plev; - Real scmlat_in, scmlon_in, iop_nudge_tq_low_in, iop_nudge_tq_high_in, iop_nudge_tscale_in, iop_perturb_high_in; - std::string iopfile_in; - bool single_column_in, scm_iop_srf_prop_in, iop_nudge_tq_in, iop_nudge_uv_in, scm_observed_aero_in, iop_dosubsidence_in, scm_multcols_in, dp_crm_in, precip_off_in, scm_zero_non_iop_tracers_in; - - void randomize(std::mt19937_64& engine) {} - - IopSetoptsData() = default; -}; - -struct SetiopupdateInitData { - // Inputs - Int plev; - - void randomize(std::mt19937_64& engine) {} -}; - -struct SetiopupdateData { - // Inputs - Int plev; - - void randomize(std::mt19937_64& engine) {} -}; - -struct ReadiopdataData : public PhysicsTestData { - // Inputs - Int plev; - bool iop_update_phase1; - Real *hyam, *hybm; - - ReadiopdataData(Int plev_, bool iop_update_phase1_) : - PhysicsTestData({{ plev_ }}, {{ &hyam, &hybm }}), plev(plev_), iop_update_phase1(iop_update_phase1_) {} - - PTD_STD_DEF(ReadiopdataData, 2, plev, iop_update_phase1); -}; - -struct IopInthtData { - // Inputs - Int plev; - - void randomize(std::mt19937_64& engine) {} -}; - -// Glue functions to call fortran from from C++ with the Data struct - -void advance_iop_forcing(AdvanceIopForcingData& d); -void advance_iop_nudging(AdvanceIopNudgingData& d); -void advance_iop_subsidence(AdvanceIopSubsidenceData& d); -void iop_setinitial(IopSetinitialData& d); -void iop_broadcast(IopBroadcastData& d); -void apply_iop_forcing(ApplyIopForcingData& d); -void iop_domain_relaxation(IopDomainRelaxationData& d); -void crm_resolved_turb(CrmResolvedTurbData& d); -void iop_default_opts(IopDefaultOptsData& d); -void iop_setopts(IopSetoptsData& d); -void setiopupdate_init(SetiopupdateInitData& d); -void setiopupdate(SetiopupdateData& d); -void readiopdata(ReadiopdataData& d); -void iop_intht(IopInthtData& d); -extern "C" { // _f function decls - -void advance_iop_forcing_f(Int plev, Int pcnst, Real scm_dt, Real ps_in, bool have_u, bool have_v, bool dp_crm, bool use_3dfrc, Real* u_in, Real* v_in, Real* t_in, Real* q_in, Real* t_phys_frc, Real* divt3d, Real* divq3d, Real* divt, Real* divq, Real* wfld, Real* uobs, Real* vobs, Real* hyai, Real* hyam, Real* hybi, Real* hybm, Real* u_update, Real* v_update, Real* t_update, Real* q_update); - -void advance_iop_nudging_f(Int plev, Real scm_dt, Real ps_in, Real* t_in, Real* q_in, Real* tobs, Real* qobs, - Real* hyai, Real* hyam, Real* hybi, Real* hybm, - Real* t_update, Real* q_update, Real* relaxt, Real* relaxq); - -void advance_iop_subsidence_f(Int plev, Int pcnst, Real scm_dt, Real ps_in, Real* u_in, Real* v_in, Real* t_in, Real* q_in, Real* hyai, Real* hyam, Real* hybi, Real* hybm, Real* wfld, Real* u_update, Real* v_update, Real* t_update, Real* q_update); - -void iop_setinitial_f(Int plev, Int pcnst, Int nelemd, Int np, Int nstep, Real psobs, bool use_replay, bool dynproc, bool have_t, bool have_q, bool have_ps, bool have_u, bool have_v, bool have_numliq, bool have_cldliq, bool have_numice, bool have_cldice, bool scm_zero_non_iop_tracers, bool is_first_restart_step, Real* qmin, Real* uobs, Real* vobs, Real* numliqobs, Real* numiceobs, Real* cldliqobs, Real* cldiceobs, Real* dx_short, tracer_t* tracers, element_t* elem, Real* dyn_dx_size, Real* tobs, Real* qobs); - -void iop_broadcast_f(); -void apply_iop_forcing_f(Int nelemd, element_t* elem, hvcoord_t* hvcoord, hybrid_t hybrid, timelevel_t tl, Int n, bool t_before_advance, Int nets, Int nete); -void iop_domain_relaxation_f(Int nelemd, Int np, Int nlev, element_t* elem, hvcoord_t hvcoord, hybrid_t hybrid, Int t1, Real* dp, Int nelemd_todo, Int np_todo, Real dt); -void crm_resolved_turb_f(Int nelemd, element_t* elem, hvcoord_t hvcoord, hybrid_t hybrid, Int t1, Int nelemd_todo, Int np_todo); -void iop_default_opts_f(Real* scmlat_out, Real* scmlon_out, char** iopfile_out, bool* single_column_out, bool* scm_iop_srf_prop_out, bool* iop_nudge_tq_out, bool* iop_nudge_uv_out, Real* iop_nudge_tq_low_out, Real* iop_nudge_tq_high_out, Real* iop_nudge_tscale_out, bool* scm_observed_aero_out, bool* iop_dosubsidence_out, bool* scm_multcols_out, bool* dp_crm_out, Real* iop_perturb_high_out, bool* precip_off_out, bool* scm_zero_non_iop_tracers_out); -void iop_setopts_f(Real scmlat_in, Real scmlon_in, const char** iopfile_in, bool single_column_in, bool scm_iop_srf_prop_in, bool iop_nudge_tq_in, bool iop_nudge_uv_in, Real iop_nudge_tq_low_in, Real iop_nudge_tq_high_in, Real iop_nudge_tscale_in, bool scm_observed_aero_in, bool iop_dosubsidence_in, bool scm_multcols_in, bool dp_crm_in, Real iop_perturb_high_in, bool precip_off_in, bool scm_zero_non_iop_tracers_in); -void setiopupdate_init_f(); -void setiopupdate_f(); -void readiopdata_f(Int plev, bool iop_update_phase1, Real* hyam, Real* hybm); -void iop_intht_f(); -} // end _f function decls - -} // namespace dp -} // namespace scream - -#endif // SCREAM_DP_FUNCTIONS_F90_HPP diff --git a/components/eamxx/src/doubly-periodic/dp_iso_c.f90 b/components/eamxx/src/doubly-periodic/dp_iso_c.f90 deleted file mode 100644 index 29d2fe55d4b2..000000000000 --- a/components/eamxx/src/doubly-periodic/dp_iso_c.f90 +++ /dev/null @@ -1,156 +0,0 @@ -module dp_iso_c - use iso_c_binding - implicit none - -#include "scream_config.f" -#ifdef SCREAM_DOUBLE_PRECISION -# define c_real c_double -#else -# define c_real c_float -#endif - -! -! This file contains bridges from scream c++ to DP fortran. -! - -contains - subroutine advance_iop_forcing_c(plev, pcnst, scm_dt, ps_in, u_in, v_in, t_in, q_in, t_phys_frc, u_update, v_update, t_update, q_update) bind(C) - !use dp, only : advance_iop_forcing - - integer(kind=c_int) , value, intent(in) :: plev, pcnst - real(kind=c_real) , value, intent(in) :: scm_dt, ps_in - real(kind=c_real) , intent(in), dimension(plev) :: u_in, v_in, t_in, t_phys_frc - real(kind=c_real) , intent(in), dimension(plev, pcnst) :: q_in - real(kind=c_real) , intent(out), dimension(plev) :: u_update, v_update, t_update - real(kind=c_real) , intent(out), dimension(plev, pcnst) :: q_update - - !call advance_iop_forcing(plev, pcnst, scm_dt, ps_in, u_in, v_in, t_in, q_in, t_phys_frc, u_update, v_update, t_update, q_update) - end subroutine advance_iop_forcing_c - subroutine advance_iop_nudging_c(plev, scm_dt, ps_in, t_in, q_in, t_update, q_update, relaxt, relaxq) bind(C) - !use dp, only : advance_iop_nudging - - integer(kind=c_int) , value, intent(in) :: plev - real(kind=c_real) , value, intent(in) :: scm_dt, ps_in - real(kind=c_real) , intent(in), dimension(plev) :: t_in, q_in - real(kind=c_real) , intent(out), dimension(plev) :: t_update, q_update, relaxt, relaxq - - !call advance_iop_nudging(plev, scm_dt, ps_in, t_in, q_in, t_update, q_update, relaxt, relaxq) - end subroutine advance_iop_nudging_c - subroutine advance_iop_subsidence_c(plev, pcnst, scm_dt, ps_in, u_in, v_in, t_in, q_in, u_update, v_update, t_update, q_update) bind(C) - !use dp, only : advance_iop_subsidence - - integer(kind=c_int) , value, intent(in) :: plev, pcnst - real(kind=c_real) , value, intent(in) :: scm_dt, ps_in - real(kind=c_real) , intent(in), dimension(plev) :: u_in, v_in, t_in - real(kind=c_real) , intent(in), dimension(plev, pcnst) :: q_in - real(kind=c_real) , intent(out), dimension(plev) :: u_update, v_update, t_update - real(kind=c_real) , intent(out), dimension(plev, pcnst) :: q_update - - !call advance_iop_subsidence(plev, pcnst, scm_dt, ps_in, u_in, v_in, t_in, q_in, u_update, v_update, t_update, q_update) - end subroutine advance_iop_subsidence_c - subroutine iop_setinitial_c(nelemd, elem) bind(C) - !use dp, only : iop_setinitial - - integer(kind=c_int) , value, intent(in) :: nelemd - type(c_ptr) , intent(inout), dimension(nelemd) :: elem - - !call iop_setinitial(nelemd, elem) - end subroutine iop_setinitial_c - subroutine iop_broadcast_c() bind(C) - !use dp, only : iop_broadcast - - !call iop_broadcast() - end subroutine iop_broadcast_c - subroutine apply_iop_forcing_c(nelemd, elem, hvcoord, hybrid, tl, n, t_before_advance, nets, nete) bind(C) - !use dp, only : apply_iop_forcing - - integer(kind=c_int) , value, intent(in) :: nelemd, n, nets, nete - type(c_ptr) , intent(inout), dimension(nelemd) :: elem - type(c_ptr) , intent(inout) :: hvcoord - type(c_ptr) , intent(in) :: hybrid - type(c_ptr) , intent(in) :: tl - logical(kind=c_bool) , value, intent(in) :: t_before_advance - - !call apply_iop_forcing(nelemd, elem, hvcoord, hybrid, tl, n, t_before_advance, nets, nete) - end subroutine apply_iop_forcing_c - subroutine iop_domain_relaxation_c(nelemd, np, nlev, elem, hvcoord, hybrid, t1, dp, nelemd_todo, np_todo, dt) bind(C) - !use dp, only : iop_domain_relaxation - - integer(kind=c_int) , value, intent(in) :: nelemd, np, nlev, t1, nelemd_todo, np_todo - type(c_ptr) , intent(inout), dimension(nelemd) :: elem - type(c_ptr) , intent(in) :: hvcoord - type(c_ptr) , intent(in) :: hybrid - real(kind=c_real) , intent(inout), dimension(np, np, nlev) :: dp - real(kind=c_real) , value, intent(in) :: dt - - !call iop_domain_relaxation(nelemd, np, nlev, elem, hvcoord, hybrid, t1, dp, nelemd_todo, np_todo, dt) - end subroutine iop_domain_relaxation_c - subroutine crm_resolved_turb_c(nelemd, elem, hvcoord, hybrid, t1, nelemd_todo, np_todo) bind(C) - !use dp, only : crm_resolved_turb - - integer(kind=c_int) , value, intent(in) :: nelemd, t1, nelemd_todo, np_todo - type(c_ptr) , intent(inout), dimension(nelemd) :: elem - type(c_ptr) , intent(in) :: hvcoord - type(c_ptr) , intent(in) :: hybrid - - !call crm_resolved_turb(nelemd, elem, hvcoord, hybrid, t1, nelemd_todo, np_todo) - end subroutine crm_resolved_turb_c - subroutine iop_default_opts_c(scmlat_out, scmlon_out, iopfile_out, & - single_column_out, scm_iop_srf_prop_out, iop_nudge_tq_out, & - iop_nudge_uv_out, iop_nudge_tq_low_out, iop_nudge_tq_high_out, & - iop_nudge_tscale_out, scm_observed_aero_out, iop_dosubsidence_out, & - scm_multcols_out, dp_crm_out, iop_perturb_high_out, precip_off_out, & - scm_zero_non_iop_tracers_out) bind(C) - !use dp, only : iop_default_opts - - real(kind=c_real) , intent(out) :: scmlat_out, scmlon_out, iop_nudge_tq_low_out, iop_nudge_tq_high_out, iop_nudge_tscale_out, iop_perturb_high_out - type(c_ptr) , intent(out) :: iopfile_out - logical(kind=c_bool) , intent(out) :: single_column_out, & - scm_iop_srf_prop_out, iop_nudge_tq_out, iop_nudge_uv_out, & - scm_observed_aero_out, iop_dosubsidence_out, scm_multcols_out, & - dp_crm_out, precip_off_out, scm_zero_non_iop_tracers_out - - !call iop_default_opts(scmlat_out, scmlon_out, iopfile_out, single_column_out, scm_iop_srf_prop_out, iop_nudge_tq_out, iop_nudge_uv_out, iop_nudge_tq_low_out, iop_nudge_tq_high_out, iop_nudge_tscale_out, scm_observed_aero_out, iop_dosubsidence_out, scm_multcols_out, dp_crm_out, iop_perturb_high_out, precip_off_out, scm_zero_non_iop_tracers_out) - end subroutine iop_default_opts_c - subroutine iop_setopts_c(scmlat_in, scmlon_in, iopfile_in, & - single_column_in, scm_iop_srf_prop_in, iop_nudge_tq_in, & - iop_nudge_uv_in, iop_nudge_tq_low_in, iop_nudge_tq_high_in, & - iop_nudge_tscale_in, scm_observed_aero_in, iop_dosubsidence_in, & - scm_multcols_in, dp_crm_in, iop_perturb_high_in, precip_off_in, & - scm_zero_non_iop_tracers_in) bind(C) - !use dp, only : iop_setopts - - real(kind=c_real) , value, intent(in) :: scmlat_in, scmlon_in, iop_nudge_tq_low_in, iop_nudge_tq_high_in, iop_nudge_tscale_in, iop_perturb_high_in - type(c_ptr) , intent(in) :: iopfile_in - logical(kind=c_bool) , value, intent(in) :: single_column_in, & - scm_iop_srf_prop_in, iop_nudge_tq_in, iop_nudge_uv_in, & - scm_observed_aero_in, iop_dosubsidence_in, scm_multcols_in, & - dp_crm_in, precip_off_in, scm_zero_non_iop_tracers_in - - !call iop_setopts(scmlat_in, scmlon_in, iopfile_in, single_column_in, scm_iop_srf_prop_in, iop_nudge_tq_in, iop_nudge_uv_in, iop_nudge_tq_low_in, iop_nudge_tq_high_in, iop_nudge_tscale_in, scm_observed_aero_in, iop_dosubsidence_in, scm_multcols_in, dp_crm_in, iop_perturb_high_in, precip_off_in, scm_zero_non_iop_tracers_in) - end subroutine iop_setopts_c - subroutine setiopupdate_init_c() bind(C) - !use dp, only : setiopupdate_init - - ! call setiopupdate_init() - end subroutine setiopupdate_init_c - subroutine setiopupdate_c() bind(C) - !use dp, only : setiopupdate - - !call setiopupdate() - end subroutine setiopupdate_c - subroutine readiopdata_c(plev, iop_update_phase1, hyam, hybm) bind(C) - !use dp, only : readiopdata - - integer(kind=c_int) , value, intent(in) :: plev - logical(kind=c_bool) , value, intent(in) :: iop_update_phase1 - real(kind=c_real) , intent(in), dimension(plev) :: hyam, hybm - - !call readiopdata(plev, iop_update_phase1, hyam, hybm) - end subroutine readiopdata_c - subroutine iop_intht_c() bind(C) - !use dp, only : iop_intht - - !call iop_intht() - end subroutine iop_intht_c -end module dp_iso_c diff --git a/components/eamxx/src/doubly-periodic/dp_iso_f.f90 b/components/eamxx/src/doubly-periodic/dp_iso_f.f90 deleted file mode 100644 index 98c48961aa4a..000000000000 --- a/components/eamxx/src/doubly-periodic/dp_iso_f.f90 +++ /dev/null @@ -1,123 +0,0 @@ -module dp_iso_f - use iso_c_binding - implicit none - -#include "scream_config.f" -#ifdef SCREAM_DOUBLE_PRECISION -# define c_real c_double -#else -# define c_real c_float -#endif - -! -! This file contains bridges from DP fortran to scream c++. -! - -interface - - subroutine advance_iop_forcing_f(plev, pcnst, scm_dt, ps_in, u_in, v_in, t_in, q_in, t_phys_frc, u_update, v_update, t_update, q_update) bind(C) - use iso_c_binding - - integer(kind=c_int) , value, intent(in) :: plev, pcnst - real(kind=c_real) , value, intent(in) :: scm_dt, ps_in - real(kind=c_real) , intent(in), dimension(plev) :: u_in, v_in, t_in, t_phys_frc - real(kind=c_real) , intent(in), dimension(plev, pcnst) :: q_in - real(kind=c_real) , intent(out), dimension(plev) :: u_update, v_update, t_update - real(kind=c_real) , intent(out), dimension(plev, pcnst) :: q_update - end subroutine advance_iop_forcing_f - subroutine advance_iop_nudging_f(plev, scm_dt, ps_in, t_in, q_in, t_update, q_update, relaxt, relaxq) bind(C) - use iso_c_binding - - integer(kind=c_int) , value, intent(in) :: plev - real(kind=c_real) , value, intent(in) :: scm_dt, ps_in - real(kind=c_real) , intent(in), dimension(plev) :: t_in, q_in - real(kind=c_real) , intent(out), dimension(plev) :: t_update, q_update, relaxt, relaxq - end subroutine advance_iop_nudging_f - subroutine advance_iop_subsidence_f(plev, pcnst, scm_dt, ps_in, u_in, v_in, t_in, q_in, u_update, v_update, t_update, q_update) bind(C) - use iso_c_binding - - integer(kind=c_int) , value, intent(in) :: plev, pcnst - real(kind=c_real) , value, intent(in) :: scm_dt, ps_in - real(kind=c_real) , intent(in), dimension(plev) :: u_in, v_in, t_in - real(kind=c_real) , intent(in), dimension(plev, pcnst) :: q_in - real(kind=c_real) , intent(out), dimension(plev) :: u_update, v_update, t_update - real(kind=c_real) , intent(out), dimension(plev, pcnst) :: q_update - end subroutine advance_iop_subsidence_f - subroutine iop_setinitial_f(nelemd, elem) bind(C) - use iso_c_binding - - integer(kind=c_int) , value, intent(in) :: nelemd - type(c_ptr) , intent(inout), dimension(nelemd) :: elem - end subroutine iop_setinitial_f - subroutine iop_broadcast_f() bind(C) - use iso_c_binding - - - end subroutine iop_broadcast_f - subroutine apply_iop_forcing_f(nelemd, elem, hvcoord, hybrid, tl, n, t_before_advance, nets, nete) bind(C) - use iso_c_binding - - integer(kind=c_int) , value, intent(in) :: nelemd, n, nets, nete - type(c_ptr) , intent(inout), dimension(nelemd) :: elem - type(c_ptr) , intent(inout) :: hvcoord - type(c_ptr) , intent(in) :: hybrid - type(c_ptr) , intent(in) :: tl - logical(kind=c_bool) , value, intent(in) :: t_before_advance - end subroutine apply_iop_forcing_f - subroutine iop_domain_relaxation_f(nelemd, np, nlev, elem, hvcoord, hybrid, t1, dp, nelemd_todo, np_todo, dt) bind(C) - use iso_c_binding - - integer(kind=c_int) , value, intent(in) :: nelemd, np, nlev, t1, nelemd_todo, np_todo - type(c_ptr) , intent(inout), dimension(nelemd) :: elem - type(c_ptr) , intent(in) :: hvcoord - type(c_ptr) , intent(in) :: hybrid - real(kind=c_real) , intent(inout), dimension(np, np, nlev) :: dp - real(kind=c_real) , value, intent(in) :: dt - end subroutine iop_domain_relaxation_f - subroutine crm_resolved_turb_f(nelemd, elem, hvcoord, hybrid, t1, nelemd_todo, np_todo) bind(C) - use iso_c_binding - - integer(kind=c_int) , value, intent(in) :: nelemd, t1, nelemd_todo, np_todo - type(c_ptr) , intent(inout), dimension(nelemd) :: elem - type(c_ptr) , intent(in) :: hvcoord - type(c_ptr) , intent(in) :: hybrid - end subroutine crm_resolved_turb_f - subroutine iop_default_opts_f(scmlat_out, scmlon_out, iopfile_out, single_column_out, scm_iop_srf_prop_out, iop_nudge_tq_out, iop_nudge_uv_out, iop_nudge_tq_low_out, iop_nudge_tq_high_out, iop_nudge_tscale_out, scm_observed_aero_out, iop_dosubsidence_out, scm_multcols_out, dp_crm_out, iop_perturb_high_out, precip_off_out, scm_zero_non_iop_tracers_out) bind(C) - use iso_c_binding - - real(kind=c_real) , intent(out) :: scmlat_out, scmlon_out, iop_nudge_tq_low_out, iop_nudge_tq_high_out, iop_nudge_tscale_out, iop_perturb_high_out - type(c_ptr) , intent(out) :: iopfile_out - logical(kind=c_bool) , intent(out) :: single_column_out, scm_iop_srf_prop_out, iop_nudge_tq_out, iop_nudge_uv_out, scm_observed_aero_out, iop_dosubsidence_out, scm_multcols_out, dp_crm_out, precip_off_out, scm_zero_non_iop_tracers_out - end subroutine iop_default_opts_f - subroutine iop_setopts_f(scmlat_in, scmlon_in, iopfile_in, single_column_in, scm_iop_srf_prop_in, iop_nudge_tq_in, iop_nudge_uv_in, iop_nudge_tq_low_in, iop_nudge_tq_high_in, iop_nudge_tscale_in, scm_observed_aero_in, iop_dosubsidence_in, scm_multcols_in, dp_crm_in, iop_perturb_high_in, precip_off_in, scm_zero_non_iop_tracers_in) bind(C) - use iso_c_binding - - real(kind=c_real) , value, intent(in) :: scmlat_in, scmlon_in, iop_nudge_tq_low_in, iop_nudge_tq_high_in, iop_nudge_tscale_in, iop_perturb_high_in - type(c_ptr) , intent(in) :: iopfile_in - logical(kind=c_bool) , value, intent(in) :: single_column_in, scm_iop_srf_prop_in, iop_nudge_tq_in, iop_nudge_uv_in, scm_observed_aero_in, iop_dosubsidence_in, scm_multcols_in, dp_crm_in, precip_off_in, scm_zero_non_iop_tracers_in - end subroutine iop_setopts_f - subroutine setiopupdate_init_f() bind(C) - use iso_c_binding - - - end subroutine setiopupdate_init_f - subroutine setiopupdate_f() bind(C) - use iso_c_binding - - - end subroutine setiopupdate_f - subroutine readiopdata_f(plev, iop_update_phase1, hyam, hybm) bind(C) - use iso_c_binding - - integer(kind=c_int) , value, intent(in) :: plev - logical(kind=c_bool) , value, intent(in) :: iop_update_phase1 - real(kind=c_real) , intent(in), dimension(plev) :: hyam, hybm - end subroutine readiopdata_f - subroutine iop_intht_f() bind(C) - use iso_c_binding - - - end subroutine iop_intht_f -end interface - -end module dp_iso_f diff --git a/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_forcing.cpp b/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_forcing.cpp deleted file mode 100644 index a0f560796a4d..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_forcing.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_advance_iop_forcing_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing advance_iop_forcing on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_nudging.cpp b/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_nudging.cpp deleted file mode 100644 index 6e5fab7e6090..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_nudging.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_advance_iop_nudging_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing advance_iop_nudging on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_subsidence.cpp b/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_subsidence.cpp deleted file mode 100644 index 1fe6c79a160a..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_subsidence.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_advance_iop_subsidence_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing advance_iop_subsidence on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_apply_iop_forcing.cpp b/components/eamxx/src/doubly-periodic/eti/dp_apply_iop_forcing.cpp deleted file mode 100644 index cb138eb9b048..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_apply_iop_forcing.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_apply_iop_forcing_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing apply_iop_forcing on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_crm_resolved_turb.cpp b/components/eamxx/src/doubly-periodic/eti/dp_crm_resolved_turb.cpp deleted file mode 100644 index 6921dc81cc5a..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_crm_resolved_turb.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_crm_resolved_turb_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing crm_resolved_turb on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_iop_broadcast.cpp b/components/eamxx/src/doubly-periodic/eti/dp_iop_broadcast.cpp deleted file mode 100644 index 083d19b97afd..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_iop_broadcast.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_iop_broadcast_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing iop_broadcast on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_iop_default_opts.cpp b/components/eamxx/src/doubly-periodic/eti/dp_iop_default_opts.cpp deleted file mode 100644 index 30fae390febb..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_iop_default_opts.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_iop_default_opts_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing iop_default_opts on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_iop_domain_relaxation.cpp b/components/eamxx/src/doubly-periodic/eti/dp_iop_domain_relaxation.cpp deleted file mode 100644 index 2b8aa9a09add..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_iop_domain_relaxation.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_iop_domain_relaxation_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing iop_domain_relaxation on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_iop_intht.cpp b/components/eamxx/src/doubly-periodic/eti/dp_iop_intht.cpp deleted file mode 100644 index b68587342a5c..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_iop_intht.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_iop_intht_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing iop_intht on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_iop_setfield.cpp b/components/eamxx/src/doubly-periodic/eti/dp_iop_setfield.cpp deleted file mode 100644 index 8641cfa0704a..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_iop_setfield.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_iop_setfield_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing iop_setfield on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_iop_setinitial.cpp b/components/eamxx/src/doubly-periodic/eti/dp_iop_setinitial.cpp deleted file mode 100644 index 9195f4f6cf3b..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_iop_setinitial.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_iop_setinitial_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing iop_setinitial on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_iop_setopts.cpp b/components/eamxx/src/doubly-periodic/eti/dp_iop_setopts.cpp deleted file mode 100644 index e46aff90bdc6..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_iop_setopts.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_iop_setopts_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing iop_setopts on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_readiopdata.cpp b/components/eamxx/src/doubly-periodic/eti/dp_readiopdata.cpp deleted file mode 100644 index 4289e61b9848..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_readiopdata.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_readiopdata_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing readiopdata on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_setiopupdate.cpp b/components/eamxx/src/doubly-periodic/eti/dp_setiopupdate.cpp deleted file mode 100644 index 56e7084e4760..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_setiopupdate.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_setiopupdate_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing setiopupdate on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_setiopupdate_init.cpp b/components/eamxx/src/doubly-periodic/eti/dp_setiopupdate_init.cpp deleted file mode 100644 index 45f26f184665..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_setiopupdate_init.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_setiopupdate_init_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing setiopupdate_init on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_forcing_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_forcing_impl.hpp deleted file mode 100644 index 5220d14e290b..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_forcing_impl.hpp +++ /dev/null @@ -1,158 +0,0 @@ -#ifndef DP_ADVANCE_IOP_FORCING_IMPL_HPP -#define DP_ADVANCE_IOP_FORCING_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp advance_iop_forcing. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -KOKKOS_FUNCTION -void Functions::plevs0( - // Input arguments - const Int& nver, - const Scalar& ps, - const uview_1d& hyai, - const uview_1d& hyam, - const uview_1d& hybi, - const uview_1d& hybm, - // Kokkos stuff - const MemberType& team, - // Output arguments - const uview_1d& pint, - const uview_1d& pmid, - const uview_1d& pdel) -{ - const auto ps0 = C::P0; - const Int nver_pack = ekat::npack(nver); - - // Set interface pressures - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, nver_pack), [&] (Int k) { - pint(k) = hyai(k)*ps0 + hybi(k)*ps; - pmid(k) = hyam(k)*ps0 + hybm(k)*ps; - }); - - // Set midpoint pressures and layer thicknesses - const auto pint_s = scalarize(pint); - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, nver_pack), [&] (Int k) { - Spack spint, spint_1; - IntSmallPack range_pack1 = ekat::range(k*Spack::n); - auto range_pack2_p1_safe = range_pack1; - range_pack2_p1_safe.set(range_pack1 > nver-1, nver-1); - ekat::index_and_shift<1>(pint_s, range_pack2_p1_safe, spint, spint_1); - pdel(k) = spint_1 - spint; - }); -} - -template -KOKKOS_FUNCTION -void Functions::advance_iop_forcing( - // Input arguments - const Int& plev, - const Int& pcnst, - const bool& have_u, - const bool& have_v, - const bool& dp_crm, - const bool& use_3dfrc, - const Scalar& scm_dt, - const Scalar& ps_in, - const uview_1d& u_in, - const uview_1d& v_in, - const uview_1d& t_in, - const uview_2d& q_in, - const uview_1d& t_phys_frc, - const uview_1d& divt3d, - const uview_2d& divq3d, - const uview_1d& divt, - const uview_2d& divq, - const uview_1d& wfld, - const uview_1d& uobs, - const uview_1d& vobs, - const uview_1d& hyai, - const uview_1d& hyam, - const uview_1d& hybi, - const uview_1d& hybm, - // Kokkos stuff - const MemberType& team, - const Workspace& workspace, - // Output arguments - const uview_1d& u_update, - const uview_1d& v_update, - const uview_1d& t_update, - const uview_2d& q_update) -{ - // Local variables - uview_1d - pmidm1, // pressure at model levels - pintm1, // pressure at model interfaces (dim=plev+1) - pdelm1; // pdel(k) = pint (k+1)-pint (k) - workspace.template take_many_contiguous_unsafe<3>( - {"pmidm1", "pintm1", "pdelm1"}, - {&pmidm1, &pintm1, &pdelm1}); - - // Get vertical level profiles - plevs0(plev, ps_in, hyai, hyam, hybi, hybm, team, pintm1, pmidm1, pdelm1); - - //////////////////////////////////////////////////////////// - // Advance T and Q due to large scale forcing - - uview_1d t_lsf; // storage for temperature large scale forcing - uview_2d q_lsf; // storage for moisture large scale forcing - - if (use_3dfrc) { - t_lsf = divt3d; - q_lsf = divq3d; - } - else { - t_lsf = divt; - q_lsf = divq; - } - - const Int plev_pack = ekat::npack(plev); - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, plev_pack), [&] (Int k) { - // Initialize thermal expansion term to zero. This term is only - // considered if using the preq-x dycore and if three dimensional - // forcing is not provided by IOP forcing file. - Spack t_expan = 0; - t_update(k) = t_in(k) + t_expan + scm_dt*(t_phys_frc(k) + t_lsf(k)); - for (Int m = 0; m < pcnst; ++m) { - q_update(m, k) = q_in(m, k) + scm_dt*q_lsf(m, k); - } - }); - - //////////////////////////////////////////////////////////// - // Set U and V fields - - uview_1d u_src, v_src; - - if (have_v && have_u && !dp_crm) { - u_src = uobs; - v_src = vobs; - } - else { - u_src = u_in; - v_src = v_in; - } - - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, plev_pack), [&] (Int k) { - u_update(k) = u_src(k); - v_update(k) = v_src(k); - }); - - workspace.template release_many_contiguous<3>( - {&pmidm1, &pintm1, &pdelm1}); -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_nudging_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_nudging_impl.hpp deleted file mode 100644 index cf0f160209a3..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_nudging_impl.hpp +++ /dev/null @@ -1,76 +0,0 @@ -#ifndef DP_ADVANCE_IOP_NUDGING_IMPL_HPP -#define DP_ADVANCE_IOP_NUDGING_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp advance_iop_nudging. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -KOKKOS_FUNCTION -void Functions::advance_iop_nudging( - // Input arguments - const Int& plev, - const Scalar& scm_dt, - const Scalar& ps_in, - const uview_1d& t_in, - const uview_1d& q_in, - const uview_1d& tobs, - const uview_1d& qobs, - const uview_1d& hyai, - const uview_1d& hyam, - const uview_1d& hybi, - const uview_1d& hybm, - // Kokkos stuff - const MemberType& team, - const Workspace& workspace, - // Output arguments - const uview_1d& t_update, - const uview_1d& q_update, - const uview_1d& relaxt, - const uview_1d& relaxq) -{ - // Local variables - uview_1d - pmidm1, // pressure at model levels - pintm1, // pressure at model interfaces (dim=plev+1) - pdelm1, // pdel(k) = pint (k+1)-pint (k) - rtau; - workspace.template take_many_contiguous_unsafe<4>( - {"pmidm1", "pintm1", "pdelm1", "rtau"}, - {&pmidm1, &pintm1, &pdelm1, &rtau}); - - // Get vertical level profiles - plevs0(plev, ps_in, hyai, hyam, hybi, hybm, team, pintm1, pmidm1, pdelm1); - - const Int plev_pack = ekat::npack(plev); - constexpr Scalar iop_nudge_tq_low = DPC::iop_nudge_tq_low; - constexpr Scalar iop_nudge_tq_high = DPC::iop_nudge_tq_high; - constexpr Scalar iop_nudge_tscale = DPC::iop_nudge_tscale; - - - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, plev_pack), [&] (Int k) { - relaxt(k) = 0; - relaxq(k) = 0; - - const auto condition = pmidm1(k) <= iop_nudge_tq_low*100 - && - pmidm1(k) >= iop_nudge_tq_high*100; - rtau(k).set(condition, ekat::impl::max(scm_dt, iop_nudge_tscale)); - relaxt(k).set(condition, (t_in(k) - tobs(k))/rtau(k)); - relaxq(k).set(condition, (q_in(k) - qobs(k))/rtau(k)); - - t_update(k).set(condition, t_in(k) + relaxt(k)*scm_dt); - q_update(k).set(condition, q_in(k) + relaxq(k)*scm_dt); - }); -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_subsidence_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_subsidence_impl.hpp deleted file mode 100644 index 3fc7a53320a8..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_subsidence_impl.hpp +++ /dev/null @@ -1,164 +0,0 @@ -#ifndef DP_ADVANCE_IOP_SUBSIDENCE_IMPL_HPP -#define DP_ADVANCE_IOP_SUBSIDENCE_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -#include "ekat/kokkos/ekat_subview_utils.hpp" - -namespace scream { -namespace dp { - -/* - * Implementation of dp advance_iop_subsidence. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -KOKKOS_INLINE_FUNCTION -void Functions::do_advance_iop_subsidence_update( - const Int& k, - const Int& plev, - const Spack& fac, - const Spack& swfldint, - const Spack& swfldint_p1, - const uview_1d& in, - const uview_1d& in_s, - const uview_1d& update) -{ - Spack sin, sin_p1, sin_m1; - - auto range_pack1 = ekat::range(k*Spack::n); - auto range_pack2_m1_safe = range_pack1; - auto range_pack2_p1_safe = range_pack1; - range_pack2_m1_safe.set(range_pack1 < 1, 1); // don't want the shift to go below zero. we mask out that result anyway - range_pack2_p1_safe.set(range_pack1 > plev-2, plev-2); // don't want the shift to go beyond pack. - ekat::index_and_shift<-1>(in_s, range_pack2_m1_safe, sin, sin_m1); - ekat::index_and_shift< 1>(in_s, range_pack2_p1_safe, sin, sin_p1); - - update(k) = in(k) - fac*(swfldint_p1*(sin_p1 - sin) + swfldint*(sin - sin_m1)); -} - -template -KOKKOS_FUNCTION -void Functions::advance_iop_subsidence( - const Int& plev, - const Int& pcnst, - const Scalar& scm_dt, - const Scalar& ps_in, - const uview_1d& u_in, - const uview_1d& v_in, - const uview_1d& t_in, - const uview_2d& q_in, - const uview_1d& hyai, - const uview_1d& hyam, - const uview_1d& hybi, - const uview_1d& hybm, - const uview_1d& wfld, - const MemberType& team, - const Workspace& workspace, - const uview_1d& u_update, - const uview_1d& v_update, - const uview_1d& t_update, - const uview_2d& q_update) -{ - // Local variables - uview_1d - pmidm1, // pressure at model levels - pintm1, // pressure at model interfaces (dim=plev+1) - pdelm1, // pdel(k) = pint (k+1)-pint (k) - wfldint;// (dim=plev+1) - workspace.template take_many_contiguous_unsafe<4>( - {"pmidm1", "pintm1", "pdelm1", "wfldint"}, - {&pmidm1, &pintm1, &pdelm1, &wfldint}); - - const Int plev_pack = ekat::npack(plev); - - // Get vertical level profiles - plevs0(plev, ps_in, hyai, hyam, hybi, hybm, team, pintm1, pmidm1, pdelm1); - - // Scalarize a bunch of views that need shift operations - auto pmidm1_s = scalarize(pmidm1); - auto pintm1_s = scalarize(pintm1); - auto wfld_s = scalarize(wfld); - auto wfldint_s = scalarize(wfldint); - auto u_in_s = scalarize(u_in); - auto v_in_s = scalarize(v_in); - auto t_in_s = scalarize(t_in); - - wfldint_s(0) = 0; - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, plev_pack), [&] (Int k) { - Spack spmidm1, spmidm1_m1, spintm1, spintm1_m1, swfld, swfld_m1; - auto range_pack1 = ekat::range(k*Spack::n); - auto range_pack2 = range_pack1; - range_pack2.set(range_pack1 < 1, 1); // don't want the shift to go below zero. we mask out that result anyway - ekat::index_and_shift<-1>(pmidm1_s, range_pack2, spmidm1, spmidm1_m1); - ekat::index_and_shift<-1>(pintm1_s, range_pack2, spintm1, spintm1_m1); - ekat::index_and_shift<-1>(wfld_s, range_pack2, swfld, swfld_m1); - Spack weight = (spintm1 - spintm1_m1) / (spmidm1 - spmidm1_m1); - wfldint(k) = (1 - weight)*swfld_m1 + weight*swfld; - }); - wfldint_s(plev) = 0; - - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, plev_pack), [&] (Int k) { - Spack swfldint, swfldint_p1; - auto range_pack1 = ekat::range(k*Spack::n); - auto range_pack2 = range_pack1; - range_pack2.set(range_pack1 > plev-1, plev-1); - - ekat::index_and_shift<1>(wfldint_s, range_pack2, swfldint, swfldint_p1); - - Spack fac = scm_dt/(2 * pdelm1(k)); - - do_advance_iop_subsidence_update(k, plev, fac, swfldint, swfldint_p1, u_in, u_in_s, u_update); - do_advance_iop_subsidence_update(k, plev, fac, swfldint, swfldint_p1, v_in, v_in_s, v_update); - do_advance_iop_subsidence_update(k, plev, fac, swfldint, swfldint_p1, t_in, t_in_s, t_update); - - for (Int m = 0; m < pcnst; ++m) { - // Grab m-th subview of q stuff - auto q_update_sub = ekat::subview(q_update, m); - auto q_in_sub = ekat::subview(q_in, m); - auto q_in_sub_s = scalarize(q_in_sub); - do_advance_iop_subsidence_update(k, plev, fac, swfldint, swfldint_p1, q_in_sub, q_in_sub_s, q_update_sub); - } - }); - - // Top and bottom levels next - Kokkos::Array bot_top = {0, plev-1}; - for (Int i = 0; i < 2; ++i) { - const auto k = bot_top[i]; - const auto pack_idx = ekat::npack(k+1) - 1; - const auto s_idx = k % Spack::n; - const auto idx1 = k == 0 ? k+1 : k; - - Scalar fac = scm_dt/(2 * pdelm1(pack_idx)[s_idx]); - u_update(pack_idx)[s_idx] = u_in_s(k) - fac*(wfldint_s(idx1)*(u_in_s(idx1) - u_in_s(idx1-1))); - v_update(pack_idx)[s_idx] = v_in_s(k) - fac*(wfldint_s(idx1)*(v_in_s(idx1) - v_in_s(idx1-1))); - t_update(pack_idx)[s_idx] = t_in_s(k) - fac*(wfldint_s(idx1)*(t_in_s(idx1) - t_in_s(idx1-1))); - - for (Int m = 0; m < pcnst; ++m) { - auto q_update_sub = ekat::subview(q_update, m); - auto q_in_sub = ekat::subview(q_in, m); - auto q_in_sub_s = scalarize(q_in_sub); - - q_update_sub(pack_idx)[s_idx] = q_in_sub_s(k) - fac*(wfldint_s(idx1)*(q_in_sub_s(idx1) - q_in_sub_s(idx1-1))); - } - } - - // thermal expansion term due to LS vertical advection - constexpr Scalar rair = C::Rair; - constexpr Scalar cpair = C::Cpair; - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, plev_pack), [&] (Int k) { - t_update(k) = t_update(k) + scm_dt*wfld(k)*t_in(k)*rair/(cpair*pmidm1(k)); - }); - - workspace.template release_many_contiguous<4>( - {&pmidm1, &pintm1, &pdelm1, &wfldint}); -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_apply_iop_forcing_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_apply_iop_forcing_impl.hpp deleted file mode 100644 index cacae8b46ddd..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_apply_iop_forcing_impl.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef DP_APPLY_IOP_FORCING_IMPL_HPP -#define DP_APPLY_IOP_FORCING_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp apply_iop_forcing. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -KOKKOS_FUNCTION -void Functions::apply_iop_forcing(const Int& nelemd, const uview_1d& elem, hvcoord_t& hvcoord, const hybrid_t& hybrid, const timelevel_t& tl, const Int& n, const bool& t_before_advance, const Int& nets, const Int& nete) -{ - // TODO - // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_crm_resolved_turb_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_crm_resolved_turb_impl.hpp deleted file mode 100644 index 2e17f7a43c5b..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_crm_resolved_turb_impl.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef DP_CRM_RESOLVED_TURB_IMPL_HPP -#define DP_CRM_RESOLVED_TURB_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp crm_resolved_turb. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -KOKKOS_FUNCTION -void Functions::crm_resolved_turb(const Int& nelemd, const uview_1d& elem, const hvcoord_t& hvcoord, const hybrid_t& hybrid, const Int& t1, const Int& nelemd_todo, const Int& np_todo) -{ - // TODO - // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_iop_broadcast_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_iop_broadcast_impl.hpp deleted file mode 100644 index ead8ec78a2b6..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_iop_broadcast_impl.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef DP_IOP_BROADCAST_IMPL_HPP -#define DP_IOP_BROADCAST_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp iop_broadcast. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -KOKKOS_FUNCTION -void Functions::iop_broadcast() -{ - // TODO - // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_iop_default_opts_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_iop_default_opts_impl.hpp deleted file mode 100644 index e5687d7558bb..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_iop_default_opts_impl.hpp +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef DP_IOP_DEFAULT_OPTS_IMPL_HPP -#define DP_IOP_DEFAULT_OPTS_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp iop_default_opts. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -void Functions::iop_default_opts(Spack& scmlat_out, Spack& scmlon_out, std::string& iopfile_out, bool& single_column_out, bool& scm_iop_srf_prop_out, bool& iop_nudge_tq_out, bool& iop_nudge_uv_out, Spack& iop_nudge_tq_low_out, Spack& iop_nudge_tq_high_out, Spack& iop_nudge_tscale_out, bool& scm_observed_aero_out, bool& iop_dosubsidence_out, bool& scm_multcols_out, bool& dp_crm_out, Spack& iop_perturb_high_out, bool& precip_off_out, bool& scm_zero_non_iop_tracers_out) -{ - // TODO - // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_iop_domain_relaxation_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_iop_domain_relaxation_impl.hpp deleted file mode 100644 index ad36a9a0ce51..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_iop_domain_relaxation_impl.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef DP_IOP_DOMAIN_RELAXATION_IMPL_HPP -#define DP_IOP_DOMAIN_RELAXATION_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp iop_domain_relaxation. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -KOKKOS_FUNCTION -void Functions::iop_domain_relaxation(const Int& nelemd, const Int& np, const Int& nlev, const uview_1d& elem, const hvcoord_t& hvcoord, const hybrid_t& hybrid, const Int& t1, const uview_1d& dp, const Int& nelemd_todo, const Int& np_todo, const Spack& dt) -{ - // TODO - // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_iop_intht_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_iop_intht_impl.hpp deleted file mode 100644 index e03631f94cb7..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_iop_intht_impl.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef DP_IOP_INTHT_IMPL_HPP -#define DP_IOP_INTHT_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp iop_intht. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -KOKKOS_FUNCTION -void Functions::iop_intht() -{ - // TODO - // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_iop_setfield_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_iop_setfield_impl.hpp deleted file mode 100644 index ee8991c472ba..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_iop_setfield_impl.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef DP_IOP_SETFIELD_IMPL_HPP -#define DP_IOP_SETFIELD_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp iop_setfield. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -KOKKOS_FUNCTION -void Functions::iop_setfield(const Int& nelemd, const uview_1d& elem, const bool& iop_update_phase1) -{ - // TODO - // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_iop_setinitial_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_iop_setinitial_impl.hpp deleted file mode 100644 index 854ced7dd48d..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_iop_setinitial_impl.hpp +++ /dev/null @@ -1,207 +0,0 @@ -#ifndef DP_IOP_SETINITIAL_IMPL_HPP -#define DP_IOP_SETINITIAL_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -#include "Context.hpp" -#include "TimeLevel.hpp" - -namespace scream { -namespace dp { - -/* - * Implementation of dp iop_setinitial. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -void Functions::iop_setinitial( - const Int& plev, - const Int& pcnst, - const Int& nelemd, - const Int& np, - const Int& nstep, - const bool& use_replay, - const bool& dynproc, - const bool& have_t, - const bool& have_q, - const bool& have_ps, - const bool& have_u, - const bool& have_v, - const bool& have_numliq, - const bool& have_cldliq, - const bool& have_numice, - const bool& have_cldice, - const bool& scm_zero_non_iop_tracers, - const bool& is_first_restart_step, - const uview_1d& qmin, - const uview_1d& uobs, - const uview_1d& vobs, - const uview_1d& numliqobs, - const uview_1d& numiceobs, - const uview_1d& cldliqobs, - const uview_1d& cldiceobs, - const Scalar& psobs, - const uview_1d& dx_short, - Scalar& dyn_dx_size, - tracer_t& tracers, - element_t& elem, - const uview_1d& tobs, - const uview_1d& qobs) -{ - // Made these up - - //FieldGroup g = ...; - //const auto& names = g.m_info->m_field_names; - constexpr Int inumliq = 1; - constexpr Int inumice = 2; - constexpr Int icldliq = 3; - constexpr Int icldice = 4; - - const Int plev_packs = ekat::npack(plev); - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(1, plev_packs); - - assert(np <= NP); - //assert(plev_packs <= Homme::NUM_LEV); - assert(tracers.inited()); - assert(elem.inited()); - - // Get time info - const Int n0 = Homme::Context::singleton().get().n0; - - if (!use_replay && nstep == 0 && dynproc) { - - Kokkos::parallel_for( - "iop_setinitial loop", - policy, - KOKKOS_LAMBDA(const MemberType& team) { - - // We cannot support parallelism at the element level because the thelev - // computation of tobs is dependent on prior iterations that may have alterted tobs - for (Int ie = 0; ie < nelemd; ++ie) { - for (Int j = 0; j < np; ++j) { - for (Int i = 0; i < np; ++i) { - - // Find level where tobs is no longer zero - Int thelev=-1; - Kokkos::parallel_reduce( - Kokkos::TeamVectorRange(team, plev_packs), [&] (int plev_pack, Int& pmin) { - auto zmask = tobs(plev_pack) != 0; - if (zmask.any()) { - pmin = plev_pack; - } - }, Kokkos::Min(thelev)); - - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, thelev+1), [&] (int k) { - auto zmask = k < thelev ? Smask(true) : tobs(k) == 0; - vector_simd for (Int p = 0; p < Spack::n; ++p) { - if (zmask[p]) { - tobs(k)[p] = elem.m_forcing.m_ft(ie,i,j,k)[p]; - qobs(k)[p] = tracers.Q(ie,0,i,j,k)[p]; // Tracer index 0 is qobs - } - } - }); - - if (scm_zero_non_iop_tracers) { - for (Int cix = 0; cix < pcnst; ++cix) { - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, plev_packs), [&] (int k) { - tracers.Q(ie, cix, i, j, k) = qmin(cix); - }); - } - } - - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, thelev, plev_packs), [&] (int k) { - auto zmask = k > thelev ? Smask(true) : tobs(k) != 0; - if (have_t) { - vector_simd for (Int p = 0; p < Spack::n; ++p) { - if (zmask[p]) { - elem.m_forcing.m_ft(ie,i,j,k)[p] = tobs(k)[p]; - } - } - } - if (have_q) { - vector_simd for (Int p = 0; p < Spack::n; ++p) { - if (zmask[p]) { - tracers.Q(ie,0,i,j,k)[p] = qobs(k)[p]; - } - } - } - }); - - if (have_ps) { - elem.m_state.m_ps_v(ie, n0, i, j) = psobs; - } - - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, plev_packs), [&] (int k) { - if (have_u) { - vector_simd for (Int p = 0; p < Spack::n; ++p) { - elem.m_state.m_v(ie, 0, 0, i, j, k)[p] = uobs(k)[p]; // [2] is 0 for u - } - } - if (have_v) { - vector_simd for (Int p = 0; p < Spack::n; ++p) { - elem.m_state.m_v(ie, 0, 1, i, j, k)[p] = vobs(k)[p]; // [2] is 1 for v - } - } - if (have_numliq) { - vector_simd for (Int p = 0; p < Spack::n; ++p) { - tracers.Q(ie, inumliq, i, j, k)[p] = numliqobs(k)[p]; - } - } - if (have_cldliq) { - vector_simd for (Int p = 0; p < Spack::n; ++p) { - tracers.Q(ie, icldliq, i, j, k)[p] = cldliqobs(k)[p]; - } - } - if (have_numice) { - vector_simd for (Int p = 0; p < Spack::n; ++p) { - tracers.Q(ie, inumice, i, j, k)[p] = numiceobs(k)[p]; - } - } - if (have_cldice) { - vector_simd for (Int p = 0; p < Spack::n; ++p) { - tracers.Q(ie, icldice, i, j, k)[p] = cldiceobs(k)[p]; - } - } - - // If DP-CRM mode we do NOT want to write over the dy-core vertical - // velocity with the large-scale one. wfld is used in forecast.F90 - // for the compuation of the large-scale subsidence. - elem.m_derived.m_omega_p(ie, i, j, k) = 0; - }); - } - } - } - }); - } - - // If DP-CRM mode then SHOC/CLUBB needs to know about grid - // length size. The calculations of this based on a sphere in the - // SHOC and CLUBB interefaces are not valid for a planar grid, thus - // save the grid length from the dycore. Note that planar dycore - // only supports uniform grids, thus we only save one value. - // Set this if it is the first time step or the first restart step - if ( (nstep == 0 || is_first_restart_step) && dynproc) { - // for (Int ie = 0; ie < nelemd; ++ie) { - // dyn_dx_size = dx_short(ie) * 1000; // why not just grab the last ie? - // } - Kokkos::parallel_reduce( - "iop_setinitial loop", - policy, - KOKKOS_LAMBDA(const MemberType& team, Scalar& dyn) { - const Int ie = team.league_rank(); - if (ie == nelemd-1) { - dyn = dx_short(nelemd-1) * 1000; - } - }, dyn_dx_size); - } -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_iop_setopts_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_iop_setopts_impl.hpp deleted file mode 100644 index 5483abdd6a41..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_iop_setopts_impl.hpp +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef DP_IOP_SETOPTS_IMPL_HPP -#define DP_IOP_SETOPTS_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp iop_setopts. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -void Functions::iop_setopts(const Spack& scmlat_in, const Spack& scmlon_in, const std::string& iopfile_in, const bool& single_column_in, const bool& scm_iop_srf_prop_in, const bool& iop_nudge_tq_in, const bool& iop_nudge_uv_in, const Spack& iop_nudge_tq_low_in, const Spack& iop_nudge_tq_high_in, const Spack& iop_nudge_tscale_in, const bool& scm_observed_aero_in, const bool& iop_dosubsidence_in, const bool& scm_multcols_in, const bool& dp_crm_in, const Spack& iop_perturb_high_in, const bool& precip_off_in, const bool& scm_zero_non_iop_tracers_in) -{ - // TODO - // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_readiopdata_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_readiopdata_impl.hpp deleted file mode 100644 index 3cd83bfcdb06..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_readiopdata_impl.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef DP_READIOPDATA_IMPL_HPP -#define DP_READIOPDATA_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp readiopdata. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -KOKKOS_FUNCTION -void Functions::readiopdata(const Int& plev, const bool& iop_update_phase1, const uview_1d& hyam, const uview_1d& hybm) -{ - // TODO - // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_setiopupdate_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_setiopupdate_impl.hpp deleted file mode 100644 index 78e1bfe7340c..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_setiopupdate_impl.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef DP_SETIOPUPDATE_IMPL_HPP -#define DP_SETIOPUPDATE_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp setiopupdate. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -KOKKOS_FUNCTION -void Functions::setiopupdate() -{ - // TODO - // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_setiopupdate_init_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_setiopupdate_init_impl.hpp deleted file mode 100644 index bc9ba8d41c61..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_setiopupdate_init_impl.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef DP_SETIOPUPDATE_INIT_IMPL_HPP -#define DP_SETIOPUPDATE_INIT_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp setiopupdate_init. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -KOKKOS_FUNCTION -void Functions::setiopupdate_init() -{ - // TODO - // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/tests/CMakeLists.txt b/components/eamxx/src/doubly-periodic/tests/CMakeLists.txt deleted file mode 100644 index 32e2883a22f6..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/CMakeLists.txt +++ /dev/null @@ -1,27 +0,0 @@ -INCLUDE (ScreamUtils) - -set(DP_TESTS_SRCS - dp_unit_tests.cpp - dp_advance_iop_forcing_tests.cpp - dp_advance_iop_nudging_tests.cpp - dp_advance_iop_subsidence_tests.cpp - dp_iop_setinitial_tests.cpp - dp_iop_broadcast_tests.cpp - dp_apply_iop_forcing_tests.cpp - dp_iop_domain_relaxation_tests.cpp - dp_crm_resolved_turb_tests.cpp - dp_iop_default_opts_tests.cpp - dp_iop_setopts_tests.cpp - dp_setiopupdate_init_tests.cpp - dp_setiopupdate_tests.cpp - dp_readiopdata_tests.cpp - dp_iop_intht_tests.cpp - ) # DP_TESTS_SRCS - -# NOTE: tests inside this if statement won't be built in a baselines-only build -if (NOT SCREAM_BASELINES_ONLY) - CreateUnitTest(dp_tests "${DP_TESTS_SRCS}" - LIBS dp - THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} - ) -endif() diff --git a/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_forcing_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_forcing_tests.cpp deleted file mode 100644 index d1f5ea89a42e..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_forcing_tests.cpp +++ /dev/null @@ -1,116 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestAdvanceIopForcing { - - static void run_bfb() - { - auto engine = setup_random_test(); - - AdvanceIopForcingData f90_data[] = { - // plev, pcnst, scm_dt, ps_in, have_u, have_v, dp_crm, use_3dfrc - AdvanceIopForcingData(72, 10, 0.1, 1000.0, true, true, true, true), - AdvanceIopForcingData(72, 10, 0.1, 1000.0, true, true, true, false), - AdvanceIopForcingData(72, 10, 0.1, 1000.0, true, true, false, true), - - AdvanceIopForcingData(27, 7, 0.1, 1000.0, true, true, true, true), - AdvanceIopForcingData(27, 7, 0.1, 1000.0, true, true, true, false), - AdvanceIopForcingData(27, 7, 0.1, 1000.0, true, true, false, true), - - AdvanceIopForcingData(32, 7, 0.1, 1000.0, true, true, true, true), - AdvanceIopForcingData(32, 7, 0.1, 1000.0, true, true, true, false), - AdvanceIopForcingData(32, 7, 0.1, 1000.0, true, true, false, true), - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(AdvanceIopForcingData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx. Needs to happen before fortran calls so that - // inout data is in original state - AdvanceIopForcingData cxx_data[num_runs] = { - AdvanceIopForcingData(f90_data[0]), - AdvanceIopForcingData(f90_data[1]), - AdvanceIopForcingData(f90_data[2]), - AdvanceIopForcingData(f90_data[3]), - AdvanceIopForcingData(f90_data[4]), - AdvanceIopForcingData(f90_data[5]), - AdvanceIopForcingData(f90_data[6]), - AdvanceIopForcingData(f90_data[7]), - AdvanceIopForcingData(f90_data[8]), - }; - - // Assume all data is in C layout - - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - advance_iop_forcing(d); - } - - // Get data from cxx - for (auto& d : cxx_data) { - d.template transpose(); // _f expects data in fortran layout - advance_iop_forcing_f(d.plev, d.pcnst, d.scm_dt, d.ps_in, d.have_u, d.have_v, d.dp_crm, d.use_3dfrc, d.u_in, d.v_in, d.t_in, d.q_in, d.t_phys_frc, d.divt3d, d.divq3d, d.divt, d.divq, d.wfld, d.uobs, d.vobs, d.hyai, d.hyam, d.hybi, d.hybm, d.u_update, d.v_update, d.t_update, d.q_update); - d.template transpose(); // go back to C layout - } - - // We can't call into fortran. Due to all the dependencies it has, it's not possible - // to build it in standalone eamxx. Without fortran, we cannot do BFB tests. -#if 0 - // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - AdvanceIopForcingData& d_f90 = f90_data[i]; - AdvanceIopForcingData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.u_update); ++k) { - REQUIRE(d_f90.total(d_f90.u_update) == d_cxx.total(d_cxx.u_update)); - REQUIRE(d_f90.u_update[k] == d_cxx.u_update[k]); - REQUIRE(d_f90.total(d_f90.u_update) == d_cxx.total(d_cxx.v_update)); - REQUIRE(d_f90.v_update[k] == d_cxx.v_update[k]); - REQUIRE(d_f90.total(d_f90.u_update) == d_cxx.total(d_cxx.t_update)); - REQUIRE(d_f90.t_update[k] == d_cxx.t_update[k]); - } - for (Int k = 0; k < d_f90.total(d_f90.q_update); ++k) { - REQUIRE(d_f90.total(d_f90.q_update) == d_cxx.total(d_cxx.q_update)); - REQUIRE(d_f90.q_update[k] == d_cxx.q_update[k]); - } - - } - } -#endif - - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("advance_iop_forcing_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestAdvanceIopForcing; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_nudging_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_nudging_tests.cpp deleted file mode 100644 index 02f9362eb7a3..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_nudging_tests.cpp +++ /dev/null @@ -1,98 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestAdvanceIopNudging { - - static void run_bfb() - { - auto engine = setup_random_test(); - - AdvanceIopNudgingData f90_data[] = { - // plev, scm_dt, ps_in - AdvanceIopNudgingData(72, 0.1, 1000), - AdvanceIopNudgingData(27, 0.1, 1000), - AdvanceIopNudgingData(32, 0.1, 1000), - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(AdvanceIopNudgingData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx. Needs to happen before fortran calls so that - // inout data is in original state - AdvanceIopNudgingData cxx_data[] = { - AdvanceIopNudgingData(f90_data[0]), - AdvanceIopNudgingData(f90_data[1]), - AdvanceIopNudgingData(f90_data[2]), - }; - - // Assume all data is in C layout - - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - advance_iop_nudging(d); - } - - // Get data from cxx - for (auto& d : cxx_data) { - advance_iop_nudging_f(d.plev, d.scm_dt, d.ps_in, d.t_in, d.q_in, d.tobs, d.qobs, - d.hyai, d.hyam, d.hybi, d.hybm, - d.t_update, d.q_update, d.relaxt, d.relaxq); - } - - // We can't call into fortran. Due to all the dependencies it has, it's not possible - // to build it in standalone eamxx. Without fortran, we cannot do BFB tests. -#if 0 - // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - AdvanceIopNudgingData& d_f90 = f90_data[i]; - AdvanceIopNudgingData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.t_update); ++k) { - REQUIRE(d_f90.total(d_f90.t_update) == d_cxx.total(d_cxx.t_update)); - REQUIRE(d_f90.t_update[k] == d_cxx.t_update[k]); - REQUIRE(d_f90.total(d_f90.t_update) == d_cxx.total(d_cxx.q_update)); - REQUIRE(d_f90.q_update[k] == d_cxx.q_update[k]); - REQUIRE(d_f90.total(d_f90.t_update) == d_cxx.total(d_cxx.relaxt)); - REQUIRE(d_f90.relaxt[k] == d_cxx.relaxt[k]); - REQUIRE(d_f90.total(d_f90.t_update) == d_cxx.total(d_cxx.relaxq)); - REQUIRE(d_f90.relaxq[k] == d_cxx.relaxq[k]); - } - - } - } -#endif - } // run_bfb -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("advance_iop_nudging_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestAdvanceIopNudging; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_subsidence_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_subsidence_tests.cpp deleted file mode 100644 index a430135f75be..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_subsidence_tests.cpp +++ /dev/null @@ -1,107 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestAdvanceIopSubsidence { - - static void run_bfb() - { - auto engine = setup_random_test(); - - AdvanceIopSubsidenceData f90_data[] = { - // plev, pcnst, scm_dt, ps_in - AdvanceIopSubsidenceData(72, 10, 0.1, 1000.0), - AdvanceIopSubsidenceData(72, 7, 0.1, 1000.0), - AdvanceIopSubsidenceData(27, 10, 0.1, 1000.0), - AdvanceIopSubsidenceData(27, 7, 0.1, 1000.0), - AdvanceIopSubsidenceData(32, 10, 0.1, 1000.0), - AdvanceIopSubsidenceData(32, 7, 0.1, 1000.0), - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(AdvanceIopSubsidenceData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx. Needs to happen before fortran calls so that - // inout data is in original state - AdvanceIopSubsidenceData cxx_data[num_runs] = { - AdvanceIopSubsidenceData(f90_data[0]), - AdvanceIopSubsidenceData(f90_data[1]), - AdvanceIopSubsidenceData(f90_data[2]), - AdvanceIopSubsidenceData(f90_data[3]), - AdvanceIopSubsidenceData(f90_data[4]), - AdvanceIopSubsidenceData(f90_data[5]), - }; - - // Assume all data is in C layout - - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - advance_iop_subsidence(d); - } - - // Get data from cxx - for (auto& d : cxx_data) { - d.template transpose(); // _f expects data in fortran layout - advance_iop_subsidence_f(d.plev, d.pcnst, d.scm_dt, d.ps_in, d.u_in, d.v_in, d.t_in, d.q_in, d.hyai, d.hyam, d.hybi, d.hybm, d.wfld, d.u_update, d.v_update, d.t_update, d.q_update); - d.template transpose(); // go back to C layout - } - - // We can't call into fortran. Due to all the dependencies it has, it's not possible - // to build it in standalone eamxx. Without fortran, we cannot do BFB tests. -#if 0 - // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - AdvanceIopSubsidenceData& d_f90 = f90_data[i]; - AdvanceIopSubsidenceData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.u_update); ++k) { - REQUIRE(d_f90.total(d_f90.u_update) == d_cxx.total(d_cxx.u_update)); - REQUIRE(d_f90.u_update[k] == d_cxx.u_update[k]); - REQUIRE(d_f90.total(d_f90.u_update) == d_cxx.total(d_cxx.v_update)); - REQUIRE(d_f90.v_update[k] == d_cxx.v_update[k]); - REQUIRE(d_f90.total(d_f90.u_update) == d_cxx.total(d_cxx.t_update)); - REQUIRE(d_f90.t_update[k] == d_cxx.t_update[k]); - } - for (Int k = 0; k < d_f90.total(d_f90.q_update); ++k) { - REQUIRE(d_f90.total(d_f90.q_update) == d_cxx.total(d_cxx.q_update)); - REQUIRE(d_f90.q_update[k] == d_cxx.q_update[k]); - } - - } - } -#endif - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("advance_iop_subsidence_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestAdvanceIopSubsidence; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_apply_iop_forcing_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_apply_iop_forcing_tests.cpp deleted file mode 100644 index 3e85e30974bc..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_apply_iop_forcing_tests.cpp +++ /dev/null @@ -1,78 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestApplyIopForcing { - - static void run_bfb() - { - auto engine = setup_random_test(); - - ApplyIopForcingData f90_data[] = { - // TODO - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(ApplyIopForcingData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx. Needs to happen before fortran calls so that - // inout data is in original state - ApplyIopForcingData cxx_data[] = { - // TODO - }; - - // Assume all data is in C layout - - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - apply_iop_forcing(d); - } - - // Get data from cxx - for (auto& d : cxx_data) { - apply_iop_forcing_f(d.nelemd, d.elem, &d.hvcoord, d.hybrid, d.tl, d.n, d.t_before_advance, d.nets, d.nete); - } - - // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - ApplyIopForcingData& d_f90 = f90_data[i]; - ApplyIopForcingData& d_cxx = cxx_data[i]; - //REQUIRE(d_f90.hvcoord == d_cxx.hvcoord); - } - } - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("apply_iop_forcing_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestApplyIopForcing; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_crm_resolved_turb_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_crm_resolved_turb_tests.cpp deleted file mode 100644 index f56d24880606..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_crm_resolved_turb_tests.cpp +++ /dev/null @@ -1,78 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestCrmResolvedTurb { - - static void run_bfb() - { - auto engine = setup_random_test(); - - CrmResolvedTurbData f90_data[] = { - // TODO - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(CrmResolvedTurbData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx. Needs to happen before fortran calls so that - // inout data is in original state - CrmResolvedTurbData cxx_data[] = { - // TODO - }; - - // Assume all data is in C layout - - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - crm_resolved_turb(d); - } - - // Get data from cxx - for (auto& d : cxx_data) { - crm_resolved_turb_f(d.nelemd, d.elem, d.hvcoord, d.hybrid, d.t1, d.nelemd_todo, d.np_todo); - } - - // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - CrmResolvedTurbData& d_f90 = f90_data[i]; - CrmResolvedTurbData& d_cxx = cxx_data[i]; - - } - } - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("crm_resolved_turb_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestCrmResolvedTurb; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_iop_broadcast_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_iop_broadcast_tests.cpp deleted file mode 100644 index 0ffb8f63e5a5..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_iop_broadcast_tests.cpp +++ /dev/null @@ -1,85 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestIopBroadcast { - - static void run_bfb() - { - auto engine = setup_random_test(); - - IopBroadcastData f90_data[max_pack_size]; //= { - // // TODO - // }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(IopBroadcastData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - -#if 0 - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that - // inout data is in original state - view_1d cxx_device("cxx_device", max_pack_size); - const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); - Kokkos::deep_copy(cxx_device, cxx_host); - - // Get data from fortran - for (auto& d : f90_data) { - iop_broadcast(d); - } - - // Get data from cxx. Run iop_broadcast from a kernel and copy results back to host - // Kokkos::parallel_for(num_test_itrs, KOKKOS_LAMBDA(const Int& i) { - // const Int offset = i * Spack::n; - - - - - // Functions::iop_broadcast(); - - - // }); - - Kokkos::deep_copy(cxx_host, cxx_device); - - // Verify BFB results - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - IopBroadcastData& d_f90 = f90_data[i]; - IopBroadcastData& d_cxx = cxx_host[i]; - } - } -#endif - } // run_bfb -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("iop_broadcast_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestIopBroadcast; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_iop_default_opts_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_iop_default_opts_tests.cpp deleted file mode 100644 index 622ca26a7264..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_iop_default_opts_tests.cpp +++ /dev/null @@ -1,106 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestIopDefaultOpts { - - static void run_bfb() - { - auto engine = setup_random_test(); - - IopDefaultOptsData f90_data[max_pack_size] = { - // TODO - }; - - static constexpr Int num_runs = 0; //sizeof(f90_data) / sizeof(IopDefaultOptsData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that - // inout data is in original state - using host_view = typename view_1d::host_mirror_type; - host_view cxx_host("cxx_host", max_pack_size); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); - - // Get data from fortran - for (auto& d : f90_data) { - iop_default_opts(d); - } - - // Get data from cxx. Run iop_default_opts from a kernel and copy results back to host - for (Int i = 0; i < num_test_itrs; ++i) { - const Int offset = i * Spack::n; - - // Init outputs - Spack iop_nudge_tq_high_out(0), iop_nudge_tq_low_out(0), iop_nudge_tscale_out(0), iop_perturb_high_out(0), scmlat_out(0), scmlon_out(0); - - Functions::iop_default_opts(scmlat_out, scmlon_out, cxx_host(0).iopfile_out, cxx_host(0).single_column_out, cxx_host(0).scm_iop_srf_prop_out, cxx_host(0).iop_nudge_tq_out, cxx_host(0).iop_nudge_uv_out, iop_nudge_tq_low_out, iop_nudge_tq_high_out, iop_nudge_tscale_out, cxx_host(0).scm_observed_aero_out, cxx_host(0).iop_dosubsidence_out, cxx_host(0).scm_multcols_out, cxx_host(0).dp_crm_out, iop_perturb_high_out, cxx_host(0).precip_off_out, cxx_host(0).scm_zero_non_iop_tracers_out); - - // Copy spacks back into cxx_host view - for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { - cxx_host(vs).iop_nudge_tq_high_out = iop_nudge_tq_high_out[s]; - cxx_host(vs).iop_nudge_tq_low_out = iop_nudge_tq_low_out[s]; - cxx_host(vs).iop_nudge_tscale_out = iop_nudge_tscale_out[s]; - cxx_host(vs).iop_perturb_high_out = iop_perturb_high_out[s]; - cxx_host(vs).scmlat_out = scmlat_out[s]; - cxx_host(vs).scmlon_out = scmlon_out[s]; - } - } - - // Verify BFB results - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - IopDefaultOptsData& d_f90 = f90_data[i]; - IopDefaultOptsData& d_cxx = cxx_host[i]; - REQUIRE(d_f90.scmlat_out == d_cxx.scmlat_out); - REQUIRE(d_f90.scmlon_out == d_cxx.scmlon_out); - REQUIRE(d_f90.iopfile_out == d_cxx.iopfile_out); - REQUIRE(d_f90.single_column_out == d_cxx.single_column_out); - REQUIRE(d_f90.scm_iop_srf_prop_out == d_cxx.scm_iop_srf_prop_out); - REQUIRE(d_f90.iop_nudge_tq_out == d_cxx.iop_nudge_tq_out); - REQUIRE(d_f90.iop_nudge_uv_out == d_cxx.iop_nudge_uv_out); - REQUIRE(d_f90.iop_nudge_tq_low_out == d_cxx.iop_nudge_tq_low_out); - REQUIRE(d_f90.iop_nudge_tq_high_out == d_cxx.iop_nudge_tq_high_out); - REQUIRE(d_f90.iop_nudge_tscale_out == d_cxx.iop_nudge_tscale_out); - REQUIRE(d_f90.scm_observed_aero_out == d_cxx.scm_observed_aero_out); - REQUIRE(d_f90.iop_dosubsidence_out == d_cxx.iop_dosubsidence_out); - REQUIRE(d_f90.scm_multcols_out == d_cxx.scm_multcols_out); - REQUIRE(d_f90.dp_crm_out == d_cxx.dp_crm_out); - REQUIRE(d_f90.iop_perturb_high_out == d_cxx.iop_perturb_high_out); - REQUIRE(d_f90.precip_off_out == d_cxx.precip_off_out); - REQUIRE(d_f90.scm_zero_non_iop_tracers_out == d_cxx.scm_zero_non_iop_tracers_out); - } - } - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("iop_default_opts_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestIopDefaultOpts; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_iop_domain_relaxation_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_iop_domain_relaxation_tests.cpp deleted file mode 100644 index ed570c6767a4..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_iop_domain_relaxation_tests.cpp +++ /dev/null @@ -1,84 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestIopDomainRelaxation { - - static void run_bfb() - { - auto engine = setup_random_test(); - - IopDomainRelaxationData f90_data[] = { - // TODO - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(IopDomainRelaxationData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx. Needs to happen before fortran calls so that - // inout data is in original state - IopDomainRelaxationData cxx_data[] = { - // TODO - }; - - // Assume all data is in C layout - - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - iop_domain_relaxation(d); - } - - // Get data from cxx - for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - iop_domain_relaxation_f(d.nelemd, d.np, d.nlev, d.elem, d.hvcoord, d.hybrid, d.t1, d.dp, d.nelemd_todo, d.np_todo, d.dt); - d.transpose(); // go back to C layout - } - - // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - IopDomainRelaxationData& d_f90 = f90_data[i]; - IopDomainRelaxationData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.dp); ++k) { - REQUIRE(d_f90.total(d_f90.dp) == d_cxx.total(d_cxx.dp)); - REQUIRE(d_f90.dp[k] == d_cxx.dp[k]); - } - - } - } - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("iop_domain_relaxation_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestIopDomainRelaxation; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_iop_intht_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_iop_intht_tests.cpp deleted file mode 100644 index f214ebb49961..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_iop_intht_tests.cpp +++ /dev/null @@ -1,84 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestIopIntht { - - static void run_bfb() - { - auto engine = setup_random_test(); - - IopInthtData f90_data[max_pack_size] = { - // TODO - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(IopInthtData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that - // inout data is in original state - view_1d cxx_device("cxx_device", max_pack_size); - const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); - Kokkos::deep_copy(cxx_device, cxx_host); - - // Get data from fortran - for (auto& d : f90_data) { - iop_intht(d); - } - - // Get data from cxx. Run iop_intht from a kernel and copy results back to host - Kokkos::parallel_for(num_test_itrs, KOKKOS_LAMBDA(const Int& i) { - const Int offset = i * Spack::n; - - - - - Functions::iop_intht(); - - - }); - - Kokkos::deep_copy(cxx_host, cxx_device); - - // Verify BFB results - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - IopInthtData& d_f90 = f90_data[i]; - IopInthtData& d_cxx = cxx_host[i]; - } - } - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("iop_intht_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestIopIntht; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_iop_setfield_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_iop_setfield_tests.cpp deleted file mode 100644 index 04921cbcbfb8..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_iop_setfield_tests.cpp +++ /dev/null @@ -1,78 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestIopSetfield { - - static void run_bfb() - { - auto engine = setup_random_test(); - - IopSetfieldData f90_data[] = { - // TODO - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(IopSetfieldData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx. Needs to happen before fortran calls so that - // inout data is in original state - IopSetfieldData cxx_data[] = { - // TODO - }; - - // Assume all data is in C layout - - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - iop_setfield(d); - } - - // Get data from cxx - for (auto& d : cxx_data) { - iop_setfield_f(d.nelemd, d.elem, d.iop_update_phase1); - } - - // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - IopSetfieldData& d_f90 = f90_data[i]; - IopSetfieldData& d_cxx = cxx_data[i]; - - } - } - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("iop_setfield_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestIopSetfield; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_iop_setinitial_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_iop_setinitial_tests.cpp deleted file mode 100644 index d7e1d6d7a7df..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_iop_setinitial_tests.cpp +++ /dev/null @@ -1,102 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestIopSetinitial { - - static void run_bfb() - { - auto engine = setup_random_test(); - - IopSetinitialData f90_data[] = { - // plev, pcnst, nelemd, np, nstep, psobs, use_replay, dynproc, have_t, have_q, have_ps, have_u, have_v, have_numliq, have_cldliq, have_numice, have_cldice, scm_zero_non_iop_tracers, is_first_restart_step - IopSetinitialData(72 , 5 , 10 , 2 , 10, 0.1, true , true , true , true , true , true , true , true , true , true , true , true , true), - IopSetinitialData(72 , 5 , 10 , 2 , 10, 0.1, false , true , true , true , true , true , true , true , true , true , true , true , true), - IopSetinitialData(72 , 5 , 10 , 2 , 1 , 0.1, true , true , true , true , true , true , true , true , true , true , true , true , true), - IopSetinitialData(72 , 5 , 10 , 2 , 1 , 0.1, false , true , true , true , true , true , true , true , true , true , true , true , true), - IopSetinitialData(72 , 5 , 10 , 2 , 1 , 0.1, true , true , true , true , true , true , true , true , true , true , true , true , false), - IopSetinitialData(72 , 5 , 10 , 2 , 1 , 0.1, false , true , true , true , true , true , true , true , true , true , true , true , false), - IopSetinitialData(72 , 5 , 10 , 2 , 0 , 0.1, true , true , true , true , true , true , true , true , true , true , true , true , true), - IopSetinitialData(72 , 5 , 10 , 2 , 0 , 0.1, false , true , true , true , true , true , true , true , true , true , true , true , true), - IopSetinitialData(72 , 5 , 10 , 2 , 0 , 0.1, true , true , true , true , true , true , true , true , true , true , true , false , true), - IopSetinitialData(72 , 5 , 10 , 2 , 0 , 0.1, false , true , true , true , true , true , true , true , true , true , true , false , true), - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(IopSetinitialData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx. Needs to happen before fortran calls so that - // inout data is in original state - IopSetinitialData cxx_data[] = { - IopSetinitialData(f90_data[0]), - IopSetinitialData(f90_data[1]), - IopSetinitialData(f90_data[2]), - IopSetinitialData(f90_data[3]), - IopSetinitialData(f90_data[4]), - IopSetinitialData(f90_data[5]), - IopSetinitialData(f90_data[6]), - IopSetinitialData(f90_data[7]), - IopSetinitialData(f90_data[8]), - IopSetinitialData(f90_data[9]), - }; - - // Assume all data is in C layout - - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - iop_setinitial(d); - } - - // Get data from cxx - for (auto& d : cxx_data) { - d.init(); - iop_setinitial_f(d.plev, d.pcnst, d.nelemd, d.np, d.nstep, d.psobs, d.use_replay, d.dynproc, d.have_t, d.have_q, d.have_ps, d.have_u, d.have_v, d.have_numliq, d.have_cldliq, d.have_numice, d.have_cldice, d.scm_zero_non_iop_tracers, d.is_first_restart_step, d.qmin, d.uobs, d.vobs, d.numliqobs, d.numiceobs, d.cldliqobs, d.cldiceobs, d.dx_short, &d.tracers, &d.elem, &d.dyn_dx_size, d.tobs, d.qobs); - } - - // We can't call into fortran. Due to all the dependencies it has, it's not possible - // to build it in standalone eamxx. Without fortran, we cannot do BFB tests. -#if 0 - // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - IopSetinitialData& d_f90 = f90_data[i]; - IopSetinitialData& d_cxx = cxx_data[i]; - - } - } -#endif - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("iop_setinitial_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestIopSetinitial; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_iop_setopts_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_iop_setopts_tests.cpp deleted file mode 100644 index d24f54fb634a..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_iop_setopts_tests.cpp +++ /dev/null @@ -1,87 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestIopSetopts { - - static void run_bfb() - { - auto engine = setup_random_test(); - - IopSetoptsData f90_data[max_pack_size] = { - // TODO - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(IopSetoptsData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that - // inout data is in original state - using host_view = typename view_1d::host_mirror_type; - host_view cxx_host("cxx_host", max_pack_size); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); - - // Get data from fortran - for (auto& d : f90_data) { - iop_setopts(d); - } - - // Get data from cxx. Run iop_setopts from a kernel and copy results back to host - for (Int i = 0; i < num_test_itrs; ++i) { - const Int offset = i * Spack::n; - - // Init pack inputs - Spack iop_nudge_tq_high_in, iop_nudge_tq_low_in, iop_nudge_tscale_in, iop_perturb_high_in, scmlat_in, scmlon_in; - for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { - iop_nudge_tq_high_in[s] = cxx_host(vs).iop_nudge_tq_high_in; - iop_nudge_tq_low_in[s] = cxx_host(vs).iop_nudge_tq_low_in; - iop_nudge_tscale_in[s] = cxx_host(vs).iop_nudge_tscale_in; - iop_perturb_high_in[s] = cxx_host(vs).iop_perturb_high_in; - scmlat_in[s] = cxx_host(vs).scmlat_in; - scmlon_in[s] = cxx_host(vs).scmlon_in; - } - - Functions::iop_setopts(scmlat_in, scmlon_in, cxx_host(0).iopfile_in, cxx_host(0).single_column_in, cxx_host(0).scm_iop_srf_prop_in, cxx_host(0).iop_nudge_tq_in, cxx_host(0).iop_nudge_uv_in, iop_nudge_tq_low_in, iop_nudge_tq_high_in, iop_nudge_tscale_in, cxx_host(0).scm_observed_aero_in, cxx_host(0).iop_dosubsidence_in, cxx_host(0).scm_multcols_in, cxx_host(0).dp_crm_in, iop_perturb_high_in, cxx_host(0).precip_off_in, cxx_host(0).scm_zero_non_iop_tracers_in); - } - - // Verify BFB results - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - IopSetoptsData& d_f90 = f90_data[i]; - IopSetoptsData& d_cxx = cxx_host[i]; - } - } - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("iop_setopts_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestIopSetopts; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_readiopdata_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_readiopdata_tests.cpp deleted file mode 100644 index bef219811845..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_readiopdata_tests.cpp +++ /dev/null @@ -1,78 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestReadiopdata { - - static void run_bfb() - { - auto engine = setup_random_test(); - - ReadiopdataData f90_data[] = { - // TODO - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(ReadiopdataData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx. Needs to happen before fortran calls so that - // inout data is in original state - ReadiopdataData cxx_data[] = { - // TODO - }; - - // Assume all data is in C layout - - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - readiopdata(d); - } - - // Get data from cxx - for (auto& d : cxx_data) { - readiopdata_f(d.plev, d.iop_update_phase1, d.hyam, d.hybm); - } - - // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - ReadiopdataData& d_f90 = f90_data[i]; - ReadiopdataData& d_cxx = cxx_data[i]; - - } - } - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("readiopdata_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestReadiopdata; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_setiopupdate_init_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_setiopupdate_init_tests.cpp deleted file mode 100644 index b60fc5f8d4cf..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_setiopupdate_init_tests.cpp +++ /dev/null @@ -1,84 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestSetiopupdateInit { - - static void run_bfb() - { - auto engine = setup_random_test(); - - SetiopupdateInitData f90_data[max_pack_size] = { - // TODO - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(SetiopupdateInitData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that - // inout data is in original state - view_1d cxx_device("cxx_device", max_pack_size); - const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); - Kokkos::deep_copy(cxx_device, cxx_host); - - // Get data from fortran - for (auto& d : f90_data) { - setiopupdate_init(d); - } - - // Get data from cxx. Run setiopupdate_init from a kernel and copy results back to host - Kokkos::parallel_for(num_test_itrs, KOKKOS_LAMBDA(const Int& i) { - const Int offset = i * Spack::n; - - - - - Functions::setiopupdate_init(); - - - }); - - Kokkos::deep_copy(cxx_host, cxx_device); - - // Verify BFB results - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - SetiopupdateInitData& d_f90 = f90_data[i]; - SetiopupdateInitData& d_cxx = cxx_host[i]; - } - } - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("setiopupdate_init_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestSetiopupdateInit; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_setiopupdate_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_setiopupdate_tests.cpp deleted file mode 100644 index 800aaab005bf..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_setiopupdate_tests.cpp +++ /dev/null @@ -1,84 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestSetiopupdate { - - static void run_bfb() - { - auto engine = setup_random_test(); - - SetiopupdateData f90_data[max_pack_size] = { - // TODO - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(SetiopupdateData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that - // inout data is in original state - view_1d cxx_device("cxx_device", max_pack_size); - const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); - Kokkos::deep_copy(cxx_device, cxx_host); - - // Get data from fortran - for (auto& d : f90_data) { - setiopupdate(d); - } - - // Get data from cxx. Run setiopupdate from a kernel and copy results back to host - Kokkos::parallel_for(num_test_itrs, KOKKOS_LAMBDA(const Int& i) { - const Int offset = i * Spack::n; - - - - - Functions::setiopupdate(); - - - }); - - Kokkos::deep_copy(cxx_host, cxx_device); - - // Verify BFB results - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - SetiopupdateData& d_f90 = f90_data[i]; - SetiopupdateData& d_cxx = cxx_host[i]; - } - } - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("setiopupdate_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestSetiopupdate; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_unit_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_unit_tests.cpp deleted file mode 100644 index bfd865a1d924..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_unit_tests.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#include "catch2/catch.hpp" - -#include "dp_unit_tests_common.hpp" - -#include "dp_functions.hpp" -#include "dp_functions_f90.hpp" - -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "ekat/util/ekat_arch.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace dp { -namespace unit_test { - -}//namespace unit_test -}//namespace dp -}//namespace scream - diff --git a/components/eamxx/src/doubly-periodic/tests/dp_unit_tests_common.hpp b/components/eamxx/src/doubly-periodic/tests/dp_unit_tests_common.hpp deleted file mode 100644 index b2a85e9e5db7..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_unit_tests_common.hpp +++ /dev/null @@ -1,79 +0,0 @@ -#ifndef DP_UNIT_TESTS_COMMON_HPP -#define DP_UNIT_TESTS_COMMON_HPP - -#include "dp_functions.hpp" -#include "share/scream_types.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "share/util/scream_setup_random_test.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -/* - * Unit test infrastructure for dp unit tests. - * - * dp entities can friend scream::dp::unit_test::UnitWrap to give unit tests - * access to private members. - * - * All unit test impls should be within an inner struct of UnitWrap::UnitTest for - * easy access to useful types. - */ - -struct UnitWrap { - - template - struct UnitTest : public KokkosTypes { - - using Device = D; - using MemberType = typename KokkosTypes::MemberType; - using TeamPolicy = typename KokkosTypes::TeamPolicy; - using RangePolicy = typename KokkosTypes::RangePolicy; - using ExeSpace = typename KokkosTypes::ExeSpace; - - template - using view_1d = typename KokkosTypes::template view_1d; - template - using view_2d = typename KokkosTypes::template view_2d; - template - using view_3d = typename KokkosTypes::template view_3d; - - template - using uview_1d = typename ekat::template Unmanaged >; - - using Functions = scream::dp::Functions; - using Scalar = typename Functions::Scalar; - using Spack = typename Functions::Spack; - using Pack = typename Functions::Pack; - using IntSmallPack = typename Functions::IntSmallPack; - using Smask = typename Functions::Smask; - using C = typename Functions::C; - - static constexpr Int max_pack_size = 16; - static constexpr Int num_test_itrs = max_pack_size / Spack::n; - - // Put struct decls here - struct TestAdvanceIopForcing; - struct TestAdvanceIopNudging; - struct TestAdvanceIopSubsidence; - struct TestIopSetinitial; - struct TestIopBroadcast; - struct TestApplyIopForcing; - struct TestIopDomainRelaxation; - struct TestCrmResolvedTurb; - struct TestIopDefaultOpts; - struct TestIopSetopts; - struct TestSetiopupdateInit; - struct TestSetiopupdate; - struct TestReadiopdata; - struct TestIopIntht; - }; - -}; - - -} // namespace unit_test -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/dynamics/homme/CMakeLists.txt b/components/eamxx/src/dynamics/homme/CMakeLists.txt index 3cf34d3eb88a..b6a69a3605f9 100644 --- a/components/eamxx/src/dynamics/homme/CMakeLists.txt +++ b/components/eamxx/src/dynamics/homme/CMakeLists.txt @@ -132,7 +132,7 @@ macro (CreateDynamicsLib HOMME_TARGET NP PLEV QSIZE) # possible something that matters will be added in the future. SetCudaFlags(${hommeLibName} CUDA_LANG) endif() - + SetOmpFlags(${hommeLibName}) ##################################### @@ -146,6 +146,7 @@ macro (CreateDynamicsLib HOMME_TARGET NP PLEV QSIZE) ${SCREAM_DYNAMICS_SRC_DIR}/eamxx_homme_process_interface.cpp ${SCREAM_DYNAMICS_SRC_DIR}/eamxx_homme_fv_phys.cpp ${SCREAM_DYNAMICS_SRC_DIR}/eamxx_homme_rayleigh_friction.cpp + ${SCREAM_DYNAMICS_SRC_DIR}/eamxx_homme_iop.cpp ${SCREAM_DYNAMICS_SRC_DIR}/physics_dynamics_remapper.cpp ${SCREAM_DYNAMICS_SRC_DIR}/homme_grids_manager.cpp ${SCREAM_DYNAMICS_SRC_DIR}/interface/homme_context_mod.F90 diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp new file mode 100644 index 000000000000..09e2c7a17b2d --- /dev/null +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -0,0 +1,444 @@ +#include "eamxx_homme_process_interface.hpp" + +// EAMxx includes +#include "control/intensive_observation_period.hpp" +#include "dynamics/homme/homme_dimensions.hpp" +#include "dynamics/homme/homme_dynamics_helpers.hpp" +#include "physics/share/physics_constants.hpp" +#include "share/util/scream_column_ops.hpp" + +// Homme includes +#include "Context.hpp" +#include "ColumnOps.hpp" +#include "ElementOps.hpp" +#include "EquationOfState.hpp" +#include "HommexxEnums.hpp" +#include "HybridVCoord.hpp" +#include "KernelVariables.hpp" +#include "SimulationParams.hpp" +#include "Types.hpp" + +// EKAT includes +#include "ekat/ekat_workspace.hpp" +#include "ekat/kokkos/ekat_kokkos_types.hpp" + +namespace scream { + +// Compute effects of large scale subsidence on T, q, u, and v. +KOKKOS_FUNCTION +void HommeDynamics:: +advance_iop_subsidence(const KT::MemberType& team, + const int nlevs, + const Real dt, + const Real ps, + const view_1d& pmid, + const view_1d& pint, + const view_1d& pdel, + const view_1d& omega, + const Workspace& workspace, + const view_1d& u, + const view_1d& v, + const view_1d& T, + const view_2d& Q) +{ + using ColOps = ColumnOps; + using C = physics::Constants; + constexpr Real Rair = C::Rair; + constexpr Real Cpair = C::Cpair; + + const auto n_q_tracers = Q.extent_int(0); + const auto nlev_packs = ekat::npack(nlevs); + + // Get some temporary views from WS + uview_1d omega_int, delta_u, delta_v, delta_T, tmp; + workspace.take_many_contiguous_unsafe<4>({"omega_int", "delta_u", "delta_v", "delta_T"}, + {&omega_int, &delta_u, &delta_v, &delta_T}); + const auto delta_Q_slot = workspace.take_macro_block("delta_Q", n_q_tracers); + uview_2d delta_Q(delta_Q_slot.data(), n_q_tracers, nlev_packs); + + auto s_pmid = ekat::scalarize(pmid); + auto s_omega = ekat::scalarize(omega); + auto s_delta_u = ekat::scalarize(delta_u); + auto s_delta_v = ekat::scalarize(delta_v); + auto s_delta_T = ekat::scalarize(delta_T); + auto s_delta_Q = ekat::scalarize(delta_Q); + auto s_omega_int = ekat::scalarize(omega_int); + + // Compute omega on the interface grid by using a weighted average in pressure + const int pack_begin = 1/Pack::n, pack_end = (nlevs-1)/Pack::n; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, pack_begin, pack_end+1), [&] (const int k){ + auto range_pack = ekat::range(k*Pack::n); + range_pack.set(range_pack<1, 1); + Pack pmid_k, pmid_km1, omega_k, omega_km1; + ekat::index_and_shift<-1>(s_pmid, range_pack, pmid_k, pmid_km1); + ekat::index_and_shift<-1>(s_omega, range_pack, omega_k, omega_km1); + + const auto weight = (pint(k) - pmid_km1)/(pmid_k - pmid_km1); + omega_int(k).set(range_pack>=1 and range_pack<=nlevs-1, + weight*omega_k + (1-weight)*omega_km1); + }); + omega_int(0)[0] = 0; + omega_int(nlevs/Pack::n)[nlevs%Pack::n] = 0; + + // Compute delta views for u, v, T, and Q (e.g., u(k+1) - u(k), k=0,...,nlevs-2) + ColOps::compute_midpoint_delta(team, nlevs-1, u, delta_u); + ColOps::compute_midpoint_delta(team, nlevs-1, v, delta_v); + ColOps::compute_midpoint_delta(team, nlevs-1, T, delta_T); + for (int iq=0; iq(k*Pack::n); + const auto at_top = range_pack==0; + const auto at_bot = range_pack==nlevs-1; + const auto at_mid = not at_top and not at_bot; + const bool any_at_top = at_top.any(); + const bool any_at_bot = at_bot.any(); + + // Get delta(k-1) packs. The range pack should not + // contain index 0 (so that we don't attempt to access + // k=-1 index) or index > nlevs-2 (since delta_* views + // are size nlevs-1). + auto range_pack_for_m1_shift = range_pack; + range_pack_for_m1_shift.set(range_pack<1, 1); + range_pack_for_m1_shift.set(range_pack>nlevs-2, nlevs-2); + Pack delta_u_k, delta_u_km1, + delta_v_k, delta_v_km1, + delta_T_k, delta_T_km1; + ekat::index_and_shift<-1>(s_delta_u, range_pack_for_m1_shift, delta_u_k, delta_u_km1); + ekat::index_and_shift<-1>(s_delta_v, range_pack_for_m1_shift, delta_v_k, delta_v_km1); + ekat::index_and_shift<-1>(s_delta_T, range_pack_for_m1_shift, delta_T_k, delta_T_km1); + + // At the top and bottom of the model, set the end points for + // delta_*_k and delta_*_km1 to be the first and last entries + // of delta_*, respectively. + if (any_at_top) { + delta_u_k.set(at_top, s_delta_u(0)); + delta_v_k.set(at_top, s_delta_v(0)); + delta_T_k.set(at_top, s_delta_T(0)); + } + if (any_at_bot) { + delta_u_km1.set(at_bot, s_delta_u(nlevs-2)); + delta_v_km1.set(at_bot, s_delta_v(nlevs-2)); + delta_T_km1.set(at_bot, s_delta_T(nlevs-2)); + } + + // Get omega_int(k+1) pack. The range pack should not + // contain index > nlevs-1 (since omega_int is size nlevs+1). + auto range_pack_for_p1_shift = range_pack; + range_pack_for_p1_shift.set(range_pack>nlevs-1, nlevs-1); + Pack omega_int_k, omega_int_kp1; + ekat::index_and_shift<1>(s_omega_int, range_pack, omega_int_k, omega_int_kp1); + + const auto fac = (dt/2)/pdel(k); + + // Update u + auto& u_k = u(k); + u_k.set(at_top, u_k - fac*omega_int_kp1*delta_u_k); + u_k.set(at_bot, u_k - fac*omega_int_k*delta_u_km1); + u_k.set(at_mid, u_k - fac*(omega_int_kp1*delta_u_k + omega_int_k*delta_u_km1)); + + // Update v + auto& v_k = v(k); + v_k.set(at_top, v_k - fac*omega_int_kp1*delta_v_k); + v_k.set(at_bot, v_k - fac*omega_int_k*delta_v_km1); + v_k.set(at_mid, v_k - fac*(omega_int_kp1*delta_v_k + omega_int_k*delta_v_km1)); + + // Before updating T, first scale using thermal + // expansion term due to LS vertical advection + auto& T_k = T(k); + T_k *= 1 + (dt*Rair/Cpair)*omega(k)/pmid(k); + + // Update T + T_k.set(at_top, T_k - fac*omega_int_kp1*delta_T_k); + T_k.set(at_bot, T_k - fac*omega_int_k*delta_T_km1); + T_k.set(at_mid, T_k - fac*(omega_int_kp1*delta_T_k + omega_int_k*delta_T_km1)); + + // Update Q + Pack delta_tracer_k, delta_tracer_km1; + for (int iq=0; iq(s_delta_tracer, range_pack_for_m1_shift, delta_tracer_k, delta_tracer_km1); + if (any_at_top) delta_tracer_k.set(at_top, s_delta_tracer(0)); + if (any_at_bot) delta_tracer_km1.set(at_bot, s_delta_tracer(nlevs-2)); + + auto& Q_k = Q(iq, k); + Q_k.set(at_top, Q_k - fac*omega_int_kp1*delta_tracer_k); + Q_k.set(at_bot, Q_k - fac*omega_int_k*delta_tracer_km1); + Q_k.set(at_mid, Q_k - fac*(omega_int_kp1*delta_tracer_k + omega_int_k*delta_tracer_km1)); + } + }); + + // Release WS views + workspace.release_macro_block(delta_Q_slot, n_q_tracers); + workspace.release_many_contiguous<4>({&omega_int, &delta_u, &delta_v, &delta_T}); +} + +// Apply large scale forcing for temperature and water vapor as provided by the IOP file +KOKKOS_FUNCTION +void HommeDynamics:: +advance_iop_forcing(const KT::MemberType& team, + const int nlevs, + const Real dt, + const view_1d& divT, + const view_1d& divq, + const view_1d& T, + const view_1d& qv) +{ + const auto nlev_packs = ekat::npack(nlevs); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&] (const int k) { + T(k) += dt*divT(k); + qv(k) += dt*divq(k); + }); +} + +void HommeDynamics:: +apply_iop_forcing(const Real dt) +{ + using ESU = ekat::ExeSpaceUtils; + + using EOS = Homme::EquationOfState; + using ElementOps = Homme::ElementOps; + using KV = Homme::KernelVariables; + + using ColOps = ColumnOps; + using C = physics::Constants; + constexpr Real Rair = C::Rair; + + // Homme objects + const auto& c = Homme::Context::singleton(); + const auto& hvcoord = c.get(); + const auto& params = c.get(); + + // Dimensions + constexpr int NGP = HOMMEXX_NP; + constexpr int NLEV = HOMMEXX_NUM_LEV; + constexpr int NLEVI = HOMMEXX_NUM_LEV_P; + const auto nelem = m_dyn_grid->get_num_local_dofs()/(NGP*NGP); + const auto total_levels = m_dyn_grid->get_num_vertical_levels(); + const auto qsize = params.qsize; + + // Sanity checks since we will be switching between ekat::Pack + // and Homme::Scalar view types + EKAT_ASSERT_MSG(NLEV == ekat::npack(total_levels), + "Error! Dimension for vectorized Homme levels does not match level dimension " + "of the packed views used here. Check that Pack typedef is using a pack size " + "consistent with Homme's vector size.\n"); + EKAT_ASSERT_MSG(NLEVI == ekat::npack(total_levels+1), + "Error! Dimension for vectorized Homme levels does not match level dimension " + "of the packed views used here. Check that Pack typedef is using a pack size " + "consistent with Homme's vector size.\n"); + + // Hybrid coord values + const auto ps0 = hvcoord.ps0; + const auto hyam = m_dyn_grid->get_geometry_data("hyam").get_view(); + const auto hybm = m_dyn_grid->get_geometry_data("hybm").get_view(); + const auto hyai = m_dyn_grid->get_geometry_data("hyai").get_view(); + const auto hybi = m_dyn_grid->get_geometry_data("hybi").get_view(); + + // Homme element states and EOS/EO classes + auto ps_dyn = get_internal_field("ps_dyn").get_view(); + auto dp3d_dyn = get_internal_field("dp3d_dyn").get_view(); + auto vtheta_dp_dyn = get_internal_field("vtheta_dp_dyn").get_view(); + auto phi_int_dyn = get_internal_field("phi_int_dyn").get_view(); + auto v_dyn = get_internal_field("v_dyn").get_view(); + auto Q_dyn = m_helper_fields.at("Q_dyn").get_view(); + auto Qdp_dyn = get_internal_field("Qdp_dyn").get_view(); + + EOS eos; + eos.init(params.theta_hydrostatic_mode, hvcoord); + + ElementOps elem_ops; + elem_ops.init(hvcoord); + const bool use_moisture = (params.moisture == Homme::MoistDry::MOIST); + + // Load data from IOP files, if necessary + m_iop->read_iop_file_data(timestamp()); + + // Define local IOP param values and views + const auto iop_dosubsidence = m_iop->get_params().get("iop_dosubsidence"); + const auto use_3d_forcing = m_iop->get_params().get("use_3d_forcing"); + const auto omega = m_iop->get_iop_field("omega").get_view(); + const auto divT = use_3d_forcing ? m_iop->get_iop_field("divT3d").get_view() + : m_iop->get_iop_field("divT").get_view(); + const auto divq = use_3d_forcing ? m_iop->get_iop_field("divq3d").get_view() + : m_iop->get_iop_field("divq").get_view(); + + // Team policy and workspace manager for both homme and scream + // related loops. We need separate policies since hommexx functions used here + // assume they are called inside nested loops for elements and Gaussian points, + // whereas EAMxx function we use expects a single level of parallelism + // for elements and Guassian points. + // TODO: scream::ColumnOps functions could take an arbitary loop boundary + // (TeamVectorRange, TeamThreadRange, ThreadVectorRange) so that + // all 3 kernel launches here could be combined. + const auto policy_homme = ESU::get_default_team_policy(nelem, NLEV); + const auto policy_eamxx = ESU::get_default_team_policy(nelem*NGP*NGP, NLEV); + + // TODO: Create a memory buffer for this class + // and add the below WSM and views + WorkspaceMgr eamxx_wsm(NLEVI, 7+qsize, policy_eamxx); + WorkspaceMgr homme_wsm(NLEV, 32, policy_homme); + view_Nd + temperature("temperature", nelem, NGP, NGP, NLEV), + exner("exner", nelem, NGP, NGP, NLEV); + + // Preprocess some homme states to get temperature and exner + Kokkos::parallel_for("compute_t_and_exner", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { + KV kv(team); + const int ie = team.league_rank(); + + // Get temp views from workspace + auto ws = homme_wsm.get_workspace(team); + auto pnh_slot = ws.take_macro_block("pnh" , NGP*NGP); + auto rstar_slot = ws.take_macro_block("rstar", NGP*NGP); + uview_2d + pnh (reinterpret_cast(pnh_slot.data()), NGP*NGP, NLEV), + rstar(reinterpret_cast(rstar_slot.data()), NGP*NGP, NLEV); + + Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { + const int igp = idx/NGP; + const int jgp = idx%NGP; + + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto phi_int_i = ekat::subview(phi_int_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto pnh_i = ekat::subview(pnh, idx); + auto rstar_i = ekat::subview(rstar, idx); + auto exner_i = ekat::subview(exner, ie, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + + // Reinterperate into views of Homme::Scalar for calling Hommexx function. + Homme::ExecViewUnmanaged dp3d_scalar(reinterpret_cast(dp3d_i.data()), NLEV); + Homme::ExecViewUnmanaged vtheta_dp_scalar(reinterpret_cast(vtheta_dp_i.data()), NLEV); + Homme::ExecViewUnmanaged phi_int_scalar(reinterpret_cast(phi_int_i.data()), NLEVI); + Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); + Homme::ExecViewUnmanaged pnh_scalar(reinterpret_cast(pnh_i.data()), NLEV); + Homme::ExecViewUnmanaged exner_scalar(reinterpret_cast(exner_i.data()), NLEV); + Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); + Homme::ExecViewUnmanaged temperature_scalar(reinterpret_cast(temperature_i.data()), NLEV); + + // Compute exner from EOS + if (params.theta_hydrostatic_mode) { + auto hydro_p_int = ws.take("hydro_p_int"); + Homme::ExecViewUnmanaged hydro_p_int_scalar(reinterpret_cast(hydro_p_int.data()), NLEVI); + elem_ops.compute_hydrostatic_p(kv, dp3d_scalar, hydro_p_int_scalar, pnh_scalar); + eos.compute_exner(kv, pnh_scalar, exner_scalar); + ws.release(hydro_p_int); + } else { + eos.compute_pnh_and_exner(kv, vtheta_dp_scalar, phi_int_scalar, pnh_scalar, exner_scalar); + } + + // Get the temperature from dynamics states + elem_ops.get_temperature(kv, eos, use_moisture, dp3d_scalar, exner_scalar, vtheta_dp_scalar, qv_scalar, rstar_scalar, temperature_scalar); + }); + + // Release WS views + ws.release_macro_block(rstar_slot, NGP*NGP); + ws.release_macro_block(pnh_slot, NGP*NGP); + }); + Kokkos::fence(); + + // Apply IOP forcing + Kokkos::parallel_for("apply_iop_forcing", policy_eamxx, KOKKOS_LAMBDA (const KT::MemberType& team) { + const int ie = team.league_rank()/(NGP*NGP); + const int igp = (team.league_rank()/NGP)%NGP; + const int jgp = team.league_rank()%NGP; + + // Get temp views from workspace + auto ws = eamxx_wsm.get_workspace(team); + uview_1d pmid, pint, pdel; + ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, + {&pmid, &pint, &pdel}); + + auto ps_i = ps_dyn(ie, igp, jgp); + auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); + auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + auto s_pint = ekat::scalarize(pint); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels+1), [&](const int& k) { + s_pint(k) = hyai(k)*ps0 + hybi(k)*ps_i; + if (k < total_levels) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + } + }); + team.team_barrier(); + ColOps::compute_midpoint_delta(team, total_levels, pint, pdel); + team.team_barrier(); + + if (iop_dosubsidence) { + // Compute subsidence due to large-scale forcing + advance_iop_subsidence(team, total_levels, dt, ps_i, pmid, pint, pdel, omega, ws, u_i, v_i, temperature_i, Q_i); + } + + // Update T and qv according to large scale forcing as specified in IOP file. + advance_iop_forcing(team, total_levels, dt, divT, divq, temperature_i, qv_i); + + // Release WS views + ws.release_many_contiguous<3>({&pmid, &pint, &pdel}); + }); + Kokkos::fence(); + + // Postprocess homme states Qdp and vtheta_dp + Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { + KV kv(team); + const int ie = team.league_rank(); + + // Get temp views from workspace + auto ws = homme_wsm.get_workspace(team); + auto rstar_slot = ws.take_macro_block("rstar", NGP*NGP); + uview_2d + rstar(reinterpret_cast(rstar_slot.data()), NGP*NGP, NLEV); + + Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { + const int igp = idx/NGP; + const int jgp = idx%NGP; + + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + auto rstar_i = ekat::subview(rstar, idx); + auto exner_i = ekat::subview(exner, ie, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + + // Reinterperate into views of Homme::Scalar for calling Hommexx function. + Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); + Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); + + // Compute Qdp from updated Q + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV*qsize), [&] (const int k) { + const int ilev = k/qsize; + const int q = k%qsize; + + Qdp_i(q, ilev) = Q_i(q, ilev)*dp3d_i(ilev); + // For BFB on restarts, Q needs to be updated after we compute Qdp + Q_i(q, ilev) = Qdp_i(q, ilev)/dp3d_i(ilev); + }); + + // Convert updated temperature back to potential temperature + elem_ops.get_R_star(kv, use_moisture, qv_scalar, rstar_scalar); + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { + vtheta_dp_i(k) = temperature_i(k)*rstar_i(k)*dp3d_i(k)/(Rair*exner_i(k)); + }); + }); + + // Release WS views + ws.release_macro_block(rstar_slot, NGP*NGP); + }); +} + +} // namespace scream diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp index c535829fbfab..f33c453c117c 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp @@ -423,6 +423,11 @@ void HommeDynamics::initialize_impl (const RunType run_type) if (run_type==RunType::Initial) { initialize_homme_state (); } else { + if (m_iop) { + // We need to reload IOP data after restarting + m_iop->read_iop_file_data(timestamp()); + } + restart_homme_state (); } @@ -547,16 +552,13 @@ void HommeDynamics::homme_pre_process (const double dt) { // T and uv tendencies are backed out on the ref grid. // Homme takes care of turning the FT tendency into a tendency for VTheta_dp. - constexpr int N = sizeof(Homme::Scalar) / sizeof(Real); - using Pack = RPack; - using namespace Homme; const auto& c = Context::singleton(); const auto& params = c.get(); const int ncols = m_phys_grid->get_num_local_dofs(); const int nlevs = m_phys_grid->get_num_vertical_levels(); - const int npacks = ekat::PackInfo::num_packs(nlevs); + const int npacks = ekat::npack(nlevs); const auto& pgn = m_phys_grid->name(); @@ -664,6 +666,10 @@ void HommeDynamics::homme_post_process (const double dt) { get_internal_field("w_int_dyn").get_header().get_alloc_properties().reset_subview_idx(tl.n0); } + if (m_iop) { + apply_iop_forcing(dt); + } + if (fv_phys_active()) { fv_phys_post_process(); // Apply Rayleigh friction to update temperature and horiz_winds @@ -675,8 +681,6 @@ void HommeDynamics::homme_post_process (const double dt) { // Remap outputs to ref grid m_d2p_remapper->remap(true); - constexpr int N = HOMMEXX_PACK_SIZE; - using Pack = RPack; using ColOps = ColumnOps; using PF = PhysicsFunctions; @@ -697,7 +701,7 @@ void HommeDynamics::homme_post_process (const double dt) { const auto ncols = m_phys_grid->get_num_local_dofs(); const auto nlevs = m_phys_grid->get_num_vertical_levels(); - const auto npacks= ekat::PackInfo::num_packs(nlevs); + const auto npacks= ekat::npack(nlevs); using ESU = ekat::ExeSpaceUtils; const auto policy = ESU::get_thread_range_parallel_scan_team_policy(ncols,npacks); @@ -797,14 +801,13 @@ void HommeDynamics::init_homme_views () { constexpr int QSZ = HOMMEXX_QSIZE_D; constexpr int NVL = HOMMEXX_NUM_LEV; constexpr int NVLI = HOMMEXX_NUM_LEV_P; - constexpr int N = HOMMEXX_PACK_SIZE; const int nelem = m_dyn_grid->get_num_local_dofs()/(NGP*NGP); const int qsize = tracers.num_tracers(); const auto ncols = m_phys_grid->get_num_local_dofs(); const auto nlevs = m_phys_grid->get_num_vertical_levels(); - const auto npacks= ekat::PackInfo::num_packs(nlevs); + const auto npacks= ekat::npack(nlevs); using ESU = ekat::ExeSpaceUtils; const auto default_policy = ESU::get_default_team_policy(ncols,npacks); @@ -931,8 +934,6 @@ void HommeDynamics::restart_homme_state () { " - field name: " + f.get_header().get_identifier().name() + "\n"); } - constexpr int N = HOMMEXX_PACK_SIZE; - using Pack = RPack; using ESU = ekat::ExeSpaceUtils; using PF = PhysicsFunctions; @@ -957,7 +958,7 @@ void HommeDynamics::restart_homme_state () { const int nlevs = m_phys_grid->get_num_vertical_levels(); const int ncols = m_phys_grid->get_num_local_dofs(); const int nelem = m_dyn_grid->get_num_local_dofs() / (NGP*NGP); - const int npacks = ekat::PackInfo::num_packs(nlevs); + const int npacks = ekat::npack(nlevs); const int qsize = params.qsize; // NOTE: when restarting stuff like T_prev, and other "previous steps" quantities that HommeDynamics @@ -1060,12 +1061,10 @@ void HommeDynamics::restart_homme_state () { void HommeDynamics::initialize_homme_state () { // Some types - using Pack = RPack; using ColOps = ColumnOps; using PF = PhysicsFunctions; using ESU = ekat::ExeSpaceUtils; using EOS = Homme::EquationOfState; - using WS = ekat::WorkspaceManager; const auto& rgn = m_cgll_grid->name(); @@ -1080,8 +1079,8 @@ void HommeDynamics::initialize_homme_state () { constexpr int NGP = HOMMEXX_NP; const int nelem = m_dyn_grid->get_num_local_dofs()/(NGP*NGP); const int qsize = params.qsize; - const int npacks_mid = ekat::PackInfo::num_packs(nlevs); - const int npacks_int = ekat::PackInfo::num_packs(nlevs+1); + const int npacks_mid = ekat::npack(nlevs); + const int npacks_int = ekat::npack(nlevs+1); // Bootstrap dp on phys grid, and let the ic remapper transfer dp on dyn grid // NOTE: HybridVCoord already stores hyai and hybi deltas as packed views, @@ -1138,7 +1137,7 @@ void HommeDynamics::initialize_homme_state () { const auto hyai0 = hvcoord.hybrid_ai0; // Need two temporaries, for pi_mid and pi_int const auto policy = ESU::get_thread_range_parallel_scan_team_policy(nelem*NGP*NGP,npacks_mid); - WS wsm(npacks_int,2,policy); + WorkspaceMgr wsm(npacks_int,2,policy); Kokkos::parallel_for(policy, KOKKOS_LAMBDA (const KT::MemberType& team) { const int ie = team.league_rank() / (NGP*NGP); const int igp = (team.league_rank() / NGP) % NGP; @@ -1149,8 +1148,8 @@ void HommeDynamics::initialize_homme_state () { // Compute p_mid auto ws = wsm.get_workspace(team); - ekat::Unmanaged > p_int, p_mid; - ws.template take_many_and_reset<2>({"p_int", "p_mid"}, {&p_int, &p_mid}); + ekat::Unmanaged > p_int, p_mid; + ws.take_many_and_reset<2>({"p_int", "p_mid"}, {&p_int, &p_mid}); ColOps::column_scan(team,nlevs,dp,p_int,ps0*hyai0); team.team_barrier(); @@ -1281,12 +1280,11 @@ copy_dyn_states_to_all_timelevels () { // TODO item to consolidate how we update the pressure during initialization and run, but // for now we have two locations where we do this. void HommeDynamics::update_pressure(const std::shared_ptr& grid) { - using Pack = RPack; using ColOps = ColumnOps; const auto ncols = grid->get_num_local_dofs(); const auto nlevs = grid->get_num_vertical_levels(); - const auto npacks= ekat::PackInfo::num_packs(nlevs); + const auto npacks= ekat::npack(nlevs); const auto& c = Homme::Context::singleton(); const auto& hvcoord = c.get(); diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp index 6f6da5321484..2749d268c9a7 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp @@ -6,12 +6,12 @@ #include "ekat/ekat_parameter_list.hpp" #include "ekat/ekat_pack.hpp" +#include "ekat/ekat_workspace.hpp" #include namespace scream { - /* * The class responsible to handle the atmosphere dynamics * @@ -23,6 +23,25 @@ namespace scream */ class HommeDynamics : public AtmosphereProcess { + // Define some types needed by class + using Pack = ekat::Pack; + using IntPack = ekat::Pack; + + using KT = KokkosTypes; + template + using view_1d = typename KT::template view_1d; + template + using view_2d = typename KT::template view_2d; + template + using view_Nd = typename KT::template view_ND; + template + using uview_1d = ekat::Unmanaged>; + template + using uview_2d = ekat::Unmanaged>; + + using WorkspaceMgr = ekat::WorkspaceManager; + using Workspace = WorkspaceMgr::Workspace; + public: // Constructor(s) and Destructor @@ -89,6 +108,33 @@ class HommeDynamics : public AtmosphereProcess void rayleigh_friction_init (); void rayleigh_friction_apply (const Real dt) const; + // IOP functions + void apply_iop_forcing(const Real dt); + + KOKKOS_FUNCTION + static void advance_iop_subsidence(const KT::MemberType& team, + const int nlevs, + const Real dt, + const Real ps, + const view_1d& pmid, + const view_1d& pint, + const view_1d& pdel, + const view_1d& omega, + const Workspace& workspace, + const view_1d& u, + const view_1d& v, + const view_1d& T, + const view_2d& Q); + + KOKKOS_FUNCTION + static void advance_iop_forcing(const KT::MemberType& team, + const int nlevs, + const Real dt, + const view_1d& divT, + const view_1d& divq, + const view_1d& T, + const view_1d& qv); + public: // Fast boolean function returning whether Physics PGN is being used. bool fv_phys_active() const; @@ -134,15 +180,8 @@ class HommeDynamics : public AtmosphereProcess std::shared_ptr m_phys_grid; // Column parameterizations grid std::shared_ptr m_cgll_grid; // Unique CGLL - template - using RPack = ekat::Pack; - - using KT = KokkosTypes; - template - using view_1d = typename KT::template view_1d; - // Rayleigh friction decay rate profile - view_1d> m_otau; + view_1d m_otau; // Rayleigh friction paramaters int m_rayk0; // Vertical level at which rayleigh friction term is centered. diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_rayleigh_friction.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_rayleigh_friction.cpp index 120dce865db6..3b4cbdb711e6 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_rayleigh_friction.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_rayleigh_friction.cpp @@ -22,12 +22,9 @@ void HommeDynamics::rayleigh_friction_init() // to 0-based for computations m_rayk0 -= 1; - constexpr int N = SCREAM_PACK_SIZE; - using Pack = RPack; - // Calculate decay rate profile, otau. const int nlevs = m_dyn_grid->get_num_vertical_levels(); - const auto npacks= ekat::PackInfo::num_packs(nlevs); + const auto npacks= ekat::npack(nlevs); m_otau = decltype(m_otau)("otau", npacks); // Local paramters @@ -48,7 +45,7 @@ void HommeDynamics::rayleigh_friction_init() Kokkos::parallel_for(KT::RangePolicy(0, npacks), KOKKOS_LAMBDA (const int ilev) { - const auto range_pack = ekat::range(ilev*N); + const auto range_pack = ekat::range(ilev*Pack::n); const Pack x = (rayk0 - range_pack)/krange; otau(ilev) = otau0*(1.0 + ekat::tanh(x))/2.0; }); @@ -56,9 +53,7 @@ void HommeDynamics::rayleigh_friction_init() void HommeDynamics::rayleigh_friction_apply(const Real dt) const { - constexpr int N = HOMMEXX_PACK_SIZE; using PF = PhysicsFunctions; - using Pack = RPack; using ESU = ekat::ExeSpaceUtils; // If m_raytau0==0, then no Rayleigh friction is applied. Return. @@ -66,7 +61,7 @@ void HommeDynamics::rayleigh_friction_apply(const Real dt) const const auto ncols = m_phys_grid->get_num_local_dofs(); const auto nlevs = m_phys_grid->get_num_vertical_levels(); - const auto npacks= ekat::PackInfo::num_packs(nlevs); + const auto npacks= ekat::npack(nlevs); const auto horiz_winds_view = get_field_out("horiz_winds").get_view(); const auto T_mid_view = get_field_out("T_mid").get_view(); diff --git a/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp b/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp index 08dfe0ccb831..23b47a59aa20 100644 --- a/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp +++ b/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp @@ -267,6 +267,7 @@ build_physics_grid (const ci_string& type, const ci_string& rebalance) { for (auto f : {hyai, hybi, hyam, hybm}) { auto f_d = get_grid("Dynamics")->get_geometry_data(f.name()); f.deep_copy(f_d); + f.sync_to_host(); } } @@ -284,7 +285,7 @@ build_physics_grid (const ci_string& type, const ci_string& rebalance) { void HommeGridsManager:: initialize_vertical_coordinates (const nonconstgrid_ptr_type& dyn_grid) { - using view_1d_host = AtmosphereInput::view_1d_host; + using view_1d_host = AtmosphereInput::view_1d_host; using vos_t = std::vector; using namespace ShortFieldTagsNames; @@ -337,7 +338,7 @@ initialize_vertical_coordinates (const nonconstgrid_ptr_type& dyn_grid) { hybi.sync_to_dev(); hyam.sync_to_dev(); hybm.sync_to_dev(); - + // Pass host views data to hvcoord init function const auto ps0 = Homme::PhysicalConstants::p0; diff --git a/components/eamxx/src/dynamics/homme/tests/CMakeLists.txt b/components/eamxx/src/dynamics/homme/tests/CMakeLists.txt index 8b500fa239bc..cc09289ff564 100644 --- a/components/eamxx/src/dynamics/homme/tests/CMakeLists.txt +++ b/components/eamxx/src/dynamics/homme/tests/CMakeLists.txt @@ -1,5 +1,5 @@ # NOTE: if you have baseline-type tests, add the subdirectory OUTSIDE the following if statement -if (NOT SCREAM_BASELINES_ONLY) +if (NOT SCREAM_ONLY_GENERATE_BASELINES) include (ScreamUtils) # Get or create the dynamics lib diff --git a/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp b/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp index 0819dd87fd39..a2534c049c26 100644 --- a/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp +++ b/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp @@ -151,12 +151,12 @@ TEST_CASE("dyn_grid_io") ekat::ParameterList out_params; out_params.set("Averaging Type","Instant"); out_params.set("filename_prefix","dyn_grid_io"); + out_params.set("MPI Ranks in Filename",true); out_params.sublist("Fields").sublist("Dynamics").set>("Field Names",fnames); out_params.sublist("Fields").sublist("Dynamics").set("IO Grid Name","Physics GLL"); out_params.sublist("output_control").set("Frequency",1); out_params.sublist("output_control").set("frequency_units","nsteps"); - out_params.sublist("output_control").set("MPI Ranks in Filename",true); out_params.set("Floating Point Precision","real"); OutputManager output; diff --git a/components/eamxx/src/physics/cosp/cosp_c2f.F90 b/components/eamxx/src/physics/cosp/cosp_c2f.F90 index 351207ae336d..957c546c4aa7 100644 --- a/components/eamxx/src/physics/cosp/cosp_c2f.F90 +++ b/components/eamxx/src/physics/cosp/cosp_c2f.F90 @@ -292,23 +292,30 @@ subroutine cosp_c2f_run(npoints, ncolumns, nlevels, ntau, nctp, & nptsperit = npoints - tca(:npoints,:nlevels) = cldfrac(:npoints,:nlevels) - cca(:npoints,:nlevels) = 0 - mr_lsliq(:npoints,:nlevels) = 0 - mr_ccliq(:npoints,:nlevels) = 0 - mr_lsice(:npoints,:nlevels) = 0 - mr_ccice(:npoints,:nlevels) = 0 - dtau_c(:npoints,:nlevels) = 0 - dtau_s(:npoints,:nlevels) = dtau067(:npoints,:nlevels) - dem_c (:npoints,:nlevels) = 0 - dem_s (:npoints,:nlevels) = 1._wp - exp(-dtau105(:npoints,:nlevels)) - mr_lsrain(:npoints,:nlevels) = 0 - mr_ccrain(:npoints,:nlevels) = 0 - mr_lssnow(:npoints,:nlevels) = 0 - mr_lssnow(:npoints,:nlevels) = 0 - mr_ccsnow(:npoints,:nlevels) = 0 - mr_lsgrpl(:npoints,:nlevels) = 0 - reff = 0 ! FIXME + ! In-cloud values are assumed. If ncolumns = 1, then convert in-cloud values to gridbox + if (ncolumns == 1) then + tca(:npoints,:nlevels) = cldfrac(:npoints,:nlevels) + dtau_s(:npoints,:nlevels) = cldfrac(:npoints,:nlevels) * dtau067(:npoints,:nlevels) + dem_s (:npoints,:nlevels) = 1._wp - exp(-cldfrac(:npoints,:nlevels) * dtau105(:npoints,:nlevels)) + else + tca(:npoints,:nlevels) = cldfrac(:npoints,:nlevels) + dtau_s(:npoints,:nlevels) = dtau067(:npoints,:nlevels) + dem_s (:npoints,:nlevels) = 1._wp - exp(-dtau105(:npoints,:nlevels)) + end if + ! Fields not currently being used (will need to be revisted when turning on additional simulators) + cca(:npoints,:nlevels) = 0 ! Cloud fraction of convective clouds; not present or used in our model + dtau_c(:npoints,:nlevels) = 0 ! Optical depth of convective clouds; not present or used in our model + dem_c (:npoints,:nlevels) = 0 ! Emissivity of convective clouds; not present or used in our model + mr_lsliq(:npoints,:nlevels) = 0 ! Mixing ratio of cloud liquid; will be needed for radar/lidar + mr_ccliq(:npoints,:nlevels) = 0 ! Mixing ratio of cloud liquid for convective clouds; not present or used in our model + mr_lsice(:npoints,:nlevels) = 0 ! Mixing ratio of cloud ice; will be needed for radar/lidar + mr_ccice(:npoints,:nlevels) = 0 ! Mixing ratio of cloud ice for convective clouds; not present or used in our model + mr_lsrain(:npoints,:nlevels) = 0 ! Mixing ratio of rain; will be needed for radar/lidar + mr_ccrain(:npoints,:nlevels) = 0 ! Mixing ratio of rain for convective clouds; not present or used in our model + mr_lssnow(:npoints,:nlevels) = 0 ! Mixing ratio of snow; will be needed for radar/lidar + mr_ccsnow(:npoints,:nlevels) = 0 ! Mixing ratio of snow for convective clouds; will be needed for radar/lidar + mr_lsgrpl(:npoints,:nlevels) = 0 ! Mixing ratio of graupel; will be needed for radar/lidar + reff = 0 ! Effective radii; will be needed for MODIS start_idx = 1 end_idx = npoints diff --git a/components/eamxx/src/physics/cosp/eamxx_cosp.cpp b/components/eamxx/src/physics/cosp/eamxx_cosp.cpp index 7aaa4be3b2b0..478fce989276 100644 --- a/components/eamxx/src/physics/cosp/eamxx_cosp.cpp +++ b/components/eamxx/src/physics/cosp/eamxx_cosp.cpp @@ -69,7 +69,7 @@ void Cosp::set_grids(const std::shared_ptr grids_manager) add_field("qv", scalar3d_layout_mid, Q, grid_name, "tracers"); add_field("qc", scalar3d_layout_mid, Q, grid_name, "tracers"); add_field("qi", scalar3d_layout_mid, Q, grid_name, "tracers"); - add_field("cldfrac_tot_for_analysis", scalar3d_layout_mid, nondim, grid_name); + add_field("cldfrac_rad", scalar3d_layout_mid, nondim, grid_name); // Optical properties, should be computed in radiation interface add_field("dtau067", scalar3d_layout_mid, nondim, grid_name); // 0.67 micron optical depth add_field("dtau105", scalar3d_layout_mid, nondim, grid_name); // 10.5 micron optical depth @@ -140,7 +140,7 @@ void Cosp::run_impl (const double dt) get_field_in("T_mid").sync_to_host(); get_field_in("p_mid").sync_to_host(); get_field_in("p_int").sync_to_host(); - get_field_in("cldfrac_tot_for_analysis").sync_to_host(); + get_field_in("cldfrac_rad").sync_to_host(); get_field_in("eff_radius_qc").sync_to_host(); get_field_in("eff_radius_qi").sync_to_host(); get_field_in("dtau067").sync_to_host(); @@ -154,7 +154,7 @@ void Cosp::run_impl (const double dt) auto T_mid = get_field_in("T_mid").get_view(); auto p_mid = get_field_in("p_mid").get_view(); auto p_int = get_field_in("p_int").get_view(); - auto cldfrac = get_field_in("cldfrac_tot_for_analysis").get_view(); + auto cldfrac = get_field_in("cldfrac_rad").get_view(); auto reff_qc = get_field_in("eff_radius_qc").get_view(); auto reff_qi = get_field_in("eff_radius_qi").get_view(); auto dtau067 = get_field_in("dtau067").get_view(); diff --git a/components/eamxx/src/physics/mam/CMakeLists.txt b/components/eamxx/src/physics/mam/CMakeLists.txt index a4e095bf5f2a..53eb4049f11b 100644 --- a/components/eamxx/src/physics/mam/CMakeLists.txt +++ b/components/eamxx/src/physics/mam/CMakeLists.txt @@ -1,11 +1,53 @@ +if (SCREAM_CIME_BUILD) + # PROJECT_SOURCE_DIR is SCREAM_ROOT/components + set(EXTERNALS_SOURCE_DIR "${PROJECT_SOURCE_DIR}/../externals") +else() + # PROJECT_SOURCE_DIR is SCREAM_ROOT/components/eamxx + set(EXTERNALS_SOURCE_DIR "${PROJECT_SOURCE_DIR}/../../externals") +endif() + +# Haero (Kokkos-based aerosol interface library) +set(HAERO_ENABLE_GPU ${EAMXX_ENABLE_GPU}) +set(HAERO_ENABLE_MPI ON) +set(HAERO_ENABLE_TESTS OFF) +if (SCREAM_DOUBLE_PRECISION) + set(HAERO_PRECISION "double") +else() + set(HAERO_PRECISION "single") +endif() +list(APPEND CMAKE_MODULE_PATH + ${EXTERNALS_SOURCE_DIR}/haero/cmake +) +add_subdirectory(${EXTERNALS_SOURCE_DIR}/haero ${CMAKE_BINARY_DIR}/externals/haero) + +# mam4xx (C++ port of MAM4) +set(ENABLE_TESTS OFF) +set(ENABLE_SKYWALKER OFF) +set(NUM_VERTICAL_LEVELS ${SCREAM_NUM_VERTICAL_LEV}) +set(HAERO_C_STANDARD ${CMAKE_C_STANDARD}) +set(HAERO_C_COMPILER ${CMAKE_C_COMPILER}) +set(HAERO_CXX_STANDARD ${CMAKE_CXX_STANDARD}) +set(HAERO_CXX_COMPILER ${CMAKE_CXX_COMPILER}) +list(APPEND CMAKE_MODULE_PATH + ${EXTERNALS_SOURCE_DIR}/mam4xx/cmake +) +#include_directories( + # ${EXTERNALS_SOURCE_DIR}/ekat/src + # ${EXTERNALS_SOURCE_DIR}/haero + # ${CMAKE_BINARY_DIR}/externals/ekat/src + # ${CMAKE_BINARY_DIR}/externals/haero + #) +add_subdirectory(${EXTERNALS_SOURCE_DIR}/mam4xx ${CMAKE_BINARY_DIR}/externals/mam4xx) + +# EAMxx mam4xx-based atmospheric processes add_library(mam eamxx_mam_microphysics_process_interface.cpp eamxx_mam_optics_process_interface.cpp) target_compile_definitions(mam PUBLIC EAMXX_HAS_MAM) -add_dependencies(mam mam4xx_proj) +add_dependencies(mam mam4xx) target_include_directories(mam PUBLIC - ${PROJECT_BINARY_DIR}/externals/haero/include - ${PROJECT_BINARY_DIR}/externals/mam4xx/include + ${EXTERNALS_SOURCE_DIR}/haero + ${EXTERNALS_SOURCE_DIR}/mam4xx/src ) target_link_libraries(mam PUBLIC physics_share scream_share mam4xx haero) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 1a89e1767522..7bfe86ef713f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -1,11 +1,18 @@ #include +#include #include #include #include "scream_config.h" // for SCREAM_CIME_BUILD +#include // for serial NetCDF file reads on MPI root #include +// NOTE: see the impl/ directory for the contents of the impl namespace +#include "impl/compute_o3_column_density.cpp" +#include "impl/compute_water_content.cpp" +#include "impl/gas_phase_chemistry.cpp" + namespace scream { @@ -13,7 +20,8 @@ MAMMicrophysics::MAMMicrophysics( const ekat::Comm& comm, const ekat::ParameterList& params) : AtmosphereProcess(comm, params), - aero_config_(), nucleation_(new mam4::NucleationProcess(aero_config_)) { + aero_config_() { + configure(params); } AtmosphereProcessType MAMMicrophysics::type() const { @@ -24,33 +32,94 @@ std::string MAMMicrophysics::name() const { return "mam4_micro"; } +namespace { + +void set_data_file(const char *name, const char *path, char location[MAX_FILENAME_LEN]) { + EKAT_REQUIRE_MSG(strlen(SCREAM_DATA_DIR) + strlen(path) < MAX_FILENAME_LEN, + "Error! " << name << " path is too long (must be < " << MAX_FILENAME_LEN << " characters)"); + sprintf(location, "%s/%s", SCREAM_DATA_DIR, path); +} + +} + +#define set_file_location(data_file, path) set_data_file(#data_file, path, data_file) + +void MAMMicrophysics::set_defaults_() { + config_.amicphys.do_cond = true; + config_.amicphys.do_rename = true; + config_.amicphys.do_newnuc = true; + config_.amicphys.do_coag = true; + + config_.amicphys.nucleation = {}; + config_.amicphys.nucleation.dens_so4a_host = 1770.0; + config_.amicphys.nucleation.mw_so4a_host = 115.0; + config_.amicphys.nucleation.newnuc_method_user_choice = 2; + config_.amicphys.nucleation.pbl_nuc_wang2008_user_choice = 1; + config_.amicphys.nucleation.adjust_factor_pbl_ratenucl = 1.0; + config_.amicphys.nucleation.accom_coef_h2so4 = 1.0; + config_.amicphys.nucleation.newnuc_adjust_factor_dnaitdt = 1.0; + + // these parameters guide the coupling between parameterizations + // NOTE: mam4xx was ported with these parameters fixed, so it's probably not + // NOTE: safe to change these without code modifications. + config_.amicphys.gaexch_h2so4_uptake_optaa = 2; + config_.amicphys.newnuc_h2so4_conc_optaa = 2; + + //=========================================================== + // default data file locations (relative to SCREAM_DATA_DIR) + //=========================================================== + + // many of these paths were extracted from + // e3smv2/bld/namelist_files/namelist_defaults_eam.xml + + // photolysis + set_file_location(config_.photolysis.rsf_file, "../waccm/phot/RSF_GT200nm_v3.0_c080811.nc"); + set_file_location(config_.photolysis.xs_long_file, "../waccm/phot/temp_prs_GT200nm_JPL10_c130206.nc"); + + // stratospheric chemistry + set_file_location(config_.linoz.chlorine_loading_file, "../cam/chem/trop_mozart/ub/Linoz_Chlorine_Loading_CMIP6_0003-2017_c20171114.nc"); + + // dry deposition + set_file_location(config_.drydep.srf_file, "../cam/chem/trop_mam/atmsrf_ne4pg2_200527.nc"); +} + +void MAMMicrophysics::configure(const ekat::ParameterList& params) { + set_defaults_(); + // FIXME: implement "namelist" parsing +} + void MAMMicrophysics::set_grids(const std::shared_ptr grids_manager) { using namespace ekat::units; - // The units of mixing ratio q are technically non-dimensional. - // Nevertheless, for output reasons, we like to see 'kg/kg'. - auto q_unit = kg/kg; + auto q_unit = kg/kg; // mass mixing ratios [kg stuff / kg air] q_unit.set_string("kg/kg"); - auto n_unit = 1/kg; + auto n_unit = 1/kg; // number mixing ratios [# / kg air] n_unit.set_string("#/kg"); Units nondim(0,0,0,0,0,0,0); + const auto m2 = m*m; + const auto s2 = s*s; grid_ = grids_manager->get_grid("Physics"); const auto& grid_name = grid_->name(); - ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank - nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + ncol_ = grid_->get_num_local_dofs(); // number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // number of levels per column - // Define the different field layouts that will be used for this process + // get column geometry and locations + col_areas_ = grid_->get_geometry_data("area").get_view(); + col_latitudes_ = grid_->get_geometry_data("lat").get_view(); + col_longitudes_ = grid_->get_geometry_data("lon").get_view(); + + // define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; - // Layout for 2D (1d horiz X 1d vertical) variable + // layout for 2D (1d horiz X 1d vertical) variable FieldLayout scalar2d_layout_col{ {COL}, {ncol_} }; - // Layout for 3D (2d horiz X 1d vertical) variables + // layout for 3D (2d horiz X 1d vertical) variables FieldLayout scalar3d_layout_mid{ {COL, LEV}, {ncol_, nlev_} }; - // Define fields needed in mam4xx. + // define fields needed in mam4xx // atmospheric quantities add_field("omega", scalar3d_layout_mid, Pa/s, grid_name); // vertical pressure velocity @@ -60,19 +129,31 @@ void MAMMicrophysics::set_grids(const std::shared_ptr grids_ add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // ice wet mixing ratio add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // ice number mixing ratio add_field("pbl_height", scalar2d_layout_col, m, grid_name); // planetary boundary layer height - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // pdel, hydrostatic pressure + add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // p_del, hydrostatic pressure + add_field("phis", scalar2d_layout_col, m2/s2, grid_name); add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction // droplet activation can alter cloud liquid and number mixing ratios add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid wet mixing ratio add_field("nc", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // cloud liquid wet number mixing ratio - // aerosol tracers of interest: mass (q) and number (n) mixing ratios - add_field("q_aitken_so4", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // sulfate mixing ratio for aitken mode - add_field("n_aitken", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // number mixing ratio of aitken mode + // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing ratios + for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); + for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); + if (strlen(int_mmr_field_name) > 0) { + add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); + } + } + } // aerosol-related gases: mass mixing ratios - add_field("q_h2so4", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // wet mixing ratio of sulfuric acid gas + for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); + } // Tracers group -- do we need this in addition to the tracers above? In any // case, this call should be idempotent, so it can't hurt. @@ -89,184 +170,99 @@ set_computed_group_impl(const FieldGroup& group) { EKAT_REQUIRE_MSG(group.m_info->m_bundled, "Error! MAM4 expects bundled fields for tracers.\n"); - // How many aerosol/gas tracers do we expect? Recall that we maintain - // both cloudborne and interstitial aerosol tracers. For now, though, we have - // 3 aerosol tracers: - // * H2SO4 gas mass mixing ratio - // * interstitial aitken-mode sulfate number + mass mixing ratios - int num_aero_tracers = 3; - /* - aero_config_.num_gas_ids() + // gas tracers - 2 * aero_config_.num_modes(); // modal number mixing ratio tracers - for (int m = 0; m < aero_config_.num_modes(); ++m) { - auto m_index = static_cast(m); - for (int a = 0; a < aero_config_.num_aerosol_ids(); ++a) { - auto a_id = static_cast(a); - if (mam4::aerosol_index_for_mode(m_index, a_id) != -1) { - num_aero_tracers += 2; // aerosol mass mixing ratios (interstitial, cloudborne) - } - } - } - */ - - EKAT_REQUIRE_MSG(group.m_info->size() >= num_aero_tracers, - "Error! MAM4 requires at least " << num_aero_tracers << " aerosol tracers."); + // how many aerosol/gas tracers do we expect? + int num_tracers = 2 * (mam_coupling::num_aero_modes() + + mam_coupling::num_aero_tracers()) + + mam_coupling::num_aero_gases(); + EKAT_REQUIRE_MSG(group.m_info->size() >= num_tracers, + "Error! MAM4 requires at least " << num_tracers << " aerosol tracers."); } size_t MAMMicrophysics::requested_buffer_size_in_bytes() const { - // number of Reals needed by local views in interface - const size_t request = sizeof(Real) * - (Buffer::num_2d_mid * ncol_ * nlev_ + - Buffer::num_2d_iface * ncol_ * (nlev_+1)); - - // FIXME: Need to figure out whether we need this stuff - /* - // Number of Reals needed by the WorkspaceManager passed to shoc_main - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(m_num_cols, nlev_packs); - const int n_wind_slots = ekat::npack(2)*Spack::n; - const int n_trac_slots = ekat::npack(m_num_tracers+3)*Spack::n; - const size_t wsm_request= WSM::get_total_bytes_needed(nlevi_packs, 13+(n_wind_slots+n_trac_slots), policy); - */ - - return request;// + wsm_request; + return mam_coupling::buffer_size(ncol_, nlev_); } void MAMMicrophysics::init_buffers(const ATMBufferManager &buffer_manager) { EKAT_REQUIRE_MSG(buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), "Error! Insufficient buffer size.\n"); - Real* mem = reinterpret_cast(buffer_manager.get_memory()); - - // set view pointers for midpoint fields - using view_2d_t = decltype(buffer_.z_mid); - view_2d_t* view_2d_mid_ptrs[Buffer::num_2d_mid] = { - &buffer_.z_mid, - &buffer_.dz, - &buffer_.qv_dry, - &buffer_.qc_dry, - &buffer_.n_qc_dry, - &buffer_.qi_dry, - &buffer_.n_qi_dry, - &buffer_.w_updraft, - &buffer_.q_h2so4_tend, - &buffer_.n_aitken_tend, - &buffer_.q_aitken_so4_tend, - }; - for (int i = 0; i < Buffer::num_2d_mid; ++i) { - *view_2d_mid_ptrs[i] = view_2d_t(mem, ncol_, nlev_); - mem += view_2d_mid_ptrs[i]->size(); - } - - // set view pointers for interface fields - view_2d_t* view_2d_iface_ptrs[Buffer::num_2d_iface] = {&buffer_.z_iface}; - for (int i = 0; i < Buffer::num_2d_iface; ++i) { - *view_2d_iface_ptrs[i] = view_2d_t(mem, ncol_, nlev_+1); - mem += view_2d_iface_ptrs[i]->size(); - } - - // WSM data - buffer_.wsm_data = mem; - - /* FIXME: this corresponds to the FIXME in the above function - // Compute workspace manager size to check used memory - // vs. requested memory - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); - const int n_wind_slots = ekat::npack(2)*Spack::n; - const int n_trac_slots = ekat::npack(m_num_tracers+3)*Spack::n; - const int wsm_size = WSM::get_total_bytes_needed(nlevi_packs, 13+(n_wind_slots+n_trac_slots), policy)/sizeof(Spack); - mem += wsm_size; - */ - - size_t used_mem = (mem - buffer_manager.get_memory())*sizeof(Real); + size_t used_mem = mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); EKAT_REQUIRE_MSG(used_mem==requested_buffer_size_in_bytes(), "Error! Used memory != requested memory for MAMMicrophysics."); } + void MAMMicrophysics::initialize_impl(const RunType run_type) { - const auto& T_mid = get_field_in("T_mid").get_view(); - const auto& p_mid = get_field_in("p_mid").get_view(); - const auto& qv = get_field_in("qv").get_view(); - const auto& pblh = get_field_in("pbl_height").get_view(); - const auto& p_del = get_field_in("pseudo_density").get_view(); - const auto& cldfrac = get_field_in("cldfrac_tot").get_view(); // FIXME: tot or liq? - const auto& qc = get_field_out("qc").get_view(); - const auto& n_qc = get_field_out("nc").get_view(); - const auto& qi = get_field_in("qi").get_view(); - const auto& n_qi = get_field_in("ni").get_view(); - const auto& omega = get_field_in("omega").get_view(); - const auto& q_h2so4 = get_field_out("q_h2so4").get_view(); - const auto& n_aitken = get_field_out("n_aitken").get_view(); - const auto& q_aitken_so4 = get_field_out("q_aitken_so4").get_view(); + step_ = 0; + + // populate the wet and dry atmosphere states with views from fields and + // the buffer + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_out("qc").get_view(); + wet_atm_.nc = get_field_out("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + wet_atm_.omega = get_field_in("omega").get_view(); + + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); // FIXME: tot or liq? + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + dry_atm_.phis = get_field_in("phis").get_view(); + dry_atm_.z_mid = buffer_.z_mid; + dry_atm_.dz = buffer_.dz; + dry_atm_.z_iface = buffer_.z_iface; + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + dry_atm_.w_updraft = buffer_.w_updraft; + dry_atm_.z_surf = 0.0; // FIXME: for now const auto& tracers = get_group_out("tracers"); const auto& tracers_info = tracers.m_info; - int num_tracers = tracers_info->size(); - - // Alias local variables from temporary buffer - auto z_mid = buffer_.z_mid; - auto dz = buffer_.dz; - auto z_iface = buffer_.z_iface; - auto qv_dry = buffer_.qv_dry; - auto qc_dry = buffer_.qc_dry; - auto n_qc_dry = buffer_.n_qc_dry; - auto qi_dry = buffer_.qi_dry; - auto n_qi_dry = buffer_.n_qi_dry; - auto w_updraft = buffer_.w_updraft; - - // Perform any initialization work. - if (run_type==RunType::Initial){ - /* e.g. - Kokkos::deep_copy(sgs_buoy_flux,0.0); - Kokkos::deep_copy(tk,0.0); - Kokkos::deep_copy(tke,0.0004); - Kokkos::deep_copy(tke_copy,0.0004); - Kokkos::deep_copy(cldfrac_liq,0.0); - */ + + // perform any initialization work + if (run_type==RunType::Initial) { } - // set atmosphere state data - T_mid_ = T_mid; - p_mid_ = p_mid; - qv_ = qv; - qc_ = qc; - n_qc_ = n_qc; - qi_ = qi; - n_qi_ = n_qi; - pdel_ = p_del; - cloud_f_ = cldfrac; - pblh_ = pblh; - q_h2so4_ = q_h2so4; - q_aitken_so4_ = q_aitken_so4; - n_aitken_ = n_aitken; - - // FIXME: For now, set z_surf to zero. - const Real z_surf = 0.0; - - // Determine indices of aerosol/gas tracers for wet<->dry conversion - auto q_aitken_so4_index = tracers_info->m_subview_idx.at("q_aitken_so4"); - auto q_h2so4_index = tracers_info->m_subview_idx.at("q_h2so4"); - int num_aero_tracers = 2; // for now, just 1 gas + aitken so4 - view_1d_int convert_wet_dry_idx_d("convert_wet_dry_idx_d", num_aero_tracers); - auto convert_wet_dry_idx_h = Kokkos::create_mirror_view(convert_wet_dry_idx_d); - for (int it=0, iq=0; it < num_tracers; ++it) { - if ((it == q_aitken_so4_index) || (it == q_h2so4_index)) { - convert_wet_dry_idx_h(iq) = it; - ++iq; + // set wet/dry aerosol state data (interstitial aerosols only) + for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + wet_aero_.int_aero_nmr[m] = get_field_out(int_nmr_field_name).get_view(); + dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; + for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); + if (strlen(int_mmr_field_name) > 0) { + wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); + dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; + } } } - Kokkos::deep_copy(convert_wet_dry_idx_d, convert_wet_dry_idx_h); - // hand views to our preprocess/postprocess functors - preprocess_.set_variables(ncol_, nlev_, z_surf, convert_wet_dry_idx_d, T_mid, - p_mid, qv, qv_dry, qc, n_qc, qc_dry, n_qc_dry, qi, n_qi, - qi_dry, n_qi_dry, z_mid, z_iface, dz, pdel_, cldfrac, omega, w_updraft, pblh, - q_h2so4_, q_aitken_so4_, n_aitken_); - postprocess_.set_variables(ncol_, nlev_, convert_wet_dry_idx_d, qv_dry, - q_h2so4_, q_aitken_so4_, n_aitken_); + // set wet/dry aerosol-related gas state data + for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char* mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = get_field_out(mmr_field_name).get_view(); + dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; + } + + // create our photolysis rate calculation table + photo_table_ = impl::read_photo_table(get_comm(), + config_.photolysis.rsf_file, + config_.photolysis.xs_long_file); + + // FIXME: read relevant land use data from drydep surface file - // Set field property checks for the fields in this process + // set up our preprocess/postprocess functors + preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); + postprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); + + // set field property checks for the fields in this process /* e.g. using Interval = FieldWithinIntervalCheck; using LowerBound = FieldLowerBoundCheck; @@ -276,21 +272,10 @@ void MAMMicrophysics::initialize_impl(const RunType run_type) { add_postcondition_check(get_field_out("tke"),m_grid,0); */ - // Setup WSM for internal local variables - // FIXME: do we need this? + // set up WSM for internal local variables + // (we'll probably need this later, but we'll just use ATMBufferManager for now) //const auto default_policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); //workspace_mgr_.setup(buffer_.wsm_data, nlev_+1, 13+(n_wind_slots+n_trac_slots), default_policy); - - // configure the nucleation parameterization - mam4::NucleationProcess::ProcessConfig nuc_config{}; - nuc_config.dens_so4a_host = 1770.0; - nuc_config.mw_so4a_host = 115.0; - nuc_config.newnuc_method_user_choice = 2; - nuc_config.pbl_nuc_wang2008_user_choice = 1; - nuc_config.adjust_factor_pbl_ratenucl = 1.0; - nuc_config.accom_coef_h2so4 = 1.0; - nuc_config.newnuc_adjust_factor_dnaitdt = 1.0; - nucleation_->init(nuc_config); } void MAMMicrophysics::run_impl(const double dt) { @@ -302,85 +287,219 @@ void MAMMicrophysics::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - // Reset internal WSM variables. + // reset internal WSM variables //workspace_mgr_.reset_internals(); - // FIXME: nothing depends on simulation time (yet), so we can just use zero for now + // NOTE: nothing depends on simulation time (yet), so we can just use zero for now double t = 0.0; - // Alias member variables - auto T_mid = T_mid_; - auto p_mid = p_mid_; - auto qv_dry = buffer_.qv_dry; - auto qc = qc_; - auto n_qc = n_qc_; - auto qi = qi_; - auto n_qi = n_qi_; - auto z_mid = buffer_.z_mid; - auto cldfrac = cloud_f_; - auto pdel = pdel_; - auto w_updraft = buffer_.w_updraft; - - // Compute nucleation tendencies on all local columns and accumulate them - // into our tracer state. + // here's where we store per-column photolysis rates + using View2D = haero::DeviceType::view_2d; + View2D photo_rates("photo_rates", nlev_, mam4::mo_photo::phtcnt); + + // climatology data for linear stratospheric chemistry + auto linoz_o3_clim = buffer_.scratch[0]; // ozone (climatology) [vmr] + auto linoz_o3col_clim = buffer_.scratch[1]; // column o3 above box (climatology) [Dobson Units (DU)] + auto linoz_t_clim = buffer_.scratch[2]; // temperature (climatology) [K] + auto linoz_PmL_clim = buffer_.scratch[3]; // P minus L (climatology) [vmr/s] + auto linoz_dPmL_dO3 = buffer_.scratch[4]; // sensitivity of P minus L to O3 [1/s] + auto linoz_dPmL_dT = buffer_.scratch[5]; // sensitivity of P minus L to T3 [K] + auto linoz_dPmL_dO3col = buffer_.scratch[6]; // sensitivity of P minus L to overhead O3 column [vmr/DU] + auto linoz_cariolle_psc = buffer_.scratch[7]; // Cariolle parameter for PSC loss of ozone [1/s] + + // it's a bit wasteful to store this for all columns, but simpler from an + // allocation perspective + auto o3_col_dens = buffer_.scratch[8]; + + // FIXME: read relevant linoz climatology data from file(s) based on time + + // FIXME: read relevant chlorine loading data from file based on time + + // loop over atmosphere columns and compute aerosol microphyscs Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { - const Int icol = team.league_rank(); // column index - - // extract column-specific atmosphere state data - haero::Atmosphere atm(nlev_, ekat::subview(T_mid_, icol), - ekat::subview(p_mid_, icol), - ekat::subview(qv_dry, icol), - ekat::subview(qc_, icol), - ekat::subview(n_qc_, icol), - ekat::subview(qi_, icol), - ekat::subview(n_qi_, icol), - ekat::subview(z_mid, icol), - ekat::subview(pdel, icol), - ekat::subview(cldfrac, icol), - ekat::subview(w_updraft, icol), - pblh_(icol)); + const int icol = team.league_rank(); // column index + + Real col_lat = col_latitudes_(icol); // column latitude (degrees?) + + // fetch column-specific atmosphere state data + auto atm = mam_coupling::atmosphere_for_column(dry_atm_, icol); + auto z_iface = ekat::subview(dry_atm_.z_iface, icol); + Real z_surf = dry_atm_.z_surf; + Real phis = dry_atm_.phis(icol); // set surface state data haero::Surface sfc{}; - // extract column-specific subviews into aerosol prognostics - using ModeIndex = mam4::ModeIndex; - using AeroId = mam4::AeroId; - using GasId = mam4::GasId; - - mam4::Prognostics progs(nlev_); - int iait = static_cast(ModeIndex::Aitken); - progs.n_mode_i[iait] = ekat::subview(n_aitken_, icol); - int iso4 = mam4::aerosol_index_for_mode(ModeIndex::Aitken, AeroId::SO4); - progs.q_aero_i[iait][iso4] = ekat::subview(q_aitken_so4_, icol); - int ih2so4 = static_cast(GasId::H2SO4); - progs.q_gas[ih2so4] = ekat::subview(q_h2so4_, icol); - - // nucleation doesn't use any diagnostics, so it's okay to leave this alone - // for now + // fetch column-specific subviews into aerosol prognostics + mam4::Prognostics progs = mam_coupling::interstitial_aerosols_for_column(dry_aero_, icol); + + // set up diagnostics mam4::Diagnostics diags(nlev_); - // grab views from the buffer to store tendencies - mam4::Tendencies tends(nlev_); - tends.q_gas[ih2so4] = ekat::subview(buffer_.q_h2so4_tend, icol); - tends.n_mode_i[iait] = ekat::subview(buffer_.n_aitken_tend, icol); - tends.q_aero_i[iait][iso4] = ekat::subview(buffer_.q_aitken_so4_tend, icol); - - // run the nucleation process to obtain tendencies - nucleation_->compute_tendencies(team, t, dt, atm, sfc, progs, diags, tends); -#ifndef NDEBUG - const int lev_idx = 0; - if (icol == 0) { - m_atm_logger->debug("tends.q_gas[ih2so4] = {}, tends.n_mode_i[iait] = {}, tends.q_aero_i[iait][iso4] = {}", - tends.q_gas[ih2so4](lev_idx), tends.n_mode_i[iait](lev_idx), tends.q_aero_i[iait][iso4](lev_idx)); - } -#endif + // calculate o3 column densities (first component of col_dens in Fortran code) + auto o3_col_dens_i = ekat::subview(o3_col_dens, icol); + impl::compute_o3_column_density(team, atm, progs, o3_col_dens_i); + + // set up photolysis work arrays for this column. + mam4::mo_photo::PhotoTableWorkArrays photo_work_arrays; + // FIXME: set views here + + // ... look up photolysis rates from our table + // NOTE: the table interpolation operates on an entire column of data, so we + // NOTE: must do it before dispatching to individual vertical levels + Real zenith_angle = 0.0; // FIXME: need to get this from EAMxx [radians] + Real surf_albedo = 0.0; // FIXME: surface albedo + Real esfact = 0.0; // FIXME: earth-sun distance factor + mam4::ColumnView lwc; // FIXME: liquid water cloud content: where do we get this? + mam4::mo_photo::table_photo(photo_rates, atm.pressure, atm.hydrostatic_dp, + atm.temperature, o3_col_dens_i, zenith_angle, surf_albedo, lwc, + atm.cloud_fraction, esfact, photo_table_, photo_work_arrays); + + // compute external forcings at time t(n+1) [molecules/cm^3/s] + constexpr int extcnt = mam4::gas_chemistry::extcnt; + view_2d extfrc; // FIXME: where to allocate? (nlev, extcnt) + mam4::mo_setext::Forcing forcings[extcnt]; // FIXME: forcings seem to require file data + mam4::mo_setext::extfrc_set(forcings, extfrc); + + // compute aerosol microphysics on each vertical level within this column + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev_), [&](const int k) { + + constexpr int num_modes = mam4::AeroConfig::num_modes(); + constexpr int num_aero_ids = mam4::AeroConfig::num_aerosol_ids(); + constexpr int gas_pcnst = mam_coupling::gas_pcnst(); + constexpr int nqtendbb = mam_coupling::nqtendbb(); + + // extract atm state variables (input) + Real temp = atm.temperature(k); + Real pmid = atm.pressure(k); + Real pdel = atm.hydrostatic_dp(k); + Real zm = atm.height(k); + Real zi = z_iface(k); + Real pblh = atm.planetary_boundary_layer_height; + Real qv = atm.vapor_mixing_ratio(k); + Real cldfrac = atm.cloud_fraction(k); + + // extract aerosol state variables into "working arrays" (mass mixing ratios) + // (in EAM, this is done in the gas_phase_chemdr subroutine defined within + // mozart/mo_gas_phase_chemdr.F90) + Real q[gas_pcnst] = {}; + Real qqcw[gas_pcnst] = {}; + mam_coupling::transfer_prognostics_to_work_arrays(progs, k, q, qqcw); + + // convert mass mixing ratios to volume mixing ratios (VMR), equivalent + // to tracer mixing ratios (TMR)) + Real vmr[gas_pcnst], vmrcw[gas_pcnst]; + mam_coupling::convert_work_arrays_to_vmr(q, qqcw, vmr, vmrcw); + + // aerosol/gas species tendencies (output) + Real vmr_tendbb[gas_pcnst][nqtendbb] = {}; + Real vmrcw_tendbb[gas_pcnst][nqtendbb] = {}; + + // create work array copies to retain "pre-chemistry" values + Real vmr_pregaschem[gas_pcnst] = {}; + Real vmr_precldchem[gas_pcnst] = {}; + Real vmrcw_precldchem[gas_pcnst] = {}; + for (int i = 0; i < gas_pcnst; ++i) { + vmr_pregaschem[i] = vmr[i]; + vmr_precldchem[i] = vmr[i]; + vmrcw_precldchem[i] = vmrcw[i]; + } + + //--------------------- + // Gas Phase Chemistry + //--------------------- + Real photo_rates_k[mam4::mo_photo::phtcnt]; + for (int i = 0; i < mam4::mo_photo::phtcnt; ++i) { + photo_rates_k[i] = photo_rates(k, i); + } + Real extfrc_k[extcnt]; + for (int i = 0; i < extcnt; ++i) { + extfrc_k[i] = extfrc(k, i); + } + constexpr int nfs = mam4::gas_chemistry::nfs; // number of "fixed species" + // NOTE: we compute invariants here and pass them out to use later with + // NOTE: setsox + Real invariants[nfs]; + impl::gas_phase_chemistry(zm, zi, phis, temp, pmid, pdel, dt, + photo_rates_k, extfrc_k, vmr, invariants); + + //---------------------- + // Aerosol microphysics + //---------------------- + // the logic below is taken from the aero_model_gasaerexch subroutine in + // eam/src/chemistry/modal_aero/aero_model.F90 + + // aqueous chemistry ... + const int loffset = 8; // offset of first tracer in work arrays + // (taken from mam4xx setsox validation test) + const Real mbar = haero::Constants::molec_weight_dry_air; + constexpr int indexm = 0; // FIXME: index of xhnm in invariants array (??) + Real cldnum = 0.0; // FIXME: droplet number concentration: where do we get this? + setsox_single_level(loffset, dt, pmid, pdel, temp, mbar, lwc(k), + cldfrac, cldnum, invariants[indexm], config_.setsox, vmrcw, vmr); + + // calculate aerosol water content using water uptake treatment + // * dry and wet diameters [m] + // * wet densities [kg/m3] + // * aerosol water mass mixing ratio [kg/kg] + Real dgncur_a[num_modes] = {}; + Real dgncur_awet[num_modes] = {}; + Real wetdens[num_modes] = {}; + Real qaerwat[num_modes] = {}; + impl::compute_water_content(progs, k, qv, temp, pmid, dgncur_a, dgncur_awet, wetdens, qaerwat); + + // do aerosol microphysics (gas-aerosol exchange, nucleation, coagulation) + impl::modal_aero_amicphys_intr(config_.amicphys, step_, dt, t, pmid, pdel, + zm, pblh, qv, cldfrac, vmr, vmrcw, vmr_pregaschem, + vmr_precldchem, vmrcw_precldchem, vmr_tendbb, + vmrcw_tendbb, dgncur_a, dgncur_awet, + wetdens, qaerwat); + + //----------------- + // LINOZ chemistry + //----------------- + + // the following things are diagnostics, which we're not + // including in the first rev + Real do3_linoz, do3_linoz_psc, ss_o3, o3col_du_diag, o3clim_linoz_diag, + zenith_angle_degrees; + + // FIXME: Need to get chlorine loading data from file + Real chlorine_loading = 0.0; + + Real rlats = col_lat * M_PI / 180.0; // convert column latitude to radians + int o3_ndx = 0; // index of "O3" in solsym array (in EAM) + mam4::lin_strat_chem::lin_strat_chem_solve_kk(o3_col_dens_i(k), temp, + zenith_angle, pmid, dt, rlats, + linoz_o3_clim(icol, k), linoz_t_clim(icol, k), linoz_o3col_clim(icol, k), + linoz_PmL_clim(icol, k), linoz_dPmL_dO3(icol, k), linoz_dPmL_dT(icol, k), + linoz_dPmL_dO3col(icol, k), linoz_cariolle_psc(icol, k), + chlorine_loading, config_.linoz.psc_T, vmr[o3_ndx], + do3_linoz, do3_linoz_psc, ss_o3, + o3col_du_diag, o3clim_linoz_diag, zenith_angle_degrees); + + // update source terms above the ozone decay threshold + if (k > nlev_ - config_.linoz.o3_lbl - 1) { + Real do3_mass; // diagnostic, not needed + mam4::lin_strat_chem::lin_strat_sfcsink_kk(dt, pdel, vmr[o3_ndx], config_.linoz.o3_sfc, + config_.linoz.o3_tau, do3_mass); + } + + // ... check for negative values and reset to zero + for (int i = 0; i < gas_pcnst; ++i) { + if (vmr[i] < 0.0) vmr[i] = 0.0; + } - // accumulate tendencies into prognostics - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev_), [&](const int klev) { - progs.q_gas[ih2so4](klev) += dt * tends.q_gas[ih2so4](klev); - progs.n_mode_i[iait](klev) += dt * tends.n_mode_i[iait](klev); - progs.q_aero_i[iait][iso4](klev) += dt * tends.q_aero_i[iait][iso4](klev); + //---------------------- + // Dry deposition (gas) + //---------------------- + + // FIXME: C++ port in progress! + //mam4::drydep::drydep_xactive(...); + + // transfer updated prognostics from work arrays + mam_coupling::convert_work_arrays_to_mmr(vmr, vmrcw, q, qqcw); + mam_coupling::transfer_work_arrays_to_prognostics(q, qqcw, progs, k); }); }); @@ -389,8 +508,7 @@ void MAMMicrophysics::run_impl(const double dt) { Kokkos::fence(); } -void MAMMicrophysics::finalize_impl() -{ +void MAMMicrophysics::finalize_impl() { } } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp index ff3652459f1e..5f5a44b846be 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp @@ -1,9 +1,11 @@ #ifndef EAMXX_MAM_MICROPHYSICS_HPP #define EAMXX_MAM_MICROPHYSICS_HPP +#include #include #include -#include + +#include "impl/mam4_amicphys.cpp" // mam4xx top-level microphysics function(s) #include #include @@ -22,8 +24,8 @@ namespace scream { -// The process responsible for handling MAM4 aerosols. The AD stores exactly ONE -// instance of this class in its list of subcomponents. +// The process responsible for handling MAM4 aerosol microphysics. The AD +// stores exactly ONE instance of this class in its list of subcomponents. class MAMMicrophysics final : public scream::AtmosphereProcess { using PF = scream::PhysicsFunctions; using KT = ekat::KokkosTypes; @@ -60,6 +62,9 @@ class MAMMicrophysics final : public scream::AtmosphereProcess { AtmosphereProcessType type() const override; std::string name() const override; + // set aerosol microphysics configuration parameters (called by constructor) + void configure(const ekat::ParameterList& params); + // grid void set_grids(const std::shared_ptr grids_manager) override; @@ -80,284 +85,144 @@ class MAMMicrophysics final : public scream::AtmosphereProcess { // number of horizontal columns and vertical levels int ncol_, nlev_; + // configuration data (for the moment, we plan to be able to move this to + // the device, so we can't use C++ strings) + struct Config { + // photolysis parameters + struct { + char rsf_file[MAX_FILENAME_LEN]; + char xs_long_file[MAX_FILENAME_LEN]; + } photolysis; + + // stratospheric chemistry parameters + struct { + int o3_lbl; // number of layers with ozone decay from the surface + int o3_sfc; // set from namelist input linoz_sfc + int o3_tau; // set from namelist input linoz_tau + Real psc_T; // set from namelist input linoz_psc_T + char chlorine_loading_file[MAX_FILENAME_LEN]; + } linoz; + + // aqueous chemistry parameters + mam4::mo_setsox::Config setsox; + + // aero microphysics configuration (see impl/mam4_amicphys.cpp) + impl::AmicPhysConfig amicphys; + + // dry deposition parameters + struct { + char srf_file[MAX_FILENAME_LEN]; + } drydep; + }; + Config config_; + // Atmosphere processes often have a pre-processing step that constructs // required variables from the set of fields stored in the field manager. // This functor implements this step, which is called during run_impl. struct Preprocess { Preprocess() = default; + // on host: initializes preprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere& wet_atm, + const mam_coupling::AerosolState& wet_aero, + const mam_coupling::DryAtmosphere& dry_atm, + const mam_coupling::AerosolState& dry_aero) { + ncol_ = ncol; + nlev_ = nlev; + wet_atm_ = wet_atm; + wet_aero_ = wet_aero; + dry_atm_ = dry_atm; + dry_aero_ = dry_aero; + } + KOKKOS_INLINE_FUNCTION void operator()(const Kokkos::TeamPolicy::member_type& team) const { const int i = team.league_rank(); // column index - // Compute vertical layer heights - const auto dz_i = ekat::subview(dz_, i); - auto z_iface_i = ekat::subview(z_iface_, i); - auto z_mid_i = ekat::subview(z_mid_, i); - PF::calculate_z_int(team, nlev_, dz_i, z_surf_, z_iface_i); - team.team_barrier(); // TODO: is this barrier necessary? - PF::calculate_z_mid(team, nlev_, z_iface_i, z_mid_i); - // barrier here allows the kernels that follow to use layer heights - team.team_barrier(); - - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_), - [&] (const int k) { - //-------------------------- - // Vertical velocity from pressure to height - //-------------------------- - const auto rho = PF::calculate_density(pdel_(i,k), dz_i(k)); - w_updraft_(i,k) = PF::calculate_vertical_velocity(omega_(i,k), rho); - - //-------------------------- - // Wet to dry mixing ratios - //-------------------------- - // - // Since tracers from the host model (or AD) are wet mixing ratios, and - // MAM4 expects these tracers in dry mixing ratios, we convert the wet - // mixing ratios to dry mixing ratios for all the tracers. - // - // The function calculate_drymmr_from_wetmmr takes 2 arguments: - // 1. wet mmr - // 2. "wet" water vapor mixing ratio - // - // Units of all tracers become [kg/kg(dry-air)] for mass mixing ratios and - // [#/kg(dry-air)] for number mixing ratios after the following - // conversion. - const auto qv_ik = qv_(i,k); - // const fields need separate storage for "dry" values - qv_dry_(i,k) = PF::calculate_drymmr_from_wetmmr(qv_(i,k), qv_ik); - qc_dry_(i,k) = PF::calculate_drymmr_from_wetmmr(qc_(i,k), qv_ik); - n_qc_dry_(i,k) = PF::calculate_drymmr_from_wetmmr(n_qc_(i,k), qv_ik); - qi_dry_(i,k) = PF::calculate_drymmr_from_wetmmr(qi_(i,k), qv_ik); - n_qi_dry_(i,k) = PF::calculate_drymmr_from_wetmmr(n_qi_(i,k), qv_ik); - - // non-const fields can be overwritten; we'll convert back to moist - // air ratios during postprocess - q_h2so4_(i,k) = PF::calculate_drymmr_from_wetmmr(q_h2so4_(i,k), qv_ik); - q_aitken_so4_(i,k) = PF::calculate_drymmr_from_wetmmr(q_aitken_so4_(i,k), qv_ik); - n_aitken_(i,k) = PF::calculate_drymmr_from_wetmmr(n_aitken_(i,k), qv_ik); - }); - // make sure all operations are done before exiting kernel + compute_vertical_layer_heights(team, dry_atm_, i); + team.team_barrier(); // allows kernels below to use layer heights + compute_updraft_velocities(team, wet_atm_, dry_atm_, i); + compute_dry_mixing_ratios(team, wet_atm_, dry_atm_, i); + compute_dry_mixing_ratios(team, wet_atm_, wet_aero_, dry_aero_, i); team.team_barrier(); } // operator() // number of horizontal columns and vertical levels int ncol_, nlev_; - // height of bottom of atmosphere - Real z_surf_; - - // used for converting between wet and dry mixing ratios - view_1d_int convert_wet_dry_idx_d_; - - // local atmospheric state column variables - const_view_2d T_mid_; // temperature at grid midpoints [K] - const_view_2d p_mid_; // total pressure at grid midpoints [Pa] - const_view_2d qv_; // water vapor specific humidity [kg vapor / kg moist air] - view_2d qv_dry_; // water vapor mixing ratio [kg vapor / kg dry air] - const_view_2d qc_; // cloud liquid water mass mixing ratio [kg vapor/kg moist air] - view_2d qc_dry_; - const_view_2d n_qc_; // cloud liquid water number mixing ratio [kg cloud water / kg moist air] - view_2d n_qc_dry_; // cloud liquid water number mixing ratio (dry air) - const_view_2d qi_; // cloud ice water mass mixing ratio - view_2d qi_dry_; // [kg vapor/kg dry air] - const_view_2d n_qi_; // cloud ice water number mixing ratio - view_2d n_qi_dry_; - view_2d z_mid_; // height at layer midpoints [m] - view_2d z_iface_; // height at layer interfaces [m] - view_2d dz_; // layer thickness [m] - const_view_2d pdel_; // hydrostatic "pressure thickness" at grid - // interfaces [Pa] - const_view_2d cloud_f_; // cloud fraction [-] - const_view_2d omega_; // vertical pressure velocity [Pa/s] - view_2d w_updraft_; // updraft velocity [m/s] - const_view_1d pblh_; // planetary boundary layer height [m] - - // local aerosol-related gases - view_2d q_h2so4_; // H2SO4 gas [kg/kg dry air] - - // local aerosols (more to appear as we improve this atm process) - view_2d n_aitken_; // aitken mode number mixing ratio [1/kg dry air] - view_2d q_aitken_so4_; // SO4 mass mixing ratio in aitken mode [kg/kg dry air] - - // assigns local variables - void set_variables(const int ncol, const int nlev, const Real z_surf, - const view_1d_int& convert_wet_dry_idx_d, - const const_view_2d& T_mid, - const const_view_2d& p_mid, - const const_view_2d& qv, - const view_2d& qv_dry, - const view_2d& qc, - const view_2d& n_qc, - const view_2d& qc_dry, - const view_2d& n_qc_dry, - const const_view_2d& qi, - const const_view_2d& n_qi, - const view_2d& qi_dry, - const view_2d& n_qi_dry, - const view_2d& z_mid, - const view_2d& z_iface, - const view_2d& dz, - const const_view_2d& pdel, - const const_view_2d& cf, - const const_view_2d& omega, - const view_2d& w_updraft, - const const_view_1d& pblh, - const view_2d& q_h2so4, - const view_2d& q_aitken_so4, - const view_2d& n_aitken) { - ncol_ = ncol; - nlev_ = nlev; - z_surf_ = z_surf; - convert_wet_dry_idx_d_ = convert_wet_dry_idx_d; - T_mid_ = T_mid; - p_mid_ = p_mid; - qv_ = qv; - qv_dry_ = qv_dry; - qc_ = qc; - n_qc_ = n_qc; - qc_dry_ = qc_dry; - n_qc_dry_ = n_qc_dry; - qi_ = qi; - n_qi_ = n_qi; - qi_dry_ = qi_dry; - n_qi_dry_ = n_qi_dry; - z_mid_ = z_mid; - z_iface_ = z_iface; - dz_ = dz; - pdel_ = pdel; - cloud_f_ = cf; - omega_ = omega; - w_updraft_ = w_updraft; - pblh_ = pblh; - q_h2so4_ = q_h2so4; - q_aitken_so4_ = q_aitken_so4; - n_aitken_ = n_aitken; - } // set_variables + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + mam_coupling::AerosolState wet_aero_, dry_aero_; + }; // MAMMicrophysics::Preprocess // Postprocessing functor struct Postprocess { Postprocess() = default; + // on host: initializes postprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere& wet_atm, + const mam_coupling::AerosolState& wet_aero, + const mam_coupling::DryAtmosphere& dry_atm, + const mam_coupling::AerosolState& dry_aero) { + ncol_ = ncol; + nlev_ = nlev; + wet_atm_ = wet_atm; + wet_aero_ = wet_aero; + dry_atm_ = dry_atm; + dry_aero_ = dry_aero; + } + KOKKOS_INLINE_FUNCTION void operator()(const Kokkos::TeamPolicy::member_type& team) const { - // After these updates, all non-const tracers are converted from dry mmr to wet mmr - const int i = team.league_rank(); - - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_), - [&] (const int k) { - const auto qv_ik = qv_dry_(i,k); - q_h2so4_(i,k) = PF::calculate_wetmmr_from_drymmr(q_h2so4_(i,k), qv_ik); - q_aitken_so4_(i,k) = PF::calculate_wetmmr_from_drymmr(q_aitken_so4_(i,k), qv_ik); - n_aitken_(i,k) = PF::calculate_wetmmr_from_drymmr(n_aitken_(i,k), qv_ik); - }); + const int i = team.league_rank(); // column index + compute_wet_mixing_ratios(team, dry_atm_, dry_aero_, wet_aero_, i); team.team_barrier(); } // operator() - // Local variables + // number of horizontal columns and vertical levels int ncol_, nlev_; - view_2d qv_dry_; - - // used for converting between wet and dry mixing ratios - view_1d_int convert_wet_dry_idx_d_; - - // local aerosol-related gases - view_2d q_h2so4_; // H2SO4 gas [kg/kg dry air] - - // local aerosols (more to appear as we improve this atm process) - view_2d q_aitken_so4_; // SO4 aerosol in aitken mode [kg/kg dry air] - // modal quantities - view_2d n_aitken_; - - // assigns local variables - void set_variables(const int ncol, - const int nlev, - const view_1d_int& convert_wet_dry_idx_d, - const view_2d& qv_dry, - const view_2d& q_h2so4, - const view_2d& q_aitken_so4, - const view_2d& n_aitken) { - ncol_ = ncol; - nlev_ = nlev; - convert_wet_dry_idx_d_ = convert_wet_dry_idx_d; - qv_dry_ = qv_dry; - q_h2so4_ = q_h2so4; - q_aitken_so4_ = q_aitken_so4; - n_aitken_ = n_aitken; - } // set_variables + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + mam_coupling::AerosolState wet_aero_, dry_aero_; }; // MAMMicrophysics::Postprocess - // storage for local variables, initialized with ATMBufferManager - struct Buffer { - // number of local fields stored at column midpoints - static constexpr int num_2d_mid = 11; - - // local column midpoint fields - uview_2d z_mid; // height at midpoints - uview_2d dz; // layer thickness - uview_2d qv_dry; // water vapor mixing ratio (dry air) - uview_2d qc_dry; // cloud water mass mixing ratio - uview_2d n_qc_dry; // cloud water number mixing ratio - uview_2d qi_dry; // cloud ice mass mixing ratio - uview_2d n_qi_dry; // cloud ice number mixing ratio - uview_2d w_updraft; // vertical wind velocity - uview_2d q_h2so4_tend; // tendency for H2SO4 gas - uview_2d n_aitken_tend; // tendency for aitken aerosol mode - uview_2d q_aitken_so4_tend; // tendency for aitken mode sulfate aerosol - - // number of local fields stored at column interfaces - static constexpr int num_2d_iface = 1; - - // local column interface fields - uview_2d z_iface; // height at interfaces - - // storage - Real* wsm_data; - }; - // MAM4 aerosol particle size description mam4::AeroConfig aero_config_; - // aerosol processes - std::unique_ptr nucleation_; - - // pre- and postprocessing scratch pads + // pre- and postprocessing scratch pads (for wet <-> dry conversions) Preprocess preprocess_; Postprocess postprocess_; - // local atmospheric state column variables - const_view_2d T_mid_; // temperature at grid midpoints [K] - const_view_2d p_mid_; // total pressure at grid midpoints [Pa] - const_view_2d qv_; // water vapor specific humidity [kg h2o vapor / kg moist air] - // we keep the specific humidity to use in the PostProcess step. - view_2d qc_; // cloud liquid mass mixing ratio - // must be converted from wet to dry [kg cloud water /kg dry air] - view_2d n_qc_; // cloud liquid number mixing ratio - // must be converted from wet to dry [1 /kg dry air] - const_view_2d qi_; // cloud ice mass mixing ratio - // must be converted from wet to dry [kg cloud water /kg dry air] - const_view_2d n_qi_; // cloud ice number mixing ratio - // must be converted from wet to dry [1 /kg dry air] - const_view_2d pdel_; // hydrostatic "pressure thickness" at grid - // interfaces [Pa] - const_view_2d cloud_f_; // cloud fraction [-] - const_view_1d pblh_; // planetary boundary layer height [m] - - // local aerosol-related gases - view_2d q_h2so4_; // H2SO3 gas [kg/kg dry air] - - // local aerosols (more to appear as we improve this atm process) - view_2d n_aitken_; // aitken mode number mixing ratio [1/kg dry air] - view_2d q_aitken_so4_; // SO4 mass mixing ratio in aitken mode [kg/kg dry air] + // atmospheric and aerosol state variables + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + mam_coupling::AerosolState wet_aero_, dry_aero_; + + // photolysis rate table (column-independent) + mam4::mo_photo::PhotoTableData photo_table_; + + // column areas, latitudes, longitudes + const_view_1d col_areas_, col_latitudes_, col_longitudes_; + + // time step number + int step_; // workspace manager for internal local variables //ekat::WorkspaceManager workspace_mgr_; - Buffer buffer_; + mam_coupling::Buffer buffer_; // physics grid for column information std::shared_ptr grid_; + + // sets defaults for "namelist parameters" + void set_defaults_(); + }; // MAMMicrophysics } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp index 85b51ff3f3dc..a827347c5385 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp @@ -1,107 +1,518 @@ +#include #include #include #include -#include "scream_config.h" // for SCREAM_CIME_BUILD - -#include +#include "scream_config.h" // for SCREAM_CIME_BUILD +#include "share/grid/point_grid.hpp" +#include "share/io/scorpio_input.hpp" -namespace scream -{ +namespace scream { -MAMOptics::MAMOptics( - const ekat::Comm& comm, - const ekat::ParameterList& params) - : AtmosphereProcess(comm, params), - aero_config_() { -} +MAMOptics::MAMOptics(const ekat::Comm &comm, const ekat::ParameterList ¶ms) + : AtmosphereProcess(comm, params), aero_config_() { + } AtmosphereProcessType MAMOptics::type() const { return AtmosphereProcessType::Physics; } -std::string MAMOptics::name() const { - return "mam4_optics"; -} +std::string MAMOptics::name() const { return "mam4_optics"; } -void MAMOptics::set_grids(const std::shared_ptr grids_manager) { +void MAMOptics::set_grids( + const std::shared_ptr grids_manager) { using namespace ekat::units; - grid_ = grids_manager->get_grid("Physics"); - const auto& grid_name = grid_->name(); + grid_ = grids_manager->get_grid("Physics"); + const auto &grid_name = grid_->name(); + auto q_unit = kg / kg; // mass mixing ratios [kg stuff / kg air] + q_unit.set_string("kg/kg"); + auto n_unit = 1 / kg; // number mixing ratios [# / kg air] + n_unit.set_string("#/kg"); + const auto m2 = m * m; + const auto s2 = s * s; - ncol_ = grid_->get_num_local_dofs(); // number of columns on this rank - nlev_ = grid_->get_num_vertical_levels(); // number of levels per column - nswbands_ = 14; // number of shortwave bands - nlwbands_ = 16; // number of longwave bands + ncol_ = grid_->get_num_local_dofs(); // number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // number of levels per column + nswbands_ = mam4::modal_aer_opt::nswbands; // number of shortwave bands + nlwbands_ = mam4::modal_aer_opt::nlwbands; // number of longwave bands // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; // Define aerosol optics fields computed by this process. auto nondim = Units::nondimensional(); - FieldLayout scalar3d_swband_layout { {COL, SWBND, LEV}, {ncol_, nswbands_, nlev_} }; - FieldLayout scalar3d_lwband_layout { {COL, LWBND, LEV}, {ncol_, nlwbands_, nlev_} }; - - // shortwave aerosol scattering asymmetry parameter [-] - add_field("aero_g_sw", scalar3d_swband_layout, nondim, grid_name); - // shortwave aerosol single-scattering albedo [-] - add_field("aero_ssa_sw", scalar3d_swband_layout, nondim, grid_name); - // shortwave aerosol optical depth [-] - add_field("aero_tau_sw", scalar3d_swband_layout, nondim, grid_name); - // longwave aerosol optical depth [-] + // 3D layout for shortwave aerosol fields: columns, number of shortwave, and nlev + FieldLayout scalar3d_swband_layout{{COL, SWBND, LEV}, + {ncol_, nswbands_, nlev_}}; + + // 3D layout for longwave aerosol fields: columns, number of shortwave, and nlev +1 s + FieldLayout scalar3d_lwband_layout{{COL, LWBND, LEV}, + {ncol_, nlwbands_, nlev_}}; + + FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + + // layout for 2D (1d horiz X 1d vertical) variable + FieldLayout scalar2d_layout_col{{COL}, {ncol_}}; + + // layout for 3D (2d horiz X 1d vertical) variables + FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; + add_field("omega", scalar3d_layout_mid, Pa / s, + grid_name); // vertical pressure velocity + add_field("T_mid", scalar3d_layout_mid, K, + grid_name); // Temperature + add_field("p_mid", scalar3d_layout_mid, Pa, + grid_name); // total pressure + + add_field("p_int", scalar3d_layout_int, Pa, + grid_name); // total pressure + add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); + add_field("pseudo_density_dry", scalar3d_layout_mid, Pa, grid_name); + + add_field("qv", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // specific humidity + add_field("qi", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // ice wet mixing ratio + add_field("ni", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // ice number mixing ratio + + // droplet activation can alter cloud liquid and number mixing ratios + add_field("qc", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // cloud liquid wet mixing ratio + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // cloud liquid wet number mixing ratio + + add_field("phis", scalar2d_layout_col, m2 / s2, grid_name); + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, + grid_name); // cloud fraction + add_field("pbl_height", scalar2d_layout_col, m, + grid_name); // planetary boundary layer height + // shortwave aerosol scattering asymmetry parameter [unitless] + add_field("aero_g_sw", scalar3d_swband_layout, nondim, grid_name); + // shortwave aerosol single-scattering albedo [unitless] + add_field("aero_ssa_sw", scalar3d_swband_layout, nondim, + grid_name); + // shortwave aerosol extinction optical depth + add_field("aero_tau_sw", scalar3d_swband_layout, nondim, + grid_name); + //longwave aerosol extinction optical depth [unitless] add_field("aero_tau_lw", scalar3d_lwband_layout, nondim, grid_name); - // FIXME: this field doesn't belong here, but this is a convenient place to - // FIXME: put it for now. - // number mixing ratio for CCN - using Spack = ekat::Pack; - using Pack = ekat::Pack; - constexpr int ps = Pack::n; - FieldLayout scalar3d_layout_mid { {COL, LEV}, {ncol_, nlev_} }; - add_field("nccn", scalar3d_layout_mid, 1/kg, grid_name, ps); + add_field("aodvis", scalar2d_layout_col, nondim, grid_name); + + // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing + // ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + + add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name, "tracers"); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + + if(strlen(int_mmr_field_name) > 0) { + add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); + } + } + } + // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + + if(strlen(cld_mmr_field_name) > 0) { + add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name); + } + } + } + + // aerosol-related gases: mass mixing ratios + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); + } } -void MAMOptics::initialize_impl(const RunType run_type) { +size_t MAMOptics::requested_buffer_size_in_bytes() const { + return mam_coupling::buffer_size(ncol_, nlev_); } -void MAMOptics::run_impl(const double dt) { +void MAMOptics::init_buffers(const ATMBufferManager &buffer_manager) { + EKAT_REQUIRE_MSG( + buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); - - // get the aerosol optics fields - auto aero_g_sw = get_field_out("aero_g_sw").get_view(); - auto aero_ssa_sw = get_field_out("aero_ssa_sw").get_view(); - auto aero_tau_sw = get_field_out("aero_tau_sw").get_view(); - auto aero_tau_lw = get_field_out("aero_tau_lw").get_view(); - - auto aero_nccn = get_field_out("nccn").get_view(); // FIXME: get rid of this - - // Compute optical properties on all local columns. - // (Strictly speaking, we don't need this parallel_for here yet, but we leave - // it in anticipation of column-specific aerosol optics to come.) - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { - const Int icol = team.league_rank(); // column index - - auto g_sw = ekat::subview(aero_g_sw, icol); - auto ssa_sw = ekat::subview(aero_ssa_sw, icol); - auto tau_sw = ekat::subview(aero_tau_sw, icol); - auto tau_lw = ekat::subview(aero_tau_lw, icol); - - // populate these fields with reasonable representative values - Kokkos::deep_copy(g_sw, 0.5); - Kokkos::deep_copy(ssa_sw, 0.7); - Kokkos::deep_copy(tau_sw, 0.0); - Kokkos::deep_copy(tau_lw, 0.0); - - // FIXME: Get rid of this - auto nccn = ekat::subview(aero_nccn, icol); - Kokkos::deep_copy(nccn, 50.0); - }); + size_t used_mem = + mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); + EKAT_REQUIRE_MSG(used_mem == requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MAMMOptics."); } -void MAMOptics::finalize_impl() -{ +void MAMOptics::initialize_impl(const RunType run_type) { + // populate the wet and dry atmosphere states with views from fields and + // the buffer + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + wet_atm_.omega = get_field_in("omega").get_view(); + + constexpr int ntot_amode = mam4::AeroConfig::num_modes(); + + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density_dry").get_view(); + p_del_ = get_field_in("pseudo_density").get_view(); + dry_atm_.cldfrac = get_field_in("cldfrac_tot") + .get_view(); // FIXME: tot or liq? + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + dry_atm_.phis = get_field_in("phis").get_view(); + + dry_atm_.z_mid = buffer_.z_mid; + dry_atm_.dz = buffer_.dz; + dry_atm_.z_iface = buffer_.z_iface; + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + dry_atm_.w_updraft = buffer_.w_updraft; + // The surface height is zero by definition. + // see eam/src/physics/cam/geopotential.F90 + dry_atm_.z_surf = 0.0; + + // set wet/dry aerosol state data (interstitial aerosols only) + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + wet_aero_.int_aero_nmr[m] = + get_field_out(int_nmr_field_name).get_view(); + dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + if(strlen(int_mmr_field_name) > 0) { + wet_aero_.int_aero_mmr[m][a] = + get_field_out(int_mmr_field_name).get_view(); + dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; + } + } + } + + // set wet/dry aerosol state data (cloud aerosols only) + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + wet_aero_.cld_aero_nmr[m] = + get_field_out(cld_nmr_field_name).get_view(); + dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + if(strlen(cld_mmr_field_name) > 0) { + wet_aero_.cld_aero_mmr[m][a] = + get_field_out(cld_mmr_field_name).get_view(); + dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; + } + } + } + + // set wet/dry aerosol-related gas state data + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = get_field_out(mmr_field_name).get_view(); + dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; + } + + // prescribed volcanic aerosols. + ssa_cmip6_sw_ = + mam_coupling::view_3d("ssa_cmip6_sw", ncol_, nlev_, nswbands_); + af_cmip6_sw_ = mam_coupling::view_3d("af_cmip6_sw", ncol_, nlev_, nswbands_); + ext_cmip6_sw_ = + mam_coupling::view_3d("ext_cmip6_sw", ncol_, nswbands_, nlev_); + ext_cmip6_lw_ = + mam_coupling::view_3d("ext_cmip6_lw_", ncol_, nlev_, nlwbands_); + // FIXME: We need to get ssa_cmip6_sw_, af_cmip6_sw_, ext_cmip6_sw_, + // ext_cmip6_lw_ from a netcdf file. + // We will rely on eamxx to interpolate/map data from netcdf files. + // The io interface in eamxx is being upgraded. + // Thus, I will wait until changes in the eamxx's side are completed. + Kokkos::deep_copy(ssa_cmip6_sw_, 0.0); + Kokkos::deep_copy(af_cmip6_sw_, 0.0); + Kokkos::deep_copy(ext_cmip6_sw_, 0.0); + Kokkos::deep_copy(ext_cmip6_lw_, 0.0); + + // set up our preprocess/postprocess functors + preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); + postprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); + + const int work_len = mam4::modal_aer_opt::get_work_len_aerosol_optics(); + work_ = mam_coupling::view_2d("work", ncol_, work_len); + + // shortwave aerosol scattering asymmetry parameter [unitless] + tau_ssa_g_sw_ = mam_coupling::view_3d("tau_ssa_g_sw_", ncol_, nswbands_, nlev_ + 1); + // shortwave aerosol single-scattering albedo [unitless] + tau_ssa_sw_ = mam_coupling::view_3d("tau_ssa_sw_", ncol_, nswbands_, nlev_ + 1); + // shortwave aerosol extinction optical depth [unitless] + tau_sw_ = mam_coupling::view_3d("tau_sw_", ncol_, nswbands_, nlev_ + 1); + //aerosol forward scattered fraction * tau * w + tau_f_sw_= mam_coupling::view_3d("tau_f_sw_", ncol_, nswbands_, nlev_ + 1); + + // read table info + { + using namespace ShortFieldTagsNames; + + using view_1d_host = typename KT::view_1d::HostMirror; + + // Note: these functions do not set values for aerosol_optics_device_data_. + mam4::modal_aer_opt::set_complex_views_modal_aero( + aerosol_optics_device_data_); + mam4::modal_aer_opt::set_aerosol_optics_data_for_modal_aero_sw_views( + aerosol_optics_device_data_); + mam4::modal_aer_opt::set_aerosol_optics_data_for_modal_aero_lw_views( + aerosol_optics_device_data_); + + mam_coupling::AerosolOpticsHostData aerosol_optics_host_data; + + std::map layouts; + std::map host_views; + ekat::ParameterList rrtmg_params; + + mam_coupling::set_parameters_table(aerosol_optics_host_data, rrtmg_params, + layouts, host_views); + + for(int imode = 0; imode < ntot_amode; imode++) { + const auto key = "mam4_mode" + std::to_string(imode+1) + + "_physical_properties_file"; + const auto& fname = m_params.get(key); + mam_coupling::read_rrtmg_table(fname, + imode, // mode No + rrtmg_params, grid_, host_views, layouts, + aerosol_optics_host_data, + aerosol_optics_device_data_); + } + + std::string table_name_water = + m_params.get("mam4_water_refindex_file"); + + // it will syn data to device. + mam_coupling::read_water_refindex(table_name_water, grid_, + aerosol_optics_device_data_.crefwlw, + aerosol_optics_device_data_.crefwsw); + // + { + // make a list of host views + std::map host_views_aero; + // defines layouts + std::map layouts_aero; + ekat::ParameterList params_aero; + std::string surname_aero = "aer"; + mam_coupling::set_refindex_names(surname_aero, params_aero, host_views_aero, + layouts_aero); + + constexpr int maxd_aspectype = mam4::ndrop::maxd_aspectype; + auto specrefndxsw_host = mam_coupling::complex_view_2d::HostMirror( + "specrefndxsw_host", nswbands_, maxd_aspectype); + + auto specrefndxlw_host = mam_coupling::complex_view_2d::HostMirror( + "specrefndxlw_host", nlwbands_, maxd_aspectype); + + // read physical properties data for aerosol species + std::map map_table_name_species_id; + map_table_name_species_id["soa"] = 4; // soa:s-organic + map_table_name_species_id["dust"] = 7; // dst:dust: + map_table_name_species_id["nacl"] = 6; // ncl:seasalt + map_table_name_species_id["so4"] = 0; // so4:sulfate + map_table_name_species_id["pom"] = 3; // pom:p-organic + map_table_name_species_id["bc"] = 5; // bc :black-c + map_table_name_species_id["mom"] = 8; // mom:m-organic + + for (const auto& item : map_table_name_species_id) { + const auto spec_name = item.first; + const int species_id = item.second; + const auto table_name = "mam4_" + spec_name + "_physical_properties_file"; + const auto& fname = m_params.get(table_name); + // read data + // need to update table name + params_aero.set("Filename", fname); + AtmosphereInput refindex_aerosol(params_aero, grid_, host_views_aero, + layouts_aero); + refindex_aerosol.read_variables(); + refindex_aerosol.finalize(); + // copy data to device + mam_coupling::set_refindex_aerosol( + species_id, host_views_aero, + specrefndxsw_host, // complex refractive index for water visible + specrefndxlw_host); + } // done ispec + // reshape specrefndxsw_host and copy it to device + mam4::modal_aer_opt::set_device_specrefindex( + aerosol_optics_device_data_.specrefindex_sw, "short_wave", + specrefndxsw_host); + mam4::modal_aer_opt::set_device_specrefindex( + aerosol_optics_device_data_.specrefindex_lw, "long_wave", + specrefndxlw_host); + } + } + //FIXME: We are hard-coding the band ordering in RRTMGP. + //TODO: We can update optics file using the ordering below (rrtmg_to_rrtmgp_swbands_). + // Mapping from old RRTMG sw bands to new band ordering in RRTMGP + // rrtmg_swband (old) = 2925, 3625, 4325, 4900, 5650, 6925, 7875, 10450, 14425, 19325, + // 25825, 33500, 44000, 1710 ; + // rrtmgp_swband (new) = 1710, 2925, 3625, 4325, 4900, 5650, 6925, 7875, 10450, 14425, + // 19325, 25825, 33500, 44000 ; + // given the rrtmg index return the rrtmgp index + std::vector temporal = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0}; + auto get_idx_rrtmgp_from_rrtmg_swbands_host =mam_coupling::view_int_1d::HostMirror(temporal.data(),nswbands_); + get_idx_rrtmgp_from_rrtmg_swbands_ = mam_coupling::view_int_1d("rrtmg_to_rrtmgp_swbands",nswbands_); + Kokkos::deep_copy(get_idx_rrtmgp_from_rrtmg_swbands_,get_idx_rrtmgp_from_rrtmg_swbands_host); } +void MAMOptics::run_impl(const double dt) { + + constexpr Real zero=0.0; + constexpr Real one=1.0; + + const auto policy = + ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + // preprocess input -- needs a scan for the calculation of atm height + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + + //tau_w_g : aerosol asymmetry parameter * tau * w + const auto tau_ssa_g_sw = tau_ssa_g_sw_; + //tau_w : aerosol single scattering albedo * tau + const auto tau_ssa_sw =tau_ssa_sw_; + // tau : aerosol extinction optical depth + const auto tau_sw = tau_sw_; + // get_field_out("aero_tau_sw_mam4").get_view(); + // aero_tau_lw ( or odap_aer) : absorption optical depth, per layer + const auto aero_tau_lw = get_field_out("aero_tau_lw").get_view(); + + const auto aero_g_sw_eamxx = get_field_out("aero_g_sw").get_view(); + + const auto aero_ssa_sw_eamxx = + get_field_out("aero_ssa_sw").get_view(); + const auto aero_tau_sw_eamxx = + get_field_out("aero_tau_sw").get_view(); + //tau_w_f : aerosol forward scattered fraction * tau * w + const auto tau_f_sw = tau_f_sw_; + const auto aodvis = get_field_out("aodvis").get_view(); + + // NOTE! we need a const mam_coupling::DryAtmosphere dry_atm for gpu access. + // We cannot use member of this class inside of the parallel_for + const mam_coupling::DryAtmosphere &dry_atm = dry_atm_; + const auto &p_del = p_del_; + const auto &ssa_cmip6_sw = ssa_cmip6_sw_; + const auto &af_cmip6_sw = af_cmip6_sw_; + const auto &ext_cmip6_sw = ext_cmip6_sw_; + const auto &ext_cmip6_lw = ext_cmip6_lw_; + const auto &work = work_; + const auto &dry_aero = dry_aero_; + const auto &aerosol_optics_device_data = aerosol_optics_device_data_; + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const ThreadTeam &team) { + const Int icol = team.league_rank(); // column index + // absorption optical depth, per layer [unitless] + auto odap_aer_icol = ekat::subview(aero_tau_lw, icol); + const auto atm = mam_coupling::atmosphere_for_column(dry_atm, icol); + + // FIXME: interface pressure [Pa] + auto pint = ekat::subview(dry_atm.p_int, icol); + // FIXME: dry mass pressure interval [Pa] + auto zi = ekat::subview(dry_atm.z_iface, icol); + auto pdel = ekat::subview(p_del, icol); + auto pdeldry = ekat::subview(dry_atm.p_del, icol); + + auto ssa_cmip6_sw_icol = ekat::subview(ssa_cmip6_sw, icol); + auto af_cmip6_sw_icol = ekat::subview(af_cmip6_sw, icol); + auto ext_cmip6_sw_icol = ekat::subview(ext_cmip6_sw, icol); + auto ext_cmip6_lw_icol = ekat::subview(ext_cmip6_lw, icol); + + // tau_w: aerosol single scattering albedo * tau + auto tau_w_icol = ekat::subview(tau_ssa_sw, icol); + // tau_w_g: aerosol assymetry + // parameter * tau * w + auto tau_w_g_icol = ekat::subview(tau_ssa_g_sw, icol); + // tau_w_f: aero_tau_forward + // forward scattered fraction * tau * w + auto tau_w_f_icol = ekat::subview(tau_f_sw, icol); + // tau: aerosol + // aerosol extinction optical depth + auto tau_icol = ekat::subview(tau_sw, icol); + + auto work_icol = ekat::subview(work, icol); + + // fetch column-specific subviews into aerosol prognostics + mam4::Prognostics progs = + mam_coupling::aerosols_for_column(dry_aero, icol); + + mam4::aer_rad_props::aer_rad_props_sw( + team, dt, progs, atm, zi, pint, pdel, pdeldry, ssa_cmip6_sw_icol, + af_cmip6_sw_icol, ext_cmip6_sw_icol, tau_icol, tau_w_icol, + tau_w_g_icol, tau_w_f_icol, aerosol_optics_device_data, aodvis(icol), work_icol); + + team.team_barrier(); + mam4::aer_rad_props::aer_rad_props_lw( + team, dt, progs, atm, pint, zi, pdel, pdeldry, ext_cmip6_lw_icol, + aerosol_optics_device_data, odap_aer_icol); + }); + Kokkos::fence(); + // TODO: We will need to generate optical inputs files with band ordering that is consistent with + // RRTMGP + // Optical files depend on the band ordering in RRTMG. As a temporary fix, + // we are correcting the band ordering of mam4xx's ouputs, so that they are consistent with + // inputs in RRTMGP. + // Mapping from old RRTMG sw bands to new band ordering in RRTMGP + // than rrtmgp mam4 layout: (ncols, nswlands, nlevs +1 ) rrtmgp in emaxx: + // (ncols, nswlands, nlevs) Here, we copy data from kk=1 in mam4xx + // Here, we are following: E3SM/components/eam/src/physics/rrtmgp + ///cam_optics.F90 + const auto& get_idx_rrtmgp_from_rrtmg_swbands = get_idx_rrtmgp_from_rrtmg_swbands_; + // postprocess output + Kokkos::parallel_for("postprocess", policy, postprocess_); + Kokkos::fence(); + + // nswbands loop is using rrtmg indexing. + Kokkos::parallel_for( + "copying data from mam4xx to eamxx", + Kokkos::MDRangePolicy >({0, 0, 0}, + {ncol_, nswbands_, nlev_}), + KOKKOS_LAMBDA(const int icol, const int iswband, const int kk) { + // Extract single scattering albedo from the product-defined fields + if (tau_sw(icol, iswband, kk + 1) > zero) { + aero_ssa_sw_eamxx(icol, get_idx_rrtmgp_from_rrtmg_swbands(iswband), kk) = + tau_ssa_sw(icol, iswband, kk + 1)/tau_sw(icol, iswband, kk + 1); + } else { + aero_ssa_sw_eamxx(icol, get_idx_rrtmgp_from_rrtmg_swbands(iswband), kk) = one; + } + // Extract assymmetry parameter from the product-defined fields + if (tau_ssa_sw(icol, iswband, kk + 1) > zero ) { + aero_g_sw_eamxx(icol, get_idx_rrtmgp_from_rrtmg_swbands(iswband), kk) = + tau_ssa_g_sw(icol, iswband, kk + 1)/tau_ssa_sw(icol, iswband, kk + 1) ; + } else { + aero_g_sw_eamxx(icol, get_idx_rrtmgp_from_rrtmg_swbands(iswband), kk) = zero; + } + // Copy cloud optical depth over directly + aero_tau_sw_eamxx(icol, get_idx_rrtmgp_from_rrtmg_swbands(iswband), kk) = + tau_sw(icol, iswband, kk + 1); + }); + Kokkos::fence(); +} + +void MAMOptics::finalize_impl() {} -} // namespace scream +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.hpp index ebd69437dee1..466cdcb9febe 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.hpp @@ -1,14 +1,14 @@ #ifndef EAMXX_MAM_OPTICS_HPP #define EAMXX_MAM_OPTICS_HPP -#include -#include -#include - #include #include #include - +#include +#include +#include +#include +#include #include #ifndef KOKKOS_ENABLE_CUDA @@ -19,8 +19,7 @@ #define private_except_cuda private #endif -namespace scream -{ +namespace scream { // The process responsible for handling MAM4 aerosol optical properties. The AD // stores exactly ONE instance of this class in its list of subcomponents. @@ -34,30 +33,121 @@ class MAMOptics final : public scream::AtmosphereProcess { // a thread team dispatched to a single vertical column using ThreadTeam = mam4::ThreadTeam; -public: - + public: // Constructor - MAMOptics(const ekat::Comm& comm, const ekat::ParameterList& params); + MAMOptics(const ekat::Comm &comm, const ekat::ParameterList ¶ms); -protected_except_cuda: + protected_except_cuda : - // -------------------------------------------------------------------------- - // AtmosphereProcess overrides (see share/atm_process/atmosphere_process.hpp) - // -------------------------------------------------------------------------- + // -------------------------------------------------------------------------- + // AtmosphereProcess overrides (see + // share/atm_process/atmosphere_process.hpp) + // -------------------------------------------------------------------------- - // process metadata - AtmosphereProcessType type() const override; + // process metadata + AtmosphereProcessType + type() const override; std::string name() const override; // grid - void set_grids(const std::shared_ptr grids_manager) override; + void set_grids( + const std::shared_ptr grids_manager) override; + + // management of common atm process memory + size_t requested_buffer_size_in_bytes() const override; + void init_buffers(const ATMBufferManager &buffer_manager) override; // process behavior void initialize_impl(const RunType run_type) override; void run_impl(const double dt) override; void finalize_impl() override; -private_except_cuda: + private_except_cuda : + // FIXME: duplicate code from microphysics: ask it can be moved to place + // where other process can see it. Atmosphere processes often have a + // pre-processing step that constructs required variables from the set of + // fields stored in the field manager. This functor implements this step, + // which is called during run_impl. + struct Preprocess { + Preprocess() = default; + + // on host: initializes preprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::AerosolState &wet_aero, + const mam_coupling::DryAtmosphere &dry_atm, + const mam_coupling::AerosolState &dry_aero) { + ncol_ = ncol; + nlev_ = nlev; + wet_atm_ = wet_atm; + wet_aero_ = wet_aero; + dry_atm_ = dry_atm; + dry_aero_ = dry_aero; + } + + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + // first, compute dry fields + compute_dry_mixing_ratios(team, wet_atm_, dry_atm_, i); + compute_dry_mixing_ratios(team, wet_atm_, wet_aero_, dry_aero_, i); + team.team_barrier(); + // second, we can use dry fields to compute dz, zmin, zint + compute_vertical_layer_heights(team, dry_atm_, i); + compute_updraft_velocities(team, wet_atm_, dry_atm_, i); + } // operator() + + // number of horizontal columns and vertical levels + int ncol_, nlev_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + mam_coupling::AerosolState wet_aero_, dry_aero_; + + }; // MAMMicrophysics::Preprocess + + // Postprocessing functor + struct Postprocess { + Postprocess() = default; + + // on host: initializes postprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::AerosolState &wet_aero, + const mam_coupling::DryAtmosphere &dry_atm, + const mam_coupling::AerosolState &dry_aero) { + ncol_ = ncol; + nlev_ = nlev; + wet_atm_ = wet_atm; + wet_aero_ = wet_aero; + dry_atm_ = dry_atm; + dry_aero_ = dry_aero; + } + + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + compute_wet_mixing_ratios(team, dry_atm_, dry_aero_, wet_aero_, i); + } // operator() + + // number of horizontal columns and vertical levels + int ncol_, nlev_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + mam_coupling::AerosolState wet_aero_, dry_aero_; + }; // MAMMicrophysics::Postprocess + + // pre- and postprocessing scratch pads (for wet <-> dry conversions) + Preprocess preprocess_; + Postprocess postprocess_; + + // state variable + // mam_coupling::view_3d state_q_, qqcw_;// odap_aer_, // number of horizontal columns and vertical levels int ncol_, nlev_; @@ -65,16 +155,33 @@ class MAMOptics final : public scream::AtmosphereProcess { // number of shortwave and longwave radiation bands int nswbands_, nlwbands_; + // FIXME: move these values to mam_coupling + mam_coupling::const_view_2d p_int_, p_del_; + // MAM4 aerosol particle size description mam4::AeroConfig aero_config_; - // aerosol processes - //std::unique_ptr optics_; + // atmospheric and aerosol state variables + // atmospheric and aerosol state variables + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + mam_coupling::AerosolState wet_aero_, dry_aero_; + mam_coupling::view_3d ssa_cmip6_sw_, af_cmip6_sw_, ext_cmip6_sw_; + // long wave extinction in the units of [1/km] + mam_coupling::view_3d ext_cmip6_lw_; + mam4::modal_aer_opt::AerosolOpticsDeviceData aerosol_optics_device_data_; // physics grid for column information std::shared_ptr grid_; -}; // MAMOptics + mam_coupling::view_2d work_; + mam_coupling::view_3d tau_ssa_g_sw_, tau_ssa_sw_, tau_sw_, tau_f_sw_; + //Mapping from old RRTMG sw bands to new band ordering in RRTMGP + // given old index swband (RRTMG) return new index swband RRTMGP + mam_coupling::view_int_1d get_idx_rrtmgp_from_rrtmg_swbands_; + + mam_coupling::Buffer buffer_; +}; // MAMOptics -} // namespace scream +} // namespace scream -#endif // EAMXX_MAM_OPTICS_HPP +#endif // EAMXX_MAM_OPTICS_HPP diff --git a/components/eamxx/src/physics/mam/impl/README.md b/components/eamxx/src/physics/mam/impl/README.md new file mode 100644 index 000000000000..f05a484a82aa --- /dev/null +++ b/components/eamxx/src/physics/mam/impl/README.md @@ -0,0 +1,11 @@ +# MAM4 Integration Code + +This folder contains C++ implementations of the higher-level MAM4 interface +routines for aerosol microphysics, cloud-aerosol interactions, and optical +properties. We've retained the overall structure of the original Fortran code +to make it easier to understand for folks who are more familiar with the +original implementation of MAM4. + +## Contents + +* `mam4_amicphys.cpp` - high-level MAM4 microphysics interface code diff --git a/components/eamxx/src/physics/mam/impl/compute_o3_column_density.cpp b/components/eamxx/src/physics/mam/impl/compute_o3_column_density.cpp new file mode 100644 index 000000000000..b31537eef83d --- /dev/null +++ b/components/eamxx/src/physics/mam/impl/compute_o3_column_density.cpp @@ -0,0 +1,44 @@ +namespace scream::impl { + +KOKKOS_INLINE_FUNCTION +void compute_o3_column_density(const ThreadTeam& team, const haero::Atmosphere& atm, + const mam4::Prognostics &progs, ColumnView o3_col_dens) { + constexpr int nabscol = mam4::gas_chemistry::nabscol; // number of absorbing densities + constexpr int gas_pcnst = mam4::gas_chemistry::gas_pcnst; // number of gas phase species + constexpr int nfs = mam4::gas_chemistry::nfs; // number of "fixed species" + constexpr Real mwdry = 1.0/haero::Constants::molec_weight_dry_air; + + Real o3_col_deltas[mam4::nlev+1] = {}; // o3 column density above model [1/cm^2] + // NOTE: if we need o2 column densities, set_ub_col and setcol must be changed + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, atm.num_levels()), [&](const int k) { + + Real temp = atm.temperature(k); + Real pmid = atm.pressure(k); + Real pdel = atm.hydrostatic_dp(k); + Real qv = atm.vapor_mixing_ratio(k); + + // ... map incoming mass mixing ratios to working array + Real q[gas_pcnst], qqcw[gas_pcnst]; + mam_coupling::transfer_prognostics_to_work_arrays(progs, k, q, qqcw); + + // ... set atmosphere mean mass to the molecular weight of dry air + // and compute water vapor vmr + Real mbar = mwdry; + Real h2ovmr = mam4::conversions::vmr_from_mmr(qv, mbar); + + // ... Xform from mmr to vmr + Real vmr[gas_pcnst], vmrcw[gas_pcnst]; + mam_coupling::convert_work_arrays_to_vmr(q, qqcw, vmr, vmrcw); + + // ... compute invariants for this level + Real invariants[nfs]; + // setinv(invariants, temp, h2ovmr, vmr, pmid); FIXME: not yet ported + + // compute the change in o3 density for this column above its neighbor + mam4::mo_photo::set_ub_col(o3_col_deltas[k+1], vmr, invariants, pdel); + }); + // sum the o3 column deltas to densities + mam4::mo_photo::setcol(o3_col_deltas, o3_col_dens); +} + +} // namespace scream::impl diff --git a/components/eamxx/src/physics/mam/impl/compute_water_content.cpp b/components/eamxx/src/physics/mam/impl/compute_water_content.cpp new file mode 100644 index 000000000000..ea7190afff91 --- /dev/null +++ b/components/eamxx/src/physics/mam/impl/compute_water_content.cpp @@ -0,0 +1,99 @@ +#include + +namespace scream::impl { + +KOKKOS_INLINE_FUNCTION +void compute_water_content(const mam4::Prognostics &progs, int k, + Real qv, Real temp, Real pmid, + Real dgncur_a[mam4::AeroConfig::num_modes()], + Real dgncur_awet[mam4::AeroConfig::num_modes()], + Real wetdens[mam4::AeroConfig::num_modes()], + Real qaerwat[mam4::AeroConfig::num_modes()]) { + constexpr int num_modes = mam4::AeroConfig::num_modes(); + constexpr int num_aero_ids = mam4::AeroConfig::num_aerosol_ids(); + + // get some information about aerosol species + // FIXME: this isn't great! + constexpr int maxd_aspectype = mam4::water_uptake::maxd_aspectype; + int nspec_amode[num_modes], lspectype_amode[maxd_aspectype][num_modes]; + Real specdens_amode[maxd_aspectype], spechygro[maxd_aspectype]; + mam4::water_uptake::get_e3sm_parameters(nspec_amode, lspectype_amode, + specdens_amode, spechygro); + + // extract aerosol tracers for this level into state_q, which is needed + // for computing dry aerosol properties below + // FIXME: we should eliminate this index translation stuff + constexpr int nvars = aero_model::pcnst; + Real state_q[nvars]; // aerosol tracers for level k + for (int imode = 0; imode < num_modes; ++imode) { + int la, lc; // interstitial and cloudborne indices within state_q + + // number mixing ratios + mam4::convproc::assign_la_lc(imode, -1, la, lc); + state_q[la] = progs.n_mode_i[imode](k); + state_q[lc] = progs.n_mode_c[imode](k); + // aerosol mass mixing ratios + for (int iaero = 0; iaero < num_aero_ids; ++iaero) { + mam4::convproc::assign_la_lc(imode, iaero, la, lc); + auto mode = static_cast(imode); + auto aero = static_cast(iaero); + int ispec = mam4::aerosol_index_for_mode(mode, aero); + if (ispec != -1) { + state_q[la] = progs.q_aero_i[imode][ispec](k); + state_q[lc] = progs.q_aero_c[imode][ispec](k); + } + } + } + + // compute the dry volume for each mode, and from it the current dry + // geometric nominal particle diameter. + // FIXME: We have to do some gymnastics here to set up the calls to + // FIXME: calcsize. This could be improved. + Real inv_densities[num_modes][num_aero_ids] = {}; + for (int imode = 0; imode < num_modes; ++imode) { + const int n_spec = mam4::num_species_mode(imode); + for (int ispec = 0; ispec < n_spec; ++ispec) { + const int iaer = static_cast(mam4::mode_aero_species(imode, ispec)); + const Real density = mam4::aero_species(iaer).density; + inv_densities[imode][ispec] = 1.0 / density; + } + } + for (int imode = 0; imode < num_modes; ++imode) { + Real dryvol_i, dryvol_c; // interstitial and cloudborne dry volumes + mam4::calcsize::compute_dry_volume_k(k, imode, inv_densities, progs, + dryvol_i, dryvol_c); + + // NOTE: there's some disagreement over whether vol2num should be called + // NOTE: num2vol here, so I'm just adopting the nomenclature used by + // NOTE: the following call to calcsize) + const mam4::Mode& mode = mam4::modes(imode); + Real vol2num_min = 1.0/mam4::conversions::mean_particle_volume_from_diameter( + mode.max_diameter, mode.mean_std_dev); + Real vol2num_max = 1.0/mam4::conversions::mean_particle_volume_from_diameter( + mode.min_diameter, mode.mean_std_dev); + Real vol2num; + mam4::calcsize::update_diameter_and_vol2num(dryvol_i, + progs.n_mode_i[imode](k), vol2num_min, vol2num_max, + mode.min_diameter, mode.max_diameter, mode.mean_std_dev, + dgncur_a[imode], vol2num); + } + + // calculate dry aerosol properties + Real hygro[num_modes], naer[num_modes], dryrad[num_modes], + dryvol[num_modes], drymass[num_modes], + rhcrystal[num_modes], rhdeliques[num_modes], specdens_1[num_modes]; + mam4::water_uptake::modal_aero_water_uptake_dryaer(nspec_amode, specdens_amode, + spechygro, lspectype_amode, state_q, dgncur_a, hygro, + naer, dryrad, dryvol, drymass, rhcrystal, rhdeliques, specdens_1); + + // calculate wet aerosol properties + Real rh = mam4::conversions::relative_humidity_from_vapor_mixing_ratio(qv, temp, pmid); + Real wetrad[num_modes], wetvol[num_modes], wtrvol[num_modes]; + mam4::water_uptake::modal_aero_water_uptake_wetaer(rhcrystal, rhdeliques, dgncur_a, + dryrad, hygro, rh, naer, dryvol, wetrad, wetvol, wtrvol, dgncur_awet, + qaerwat); + mam4::water_uptake::modal_aero_water_uptake_wetdens(wetvol, wtrvol, + drymass, specdens_1, wetdens); +} + +} // namespace scream::impl diff --git a/components/eamxx/src/physics/mam/impl/gas_phase_chemistry.cpp b/components/eamxx/src/physics/mam/impl/gas_phase_chemistry.cpp new file mode 100644 index 000000000000..98e33b3c2b7c --- /dev/null +++ b/components/eamxx/src/physics/mam/impl/gas_phase_chemistry.cpp @@ -0,0 +1,387 @@ +#include + +namespace scream::impl { + +using mam4::utils::min_max_bound; + +using HostView1D = haero::DeviceType::view_1d::HostMirror; +using HostViewInt1D = haero::DeviceType::view_1d::HostMirror; + +//------------------------------------------------------------------------- +// Reading the photolysis table +//------------------------------------------------------------------------- +// This logic is currently implemented using serial NetCDF calls for +// clarity of purpose. We should probably read the data for the photolysis +// table using SCREAM's SCORPIO interface instead, but I wanted to make +// clear what we're trying to do in terms of "elementary" operations first. + +// ON HOST (MPI root rank only), reads the dimension of a NetCDF variable from +// the file with the given ID +int nc_dimension(const char *file, int nc_id, const char *dim_name) { + int dim_id; + int result = nc_inq_dimid(nc_id, dim_name, &dim_id); + EKAT_REQUIRE_MSG(result == 0, "Error! Couldn't fetch " << dim_name << + " dimension ID from NetCDF file '" << file << "'\n"); + size_t dim; + result = nc_inq_dimlen(nc_id, dim_id, &dim); + EKAT_REQUIRE_MSG(result == 0, "Error! Couldn't fetch " << dim_name << + " dimension from NetCDF file '" << file << "'\n"); + return static_cast(dim); +} + +// ON HOST (MPI root rank only), reads data from the given NetCDF variable from +// the file with the given ID into the given Kokkos host View +template +void read_nc_var(const char *file, int nc_id, const char *var_name, V host_view) { + int var_id; + int result = nc_inq_varid(nc_id, var_name, &var_id); + EKAT_REQUIRE_MSG(result == 0, "Error! Couldn't fetch ID for variable '" << var_name << + "' from NetCDF file '" << file << "'\n"); + result = nc_get_var(nc_id, var_id, host_view.data()); + EKAT_REQUIRE_MSG(result == 0, "Error! Couldn't read data for variable '" << var_name << + "' from NetCDF file '" << file << "'\n"); +} + +// ON HOST (MPI root rank only), reads data from the NetCDF variable with the +// given ID, from the file with the given ID, into the given Kokkos host View +template +void read_nc_var(const char *file, int nc_id, int var_id, V host_view) { + int result = nc_get_var(nc_id, var_id, host_view.data()); + EKAT_REQUIRE_MSG(result == 0, "Error! Couldn't read data for variable with ID " << + var_id << " from NetCDF file '" << file << "'\n"); +} + +// ON HOST (MPI root only), sets the lng_indexer and pht_alias_mult_1 host views +// according to parameters in our (hardwired) chemical mechanism +void set_lng_indexer_and_pht_alias_mult_1(const char *file, int nc_id, + HostViewInt1D lng_indexer, + HostView1D pht_alias_mult_1) { + // NOTE: it seems that the chemical mechanism we're using + // NOTE: 1. sets pht_alias_lst to a blank string [1] + // NOTE: 2. sets pht_alias_mult_1 to 1.0 [1] + // NOTE: 3. sets rxt_tag_lst to ['jh2o2', 'usr_HO2_HO2', 'usr_SO2_OH', 'usr_DMS_OH'] [2] + // NOTE: References: + // NOTE: [1] (https://github.com/eagles-project/e3sm_mam4_refactor/blob/refactor-maint-2.0/components/eam/src/chemistry/pp_linoz_mam4_resus_mom_soag/mo_sim_dat.F90#L117) + // NOTE: [2] (https://github.com/eagles-project/e3sm_mam4_refactor/blob/refactor-maint-2.0/components/eam/src/chemistry/pp_linoz_mam4_resus_mom_soag/mo_sim_dat.F90#L99) + + // populate lng_indexer (see https://github.com/eagles-project/e3sm_mam4_refactor/blob/refactor-maint-2.0/components/eam/src/chemistry/mozart/mo_jlong.F90#L180) + static const char *var_names[4] = {"jh2o2", "usr_HO2_HO2", "usr_SO2_OH", "usr_DMS_OH"}; + for (int m = 0; m < mam4::mo_photo::phtcnt; ++m) { + int var_id; + int result = nc_inq_varid(nc_id, var_names[m], &var_id); + EKAT_REQUIRE_MSG(result == 0, "Error! Couldn't fetch ID for variable '" + << var_names[m] << "' from NetCDF file '" << file << "'\n"); + lng_indexer(m) = var_id; + } + + // set pht_alias_mult_1 to 1 + Kokkos::deep_copy(pht_alias_mult_1, 1.0); +} + +// ON HOST (MPI root only), populates the etfphot view using rebinned +// solar data from our solar_data_file +void populate_etfphot(HostView1D we, HostView1D etfphot) { + // FIXME: It looks like EAM is relying on a piece of infrastructure that + // FIXME: we just don't have in EAMxx (eam/src/chemistry/utils/solar_data.F90). + // FIXME: I have no idea whether EAMxx has a plan for supporting this + // FIXME: solar irradiance / photon flux data, and I'm not going to recreate + // FIXME: that capability here. So this is an unplugged hole. + // FIXME: + // FIXME: If we are going to do this the way EAM does it, the relevant logic + // FIXME: is the call to rebin() in eam/src/chemistry/mozart/mo_jlong.F90, + // FIXME: around line 104. + + // FIXME: zero the photon flux for now + Kokkos::deep_copy(etfphot, 0); +} + +// ON HOST, reads the photolysis table (used for gas phase chemistry) from the +// files with the given names +mam4::mo_photo::PhotoTableData read_photo_table(const ekat::Comm& comm, + const char *rsf_file, + const char* xs_long_file) { + // NOTE: at the time of development, SCREAM's SCORPIO interface seems intended + // NOTE: for domain-decomposed grid data. The files we're reading here are not + // NOTE: spatial data, and should be the same everywhere, so we read them + // NOTE: using serial NetCDF calls on MPI rank 0 and broadcast to other ranks. + const int mpi_root = 0; + int rsf_id, xs_long_id; // NetCDF file IDs (used only on MPI root) + int nw, nump, numsza, numcolo3, numalb, nt, np_xs; // table dimensions + if (comm.rank() == mpi_root) { // read dimension data from files and broadcast + // open files + int result = nc_open(rsf_file, NC_NOWRITE, &rsf_id); + EKAT_REQUIRE_MSG(result == 0, "Error! Couldn't open rsf_file '" << rsf_file << "'\n"); + result = nc_open(xs_long_file, NC_NOWRITE, &xs_long_id); + EKAT_REQUIRE_MSG(result == 0, "Error! Couldn't open xs_long_file '" << xs_long_file << "'\n"); + + // read and broadcast dimension data + nump = nc_dimension(rsf_file, rsf_id, "numz"); + numsza = nc_dimension(rsf_file, rsf_id, "numsza"); + numalb = nc_dimension(rsf_file, rsf_id, "numalb"); + numcolo3 = nc_dimension(rsf_file, rsf_id, "numcolo3fact"); + nt = nc_dimension(xs_long_file, xs_long_id, "numtemp"); + nw = nc_dimension(xs_long_file, xs_long_id, "numwl"); + np_xs = nc_dimension(xs_long_file, xs_long_id, "numprs"); + + int dim_data[7] = {nump, numsza, numcolo3, numalb, nt, nw, np_xs}; + comm.broadcast(dim_data, 7, mpi_root); + } else { // receive broadcasted dimension data from root rank + int dim_data[7]; + comm.broadcast(dim_data, 7, mpi_root); + nump = dim_data[0]; + numsza = dim_data[1]; + numcolo3 = dim_data[2]; + numalb = dim_data[3]; + nt = dim_data[4]; + nw = dim_data[5]; + np_xs = dim_data[6]; + } + + // set up the lng_indexer and pht_alias_mult_1 views based on our + // (hardwired) chemical mechanism + HostViewInt1D lng_indexer_h("lng_indexer(host)", mam4::mo_photo::phtcnt); + HostView1D pht_alias_mult_1_h("pht_alias_mult_1(host)", 2); + if (comm.rank() == mpi_root) { + set_lng_indexer_and_pht_alias_mult_1(xs_long_file, xs_long_id, + lng_indexer_h, pht_alias_mult_1_h); + } + comm.broadcast(lng_indexer_h.data(), mam4::mo_photo::phtcnt, mpi_root); + comm.broadcast(pht_alias_mult_1_h.data(), 2, mpi_root); + + // compute the size of the foremost dimension of xsqy using lng_indexer + int numj = 0; + for (int m = 0; m < mam4::mo_photo::phtcnt; ++m) { + if (lng_indexer_h(m) > 0) { + for (int mm = 0; mm < m; ++mm) { + if (lng_indexer_h(mm) == lng_indexer_h(m)) { + break; + } + ++numj; + } + } + } + + // allocate the photolysis table + auto table = mam4::mo_photo::create_photo_table_data(nw, nt, np_xs, numj, + nump, numsza, numcolo3, + numalb); + + // allocate host views for table data + auto rsf_tab_h = Kokkos::create_mirror_view(table.rsf_tab); + auto xsqy_h = Kokkos::create_mirror_view(table.xsqy); + auto sza_h = Kokkos::create_mirror_view(table.sza); + auto alb_h = Kokkos::create_mirror_view(table.alb); + auto press_h = Kokkos::create_mirror_view(table.press); + auto colo3_h = Kokkos::create_mirror_view(table.colo3); + auto o3rat_h = Kokkos::create_mirror_view(table.o3rat); + auto etfphot_h = Kokkos::create_mirror_view(table.etfphot); + auto prs_h = Kokkos::create_mirror_view(table.prs); + + if (comm.rank() == mpi_root) { // read data from files and broadcast + // read file data into our host views + read_nc_var(rsf_file, rsf_id, "pm", press_h); + read_nc_var(rsf_file, rsf_id, "sza", sza_h); + read_nc_var(rsf_file, rsf_id, "alb", alb_h); + read_nc_var(rsf_file, rsf_id, "colo3fact", o3rat_h); + read_nc_var(rsf_file, rsf_id, "colo3", colo3_h); + read_nc_var(rsf_file, rsf_id, "RSF", rsf_tab_h); + + read_nc_var(xs_long_file, xs_long_id, "pressure", prs_h); + + // read xsqy data (using lng_indexer_h for the first index) + int ndx = 0; + for (int m = 0; m < mam4::mo_photo::phtcnt; ++m) { + if (lng_indexer_h(m) > 0) { + auto xsqy_ndx_h = ekat::subview(xsqy_h, ndx); + read_nc_var(xs_long_file, xs_long_id, lng_indexer_h(m), xsqy_ndx_h); + ++ndx; + } + } + + // populate etfphot by rebinning solar data + HostView1D wc_h("wc", nw), wlintv_h("wlintv", nw), we_h("we", nw+1); + read_nc_var(rsf_file, rsf_id, "wc", wc_h); + read_nc_var(rsf_file, rsf_id, "wlintv", wlintv_h); + for (int i = 0; i < nw; ++i) { + we_h(i) = wc_h(i) - 0.5 * wlintv_h(i); + } + we_h(nw) = wc_h(nw-1) - 0.5 * wlintv_h(nw-1); + populate_etfphot(we_h, etfphot_h); + + // close the files + nc_close(rsf_id); + nc_close(xs_long_id); + } + + // broadcast host views from MPI root to others + comm.broadcast(rsf_tab_h.data(), nw*numalb*numcolo3*numsza*nump, mpi_root); + comm.broadcast(xsqy_h.data(), numj*nw*nt*np_xs, mpi_root); + comm.broadcast(sza_h.data(), numsza, mpi_root); + comm.broadcast(alb_h.data(), numalb, mpi_root); + comm.broadcast(press_h.data(), nump, mpi_root); + comm.broadcast(o3rat_h.data(), numcolo3, mpi_root); + comm.broadcast(colo3_h.data(), nump, mpi_root); + comm.broadcast(etfphot_h.data(), nw, mpi_root); + comm.broadcast(prs_h.data(), np_xs, mpi_root); + + // copy host photolysis table into place on device + Kokkos::deep_copy(table.rsf_tab, rsf_tab_h); + Kokkos::deep_copy(table.xsqy, xsqy_h); + Kokkos::deep_copy(table.sza, sza_h); + Kokkos::deep_copy(table.alb, alb_h); + Kokkos::deep_copy(table.press, press_h); + Kokkos::deep_copy(table.colo3, colo3_h); + Kokkos::deep_copy(table.o3rat, o3rat_h); + Kokkos::deep_copy(table.etfphot, etfphot_h); + Kokkos::deep_copy(table.prs, prs_h); + Kokkos::deep_copy(table.pht_alias_mult_1, pht_alias_mult_1_h); + Kokkos::deep_copy(table.lng_indexer, lng_indexer_h); + + // compute gradients (on device) + Kokkos::parallel_for("del_p", nump-1, KOKKOS_LAMBDA(int i) { + table.del_p(i) = 1.0/::abs(table.press(i)- table.press(i+1)); + }); + Kokkos::parallel_for("del_sza", numsza-1, KOKKOS_LAMBDA(int i) { + table.del_sza(i) = 1.0/(table.sza(i+1) - table.sza(i)); + }); + Kokkos::parallel_for("del_alb", numalb-1, KOKKOS_LAMBDA(int i) { + table.del_alb(i) = 1.0/(table.alb(i+1) - table.alb(i)); + }); + Kokkos::parallel_for("del_o3rat", numcolo3-1, KOKKOS_LAMBDA(int i) { + table.del_o3rat(i) = 1.0/(table.o3rat(i+1) - table.o3rat(i)); + }); + Kokkos::parallel_for("dprs", np_xs-1, KOKKOS_LAMBDA(int i) { + table.dprs(i) = 1.0/(table.prs(i) - table.prs(i+1)); + }); + + return table; +} + +// performs gas phase chemistry calculations on a single level of a single +// atmospheric column +KOKKOS_INLINE_FUNCTION +void gas_phase_chemistry(Real zm, Real zi, Real phis, Real temp, Real pmid, Real pdel, Real dt, + const Real photo_rates[mam4::mo_photo::phtcnt], // in + const Real extfrc[mam4::gas_chemistry::extcnt], // in + Real q[mam4::gas_chemistry::gas_pcnst], // VMRs, inout + Real invariants[mam4::gas_chemistry::nfs]) { // out + constexpr Real rga = 1.0/haero::Constants::gravity; + constexpr Real m2km = 0.01; // converts m -> km + + // The following things are chemical mechanism dependent! See mam4xx/src/mam4xx/gas_chem_mechanism.hpp) + constexpr int gas_pcnst = mam4::gas_chemistry::gas_pcnst; // number of gas phase species + constexpr int rxntot = mam4::gas_chemistry::rxntot; // number of chemical reactions + constexpr int extcnt = mam4::gas_chemistry::extcnt; // number of species with external forcing + constexpr int nabscol = mam4::gas_chemistry::nabscol; // number of "absorbing column densities" + constexpr int indexm = 0; // FIXME: index of total atm density in invariants array + + constexpr int phtcnt = mam4::mo_photo::phtcnt; // number of photolysis reactions + + constexpr int itermax = mam4::gas_chemistry::itermax; + constexpr int clscnt4 = mam4::gas_chemistry::clscnt4; + + // NOTE: vvv these arrays were copied from mam4xx/gas_chem_mechanism.hpp vvv + constexpr int permute_4[gas_pcnst] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29}; + constexpr int clsmap_4[gas_pcnst] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30}; + + // These indices for species are fixed by the chemical mechanism + // std::string solsym[] = {"O3", "H2O2", "H2SO4", "SO2", "DMS", "SOAG", + // "so4_a1", "pom_a1", "soa_a1", "bc_a1", "dst_a1", + // "ncl_a1", "mom_a1", "num_a1", "so4_a2", "soa_a2", + // "ncl_a2", "mom_a2", "num_a2", "dst_a3", "ncl_a3", + // "so4_a3", "bc_a3", "pom_a3", "soa_a3", "mom_a3", + // "num_a3", "pom_a4", "bc_a4", "mom_a4", "num_a4"}; + constexpr int ndx_h2so4 = 2; + constexpr int o3_ndx = 0; + // std::string extfrc_list[] = {"SO2", "so4_a1", "so4_a2", "pom_a4", "bc_a4", + // "num_a1", "num_a2", "num_a3", "num_a4", "SOAG"}; + constexpr int synoz_ndx = -1; + + // fetch the zenith angle (not its cosine!) in degrees for this column. + // FIXME: For now, we fix the zenith angle. At length, we need to compute it + // FIXME: from EAMxx's current set of orbital parameters, which requires some + // FIXME: conversation with the EAMxx team. + Real zen_angle = 0.0; // [deg] + + // xform geopotential height from m to km and pressure from Pa to mb + Real zsurf = rga * phis; + Real zintr = m2km * zi; + Real zmid = m2km * (zm + zsurf); + Real zint = m2km * (zi + zsurf); + + // ... compute the column's invariants + Real h2ovmr = q[0]; + // setinv(invariants, temp, h2ovmr, q, pmid); FIXME: not ported yet + + // ... set rates for "tabular" and user specified reactions + Real reaction_rates[rxntot]; + mam4::gas_chemistry::setrxt(reaction_rates, temp); + + // set reaction rates based on chemical invariants + // (indices (ndxes?) are taken from mam4 validation data and translated from + // 1-based indices to 0-based indices) + int usr_HO2_HO2_ndx = 1, usr_DMS_OH_ndx = 5, + usr_SO2_OH_ndx = 3, inv_h2o_ndx = 3; + mam4::gas_chemistry::usrrxt(reaction_rates, temp, invariants, invariants[indexm], + usr_HO2_HO2_ndx, usr_DMS_OH_ndx, + usr_SO2_OH_ndx, inv_h2o_ndx); + mam4::gas_chemistry::adjrxt(reaction_rates, invariants, invariants[indexm]); + + //=================================== + // Photolysis rates at time = t(n+1) + //=================================== + + // compute the rate of change from forcing + Real extfrc_rates[extcnt]; // [1/cm^3/s] + for (int mm = 0; mm < extcnt; ++mm) { + if (mm != synoz_ndx) { + extfrc_rates[mm] = extfrc[mm] / invariants[indexm]; + } + } + + // ... Form the washout rates + Real het_rates[gas_pcnst]; + // FIXME: not ported yet + //sethet(het_rates, pmid, zmid, phis, temp, cmfdqr, prain, nevapr, delt, + // invariants[indexm], q); + + int ltrop_sol = 0; // apply solver to all levels + + // save h2so4 before gas phase chem (for later new particle nucleation) + Real del_h2so4_gasprod = q[ndx_h2so4]; + + //=========================== + // Class solution algorithms + //=========================== + + // copy photolysis rates into reaction_rates (assumes photolysis rates come first) + for (int i = 0; i < phtcnt; ++i) { + reaction_rates[i] = photo_rates[i]; + } + + // ... solve for "Implicit" species + bool factor[itermax]; + for (int i = 0; i < itermax; ++i) { + factor[i] = true; + } + + // initialize error tolerances + Real epsilon[clscnt4]; + mam4::gas_chemistry::imp_slv_inti(epsilon); + + // solve chemical system implicitly + Real prod_out[clscnt4], loss_out[clscnt4]; + mam4::gas_chemistry::imp_sol(q, reaction_rates, het_rates, extfrc_rates, dt, + permute_4, clsmap_4, factor, epsilon, prod_out, loss_out); + + // save h2so4 change by gas phase chem (for later new particle nucleation) + if (ndx_h2so4 > 0) { + del_h2so4_gasprod = q[ndx_h2so4] - del_h2so4_gasprod; + } +} + +} // namespace scream::impl diff --git a/components/eamxx/src/physics/mam/impl/mam4_amicphys.cpp b/components/eamxx/src/physics/mam/impl/mam4_amicphys.cpp new file mode 100644 index 000000000000..71cee1eeb407 --- /dev/null +++ b/components/eamxx/src/physics/mam/impl/mam4_amicphys.cpp @@ -0,0 +1,1769 @@ +#include +#include +#include +#include +#include +#include + +namespace scream::impl { + +#define MAX_FILENAME_LEN 256 + +using namespace mam4; + +// number of constituents in gas chemistry "work arrays" +KOKKOS_INLINE_FUNCTION +constexpr int gas_pcnst() { + constexpr int gas_pcnst_ = mam4::gas_chemistry::gas_pcnst; + return gas_pcnst_; +} + +// number of aerosol/gas species tendencies +KOKKOS_INLINE_FUNCTION +constexpr int nqtendbb() { return 4; } + +// MAM4 aerosol microphysics configuration data +struct AmicPhysConfig { + // these switches activate various aerosol microphysics processes + bool do_cond; // condensation (a.k.a gas-aerosol exchange) + bool do_rename; // mode "renaming" + bool do_newnuc; // gas -> aerosol nucleation + bool do_coag; // aerosol coagulation + + // configurations for specific aerosol microphysics + mam4::GasAerExchProcess::ProcessConfig condensation; + mam4::NucleationProcess::ProcessConfig nucleation; + + // controls treatment of h2so4 condensation in mam_gasaerexch_1subarea + // 1 = sequential calc. of gas-chem prod then condensation loss + // 2 = simultaneous calc. of gas-chem prod and condensation loss + int gaexch_h2so4_uptake_optaa; + + // controls how nucleation interprets h2so4 concentrations + int newnuc_h2so4_conc_optaa; +}; + +namespace { + +KOKKOS_INLINE_FUNCTION constexpr int nqtendaa() { return 5; } +KOKKOS_INLINE_FUNCTION constexpr int nqqcwtendaa() { return 1; } +KOKKOS_INLINE_FUNCTION constexpr int nqqcwtendbb() { return 1; } +KOKKOS_INLINE_FUNCTION constexpr int iqtend_cond() { return 0; } +KOKKOS_INLINE_FUNCTION constexpr int iqtend_rnam() { return 1; } +KOKKOS_INLINE_FUNCTION constexpr int iqtend_nnuc() { return 2; } +KOKKOS_INLINE_FUNCTION constexpr int iqtend_coag() { return 3; } +KOKKOS_INLINE_FUNCTION constexpr int iqtend_cond_only() { return 4; } +KOKKOS_INLINE_FUNCTION constexpr int iqqcwtend_rnam() { return 0; } +KOKKOS_INLINE_FUNCTION constexpr int maxsubarea() { return 2; } + +// conversion factors +KOKKOS_INLINE_FUNCTION Real fcvt_gas(int gas_id) { + static const Real fcvt_gas_[AeroConfig::num_gas_ids()] = {1, 1, 1}; + return fcvt_gas_[gas_id]; +} +KOKKOS_INLINE_FUNCTION Real fcvt_aer(int aero_id) { + static const Real fcvt_aer_[AeroConfig::num_aerosol_ids()] = {1, 1, 1, 1, 1, 1, 1}; + return fcvt_aer_[aero_id]; +} + +// leave number mix-ratios unchanged (#/kmol-air) +KOKKOS_INLINE_FUNCTION Real fcvt_num() { return 1.0; } +// factor for converting aerosol water mix-ratios from (kg/kg) to (mol/mol) +KOKKOS_INLINE_FUNCTION Real fcvt_wtr() { return 1.0; } + +KOKKOS_INLINE_FUNCTION constexpr int lmapcc_val_nul() { return 0; } +KOKKOS_INLINE_FUNCTION constexpr int lmapcc_val_gas() { return 1; } +KOKKOS_INLINE_FUNCTION constexpr int lmapcc_val_aer() { return 2; } +KOKKOS_INLINE_FUNCTION constexpr int lmapcc_val_num() { return 3; } +KOKKOS_INLINE_FUNCTION int lmapcc_all(int index) { + static const int lmapcc_all_[gas_pcnst()] = { + lmapcc_val_nul(), lmapcc_val_gas(), lmapcc_val_nul(), lmapcc_val_nul(), + lmapcc_val_gas(), lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), + lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), + lmapcc_val_num(), lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), + lmapcc_val_aer(), lmapcc_val_num(), lmapcc_val_aer(), lmapcc_val_aer(), + lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), + lmapcc_val_aer(), lmapcc_val_num(), lmapcc_val_aer(), lmapcc_val_aer(), + lmapcc_val_aer(), lmapcc_val_num()}; + return lmapcc_all_[index]; +} + +// Where lmapcc_val_num are defined in lmapcc_all +KOKKOS_INLINE_FUNCTION int numptr_amode(int mode) { + static const int numptr_amode_[AeroConfig::num_modes()] = {12, 17, 25, 29}; + return numptr_amode_[mode]; +} + +// Where lmapcc_val_gas are defined in lmapcc_all +KOKKOS_INLINE_FUNCTION int lmap_gas(int mode) { + static const int lmap_gas_[AeroConfig::num_modes()] = {4, 1}; + return lmap_gas_[mode]; +} +// Where lmapcc_val_aer are defined in lmapcc_all +KOKKOS_INLINE_FUNCTION int lmassptr_amode(int aero_id, int mode) { + static const int lmassptr_amode_[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()] = { + {5, 13, 18, 26}, {6, 14, 19, 27}, {7, 15, 20, 28}, {8, 16, 21, -6}, + {9, -6, 22, -6}, {10, -6, 23, -6}, {11, -6, 24, -6}}; + return lmassptr_amode_[aero_id][mode]; +} + +KOKKOS_INLINE_FUNCTION +void subarea_partition_factors( + const Real + q_int_cell_avg, // in grid cell mean interstitial aerosol mixing ratio + const Real + q_cbn_cell_avg, // in grid cell mean cloud-borne aerosol mixing ratio + const Real fcldy, // in cloudy fraction of the grid cell + const Real fclea, // in clear fraction of the grid cell + Real &part_fac_q_int_clea, // out + Real &part_fac_q_int_cldy) // out +{ + // Calculate mixing ratios of each subarea + + // cloud-borne, cloudy subarea + const Real tmp_q_cbn_cldy = q_cbn_cell_avg / fcldy; + // interstitial, cloudy subarea + const Real tmp_q_int_cldy = + haero::max(0.0, ((q_int_cell_avg + q_cbn_cell_avg) - tmp_q_cbn_cldy)); + // interstitial, clear subarea + const Real tmp_q_int_clea = (q_int_cell_avg - fcldy * tmp_q_int_cldy) / fclea; + + // Calculate the corresponding paritioning factors for interstitial aerosols + // using the above-derived subarea mixing ratios plus the constraint that + // the cloud fraction weighted average of subarea mean need to match grid box + // mean. + + // *** question *** + // use same part_fac_q_int_clea/cldy for everything ? + // use one for number and one for all masses (based on total mass) ? + // use separate ones for everything ? + // maybe one for number and one for all masses is best, + // because number and mass have different activation fractions + // *** question *** + + Real tmp_aa = haero::max(1.e-35, tmp_q_int_clea * fclea) / + haero::max(1.e-35, q_int_cell_avg); + tmp_aa = haero::max(0.0, haero::min(1.0, tmp_aa)); + + part_fac_q_int_clea = tmp_aa / fclea; + part_fac_q_int_cldy = (1.0 - tmp_aa) / fcldy; +} + +KOKKOS_INLINE_FUNCTION +void construct_subareas_1gridcell( + const Real cld, // in + const Real relhumgcm, // in + const Real q_pregaschem[gas_pcnst()], // in q TMRs before + // gas-phase chemistry + const Real q_precldchem[gas_pcnst()], // in q TMRs before + // cloud chemistry + const Real qqcw_precldchem[gas_pcnst()], // in qqcw TMRs before + // cloud chemistry + const Real q[gas_pcnst()], // in current tracer mixing ratios (TMRs) + // *** MUST BE #/kmol-air for number + // *** MUST BE mol/mol-air for mass + const Real qqcw[gas_pcnst()], // in like q but for + // cloud-borner tracers + int &nsubarea, // out + int &ncldy_subarea, // out + int &jclea, // out + int &jcldy, // out + bool iscldy_subarea[maxsubarea()], // out + Real afracsub[maxsubarea()], // out + Real relhumsub[maxsubarea()], // out + Real qsub1[gas_pcnst()][maxsubarea()], // out interstitial + Real qsub2[gas_pcnst()][maxsubarea()], // out interstitial + Real qsub3[gas_pcnst()][maxsubarea()], // out interstitial + Real qqcwsub1[gas_pcnst()][maxsubarea()], // out cloud-borne + Real qqcwsub2[gas_pcnst()][maxsubarea()], // out cloud-borne + Real qqcwsub3[gas_pcnst()][maxsubarea()], // outcloud-borne + Real qaerwatsub3[AeroConfig::num_modes()] + [maxsubarea()], // out aerosol water mixing ratios (mol/mol) + Real qaerwat[AeroConfig::num_modes()] // in aerosol water mixing ratio + // (kg/kg, NOT mol/mol) +) { + static constexpr int num_modes = AeroConfig::num_modes(); + // cloud chemistry is only on when cld(i,k) >= 1.0e-5_wp + // it may be that the macrophysics has a higher threshold that this + const Real fcld_locutoff = 1.0e-5; + const Real fcld_hicutoff = 0.999; + + // qgcmN and qqcwgcmN (N=1:4) are grid-cell mean tracer mixing ratios (TMRs, + // mol/mol or #/kmol) + // N=1 - before gas-phase chemistry + // N=2 - before cloud chemistry + // N=3 - incoming values (before gas-aerosol exchange, newnuc, coag) + // qgcm1, qgcm2, qgcm3 + // qqcwgcm2, qqcwgcm3 + // qaerwatgcm3 ! aerosol water mixing ratios (mol/mol) + + // -------------------------------------------------------------------------------------- + // Determine the number of sub-areas, their fractional areas, and relative + // humidities + // -------------------------------------------------------------------------------------- + // if cloud fraction ~= 0, the grid-cell has a single clear sub-area + // (nsubarea = 1) if cloud fraction ~= 1, the grid-cell has a single cloudy + // sub-area (nsubarea = 1) otherwise, the grid-cell has a + // clear and a cloudy sub-area (nsubarea = 2) + + Real zfcldy = 0; + nsubarea = 0; + ncldy_subarea = 0; + jclea = 0; + jcldy = 0; + + if (cld < fcld_locutoff) { + nsubarea = 1; + jclea = 1; + } else if (cld > fcld_hicutoff) { + zfcldy = 1.0; + nsubarea = 1; + ncldy_subarea = 1; + jcldy = 1; + } else { + zfcldy = cld; + nsubarea = 2; + ncldy_subarea = 1; + jclea = 1; + jcldy = 2; + } + + const Real zfclea = 1.0 - zfcldy; + for (int i = 0; i < maxsubarea(); ++i) + iscldy_subarea[i] = false; + if (jcldy > 0) + iscldy_subarea[jcldy - 1] = true; + for (int i = 0; i < maxsubarea(); ++i) + afracsub[i] = 0.0; + if (jclea > 0) + afracsub[jclea - 1] = zfclea; + if (jcldy > 0) + afracsub[jcldy - 1] = zfcldy; + + // cldy_rh_sameas_clear is just to match mam_refactor. Compiler should + // optimize away. + const int cldy_rh_sameas_clear = 0; + if (ncldy_subarea <= 0) { + for (int i = 0; i < maxsubarea(); ++i) + relhumsub[i] = relhumgcm; + } else if (cldy_rh_sameas_clear > 0) { + for (int i = 0; i < maxsubarea(); ++i) + relhumsub[i] = relhumgcm; + } else { + if (jcldy > 0) { + relhumsub[jcldy - 1] = 1.0; + if (jclea > 0) { + const Real tmpa = + (relhumgcm - afracsub[jcldy - 1]) / afracsub[jclea - 1]; + relhumsub[jclea - 1] = haero::max(0.0, haero::min(1.0, tmpa)); + } + } + } + + // ---------------------------------------------------------------------------- + // Copy grid cell mean mixing ratios. + // These values, together with cloud fraction and a few assumptions, are used + // in the remainder of the subroutine to calculate the sub-area mean mixing + // ratios. + // ---------------------------------------------------------------------------- + // Interstitial aerosols + Real qgcm1[gas_pcnst()], qgcm2[gas_pcnst()], qgcm3[gas_pcnst()]; + for (int i = 0; i < gas_pcnst(); ++i) { + qgcm1[i] = haero::max(0.0, q_pregaschem[i]); + qgcm2[i] = haero::max(0.0, q_precldchem[i]); + qgcm3[i] = haero::max(0.0, q[i]); + } + + // Cloud-borne aerosols + Real qqcwgcm2[gas_pcnst()], qqcwgcm3[gas_pcnst()]; + for (int i = 0; i < gas_pcnst(); ++i) { + qqcwgcm2[i] = haero::max(0.0, qqcw_precldchem[i]); + qqcwgcm3[i] = haero::max(0.0, qqcw[i]); + } + + // aerosol water + Real qaerwatgcm3[num_modes] = {}; + for (int i = 0; i < num_modes; ++i) { + qaerwatgcm3[i] = haero::max(0.0, qaerwat[i]); + } + + // ---------------------------------------------------------------------------- + // Initialize the subarea mean mixing ratios + // ---------------------------------------------------------------------------- + { + const int n = haero::min(maxsubarea(), nsubarea + 1); + for (int i = 0; i < n; ++i) { + for (int j = 0; j < gas_pcnst(); ++j) { + qsub1[j][i] = 0.0; + qsub2[j][i] = 0.0; + qsub3[j][i] = 0.0; + qqcwsub1[j][i] = 0.0; + qqcwsub2[j][i] = 0.0; + qqcwsub3[j][i] = 0.0; + } + for (int j = 0; j < num_modes; ++j) { + qaerwatsub3[j][i] = 0.0; + } + } + } + + // ************************************************************************************************* + // Calculate initial (i.e., before cond/rnam/nnuc/coag) tracer mixing + // ratios within the sub-areas + // - for all-clear or all-cloudy cases, the sub-area TMRs are equal to the + // grid-cell means + // - for partly cloudy case, they are different. This is primarily + // because the + // interstitial aerosol mixing ratios are assumed lower in the cloudy + // sub-area than in the clear sub-area, because much of the aerosol is + // activated in the cloudy sub-area. + // ************************************************************************************************* + // Category I: partly cloudy case + // ************************************************************************************************* + if ((jclea > 0) && (jcldy > 0) && (jclea + jcldy == 3) && (nsubarea == 2)) { + + // --------------------------------------------------------------------- + // Set GAS mixing ratios in sub-areas (for the condensing gases only!!) + // --------------------------------------------------------------------- + for (int lmz = 0; lmz < gas_pcnst(); ++lmz) { + if (lmapcc_all(lmz) == lmapcc_val_gas()) { + + // assume gas in both sub-areas before gas-chem and cloud-chem equal + // grid-cell mean + for (int i = 0; i < nsubarea; ++i) { + qsub1[lmz][i] = qgcm1[lmz]; + qsub2[lmz][i] = qgcm2[lmz]; + } + // assume gas in clear sub-area after cloud-chem equals before + // cloud-chem value + qsub3[lmz][jclea - 1] = qsub2[lmz][jclea - 1]; + // gas in cloud sub-area then determined by grid-cell mean and clear + // values + qsub3[lmz][jcldy - 1] = + (qgcm3[lmz] - zfclea * qsub3[lmz][jclea - 1]) / zfcldy; + + // check that this does not produce a negative value + if (qsub3[lmz][jcldy - 1] < 0.0) { + qsub3[lmz][jcldy - 1] = 0.0; + qsub3[lmz][jclea - 1] = qgcm3[lmz] / zfclea; + } + } + } + // --------------------------------------------------------------------- + // Set CLOUD-BORNE AEROSOL mixing ratios in sub-areas. + // This is straightforward, as the same partitioning factors (0 or 1/f) + // are applied to all mass and number mixing ratios in all modes. + // --------------------------------------------------------------------- + // loop thru log-normal modes + for (int n = 0; n < num_modes; ++n) { + // number - then mass of individual species - of a mode + for (int l2 = -1; l2 < num_species_mode(n); ++l2) { + int lc; + if (l2 == -1) + lc = numptr_amode(n); + else + lc = lmassptr_amode(l2, n); + qqcwsub2[lc][jclea - 1] = 0.0; + qqcwsub2[lc][jcldy - 1] = qqcwgcm2[lc] / zfcldy; + qqcwsub3[lc][jclea - 1] = 0.0; + qqcwsub3[lc][jcldy - 1] = qqcwgcm3[lc] / zfcldy; + } + } + + // --------------------------------------------------------------------- + // Set INTERSTITIAL AEROSOL mixing ratios in sub-areas. + // --------------------------------------------------------------------- + for (int n = 0; n < num_modes; ++n) { + // ------------------------------------- + // Aerosol number + // ------------------------------------- + // grid cell mean, interstitial + Real tmp_q_cellavg_int = qgcm2[numptr_amode(n)]; + // grid cell mean, cloud-borne + Real tmp_q_cellavg_cbn = qqcwgcm2[numptr_amode(n)]; + + Real nmbr_part_fac_clea = 0; + Real nmbr_part_fac_cldy = 0; + subarea_partition_factors(tmp_q_cellavg_int, tmp_q_cellavg_cbn, zfcldy, + zfclea, nmbr_part_fac_clea, nmbr_part_fac_cldy); + + // Apply the partitioning factors to calculate sub-area mean number + // mixing ratios + + const int la = numptr_amode(n); + + qsub2[la][jclea - 1] = qgcm2[la] * nmbr_part_fac_clea; + qsub2[la][jcldy - 1] = qgcm2[la] * nmbr_part_fac_cldy; + qsub3[la][jclea - 1] = qgcm3[la] * nmbr_part_fac_clea; + qsub3[la][jcldy - 1] = qgcm3[la] * nmbr_part_fac_cldy; + + //------------------------------------- + // Aerosol mass + //------------------------------------- + // For aerosol mass, we use the total grid cell mean + // interstitial/cloud-borne mass mixing ratios to come up with the same + // partitioning for all species in the mode. + + // Compute the total mixing ratios by summing up the individual species + + tmp_q_cellavg_int = 0.0; // grid cell mean, interstitial + tmp_q_cellavg_cbn = 0.0; // grid cell mean, cloud-borne + + for (int l2 = 0; l2 < num_species_mode(n); ++l2) { + tmp_q_cellavg_int += qgcm2[lmassptr_amode(l2, n)]; + tmp_q_cellavg_cbn += qqcwgcm2[lmassptr_amode(l2, n)]; + } + Real mass_part_fac_clea = 0; + Real mass_part_fac_cldy = 0; + // Calculate the partitioning factors + subarea_partition_factors(tmp_q_cellavg_int, tmp_q_cellavg_cbn, zfcldy, + zfclea, mass_part_fac_clea, mass_part_fac_cldy); + + // Apply the partitioning factors to calculate sub-area mean mass mixing + // ratios + + for (int l2 = 0; l2 < num_species_mode(n); ++l2) { + const int la = lmassptr_amode(l2, n); + + qsub2[la][jclea - 1] = qgcm2[la] * mass_part_fac_clea; + qsub2[la][jcldy - 1] = qgcm2[la] * mass_part_fac_cldy; + qsub3[la][jclea - 1] = qgcm3[la] * mass_part_fac_clea; + qsub3[la][jcldy - 1] = qgcm3[la] * mass_part_fac_cldy; + } + } + + // ************************************************************************************************* + // Category II: all clear, or cld < 1e-5 + // In this case, zfclea=1 and zfcldy=0 + // ************************************************************************************************* + } else if ((jclea == 1) && (jcldy == 0) && (nsubarea == 1)) { + // + // put all the gases and interstitial aerosols in the clear sub-area + // and set mix-ratios = 0 in cloudy sub-area + // for cloud-borne aerosol, do nothing + // because the grid-cell-mean cloud-borne aerosol will be left + // unchanged (i.e., this routine only changes qqcw when cld >= 1e-5) + // + + for (int lmz = 0; lmz < gas_pcnst(); ++lmz) { + if (0 < lmapcc_all(lmz)) { + qsub1[lmz][jclea - 1] = qgcm1[lmz]; + qsub2[lmz][jclea - 1] = qgcm2[lmz]; + qsub3[lmz][jclea - 1] = qgcm3[lmz]; + qqcwsub2[lmz][jclea - 1] = qqcwgcm2[lmz]; + qqcwsub3[lmz][jclea - 1] = qqcwgcm3[lmz]; + } + } + // ************************************************************************************************* + // Category III: all cloudy, or cld > 0.999 + // in this case, zfcldy= and zfclea=0 + // ************************************************************************************************* + } else if ((jclea == 0) && (jcldy == 1) && (nsubarea == 1)) { + // + // put all the gases and interstitial aerosols in the cloudy sub-area + // and set mix-ratios = 0 in clear sub-area + // + for (int lmz = 0; lmz < gas_pcnst(); ++lmz) { + if (0 < lmapcc_all(lmz)) { + qsub1[lmz][jcldy - 1] = qgcm1[lmz]; + qsub2[lmz][jcldy - 1] = qgcm2[lmz]; + qsub3[lmz][jcldy - 1] = qgcm3[lmz]; + qqcwsub2[lmz][jcldy - 1] = qqcwgcm2[lmz]; + qqcwsub3[lmz][jcldy - 1] = qqcwgcm3[lmz]; + } + } + // ************************************************************************************************* + } else { // this should not happen + EKAT_KERNEL_REQUIRE_MSG(false, "*** modal_aero_amicphys - bad jclea, jcldy, nsubarea!"); + } + // ************************************************************************************************* + + // ------------------------------------------------------------------------------------ + // aerosol water -- how to treat this in sub-areas needs more work/thinking + // currently modal_aero_water_uptake calculates qaerwat using + // the grid-cell mean interstital-aerosol mix-rats and the clear-area rh + for (int jsub = 0; jsub < nsubarea; ++jsub) + for (int i = 0; i < num_modes; ++i) + qaerwatsub3[i][jsub] = qaerwatgcm3[i]; + + // ------------------------------------------------------------------------------------ + if (nsubarea == 1) { + // the j=1 subarea is used for some diagnostics + // but is not used in actual calculations + const int j = 1; + for (int i = 0; i < gas_pcnst(); ++i) { + qsub1[i][j] = 0.0; + qsub2[i][j] = 0.0; + qsub3[i][j] = 0.0; + qqcwsub2[i][j] = 0.0; + qqcwsub3[i][j] = 0.0; + } + } +} + +KOKKOS_INLINE_FUNCTION +void mam_amicphys_1subarea_clear( + const AmicPhysConfig& config, const int nstep, const Real deltat, const int jsub, + const int nsubarea, const bool iscldy_subarea, const Real afracsub, + const Real temp, const Real pmid, const Real pdel, const Real zmid, + const Real pblh, const Real relhum, Real dgn_a[AeroConfig::num_modes()], + Real dgn_awet[AeroConfig::num_modes()], + Real wetdens[AeroConfig::num_modes()], + const Real qgas1[AeroConfig::num_gas_ids()], + const Real qgas3[AeroConfig::num_gas_ids()], + Real qgas4[AeroConfig::num_gas_ids()], + Real qgas_delaa[AeroConfig::num_gas_ids()][nqtendaa()], + const Real qnum3[AeroConfig::num_modes()], + Real qnum4[AeroConfig::num_modes()], + Real qnum_delaa[AeroConfig::num_modes()][nqtendaa()], + const Real qaer3[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], + Real qaer4[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], + Real qaer_delaa[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()] + [nqtendaa()], + const Real qwtr3[AeroConfig::num_modes()], + Real qwtr4[AeroConfig::num_modes()]) { + static constexpr int num_gas_ids = AeroConfig::num_gas_ids(); + static constexpr int num_modes = AeroConfig::num_modes(); + static constexpr int num_aerosol_ids = AeroConfig::num_aerosol_ids(); + + static constexpr int igas_h2so4 = static_cast(GasId::H2SO4); + // Turn off nh3 for now. This is a future enhancement. + static constexpr int igas_nh3 = -999888777; // Same as mam_refactor + static constexpr int iaer_so4 = static_cast(AeroId::SO4); + static constexpr int iaer_pom = static_cast(AeroId::POM); + + const AeroId gas_to_aer[num_gas_ids] = {AeroId::SOA, AeroId::SO4, + AeroId::None}; + + const bool l_gas_condense_to_mode[num_gas_ids][num_modes] = { + {true, true, true, true}, + {true, true, true, true}, + {false, false, false, false}}; + enum { NA, ANAL, IMPL }; + const int eqn_and_numerics_category[num_gas_ids] = {IMPL, ANAL, ANAL}; + + // air molar density (kmol/m3) + // const Real r_universal = Constants::r_gas; // [mJ/(K mol)] + const Real r_universal = 8.314467591; // [mJ/(mol)] as in mam_refactor + const Real aircon = pmid / (1000 * r_universal * temp); + const Real alnsg_aer[num_modes] = {0.58778666490211906, 0.47000362924573563, + 0.58778666490211906, 0.47000362924573563}; + const Real uptk_rate_factor[num_gas_ids] = {0.81, 1.0, 1.0}; + // calculates changes to gas and aerosol sub-area TMRs (tracer mixing ratios) + // qgas3, qaer3, qnum3 are the current incoming TMRs + // qgas4, qaer4, qnum4 are the updated outgoing TMRs + // + // this routine calculates changes involving + // gas-aerosol exchange (condensation/evaporation) + // growth from smaller to larger modes (renaming) due to condensation + // new particle nucleation + // coagulation + // transfer of particles from hydrophobic modes to hydrophilic modes + // (aging) + // due to condensation and coagulation + // + // qXXXN (X=gas,aer,wat,num; N=1:4) are sub-area mixing ratios + // XXX=gas - gas species + // XXX=aer - aerosol mass species (excluding water) + // XXX=wat - aerosol water + // XXX=num - aerosol number + // N=1 - before gas-phase chemistry + // N=2 - before cloud chemistry + // N=3 - current incoming values (before gas-aerosol exchange, newnuc, + // coag) N=4 - updated outgoing values (after gas-aerosol exchange, + // newnuc, coag) + // + // qXXX_delaa are TMR changes (not tendencies) + // for different processes, which are used to produce history output + // for a clear sub-area, the processes are condensation/evaporation (and + // associated aging), renaming, coagulation, and nucleation + + Real qgas_cur[num_gas_ids]; + for (int i = 0; i < num_gas_ids; ++i) + qgas_cur[i] = qgas3[i]; + Real qaer_cur[num_aerosol_ids][num_modes]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_cur[i][j] = qaer3[i][j]; + + Real qnum_cur[num_modes]; + for (int j = 0; j < num_modes; ++j) + qnum_cur[j] = qnum3[j]; + Real qwtr_cur[num_modes]; + for (int j = 0; j < num_modes; ++j) + qwtr_cur[j] = qwtr3[j]; + + // qgas_netprod_otrproc = gas net production rate from other processes + // such as gas-phase chemistry and emissions (mol/mol/s) + // this allows the condensation (gasaerexch) routine to apply production and + // condensation loss + // together, which is more accurate numerically + // NOTE - must be >= zero, as numerical method can fail when it is negative + // NOTE - currently only the values for h2so4 and nh3 should be non-zero + Real qgas_netprod_otrproc[num_gas_ids] = {}; + if (config.do_cond && config.gaexch_h2so4_uptake_optaa == 2) { + for (int igas = 0; igas < num_gas_ids; ++igas) { + if (igas == igas_h2so4 || igas == igas_nh3) { + // if config.gaexch_h2so4_uptake_optaa == 2, then + // if qgas increases from pre-gaschem to post-cldchem, + // start from the pre-gaschem mix-ratio and add in the production + // during the integration + // if it decreases, + // start from post-cldchem mix-ratio + // *** currently just do this for h2so4 and nh3 + qgas_netprod_otrproc[igas] = (qgas3[igas] - qgas1[igas]) / deltat; + if (qgas_netprod_otrproc[igas] >= 0.0) + qgas_cur[igas] = qgas1[igas]; + else + qgas_netprod_otrproc[igas] = 0.0; + } + } + } + Real qgas_del_cond[num_gas_ids] = {}; + Real qgas_del_nnuc[num_gas_ids] = {}; + Real qgas_del_cond_only[num_gas_ids] = {}; + Real qaer_del_cond[num_aerosol_ids][num_modes] = {}; + Real qaer_del_rnam[num_aerosol_ids][num_modes] = {}; + Real qaer_del_nnuc[num_aerosol_ids][num_modes] = {}; + Real qaer_del_coag[num_aerosol_ids][num_modes] = {}; + Real qaer_delsub_coag_in[num_aerosol_ids][AeroConfig::max_agepair()] = {}; + Real qaer_delsub_cond[num_aerosol_ids][num_modes] = {}; + Real qaer_delsub_coag[num_aerosol_ids][num_modes] = {}; + Real qaer_del_cond_only[num_aerosol_ids][num_modes] = {}; + Real qnum_del_cond[num_modes] = {}; + Real qnum_del_rnam[num_modes] = {}; + Real qnum_del_nnuc[num_modes] = {}; + Real qnum_del_coag[num_modes] = {}; + Real qnum_delsub_cond[num_modes] = {}; + Real qnum_delsub_coag[num_modes] = {}; + Real qnum_del_cond_only[num_modes] = {}; + Real dnclusterdt = 0.0; + + const int ntsubstep = 1; + Real dtsubstep = deltat; + if (ntsubstep > 1) + dtsubstep = deltat / ntsubstep; + Real del_h2so4_gasprod = + haero::max(qgas3[igas_h2so4] - qgas1[igas_h2so4], 0.0) / ntsubstep; + + // loop over multiple time sub-steps + for (int jtsubstep = 1; jtsubstep <= ntsubstep; ++jtsubstep) { + // gas-aerosol exchange + Real uptkrate_h2so4 = 0.0; + Real del_h2so4_aeruptk = 0.0; + Real qaer_delsub_grow4rnam[num_aerosol_ids][num_modes] = {}; + Real qgas_avg[num_gas_ids] = {}; + Real qnum_sv1[num_modes] = {}; + Real qaer_sv1[num_aerosol_ids][num_modes] = {}; + Real qgas_sv1[num_gas_ids] = {}; + + if (config.do_cond) { + + const bool l_calc_gas_uptake_coeff = jtsubstep == 1; + Real uptkaer[num_gas_ids][num_modes] = {}; + + for (int i = 0; i < num_gas_ids; ++i) + qgas_sv1[i] = qgas_cur[i]; + for (int i = 0; i < num_modes; ++i) + qnum_sv1[i] = qnum_cur[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_sv1[j][i] = qaer_cur[j][i]; + + // time sub-step + const Real dtsub_soa_fixed = -1.0; + // Integration order + const int nghq = 2; + const int ntot_soamode = 4; + int niter_out = 0; + Real g0_soa_out = 0; + gasaerexch::mam_gasaerexch_1subarea( + nghq, igas_h2so4, igas_nh3, ntot_soamode, gas_to_aer, iaer_so4, + iaer_pom, l_calc_gas_uptake_coeff, l_gas_condense_to_mode, + eqn_and_numerics_category, dtsubstep, dtsub_soa_fixed, temp, pmid, + aircon, num_gas_ids, qgas_cur, qgas_avg, qgas_netprod_otrproc, + qaer_cur, qnum_cur, dgn_awet, alnsg_aer, uptk_rate_factor, uptkaer, + uptkrate_h2so4, niter_out, g0_soa_out); + + if (config.newnuc_h2so4_conc_optaa == 11) + qgas_avg[igas_h2so4] = + 0.5 * (qgas_sv1[igas_h2so4] + qgas_cur[igas_h2so4]); + else if (config.newnuc_h2so4_conc_optaa == 12) + qgas_avg[igas_h2so4] = qgas_cur[igas_h2so4]; + + for (int i = 0; i < num_gas_ids; ++i) + qgas_del_cond[i] += + (qgas_cur[i] - (qgas_sv1[i] + qgas_netprod_otrproc[i] * dtsubstep)); + + for (int i = 0; i < num_modes; ++i) + qnum_delsub_cond[i] = qnum_cur[i] - qnum_sv1[i]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_delsub_cond[i][j] = qaer_cur[i][j] - qaer_sv1[i][j]; + + // qaer_del_grow4rnam = change in qaer_del_cond during latest condensation + // calculations + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_delsub_grow4rnam[i][j] = qaer_cur[i][j] - qaer_sv1[i][j]; + for (int i = 0; i < num_gas_ids; ++i) + qgas_del_cond_only[i] = qgas_del_cond[i]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_del_cond_only[i][j] = qaer_delsub_cond[i][j]; + for (int i = 0; i < num_modes; ++i) + qnum_del_cond_only[i] = qnum_delsub_cond[i]; + del_h2so4_aeruptk = + qgas_cur[igas_h2so4] - + (qgas_sv1[igas_h2so4] + qgas_netprod_otrproc[igas_h2so4] * dtsubstep); + } else { + for (int i = 0; i < num_gas_ids; ++i) + qgas_avg[i] = qgas_cur[i]; + } + + // renaming after "continuous growth" + if (config.do_rename) { + constexpr int nmodes = AeroConfig::num_modes(); + constexpr int naerosol_species = AeroConfig::num_aerosol_ids(); + const Real smallest_dryvol_value = 1.0e-25; // BAD_CONSTANT + const int dest_mode_of_mode[nmodes] = {-1, 0, -1, -1}; + + Real qnumcw_cur[num_modes] = {}; + Real qaercw_cur[num_aerosol_ids][num_modes] = {}; + Real qaercw_delsub_grow4rnam[num_aerosol_ids][num_modes] = {}; + Real mean_std_dev[nmodes]; + Real fmode_dist_tail_fac[nmodes]; + Real v2n_lo_rlx[nmodes]; + Real v2n_hi_rlx[nmodes]; + Real ln_diameter_tail_fac[nmodes]; + int num_pairs = 0; + Real diameter_cutoff[nmodes]; + Real ln_dia_cutoff[nmodes]; + Real diameter_threshold[nmodes]; + Real mass_2_vol[naerosol_species] = {0.15, + 6.4971751412429377e-002, + 0.15, + 7.0588235294117650e-003, + 3.0789473684210526e-002, + 5.1923076923076926e-002, + 156.20986883198000}; + + rename::find_renaming_pairs(dest_mode_of_mode, // in + mean_std_dev, // out + fmode_dist_tail_fac, // out + v2n_lo_rlx, // out + v2n_hi_rlx, // out + ln_diameter_tail_fac, // out + num_pairs, // out + diameter_cutoff, // out + ln_dia_cutoff, diameter_threshold); + + for (int i = 0; i < num_modes; ++i) + qnum_sv1[i] = qnum_cur[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_sv1[j][i] = qaer_cur[j][i]; + Real dgnum_amode[nmodes]; + for (int m = 0; m < nmodes; ++m) { + dgnum_amode[m] = modes(m).nom_diameter; + } + + { + Real qmol_i_cur[num_modes][num_aerosol_ids]; + Real qmol_i_del[num_modes][num_aerosol_ids]; + Real qmol_c_cur[num_modes][num_aerosol_ids]; + Real qmol_c_del[num_modes][num_aerosol_ids]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) { + qmol_i_cur[i][j] = qaer_cur[j][i]; + qmol_i_del[i][j] = qaer_delsub_grow4rnam[j][i]; + qmol_c_cur[i][j] = qaercw_cur[j][i]; + qmol_c_del[i][j] = qaercw_delsub_grow4rnam[j][i]; + } + Rename rename; + rename.mam_rename_1subarea_( + iscldy_subarea, smallest_dryvol_value, dest_mode_of_mode, + mean_std_dev, fmode_dist_tail_fac, v2n_lo_rlx, v2n_hi_rlx, + ln_diameter_tail_fac, num_pairs, diameter_cutoff, ln_dia_cutoff, + diameter_threshold, mass_2_vol, dgnum_amode, qnum_cur, qmol_i_cur, + qmol_i_del, qnumcw_cur, qmol_c_cur, qmol_c_del); + + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) { + qaer_cur[j][i] = qmol_i_cur[i][j]; + qaer_delsub_grow4rnam[j][i] = qmol_i_del[i][j]; + qaercw_cur[j][i] = qmol_c_cur[i][j]; + qaercw_delsub_grow4rnam[j][i] = qmol_c_del[i][j]; + } + } + + for (int i = 0; i < num_modes; ++i) + qnum_del_rnam[i] += qnum_cur[i] - qnum_sv1[i]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_del_rnam[i][j] += qaer_cur[i][j] - qaer_sv1[i][j]; + } + + // new particle formation (nucleation) + if (config.do_newnuc) { + for (int i = 0; i < num_gas_ids; ++i) + qgas_sv1[i] = qgas_cur[i]; + for (int i = 0; i < num_modes; ++i) + qnum_sv1[i] = qnum_cur[i]; + Real qaer_cur_tmp[num_modes][num_aerosol_ids]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) { + qaer_sv1[j][i] = qaer_cur[j][i]; + qaer_cur_tmp[i][j] = qaer_cur[j][i]; + } + Real dnclusterdt_substep = 0; + Real dndt_ait = 0; + Real dmdt_ait = 0; + Real dso4dt_ait = 0; + Real dnh4dt_ait = 0; + Nucleation nucleation; + Nucleation::Config config; + config.dens_so4a_host = 1770; + config.mw_nh4a_host = 115; + config.mw_so4a_host = 115; + config.accom_coef_h2so4 = 0.65; + AeroConfig aero_config; + nucleation.init(aero_config, config); + nucleation.compute_tendencies_( + dtsubstep, temp, pmid, aircon, zmid, pblh, relhum, uptkrate_h2so4, + del_h2so4_gasprod, del_h2so4_aeruptk, qgas_cur, qgas_avg, qnum_cur, + qaer_cur_tmp, qwtr_cur, dndt_ait, dmdt_ait, dso4dt_ait, dnh4dt_ait, + dnclusterdt_substep); + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_cur[j][i] = qaer_cur_tmp[i][j]; + + //! Apply the tendencies to the prognostics. + const int nait = static_cast(ModeIndex::Aitken); + qnum_cur[nait] += dndt_ait * dtsubstep; + + if (dso4dt_ait > 0.0) { + static constexpr int iaer_so4 = static_cast(AeroId::SO4); + static constexpr int igas_h2so4 = static_cast(GasId::H2SO4); + + Real delta_q = dso4dt_ait * dtsubstep; + qaer_cur[iaer_so4][nait] += delta_q; + delta_q = haero::min(delta_q, qgas_cur[igas_h2so4]); + qgas_cur[igas_h2so4] -= delta_q; + } + + if (igas_nh3 > 0 && dnh4dt_ait > 0.0) { + static constexpr int iaer_nh4 = + -9999999; // static_cast(AeroId::NH4); + + Real delta_q = dnh4dt_ait * dtsubstep; + qaer_cur[iaer_nh4][nait] += delta_q; + delta_q = haero::min(delta_q, qgas_cur[igas_nh3]); + qgas_cur[igas_nh3] -= delta_q; + } + for (int i = 0; i < num_gas_ids; ++i) + qgas_del_nnuc[i] += (qgas_cur[i] - qgas_sv1[i]); + for (int i = 0; i < num_modes; ++i) + qnum_del_nnuc[i] += (qnum_cur[i] - qnum_sv1[i]); + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_del_nnuc[j][i] += (qaer_cur[j][i] - qaer_sv1[j][i]); + + dnclusterdt = dnclusterdt + dnclusterdt_substep * (dtsubstep / deltat); + } + + // coagulation part + if (config.do_coag) { + for (int i = 0; i < num_modes; ++i) + qnum_sv1[i] = qnum_cur[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_sv1[j][i] = qaer_cur[j][i]; + coagulation::mam_coag_1subarea(dtsubstep, temp, pmid, aircon, dgn_a, + dgn_awet, wetdens, qnum_cur, qaer_cur, + qaer_delsub_coag_in); + for (int i = 0; i < num_modes; ++i) + qnum_delsub_coag[i] = qnum_cur[i] - qnum_sv1[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_delsub_coag[j][i] = qaer_cur[j][i] - qaer_sv1[j][i]; + } + + // primary carbon aging + + aging::mam_pcarbon_aging_1subarea( + dgn_a, qnum_cur, qnum_delsub_cond, qnum_delsub_coag, qaer_cur, + qaer_delsub_cond, qaer_delsub_coag, qaer_delsub_coag_in); + + // accumulate sub-step q-dels + if (config.do_coag) { + for (int i = 0; i < num_modes; ++i) + qnum_del_coag[i] += qnum_delsub_coag[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_del_coag[j][i] += qaer_delsub_coag[j][i]; + } + if (config.do_cond) { + for (int i = 0; i < num_modes; ++i) + qnum_del_cond[i] += qnum_delsub_cond[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_del_cond[j][i] += qaer_delsub_cond[j][i]; + } + } + + // final mix ratios + for (int i = 0; i < num_gas_ids; ++i) + qgas4[i] = qgas_cur[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer4[j][i] = qaer_cur[j][i]; + for (int i = 0; i < num_modes; ++i) + qnum4[i] = qnum_cur[i]; + for (int i = 0; i < num_modes; ++i) + qwtr4[i] = qwtr_cur[i]; + + // final mix ratio changes + for (int i = 0; i < num_gas_ids; ++i) { + qgas_delaa[i][iqtend_cond()] = qgas_del_cond[i]; + qgas_delaa[i][iqtend_rnam()] = 0.0; + qgas_delaa[i][iqtend_nnuc()] = qgas_del_nnuc[i]; + qgas_delaa[i][iqtend_coag()] = 0.0; + qgas_delaa[i][iqtend_cond_only()] = qgas_del_cond_only[i]; + } + for (int i = 0; i < num_modes; ++i) { + qnum_delaa[i][iqtend_cond()] = qnum_del_cond[i]; + qnum_delaa[i][iqtend_rnam()] = qnum_del_rnam[i]; + qnum_delaa[i][iqtend_nnuc()] = qnum_del_nnuc[i]; + qnum_delaa[i][iqtend_coag()] = qnum_del_coag[i]; + qnum_delaa[i][iqtend_cond_only()] = qnum_del_cond_only[i]; + } + for (int j = 0; j < num_aerosol_ids; ++j) { + for (int i = 0; i < num_modes; ++i) { + qaer_delaa[j][i][iqtend_cond()] = qaer_del_cond[j][i]; + qaer_delaa[j][i][iqtend_rnam()] = qaer_del_rnam[j][i]; + qaer_delaa[j][i][iqtend_nnuc()] = qaer_del_nnuc[j][i]; + qaer_delaa[j][i][iqtend_coag()] = qaer_del_coag[j][i]; + qaer_delaa[j][i][iqtend_cond_only()] = qaer_del_cond_only[j][i]; + } + } +} + +KOKKOS_INLINE_FUNCTION +void mam_amicphys_1subarea_cloudy( + const AmicPhysConfig& config, const int nstep, const Real deltat, const int jsub, + const int nsubarea, const bool iscldy_subarea, const Real afracsub, + const Real temp, const Real pmid, const Real pdel, const Real zmid, + const Real pblh, const Real relhum, Real dgn_a[AeroConfig::num_modes()], + Real dgn_awet[AeroConfig::num_modes()], + Real wetdens[AeroConfig::num_modes()], + const Real qgas1[AeroConfig::num_gas_ids()], + const Real qgas3[AeroConfig::num_gas_ids()], + Real qgas4[AeroConfig::num_gas_ids()], + Real qgas_delaa[AeroConfig::num_gas_ids()][nqtendaa()], + const Real qnum3[AeroConfig::num_modes()], + Real qnum4[AeroConfig::num_modes()], + Real qnum_delaa[AeroConfig::num_modes()][nqtendaa()], + const Real qaer2[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], + const Real qaer3[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], + Real qaer4[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], + Real qaer_delaa[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()] + [nqtendaa()], + const Real qwtr3[AeroConfig::num_modes()], + Real qwtr4[AeroConfig::num_modes()], + const Real qnumcw3[AeroConfig::num_modes()], + Real qnumcw4[AeroConfig::num_modes()], + Real qnumcw_delaa[AeroConfig::num_modes()][nqqcwtendaa()], + const Real qaercw2[AeroConfig::num_gas_ids()][AeroConfig::num_modes()], + const Real qaercw3[AeroConfig::num_gas_ids()][AeroConfig::num_modes()], + Real qaercw4[AeroConfig::num_gas_ids()][AeroConfig::num_modes()], + Real qaercw_delaa[AeroConfig::num_gas_ids()][AeroConfig::num_modes()] + [nqqcwtendaa()]) { + + // + // calculates changes to gas and aerosol sub-area TMRs (tracer mixing ratios) + // qgas3, qaer3, qaercw3, qnum3, qnumcw3 are the current incoming TMRs + // qgas4, qaer4, qaercw4, qnum4, qnumcw4 are the updated outgoing TMRs + // + // when config.do_cond = false, this routine only calculates changes involving + // growth from smaller to larger modes (renaming) following cloud chemistry + // so gas TMRs are not changed + // when config.do_cond = true, this routine also calculates changes involving + // gas-aerosol exchange (condensation/evaporation) + // transfer of particles from hydrophobic modes to hydrophilic modes + // (aging) + // due to condensation + // currently this routine does not do + // new particle nucleation - because h2so4 gas conc. should be very low in + // cloudy air coagulation - because cloud-borne aerosol would need to be + // included + // + + // qXXXN (X=gas,aer,wat,num; N=1:4) are sub-area mixing ratios + // XXX=gas - gas species + // XXX=aer - aerosol mass species (excluding water) + // XXX=wat - aerosol water + // XXX=num - aerosol number + // N=1 - before gas-phase chemistry + // N=2 - before cloud chemistry + // N=3 - current incoming values (before gas-aerosol exchange, newnuc, + // coag) N=4 - updated outgoing values (after gas-aerosol exchange, + // newnuc, coag) + // + // qXXX_delaa are TMR changes (not tendencies) + // for different processes, which are used to produce history output + // for a clear sub-area, the processes are condensation/evaporation (and + // associated aging), + // renaming, coagulation, and nucleation + + // qxxx_del_yyyy are mix-ratio changes over full time step (deltat) + // qxxx_delsub_yyyy are mix-ratio changes over time sub-step (dtsubstep) + + static constexpr int num_gas_ids = AeroConfig::num_gas_ids(); + static constexpr int num_modes = AeroConfig::num_modes(); + static constexpr int num_aerosol_ids = AeroConfig::num_aerosol_ids(); + + static constexpr int igas_h2so4 = static_cast(GasId::H2SO4); + // Turn off nh3 for now. This is a future enhancement. + static constexpr int igas_nh3 = -999888777; // Same as mam_refactor + static constexpr int iaer_so4 = static_cast(AeroId::SO4); + static constexpr int iaer_pom = static_cast(AeroId::POM); + + const AeroId gas_to_aer[num_gas_ids] = {AeroId::SOA, AeroId::SO4, + AeroId::None}; + const bool l_gas_condense_to_mode[num_gas_ids][num_modes] = { + {true, true, true, true}, + {true, true, true, true}, + {false, false, false, false}}; + enum { NA, ANAL, IMPL }; + const int eqn_and_numerics_category[num_gas_ids] = {IMPL, ANAL, ANAL}; + // air molar density (kmol/m3) + // In order to try to match the results in mam_refactor + // set r_universal as [mJ/(mol)] as in mam_refactor. + // const Real r_universal = Constants::r_gas; // [mJ/(K mol)] + const Real r_universal = 8.314467591; // [mJ/(mol)] as in mam_refactor + const Real aircon = pmid / (1000 * r_universal * temp); + const Real alnsg_aer[num_modes] = {0.58778666490211906, 0.47000362924573563, + 0.58778666490211906, 0.47000362924573563}; + const Real uptk_rate_factor[num_gas_ids] = {0.81, 1.0, 1.0}; + + Real qgas_cur[num_gas_ids]; + for (int i = 0; i < num_gas_ids; ++i) + qgas_cur[i] = qgas3[i]; + Real qaer_cur[num_aerosol_ids][num_modes]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_cur[i][j] = qaer3[i][j]; + + Real qnum_cur[num_modes]; + for (int j = 0; j < num_modes; ++j) + qnum_cur[j] = qnum3[j]; + Real qwtr_cur[num_modes]; + for (int j = 0; j < num_modes; ++j) + qwtr_cur[j] = qwtr3[j]; + + Real qnumcw_cur[num_modes]; + for (int j = 0; j < num_modes; ++j) + qnumcw_cur[j] = qnumcw3[j]; + + Real qaercw_cur[num_gas_ids][num_modes]; + for (int i = 0; i < num_gas_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaercw_cur[i][j] = qaercw3[i][j]; + + Real qgas_netprod_otrproc[num_gas_ids] = {}; + if (config.do_cond && config.gaexch_h2so4_uptake_optaa == 2) { + for (int igas = 0; igas < num_gas_ids; ++igas) { + if (igas == igas_h2so4 || igas == igas_nh3) { + // if gaexch_h2so4_uptake_optaa == 2, then + // if qgas increases from pre-gaschem to post-cldchem, + // start from the pre-gaschem mix-ratio and add in the production + // during the integration + // if it decreases, + // start from post-cldchem mix-ratio + // *** currently just do this for h2so4 and nh3 + qgas_netprod_otrproc[igas] = (qgas3[igas] - qgas1[igas]) / deltat; + if (qgas_netprod_otrproc[igas] >= 0.0) + qgas_cur[igas] = qgas1[igas]; + else + qgas_netprod_otrproc[igas] = 0.0; + } + } + } + Real qgas_del_cond[num_gas_ids] = {}; + Real qgas_del_nnuc[num_gas_ids] = {}; + Real qgas_del_cond_only[num_gas_ids] = {}; + Real qaer_del_cond[num_aerosol_ids][num_modes] = {}; + Real qaer_del_rnam[num_aerosol_ids][num_modes] = {}; + Real qaer_del_nnuc[num_aerosol_ids][num_modes] = {}; + Real qaer_del_coag[num_aerosol_ids][num_modes] = {}; + Real qaer_delsub_cond[num_aerosol_ids][num_modes] = {}; + Real qaer_delsub_coag[num_aerosol_ids][num_modes] = {}; + Real qaer_del_cond_only[num_aerosol_ids][num_modes] = {}; + Real qaercw_del_rnam[num_aerosol_ids][num_modes] = {}; + Real qnum_del_cond[num_modes] = {}; + Real qnum_del_rnam[num_modes] = {}; + Real qnum_del_nnuc[num_modes] = {}; + Real qnum_del_coag[num_modes] = {}; + Real qnum_delsub_cond[num_modes] = {}; + Real qnum_delsub_coag[num_modes] = {}; + Real qnum_del_cond_only[num_modes] = {}; + Real qnumcw_del_rnam[num_modes] = {}; + Real qaer_delsub_coag_in[num_aerosol_ids][AeroConfig::max_agepair()] = {}; + + const int ntsubstep = 1; + Real dtsubstep = deltat; + if (ntsubstep > 1) + dtsubstep = deltat / ntsubstep; + + // loop over multiple time sub-steps + + for (int jtsubstep = 1; jtsubstep <= ntsubstep; ++jtsubstep) { + // gas-aerosol exchange + Real uptkrate_h2so4 = 0.0; + Real qgas_avg[num_gas_ids] = {}; + Real qgas_sv1[num_gas_ids] = {}; + Real qnum_sv1[num_modes] = {}; + Real qaer_sv1[num_aerosol_ids][num_modes] = {}; + Real qaer_delsub_grow4rnam[num_aerosol_ids][num_modes] = {}; + + if (config.do_cond) { + + const bool l_calc_gas_uptake_coeff = jtsubstep == 1; + Real uptkaer[num_gas_ids][num_modes] = {}; + + for (int i = 0; i < num_gas_ids; ++i) + qgas_sv1[i] = qgas_cur[i]; + for (int i = 0; i < num_modes; ++i) + qnum_sv1[i] = qnum_cur[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_sv1[j][i] = qaer_cur[j][i]; + + const int nghq = 2; + const int ntot_soamode = 4; + int niter_out = 0; + Real g0_soa_out = 0; + // time sub-step + const Real dtsub_soa_fixed = -1.0; + gasaerexch::mam_gasaerexch_1subarea( + nghq, igas_h2so4, igas_nh3, ntot_soamode, gas_to_aer, iaer_so4, + iaer_pom, l_calc_gas_uptake_coeff, l_gas_condense_to_mode, + eqn_and_numerics_category, dtsubstep, dtsub_soa_fixed, temp, pmid, + aircon, num_gas_ids, qgas_cur, qgas_avg, qgas_netprod_otrproc, + qaer_cur, qnum_cur, dgn_awet, alnsg_aer, uptk_rate_factor, uptkaer, + uptkrate_h2so4, niter_out, g0_soa_out); + + if (config.newnuc_h2so4_conc_optaa == 11) + qgas_avg[igas_h2so4] = + 0.5 * (qgas_sv1[igas_h2so4] + qgas_cur[igas_h2so4]); + else if (config.newnuc_h2so4_conc_optaa == 12) + qgas_avg[igas_h2so4] = qgas_cur[igas_h2so4]; + + for (int i = 0; i < num_gas_ids; ++i) + qgas_del_cond[i] += + (qgas_cur[i] - (qgas_sv1[i] + qgas_netprod_otrproc[i] * dtsubstep)); + + for (int i = 0; i < num_modes; ++i) + qnum_delsub_cond[i] = qnum_cur[i] - qnum_sv1[i]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_delsub_cond[i][j] = qaer_cur[i][j] - qaer_sv1[i][j]; + + // qaer_del_grow4rnam = change in qaer_del_cond during latest condensation + // calculations + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_delsub_grow4rnam[i][j] = qaer_cur[i][j] - qaer_sv1[i][j]; + for (int i = 0; i < num_gas_ids; ++i) + qgas_del_cond_only[i] = qgas_del_cond[i]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_del_cond_only[i][j] = qaer_delsub_cond[i][j]; + for (int i = 0; i < num_modes; ++i) + qnum_del_cond_only[i] = qnum_delsub_cond[i]; + + } else { + for (int i = 0; i < num_gas_ids; ++i) + qgas_avg[i] = qgas_cur[i]; + } + // renaming after "continuous growth" + if (config.do_rename) { + constexpr int nmodes = AeroConfig::num_modes(); + constexpr int naerosol_species = AeroConfig::num_aerosol_ids(); + const Real smallest_dryvol_value = 1.0e-25; // BAD_CONSTANT + const int dest_mode_of_mode[nmodes] = {-1, 0, -1, -1}; + + Real qnumcw_cur[num_modes] = {}; + Real qaercw_cur[num_aerosol_ids][num_modes] = {}; + Real qaercw_delsub_grow4rnam[num_aerosol_ids][num_modes] = {}; + Real mean_std_dev[nmodes]; + Real fmode_dist_tail_fac[nmodes]; + Real v2n_lo_rlx[nmodes]; + Real v2n_hi_rlx[nmodes]; + Real ln_diameter_tail_fac[nmodes]; + int num_pairs = 0; + Real diameter_cutoff[nmodes]; + Real ln_dia_cutoff[nmodes]; + Real diameter_threshold[nmodes]; + Real mass_2_vol[naerosol_species] = {0.15, + 6.4971751412429377e-002, + 0.15, + 7.0588235294117650e-003, + 3.0789473684210526e-002, + 5.1923076923076926e-002, + 156.20986883198000}; + + rename::find_renaming_pairs(dest_mode_of_mode, // in + mean_std_dev, // out + fmode_dist_tail_fac, // out + v2n_lo_rlx, // out + v2n_hi_rlx, // out + ln_diameter_tail_fac, // out + num_pairs, // out + diameter_cutoff, // out + ln_dia_cutoff, diameter_threshold); + + for (int i = 0; i < num_modes; ++i) + qnum_sv1[i] = qnum_cur[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_sv1[j][i] = qaer_cur[j][i]; + Real dgnum_amode[nmodes]; + for (int m = 0; m < nmodes; ++m) { + dgnum_amode[m] = modes(m).nom_diameter; + } + + // qaercw_delsub_grow4rnam = change in qaercw from cloud chemistry + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaercw_delsub_grow4rnam[i][j] = + (qaercw3[i][j] - qaercw2[i][j]) / ntsubstep; + Real qnumcw_sv1[num_modes]; + for (int i = 0; i < num_modes; ++i) + qnumcw_sv1[i] = qnumcw_cur[i]; + Real qaercw_sv1[num_aerosol_ids][num_modes]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaercw_sv1[i][j] = qaercw_cur[i][j]; + + { + Real qmol_i_cur[num_modes][num_aerosol_ids]; + Real qmol_i_del[num_modes][num_aerosol_ids]; + Real qmol_c_cur[num_modes][num_aerosol_ids]; + Real qmol_c_del[num_modes][num_aerosol_ids]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) { + qmol_i_cur[i][j] = qaer_cur[j][i]; + qmol_i_del[i][j] = qaer_delsub_grow4rnam[j][i]; + qmol_c_cur[i][j] = qaercw_cur[j][i]; + qmol_c_del[i][j] = qaercw_delsub_grow4rnam[j][i]; + } + + Rename rename; + rename.mam_rename_1subarea_( + iscldy_subarea, smallest_dryvol_value, dest_mode_of_mode, + mean_std_dev, fmode_dist_tail_fac, v2n_lo_rlx, v2n_hi_rlx, + ln_diameter_tail_fac, num_pairs, diameter_cutoff, ln_dia_cutoff, + diameter_threshold, mass_2_vol, dgnum_amode, qnum_cur, qmol_i_cur, + qmol_i_del, qnumcw_cur, qmol_c_cur, qmol_c_del); + + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) { + qaer_cur[j][i] = qmol_i_cur[i][j]; + qaer_delsub_grow4rnam[j][i] = qmol_i_del[i][j]; + qaercw_cur[j][i] = qmol_c_cur[i][j]; + qaercw_delsub_grow4rnam[j][i] = qmol_c_del[i][j]; + } + } + for (int i = 0; i < num_modes; ++i) + qnum_del_rnam[i] += qnum_cur[i] - qnum_sv1[i]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_del_rnam[i][j] += qaer_cur[i][j] - qaer_sv1[i][j]; + for (int i = 0; i < num_modes; ++i) + qnumcw_del_rnam[i] += qnumcw_cur[i] - qnumcw_sv1[i]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaercw_del_rnam[i][j] += qaercw_cur[i][j] - qaercw_sv1[i][j]; + } + + // primary carbon aging + if (config.do_cond) { + aging::mam_pcarbon_aging_1subarea( + dgn_a, qnum_cur, qnum_delsub_cond, qnum_delsub_coag, qaer_cur, + qaer_delsub_cond, qaer_delsub_coag, qaer_delsub_coag_in); + } + // accumulate sub-step q-dels + if (config.do_cond) { + for (int i = 0; i < num_modes; ++i) + qnum_del_cond[i] += qnum_delsub_cond[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_del_cond[j][i] += qaer_delsub_cond[j][i]; + } + } + // final mix ratios + for (int i = 0; i < num_gas_ids; ++i) + qgas4[i] = qgas_cur[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer4[j][i] = qaer_cur[j][i]; + for (int i = 0; i < num_modes; ++i) + qnum4[i] = qnum_cur[i]; + for (int i = 0; i < num_modes; ++i) + qwtr4[i] = qwtr_cur[i]; + for (int i = 0; i < num_modes; ++i) + qnumcw4[i] = qnumcw_cur[i]; + for (int i = 0; i < num_gas_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaercw4[i][j] = qaercw_cur[i][j]; + + // final mix ratio changes + for (int i = 0; i < num_gas_ids; ++i) { + qgas_delaa[i][iqtend_cond()] = qgas_del_cond[i]; + qgas_delaa[i][iqtend_rnam()] = 0.0; + qgas_delaa[i][iqtend_nnuc()] = qgas_del_nnuc[i]; + qgas_delaa[i][iqtend_coag()] = 0.0; + qgas_delaa[i][iqtend_cond_only()] = qgas_del_cond_only[i]; + } + for (int i = 0; i < num_modes; ++i) { + qnum_delaa[i][iqtend_cond()] = qnum_del_cond[i]; + qnum_delaa[i][iqtend_rnam()] = qnum_del_rnam[i]; + qnum_delaa[i][iqtend_nnuc()] = qnum_del_nnuc[i]; + qnum_delaa[i][iqtend_coag()] = qnum_del_coag[i]; + qnum_delaa[i][iqtend_cond_only()] = qnum_del_cond_only[i]; + } + for (int j = 0; j < num_aerosol_ids; ++j) { + for (int i = 0; i < num_modes; ++i) { + qaer_delaa[j][i][iqtend_cond()] = qaer_del_cond[j][i]; + qaer_delaa[j][i][iqtend_rnam()] = qaer_del_rnam[j][i]; + qaer_delaa[j][i][iqtend_nnuc()] = qaer_del_nnuc[j][i]; + qaer_delaa[j][i][iqtend_coag()] = qaer_del_coag[j][i]; + qaer_delaa[j][i][iqtend_cond_only()] = qaer_del_cond_only[j][i]; + } + } + for (int i = 0; i < num_modes; ++i) + qnumcw_delaa[i][iqqcwtend_rnam()] = qnumcw_del_rnam[i]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaercw_delaa[i][j][iqqcwtend_rnam()] = qaercw_del_rnam[i][j]; +} + +KOKKOS_INLINE_FUNCTION +void mam_amicphys_1gridcell( + const AmicPhysConfig& config, const int nstep, const Real deltat, const int nsubarea, + const int ncldy_subarea, const bool iscldy_subarea[maxsubarea()], + const Real afracsub[maxsubarea()], const Real temp, const Real pmid, + const Real pdel, const Real zmid, const Real pblh, + const Real relhumsub[maxsubarea()], Real dgn_a[AeroConfig::num_modes()], + Real dgn_awet[AeroConfig::num_modes()], + Real wetdens[AeroConfig::num_modes()], + const Real qsub1[AeroConfig::num_gas_ids()][maxsubarea()], + const Real qsub2[AeroConfig::num_gas_ids()][maxsubarea()], + const Real qqcwsub2[AeroConfig::num_gas_ids()][maxsubarea()], + const Real qsub3[AeroConfig::num_gas_ids()][maxsubarea()], + const Real qqcwsub3[AeroConfig::num_gas_ids()][maxsubarea()], + Real qaerwatsub3[AeroConfig::num_modes()][maxsubarea()], + Real qsub4[AeroConfig::num_gas_ids()][maxsubarea()], + Real qqcwsub4[AeroConfig::num_gas_ids()][maxsubarea()], + Real qaerwatsub4[AeroConfig::num_modes()][maxsubarea()], + Real qsub_tendaa[AeroConfig::num_gas_ids()][nqtendaa()][maxsubarea()], + Real qqcwsub_tendaa[AeroConfig::num_gas_ids()][nqqcwtendaa()][maxsubarea()]) { + + // + // calculates changes to gas and aerosol sub-area TMRs (tracer mixing ratios) + // qsub3 and qqcwsub3 are the incoming current TMRs + // qsub4 and qqcwsub4 are the outgoing updated TMRs + // + // qsubN and qqcwsubN (N=1:4) are tracer mixing ratios (TMRs, mol/mol or + // #/kmol) in sub-areas + // currently there are just clear and cloudy sub-areas + // the N=1:4 have same meanings as for qgcmN + // N=1 - before gas-phase chemistry + // N=2 - before cloud chemistry + // N=3 - incoming values (before gas-aerosol exchange, newnuc, coag) + // N=4 - outgoing values (after gas-aerosol exchange, newnuc, coag) + // qsub_tendaa and qqcwsub_tendaa are TMR tendencies + // for different processes, which are used to produce history output + // the processes are condensation/evaporation (and associated aging), + // renaming, coagulation, and nucleation + + static constexpr int num_gas_ids = AeroConfig::num_gas_ids(); + static constexpr int num_modes = AeroConfig::num_modes(); + static constexpr int num_aerosol_ids = AeroConfig::num_aerosol_ids(); + + // the q--4 values will be equal to q--3 values unless they get changed + for (int i = 0; i < num_gas_ids; ++i) + for (int j = 0; j < maxsubarea(); ++j) { + qsub4[i][j] = qsub3[i][j]; + qqcwsub4[i][j] = qqcwsub3[i][j]; + } + for (int i = 0; i < num_modes; ++i) + for (int j = 0; j < maxsubarea(); ++j) + qaerwatsub4[i][j] = qaerwatsub3[i][j]; + for (int i = 0; i < num_gas_ids; ++i) + for (int j = 0; j < nqtendaa(); ++j) + for (int k = 0; k < maxsubarea(); ++k) + qsub_tendaa[i][j][k] = 0; + for (int i = 0; i < num_gas_ids; ++i) + for (int j = 0; j < nqqcwtendaa(); ++j) + for (int k = 0; k < maxsubarea(); ++k) + qqcwsub_tendaa[i][j][k] = 0.0; + + for (int jsub = 0; jsub < nsubarea; ++jsub) { + AmicPhysConfig sub_config = config; + if (iscldy_subarea[jsub]) { + sub_config.do_cond = config.do_cond; + sub_config.do_rename = config.do_rename; + sub_config.do_newnuc = false; + sub_config.do_coag = false; + } + const bool do_map_gas_sub = sub_config.do_cond || sub_config.do_newnuc; + + // map incoming sub-area mix-ratios to gas/aer/num arrays + Real qgas1[num_gas_ids] = {}; + Real qgas3[num_gas_ids] = {}; + Real qgas4[num_gas_ids] = {}; + if (do_map_gas_sub) { + // for cldy subarea, only do gases if doing gaexch + for (int igas = 0; igas < 2; ++igas) { + const int l = lmap_gas(igas); + qgas1[igas] = qsub1[l][jsub] * fcvt_gas(igas); + qgas3[igas] = qsub3[l][jsub] * fcvt_gas(igas); + qgas4[igas] = qgas3[igas]; + } + } + Real qaer2[num_aerosol_ids][num_modes] = {}; + Real qaer3[num_aerosol_ids][num_modes] = {}; + Real qnum3[num_modes] = {}; + Real qaer4[num_aerosol_ids][num_modes] = {}; + Real qnum4[num_modes] = {}; + Real qwtr3[num_modes] = {}; + Real qwtr4[num_modes] = {}; + for (int n = 0; n < num_modes; ++n) { + qnum3[n] = qsub3[n][jsub] * fcvt_num(); + qnum4[n] = qnum3[n]; + for (int iaer = 0; iaer < num_aerosol_ids; ++iaer) { + qaer2[iaer][n] = qsub2[iaer][jsub] * fcvt_aer(iaer); + qaer3[iaer][n] = qsub3[iaer][jsub] * fcvt_aer(iaer); + qaer4[iaer][n] = qaer3[iaer][n]; + } + qwtr3[n] = qaerwatsub3[n][jsub] * fcvt_wtr(); + qwtr4[n] = qwtr3[n]; + } + Real qaercw2[num_aerosol_ids][num_modes] = {}; + Real qaercw3[num_aerosol_ids][num_modes] = {}; + Real qnumcw3[num_modes] = {}; + Real qaercw4[num_aerosol_ids][num_modes] = {}; + Real qnumcw4[num_modes] = {}; + if (iscldy_subarea[jsub]) { + // only do cloud-borne for cloudy + for (int n = 0; n < num_modes; ++n) { + qnumcw3[n] = qqcwsub3[n][jsub] * fcvt_num(); + qnumcw4[n] = qnumcw3[n]; + for (int iaer = 0; iaer < num_aerosol_ids; ++iaer) { + qaercw2[iaer][n] = qqcwsub2[n][jsub] * fcvt_aer(iaer); + qaercw3[iaer][n] = qqcwsub3[n][jsub] * fcvt_aer(iaer); + qaercw4[iaer][n] = qaercw3[iaer][n]; + } + } + } + + Real qgas_delaa[num_gas_ids][nqtendaa()] = {}; + Real qnum_delaa[num_modes][nqtendaa()] = {}; + Real qnumcw_delaa[num_modes][nqqcwtendaa()] = {}; + Real qaer_delaa[num_aerosol_ids][num_modes][nqtendaa()] = {}; + Real qaercw_delaa[num_aerosol_ids][num_modes][nqqcwtendaa()] = {}; + + if (iscldy_subarea[jsub]) { + mam_amicphys_1subarea_cloudy(sub_config, nstep, deltat, + jsub, nsubarea, iscldy_subarea[jsub], afracsub[jsub], temp, pmid, + pdel, zmid, pblh, relhumsub[jsub], dgn_a, dgn_awet, wetdens, qgas1, + qgas3, qgas4, qgas_delaa, qnum3, qnum4, qnum_delaa, qaer2, qaer3, + qaer4, qaer_delaa, qwtr3, qwtr4, qnumcw3, qnumcw4, qnumcw_delaa, + qaercw2, qaercw3, qaercw4, qaercw_delaa); + } else { + mam_amicphys_1subarea_clear(sub_config, nstep, deltat, + jsub, nsubarea, iscldy_subarea[jsub], afracsub[jsub], temp, pmid, + pdel, zmid, pblh, relhumsub[jsub], dgn_a, dgn_awet, wetdens, qgas1, + qgas3, qgas4, qgas_delaa, qnum3, qnum4, qnum_delaa, qaer3, qaer4, + qaer_delaa, qwtr3, qwtr4); + // map gas/aer/num arrays (mix-ratio and del=change) back to sub-area + // arrays + + if (do_map_gas_sub) { + for (int igas = 0; igas < 2; ++igas) { + const int l = lmap_gas(igas); + qsub4[l][jsub] = qgas4[igas] / fcvt_gas(igas); + for (int i = 0; i < nqtendaa(); ++i) + qsub_tendaa[l][i][jsub] = + qgas_delaa[igas][i] / (fcvt_gas(igas) * deltat); + } + } + for (int n = 0; n < num_modes; ++n) { + qsub4[n][jsub] = qnum4[n] / fcvt_num(); + for (int i = 0; i < nqtendaa(); ++i) + qsub_tendaa[n][i][jsub] = qnum_delaa[n][i] / (fcvt_num() * deltat); + for (int iaer = 0; iaer < num_aerosol_ids; ++iaer) { + qsub4[iaer][jsub] = qaer4[iaer][n] / fcvt_aer(iaer); + for (int i = 0; i < nqtendaa(); ++i) + qsub_tendaa[iaer][i][jsub] = + qaer_delaa[iaer][n][i] / (fcvt_aer(iaer) * deltat); + } + qaerwatsub4[n][jsub] = qwtr4[n] / fcvt_wtr(); + + if (iscldy_subarea[jsub]) { + qqcwsub4[n][jsub] = qnumcw4[n] / fcvt_num(); + for (int i = 0; i < nqqcwtendaa(); ++i) + qqcwsub_tendaa[n][i][jsub] = + qnumcw_delaa[n][i] / (fcvt_num() * deltat); + for (int iaer = 0; iaer < num_aerosol_ids; ++iaer) { + qqcwsub4[iaer][jsub] = qaercw4[iaer][n] / fcvt_aer(iaer); + for (int i = 0; i < nqqcwtendaa(); ++i) + qqcwsub_tendaa[iaer][i][jsub] = + qaercw_delaa[iaer][n][i] / (fcvt_aer(iaer) * deltat); + } + } + } + } + } +} + +} // anonymous namespace + +KOKKOS_INLINE_FUNCTION +void modal_aero_amicphys_intr( + const AmicPhysConfig& config, const int nstep, const Real deltat, const Real t, const Real pmid, const Real pdel, + const Real zm, const Real pblh, const Real qv, const Real cld, + Real q[gas_pcnst()], Real qqcw[gas_pcnst()], const Real q_pregaschem[gas_pcnst()], + const Real q_precldchem[gas_pcnst()], const Real qqcw_precldchem[gas_pcnst()], + Real q_tendbb[gas_pcnst()][nqtendbb()], Real qqcw_tendbb[gas_pcnst()][nqtendbb()], + Real dgncur_a[AeroConfig::num_modes()], + Real dgncur_awet[AeroConfig::num_modes()], + Real wetdens_host[AeroConfig::num_modes()], + Real qaerwat[AeroConfig::num_modes()]) { + + /* + nstep ! model time-step number + nqtendbb ! dimension for q_tendbb + nqqcwtendbb ! dimension f + deltat ! + q(ncol,pver,pcnstxx) ! current tracer mixing ratios (TMRs) + these values are updated (so out /= in) + *** MUST BE #/kmol-air for number + *** MUST BE mol/mol-air for mass + *** NOTE ncol dimension + qqcw(ncol,pver,pcnstxx) + like q but for cloud-borner tracers + these values are updated + q_pregaschem(ncol,pver,pcnstxx) ! q TMRs before gas-phase + chemistry q_precldchem(ncol,pver,pcnstxx) ! q TMRs before cloud + chemistry qqcw_precldchem(ncol,pver,pcnstxx) ! qqcw TMRs before cloud + chemistry q_tendbb(ncol,pver,pcnstxx,nqtendbb()) ! TMR tendencies for + box-model diagnostic output qqcw_tendbb(ncol,pver,pcnstx t(pcols,pver) ! + temperature at model levels (K) pmid(pcols,pver) ! pressure at model + level centers (Pa) pdel(pcols,pver) ! pressure thickness of levels + (Pa) zm(pcols,pver) ! altitude (above ground) at level centers (m) + pblh(pcols) ! planetary boundary layer depth (m) + qv(pcols,pver) ! specific humidity (kg/kg) + cld(ncol,pver) ! cloud fraction (-) *** NOTE ncol dimension + dgncur_a(pcols,pver,ntot_amode) + dgncur_awet(pcols,pver,ntot_amode) + ! dry & wet geo. mean dia. (m) of + number distrib. wetdens_host(pcols,pver,ntot_amode) ! interstitial + aerosol wet density (kg/m3) + + qaerwat(pcols,pver,ntot_amode aerosol water mixing ratio (kg/kg, + NOT mol/mol) + + */ + + // !DESCRIPTION: + // calculates changes to gas and aerosol TMRs (tracer mixing ratios) from + // gas-aerosol exchange (condensation/evaporation) + // growth from smaller to larger modes (renaming) due to both + // condensation and cloud chemistry + // new particle nucleation + // coagulation + // transfer of particles from hydrophobic modes to hydrophilic modes + // (aging) + // due to condensation and coagulation + // + // the incoming mixing ratios (q and qqcw) are updated before output + // + // !REVISION HISTORY: + // RCE 07.04.13: Adapted from earlier version of CAM5 modal aerosol + // routines + // for these processes + // + + static constexpr int num_modes = AeroConfig::num_modes(); + + // qgcmN and qqcwgcmN (N=1:4) are grid-cell mean tracer mixing ratios + // (TMRs, mol/mol or #/kmol) + // N=1 - before gas-phase chemistry + // N=2 - before cloud chemistry + // N=3 - incoming values (before gas-aerosol exchange, newnuc, coag) + // N=4 - outgoing values (after gas-aerosol exchange, newnuc, coag) + + // qsubN and qqcwsubN (N=1:4) are TMRs in sub-areas + // currently there are just clear and cloudy sub-areas + // the N=1:4 have same meanings as for qgcmN + + // q_coltendaa and qqcw_coltendaa are column-integrated tendencies + // for different processes, which are output to history + // the processes are condensation/evaporation (and associated aging), + // renaming, coagulation, and nucleation + + for (int i = 0; i < gas_pcnst(); ++i) + for (int j = 0; j < nqtendbb(); ++j) + q_tendbb[i][j] = 0.0, qqcw_tendbb[i][j] = 0.0; + + // get saturation mixing ratio + // call qsat( t(1:ncol,1:pver), pmid(1:ncol,1:pvnner), & + // ev_sat(1:ncol,1:pver), qv_sat(1:ncol,1:pver) ) + const Real epsqs = haero::Constants::weight_ratio_h2o_air; + // Saturation vapor pressure + const Real ev_sat = conversions::vapor_saturation_pressure_magnus(t, pmid); + // Saturation specific humidity + const Real qv_sat = epsqs * ev_sat / (pmid - (1 - epsqs) * ev_sat); + + const Real relhumgcm = haero::max(0.0, haero::min(1.0, qv / qv_sat)); + + // Set up cloudy/clear subareas inside a grid cell + int nsubarea, ncldy_subarea, jclea, jcldy; + bool iscldy_subarea[maxsubarea()]; + Real afracsub[maxsubarea()]; + Real relhumsub[maxsubarea()]; + Real qsub1[gas_pcnst()][maxsubarea()]; + Real qsub2[gas_pcnst()][maxsubarea()]; + Real qsub3[gas_pcnst()][maxsubarea()]; + Real qqcwsub1[gas_pcnst()][maxsubarea()]; + Real qqcwsub2[gas_pcnst()][maxsubarea()]; + Real qqcwsub3[gas_pcnst()][maxsubarea()]; + // aerosol water mixing ratios (mol/mol) + Real qaerwatsub3[AeroConfig::num_modes()][maxsubarea()]; + construct_subareas_1gridcell(cld, relhumgcm, // in + q_pregaschem, q_precldchem, // in + qqcw_precldchem, // in + q, qqcw, // in + nsubarea, ncldy_subarea, jclea, jcldy, // out + iscldy_subarea, afracsub, relhumsub, // out + qsub1, qsub2, qsub3, // out + qqcwsub1, qqcwsub2, qqcwsub3, qaerwatsub3, // out + qaerwat // in + ); + + // Initialize the "after-amicphys" values + Real qsub4[gas_pcnst()][maxsubarea()] = {}; + Real qqcwsub4[gas_pcnst()][maxsubarea()] = {}; + Real qaerwatsub4[AeroConfig::num_modes()][maxsubarea()] = {}; + + // + // start integration + // + Real dgn_a[num_modes], dgn_awet[num_modes], wetdens[num_modes]; + for (int n = 0; n < num_modes; ++n) { + dgn_a[n] = dgncur_a[n]; + dgn_awet[n] = dgncur_awet[n]; + wetdens[n] = haero::max(1000.0, wetdens_host[n]); + } + Real qsub_tendaa[gas_pcnst()][nqtendaa()][maxsubarea()] = {}; + Real qqcwsub_tendaa[gas_pcnst()][nqqcwtendaa()][maxsubarea()] = {}; + mam_amicphys_1gridcell(config, nstep, deltat, + nsubarea, ncldy_subarea, iscldy_subarea, afracsub, t, + pmid, pdel, zm, pblh, relhumsub, dgn_a, dgn_awet, + wetdens, qsub1, qsub2, qqcwsub2, qsub3, qqcwsub3, + qaerwatsub3, qsub4, qqcwsub4, qaerwatsub4, qsub_tendaa, + qqcwsub_tendaa); + + // + // form new grid-mean mix-ratios + Real qgcm4[gas_pcnst()]; + Real qgcm_tendaa[gas_pcnst()][nqtendaa()]; + Real qaerwatgcm4[num_modes]; + if (nsubarea == 1) { + for (int i = 0; i < gas_pcnst(); ++i) + qgcm4[i] = qsub4[i][0]; + for (int i = 0; i < gas_pcnst(); ++i) + for (int j = 0; j < nqtendaa(); ++j) + qgcm_tendaa[i][j] = qsub_tendaa[i][j][0]; + for (int i = 0; i < num_modes; ++i) + qaerwatgcm4[i] = qaerwatsub4[i][0]; + } else { + for (int i = 0; i < gas_pcnst(); ++i) + qgcm4[i] = 0.0; + for (int i = 0; i < gas_pcnst(); ++i) + for (int j = 0; j < nqtendaa(); ++j) + qgcm_tendaa[i][j] = 0.0; + for (int n = 0; n < nsubarea; ++n) { + for (int i = 0; i < gas_pcnst(); ++i) + qgcm4[i] += qsub4[i][n] * afracsub[n]; + for (int i = 0; i < gas_pcnst(); ++i) + for (int j = 0; j < nqtendaa(); ++j) + qgcm_tendaa[i][j] = + qgcm_tendaa[i][j] + qsub_tendaa[i][j][n] * afracsub[n]; + } + for (int i = 0; i < num_modes; ++i) + // for aerosol water use the clear sub-area value + qaerwatgcm4[i] = qaerwatsub4[i][jclea - 1]; + } + Real qqcwgcm4[gas_pcnst()]; + Real qqcwgcm_tendaa[gas_pcnst()][nqqcwtendaa()]; + if (ncldy_subarea <= 0) { + for (int i = 0; i < gas_pcnst(); ++i) + qqcwgcm4[i] = haero::max(0.0, qqcw[i]); + for (int i = 0; i < gas_pcnst(); ++i) + for (int j = 0; j < nqqcwtendaa(); ++j) + qqcwgcm_tendaa[i][j] = 0.0; + } else if (nsubarea == 1) { + for (int i = 0; i < gas_pcnst(); ++i) + qqcwgcm4[i] = qqcwsub4[i][0]; + for (int i = 0; i < gas_pcnst(); ++i) + for (int j = 0; j < nqqcwtendaa(); ++j) + qqcwgcm_tendaa[i][j] = qqcwsub_tendaa[i][j][0]; + } else { + for (int i = 0; i < gas_pcnst(); ++i) + qqcwgcm4[i] = 0.0; + for (int i = 0; i < gas_pcnst(); ++i) + for (int j = 0; j < nqqcwtendaa(); ++j) + qqcwgcm_tendaa[i][j] = 0.0; + for (int n = 0; n < nsubarea; ++n) { + if (iscldy_subarea[n]) { + for (int i = 0; i < gas_pcnst(); ++i) + qqcwgcm4[i] += qqcwsub4[i][n] * afracsub[n]; + for (int i = 0; i < gas_pcnst(); ++i) + for (int j = 0; j < nqqcwtendaa(); ++j) + qqcwgcm_tendaa[i][j] += qqcwsub_tendaa[i][j][n] * afracsub[n]; + } + } + } + + for (int lmz = 0; lmz < gas_pcnst(); ++lmz) { + if (lmapcc_all(lmz) > 0) { + // HW, to ensure non-negative + q[lmz] = haero::max(qgcm4[lmz], 0.0); + if (lmapcc_all(lmz) >= lmapcc_val_aer()) { + // HW, to ensure non-negative + qqcw[lmz] = haero::max(qqcwgcm4[lmz], 0.0); + } + } + } + for (int i = 0; i < gas_pcnst(); ++i) { + if (iqtend_cond() < nqtendbb()) + q_tendbb[i][iqtend_cond()] = qgcm_tendaa[i][iqtend_cond()]; + if (iqtend_rnam() < nqtendbb()) + q_tendbb[i][iqtend_rnam()] = qgcm_tendaa[i][iqtend_rnam()]; + if (iqtend_nnuc() < nqtendbb()) + q_tendbb[i][iqtend_nnuc()] = qgcm_tendaa[i][iqtend_nnuc()]; + if (iqtend_coag() < nqtendbb()) + q_tendbb[i][iqtend_coag()] = qgcm_tendaa[i][iqtend_coag()]; + if (iqqcwtend_rnam() < nqqcwtendbb()) + qqcw_tendbb[i][iqqcwtend_rnam()] = qqcwgcm_tendaa[i][iqqcwtend_rnam()]; + } + for (int i = 0; i < num_modes; ++i) + qaerwat[i] = qaerwatgcm4[i]; +} + +} // namespace scream::impl diff --git a/components/eamxx/src/physics/mam/mam_aerosol_optics_read_tables.hpp b/components/eamxx/src/physics/mam/mam_aerosol_optics_read_tables.hpp new file mode 100644 index 000000000000..d1d54e97fda0 --- /dev/null +++ b/components/eamxx/src/physics/mam/mam_aerosol_optics_read_tables.hpp @@ -0,0 +1,383 @@ +#ifndef MAM_AEROSOL_OPTICS_READ_TABLES_HPP +#define MAM_AEROSOL_OPTICS_READ_TABLES_HPP + +#include "ekat/ekat_parameter_list.hpp" +#include "mam_coupling.hpp" +#include "share/field/field_manager.hpp" +#include "share/grid/abstract_grid.hpp" +#include "share/grid/grids_manager.hpp" +#include "share/io/scorpio_input.hpp" +#include "share/io/scream_scorpio_interface.hpp" + +// later to mam_coupling.hpp +namespace scream::mam_coupling { + +using view_1d_host = typename KT::view_1d::HostMirror; +using view_2d_host = typename KT::view_2d::HostMirror; +using view_5d_host = typename KT::view_ND::HostMirror; +using complex_view_1d = typename KT::view_1d>; + +constexpr int nlwbands = mam4::modal_aer_opt::nlwbands; +constexpr int nswbands = mam4::modal_aer_opt::nswbands; + +struct AerosolOpticsHostData { + // host views + view_2d_host refindex_real_sw_host; + view_2d_host refindex_im_sw_host; + view_2d_host refindex_real_lw_host; + view_2d_host refindex_im_lw_host; + + view_5d_host absplw_host; + view_5d_host abspsw_host; + view_5d_host asmpsw_host; + view_5d_host extpsw_host; +}; + +using AerosolOpticsDeviceData = mam4::modal_aer_opt::AerosolOpticsDeviceData; + +inline void set_parameters_table( + AerosolOpticsHostData &aerosol_optics_host_data, + ekat::ParameterList &rrtmg_params, + std::map &layouts, + std::map &host_views) { + // Set up input structure to read data from file. + using strvec_t = std::vector; + using namespace ShortFieldTagsNames; + + constexpr int refindex_real = mam4::modal_aer_opt::refindex_real; + constexpr int refindex_im = mam4::modal_aer_opt::refindex_im; + constexpr int coef_number = mam4::modal_aer_opt::coef_number; + + auto refindex_real_lw_host = + view_2d_host("refrtablw_real_host", nlwbands, refindex_real); + auto refindex_im_lw_host = + view_2d_host("refrtablw_im_host", nlwbands, refindex_im); + + auto refindex_real_sw_host = + view_2d_host("refrtabsw_real_host", nswbands, refindex_real); + auto refindex_im_sw_host = + view_2d_host("refrtabsw_im_host", nswbands, refindex_im); + + // absplw(lw_band, mode, refindex_im, refindex_real, coef_number) + auto absplw_host = view_5d_host("absplw_host", nlwbands, 1, refindex_im, + refindex_real, coef_number); + + auto asmpsw_host = view_5d_host("asmpsw_host", nswbands, 1, refindex_im, + refindex_real, coef_number); + auto extpsw_host = view_5d_host("extpsw_host", nswbands, 1, refindex_im, + refindex_real, coef_number); + auto abspsw_host = view_5d_host("abspsw_host", nswbands, 1, refindex_im, + refindex_real, coef_number); + + aerosol_optics_host_data.refindex_real_lw_host = refindex_real_lw_host; + aerosol_optics_host_data.refindex_im_lw_host = refindex_im_lw_host; + aerosol_optics_host_data.refindex_real_sw_host = refindex_real_sw_host; + aerosol_optics_host_data.refindex_im_sw_host = refindex_im_sw_host; + aerosol_optics_host_data.absplw_host = absplw_host; + aerosol_optics_host_data.asmpsw_host = asmpsw_host; + aerosol_optics_host_data.extpsw_host = extpsw_host; + aerosol_optics_host_data.abspsw_host = abspsw_host; + + FieldLayout scalar_refindex_real_lw_layout{{LWBND, NREFINDEX_REAL}, + {nlwbands, refindex_real}}; + + FieldLayout scalar_refindex_im_lw_layout{{LWBND, NREFINDEX_IM}, + {nlwbands, refindex_im}}; + FieldLayout scalar_refindex_real_sw_layout{{SWBND, NREFINDEX_REAL}, + {nswbands, refindex_real}}; + + FieldLayout scalar_refindex_im_sw_layout{{SWBND, NREFINDEX_IM}, + {nswbands, refindex_im}}; + + FieldLayout scalar_absplw_layout{ + {LWBND, MODE, NREFINDEX_IM, NREFINDEX_REAL, NCOEF_NUMBER}, + {nlwbands, 1, refindex_im, refindex_real, coef_number}}; + // use also for extpsw, abspsw + FieldLayout scalar_asmpsw_layout{ + {SWBND, MODE, NREFINDEX_IM, NREFINDEX_REAL, NCOEF_NUMBER}, + {nswbands, 1, refindex_im, refindex_real, coef_number}}; + + rrtmg_params.set( + "Field Names", + {"asmpsw", "extpsw", "abspsw", "absplw", "refindex_real_sw", + "refindex_im_sw", "refindex_real_lw", "refindex_im_lw"}); + + rrtmg_params.set("Skip_Grid_Checks", true); + + host_views["refindex_real_sw"] = + view_1d_host(refindex_real_sw_host.data(), refindex_real_sw_host.size()); + + host_views["refindex_im_sw"] = + view_1d_host(refindex_im_sw_host.data(), refindex_im_sw_host.size()); + + host_views["refindex_real_lw"] = + view_1d_host(refindex_real_lw_host.data(), refindex_real_lw_host.size()); + + host_views["refindex_im_lw"] = + view_1d_host(refindex_im_lw_host.data(), refindex_im_lw_host.size()); + + host_views["absplw"] = view_1d_host(absplw_host.data(), absplw_host.size()); + + host_views["asmpsw"] = view_1d_host(asmpsw_host.data(), asmpsw_host.size()); + + host_views["extpsw"] = view_1d_host(extpsw_host.data(), extpsw_host.size()); + + host_views["abspsw"] = view_1d_host(abspsw_host.data(), abspsw_host.size()); + + layouts.emplace("refindex_real_lw", scalar_refindex_real_lw_layout); + layouts.emplace("refindex_im_lw", scalar_refindex_im_lw_layout); + layouts.emplace("refindex_real_sw", scalar_refindex_real_sw_layout); + layouts.emplace("refindex_im_sw", scalar_refindex_im_sw_layout); + layouts.emplace("absplw", scalar_absplw_layout); + layouts.emplace("asmpsw", scalar_asmpsw_layout); + layouts.emplace("extpsw", scalar_asmpsw_layout); + layouts.emplace("abspsw", scalar_asmpsw_layout); +} +// KOKKOS_INLINE_FUNCTION +inline void read_rrtmg_table( + const std::string &table_filename, const int imode, + ekat::ParameterList ¶ms, + const std::shared_ptr &grid, + const std::map &host_views_1d, + const std::map &layouts, + const AerosolOpticsHostData &aerosol_optics_host_data, + const AerosolOpticsDeviceData &aerosol_optics_device_data) { + constexpr int refindex_real = mam4::modal_aer_opt::refindex_real; + constexpr int refindex_im = mam4::modal_aer_opt::refindex_im; + constexpr int coef_number = mam4::modal_aer_opt::coef_number; + + using view_3d_host = typename KT::view_3d::HostMirror; + + // temp views: + view_3d_host temp_lw_3d_host("temp_absplw_host", coef_number, refindex_real, + refindex_im); + + params.set("Filename", table_filename); + AtmosphereInput rrtmg(params, grid, host_views_1d, layouts); + rrtmg.read_variables(); + rrtmg.finalize(); + + // copy data from host to device for mode 1 + int d1 = imode; + for(int d3 = 0; d3 < nswbands; ++d3) { + auto real_host_d3 = ekat::subview( + aerosol_optics_host_data.refindex_real_sw_host, d3); + Kokkos::deep_copy(aerosol_optics_device_data.refrtabsw[d1][d3], + real_host_d3); + auto im_host_d3 = ekat::subview( + aerosol_optics_host_data.refindex_im_sw_host, d3); + Kokkos::deep_copy(aerosol_optics_device_data.refitabsw[d1][d3], im_host_d3); + } // d3 + + for(int d3 = 0; d3 < nlwbands; ++d3) { + auto real_host_d3 = ekat::subview( + aerosol_optics_host_data.refindex_real_lw_host, d3); + Kokkos::deep_copy(aerosol_optics_device_data.refrtablw[d1][d3], + real_host_d3); + auto im_host_d3 = ekat::subview( + aerosol_optics_host_data.refindex_im_lw_host, d3); + Kokkos::deep_copy(aerosol_optics_device_data.refitablw[d1][d3], im_host_d3); + } // d3 + + // NOTE: we need to reorder dimensions in absplw + // netcfd : (lw_band, mode, refindex_im, refindex_real, coef_number) + // mam4xx : (mode, lw_band, coef_number, refindex_real, refindex_im ) + // e3sm : (ntot_amode,coef_number,refindex_real,refindex_im,nlwbands) + + for(int d5 = 0; d5 < nlwbands; ++d5) { + // reshape data: + Kokkos::parallel_for( + "reshaping absplw", + Kokkos::MDRangePolicy,Kokkos::DefaultHostExecutionSpace >({0, 0, 0}, + {coef_number, refindex_real, refindex_im}), + [&](const int d2, const int d3, const int d4) { + temp_lw_3d_host(d2, d3, d4) = + aerosol_optics_host_data.absplw_host(d5, 0, d4, d3, d2); + }); + Kokkos::fence(); + + // syn data to device + Kokkos::deep_copy(aerosol_optics_device_data.absplw[d1][d5], + temp_lw_3d_host); + } // d5 + // asmpsw, abspsw, extpsw + // netcfd : (sw_band, mode, refindex_im, refindex_real, coef_number) + // mam4xx : (mode, sw_band, coef_number, refindex_real, refindex_im ) + + for(int d5 = 0; d5 < nswbands; ++d5) { + // reshape data + Kokkos::parallel_for( + "reshaping asmpsw", + Kokkos::MDRangePolicy,Kokkos::DefaultHostExecutionSpace >({0, 0, 0}, + {coef_number, refindex_real, refindex_im}), + [&](const int d2, const int d3, const int d4) { + temp_lw_3d_host(d2, d3, d4) = + aerosol_optics_host_data.asmpsw_host(d5, 0, d4, d3, d2); + }); + Kokkos::fence(); + // syn data to device + Kokkos::deep_copy(aerosol_optics_device_data.asmpsw[d1][d5], + temp_lw_3d_host); + // reshape data + Kokkos::parallel_for( + "reshaping abspsw", + Kokkos::MDRangePolicy,Kokkos::DefaultHostExecutionSpace >({0, 0, 0}, + {coef_number, refindex_real, refindex_im}), + [&](const int d2, const int d3, const int d4) { + temp_lw_3d_host(d2, d3, d4) = + aerosol_optics_host_data.abspsw_host(d5, 0, d4, d3, d2); + }); + Kokkos::fence(); + // syn data to device + Kokkos::deep_copy(aerosol_optics_device_data.abspsw[d1][d5], + temp_lw_3d_host); + // reshape data + Kokkos::parallel_for( + "reshaping extpsw", + Kokkos::MDRangePolicy,Kokkos::DefaultHostExecutionSpace >({0, 0, 0}, + {coef_number, refindex_real, refindex_im}), + [&](const int d2, const int d3, const int d4) { + temp_lw_3d_host(d2, d3, d4) = + aerosol_optics_host_data.extpsw_host(d5, 0, d4, d3, d2); + }); + + Kokkos::fence(); + // syn data to device + Kokkos::deep_copy(aerosol_optics_device_data.extpsw[d1][d5], + temp_lw_3d_host); + + } // d5 +} + +inline void read_water_refindex(const std::string &table_filename, + const std::shared_ptr &grid, + const complex_view_1d &crefwlw, + const complex_view_1d &crefwsw) { + // refractive index for water read in read_water_refindex + // crefwsw(nswbands) ! complex refractive index for water visible + // crefwlw(nlwbands) ! complex refractive index for water infrared + + using namespace ShortFieldTagsNames; + using view_1d_host = typename KT::view_1d::HostMirror; + // Set up input structure to read data from file. + using strvec_t = std::vector; + + // here a made a list of variables that I want to read from netcdf files + ekat::ParameterList params; + params.set("Filename", table_filename); + params.set("Skip_Grid_Checks", true); + + params.set("Field Names", + {"refindex_im_water_lw", "refindex_im_water_sw", + "refindex_real_water_lw", "refindex_real_water_sw"}); + // make a list of host views + std::map host_views_water; + // fist allocate host views. + view_1d_host refindex_im_water_sw_host("refindex_im_water_sw_host", nswbands); + view_1d_host refindex_real_water_sw_host("refindex_real_water_sw_host", + nswbands); + view_1d_host refindex_im_water_lw_host("refindex_im_water_lw_host", nlwbands); + view_1d_host refindex_real_water_lw_host("refindex_real_water_lw_host", + nlwbands); + + host_views_water["refindex_im_water_sw"] = refindex_im_water_sw_host; + host_views_water["refindex_real_water_sw"] = refindex_real_water_sw_host; + host_views_water["refindex_im_water_lw"] = refindex_im_water_lw_host; + host_views_water["refindex_real_water_lw"] = refindex_real_water_lw_host; + + // defines layouts + std::map layouts_water; + FieldLayout scalar_refindex_water_sw_layout{{SWBND}, {nswbands}}; + FieldLayout scalar_refindex_water_lw_layout{{LWBND}, {nlwbands}}; + + layouts_water.emplace("refindex_im_water_sw", + scalar_refindex_water_sw_layout); + layouts_water.emplace("refindex_real_water_sw", + scalar_refindex_water_sw_layout); + layouts_water.emplace("refindex_im_water_lw", + scalar_refindex_water_lw_layout); + layouts_water.emplace("refindex_real_water_lw", + scalar_refindex_water_lw_layout); + + // create a object to read data + AtmosphereInput refindex_water(params, grid, host_views_water, layouts_water); + refindex_water.read_variables(); + refindex_water.finalize(); + + // maybe make a 1D vied of Kokkos::complex + const auto crefwlw_host = Kokkos::create_mirror_view(crefwlw); + const auto crefwsw_host = Kokkos::create_mirror_view(crefwsw); + for(int i = 0; i < nlwbands; ++i) { + // Kokkos::complex temp; + crefwlw_host(i).real() = refindex_real_water_lw_host(i); + crefwlw_host(i).imag() = haero::abs(refindex_im_water_lw_host(i)); + } + Kokkos::deep_copy(crefwlw, crefwlw_host); + // set complex representation of refractive indices as module data + for(int i = 0; i < nswbands; ++i) { + // Kokkos::complex temp; + crefwsw_host(i).real() = refindex_real_water_sw_host(i); + crefwsw_host(i).imag() = haero::abs(refindex_im_water_sw_host(i)); + } + Kokkos::deep_copy(crefwsw, crefwsw_host); +} +// read_refindex_aero + +inline void set_refindex_names(std::string surname, ekat::ParameterList ¶ms, + std::map &host_views, + std::map &layouts) { + // set variables names + using view_1d_host = typename KT::view_1d::HostMirror; + using strvec_t = std::vector; + using namespace ShortFieldTagsNames; + + std::string refindex_real_sw = "refindex_real_" + surname + "_sw"; + std::string refindex_im_sw = "refindex_im_" + surname + "_sw"; + std::string refindex_real_lw = "refindex_real_" + surname + "_lw"; + std::string refindex_im_lw = "refindex_im_" + surname + "_lw"; + + params.set("Skip_Grid_Checks", true); + params.set("Field Names", {refindex_real_sw, refindex_im_sw, + refindex_real_lw, refindex_im_lw}); + // allocate host views + host_views[refindex_real_sw] = view_1d_host(refindex_real_sw, nswbands); + host_views[refindex_im_sw] = view_1d_host(refindex_im_sw, nswbands); + host_views[refindex_real_lw] = view_1d_host(refindex_real_lw, nlwbands); + host_views[refindex_im_lw] = view_1d_host(refindex_im_lw, nlwbands); + + FieldLayout scalar_refindex_sw_layout{{SWBND}, {nswbands}}; + FieldLayout scalar_refindex_lw_layout{{LWBND}, {nlwbands}}; + + layouts.emplace(refindex_real_sw, scalar_refindex_sw_layout); + layouts.emplace(refindex_im_sw, scalar_refindex_sw_layout); + layouts.emplace(refindex_real_lw, scalar_refindex_lw_layout); + layouts.emplace(refindex_im_lw, scalar_refindex_lw_layout); + +} // set_refindex_aero + +inline void set_refindex_aerosol( + const int species_id, std::map &host_views, + mam_coupling::complex_view_2d::HostMirror + &specrefndxsw_host, // complex refractive index for water visible + mam_coupling::complex_view_2d::HostMirror &specrefndxlw_host) { + std::string sw_real_name = "refindex_real_aer_sw"; + std::string lw_real_name = "refindex_real_aer_lw"; + std::string sw_im_name = "refindex_im_aer_sw"; + std::string lw_im_name = "refindex_im_aer_lw"; + + for(int i = 0; i < nswbands; i++) { + specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); + specrefndxsw_host(i, species_id).imag() = + haero::abs(host_views[sw_im_name](i)); + } + for(int i = 0; i < nlwbands; i++) { + specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); + specrefndxlw_host(i, species_id).imag() = + haero::abs(host_views[lw_im_name](i)); + } + +} // copy_refindex_to_device + +} // namespace scream::mam_coupling + +#endif diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp new file mode 100644 index 000000000000..e8a2d127c0d2 --- /dev/null +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -0,0 +1,916 @@ +#ifndef MAM_COUPLING_HPP +#define MAM_COUPLING_HPP + +#include +#include +#include +#include +#include + +// These data structures and functions are used to move data between EAMxx +// and mam4xx. This file must be adjusted whenever the aerosol modes and +// species are modified. + +namespace scream::mam_coupling { + +using KT = ekat::KokkosTypes; + +// views for single- and multi-column data +using view_1d = typename KT::template view_1d; +using view_2d = typename KT::template view_2d; +using view_3d = typename KT::template view_3d; +using const_view_1d = typename KT::template view_1d; +using const_view_2d = typename KT::template view_2d; + +using complex_view_3d = typename KT::template view_3d>; +using complex_view_2d = typename KT::template view_2d>; + +// Kokkos thread team (league member) +using Team = Kokkos::TeamPolicy::member_type; + +// unmanaged views (for buffer and workspace manager) +using uview_1d = typename ekat::template Unmanaged>; +using uview_2d = typename ekat::template Unmanaged>; + +using PF = scream::PhysicsFunctions; + +using view_int_1d = typename KT::template view_1d; + +// number of constituents in gas chemistry "work arrays" +KOKKOS_INLINE_FUNCTION +constexpr int gas_pcnst() { + constexpr int gas_pcnst_ = mam4::gas_chemistry::gas_pcnst; + return gas_pcnst_; +} + +// number of aerosol/gas species tendencies +KOKKOS_INLINE_FUNCTION +constexpr int nqtendbb() { return 4; } + +// returns the number of distinct aerosol modes +KOKKOS_INLINE_FUNCTION +constexpr int num_aero_modes() { + return mam4::AeroConfig::num_modes(); +} + +// returns the number of distinct aerosol species +KOKKOS_INLINE_FUNCTION +constexpr int num_aero_species() { + return mam4::AeroConfig::num_aerosol_ids(); +} + +// returns the number of distinct aerosol-related gases +KOKKOS_INLINE_FUNCTION +constexpr int num_aero_gases() { + return mam4::AeroConfig::num_gas_ids(); +} + +// returns the total number of aerosol tracers (i.e. the total number of +// distinct valid mode-species pairs) +KOKKOS_INLINE_FUNCTION +constexpr int num_aero_tracers() { + // see mam4::mode_aero_species() for valid per-mode aerosol species + // (in mam4xx/aero_modes.hpp) + return 7 + 4 + 7 + 3; +} + +// Given a MAM aerosol mode index, returns a string denoting the symbolic +// name of the mode. +KOKKOS_INLINE_FUNCTION +const char* aero_mode_name(const int mode) { + static const char *mode_names[num_aero_modes()] = { + "1", + "2", + "3", + "4", + }; + return mode_names[mode]; +} + +// Given a MAM aerosol species ID, returns a string denoting the symbolic +// name of the species. +KOKKOS_INLINE_FUNCTION +const char* aero_species_name(const int species_id) { + static const char *species_names[num_aero_species()] = { + "soa", + "so4", + "pom", + "bc", + "nacl", + "dst", + "mom", + }; + return species_names[species_id]; +} + +// Given a MAM aerosol-related gas ID, returns a string denoting the symbolic +// name of the gas species. +KOKKOS_INLINE_FUNCTION +const char* gas_species_name(const int gas_id) { + static const char *species_names[num_aero_gases()] = { + "O3", + "H2O2", + "H2SO4", + "SO2", + "DMS", + "SOAG" + }; + return species_names[gas_id]; +} + +// here we provide storage for names of fields generated by the functions below +namespace { + +KOKKOS_INLINE_FUNCTION +constexpr int max_field_name_len() { + return 128; +} + +KOKKOS_INLINE_FUNCTION +size_t gpu_strlen(const char* s) { + size_t l = 0; + while (s[l]) ++l; + return l; +} + +KOKKOS_INLINE_FUNCTION +void concat_2_strings(const char *s1, const char *s2, char *concatted) { + size_t len1 = gpu_strlen(s1); + for (size_t i = 0; i < len1; ++i) + concatted[i] = s1[i]; + size_t len2 = gpu_strlen(s2); + for (size_t i = 0; i < len2; ++i) + concatted[i + len1] = s2[i]; + concatted[len1+len2] = 0; +} + +KOKKOS_INLINE_FUNCTION +void concat_3_strings(const char *s1, const char *s2, const char *s3, char *concatted) { + size_t len1 = gpu_strlen(s1); + for (size_t i = 0; i < len1; ++i) + concatted[i] = s1[i]; + size_t len2 = gpu_strlen(s2); + for (size_t i = 0; i < len2; ++i) + concatted[i + len1] = s2[i]; + size_t len3 = gpu_strlen(s3); + for (size_t i = 0; i < len3; ++i) + concatted[i + len1 + len2] = s3[i]; + concatted[len1+len2+len3] = 0; +} + +KOKKOS_INLINE_FUNCTION +char* int_aero_nmr_names(int mode) { + static char int_aero_nmr_names_[num_aero_modes()][max_field_name_len()] = {}; + return int_aero_nmr_names_[mode]; +} + +KOKKOS_INLINE_FUNCTION +char* cld_aero_nmr_names(int mode) { + static char cld_aero_nmr_names_[num_aero_modes()][max_field_name_len()] = {}; + return cld_aero_nmr_names_[mode]; +} + +KOKKOS_INLINE_FUNCTION +char* int_aero_mmr_names(int mode, int species) { + static char int_aero_mmr_names_[num_aero_modes()][num_aero_species()][max_field_name_len()] = {}; + return int_aero_mmr_names_[mode][species]; +} + +KOKKOS_INLINE_FUNCTION +char* cld_aero_mmr_names(int mode, int species) { + static char cld_aero_mmr_names_[num_aero_modes()][num_aero_species()][max_field_name_len()] = {}; + return cld_aero_mmr_names_[mode][species]; +} + +KOKKOS_INLINE_FUNCTION +char* gas_mmr_names(int gas_id) { + static char gas_mmr_names_[num_aero_gases()][max_field_name_len()] = {}; + return gas_mmr_names_[gas_id]; +} + +} // end anonymous namespace + +// Given a MAM aerosol mode index, returns the name of the related interstitial +// modal number mixing ratio field in EAMxx ("num_a<1-based-mode-index>") +KOKKOS_INLINE_FUNCTION +const char* int_aero_nmr_field_name(const int mode) { + if (!int_aero_nmr_names(mode)[0]) { + concat_2_strings("num_a", aero_mode_name(mode), int_aero_nmr_names(mode)); + } + return const_cast(int_aero_nmr_names(mode)); +} + +// Given a MAM aerosol mode index, returns the name of the related cloudborne +// modal number mixing ratio field in EAMxx ("num_c<1-based-mode-index>>") +KOKKOS_INLINE_FUNCTION +const char* cld_aero_nmr_field_name(const int mode) { + if (!cld_aero_nmr_names(mode)[0]) { + concat_2_strings("num_c", aero_mode_name(mode), cld_aero_nmr_names(mode)); + } + return const_cast(cld_aero_nmr_names(mode)); +} + +// Given a MAM aerosol mode index and the index of the MAM aerosol species +// within it, returns the name of the relevant interstitial mass mixing ratio +// field in EAMxx. The form of the field name is "_a<1-based-mode-index>". +// If the desired species is not present within the desire mode, returns a blank +// string (""). +KOKKOS_INLINE_FUNCTION +const char* int_aero_mmr_field_name(const int mode, const int species) { + if (!int_aero_mmr_names(mode, species)[0]) { + const auto aero_id = mam4::mode_aero_species(mode, species); + if (aero_id != mam4::AeroId::None) { + concat_3_strings(aero_species_name(static_cast(aero_id)), + "_a", aero_mode_name(mode), + int_aero_mmr_names(mode, species)); + } + } + return const_cast(int_aero_mmr_names(mode, species)); +}; + +// Given a MAM aerosol mode index and the index of the MAM aerosol species +// within it, returns the name of the relevant cloudborne mass mixing ratio +// field in EAMxx. The form of the field name is "_c<1-based-mode-index>". +// If the desired species is not present within the desire mode, returns a blank +// string (""). +KOKKOS_INLINE_FUNCTION +const char* cld_aero_mmr_field_name(const int mode, const int species) { + if (!cld_aero_mmr_names(mode, species)[0]) { + const auto aero_id = mam4::mode_aero_species(mode, species); + if (aero_id != mam4::AeroId::None) { + concat_3_strings(aero_species_name(static_cast(aero_id)), + "_c", aero_mode_name(mode), + cld_aero_mmr_names(mode, species)); + } + } + return const_cast(cld_aero_mmr_names(mode, species)); +}; + +// Given a MAM aerosol-related gas identifier, returns the name of its mass +// mixing ratio field in EAMxx +KOKKOS_INLINE_FUNCTION +const char* gas_mmr_field_name(const int gas) { + return const_cast(gas_species_name(gas)); +} + +// This type stores multi-column views related specifically to the wet +// atmospheric state used by EAMxx. +struct WetAtmosphere { + const_view_2d qv; // wet water vapor specific humidity [kg vapor / kg moist air] + const_view_2d qc; // wet cloud liquid water mass mixing ratio [kg cloud water/kg moist air] + const_view_2d nc; // wet cloud liquid water number mixing ratio [# / kg moist air] + const_view_2d qi; // wet cloud ice water mass mixing ratio [kg cloud ice water / kg moist air] + const_view_2d ni; // wet cloud ice water number mixing ratio [# / kg moist air] + const_view_2d omega; // vertical pressure velocity [Pa/s] +}; + +// This type stores multi-column views related to the dry atmospheric state +// used by MAM. +struct DryAtmosphere { + Real z_surf; // height of bottom of atmosphere [m] + const_view_2d T_mid; // temperature at grid midpoints [K] + const_view_2d p_mid; // total pressure at grid midpoints [Pa] + view_2d qv; // dry water vapor mixing ratio [kg vapor / kg dry air] + view_2d qc; // dry cloud liquid water mass mixing ratio [kg cloud water/kg dry air] + view_2d nc; // dry cloud liquid water number mixing ratio [# / kg dry air] + view_2d qi; // dry cloud ice water mass mixing ratio [kg cloud ice water / kg dry air] + view_2d ni; // dry cloud ice water number mixing ratio [# / kg dry air] + view_2d z_mid; // height at layer midpoints [m] + view_2d z_iface; // height at layer interfaces [m] + view_2d dz; // layer thickness [m] + const_view_2d p_del; // hydrostatic "pressure thickness" at grid interfaces [Pa] + const_view_2d p_int; // total pressure at grid interfaces [Pa] + const_view_2d cldfrac; // cloud fraction [-] + view_2d w_updraft; // updraft velocity [m/s] + const_view_1d pblh; // planetary boundary layer height [m] + const_view_1d phis; // surface geopotential [m2/s2] +}; + +// This type stores aerosol number and mass mixing ratios evolved by MAM. It +// can be used to represent wet and dry aerosols. When you declare an +// AerosolState, you must decide whether it's a dry or wet aerosol state (with +// mixing ratios in terms of dry or wet parcels of air, respectively). +// These mixing ratios are organized by mode (and species, for mass mixing ratio) +// in the same way as they are in mam4xx, and indexed using mam4::AeroConfig. +struct AerosolState { + view_2d int_aero_nmr[num_aero_modes()]; // modal interstitial aerosol number mixing ratios [# / kg air] + view_2d cld_aero_nmr[num_aero_modes()]; // modal cloudborne aerosol number mixing ratios [# / kg air] + view_2d int_aero_mmr[num_aero_modes()][num_aero_species()]; // interstitial aerosol mass mixing ratios [kg aerosol / kg air] + view_2d cld_aero_mmr[num_aero_modes()][num_aero_species()]; // cloudborne aerosol mass mixing ratios [kg aerosol / kg air] + view_2d gas_mmr[num_aero_gases()]; // gas mass mixing ratios [kg gas / kg air] +}; + +// storage for variables used within MAM atmosphere processes, initialized with +// ATMBufferManager +struct Buffer { + // ====================== + // column midpoint fields + // ====================== + + // number of "scratch" fields that hold process-specific data + // (e.g. gas-phase chemistry fields that are only needed by aerosol + // microphysics) + static constexpr int num_2d_scratch = 10; + + // number of local fields stored at column midpoints + static constexpr int num_2d_mid = 8 + // number of dry atm fields + 2 * (num_aero_modes() + num_aero_tracers()) + + num_aero_gases() + + num_2d_scratch; + + // (dry) atmospheric state + uview_2d z_mid; // height at midpoints + uview_2d dz; // layer thickness + uview_2d qv_dry; // dry water vapor mixing ratio (dry air) + uview_2d qc_dry; // dry cloud water mass mixing ratio + uview_2d nc_dry; // dry cloud water number mixing ratio + uview_2d qi_dry; // cloud ice mass mixing ratio + uview_2d ni_dry; // dry cloud ice number mixing ratio + uview_2d w_updraft; // vertical wind velocity + + // aerosol dry interstitial/cloudborne number/mass mixing ratios + // (because the number of species per mode varies, not all of these will + // be used) + uview_2d dry_int_aero_nmr[num_aero_modes()]; + uview_2d dry_cld_aero_nmr[num_aero_modes()]; + uview_2d dry_int_aero_mmr[num_aero_modes()][num_aero_species()]; + uview_2d dry_cld_aero_mmr[num_aero_modes()][num_aero_species()]; + + // aerosol-related dry gas mass mixing ratios + uview_2d dry_gas_mmr[num_aero_gases()]; + + // undedicated scratch fields for process-specific data + uview_2d scratch[num_2d_scratch]; + + // ======================= + // column interface fields + // ======================= + + // number of local fields stored at column interfaces + static constexpr int num_2d_iface = 1; + + uview_2d z_iface; // height at interfaces + + // storage + Real* wsm_data; +}; + +// ON HOST, returns the number of bytes of device memory needed by the above +// Buffer type given the number of columns and vertical levels +inline size_t buffer_size(const int ncol, const int nlev) { + return sizeof(Real) * + (Buffer::num_2d_mid * ncol * nlev + + Buffer::num_2d_iface * ncol * (nlev+1)); +} + +// ON HOST, initializeѕ the Buffer type with sufficient memory to store +// intermediate (dry) quantities on the given number of columns with the given +// number of vertical levels. Returns the number of bytes allocated. +inline size_t init_buffer(const ATMBufferManager &buffer_manager, + const int ncol, const int nlev, + Buffer &buffer) { + Real* mem = reinterpret_cast(buffer_manager.get_memory()); + + // set view pointers for midpoint fields + uview_2d* view_2d_mid_ptrs[Buffer::num_2d_mid] = { + &buffer.z_mid, + &buffer.dz, + &buffer.qv_dry, + &buffer.qc_dry, + &buffer.nc_dry, + &buffer.qi_dry, + &buffer.ni_dry, + &buffer.w_updraft, + + // aerosol modes + &buffer.dry_int_aero_nmr[0], + &buffer.dry_int_aero_nmr[1], + &buffer.dry_int_aero_nmr[2], + &buffer.dry_int_aero_nmr[3], + &buffer.dry_cld_aero_nmr[0], + &buffer.dry_cld_aero_nmr[1], + &buffer.dry_cld_aero_nmr[2], + &buffer.dry_cld_aero_nmr[3], + + // the following requires knowledge of mam4's mode-species layout + // (see mode_aero_species() in mam4xx/aero_modes.hpp) + + // accumulation mode + &buffer.dry_int_aero_mmr[0][0], + &buffer.dry_int_aero_mmr[0][1], + &buffer.dry_int_aero_mmr[0][2], + &buffer.dry_int_aero_mmr[0][3], + &buffer.dry_int_aero_mmr[0][4], + &buffer.dry_int_aero_mmr[0][5], + &buffer.dry_int_aero_mmr[0][6], + &buffer.dry_cld_aero_mmr[0][0], + &buffer.dry_cld_aero_mmr[0][1], + &buffer.dry_cld_aero_mmr[0][2], + &buffer.dry_cld_aero_mmr[0][3], + &buffer.dry_cld_aero_mmr[0][4], + &buffer.dry_cld_aero_mmr[0][5], + &buffer.dry_cld_aero_mmr[0][6], + + // aitken mode + &buffer.dry_int_aero_mmr[1][0], + &buffer.dry_int_aero_mmr[1][1], + &buffer.dry_int_aero_mmr[1][2], + &buffer.dry_int_aero_mmr[1][3], + &buffer.dry_cld_aero_mmr[1][0], + &buffer.dry_cld_aero_mmr[1][1], + &buffer.dry_cld_aero_mmr[1][2], + &buffer.dry_cld_aero_mmr[1][3], + + // coarse mode + &buffer.dry_int_aero_mmr[2][0], + &buffer.dry_int_aero_mmr[2][1], + &buffer.dry_int_aero_mmr[2][2], + &buffer.dry_int_aero_mmr[2][3], + &buffer.dry_int_aero_mmr[2][4], + &buffer.dry_int_aero_mmr[2][5], + &buffer.dry_int_aero_mmr[2][6], + &buffer.dry_cld_aero_mmr[2][0], + &buffer.dry_cld_aero_mmr[2][1], + &buffer.dry_cld_aero_mmr[2][2], + &buffer.dry_cld_aero_mmr[2][3], + &buffer.dry_cld_aero_mmr[2][4], + &buffer.dry_cld_aero_mmr[2][5], + &buffer.dry_cld_aero_mmr[2][6], + + // primary carbon mode + &buffer.dry_int_aero_mmr[3][0], + &buffer.dry_int_aero_mmr[3][1], + &buffer.dry_int_aero_mmr[3][2], + &buffer.dry_cld_aero_mmr[3][0], + &buffer.dry_cld_aero_mmr[3][1], + &buffer.dry_cld_aero_mmr[3][2], + + // aerosol gases + &buffer.dry_gas_mmr[0], + &buffer.dry_gas_mmr[1], + &buffer.dry_gas_mmr[2], + &buffer.dry_gas_mmr[3], + &buffer.dry_gas_mmr[4], + &buffer.dry_gas_mmr[5] + }; + for (int i = 0; i < Buffer::num_2d_scratch; ++i) { + view_2d_mid_ptrs[Buffer::num_2d_mid+i-Buffer::num_2d_scratch] = &buffer.scratch[i]; + } + + for (int i = 0; i < Buffer::num_2d_mid; ++i) { + *view_2d_mid_ptrs[i] = view_2d(mem, ncol, nlev); + mem += view_2d_mid_ptrs[i]->size(); + } + + // set view pointers for interface fields + uview_2d* view_2d_iface_ptrs[Buffer::num_2d_iface] = { + &buffer.z_iface + }; + for (int i = 0; i < Buffer::num_2d_iface; ++i) { + *view_2d_iface_ptrs[i] = view_2d(mem, ncol, nlev+1); + mem += view_2d_iface_ptrs[i]->size(); + } + + // WSM data + buffer.wsm_data = mem; + + /* + // Compute workspace manager size to check used memory vs. requested memory + // (if needed) + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); + const int n_wind_slots = ekat::npack(2)*Spack::n; + const int n_trac_slots = ekat::npack(m_num_tracers+3)*Spack::n; + const int wsm_size = WSM::get_total_bytes_needed(nlevi_packs, 13+(n_wind_slots+n_trac_slots), policy)/sizeof(Spack); + mem += wsm_size; + */ + + // return the number of bytes allocated + return (mem - buffer_manager.get_memory())*sizeof(Real); +} + +// Given a dry atmosphere state, creates a haero::Atmosphere object for the +// column with the given index. This object can be provided to mam4xx for the +// column. +KOKKOS_INLINE_FUNCTION +haero::Atmosphere atmosphere_for_column(const DryAtmosphere& dry_atm, + const int column_index) { + EKAT_KERNEL_ASSERT_MSG(dry_atm.T_mid.data() != nullptr, + "T_mid not defined for dry atmosphere state!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.p_mid.data() != nullptr, + "p_mid not defined for dry atmosphere state!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.qv.data() != nullptr, + "qv not defined for dry atmosphere state!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.qc.data() != nullptr, + "qc not defined for dry atmosphere state!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.nc.data() != nullptr, + "nc not defined for dry atmosphere state!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.qi.data() != nullptr, + "qi not defined for dry atmosphere state!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.ni.data() != nullptr, + "ni not defined for dry atmosphere state!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.z_mid.data() != nullptr, + "z_mid not defined for dry atmosphere state!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del.data() != nullptr, + "p_del not defined for dry atmosphere state!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.p_int.data() != nullptr, + "p_int not defined for dry atmosphere state!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.cldfrac.data() != nullptr, + "cldfrac not defined for dry atmosphere state!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.w_updraft.data() != nullptr, + "w_updraft not defined for dry atmosphere state!"); + return haero::Atmosphere(mam4::nlev, + ekat::subview(dry_atm.T_mid, column_index), + ekat::subview(dry_atm.p_mid, column_index), + ekat::subview(dry_atm.qv, column_index), + ekat::subview(dry_atm.qc, column_index), + ekat::subview(dry_atm.nc, column_index), + ekat::subview(dry_atm.qi, column_index), + ekat::subview(dry_atm.ni, column_index), + ekat::subview(dry_atm.z_mid, column_index), + ekat::subview(dry_atm.p_del, column_index), + ekat::subview(dry_atm.p_int, column_index), + ekat::subview(dry_atm.cldfrac, column_index), + ekat::subview(dry_atm.w_updraft, column_index), + dry_atm.pblh(column_index)); +} + +// Given an AerosolState with views for dry aerosol quantities, creates a +// mam4::Prognostics object for the column with the given index with +// ONLY INTERSTITIAL AEROSOL VIEWS DEFINED. This object can be provided to +// mam4xx for the column. +KOKKOS_INLINE_FUNCTION +mam4::Prognostics interstitial_aerosols_for_column(const AerosolState& dry_aero, + const int column_index) { + constexpr int nlev = mam4::nlev; + mam4::Prognostics progs(nlev); + for (int m = 0; m < num_aero_modes(); ++m) { + EKAT_KERNEL_ASSERT_MSG(dry_aero.int_aero_nmr[m].data(), + "int_aero_nmr not defined for dry aerosol state!"); + progs.n_mode_i[m] = ekat::subview(dry_aero.int_aero_nmr[m], column_index); + for (int a = 0; a < num_aero_species(); ++a) { + if (dry_aero.int_aero_mmr[m][a].data()) { + progs.q_aero_i[m][a] = ekat::subview(dry_aero.int_aero_mmr[m][a], column_index); + } + } + } + for (int g = 0; g < num_aero_gases(); ++g) { + EKAT_KERNEL_ASSERT_MSG(dry_aero.gas_mmr[g].data(), + "gas_mmr not defined for dry aerosol state!"); + progs.q_gas[g] = ekat::subview(dry_aero.gas_mmr[g], column_index); + } + return progs; +} + +// Given a dry aerosol state, creates a mam4::Prognostics object for the column +// with the given index with interstitial and cloudborne aerosol views defined. +// This object can be provided to mam4xx for the column. +KOKKOS_INLINE_FUNCTION +mam4::Prognostics aerosols_for_column(const AerosolState& dry_aero, + const int column_index) { + auto progs = interstitial_aerosols_for_column(dry_aero, column_index); + for (int m = 0; m < num_aero_modes(); ++m) { + EKAT_KERNEL_ASSERT_MSG(dry_aero.cld_aero_nmr[m].data(), + "dry_cld_aero_nmr not defined for aerosol state!"); + progs.n_mode_c[m] = ekat::subview(dry_aero.cld_aero_nmr[m], column_index); + for (int a = 0; a < num_aero_species(); ++a) { + if (dry_aero.cld_aero_mmr[m][a].data()) { + progs.q_aero_c[m][a] = ekat::subview(dry_aero.cld_aero_mmr[m][a], column_index); + } + } + } + return progs; +} + +// Given a thread team and a dry atmosphere state, dispatches threads from the +// team to compute vertical layer heights and interfaces for the column with +// the given index. +KOKKOS_INLINE_FUNCTION +void compute_vertical_layer_heights(const Team& team, + const DryAtmosphere& dry_atm, + const int column_index) { + EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), + "Given column index does not correspond to given team!"); + + const auto dz = ekat::subview(dry_atm.dz, column_index); + const auto z_iface = ekat::subview(dry_atm.z_iface, column_index); + const auto z_mid = ekat::subview(dry_atm.z_mid, column_index); + const auto qv = ekat::subview(dry_atm.qv, column_index); + const auto p_mid = ekat::subview(dry_atm.p_mid, column_index); + const auto T_mid = ekat::subview(dry_atm.T_mid, column_index); + const auto pseudo_density = ekat::subview(dry_atm.p_del, column_index); + PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, dz); + team.team_barrier(); + PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, z_iface); + team.team_barrier(); // likely necessary to have z_iface up to date + PF::calculate_z_mid(team, mam4::nlev, z_iface, z_mid); +} + +// Given a thread team and wet and dry atmospheres, dispatches threads from the +// team to compute the vertical updraft velocity for the column with the given +// index. +KOKKOS_INLINE_FUNCTION +void compute_updraft_velocities(const Team& team, + const WetAtmosphere& wet_atm, + const DryAtmosphere& dry_atm, + const int column_index) { + EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), + "Given column index does not correspond to given team!"); + + constexpr int nlev = mam4::nlev; + int i = column_index; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { + const auto rho = PF::calculate_density(dry_atm.p_del(i,k), dry_atm.dz(i,k)); + dry_atm.w_updraft(i,k) = PF::calculate_vertical_velocity(wet_atm.omega(i,k), rho); + }); +} + +// Given a thread team and a wet atmosphere state, dispatches threads +// from the team to compute mixing ratios for a dry atmosphere state in th +// column with the given index. +KOKKOS_INLINE_FUNCTION +void compute_dry_mixing_ratios(const Team& team, + const WetAtmosphere& wet_atm, + const DryAtmosphere& dry_atm, + const int column_index) { + EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), + "Given column index does not correspond to given team!"); + + constexpr int nlev = mam4::nlev; + int i = column_index; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { + const auto qv_ik = wet_atm.qv(i,k); + dry_atm.qv(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); + dry_atm.qc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); + dry_atm.nc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); + dry_atm.qi(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); + dry_atm.ni(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); + }); +} + +// Given a thread team and wet atmospheric and aerosol states, dispatches threads +// from the team to compute mixing ratios for the given dry interstitial aerosol +// state for the column with the given index. +KOKKOS_INLINE_FUNCTION +void compute_dry_mixing_ratios(const Team& team, + const WetAtmosphere& wet_atm, + const AerosolState& wet_aero, + const AerosolState& dry_aero, + const int column_index) { + EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), + "Given column index does not correspond to given team!"); + + constexpr int nlev = mam4::nlev; + int i = column_index; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { + const auto qv_ik = wet_atm.qv(i,k); + for (int m = 0; m < num_aero_modes(); ++m) { + dry_aero.int_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); + if (dry_aero.cld_aero_nmr[m].data()) { + dry_aero.cld_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); + } + for (int a = 0; a < num_aero_species(); ++a) { + if (dry_aero.int_aero_mmr[m][a].data()) { + dry_aero.int_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); + } + if (dry_aero.cld_aero_mmr[m][a].data()) { + dry_aero.cld_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); + } + } + } + for (int g = 0; g < num_aero_gases(); ++g) { + dry_aero.gas_mmr[g](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); + } + }); +} + +// Given a thread team and dry atmospheric and aerosol states, dispatches threads +// from the team to compute mixing ratios for the given wet interstitial aerosol +// state for the column with the given index. +KOKKOS_INLINE_FUNCTION +void compute_wet_mixing_ratios(const Team& team, + const DryAtmosphere& dry_atm, + const AerosolState& dry_aero, + const AerosolState& wet_aero, + const int column_index) { + EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), + "Given column index does not correspond to given team!"); + + constexpr int nlev = mam4::nlev; + int i = column_index; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { + const auto qv_ik = dry_atm.qv(i,k); + for (int m = 0; m < num_aero_modes(); ++m) { + wet_aero.int_aero_nmr[m](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.int_aero_nmr[m](i,k), qv_ik); + if (wet_aero.cld_aero_nmr[m].data()) { + wet_aero.cld_aero_nmr[m](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.cld_aero_nmr[m](i,k), qv_ik); + } + for (int a = 0; a < num_aero_species(); ++a) { + if (wet_aero.int_aero_mmr[m][a].data()) { + wet_aero.int_aero_mmr[m][a](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.int_aero_mmr[m][a](i,k), qv_ik); + } + if (wet_aero.cld_aero_mmr[m][a].data()) { + wet_aero.cld_aero_mmr[m][a](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.cld_aero_mmr[m][a](i,k), qv_ik); + } + } + } + for (int g = 0; g < num_aero_gases(); ++g) { + wet_aero.gas_mmr[g](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.gas_mmr[g](i,k), qv_ik); + } + }); +} + +// Because CUDA C++ doesn't allow us to declare and use constants outside of +// KOKKOS_INLINE_FUNCTIONS, we define this macro that allows us to (re)define +// these constants where needed within two such functions so we don't define +// them inconsistently. Yes, it's the 21st century and we're still struggling +// with these basic things. +#define DECLARE_PROG_TRANSFER_CONSTANTS \ + /* mapping of constituent indices to aerosol modes */ \ + const auto Accum = mam4::ModeIndex::Accumulation; \ + const auto Aitken = mam4::ModeIndex::Aitken; \ + const auto Coarse = mam4::ModeIndex::Coarse; \ + const auto PC = mam4::ModeIndex::PrimaryCarbon; \ + const auto NoMode = mam4::ModeIndex::None; \ + static const mam4::ModeIndex mode_for_cnst[gas_pcnst()] = { \ + NoMode, NoMode, NoMode, NoMode, NoMode, NoMode, /* gases (not aerosols) */ \ + Accum, Accum, Accum, Accum, Accum, Accum, Accum, Accum, /* 7 aero species + NMR */ \ + Aitken, Aitken, Aitken, Aitken, Aitken, /* 4 aero species + NMR */ \ + Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, /* 7 aero species + NMR */ \ + PC, PC, PC, PC, /* 3 aero species + NMR */ \ + }; \ + /* mapping of constituent indices to aerosol species */ \ + const auto SOA = mam4::AeroId::SOA; \ + const auto SO4 = mam4::AeroId::SO4; \ + const auto POM = mam4::AeroId::POM; \ + const auto BC = mam4::AeroId::BC; \ + const auto NaCl = mam4::AeroId::NaCl; \ + const auto DST = mam4::AeroId::DST; \ + const auto MOM = mam4::AeroId::MOM; \ + const auto NoAero = mam4::AeroId::None; \ + static const mam4::AeroId aero_for_cnst[gas_pcnst()] = { \ + NoAero, NoAero, NoAero, NoAero, NoAero, NoAero, /* gases (not aerosols) */ \ + SO4, POM, SOA, BC, DST, NaCl, MOM, NoAero, /* accumulation mode */ \ + SO4, SOA, NaCl, MOM, NoAero, /* aitken mode */ \ + DST, NaCl, SO4, BC, POM, SOA, MOM, NoAero, /* coarse mode */ \ + POM, BC, MOM, NoAero, /* primary carbon mode */ \ + }; \ + /* mapping of constituent indices to gases */ \ + const auto O3 = mam4::GasId::O3; \ + const auto H2O2 = mam4::GasId::H2O2; \ + const auto H2SO4 = mam4::GasId::H2SO4; \ + const auto SO2 = mam4::GasId::SO2; \ + const auto DMS = mam4::GasId::DMS; \ + const auto SOAG = mam4::GasId::SOAG; \ + const auto NoGas = mam4::GasId::None; \ + static const mam4::GasId gas_for_cnst[gas_pcnst()] = { \ + O3, H2O2, H2SO4, SO2, DMS, SOAG, \ + NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ + NoGas, NoGas, NoGas, NoGas, NoGas, \ + NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ + NoGas, NoGas, NoGas, NoGas, \ + }; + +// Given a Prognostics object, transfers data for interstitial aerosols to the +// chemistry work array q, and cloudborne aerosols to the chemistry work array +// qqcw, both at vertical level k. The input and output quantities are stored as +// number/mass mixing ratios. +// NOTE: this mapping is chemistry-mechanism-specific (see mo_sim_dat.F90 +// NOTE: in the relevant preprocessed chemical mechanism) +// NOTE: see mam4xx/aero_modes.hpp to interpret these mode/aerosol/gas +// NOTE: indices +KOKKOS_INLINE_FUNCTION +void transfer_prognostics_to_work_arrays(const mam4::Prognostics &progs, + const int k, + Real q[gas_pcnst()], + Real qqcw[gas_pcnst()]) { + DECLARE_PROG_TRANSFER_CONSTANTS + + // copy number/mass mixing ratios from progs to q and qqcw at level k, + // converting them to VMR + for (int i = 0; i < gas_pcnst(); ++i) { + auto mode_index = mode_for_cnst[i]; + auto aero_id = aero_for_cnst[i]; + auto gas_id = gas_for_cnst[i]; + if (gas_id != NoGas) { // constituent is a gas + int g = static_cast(gas_id); + q[i] = progs.q_gas[g](k); + qqcw[i] = progs.q_gas[g](k); + } else { + int m = static_cast(mode_index); + if (aero_id != NoAero) { // constituent is an aerosol species + int a = aerosol_index_for_mode(mode_index, aero_id); + q[i] = progs.q_aero_i[m][a](k); + qqcw[i] = progs.q_aero_c[m][a](k); + } else { // constituent is a modal number mixing ratio + int m = static_cast(mode_index); + q[i] = progs.n_mode_i[m](k); + qqcw[i] = progs.n_mode_c[m](k); + } + } + } +} + +// converts the quantities in the work arrays q and qqcw from mass/number +// mixing ratios to volume/number mixing ratios +KOKKOS_INLINE_FUNCTION +void convert_work_arrays_to_vmr(const Real q[gas_pcnst()], + const Real qqcw[gas_pcnst()], + Real vmr[gas_pcnst()], + Real vmrcw[gas_pcnst()]) { + DECLARE_PROG_TRANSFER_CONSTANTS + + for (int i = 0; i < gas_pcnst(); ++i) { + auto mode_index = mode_for_cnst[i]; + auto aero_id = aero_for_cnst[i]; + auto gas_id = gas_for_cnst[i]; + if (gas_id != NoGas) { // constituent is a gas + int g = static_cast(gas_id); + const Real mw = mam4::gas_species(g).molecular_weight; + vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); + vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); + } else { + int m = static_cast(mode_index); + if (aero_id != NoAero) { // constituent is an aerosol species + int a = aerosol_index_for_mode(mode_index, aero_id); + const Real mw = mam4::aero_species(a).molecular_weight; + vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); + vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); + } else { // constituent is a modal number mixing ratio + vmr[i] = q[i]; + vmrcw[i] = qqcw[i]; + } + } + } +} + +// converts the quantities in the work arrays vmrs and vmrscw from mass/number +// mixing ratios to volume/number mixing ratios +KOKKOS_INLINE_FUNCTION +void convert_work_arrays_to_mmr(const Real vmr[gas_pcnst()], + const Real vmrcw[gas_pcnst()], + Real q[gas_pcnst()], + Real qqcw[gas_pcnst()]) { + DECLARE_PROG_TRANSFER_CONSTANTS + + for (int i = 0; i < gas_pcnst(); ++i) { + auto mode_index = mode_for_cnst[i]; + auto aero_id = aero_for_cnst[i]; + auto gas_id = gas_for_cnst[i]; + if (gas_id != NoGas) { // constituent is a gas + int g = static_cast(gas_id); + const Real mw = mam4::gas_species(g).molecular_weight; + q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); + qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); + } else { + int m = static_cast(mode_index); + if (aero_id != NoAero) { // constituent is an aerosol species + int a = aerosol_index_for_mode(mode_index, aero_id); + const Real mw = mam4::aero_species(a).molecular_weight; + q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); + qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); + } else { // constituent is a modal number mixing ratio + q[i] = vmr[i]; + qqcw[i] = vmrcw[i]; + } + } + } +} + +// Given work arrays with interstitial and cloudborne aerosol data, transfers +// them to the given Prognostics object at the kth vertical level. This is the +// "inverse operator" for transfer_prognostics_to_work_arrays, above. +KOKKOS_INLINE_FUNCTION +void transfer_work_arrays_to_prognostics(const Real q[gas_pcnst()], + const Real qqcw[gas_pcnst()], + mam4::Prognostics &progs, const int k) { + DECLARE_PROG_TRANSFER_CONSTANTS + + // copy number/mass mixing ratios from progs to q and qqcw at level k, + // converting them to VMR + for (int i = 0; i < gas_pcnst(); ++i) { + auto mode_index = mode_for_cnst[i]; + auto aero_id = aero_for_cnst[i]; + auto gas_id = gas_for_cnst[i]; + if (gas_id != NoGas) { // constituent is a gas + int g = static_cast(gas_id); + progs.q_gas[g](k) = q[i]; + } else { + int m = static_cast(mode_index); + if (aero_id != NoAero) { // constituent is an aerosol species + int a = aerosol_index_for_mode(mode_index, aero_id); + progs.q_aero_i[m][a](k) = q[i]; + progs.q_aero_c[m][a](k) = qqcw[i]; + } else { // constituent is a modal number mixing ratio + int m = static_cast(mode_index); + progs.n_mode_i[m](k) = q[i]; + progs.n_mode_c[m](k) = qqcw[i]; + } + } + } +} + +#undef DECLARE_PROG_TRANSFER_CONSTANTS + +} // namespace scream::mam_coupling + +#endif diff --git a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp index 1dff45c7d032..3fefd1c84123 100644 --- a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp +++ b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp @@ -1,7 +1,13 @@ #include "eamxx_nudging_process_interface.hpp" + #include "share/util/scream_universal_constants.hpp" #include "share/grid/remap/refining_remapper_p2p.hpp" #include "share/grid/remap/do_nothing_remapper.hpp" +#include "share/util/scream_utils.hpp" + +#include +#include +#include namespace scream { @@ -10,10 +16,12 @@ namespace scream Nudging::Nudging (const ekat::Comm& comm, const ekat::ParameterList& params) : AtmosphereProcess(comm, params) { - m_datafiles = m_params.get>("nudging_filename"); + m_datafiles = filename_glob(m_params.get>("nudging_filenames_patterns")); m_timescale = m_params.get("nudging_timescale",0); + m_fields_nudge = m_params.get>("nudging_fields"); m_use_weights = m_params.get("use_nudging_weights",false); + m_skip_vert_interpolation = m_params.get("skip_vert_interpolation",false); // If we are doing horizontal refine-remapping, we need to get the mapfile from user m_refine_remap_file = m_params.get( "nudging_refine_remap_mapfile", "no-file-given"); @@ -26,9 +34,17 @@ Nudging::Nudging (const ekat::Comm& comm, const ekat::ParameterList& params) m_src_pres_type = STATIC_1D_VERTICAL_PROFILE; // Check for a designated source pressure file, default to first nudging data source if not given. m_static_vertical_pressure_file = m_params.get("source_pressure_file",m_datafiles[0]); + EKAT_REQUIRE_MSG(m_skip_vert_interpolation == false, + "Error! It makes no sense to not interpolate if src press is uniform and constant "); } else { EKAT_ERROR_MSG("ERROR! Nudging::parameter_list - unsupported source_pressure_type provided. Current options are [TIME_DEPENDENT_3D_PROFILE,STATIC_1D_VERTICAL_PROFILE]. Please check"); } + int first_file_levs = scorpio::get_dimlen(m_datafiles[0],"lev"); + for (const auto& file : m_datafiles) { + int current_file_levs = scorpio::get_dimlen(file,"lev"); + EKAT_REQUIRE_MSG(current_file_levs == first_file_levs, + "Error! Inconsistent 'lev' dimension found in nudging data files."); + } // use nudging weights if (m_use_weights) m_weights_file = m_params.get("nudging_weights_file"); @@ -84,21 +100,26 @@ void Nudging::set_grids(const std::shared_ptr grids_manager) } else { m_num_src_levs = scorpio::get_dimlen(m_static_vertical_pressure_file,"lev"); } + if (m_skip_vert_interpolation) { + EKAT_REQUIRE_MSG(m_num_src_levs == m_num_levs, + "Error! skip_vert_interpolation requires the vertical level to be " + << " the same as model vertical level "); + } /* Check for consistency between nudging files, map file, and remapper */ // Number of columns globally - auto m_num_cols_global = m_grid->get_num_global_dofs(); + auto num_cols_global = m_grid->get_num_global_dofs(); // Get the information from the first nudging data file int num_cols_src = scorpio::get_dimlen(m_datafiles[0],"ncol"); - if (num_cols_src != m_num_cols_global) { + if (num_cols_src != num_cols_global) { // If differing cols, check if remap file is provided EKAT_REQUIRE_MSG(m_refine_remap_file != "no-file-given", "Error! Nudging::set_grids - the number of columns in the nudging data file " << std::to_string(num_cols_src) << " does not match the number of columns in the " - << "model grid " << std::to_string(m_num_cols_global) << ". Please check the " + << "model grid " << std::to_string(num_cols_global) << ". Please check the " << "nudging data file and/or the model grid."); // If remap file is provided, check if it is consistent with the nudging data file // First get the data from the mapfile @@ -110,9 +131,9 @@ void Nudging::set_grids(const std::shared_ptr grids_manager) << std::to_string(num_cols_src) << " does not match the number of columns in the " << "mapfile " << std::to_string(num_cols_remap_a) << ". Please check the " << "nudging data file and/or the mapfile."); - EKAT_REQUIRE_MSG(num_cols_remap_b == m_num_cols_global, + EKAT_REQUIRE_MSG(num_cols_remap_b == num_cols_global, "Error! Nudging::set_grids - the number of columns in the model grid " - << std::to_string(m_num_cols_global) << " does not match the number of columns in the " + << std::to_string(num_cols_global) << " does not match the number of columns in the " << "mapfile " << std::to_string(num_cols_remap_b) << ". Please check the " << "model grid and/or the mapfile."); EKAT_REQUIRE_MSG(m_use_weights == false, @@ -126,232 +147,189 @@ void Nudging::set_grids(const std::shared_ptr grids_manager) // If the number of columns is the same, we don't need to do any remapping, // but print a warning if the user provided a mapfile if (m_refine_remap_file != "no-file-given") { - std::cout << "Warning! Nudging::set_grids - the number of columns in the nudging data file " - << std::to_string(num_cols_src) << " matches the number of columns in the " - << "model grid " << std::to_string(m_num_cols_global) << ". The mapfile " - << m_refine_remap_file << " will NOT be used. Please check the " - << "nudging data file and/or the model grid." << std::endl; + m_atm_logger->warn( + "[Nudging::set_grids] Warning! Map file provided, but it is not needed.\n" + " - num cols in nudging data file: " + std::to_string(num_cols_src) + "\n" + " - num cols in model grid : " + std::to_string(num_cols_global) + "\n" + " Please, make sure the nudging data file and/or model grid are correct.\n" + " The map file is only needed if the above two numbers differ."); } // If the user gives us the vertical cutoff, warn them if (m_refine_remap_vert_cutoff > 0.0) { - std::cout << "Warning! Nudging::set_grids - the vertical cutoff " - << std::to_string(m_refine_remap_vert_cutoff) - << " is larger than zero, but we are not remapping. Please " - "check your settings." << std::endl; + m_atm_logger->warn( + "[Nudging::set_grids] Warning! Non-zero vertical cutoff provided, but it is not needed\n" + " - vertical cutoff: " + std::to_string(m_refine_remap_vert_cutoff) + "\n" + " Please, check your settings. This parameter is only needed if we are remapping."); } // Set m_refine_remap to false m_refine_remap = false; } } // ========================================================================================= -void Nudging::apply_tendency(Field& base, const Field& next, const Real dt) +void Nudging::apply_tendency(Field& state, const Field& nudge, const Real dt) const { // Calculate the weight to apply the tendency - const Real dtend = dt/Real(m_timescale); - EKAT_REQUIRE_MSG(dtend>=0,"Error! Nudging::apply_tendency - timescale tendency of " << std::to_string(dt) - << " / " << std::to_string(m_timescale) << " = " << std::to_string(dtend) - << " is invalid. Please check the timescale and/or dt"); - // Now apply the tendency. - Field tend = base.clone(); - // Use update internal to set tendency, will be (1.0*next - 1.0*base), note tend=base at this point. - tend.update(next,Real(1.0),Real(-1.0)); - base.update(tend,dtend,Real(1.0)); -} -// ========================================================================================= -void Nudging::apply_weighted_tendency(Field& base, const Field& next, const Field& weights, const Real dt) -{ - // Calculate the weight to apply the tendency - const Real dtend = dt/Real(m_timescale); - EKAT_REQUIRE_MSG(dtend>=0,"Error! Nudging::apply_tendency - timescale tendency of " << std::to_string(dt) - << " / " << std::to_string(m_timescale) << " = " << std::to_string(dtend) - << " is invalid. Please check the timescale and/or dt"); - // Now apply the tendency. - Field tend = base.clone(); - - // Use update internal to set tendency, will be (weights*next - weights*base), note tend=base at this point. - auto base_view = base.get_view(); - auto tend_view = tend.get_view< Real**>(); - auto next_view = next.get_view< Real**>(); - auto w_view = weights.get_view< Real**>(); - - const int num_cols = base_view.extent(0); - const int num_vert_packs = base_view.extent(1); - Kokkos::parallel_for(Kokkos::MDRangePolicy>({0, 0}, {num_cols, num_vert_packs}), KOKKOS_LAMBDA(int i, int j) { - tend_view(i,j) = next_view(i,j)*w_view(i,j) - base_view(i,j)*w_view(i,j); - }); - base.update(tend, dtend, Real(1.0)); -} -// ========================================================================================= -void Nudging::apply_vert_cutoff_tendency(Field &base, const Field &next, - const Field &p_mid, const Real cutoff, - const Real dt) { - // Calculate the weight to apply the tendency - const Real dtend = dt / Real(m_timescale); - EKAT_REQUIRE_MSG(dtend >= 0, - "Error! Nudging::apply_tendency - timescale tendency of " - << std::to_string(dt) << " / " - << std::to_string(m_timescale) << " = " - << std::to_string(dtend) - << " is invalid. Please check the timescale and/or dt"); - // Now apply the tendency. - Field tend = base.clone(); - - // Use update internal to set tendency, will be (weights*next - weights*base), - // note tend=base at this point. - auto base_view = base.get_view(); - auto tend_view = tend.get_view(); - auto next_view = next.get_view(); - auto pmid_view = p_mid.get_view(); - - const int num_cols = base_view.extent(0); - const int num_vert_packs = base_view.extent(1); - Kokkos::parallel_for( - Kokkos::MDRangePolicy>({0, 0}, - {num_cols, num_vert_packs}), - KOKKOS_LAMBDA(int i, int j) { - // If the pressure is above the cutoff, then we are closer to the surface - if (pmid_view(i, j) > cutoff) { - // Don't apply the tendency closer to the surface - tend_view(i, j) = 0.0; - } else { - // Apply the tendency farther up from the surface - tend_view(i, j) = next_view(i, j) - base_view(i, j); - } - }); - base.update(tend, dtend, Real(1.0)); + const Real dtend = dt / m_timescale; + + using cview_2d = decltype(state.get_view()); + + auto state_view = state.get_view(); + auto nudge_view = nudge.get_view(); + cview_2d w_view, pmid_view; + + if (m_use_weights) { + auto weights = get_helper_field("nudging_weights"); + w_view = weights.get_view(); + } + if (m_refine_remap_vert_cutoff>0) { + pmid_view = get_field_in("p_mid").get_view(); + } + + auto use_weights = m_use_weights; + auto cutoff = m_refine_remap_vert_cutoff; + auto policy = Kokkos::MDRangePolicy>({0, 0}, {m_num_cols, m_num_levs}); + auto update = KOKKOS_LAMBDA(const int& i, const int& j) { + if (cutoff>0 and pmid_view(i,j)>=cutoff) { + return; + } + + auto tend = nudge_view(i,j) - state_view(i,j); + if (use_weights) { + tend *= w_view(i,j); + } + state_view(i,j) += dtend * tend; + }; + Kokkos::parallel_for(policy,update); } // ============================================================================================================= void Nudging::initialize_impl (const RunType /* run_type */) { using namespace ShortFieldTagsNames; - // Set up pointers for grids - // external grid: from source data - std::shared_ptr grid_ext; - // temporary grid: after vertical interpolation - std::shared_ptr grid_tmp; - // internal grid: after horizontal interpolation - std::shared_ptr grid_int; - - // Initialize the refining remapper stuff at the outset, - // because we need to know the grid information; - // for now, we are doing the horizontal interpolation last, - // so we use the m_grid (model physics) as the target grid - grid_int = m_grid->clone(m_grid->name(), true); + + // The first thing we do is time interpolation. + // The second thing we do is horiz interpolation. The reason for doing horizontal + // before vertical is that we do not have a coarse p_mid (which would be needed + // as tgt pressure during vert remap). To get it, we'd have to "remap back" p_mid, + // but that seems overly complicated. For horiz remap we do not need anything + // on the target grid. + + // The "intermediate" grid, is the grid after horiz remap, and before vert remap + auto grid_tmp = m_grid->clone("after_horiz_before_vert",true); + grid_tmp->reset_num_vertical_lev(m_num_src_levs); + if (m_refine_remap) { // P2P remapper - m_refine_remapper = - std::make_shared(grid_int, m_refine_remap_file); - // Get grid from remapper, and clone it - auto grid_ext_const = m_refine_remapper->get_src_grid(); - grid_ext = grid_ext_const->clone(grid_ext_const->name(), true); - // Finally, grid_ext may have different levels - grid_ext->reset_num_vertical_lev(m_num_src_levs); + m_horiz_remapper = std::make_shared(grid_tmp, m_refine_remap_file); } else { - // We set up a DoNothingRemapper, which will do nothing - m_refine_remapper = std::make_shared(grid_int, grid_int); - // We clone physics grid, but maybe we have different levels - grid_ext = m_grid->clone(m_grid->name(), true); - grid_ext->reset_num_vertical_lev(m_num_src_levs); + // We set up an IdentityRemapper, specifying that tgt is an alias + // of src, so that the remap method will do nothing + auto r = std::make_shared(grid_tmp); + r->set_aliasing(IdentityRemapper::TgtAliasSrc); + m_horiz_remapper = r; } - // The temporary grid is the external grid, but with - // the same number of levels as the internal (physics) grid - grid_tmp = grid_ext->clone(grid_ext->name(), true); - grid_tmp->reset_num_vertical_lev(m_num_levs); - - // Declare the layouts for the helper fields (int: internal) - FieldLayout scalar2d_layout_mid { {LEV}, {m_num_levs} }; - FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols, m_num_levs} }; - // Get the number of external cols on current rank - auto m_num_cols_ext = grid_ext->get_num_local_dofs(); - // Declare the layouts for the helper fields (tmp: temporary)) - FieldLayout scalar2d_layout_mid_tmp { {LEV}, {m_num_levs}}; - FieldLayout scalar3d_layout_mid_tmp { {COL,LEV}, {m_num_cols_ext, m_num_levs} }; - // Declare the layouts for the helper fields (ext: external) - FieldLayout scalar2d_layout_mid_ext { {LEV}, {m_num_src_levs}}; - FieldLayout scalar3d_layout_mid_ext { {COL,LEV}, {m_num_cols_ext, m_num_src_levs} }; + // Now that we have the remapper, we can grab the grid where the input data lives + auto grid_ext = m_horiz_remapper->get_src_grid(); - // Initialize the time interpolator + // Initialize the time interpolator and horiz remapper m_time_interp = util::TimeInterpolation(grid_ext, m_datafiles); + m_time_interp.set_logger(m_atm_logger,"[EAMxx::Nudging] Reading nudging data"); - constexpr int ps = SCREAM_PACK_SIZE; - // To be extra careful, this should be the ext_grid - const auto& grid_ext_name = grid_ext->name(); - if (m_src_pres_type == TIME_DEPENDENT_3D_PROFILE) { - auto pmid_ext = create_helper_field("p_mid_ext", scalar3d_layout_mid_ext, grid_ext_name, ps); - m_time_interp.add_field(pmid_ext.alias("p_mid"),true); - } else if (m_src_pres_type == STATIC_1D_VERTICAL_PROFILE) { - // Load p_levs from source data file - ekat::ParameterList in_params; - in_params.set("Filename",m_static_vertical_pressure_file); - in_params.set("Skip_Grid_Checks",true); // We need to skip grid checks because multiple ranks may want the same column of source data. - std::map> host_views; - std::map layouts; - auto pmid_ext = create_helper_field("p_mid_ext", scalar2d_layout_mid_ext, grid_ext_name, ps); - auto pmid_ext_v = pmid_ext.get_view(); - in_params.set>("Field Names",{"p_levs"}); - host_views["p_levs"] = pmid_ext_v; - layouts.emplace("p_levs",scalar2d_layout_mid_ext); - AtmosphereInput src_input(in_params,grid_ext,host_views,layouts); - src_input.read_variables(-1); - src_input.finalize(); - pmid_ext.sync_to_dev(); - } - - // Open the registration! - m_refine_remapper->registration_begins(); - - // To create helper fields for later; we do both tmp and ext... + // NOTE: we are ASSUMING all fields are 3d and scalar! + const auto layout_ext = grid_ext->get_3d_scalar_layout(true); + const auto layout_tmp = grid_tmp->get_3d_scalar_layout(true); + const auto layout_atm = m_grid->get_3d_scalar_layout(true); + m_horiz_remapper->registration_begins(); for (auto name : m_fields_nudge) { std::string name_ext = name + "_ext"; std::string name_tmp = name + "_tmp"; - // Helper fields that will temporarily store the target state, which can then - // be used to back out a nudging tendency - auto grid_int_name = grid_int->name(); - auto grid_ext_name = grid_ext->name(); - auto grid_tmp_name = grid_tmp->name(); - auto field = get_field_out_wrap(name); - auto layout = field.get_header().get_identifier().get_layout(); - auto field_ext = create_helper_field(name_ext, scalar3d_layout_mid_ext, grid_ext_name, ps); - auto field_tmp = create_helper_field(name_tmp, scalar3d_layout_mid_tmp, grid_tmp_name, ps); - Field field_int; + + // First copy of the field: what's read from file, and time-interpolated. + auto field_ext = create_helper_field(name_ext, layout_ext, grid_ext->name()); + + // Second copy of the field: after horiz interp (alias "ext" if no remap) + Field field_tmp; if (m_refine_remap) { - field_int = create_helper_field(name, scalar3d_layout_mid, grid_int_name, ps); + field_tmp = create_helper_field(name_tmp, layout_tmp, grid_tmp->name()); } else { - field_int = field_tmp.alias(name); - m_helper_fields[name] = field_int; + field_tmp = field_ext.alias(name_tmp); + m_helper_fields[name_tmp] = field_tmp; } - // Register the fields with the remapper - m_refine_remapper->register_field(field_tmp, field_int); - // Add the fields to the time interpolator + // Add the field to the time interpolator m_time_interp.add_field(field_ext.alias(name), true); + + // Register the fields with the remapper + m_horiz_remapper->register_field(field_ext, field_tmp); + + if (m_timescale>0) { + // Third copy of the field: after vert interpolation. + // We cannot store directly in get_field_out(name), + // since we need to back out tendencies + create_helper_field(name, layout_atm, m_grid->name()); + } else { + // We do not need to back out any tendency; the input data is used + // to directly replace the atm state + m_helper_fields[name] = get_field_out_wrap(name); + } } - m_time_interp.initialize_data_from_files(); - // Close the registration! - m_refine_remapper->registration_ends(); + // A helper field, where we copy each field after horiz remap, padding it + // at top/bot, to allow vert lin interp to extrapolate outside the bounds of p_mid + FieldLayout layout_padded ({COL,LEV},{m_num_cols,m_num_src_levs+2}); + create_helper_field("padded_field",layout_padded,""); + + if (m_src_pres_type == TIME_DEPENDENT_3D_PROFILE && !m_skip_vert_interpolation) { + // If the pressure profile is 3d and time-dep, we need to interpolate (in time/horiz) + auto pmid_ext = create_helper_field("p_mid_ext", layout_ext, grid_ext->name()); + m_time_interp.add_field(pmid_ext.alias("p_mid"),true); + Field pmid_tmp; + if (m_refine_remap) { + pmid_tmp = create_helper_field("p_mid_tmp", layout_tmp, grid_tmp->name()); + } else { + pmid_tmp = pmid_ext.alias("p_mid_tmp"); + m_helper_fields["p_mid_tmp"] = pmid_tmp; + } + m_horiz_remapper->register_field(pmid_ext,pmid_tmp); + create_helper_field("padded_p_mid_tmp",layout_padded,""); + } else if (m_src_pres_type == STATIC_1D_VERTICAL_PROFILE) { + // For static 1D profile, we can read p_mid now + auto pmid_ext = create_helper_field("p_mid_ext", grid_ext->get_vertical_layout(true), grid_ext->name()); + AtmosphereInput src_input(m_static_vertical_pressure_file,grid_ext,{pmid_ext.alias("p_levs")},true); + src_input.read_variables(-1); + + // For static 1d profile, p_mid_tmp is an alias of p_mid_ext + m_helper_fields["p_mid_tmp"] = pmid_ext.alias("p_mid_tmp"); + + // The padded p_mid is also 1d + FieldLayout pmid1d_padded_layout({COL},{m_num_src_levs+2}); + create_helper_field("padded_p_mid_tmp",pmid1d_padded_layout,""); + } + + // Close the registration + m_time_interp.initialize_data_from_files(); + m_horiz_remapper->registration_ends(); // load nudging weights from file // NOTE: the regional nudging use the same grid as the run, no need to // do the interpolation. - if (m_use_weights) - { - auto grid_name = m_grid->name(); - auto nudging_weights = create_helper_field("nudging_weights", scalar3d_layout_mid, grid_name, ps); - std::vector fields; - fields.push_back(nudging_weights); - AtmosphereInput src_weights_input(m_weights_file, m_grid, fields); + if (m_use_weights) { + auto nudging_weights = create_helper_field("nudging_weights", layout_atm, m_grid->name()); + AtmosphereInput src_weights_input(m_weights_file, m_grid, {nudging_weights},true); src_weights_input.read_variables(); - src_weights_input.finalize(); - nudging_weights.sync_to_dev(); } } // ========================================================================================= void Nudging::run_impl (const double dt) { - using namespace scream::vinterp; + using KT = KokkosTypes; + using RangePolicy = typename KT::RangePolicy; + using MemberType = typename KT::MemberType; + using ESU = ekat::ExeSpaceUtils; + using PackT = ekat::Pack; + using view_1d = KT::view_1d; + using view_2d = KT::view_2d; // Have to add dt because first time iteration is at 0 seconds where you will // not have any data from the field. The timestamp is only iterated at the @@ -361,181 +339,204 @@ void Nudging::run_impl (const double dt) // Perform time interpolation m_time_interp.perform_time_interpolation(ts); - // Process data and nudge the atmosphere state - const auto& p_mid_v = get_field_in("p_mid").get_view(); - view_Nd p_mid_ext_p; - view_Nd p_mid_ext_1d; - if (m_src_pres_type == TIME_DEPENDENT_3D_PROFILE) { - p_mid_ext_p = get_helper_field("p_mid_ext").get_view(); - } else if (m_src_pres_type == STATIC_1D_VERTICAL_PROFILE) { - p_mid_ext_1d = get_helper_field("p_mid_ext").get_view(); - } + // If the input data contains "masked" values (sometimes also called "filled" values), + // the horiz remapping would smear them around. To prevent that, we need to "cure" + // these values. Masked values can only happen at top/bot of the model (with top + // being not common), and they must be a contiguous set of entries. So to cure them, + // we simply set all bot/top masked entries equal to the first non-masked value + // from the bot/top respectively. This corresponds to a constant extrapolation. + // NOTE: we need to do a tol check, since time interpolation may not return fillValue, + // even if both f(t_beg)/f(t_end) are equal to fillValue (due to rounding). + // NOTE: if f(t_beg)==fillValue!=f(t_end), or viceversa, the time-interpolated value can + // substantially differ from fillValue. Here, we assume it didn't happen. + auto correct_masked_values = [&](const Field f) { + const auto fl = f.get_header().get_identifier().get_layout(); + const auto v = f.get_view(); - for (auto name : m_fields_nudge) { - auto atm_state_field = get_field_out_wrap(name); // int horiz, int vert - auto int_state_field = get_helper_field(name); // int horiz, int vert - auto ext_state_field = get_helper_field(name+"_ext"); // ext horiz, ext vert - auto tmp_state_field = get_helper_field(name+"_tmp"); // ext horiz, int vert - auto ext_state_view = ext_state_field.get_view(); - auto tmp_state_view = tmp_state_field.get_view(); - auto atm_state_view = atm_state_field.get_view(); // TODO: Right now assume whatever field is defined on COLxLEV - auto int_state_view = int_state_field.get_view(); - auto int_mask_view = m_buffer.int_mask_view; - // Masked values in the source data can lead to strange behavior in the vertical interpolation. - // We pre-process the data and map any masked values (sometimes called "filled" values) to the - // nearest un-masked value. - // Here we are updating the ext_state_view, which is the time interpolated values taken from the nudging - // data. Real var_fill_value = constants::DefaultFillValue().value; // Query the helper field for the fill value, if not present use default - if (ext_state_field.get_header().has_extra_data("mask_value")) { - var_fill_value = ext_state_field.get_header().get_extra_data("mask_value"); + if (f.get_header().has_extra_data("mask_value")) { + var_fill_value = f.get_header().get_extra_data("mask_value"); } - const int num_cols = ext_state_view.extent(0); - const int num_vert_packs = ext_state_view.extent(1); - const int num_src_levs = m_num_src_levs; - const auto policy = ESU::get_default_team_policy(num_cols, num_vert_packs); - Kokkos::parallel_for("correct_for_masked_values", policy, - KOKKOS_LAMBDA(MemberType const& team) { - const int icol = team.league_rank(); - auto ext_state_view_1d = ekat::subview(ext_state_view,icol); - Real fill_value; - int fill_idx = -1; - // Scan top to surf and backfill all values near TOM that are masked. - for (int kk=0; kkthresh) { + // This entry is substantially different from var_fill_value, so it's good + first_good = ekat::impl::min(first_good,k); + last_good = ekat::impl::max(last_good,k); + } } - }); - - // Vertical Interpolation onto atmosphere state pressure levels - // Note that we are going from ext to tmp here - if (m_src_pres_type == TIME_DEPENDENT_3D_PROFILE) { - perform_vertical_interpolation(p_mid_ext_p, - p_mid_v, - ext_state_view, - tmp_state_view, - int_mask_view, - m_num_src_levs, - m_num_levs); - } else if (m_src_pres_type == STATIC_1D_VERTICAL_PROFILE) { - perform_vertical_interpolation(p_mid_ext_1d, - p_mid_v, - ext_state_view, - tmp_state_view, - int_mask_view, - m_num_src_levs, - m_num_levs); - } + EKAT_KERNEL_REQUIRE_MSG (first_good=0, + "[Nudging] Error! Could not locate a non-masked entry in a column.\n"); - // Check that none of the nudging targets are masked, if they are, set value to - // nearest unmasked value above. - // NOTE: We use an algorithm whichs scans from TOM to the surface. - // If TOM is masked we keep scanning until we hit an unmasked value, - // we then set all masked values above to the unmasked value. - // We continue scanning towards the surface until we hit an unmasked value, we - // then assign that masked value the most recent unmasked value, until we hit the - // surface. - // Here we change the int_state_view which represents the vertically interpolated fields onto - // the simulation grid. - const int num_levs = m_num_levs; - Kokkos::parallel_for("correct_for_masked_values", policy, - KOKKOS_LAMBDA(MemberType const& team) { - const int icol = team.league_rank(); - auto int_mask_view_1d = ekat::subview(int_mask_view,icol); - auto tmp_state_view_1d = ekat::subview(tmp_state_view,icol); - Real fill_value; - int fill_idx = -1; - // Scan top to surf and backfill all values near TOM that are masked. - for (int kk=0; kkremap(true); + // Perform horizontal remap (if needed) + m_horiz_remapper->remap(true); - for (auto name : m_fields_nudge) { - auto atm_state_field = get_field_out_wrap(name); // int horiz, int vert - auto int_state_field = get_helper_field(name); // int horiz, int vert - auto atm_state_view = atm_state_field.get_view(); // TODO: Right now assume whatever field is defined on COLxLEV - auto int_state_view = int_state_field.get_view(); - // Apply the nudging tendencies to the ATM state - if (m_timescale <= 0) { - // We do direct replacement - Kokkos::deep_copy(atm_state_view,int_state_view); + // bypass copy_and_pad and vert_interp for skip_vert_interpolation: + if (m_skip_vert_interpolation) { + for (const auto& name : m_fields_nudge) { + auto tmp_state_field = get_helper_field(name+"_tmp"); + + if (m_timescale > 0) { + auto atm_state_field = get_field_out_wrap(name); + apply_tendency(atm_state_field,tmp_state_field,dt); + } + } + return; + } + + // Copy remapper tgt fields into padded views, to allow extrapolation at top/bot, + // then call remapping routines + + const int ncols = m_num_cols; + const int nlevs_src = m_num_src_levs; + auto copy_and_pad = [&](const Field from, const Field to, const bool is_pmid) { + auto from_view = from.get_view(); + auto to_view = to.get_view(); + auto fl = from.get_header().get_identifier().get_layout(); + + auto copy_3d = KOKKOS_LAMBDA (const MemberType& team) { + int icol = team.league_rank(); + + auto copy_col = [&](const int k) { + to_view(icol,k+1) = from_view(icol,k); + }; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,nlevs_src),copy_col); + + // Set the first/last entries of data, so that linear interp + // can extrapolate if the p_tgt is outside the p_src bounds + Kokkos::single(Kokkos::PerTeam(team),[&]{ + to_view(icol,0) = 0; // Does this make sense for *every field*? + if (is_pmid) { + // For pmid, we put a very large value, so that any p_mid_tgt + // that is larger than input p_mid bnds will end up in the + // last interval. + to_view(icol,nlevs_src+1) = 1e7; + } else { + // For data, we set last entry equal to second-to-last. + // This will cause constant extrapolation outside of + // the input p_mid bounds + to_view(icol,nlevs_src+1) = from_view(icol,nlevs_src-1); + } + }); + }; + + auto policy = ESU::get_default_team_policy(ncols,nlevs_src); + Kokkos::parallel_for("", policy, copy_3d); + }; + + // First, copy/pad p_mid, and extract the right copy (1d vs 3d) + if (m_src_pres_type==TIME_DEPENDENT_3D_PROFILE) { + copy_and_pad (get_helper_field("p_mid_tmp"),get_helper_field("padded_p_mid_tmp"),true); + } else { + // pmid is a 1d view. Just pad by hand + auto from = get_helper_field("p_mid_tmp"); + auto to = get_helper_field("padded_p_mid_tmp"); + auto from_v = from.get_view(); + auto to_v = to.get_view(); + auto lambda = KOKKOS_LAMBDA(const int& lev) { + to_v(lev+1) = from_v(lev); + if (lev==0) { + to_v(0) = 0; + } else if (lev==nlevs_src-1) { + to_v(lev+2) = to_v(lev+1); + } + }; + Kokkos::parallel_for(RangePolicy(0,nlevs_src),lambda); + } + const auto& p_mid_v = get_field_in("p_mid").get_view(); + view_2d p_mid_tmp_3d; + view_1d p_mid_tmp_1d; + bool src_pmid_3d; + if (m_src_pres_type == TIME_DEPENDENT_3D_PROFILE) { + p_mid_tmp_3d = get_helper_field("padded_p_mid_tmp").get_view(); + src_pmid_3d = true; + } else { + p_mid_tmp_1d = get_helper_field("padded_p_mid_tmp").get_view(); + src_pmid_3d = false; + } + + // Setup the linear interpolation object + using LI = ekat::LinInterp; + const int nlevs_tgt = m_num_levs; + LI vert_interp(ncols,nlevs_src+2,nlevs_tgt); + const auto policy_vinterp = ESU::get_default_team_policy(ncols, nlevs_tgt); + auto p_tgt = get_field_in("p_mid").get_view(); + Kokkos::parallel_for("nudging_vert_interp_setup_loop", policy_vinterp, + KOKKOS_LAMBDA(const MemberType& team) { + + const int icol = team.league_rank(); + + // Setup + if (src_pmid_3d) { + vert_interp.setup(team, ekat::subview(p_mid_tmp_3d,icol), + ekat::subview(p_tgt,icol)); } else { - // Back out a tendency and apply it. - if (m_use_weights) { - // get nudging weights field - // NOTES: do we really need the vertical interpolation for nudging weights? Since we are going to - // use the same grids as the case by providing the nudging weights file. - // I would not apply the vertical interpolation here, but it depends... - // - auto nudging_weights_field = get_helper_field("nudging_weights"); - // appply the nudging tendencies to the ATM states - apply_weighted_tendency(atm_state_field, int_state_field, nudging_weights_field, dt); - } else if (m_refine_remap_vert_cutoff > 0.0) { - // If we have a cutoff, we apply the tendency with p_mid cutoff - // First, get p_mid the field in the atm (i.e., int) state - auto p_mid_field = get_field_in("p_mid"); - // Then, call the tendency with a Heaviside-like cutoff - apply_vert_cutoff_tendency(atm_state_field, int_state_field, - p_mid_field, m_refine_remap_vert_cutoff, dt); + vert_interp.setup(team, p_mid_tmp_1d, + ekat::subview(p_tgt,icol)); + } + }); + Kokkos::fence(); + + // Then loop over fields, and do copy_and_pad + vremap + auto padded_field = get_helper_field("padded_field"); + for (const auto& name : m_fields_nudge) { + copy_and_pad(get_helper_field(name+"_tmp"),padded_field,false); + + auto field_after_vinterp = get_helper_field(name); + auto view_in = padded_field.get_view(); + auto view_out = field_after_vinterp.get_view(); + // Now use the interpolation object in || over all variables. + auto vinterp = KOKKOS_LAMBDA(const MemberType& team) { + const int icol = team.league_rank(); + + view_1d x1; + if (src_pmid_3d) { + x1 = ekat::subview(p_mid_tmp_3d,icol); } else { - apply_tendency(atm_state_field, int_state_field, dt); + x1 = p_mid_tmp_1d; } + auto x2 = ekat::subview(p_tgt,icol); + + auto y1 = ekat::subview(view_in, icol); + auto y2 = ekat::subview(view_out,icol); + + vert_interp.lin_interp(team, x1, x2, y1, y2, icol); + }; + Kokkos::parallel_for("nudging_vert_interp_loop", policy_vinterp, vinterp); + Kokkos::fence(); + + // If timescale==0, the call get_helper_field(name) returns the same + // fields as get_field_out_wrap(name) they are alias, so nothing to do. + // If timescale>0, then we need to back out a tendency. + if (m_timescale > 0) { + auto atm_state_field = get_field_out_wrap(name); + apply_tendency(atm_state_field,field_after_vinterp,dt); } } } @@ -552,16 +553,13 @@ Field Nudging::create_helper_field (const std::string& name, const int ps) { using namespace ekat::units; + // For helper fields we don't bother w/ units, so we set them to non-dimensional FieldIdentifier id(name,layout,Units::nondimensional(),grid_name); // Create the field. Init with NaN's, so we spot instances of uninited memory usage Field f(id); - if (ps>=0) { - f.get_header().get_alloc_properties().request_allocation(ps); - } else { - f.get_header().get_alloc_properties().request_allocation(); - } + f.get_header().get_alloc_properties().request_allocation(ps); f.allocate_view(); f.deep_copy(ekat::ScalarTraits::invalid()); @@ -569,23 +567,6 @@ Field Nudging::create_helper_field (const std::string& name, return m_helper_fields[name]; } -// ========================================================================================= -size_t Nudging::requested_buffer_size_in_bytes() const { - return m_buffer.num_2d_midpoint_mask_views*m_num_cols*m_num_levs*sizeof(mMask); -} - -// ========================================================================================= -void Nudging::init_buffers(const ATMBufferManager& buffer_manager) { - EKAT_REQUIRE_MSG(buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), - "Error, Nudging::init_buffers! Buffers size not sufficient.\n"); - mMask* mem = reinterpret_cast(buffer_manager.get_memory()); - - m_buffer.int_mask_view = decltype(m_buffer.int_mask_view)(mem,m_num_cols,m_num_levs); - mem += m_buffer.int_mask_view.size(); - - size_t used_mem = (reinterpret_cast(mem) - buffer_manager.get_memory())*sizeof(Real); - EKAT_REQUIRE_MSG(used_mem==requested_buffer_size_in_bytes(), "Error: Nudging::init_buffers! Used memory != requested memory."); -} // ========================================================================================= Field Nudging::get_field_out_wrap(const std::string& field_name) { if (field_name == "U" or field_name == "V") { diff --git a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.hpp b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.hpp index 89d05728bcd8..604360b8dc2b 100644 --- a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.hpp +++ b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.hpp @@ -1,20 +1,12 @@ #ifndef SCREAM_NUDGING_HPP #define SCREAM_NUDGING_HPP -#include "share/util/eamxx_time_interpolation.hpp" #include "share/atm_process/atmosphere_process.hpp" -#include "ekat/ekat_parameter_list.hpp" -#include "ekat/util/ekat_lin_interp.hpp" -#include "share/io/scream_output_manager.hpp" -#include "share/io/scorpio_output.hpp" -#include "share/io/scorpio_input.hpp" -#include "share/io/scream_scorpio_interface.hpp" -#include "share/grid/mesh_free_grids_manager.hpp" -#include "share/grid/point_grid.hpp" -#include "share/util/scream_vertical_interpolation.hpp" -#include "share/util/scream_time_stamp.hpp" + +#include "share/util/eamxx_time_interpolation.hpp" #include "share/grid/remap/abstract_remapper.hpp" +#include #include namespace scream @@ -23,102 +15,56 @@ namespace scream /* * The class responsible to handle the nudging of variables */ - -// enum to track how the source pressure levels are defined -enum SourcePresType { - TIME_DEPENDENT_3D_PROFILE = 0, // DEFAULT - source data should include time/spatially varying p_mid with dimensions (time, col, lev) - STATIC_1D_VERTICAL_PROFILE = 1, // source data includes p_levs which is a static set of levels in both space and time, with dimensions (lev) -}; - class Nudging : public AtmosphereProcess { public: - using mPack = ekat::Pack; - using mMask = ekat::Mask<1>; - using KT = KokkosTypes; - - template - using view_1d = typename KT::template view_1d; - - template - using view_2d = typename KT::template view_2d; - - using uview_2d_mask = Unmanaged>; - - template - using view_Nd_host = typename KT::template view_ND::HostMirror; - - template - using view_1d_host = view_Nd_host; - - template - using view_2d_host = view_Nd_host; + // enum to track how the source pressure levels are defined + enum SourcePresType { + // DEFAULT - source data should include time/spatially varying p_mid with dimensions (time, col, lev) + TIME_DEPENDENT_3D_PROFILE, + // source data includes p_levs which is a static set of levels in both space and time, with dimensions (lev), + STATIC_1D_VERTICAL_PROFILE + }; // Constructors Nudging (const ekat::Comm& comm, const ekat::ParameterList& params); // The type of subcomponent - AtmosphereProcessType type () const { return AtmosphereProcessType::Physics; } + AtmosphereProcessType type () const override { return AtmosphereProcessType::Physics; } // The name of the subcomponent - std::string name () const { return "Nudging"; } + std::string name () const override { return "Nudging"; } // Set the grid - void set_grids (const std::shared_ptr grids_manager); - - // Internal function to apply nudging at specific timescale with weights - void apply_weighted_tendency(Field& base, const Field& next, const Field& weights, const Real dt); - - // Structure for storing local variables initialized using the ATMBufferManager - struct Buffer { - // 2D view - uview_2d_mask int_mask_view; - - // Total number of 2d views - static constexpr int num_2d_midpoint_mask_views = 1; - }; + void set_grids (const std::shared_ptr grids_manager) override; #ifndef KOKKOS_ENABLE_CUDA // Cuda requires methods enclosing __device__ lambda's to be public protected: #endif - void run_impl (const double dt); + void run_impl (const double dt) override; + + // Internal function to apply nudging at specific timescale + // NOTE: this method will handle weighted and cutoff cases as well + void apply_tendency (Field &state, const Field &nudge, const Real dt) const; - /* Nudge from coarse data */ - // See more details later in this file - // Must add this here to make it public for CUDA - // (refining) remapper vertically-weighted tendency application - void apply_vert_cutoff_tendency(Field &base, const Field &next, - const Field &p_mid, const Real cutoff, - const Real dt); protected: Field get_field_out_wrap(const std::string& field_name); // The two other main overrides for the subcomponent - void initialize_impl (const RunType run_type); - void finalize_impl (); - - // Computes total number of bytes needed for local variables - size_t requested_buffer_size_in_bytes() const; - - // Set local variables using memory provided by - // the ATMBufferManager - void init_buffers(const ATMBufferManager &buffer_manager); + void initialize_impl (const RunType run_type) override; + void finalize_impl () override; // Creates an helper field, not to be shared with the AD's FieldManager Field create_helper_field (const std::string& name, const FieldLayout& layout, const std::string& grid_name, - const int ps=0); + const int ps = 1); - // Query if a local field exists - bool has_helper_field (const std::string& name) const { return m_helper_fields.find(name)!=m_helper_fields.end(); } // Retrieve a helper field Field get_helper_field (const std::string& name) const { return m_helper_fields.at(name); } - // Internal function to apply nudging at specific timescale - void apply_tendency(Field& base, const Field& next, const Real dt); std::shared_ptr m_grid; // Keep track of field dimensions and the iteration count @@ -127,6 +73,7 @@ class Nudging : public AtmosphereProcess int m_num_src_levs; int m_timescale; bool m_use_weights; + bool m_skip_vert_interpolation; std::vector m_datafiles; std::string m_static_vertical_pressure_file; // add nudging weights for regional nudging update @@ -134,8 +81,6 @@ class Nudging : public AtmosphereProcess SourcePresType m_src_pres_type; - - // Some helper fields. std::map m_helper_fields; std::vector m_fields_nudge; @@ -146,13 +91,11 @@ class Nudging : public AtmosphereProcess // file containing coarse data mapping std::string m_refine_remap_file; // (refining) remapper object - std::shared_ptr m_refine_remapper; + std::shared_ptr m_horiz_remapper; // (refining) remapper vertical cutoff Real m_refine_remap_vert_cutoff; util::TimeInterpolation m_time_interp; - - Buffer m_buffer; }; // class Nudging } // namespace scream diff --git a/components/eamxx/src/physics/nudging/tests/CMakeLists.txt b/components/eamxx/src/physics/nudging/tests/CMakeLists.txt index 0187b85d2476..75f1c33d0473 100644 --- a/components/eamxx/src/physics/nudging/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/nudging/tests/CMakeLists.txt @@ -1,8 +1,19 @@ -if (NOT SCREAM_BASELINES_ONLY) +if (NOT SCREAM_ONLY_GENERATE_BASELINES) include(ScreamUtils) + CreateUnitTest (nudging_data "create_nudging_data.cpp" + LIBS scream_io + FIXTURES_SETUP nudging_data + ) + CreateUnitTest (nudging_map_files "create_map_file.cpp" + LIBS scream_io + FIXTURES_SETUP nudging_map_files + ) CreateUnitTest(nudging_tests "nudging_tests.cpp" - LIBS nudging scream_io - LABELS "physics_nudging" ) + LIBS nudging scream_io + MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} + LABELS physics nudging + FIXTURES_REQUIRED "nudging_map_files;nudging_data" + ) endif() diff --git a/components/eamxx/src/physics/nudging/tests/create_map_file.cpp b/components/eamxx/src/physics/nudging/tests/create_map_file.cpp new file mode 100644 index 000000000000..1a83abb5f7bb --- /dev/null +++ b/components/eamxx/src/physics/nudging/tests/create_map_file.cpp @@ -0,0 +1,63 @@ +#include + +#include "share/io/scream_scorpio_interface.hpp" + +TEST_CASE("create_map_file") +{ + using namespace scream; + + ekat::Comm comm(MPI_COMM_WORLD); + scorpio::eam_init_pio_subsystem(comm); + + // Add a dof in the middle of two coarse dofs + const int ngdofs_src = 12; + const int ngdofs_tgt = 2*ngdofs_src-1; + + std::string filename = "map_ncol" + std::to_string(ngdofs_src) + + "_to_" + std::to_string(ngdofs_tgt) + ".nc"; + + // Existing dofs are "copied", added dofs are averaged from neighbors + const int nnz = ngdofs_src + 2*(ngdofs_src-1); + + scorpio::register_file(filename, scorpio::FileMode::Write); + + scorpio::register_dimension(filename, "n_a", "n_a", ngdofs_src, false); + scorpio::register_dimension(filename, "n_b", "n_b", ngdofs_tgt, false); + scorpio::register_dimension(filename, "n_s", "n_s", nnz, false); + + scorpio::register_variable(filename, "col", "col", "1", {"n_s"}, "int", "int", ""); + scorpio::register_variable(filename, "row", "row", "1", {"n_s"}, "int", "int", ""); + scorpio::register_variable(filename, "S", "S", "1", {"n_s"}, "double", "double", ""); + + std::vector dofs(nnz); + std::iota(dofs.begin(),dofs.end(),0); + scorpio::set_dof(filename,"col",dofs.size(),dofs.data()); + scorpio::set_dof(filename,"row",dofs.size(),dofs.data()); + scorpio::set_dof(filename,"S", dofs.size(),dofs.data()); + + scorpio::eam_pio_enddef(filename); + + std::vector col(nnz), row(nnz); + std::vector S(nnz); + for (int i=0; iget_grid("Point Grid"); + + // Create a field manager, and init fields (since OM's need t0 values) + const auto fm1 = create_fm(grid); + const auto fm2 = create_fm(grid); + const auto fm3 = create_fm(grid); + compute_fields(fm1,t0,comm,0); + compute_fields(fm2,t0,comm,nlevs_filled); + compute_fields(fm3,t0,comm,0); + + // Create output manager + const auto om1 = create_om("nudging_data",fm1,gm,t0,comm); + const auto om2 = create_om("nudging_data_filled",fm2,gm,t0,comm); + const auto om3 = create_om("nudging_data_nonconst_p",fm3,gm,t0,comm); + + auto time = t0; + for (int istep=1; istep<=nsteps; ++istep) { + time += dt; + + // Compute fields, but keep p_mid constnat in fm1 and fm2, to avoid vinterp + compute_fields(fm1,time,comm,0,false); + compute_fields(fm2,time,comm,nlevs_filled,false); + compute_fields(fm3,time,comm,0); + + om1->run(time); + om2->run(time); + om3->run(time); + } + om1->finalize(); + om2->finalize(); + om3->finalize(); + + scorpio::eam_pio_finalize(); +} diff --git a/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp b/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp index 9851f2c72714..0f0742875fd1 100644 --- a/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp +++ b/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp @@ -1,390 +1,466 @@ #include "catch2/catch.hpp" + #include "physics/nudging/eamxx_nudging_process_interface.hpp" -#include "share/grid/mesh_free_grids_manager.hpp" -#include "share/io/scream_output_manager.hpp" -#include "share/io/scorpio_output.hpp" -#include "share/io/scorpio_input.hpp" -#include "share/io/scream_scorpio_interface.hpp" +#include "nudging_tests_helpers.hpp" -#include "share/grid/mesh_free_grids_manager.hpp" -#include "share/grid/point_grid.hpp" +#include "share/field/field_utils.hpp" -#include "share/field/field_identifier.hpp" -#include "share/field/field_header.hpp" -#include "share/field/field.hpp" -#include "share/field/field_manager.hpp" +using namespace scream; -#include "share/util/scream_time_stamp.hpp" -#include "share/scream_types.hpp" -#include "scream_config.h" +std::shared_ptr +create_nudging (const ekat::Comm& comm, + const ekat::ParameterList& params, + const std::shared_ptr& fm, + const std::shared_ptr& gm, + const util::TimeStamp& t0) +{ + auto nudging = std::make_shared(comm,params); + nudging->set_grids(gm); + for (const auto& req : nudging->get_required_field_requests()) { + auto f = fm->get_field(req.fid.name()); + nudging->set_required_field(f); + } + for (const auto& req : nudging->get_computed_field_requests()) { + auto f = fm->get_field(req.fid.name()); + nudging->set_computed_field(f); + } + nudging->initialize(t0,RunType::Initial); -using namespace scream; + return nudging; +} + +TEST_CASE("nudging_tests") { + auto& catch_capture = Catch::getResultCapture(); -std::shared_ptr -create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { + using strvec_t = std::vector; - const int num_global_cols = ncols*comm.size(); + ekat::Comm comm(MPI_COMM_WORLD); - using vos_t = std::vector; - ekat::ParameterList gm_params; - gm_params.set("grids_names",vos_t{"Point Grid"}); - auto& pl = gm_params.sublist("Point Grid"); - pl.set("type","point_grid"); - pl.set("aliases",vos_t{"Physics"}); - pl.set("number_of_global_columns", num_global_cols); - pl.set("number_of_vertical_levels", nlevs); + auto root_print = [&](const std::string& msg) { + if (comm.am_i_root()) { + printf("%s",msg.c_str()); + } + }; + + // Init scorpio + scorpio::eam_init_pio_subsystem(comm); + + // A refined grid, with one extra node in between each of the coarse ones + const int ngcols_fine = 2*ngcols_data - 1; + const int nlevs_fine = 2*nlevs_data -1; + + // Files names + auto postfix = ".INSTANT.nsteps_x1." + get_t0().to_string() + ".nc"; + auto nudging_data = "nudging_data" + postfix; + auto nudging_data_filled = "nudging_data_filled" + postfix; + auto map_file = "map_ncol" + std::to_string(ngcols_data) + + "_to_" + std::to_string(ngcols_fine) + ".nc"; + + // For grids managers, depending on whether ncols/nlevs match the (coarse) + // values used to generate the data or are finer + auto gm_data = create_gm (comm,ngcols_data,nlevs_data); + auto gm_fine_h = create_gm (comm,ngcols_fine,nlevs_data); + auto gm_fine_v = create_gm (comm,ngcols_data,nlevs_fine); + auto gm_fine = create_gm (comm,ngcols_fine,nlevs_fine); + + auto grid_data = gm_data->get_grid("Point Grid"); + auto grid_fine_h = gm_fine_h->get_grid("Point Grid"); + auto grid_fine_v = gm_fine_v->get_grid("Point Grid"); + auto grid_fine = gm_fine->get_grid("Point Grid"); + + const int ncols_data = grid_data->get_num_local_dofs(); + + // First section tests nudging when there is no horiz-vert interp + SECTION ("no-horiz-no-vert") { + ekat::ParameterList params; + params.set("nudging_filenames_patterns",{nudging_data}); + params.set("source_pressure_type","TIME_DEPENDENT_3D_PROFILE"); + params.set("nudging_fields",{"U"}); + params.get("log_level","warn"); - auto gm = create_mesh_free_grids_manager(comm,gm_params); - gm->build_grids(); + // Create fm. Init p_mid, since it's constant in this file + auto fm = create_fm(grid_data); + compute_field(fm->get_field("p_mid"),get_t0(),comm,0); - return gm; -} + auto U = fm->get_field("U"); + SECTION ("same-time") { + std::string msg = " -> Testing same time/horiz/vert grid as data ..........."; + root_print (msg + "\n"); + bool ok = true; + + // Create and init nudging process + auto nudging = create_nudging(comm,params,fm,gm_data,get_t0()); + + auto time = get_t0(); + + auto U_tgt = U.clone("U_tgt"); + for (int n=0; ok and nrun(dt_data); -TEST_CASE("nudging") { - - //This test makes a netcdf file with 3 columns and 34 levels - //with 4 fields, T_mid, p_mid, qv, and horiz_winds - //The p_mid field is filled based on the following equation: - //2j+1, where j is the level - //The T_mid, qv, and horiz_winds fields are filled based on the equations: - //(i-1)*10000+200*j+10*(dt/250.)*ii, where i is the column, j is the level, - //and ii in the timestep - //It then runs then nudging module with the netcdf file as input to nudge - //And then checks that the output fields of the nudging module match - //what should be in the netcdf file - - using namespace ekat::units; - using namespace ShortFieldTagsNames; - using FL = FieldLayout; - - // A time stamp - util::TimeStamp t0 ({2000,1,1},{0,0,0}); - - ekat::Comm io_comm(MPI_COMM_WORLD); // MPI communicator group used for I/O set as ekat object. - const int packsize = SCREAM_PACK_SIZE; - Int num_levs = 34; - - // Initialize the pio_subsystem for this test: - // MPI communicator group used for I/O. - // In our simple test we use MPI_COMM_WORLD, however a subset could be used. - MPI_Fint fcomm = MPI_Comm_c2f(io_comm.mpi_comm()); - // Gather the initial PIO subsystem data creater by component coupler - scorpio::eam_init_pio_subsystem(fcomm); - - // First set up a field manager and grids manager to interact - // with the output functions - auto gm2 = create_gm(io_comm,3,num_levs); - auto grid2 = gm2->get_grid("Point Grid"); - int num_lcols = grid2->get_num_local_dofs(); - - IOControl io_control; // Needed for testing input. - io_control.timestamp_of_last_write = t0; - io_control.nsamples_since_last_write = 0; - io_control.frequency_units = "nsteps"; - std::vector output_stamps; - - const Int dt = 250; - const Int max_steps = 12; - { - util::TimeStamp time = t0; - util::TimeStamp time0(t0); - - // Re-create the fm anew, so the fields are re-inited for each output type - //auto field_manager = get_test_fm(grid); - - using namespace ShortFieldTagsNames; - using FR = FieldRequest; - - // Create a fm - auto fm = std::make_shared(grid2); - - const int num_levs = grid2->get_num_vertical_levels(); - - // Create some fields for this fm - std::vector tag_h = {COL}; - std::vector tag_v = {LEV}; - std::vector tag_2d = {COL,LEV}; - std::vector tag_2d_vec = {COL,CMP,LEV}; - - std::vector dims_h = {num_lcols}; - std::vector dims_v = {num_levs}; - std::vector dims_2d = {num_lcols,num_levs}; - std::vector dims_2d_vec = {num_lcols,2,num_levs}; - - const std::string& gn = grid2->name(); - - FieldIdentifier fid1("p_mid",FL{tag_2d,dims_2d},Pa,gn); - FieldIdentifier fid2("T_mid",FL{tag_2d,dims_2d},K,gn); - FieldIdentifier fid3("qv",FL{tag_2d,dims_2d},kg/kg,gn); - FieldIdentifier fidhw("horiz_winds",FL{tag_2d_vec,dims_2d_vec},m/s,gn); - - // Register fields with fm - fm->registration_begins(); - fm->register_field(FR{fid1,"output",packsize}); - fm->register_field(FR{fid2,"output",packsize}); - fm->register_field(FR{fid3,"output",packsize}); - fm->register_field(FR{fidhw,"output",packsize}); - fm->registration_ends(); - - // Initialize these fields - auto f1 = fm->get_field(fid1); - auto f1_host = f1.get_view(); - - auto f2 = fm->get_field(fid2); - auto f2_host = f2.get_view(); - - auto f3 = fm->get_field(fid3); - auto f3_host = f3.get_view(); - - auto fhw = fm->get_field(fidhw); - auto fhw_host = fhw.get_view(); - - for (int ii=0;iiinit_fields_time_stamp(time); - f1.sync_to_dev(); - f2.sync_to_dev(); - f3.sync_to_dev(); - fhw.sync_to_dev(); - - // Add subfields U and V to field manager - { - auto hw = fm->get_field("horiz_winds"); - const auto& fid = hw.get_header().get_identifier(); - const auto& layout = fid.get_layout(); - const int vec_dim = layout.get_vector_dim(); - const auto& units = fid.get_units(); - auto fU = hw.subfield("U",units,vec_dim,0); - auto fV = hw.subfield("V",units,vec_dim,1); - fm->add_field(fU); - fm->add_field(fV); + + // Test case where model times are in the middle of input data time intervals + SECTION ("half-time") { + std::string msg = " -> Testing same horiz/vert grid, different time grid ..."; + root_print (msg + "\n"); + bool ok = true; + + // Init time as t0-dt/2, so we're "half way" between data slices + auto time = get_t0() - dt_data/2; + + // Create and init nudging process + auto nudging = create_nudging(comm,params,fm,gm_data,time); + + auto tmp1 = U.clone(""); + auto tmp2 = U.clone(""); + + auto check_f = [&](const Field& f, + const util::TimeStamp& t_prev, + const util::TimeStamp& t_next) { + compute_field(tmp1,t_prev,comm,0); + compute_field(tmp2,t_next,comm,0); + tmp1.update(tmp2,0.5,0.5); + + // Since input data are integers, and the time-interp coeff is 0.5, + // we should be getting the exact answer + CHECK (views_are_equal(f,tmp1)); + ok &= catch_capture.lastAssertionPassed(); + }; + + auto t_prev = get_t0(); + for (int n=1; ok and nrun(dt_data); + + // Compare the two. Since we're exactly half way, we should get exact fp representation + check_f(U,t_prev,t_next); + + t_prev = t_next; + } + root_print (msg + (ok ? " PASS\n" : " FAIL\n")); } + } - // Set up parameter list control for output - ekat::ParameterList params; - params.set("filename_prefix","io_output_test"); - params.set("Averaging Type","Instant"); - params.set("Max Snapshots Per File",15); - std::vector fnames = {"T_mid","p_mid","qv","U","V"}; - params.set>("Field Names",fnames); - auto& params_sub = params.sublist("output_control"); - params_sub.set("frequency_units","nsteps"); - params_sub.set("Frequency",1); - params_sub.set("MPI Ranks in Filename",true); - io_control.frequency = params_sub.get("Frequency"); - - // Set up output manager. - OutputManager om; - om.setup(io_comm,params,fm,gm2,t0,t0,false); - io_comm.barrier(); - - const auto& out_fields = fm->get_groups_info().at("output"); - using namespace ShortFieldTagsNames; - // Time loop - for (Int ii=0;iim_fields_names) { - auto f = fm->get_field(fname); - f.sync_to_host(); - auto fl = f.get_header().get_identifier().get_layout(); - switch (fl.rank()) { - case 2: - { - auto v = f.get_view(); - for (int i=0; i(); - for (int i=0; i(); + auto data_h = data.get_view(); + const bool is_pmid = data.name()=="p_mid"; + for (int icol=0; icol("nudging_filenames_patterns",{nudging_data}); + params.set("source_pressure_type","TIME_DEPENDENT_3D_PROFILE"); + params.set("nudging_fields",{"U"}); + params.get("log_level","warn"); + + std::string msg = " -> Testing same time/horiz grid, different vert grid"; + root_print (msg + "\n"); + SECTION ("pmid_in_bounds") { + std::string msg = " -> Target pressure within source pressure bounds ....."; + root_print (msg + "\n"); + bool ok = true; + + // Create fm + auto fm = create_fm(grid_fine_v); + auto U = fm->get_field("U"); + auto p_mid = fm->get_field("p_mid"); + + // Create and init nudging process + auto nudging = create_nudging(comm,params,fm,gm_fine_v,get_t0()); + + // Compute pmid on data grid + auto layout_data = grid_data->get_3d_scalar_layout(true); + Field p_mid_data(FieldIdentifier("p_mid",layout_data,Pa,grid_data->name())); + p_mid_data.allocate_view(); + compute_field(p_mid_data,get_t0(),comm,0); + + manual_interp(p_mid_data,p_mid,true); + + auto time = get_t0(); + Field tmp_data = p_mid_data.clone("tmp data"); + Field tmp_fine = p_mid.clone("tmp fine"); + for (int n=0; ok and nrun(dt_data); + + // Compute data on fine grid, by manually interpolating + // (recall that nudging runs at t+dt) + compute_field(tmp_data,time+dt_data,comm,0); + manual_interp(tmp_data,tmp_fine,true); + + CHECK (views_are_equal(tmp_fine,U)); + ok &= catch_capture.lastAssertionPassed(); + time += dt_data; } + root_print (msg + (ok ? " PASS\n" : " FAIL\n")); } - // Finalize the output manager (close files) - om.finalize(); - } - - // Create a grids manager - const int ncols = 3; - const int nlevs = 35; - auto gm = create_gm(io_comm,ncols,nlevs); - auto grid = gm->get_grid("Physics"); - - ekat::ParameterList params_mid; - std::string nudging_f = "io_output_test.INSTANT.nsteps_x1."\ - "np1.2000-01-01-00000.nc"; - params_mid.set>("nudging_filename",{nudging_f}); - params_mid.set>("nudging_fields",{"T_mid","qv","U","V"}); - params_mid.set("use_nudging_weights",false); - std::shared_ptr nudging_mid = std::make_shared(io_comm,params_mid); - - nudging_mid->set_grids(gm); - - std::map input_fields; - std::map output_fields; - for (const auto& req : nudging_mid->get_required_field_requests()) { - Field f(req.fid); - auto & f_ap = f.get_header().get_alloc_properties(); - f_ap.request_allocation(packsize); - f.allocate_view(); - const auto name = f.name(); - f.get_header().get_tracking().update_time_stamp(t0); - nudging_mid->set_required_field(f); - input_fields.emplace(name,f); - if (name != "p_mid"){ - nudging_mid->set_computed_field(f); - output_fields.emplace(name,f); + SECTION ("pmid_out_of_bounds") { + std::string msg = " -> Target pressure outside source pressure bounds ...."; + root_print (msg + "\n"); + bool ok = true; + + // Create fm + auto fm = create_fm(grid_fine_v); + auto U = fm->get_field("U"); + auto p_mid = fm->get_field("p_mid"); + + // Create and init nudging process + auto nudging = create_nudging(comm,params,fm,gm_fine_v,get_t0()); + + // Compute pmid on data grid + auto layout_data = grid_data->get_3d_scalar_layout(true); + Field p_mid_data(FieldIdentifier("p_mid",layout_data,Pa,grid_data->name())); + p_mid_data.allocate_view(); + compute_field(p_mid_data,get_t0(),comm,0); + + manual_interp(p_mid_data,p_mid,false); + + auto time = get_t0(); + Field tmp_data = p_mid_data.clone("tmp data"); + Field tmp_fine = p_mid.clone("tmp fine"); + for (int n=0; ok and nrun(dt_data); + + // Compute data on fine grid, by manually interpolating + // (recall that nudging runs at t+dt) + compute_field(tmp_data,time+dt_data,comm,0); + manual_interp(tmp_data,tmp_fine,false); + + CHECK (views_are_equal(tmp_fine,U)); + ok &= catch_capture.lastAssertionPassed(); + time += dt_data; + } + root_print (msg + (ok ? " PASS\n" : " FAIL\n")); } } - //initialize - nudging_mid->initialize(t0,RunType::Initial); - Field p_mid = input_fields["p_mid"]; - Field T_mid = input_fields["T_mid"]; - Field qv = input_fields["qv"]; - Field hw = input_fields["horiz_winds"]; - Field T_mid_o = output_fields["T_mid"]; - Field qv_mid_o = output_fields["qv"]; - Field hw_o = output_fields["horiz_winds"]; - // Initialize memory buffer for all atm processes - auto memory_buffer = std::make_shared(); - memory_buffer->request_bytes(nudging_mid->requested_buffer_size_in_bytes()); - memory_buffer->allocate(); - nudging_mid->init_buffers(*memory_buffer); - - //fill data - //Don't fill T,qv,u,v because they will be nudged anyways - auto p_mid_v_h = p_mid.get_view(); - for (int icol=0; icol(); - auto qv_h_o = qv_mid_o.get_view(); - auto hw_h_o = hw_o.get_view(); - nudging_mid->run(100); - T_mid_o.sync_to_host(); - qv_mid_o.sync_to_host(); - hw_o.sync_to_host(); - - for (int icol=0; icol(); + auto glb_view_h = glb.get_view(); + for (int rank=0; rank(); + auto glb_data_h = glb_data.get_view(); + for (int icol=0; icolget_num_local_dofs(); + auto offset = ncols_fine; + comm.scan(&offset,1,MPI_SUM); + offset -= ncols_fine; + + auto fine_h = fine.get_view(); + for (int i=0; i("nudging_filenames_patterns",{nudging_data}); + params.set("source_pressure_type","TIME_DEPENDENT_3D_PROFILE"); + params.set("nudging_refine_remap_mapfile",map_file); + params.set("nudging_fields",{"U"}); + params.get("log_level","warn"); + + // Create fm + auto fm = create_fm(grid_fine_h); + auto U = fm->get_field("U"); + auto p_mid = fm->get_field("p_mid"); + + // Create and init nudging process + auto nudging = create_nudging(comm,params,fm,gm_fine_h,get_t0()); + + // Compute pmid on data grid + auto layout_data = grid_data->get_3d_scalar_layout(true); + Field p_mid_data(FieldIdentifier("p_mid",layout_data,Pa,grid_data->name())); + p_mid_data.allocate_view(); + compute_field(p_mid_data,get_t0(),comm,0); + + manual_interp(p_mid_data,p_mid); + + auto time = get_t0(); + Field tmp_data = p_mid_data.clone("tmp data"); + Field tmp_fine = p_mid.clone("tmp fine"); + for (int n=0; ok and nrun(dt_data); + + // Compute data on fine grid, by manually interpolating + // (recall that nudging runs at t+dt) + compute_field(tmp_data,time+dt_data,comm,0); + manual_interp(tmp_data,tmp_fine); + + CHECK (views_are_equal(tmp_fine,U)); + ok &= catch_capture.lastAssertionPassed(); + time += dt_data; } + root_print (msg + (ok ? " PASS\n" : " FAIL\n")); + } -} + SECTION ("filled-data") { + std::string msg = " -> Testing data with top/bot levels filled ............."; + root_print (msg + "\n"); + bool ok = true; + + // Helper lambda, to manually cure filled levels + auto manual_cure = [&](const Field& f) { + auto f_h = f.get_view(); + auto ncols = f.get_header().get_identifier().get_layout().dim(0); + auto nlevs = f.get_header().get_identifier().get_layout().dim(1); + auto first_good = nlevs_filled; + auto last_good = nlevs - nlevs_filled - 1; + for (int icol=0; icolfinalize(); + ekat::ParameterList params; + params.set("nudging_filenames_patterns",{nudging_data_filled}); + params.set("source_pressure_type","TIME_DEPENDENT_3D_PROFILE"); + params.set("nudging_fields",{"U"}); + params.get("log_level","warn"); + + // Create fm. Init p_mid, since it's constant in this file + auto fm = create_fm(grid_data); + auto U = fm->get_field("U"); + auto p_mid = fm->get_field("p_mid"); + compute_field(p_mid,get_t0(),comm,0); + + // Create and init nudging process + auto nudging = create_nudging(comm,params,fm,gm_data,get_t0()); + + auto time = get_t0(); + Field tmp = p_mid.clone("tmp"); + for (int n=0; ok and nrun(dt_data); + + // Compute data on fine grid, by manually interpolating + // (recall that nudging runs at t+dt) + compute_field(tmp,time+dt_data,comm,0); + manual_cure(tmp); + + CHECK (views_are_equal(tmp,U)); + ok &= catch_capture.lastAssertionPassed(); + time += dt_data; + } + root_print (msg + (ok ? " PASS\n" : " FAIL\n")); + } + // Clean up scorpio + scorpio::eam_pio_finalize(); } - diff --git a/components/eamxx/src/physics/nudging/tests/nudging_tests_helpers.hpp b/components/eamxx/src/physics/nudging/tests/nudging_tests_helpers.hpp new file mode 100644 index 000000000000..52e2b8ca0e30 --- /dev/null +++ b/components/eamxx/src/physics/nudging/tests/nudging_tests_helpers.hpp @@ -0,0 +1,167 @@ +#include "share/io/scream_output_manager.hpp" +#include "share/grid/mesh_free_grids_manager.hpp" +#include "share/field/field.hpp" +#include "share/field/field_manager.hpp" +#include "share/util/scream_time_stamp.hpp" +#include "share/scream_types.hpp" + +namespace scream +{ + +constexpr int ngcols_data = 12; +constexpr int nlevs_data = 20; +constexpr int nsteps_data = 5; +constexpr int dt_data = 100; +constexpr int nlevs_filled = 2; +constexpr double fill_val = 1e30; + +util::TimeStamp get_t0 () { + return util::TimeStamp ({2000,1,1},{12,0,0}); +} + +std::shared_ptr +create_gm (const ekat::Comm& comm, const int ngcols, const int nlevs) { + + using vos_t = std::vector; + ekat::ParameterList gm_params; + gm_params.set("grids_names",vos_t{"Point Grid"}); + auto& pl = gm_params.sublist("Point Grid"); + pl.set("type","point_grid"); + pl.set("aliases",vos_t{"Physics"}); + pl.set("number_of_global_columns", ngcols); + pl.set("number_of_vertical_levels", nlevs); + + auto gm = create_mesh_free_grids_manager(comm,gm_params); + gm->build_grids(); + + return gm; +} + +std::shared_ptr +create_fm (const std::shared_ptr& grid) +{ + using namespace ShortFieldTagsNames; + using namespace ekat::units; + using FR = FieldRequest; + + auto fm = std::make_shared(grid); + + const std::string& gn = grid->name(); + + auto scalar3d = grid->get_3d_scalar_layout(true); + auto vector3d = grid->get_3d_vector_layout(true,CMP,2); + + FieldIdentifier fid1("p_mid",scalar3d,Pa,gn); + FieldIdentifier fid2("horiz_winds",vector3d,m/s,gn); + + // Register fields with fm + fm->registration_begins(); + fm->register_field(FR(fid1)); + fm->register_field(FR(fid2)); + fm->registration_ends(); + + auto U = fm->get_field("horiz_winds").subfield("U",1,0); + auto V = fm->get_field("horiz_winds").subfield("V",1,1); + fm->add_field(U); + fm->add_field(V); + + return fm; +} + +void compute_field (Field f, + const util::TimeStamp& time, + const ekat::Comm& comm, + const int num_masked_levs = 0) +{ + const auto& fl = f.get_header().get_identifier().get_layout(); + + const int ncols = fl.dim(0); + const int nlevs = fl.dim(1); + + int offset = ncols; + comm.scan(&offset,1,MPI_SUM); + offset -= ncols; + + const int step = time.get_num_steps(); + const int lev_beg = num_masked_levs; + const int lev_end = nlevs - num_masked_levs; + + if (fl.rank()==2) { + const auto f_h = f.get_view(); + for (int icol=0;icol(); + for (int icol=0;icol& fm, + const util::TimeStamp& time, + const ekat::Comm& comm, + const int num_masked_levs = 0, + const bool update_p_mid = true) +{ + if (update_p_mid) { + // Don't mask pressure + compute_field(fm->get_field("p_mid"),time,comm,0); + } + compute_field(fm->get_field("U"),time,comm,num_masked_levs); + compute_field(fm->get_field("V"),time,comm,num_masked_levs); + + // Not sure if we need it, since we don't handle horiz_winds directly, I think + fm->get_field("horiz_winds").get_header().get_tracking().update_time_stamp(time); +} + +std::shared_ptr +create_om (const std::string& filename_prefix, + const std::shared_ptr& fm, + const std::shared_ptr& gm, + const util::TimeStamp& t0, + const ekat::Comm& comm) +{ + using strvec_t = std::vector; + + // NOTE: ask "real" fp precision, so even when building in double precision + // we can retrieve exactly the nudging data (if no remapping happens) + ekat::ParameterList params; + params.set("Averaging Type","INSTANT"); + params.set("filename_prefix",filename_prefix); + params.set("Floating Point Precision","real"); + params.set("MPI Ranks in Filename", false); + params.set("Field Names",strvec_t{"p_mid","U","V"}); + params.set("fill_value",fill_val); + + auto& ctrl_pl = params.sublist("output_control"); + ctrl_pl.set("frequency_units","nsteps"); + ctrl_pl.set("Frequency",1); + ctrl_pl.set("save_grid_data",false); + + auto om = std::make_shared(); + om->setup(comm,params,fm,gm,t0,t0,false); + return om; +} + +} // namespace scream diff --git a/components/eamxx/src/physics/p3/CMakeLists.txt b/components/eamxx/src/physics/p3/CMakeLists.txt index fb90ca03be2c..7ea0aeeb8145 100644 --- a/components/eamxx/src/physics/p3/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/CMakeLists.txt @@ -78,7 +78,7 @@ if (SCREAM_SMALL_KERNELS) add_library(p3 ${P3_SRCS} ${P3_SK_SRCS}) else() add_library(p3 ${P3_SRCS}) - if (NOT SCREAM_LIBS_ONLY AND NOT SCREAM_BASELINES_ONLY) + if (NOT SCREAM_LIBS_ONLY AND NOT SCREAM_ONLY_GENERATE_BASELINES) add_library(p3_sk ${P3_SRCS} ${P3_SK_SRCS}) # Always build shoc_sk with SCREAM_SMALL_KERNELS on target_compile_definitions(p3_sk PUBLIC "SCREAM_SMALL_KERNELS") diff --git a/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp index e26d6fe18743..0d2aad5247dd 100644 --- a/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp @@ -28,7 +28,8 @@ ::ice_sedimentation_disp( const view_ice_table& ice_table_vals, const uview_1d& precip_ice_surf, const uview_1d& nucleationPossible, - const uview_1d& hydrometeorsPresent) + const uview_1d& hydrometeorsPresent, + const physics::P3_Constants & p3constants) { using ExeSpace = typename KT::ExeSpace; const Int nk_pack = ekat::npack(nk); @@ -49,7 +50,7 @@ ::ice_sedimentation_disp( ekat::subview(inv_dz, i), team, workspace, nk, ktop, kbot, kdir, dt, inv_dt, ekat::subview(qi, i), ekat::subview(qi_incld, i), ekat::subview(ni, i), ekat::subview(ni_incld, i), ekat::subview(qm, i), ekat::subview(qm_incld, i), ekat::subview(bm, i), ekat::subview(bm_incld, i), ekat::subview(qi_tend, i), ekat::subview(ni_tend, i), - ice_table_vals, precip_ice_surf(i)); + ice_table_vals, precip_ice_surf(i), p3constants); }); } diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp index 5b075439aa91..c3692f3f814f 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp @@ -35,6 +35,7 @@ ::p3_main_init_disp( { using ExeSpace = typename KT::ExeSpace; const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); + Kokkos::parallel_for("p3_main_init", policy, KOKKOS_LAMBDA(const MemberType& team) { @@ -108,7 +109,8 @@ ::p3_main_internal_disp( const P3LookupTables& lookup_tables, const WorkspaceManager& workspace_mgr, Int nj, - Int nk) + Int nk, + const physics::P3_Constants & p3constants) { using ExeSpace = typename KT::ExeSpace; @@ -231,7 +233,7 @@ ::p3_main_internal_disp( T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, qv, th, qc, nc, qr, nr, qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, - ni_incld, bm_incld, nucleationPossible, hydrometeorsPresent); + ni_incld, bm_incld, nucleationPossible, hydrometeorsPresent, p3constants); // ------------------------------------------------------------------------------------------ // main k-loop (for processes): @@ -247,7 +249,7 @@ ::p3_main_internal_disp( nr_incld, ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, mu_r, lamr, logn0r, qv2qi_depos_tend, precip_total_tend, nevapr, qr_evap_tend, vap_liq_exchange, vap_ice_exchange, liq_ice_exchange, - pratot, prctot, nucleationPossible, hydrometeorsPresent); + pratot, prctot, nucleationPossible, hydrometeorsPresent, p3constants); //NOTE: At this point, it is possible to have negative (but small) nc, nr, ni. This is not // a problem; those values get clipped to zero in the sedimentation section (if necessary). @@ -273,14 +275,14 @@ ::p3_main_internal_disp( rho, inv_rho, rhofacr, cld_frac_r, inv_dz, qr_incld, workspace_mgr, lookup_tables.vn_table_vals, lookup_tables.vm_table_vals, nj, nk, ktop, kbot, kdir, infrastructure.dt, inv_dt, qr, nr, nr_incld, mu_r, lamr, precip_liq_flux, qtend_ignore, ntend_ignore, - diagnostic_outputs.precip_liq_surf, nucleationPossible, hydrometeorsPresent); + diagnostic_outputs.precip_liq_surf, nucleationPossible, hydrometeorsPresent, p3constants); // Ice sedimentation: (adaptive substepping) ice_sedimentation_disp( rho, inv_rho, rhofaci, cld_frac_i, inv_dz, workspace_mgr, nj, nk, ktop, kbot, kdir, infrastructure.dt, inv_dt, qi, qi_incld, ni, ni_incld, qm, qm_incld, bm, bm_incld, qtend_ignore, ntend_ignore, - lookup_tables.ice_table_vals, diagnostic_outputs.precip_ice_surf, nucleationPossible, hydrometeorsPresent); + lookup_tables.ice_table_vals, diagnostic_outputs.precip_ice_surf, nucleationPossible, hydrometeorsPresent, p3constants); // homogeneous freezing f cloud and rain homogeneous_freezing_disp( @@ -296,7 +298,8 @@ ::p3_main_internal_disp( rho, inv_rho, rhofaci, qv, th, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, - rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr, nucleationPossible, hydrometeorsPresent); + rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr, nucleationPossible, hydrometeorsPresent, + p3constants); // // merge ice categories with similar properties diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp index a9c0e5fe1445..3dae805255b0 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp @@ -62,7 +62,8 @@ ::p3_main_part1_disp( const uview_2d& ni_incld, const uview_2d& bm_incld, const uview_1d& nucleationPossible, - const uview_1d& hydrometeorsPresent) + const uview_1d& hydrometeorsPresent, + const physics::P3_Constants & p3constants) { using ExeSpace = typename KT::ExeSpace; const Int nk_pack = ekat::npack(nk); @@ -83,7 +84,7 @@ ::p3_main_part1_disp( ekat::subview(qv, i), ekat::subview(th_atm, i), ekat::subview(qc, i), ekat::subview(nc, i), ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), ekat::subview(qc_incld, i), ekat::subview(qr_incld, i), ekat::subview(qi_incld, i), ekat::subview(qm_incld, i), ekat::subview(nc_incld, i), ekat::subview(nr_incld, i), ekat::subview(ni_incld, i), ekat::subview(bm_incld, i), - nucleationPossible(i), hydrometeorsPresent(i)); + nucleationPossible(i), hydrometeorsPresent(i), p3constants); }); } diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp index 02073bd4f5af..2b619d54bf33 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp @@ -89,11 +89,14 @@ ::p3_main_part2_disp( const uview_2d& pratot, const uview_2d& prctot, const uview_1d& nucleationPossible, - const uview_1d& hydrometeorsPresent) + const uview_1d& hydrometeorsPresent, + const physics::P3_Constants & p3constants) { using ExeSpace = typename KT::ExeSpace; const Int nk_pack = ekat::npack(nk); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); + + // p3_cloud_sedimentation loop Kokkos::parallel_for( "p3_main_part2_disp", @@ -122,7 +125,7 @@ ::p3_main_part2_disp( ekat::subview(cdist1, i), ekat::subview(cdistr, i), ekat::subview(mu_r, i), ekat::subview(lamr, i), ekat::subview(logn0r, i), ekat::subview(qv2qi_depos_tend, i), ekat::subview(precip_total_tend, i), ekat::subview(nevapr, i), ekat::subview(qr_evap_tend, i), ekat::subview(vap_liq_exchange, i), ekat::subview(vap_ice_exchange, i), ekat::subview(liq_ice_exchange, i), - ekat::subview(pratot, i), ekat::subview(prctot, i), hydrometeorsPresent(i), nk); + ekat::subview(pratot, i), ekat::subview(prctot, i), hydrometeorsPresent(i), nk, p3constants); if (!hydrometeorsPresent(i)) return; }); diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp index 8e442cf2fc53..6d591a7e9328 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp @@ -56,7 +56,8 @@ ::p3_main_part3_disp( const uview_2d& diag_eff_radius_qc, const uview_2d& diag_eff_radius_qr, const uview_1d& nucleationPossible, - const uview_1d& hydrometeorsPresent) + const uview_1d& hydrometeorsPresent, + const physics::P3_Constants & p3constants) { using ExeSpace = typename KT::ExeSpace; const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); @@ -82,7 +83,7 @@ ::p3_main_part3_disp( ekat::subview(mu_c, i), ekat::subview(nu, i), ekat::subview(lamc, i), ekat::subview(mu_r, i), ekat::subview(lamr, i), ekat::subview(vap_liq_exchange, i), ekat::subview(ze_rain, i), ekat::subview(ze_ice, i), ekat::subview(diag_vm_qi, i), ekat::subview(diag_eff_radius_qi, i), ekat::subview(diag_diam_qi, i), ekat::subview(rho_qi, i), ekat::subview(diag_equiv_reflectivity, i), ekat::subview(diag_eff_radius_qc, i), - ekat::subview(diag_eff_radius_qr, i)); + ekat::subview(diag_eff_radius_qr, i), p3constants); }); } diff --git a/components/eamxx/src/physics/p3/disp/p3_rain_sed_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_rain_sed_impl_disp.cpp index 6a9a36b4b353..e77c91035174 100644 --- a/components/eamxx/src/physics/p3/disp/p3_rain_sed_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_rain_sed_impl_disp.cpp @@ -27,7 +27,8 @@ ::rain_sedimentation_disp( const uview_2d& nr_tend, const uview_1d& precip_liq_surf, const uview_1d& nucleationPossible, - const uview_1d& hydrometeorsPresent) + const uview_1d& hydrometeorsPresent, + const physics::P3_Constants & p3constants) { using ExeSpace = typename KT::ExeSpace; const Int nk_pack = ekat::npack(nk); @@ -49,7 +50,7 @@ ::rain_sedimentation_disp( team, workspace, vn_table_vals, vm_table_vals, nk, ktop, kbot, kdir, dt, inv_dt, ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(nr_incld, i), ekat::subview(mu_r, i), ekat::subview(lamr, i), ekat::subview(precip_liq_flux, i), - ekat::subview(qr_tend, i), ekat::subview(nr_tend, i), precip_liq_surf(i)); + ekat::subview(qr_tend, i), ekat::subview(nr_tend, i), precip_liq_surf(i), p3constants); }); } diff --git a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp index 04b304e2691c..d6264c38b8b7 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp @@ -3,6 +3,7 @@ #include "share/property_checks/field_lower_bound_check.hpp" // Needed for p3_init, the only F90 code still used. #include "physics/p3/p3_functions.hpp" +#include "physics/share/physics_constants.hpp" #include "physics/p3/p3_f90.hpp" #include "ekat/ekat_assert.hpp" @@ -204,6 +205,12 @@ void P3Microphysics::initialize_impl (const RunType /* run_type */) { // Gather runtime options runtime_options.max_total_ni = m_params.get("max_total_ni"); + + // setting P3 constants in a struct + m_p3constants.set_p3_from_namelist(m_params); + m_p3constants.print_p3constants(m_atm_logger); + // done setting P3 constants + // Set property checks for fields in this process add_invariant_check(get_field_out("T_mid"),m_grid,100.0,500.0,false); add_invariant_check(get_field_out("qv"),m_grid,1e-13,0.2,true); diff --git a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp index 6a9c945e5176..47d543ded022 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp @@ -22,6 +22,7 @@ namespace scream class P3Microphysics : public AtmosphereProcess { using P3F = p3::Functions; + using CP3 = physics::P3_Constants; using Spack = typename P3F::Spack; using Smask = typename P3F::Smask; using Pack = ekat::Pack; @@ -53,6 +54,8 @@ class P3Microphysics : public AtmosphereProcess // Set the grid void set_grids (const std::shared_ptr grids_manager); + CP3 m_p3constants; + /*--------------------------------------------------------------------------------------------*/ // Most individual processes have a pre-processing step that constructs needed variables from // the set of fields stored in the field manager. A structure like this defines those operations, diff --git a/components/eamxx/src/physics/p3/eamxx_p3_run.cpp b/components/eamxx/src/physics/p3/eamxx_p3_run.cpp index d4d1fdd6fdca..9ccfdaf20220 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_run.cpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_run.cpp @@ -29,7 +29,7 @@ void P3Microphysics::run_impl (const double dt) get_field_out("micro_vap_ice_exchange").deep_copy(0.0); P3F::p3_main(runtime_options, prog_state, diag_inputs, diag_outputs, infrastructure, - history_only, lookup_tables, workspace_mgr, m_num_cols, m_num_levs); + history_only, lookup_tables, workspace_mgr, m_num_cols, m_num_levs, m_p3constants); // Conduct the post-processing of the p3_main output. Kokkos::parallel_for( diff --git a/components/eamxx/src/physics/p3/impl/p3_autoconversion_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_autoconversion_impl.hpp index a939a7ab4a5f..43b1f21b7a74 100644 --- a/components/eamxx/src/physics/p3/impl/p3_autoconversion_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_autoconversion_impl.hpp @@ -3,6 +3,7 @@ #include "p3_functions.hpp" // for ETI only but harmless for GPU #include "p3_subgrid_variance_scaling_impl.hpp" +#include "physics/share/physics_constants.hpp" namespace scream { namespace p3 { @@ -13,18 +14,25 @@ void Functions ::cloud_water_autoconversion( const Spack& rho, const Spack& qc_incld, const Spack& nc_incld, const Spack& inv_qc_relvar, Spack& qc2qr_autoconv_tend, Spack& nc2nr_autoconv_tend, Spack& ncautr, + const physics::P3_Constants & p3constants, const Smask& context) { + // Khroutdinov and Kogan (2000) const auto qc_not_small = qc_incld >= 1e-8 && context; constexpr Scalar CONS3 = C::CONS3; + + const Scalar p3_autoconversion_prefactor = p3constants.p3_autoconversion_prefactor; + +//printf(" hey inside AAAAAAAAAAAAAAAA %13.6f \n", p3_autoconversion_factor); + if(qc_not_small.any()){ Spack sgs_var_coef; // sgs_var_coef = subgrid_variance_scaling(inv_qc_relvar, sp(2.47) ); sgs_var_coef = 1; qc2qr_autoconv_tend.set(qc_not_small, - sgs_var_coef*1350*pow(qc_incld,sp(2.47))*pow(nc_incld*sp(1.e-6)*rho,sp(-1.79))); + sgs_var_coef*p3_autoconversion_prefactor*pow(qc_incld,sp(2.47))*pow(nc_incld*sp(1.e-6)*rho,sp(-1.79))); // note: ncautr is change in Nr; nc2nr_autoconv_tend is change in Nc ncautr.set(qc_not_small, qc2qr_autoconv_tend*CONS3); nc2nr_autoconv_tend.set(qc_not_small, qc2qr_autoconv_tend*nc_incld/qc_incld); diff --git a/components/eamxx/src/physics/p3/impl/p3_cldliq_imm_freezing_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_cldliq_imm_freezing_impl.hpp index ea6597f43a75..2abcc3ef135e 100644 --- a/components/eamxx/src/physics/p3/impl/p3_cldliq_imm_freezing_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_cldliq_imm_freezing_impl.hpp @@ -20,20 +20,21 @@ ::cldliq_immersion_freezing( const Spack& mu_c, const Spack& cdist1, const Spack& qc_incld, const Spack& inv_qc_relvar, Spack& qc2qi_hetero_freeze_tend, Spack& nc2ni_immers_freeze_tend, + const physics::P3_Constants & p3constants, const Smask& context) { constexpr Scalar qsmall = C::QSMALL; - constexpr Scalar AIMM = C::AIMM; constexpr Scalar T_rainfrz = C::T_rainfrz; constexpr Scalar T_zerodegc = C::T_zerodegc; constexpr Scalar CONS5 = C::CONS5; constexpr Scalar CONS6 = C::CONS6; + const Scalar p3_a_imm = p3constants.p3_a_imm; const auto qc_not_small_and_t_freezing = (qc_incld >= qsmall) && (T_atm <= T_rainfrz) && context; if (qc_not_small_and_t_freezing.any()) { Spack expAimmDt, inv_lamc3; - expAimmDt.set(qc_not_small_and_t_freezing, exp(AIMM * (T_zerodegc-T_atm))); + expAimmDt.set(qc_not_small_and_t_freezing, exp(p3_a_imm * (T_zerodegc-T_atm))); inv_lamc3.set(qc_not_small_and_t_freezing, cube(1/lamc)); Spack sgs_var_coef; diff --git a/components/eamxx/src/physics/p3/impl/p3_cloud_rain_acc_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_cloud_rain_acc_impl.hpp index 74e605c0e4f3..5d64cb593afb 100644 --- a/components/eamxx/src/physics/p3/impl/p3_cloud_rain_acc_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_cloud_rain_acc_impl.hpp @@ -20,10 +20,13 @@ ::cloud_rain_accretion( const Spack& qc_incld, const Spack& nc_incld, const Spack& qr_incld, const Spack& inv_qc_relvar, Spack& qc2qr_accret_tend, Spack& nc_accret_tend, + const physics::P3_Constants & p3constants, const Smask& context) { constexpr Scalar qsmall = C::QSMALL; + const Scalar p3_k_accretion = p3constants.p3_k_accretion;; + Spack sgs_var_coef; // sgs_var_coef = subgrid_variance_scaling(inv_qc_relvar, sp(1.15) ); sgs_var_coef = 1; @@ -32,7 +35,7 @@ ::cloud_rain_accretion( if (qr_and_qc_not_small.any()) { // Khroutdinov and Kogan (2000) qc2qr_accret_tend.set(qr_and_qc_not_small, - sgs_var_coef * sp(67.0) * pow(qc_incld * qr_incld, sp(1.15))); + sgs_var_coef * sp(p3_k_accretion) * pow(qc_incld * qr_incld, sp(1.15))); nc_accret_tend.set(qr_and_qc_not_small, qc2qr_accret_tend * nc_incld / qc_incld); qc2qr_accret_tend.set(nc_accret_tend == 0 && context, 0); diff --git a/components/eamxx/src/physics/p3/impl/p3_dsd2_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_dsd2_impl.hpp index a503248ad855..c7c434c0daee 100644 --- a/components/eamxx/src/physics/p3/impl/p3_dsd2_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_dsd2_impl.hpp @@ -76,7 +76,8 @@ KOKKOS_FUNCTION void Functions:: get_rain_dsd2 ( const Spack& qr, Spack& nr, Spack& mu_r, - Spack& lamr, Spack& cdistr, Spack& logn0r, + Spack& lamr, + const physics::P3_Constants & p3constants, const Smask& context) { constexpr auto nsmall = C::NSMALL; @@ -84,27 +85,24 @@ get_rain_dsd2 ( constexpr auto cons1 = C::CONS1; lamr.set(context , 0); - cdistr.set(context, 0); - logn0r.set(context, 0); const auto qr_gt_small = qr >= qsmall && context; + const Scalar mu_r_const = p3constants.p3_mu_r_constant; + if (qr_gt_small.any()) { - constexpr Scalar mu_r_const = C::mu_r_const; // use lookup table to get mu // mu-lambda relationship is from Cao et al. (2008), eq. (7) // find spot in lookup table // (scaled N/q for lookup table parameter space) const auto nr_lim = max(nr, nsmall); - Spack inv_dum(0); - inv_dum.set(qr_gt_small, cbrt(qr / (cons1 * nr_lim * 6))); // Apply constant mu_r: Recall the switch to v4 tables means constant mu_r mu_r.set(qr_gt_small, mu_r_const); // recalculate slope based on mu_r - lamr.set(qr_gt_small, cbrt(cons1 * nr_lim * (mu_r + 3) * - (mu_r + 2) * (mu_r + 1)/qr)); + const auto mass_to_d3_factor = cons1 * (mu_r + 3) * (mu_r + 2) * (mu_r + 1); + lamr.set(qr_gt_small, cbrt(mass_to_d3_factor * nr_lim / qr)); // check for slope const auto lammax = (mu_r+1.)*sp(1.e+5); @@ -121,15 +119,31 @@ get_rain_dsd2 ( lamr.set(lt, lammin); lamr.set(gt, lammax); ekat_masked_loop(either, s) { - nr[s] = std::exp(3*std::log(lamr[s]) + std::log(qr[s]) + - std::log(std::tgamma(mu_r[s] + 1)) - std::log(std::tgamma(mu_r[s] + 4))) - / cons1; + nr[s] = lamr[s]*lamr[s]*lamr[s] * qr[s] / mass_to_d3_factor[s]; } } + } +} + +template +KOKKOS_FUNCTION +void Functions:: +get_cdistr_logn0r ( + const Spack& qr, const Spack& nr, const Spack& mu_r, + const Spack& lamr, Spack& cdistr, Spack& logn0r, + const Smask& context) +{ + constexpr auto qsmall = C::QSMALL; + cdistr.set(context, 0); + logn0r.set(context, 0); + + const auto qr_gt_small = qr >= qsmall && context; + + if (qr_gt_small.any()) { cdistr.set(qr_gt_small, nr/tgamma(mu_r + 1)); // note: logn0r is calculated as log10(n0r) - logn0r.set(qr_gt_small, log10(nr) + (mu_r + 1) * log10(lamr) - log10(tgamma(mu_r+1))); + logn0r.set(qr_gt_small, log10(cdistr) + (mu_r + 1) * log10(lamr)); } } diff --git a/components/eamxx/src/physics/p3/impl/p3_ice_collection_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_ice_collection_impl.hpp index 6cb9e0aed049..2c3d5f6fc22c 100644 --- a/components/eamxx/src/physics/p3/impl/p3_ice_collection_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_ice_collection_impl.hpp @@ -15,6 +15,7 @@ ::ice_cldliq_collection( const Spack& qi_incld, const Spack& qc_incld, const Spack& ni_incld, const Spack& nc_incld, Spack& qc2qi_collect_tend, Spack& nc_collect_tend, Spack& qc2qr_ice_shed_tend, Spack& ncshdc, + const physics::P3_Constants & p3constants, const Smask& context) { constexpr Scalar qsmall = C::QSMALL; @@ -27,17 +28,17 @@ ::ice_cldliq_collection( const auto both_ge_small = qi_incld_ge_small && qc_incld_ge_small && context; const auto both_ge_small_pos_t = both_ge_small && !t_is_negative; - constexpr auto eci = C::eci; + const Scalar p3_eci = p3constants.p3_eci; constexpr auto inv_dropmass = C::ONE/C::dropmass; qc2qi_collect_tend.set(both_ge_small && t_is_negative, - rhofaci*table_val_qc2qi_collect*qc_incld*eci*rho*ni_incld); - nc_collect_tend.set(both_ge_small, rhofaci*table_val_qc2qi_collect*nc_incld*eci*rho*ni_incld); + rhofaci*table_val_qc2qi_collect*qc_incld*p3_eci*rho*ni_incld); + nc_collect_tend.set(both_ge_small, rhofaci*table_val_qc2qi_collect*nc_incld*p3_eci*rho*ni_incld); // for T_atm > 273.15, assume cloud water is collected and shed as rain drops // sink for cloud water mass and number, note qcshed is source for rain mass - qc2qr_ice_shed_tend.set(both_ge_small_pos_t, rhofaci*table_val_qc2qi_collect*qc_incld*eci*rho*ni_incld); - nc_collect_tend.set(both_ge_small_pos_t, rhofaci*table_val_qc2qi_collect*nc_incld*eci*rho*ni_incld); + qc2qr_ice_shed_tend.set(both_ge_small_pos_t, rhofaci*table_val_qc2qi_collect*qc_incld*p3_eci*rho*ni_incld); + nc_collect_tend.set(both_ge_small_pos_t, rhofaci*table_val_qc2qi_collect*nc_incld*p3_eci*rho*ni_incld); // source for rain number, assume 1 mm drops are shed ncshdc.set(both_ge_small_pos_t, qc2qr_ice_shed_tend*inv_dropmass); } @@ -52,6 +53,7 @@ ::ice_rain_collection( const Spack& qi_incld, const Spack& ni_incld, const Spack& qr_incld, Spack& qr2qi_collect_tend, Spack& nr_collect_tend, + const physics::P3_Constants & p3constants, const Smask& context) { constexpr Scalar qsmall = C::QSMALL; @@ -65,11 +67,11 @@ ::ice_rain_collection( const auto both_ge_small_neg_t = both_ge_small && t_is_negative; constexpr Scalar ten = 10.0; - constexpr auto eri = C::eri; + const Scalar p3_eri = p3constants.p3_eri; // note: table_val_qr2qi_collect and logn0r are already calculated as log_10 - qr2qi_collect_tend.set(both_ge_small_neg_t, pow(ten, table_val_qr2qi_collect+logn0r)*rho*rhofaci*eri*ni_incld); - nr_collect_tend.set(both_ge_small_neg_t, pow(ten, table_val_nr_collect+logn0r)*rho*rhofaci*eri*ni_incld); + qr2qi_collect_tend.set(both_ge_small_neg_t, pow(ten, table_val_qr2qi_collect+logn0r)*rho*rhofaci*p3_eri*ni_incld); + nr_collect_tend.set(both_ge_small_neg_t, pow(ten, table_val_nr_collect+logn0r)*rho*rhofaci*p3_eri*ni_incld); // rain number sink due to collection // for T_atm > 273.15, assume collected rain number is shed as @@ -78,7 +80,7 @@ ::ice_rain_collection( // rate of ice mass due to melting // collection of rain above freezing does not impact total rain mass nr_collect_tend.set(both_ge_small && !t_is_negative, - pow(ten, table_val_nr_collect + logn0r)*rho*rhofaci*eri*ni_incld); + pow(ten, table_val_nr_collect + logn0r)*rho*rhofaci*p3_eri*ni_incld); // for now neglect shedding of ice collecting rain above freezing, since snow is // not expected to shed in these conditions (though more hevaily rimed ice would be // expected to lead to shedding) diff --git a/components/eamxx/src/physics/p3/impl/p3_ice_nucleation_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_ice_nucleation_impl.hpp index 7f84b42d21ec..27313934fa7e 100644 --- a/components/eamxx/src/physics/p3/impl/p3_ice_nucleation_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_ice_nucleation_impl.hpp @@ -13,6 +13,7 @@ ::ice_nucleation( const Spack& temp, const Spack& inv_rho, const Spack& ni, const Spack& ni_activated, const Spack& qv_supersat_i, const Scalar& inv_dt, const bool& do_predict_nc, const bool& do_prescribed_CCN, Spack& qv2qi_nucleat_tend, Spack& ni_nucleat_tend, + const physics::P3_Constants & p3constants, const Smask& context) { constexpr Scalar nsmall = C::NSMALL; @@ -22,6 +23,8 @@ ::ice_nucleation( constexpr Scalar piov3 = C::PIOV3; constexpr Scalar mi0 = sp(4.0)*piov3*sp(900.0)*sp(1.e-18); + const Scalar p3_dep_nucleation_exponent = p3constants.p3_dep_nucleation_exponent; + const auto t_lt_T_icenuc = temp < T_icenuc; const auto qv_supersat_i_ge_005 = qv_supersat_i >= 0.05; @@ -33,7 +36,7 @@ ::ice_nucleation( Spack dum{0.0}, N_nuc{0.0}, Q_nuc{0.0}; if (any_if_not_log.any()) { - dum = sp(0.005)*exp(sp(0.304)*(tmelt-temp))*sp(1.0e3)*inv_rho; + dum = sp(0.005)*exp(sp(p3_dep_nucleation_exponent)*(tmelt-temp))*sp(1.0e3)*inv_rho; dum = min(dum, sp(1.0e5)*inv_rho); diff --git a/components/eamxx/src/physics/p3/impl/p3_ice_sed_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_ice_sed_impl.hpp index 3ede317333a1..29232c69e68f 100644 --- a/components/eamxx/src/physics/p3/impl/p3_ice_sed_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_ice_sed_impl.hpp @@ -17,12 +17,13 @@ typename Functions::Spack Functions ::calc_bulk_rho_rime( const Spack& qi_tot, Spack& qi_rim, Spack& bi_rim, + const physics::P3_Constants & p3constants, const Smask& context) { constexpr Scalar bsmall = C::BSMALL; constexpr Scalar qsmall = C::QSMALL; - constexpr Scalar rho_rime_min = C::rho_rimeMin; - constexpr Scalar rho_rime_max = C::rho_rimeMax; + const Scalar p3_rho_rime_min = p3constants.p3_rho_rime_min; + const Scalar p3_rho_rime_max = p3constants.p3_rho_rime_max; Spack rho_rime(0); @@ -32,12 +33,12 @@ ::calc_bulk_rho_rime( rho_rime.set(bi_rim_gt_small, qi_rim / bi_rim); } - Smask rho_rime_lt_min = rho_rime < rho_rime_min; - Smask rho_rime_gt_max = rho_rime > rho_rime_max; + Smask rho_rime_lt_min = rho_rime < p3_rho_rime_min; + Smask rho_rime_gt_max = rho_rime > p3_rho_rime_max; // impose limits on rho_rime; adjust bi_rim if needed - rho_rime.set(bi_rim_gt_small && rho_rime_lt_min, rho_rime_min); - rho_rime.set(bi_rim_gt_small && rho_rime_gt_max, rho_rime_max); + rho_rime.set(bi_rim_gt_small && rho_rime_lt_min, p3_rho_rime_min); + rho_rime.set(bi_rim_gt_small && rho_rime_gt_max, p3_rho_rime_max); Smask adjust = bi_rim_gt_small && (rho_rime_gt_max || rho_rime_lt_min); if (adjust.any()) { bi_rim.set(adjust, qi_rim / rho_rime); @@ -85,7 +86,8 @@ ::ice_sedimentation( const uview_1d& qi_tend, const uview_1d& ni_tend, const view_ice_table& ice_table_vals, - Scalar& precip_ice_surf) + Scalar& precip_ice_surf, + const physics::P3_Constants & p3constants) { // Get temporary workspaces needed for the ice-sed calculation uview_1d V_qit, V_nit, flux_nit, flux_bir, flux_qir, flux_qit; @@ -102,6 +104,9 @@ ::ice_sedimentation( const auto sqi = scalarize(qi); constexpr Scalar qsmall = C::QSMALL; constexpr Scalar nsmall = C::NSMALL; + + const Scalar p3_ice_sed_knob = p3constants.p3_ice_sed_knob; + bool log_qxpresent; const Int k_qxtop = find_top(team, sqi, qsmall, kbot, ktop, kdir, log_qxpresent); @@ -140,7 +145,7 @@ ::ice_sedimentation( // impose lower limits to prevent log(<0) ni_incld(pk).set(qi_gt_small, max(ni_incld(pk), nsmall)); - const auto rhop = calc_bulk_rho_rime(qi_incld(pk), qm_incld(pk), bm_incld(pk), qi_gt_small); + const auto rhop = calc_bulk_rho_rime(qi_incld(pk), qm_incld(pk), bm_incld(pk), p3constants, qi_gt_small); qm(pk).set(qi_gt_small, qm_incld(pk)*cld_frac_i(pk) ); bm(pk).set(qi_gt_small, bm_incld(pk)*cld_frac_i(pk) ); @@ -158,8 +163,8 @@ ::ice_sedimentation( ni_incld(pk).set(qi_gt_small, max(ni_incld(pk), table_val_ni_lammin * ni_incld(pk))); ni(pk).set(qi_gt_small, ni_incld(pk) * cld_frac_i(pk)); - V_qit(pk).set(qi_gt_small, table_val_qi_fallspd * rhofaci(pk)); // mass-weighted fall speed (with density factor) - V_nit(pk).set(qi_gt_small, table_val_ni_fallspd * rhofaci(pk)); // number-weighted fall speed (with density factor) + V_qit(pk).set(qi_gt_small, p3_ice_sed_knob * table_val_qi_fallspd * rhofaci(pk)); // mass-weighted fall speed (with density factor) + V_nit(pk).set(qi_gt_small, p3_ice_sed_knob * table_val_ni_fallspd * rhofaci(pk)); // number-weighted fall speed (with density factor) } const auto Co_max_local = max(qi_gt_small, 0, V_qit(pk) * dt_left * inv_dz(pk)); diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp index 22819e64aad1..b05b713aec29 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp @@ -83,7 +83,8 @@ ::p3_main_internal( const P3LookupTables& lookup_tables, const WorkspaceManager& workspace_mgr, Int nj, - Int nk) + Int nk, + const physics::P3_Constants & p3constants) { using ExeSpace = typename KT::ExeSpace; @@ -233,7 +234,7 @@ ::p3_main_internal( T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, oqv, oth, oqc, onc, oqr, onr, oqi, oni, oqm, obm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, - ni_incld, bm_incld, nucleationPossible, hydrometeorsPresent); + ni_incld, bm_incld, nucleationPossible, hydrometeorsPresent, p3constants); // There might not be any work to do for this team if (!(nucleationPossible || hydrometeorsPresent)) { @@ -253,7 +254,7 @@ ::p3_main_internal( nr_incld, ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, mu_r, lamr, logn0r, oqv2qi_depos_tend, precip_total_tend, nevapr, qr_evap_tend, ovap_liq_exchange, ovap_ice_exchange, oliq_ice_exchange, - pratot, prctot, hydrometeorsPresent, nk); + pratot, prctot, hydrometeorsPresent, nk, p3constants); //NOTE: At this point, it is possible to have negative (but small) nc, nr, ni. This is not // a problem; those values get clipped to zero in the sedimentation section (if necessary). @@ -281,14 +282,14 @@ ::p3_main_internal( rho, inv_rho, rhofacr, ocld_frac_r, inv_dz, qr_incld, team, workspace, lookup_tables.vn_table_vals, lookup_tables.vm_table_vals, nk, ktop, kbot, kdir, infrastructure.dt, inv_dt, oqr, onr, nr_incld, mu_r, lamr, oprecip_liq_flux, qtend_ignore, ntend_ignore, - diagnostic_outputs.precip_liq_surf(i)); + diagnostic_outputs.precip_liq_surf(i), p3constants); // Ice sedimentation: (adaptive substepping) ice_sedimentation( rho, inv_rho, rhofaci, ocld_frac_i, inv_dz, team, workspace, nk, ktop, kbot, kdir, infrastructure.dt, inv_dt, oqi, qi_incld, oni, ni_incld, oqm, qm_incld, obm, bm_incld, qtend_ignore, ntend_ignore, - lookup_tables.ice_table_vals, diagnostic_outputs.precip_ice_surf(i)); + lookup_tables.ice_table_vals, diagnostic_outputs.precip_ice_surf(i), p3constants); // homogeneous freezing of cloud and rain homogeneous_freezing( @@ -304,7 +305,7 @@ ::p3_main_internal( rho, inv_rho, rhofaci, oqv, oth, oqc, onc, oqr, onr, oqi, oni, oqm, obm, olatent_heat_vapor, olatent_heat_sublim, mu_c, nu, lamc, mu_r, lamr, ovap_liq_exchange, ze_rain, ze_ice, diag_vm_qi, odiag_eff_radius_qi, diag_diam_qi, - orho_qi, diag_equiv_reflectivity, odiag_eff_radius_qc, odiag_eff_radius_qr); + orho_qi, diag_equiv_reflectivity, odiag_eff_radius_qc, odiag_eff_radius_qr, p3constants); // // merge ice categories with similar properties @@ -343,7 +344,8 @@ ::p3_main( const P3LookupTables& lookup_tables, const WorkspaceManager& workspace_mgr, Int nj, - Int nk) + Int nk, + const physics::P3_Constants & p3constants) { #ifndef SCREAM_SMALL_KERNELS return p3_main_internal(runtime_options, @@ -354,7 +356,7 @@ ::p3_main( history_only, lookup_tables, workspace_mgr, - nj, nk); + nj, nk, p3constants); #else return p3_main_internal_disp(runtime_options, prognostic_state, @@ -364,7 +366,7 @@ ::p3_main( history_only, lookup_tables, workspace_mgr, - nj, nk); + nj, nk, p3constants); #endif } } // namespace p3 diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp index 28be9ab9040b..439b0544adf9 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp @@ -65,7 +65,8 @@ ::p3_main_part1( const uview_1d& ni_incld, const uview_1d& bm_incld, bool& nucleationPossible, - bool& hydrometeorsPresent) + bool& hydrometeorsPresent, + const physics::P3_Constants & p3constants) { // Get access to saturation functions using physics = scream::physics::Functions; @@ -80,6 +81,8 @@ ::p3_main_part1( constexpr Scalar qsmall = C::QSMALL; constexpr Scalar inv_cp = C::INV_CP; + const Scalar p3_spa_to_nc = p3constants.p3_spa_to_nc; + nucleationPossible = false; hydrometeorsPresent = false; team.team_barrier(); @@ -131,7 +134,7 @@ ::p3_main_part1( // prescribe that value if (do_prescribed_CCN) { - nc(k).set(not_drymass, max(nc(k), nccn_prescribed(k)/inv_cld_frac_l(k))); + nc(k).set(not_drymass, max(nc(k), p3_spa_to_nc*nccn_prescribed(k)/inv_cld_frac_l(k))); } else if (predictNc) { nc(k).set(not_drymass, max(nc(k) + nc_nuceat_tend(k) * dt, 0.0)); } else { diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp index 29fc2fce11d6..c32fb7202ebb 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp @@ -94,7 +94,8 @@ ::p3_main_part2( const uview_1d& liq_ice_exchange, const uview_1d& pratot, const uview_1d& prctot, - bool& hydrometeorsPresent, const Int& nk) + bool& hydrometeorsPresent, const Int& nk, + const physics::P3_Constants & p3constants) { constexpr Scalar qsmall = C::QSMALL; constexpr Scalar nsmall = C::NSMALL; @@ -215,7 +216,8 @@ ::p3_main_part2( lamc(k), cdist(k), cdist1(k), not_skip_micro); nc(k).set(not_skip_micro, nc_incld(k) * cld_frac_l(k)); - get_rain_dsd2(qr_incld(k), nr_incld(k), mu_r(k), lamr(k), cdistr(k), logn0r(k), not_skip_micro); + get_rain_dsd2(qr_incld(k), nr_incld(k), mu_r(k), lamr(k), p3constants, not_skip_micro); + get_cdistr_logn0r(qr_incld(k), nr_incld(k), mu_r(k), lamr(k), cdistr(k), logn0r(k), not_skip_micro); nr(k).set(not_skip_micro, nr_incld(k) * cld_frac_r(k)); impose_max_total_ni(ni_incld(k), max_total_ni, inv_rho(k), not_skip_micro); @@ -227,7 +229,7 @@ ::p3_main_part2( ni_incld(k).set(qi_gt_small, max(ni_incld(k), nsmall)); nr_incld(k).set(qi_gt_small, max(nr_incld(k), nsmall)); - const auto rhop = calc_bulk_rho_rime(qi_incld(k), qm_incld(k), bm_incld(k), qi_gt_small); + const auto rhop = calc_bulk_rho_rime(qi_incld(k), qm_incld(k), bm_incld(k), p3constants, qi_gt_small); qm(k).set(qi_gt_small, qm_incld(k)*cld_frac_i(k) ); bm(k).set(qi_gt_small, bm_incld(k)*cld_frac_i(k) ); @@ -268,12 +270,12 @@ ::p3_main_part2( // collection of droplets ice_cldliq_collection( rho(k), T_atm(k), rhofaci(k), table_val_qc2qi_collect, qi_incld(k), qc_incld(k), ni_incld(k), nc_incld(k), - qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc, not_skip_micro); + qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc, p3constants, not_skip_micro); // collection of rain ice_rain_collection( rho(k), T_atm(k), rhofaci(k), logn0r(k), table_val_nr_collect, table_val_qr2qi_collect, qi_incld(k), ni_incld(k), qr_incld(k), - qr2qi_collect_tend, nr_collect_tend, not_skip_micro); + qr2qi_collect_tend, nr_collect_tend, p3constants, not_skip_micro); // collection between ice categories @@ -309,12 +311,12 @@ ::p3_main_part2( // contact and immersion freezing droplets cldliq_immersion_freezing( T_atm(k), lamc(k), mu_c(k), cdist1(k), qc_incld(k), inv_qc_relvar(k), - qc2qi_hetero_freeze_tend, nc2ni_immers_freeze_tend, not_skip_micro); + qc2qi_hetero_freeze_tend, nc2ni_immers_freeze_tend, p3constants, not_skip_micro); // for future: get rid of log statements below for rain freezing rain_immersion_freezing( T_atm(k), lamr(k), mu_r(k), cdistr(k), qr_incld(k), - qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend, not_skip_micro); + qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend, p3constants, not_skip_micro); // rime splintering (Hallet-Mossop 1974) // PMC comment: Morrison and Milbrandt 2015 part 1 and 2016 part 3 both say @@ -343,13 +345,13 @@ ::p3_main_part2( // deposition/condensation-freezing nucleation ice_nucleation( T_atm(k), inv_rho(k), ni(k), ni_activated(k), qv_supersat_i(k), inv_dt, predictNc, do_prescribed_CCN, - qv2qi_nucleat_tend, ni_nucleat_tend, not_skip_all); + qv2qi_nucleat_tend, ni_nucleat_tend, p3constants, not_skip_all); // cloud water autoconversion // NOTE: cloud_water_autoconversion must be called before droplet_self_collection cloud_water_autoconversion( rho(k), qc_incld(k), nc_incld(k), inv_qc_relvar(k), - qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr, not_skip_all); + qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr, p3constants, not_skip_all); // self-collection of droplets droplet_self_collection( @@ -359,13 +361,13 @@ ::p3_main_part2( // accretion of cloud by rain cloud_rain_accretion( rho(k), inv_rho(k), qc_incld(k), nc_incld(k), qr_incld(k), inv_qc_relvar(k), - qc2qr_accret_tend, nc_accret_tend, not_skip_all); + qc2qr_accret_tend, nc_accret_tend, p3constants, not_skip_all); // self-collection and breakup of rain // (breakup following modified Verlinde and Cotton scheme) rain_self_collection( rho(k), qr_incld(k), nr_incld(k), - nr_selfcollect_tend, not_skip_all); + nr_selfcollect_tend, p3constants, not_skip_all); // Here we map the microphysics tendency rates back to CELL-AVERAGE quantities for updating // cell-average quantities. diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp index 6791248abca4..441bcb9feb9b 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp @@ -57,7 +57,8 @@ ::p3_main_part3( const uview_1d& rho_qi, const uview_1d& diag_equiv_reflectivity, const uview_1d& diag_eff_radius_qc, - const uview_1d& diag_eff_radius_qr) + const uview_1d& diag_eff_radius_qr, + const physics::P3_Constants & p3constants) { constexpr Scalar qsmall = C::QSMALL; constexpr Scalar inv_cp = C::INV_CP; @@ -107,7 +108,7 @@ ::p3_main_part3( auto nr_incld = nr(k)/cld_frac_r(k); //nr_incld is updated in get_rain_dsd2 but isn't used again get_rain_dsd2( - qr_incld, nr_incld, mu_r(k), lamr(k), ignore1, ignore2, qr_gt_small); + qr_incld, nr_incld, mu_r(k), lamr(k), p3constants, qr_gt_small); //Note that integrating over the drop-size PDF as done here should only be done to in-cloud //quantities but radar reflectivity is likely meant to be a cell ave. Thus nr in the next line @@ -142,7 +143,7 @@ ::p3_main_part3( auto qm_incld = qm(k)/cld_frac_i(k); auto bm_incld = bm(k)/cld_frac_i(k); - const auto rhop = calc_bulk_rho_rime(qi_incld, qm_incld, bm_incld, qi_gt_small); + const auto rhop = calc_bulk_rho_rime(qi_incld, qm_incld, bm_incld, p3constants, qi_gt_small); qm(k).set(qi_gt_small, qm_incld*cld_frac_i(k) ); bm(k).set(qi_gt_small, bm_incld*cld_frac_i(k) ); diff --git a/components/eamxx/src/physics/p3/impl/p3_rain_imm_freezing_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_rain_imm_freezing_impl.hpp index acee2805bf77..c960698e4b18 100644 --- a/components/eamxx/src/physics/p3/impl/p3_rain_imm_freezing_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_rain_imm_freezing_impl.hpp @@ -17,26 +17,28 @@ void Functions ::rain_immersion_freezing(const Spack& T_atm, const Spack& lamr, const Spack& mu_r, const Spack& cdistr, const Spack& qr_incld, Spack& qr2qi_immers_freeze_tend, Spack& nr2ni_immers_freeze_tend, - const Smask& context) + const physics::P3_Constants & p3constants, + const Smask& context) { constexpr Scalar qsmall = C::QSMALL; constexpr Scalar T_rainfrz = C::T_rainfrz; constexpr Scalar T_zerodegc = C::T_zerodegc; - constexpr Scalar AIMM = C::AIMM; constexpr Scalar CONS5 = C::CONS5; constexpr Scalar CONS6 = C::CONS6; + const Scalar p3_a_imm = p3constants.p3_a_imm; + const auto qr_not_small_and_t_freezing = (qr_incld >= qsmall) && (T_atm <= T_rainfrz) && context; if (qr_not_small_and_t_freezing.any()) { qr2qi_immers_freeze_tend.set(qr_not_small_and_t_freezing, CONS6 * exp(log(cdistr) + log(tgamma(sp(7.)+mu_r)) - sp(6.)*log(lamr)) * - exp(AIMM*(T_zerodegc-T_atm))); + exp(p3_a_imm*(T_zerodegc-T_atm))); nr2ni_immers_freeze_tend.set(qr_not_small_and_t_freezing, CONS5 * exp(log(cdistr) + log(tgamma(sp(4.)+mu_r)) - sp(3.)*log(lamr)) * - exp(AIMM*(T_zerodegc-T_atm))); + exp(p3_a_imm*(T_zerodegc-T_atm))); } } diff --git a/components/eamxx/src/physics/p3/impl/p3_rain_sed_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_rain_sed_impl.hpp index d54db725caee..558264305f17 100644 --- a/components/eamxx/src/physics/p3/impl/p3_rain_sed_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_rain_sed_impl.hpp @@ -18,11 +18,11 @@ ::compute_rain_fall_velocity( const view_2d_table& vn_table_vals, const view_2d_table& vm_table_vals, const Spack& qr_incld, const Spack& rhofacr, Spack& nr_incld, Spack& mu_r, Spack& lamr, Spack& V_qr, Spack& V_nr, + const physics::P3_Constants & p3constants, const Smask& context) { Table3 table; - Spack tmp1, tmp2; //ignore - get_rain_dsd2(qr_incld, nr_incld, mu_r, lamr, tmp1, tmp2, context); + get_rain_dsd2(qr_incld, nr_incld, mu_r, lamr, p3constants, context); if (context.any()) { lookup(mu_r, lamr, table, context); @@ -55,7 +55,8 @@ ::rain_sedimentation( const uview_1d& precip_liq_flux, const uview_1d& qr_tend, const uview_1d& nr_tend, - Scalar& precip_liq_surf) + Scalar& precip_liq_surf, + const physics::P3_Constants & p3constants) { // Get temporary workspaces needed for the ice-sed calculation uview_1d V_qr, V_nr, flux_qx, flux_nx; @@ -111,7 +112,7 @@ ::rain_sedimentation( compute_rain_fall_velocity(vn_table_vals, vm_table_vals, qr_incld(pk), rhofacr(pk), nr_incld(pk), mu_r(pk), lamr(pk), - V_qr(pk), V_nr(pk), qr_gt_small); + V_qr(pk), V_nr(pk), p3constants, qr_gt_small); //in compute_rain_fall_velocity, get_rain_dsd2 keeps the drop-size //distribution within reasonable bounds by modifying nr_incld. diff --git a/components/eamxx/src/physics/p3/impl/p3_rain_self_collection_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_rain_self_collection_impl.hpp index 2df0c5790311..807e928948cb 100644 --- a/components/eamxx/src/physics/p3/impl/p3_rain_self_collection_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_rain_self_collection_impl.hpp @@ -11,6 +11,7 @@ KOKKOS_FUNCTION void Functions ::rain_self_collection( const Spack& rho, const Spack& qr_incld, const Spack& nr_incld, Spack& nr_selfcollect_tend, + const physics::P3_Constants & p3constants, const Smask& context) { // ------------------------------------------------------ @@ -21,18 +22,20 @@ ::rain_self_collection( constexpr Scalar rho_h2o = C::RHO_H2O; constexpr Scalar pi = C::Pi; + const Scalar p3_d_breakup_cutoff = p3constants.p3_d_breakup_cutoff; + const auto qr_incld_not_small = qr_incld >= qsmall && context; if (qr_incld_not_small.any()) { - const Real dum1 = 280.e-6; + const auto dum2 = cbrt((qr_incld)/(pi*rho_h2o*nr_incld)); Spack dum; - const auto dum2_lt_dum1 = dum2 < dum1 && qr_incld_not_small; - const auto dum2_gt_dum1 = dum2 >= dum1 && qr_incld_not_small; + const auto dum2_lt_dum1 = dum2 < p3_d_breakup_cutoff && qr_incld_not_small; + const auto dum2_gt_dum1 = dum2 >= p3_d_breakup_cutoff && qr_incld_not_small; dum.set(dum2_lt_dum1, 1); if (dum2_gt_dum1.any()) { - dum.set(dum2_gt_dum1, 2 - exp(2300 * (dum2-dum1))); + dum.set(dum2_gt_dum1, 2 - exp(2300 * (dum2-p3_d_breakup_cutoff))); } nr_selfcollect_tend.set(qr_incld_not_small, dum*sp(5.78)*nr_incld*qr_incld*rho); diff --git a/components/eamxx/src/physics/p3/p3_functions.hpp b/components/eamxx/src/physics/p3/p3_functions.hpp index 6a8fbc343b04..1749a4ae7346 100644 --- a/components/eamxx/src/physics/p3/p3_functions.hpp +++ b/components/eamxx/src/physics/p3/p3_functions.hpp @@ -72,6 +72,7 @@ struct Functions using KT = KokkosTypes; using C = scream::physics::Constants; + using CP3 = scream::physics::P3_Constants; template using view_1d = typename KT::template view_1d; @@ -460,7 +461,8 @@ struct Functions const uview_1d& precip_liq_flux, const uview_1d& qr_tend, const uview_1d& nr_tend, - Scalar& precip_liq_surf); + Scalar& precip_liq_surf, + const physics::P3_Constants & p3constants); #ifdef SCREAM_SMALL_KERNELS static void rain_sedimentation_disp( @@ -483,7 +485,8 @@ struct Functions const uview_2d& nr_tend, const uview_1d& precip_liq_surf, const uview_1d& is_nucleat_possible, - const uview_1d& is_hydromet_present); + const uview_1d& is_hydromet_present, + const physics::P3_Constants & p3constants); #endif // TODO: comment @@ -508,7 +511,8 @@ struct Functions const uview_1d& qi_tend, const uview_1d& ni_tend, const view_ice_table& ice_table_vals, - Scalar& precip_ice_surf); + Scalar& precip_ice_surf, + const physics::P3_Constants & p3constants); #ifdef SCREAM_SMALL_KERNELS static void ice_sedimentation_disp( @@ -532,7 +536,8 @@ struct Functions const view_ice_table& ice_table_vals, const uview_1d& precip_ice_surf, const uview_1d& is_nucleat_possible, - const uview_1d& is_hydromet_present); + const uview_1d& is_hydromet_present, + const physics::P3_Constants & p3constants); #endif // homogeneous freezing of cloud and rain @@ -620,7 +625,15 @@ struct Functions KOKKOS_FUNCTION static void get_rain_dsd2 ( const Spack& qr, Spack& nr, Spack& mu_r, - Spack& lamr, Spack& cdistr, Spack& logn0r, + Spack& lamr, + const physics::P3_Constants & p3constants, + const Smask& context = Smask(true) ); + + // Computes and returns additional rain size distribution parameters + KOKKOS_FUNCTION + static void get_cdistr_logn0r ( + const Spack& qr, const Spack& nr, const Spack& mu_r, + const Spack& lamr, Spack& cdistr, Spack& logn0r, const Smask& context = Smask(true) ); // Calculates rime density @@ -636,6 +649,7 @@ struct Functions static void cldliq_immersion_freezing(const Spack& T_atm, const Spack& lamc, const Spack& mu_c, const Spack& cdist1, const Spack& qc_incld, const Spack& inv_qc_relvar, Spack& qc2qi_hetero_freeze_tend, Spack& nc2ni_immers_freeze_tend, + const physics::P3_Constants & p3constants, const Smask& context = Smask(true) ); // Computes the immersion freezing of rain @@ -643,6 +657,7 @@ struct Functions static void rain_immersion_freezing(const Spack& T_atm, const Spack& lamr, const Spack& mu_r, const Spack& cdistr, const Spack& qr_incld, Spack& qr2qi_immers_freeze_tend, Spack& nr2ni_immers_freeze_tend, + const physics::P3_Constants & p3constants, const Smask& context = Smask(true) ); // Computes droplet self collection @@ -657,6 +672,7 @@ struct Functions static void cloud_rain_accretion(const Spack& rho, const Spack& inv_rho, const Spack& qc_incld, const Spack& nc_incld, const Spack& qr_incld, const Spack& inv_qc_relvar, Spack& qc2qr_accret_tend, Spack& nc_accret_tend, + const physics::P3_Constants & p3constants, const Smask& context = Smask(true) ); // Computes cloud water autoconversion process rate @@ -664,11 +680,13 @@ struct Functions static void cloud_water_autoconversion(const Spack& rho, const Spack& qc_incld, const Spack& nc_incld, const Spack& inv_qc_relvar, Spack& qc2qr_autoconv_tend, Spack& nc2nr_autoconv_tend, Spack& ncautr, - const Smask& context = Smask(true) ); + const physics::P3_Constants & p3constants, + const Smask& context = Smask(true)); // Computes rain self collection process rate KOKKOS_FUNCTION static void rain_self_collection(const Spack& rho, const Spack& qr_incld, const Spack& nr_incld, Spack& nr_selfcollect_tend, + const physics::P3_Constants & p3constants, const Smask& context = Smask(true) ); // Impose maximum ice number @@ -683,6 +701,7 @@ struct Functions KOKKOS_FUNCTION static Spack calc_bulk_rho_rime( const Spack& qi_tot, Spack& qi_rim, Spack& bi_rim, + const physics::P3_Constants & p3constants, const Smask& context = Smask(true) ); // TODO - comment @@ -691,6 +710,7 @@ struct Functions const view_2d_table& vn_table_vals, const view_2d_table& vm_table_vals, const Spack& qr_incld, const Spack& rhofacr, Spack& nr_incld, Spack& mu_r, Spack& lamr, Spack& V_qr, Spack& V_nr, + const physics::P3_Constants & p3constants, const Smask& context = Smask(true)); //--------------------------------------------------------------------------------- @@ -726,7 +746,8 @@ struct Functions const Spack& qi_incld, const Spack& qc_incld, const Spack& ni_incld, const Spack& nc_incld, Spack& qc2qi_collect_tend, Spack& nc_collect_tend, Spack& qc2qr_ice_shed_tend, Spack& ncshdc, - const Smask& context = Smask(true)); + const physics::P3_Constants & p3constants, + const Smask& context = Smask(true)); // TODO (comments) KOKKOS_FUNCTION @@ -736,6 +757,7 @@ struct Functions const Spack& qi_incld, const Spack& ni_incld, const Spack& qr_incld, Spack& qr2qi_collect_tend, Spack& nr_collect_tend, + const physics::P3_Constants & p3constants, const Smask& context = Smask(true)); // TODO (comments) @@ -820,6 +842,7 @@ struct Functions const Spack& qv_supersat_i, const Scalar& inv_dt, const bool& do_predict_nc, const bool& do_prescribed_CCN, Spack& qv2qi_nucleat_tend, Spack& ni_nucleat_tend, + const physics::P3_Constants & p3constants, const Smask& context = Smask(true)); KOKKOS_FUNCTION @@ -958,7 +981,8 @@ struct Functions const uview_1d& ni_incld, const uview_1d& bm_incld, bool& is_nucleat_possible, - bool& is_hydromet_present); + bool& is_hydromet_present, + const physics::P3_Constants & p3constants); #ifdef SCREAM_SMALL_KERNELS static void p3_main_part1_disp( @@ -1008,7 +1032,8 @@ struct Functions const uview_2d& ni_incld, const uview_2d& bm_incld, const uview_1d& is_nucleat_possible, - const uview_1d& is_hydromet_present); + const uview_1d& is_hydromet_present, + const physics::P3_Constants & p3constants); #endif KOKKOS_FUNCTION @@ -1089,7 +1114,8 @@ struct Functions const uview_1d& pratot, const uview_1d& prctot, bool& is_hydromet_present, - const Int& nk); + const Int& nk, + const physics::P3_Constants & p3constants); #ifdef SCREAM_SMALL_KERNELS static void p3_main_part2_disp( @@ -1169,7 +1195,8 @@ struct Functions const uview_2d& pratot, const uview_2d& prctot, const uview_1d& is_nucleat_possible, - const uview_1d& is_hydromet_present); + const uview_1d& is_hydromet_present, + const physics::P3_Constants & p3constants); #endif KOKKOS_FUNCTION @@ -1212,7 +1239,8 @@ struct Functions const uview_1d& rho_qi, const uview_1d& diag_equiv_reflectivity, const uview_1d& diag_eff_radius_qc, - const uview_1d& diag_eff_radius_qr); + const uview_1d& diag_eff_radius_qr, + const physics::P3_Constants & p3constants); #ifdef SCREAM_SMALL_KERNELS static void p3_main_part3_disp( @@ -1256,7 +1284,8 @@ struct Functions const uview_2d& diag_eff_radius_qc, const uview_2d& diag_eff_radius_qr, const uview_1d& is_nucleat_possible, - const uview_1d& is_hydromet_present); + const uview_1d& is_hydromet_present, + const physics::P3_Constants & p3constants); #endif // Return microseconds elapsed @@ -1270,7 +1299,8 @@ struct Functions const P3LookupTables& lookup_tables, const WorkspaceManager& workspace_mgr, Int nj, // number of columns - Int nk); // number of vertical cells per column + Int nk, // number of vertical cells per column + const physics::P3_Constants & p3constants); static Int p3_main_internal( const P3Runtime& runtime_options, @@ -1282,7 +1312,8 @@ struct Functions const P3LookupTables& lookup_tables, const WorkspaceManager& workspace_mgr, Int nj, // number of columns - Int nk); // number of vertical cells per column + Int nk, // number of vertical cells per column + const physics::P3_Constants & p3constants); #ifdef SCREAM_SMALL_KERNELS static Int p3_main_internal_disp( @@ -1295,7 +1326,8 @@ struct Functions const P3LookupTables& lookup_tables, const WorkspaceManager& workspace_mgr, Int nj, // number of columns - Int nk); // number of vertical cells per column + Int nk, // number of vertical cells per column + const physics::P3_Constants & p3constants); #endif KOKKOS_FUNCTION diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.cpp b/components/eamxx/src/physics/p3/p3_functions_f90.cpp index 1f6a9d26d871..83aeb5b16d3d 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.cpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.cpp @@ -1329,7 +1329,7 @@ void ice_sedimentation_f( nk, ktop, kbot, kdir, dt, inv_dt, qi_d, qi_incld_d, ni_d, ni_incld_d, qm_d, qm_incld_d, bm_d, bm_incld_d, qi_tend_d, ni_tend_d, ice_table_vals, - precip_ice_surf_k); + precip_ice_surf_k, physics::P3_Constants()); }, my_precip_ice_surf); *precip_ice_surf += my_precip_ice_surf; @@ -1402,7 +1402,7 @@ void rain_sedimentation_f( team, wsm.get_workspace(team), vn_table_vals, vm_table_vals, nk, ktop, kbot, kdir, dt, inv_dt, qr_d, nr_d, nr_incld_d, mu_r_d, lamr_d, precip_liq_flux_d, qr_tend_d, nr_tend_d, - precip_liq_surf_k); + precip_liq_surf_k, physics::P3_Constants()); }, my_precip_liq_surf); *precip_liq_surf += my_precip_liq_surf; @@ -1634,7 +1634,7 @@ void p3_main_part1_f( t_d, rho_d, inv_rho_d, qv_sat_l_d, qv_sat_i_d, qv_supersat_i_d, rhofacr_d, rhofaci_d, acn_d, qv_d, th_atm_d, qc_d, nc_d, qr_d, nr_d, qi_d, ni_d, qm_d, bm_d, qc_incld_d, qr_incld_d, qi_incld_d, qm_incld_d, nc_incld_d, nr_incld_d, ni_incld_d, bm_incld_d, - bools_d(0), bools_d(1)); + bools_d(0), bools_d(1), physics::P3_Constants()); }); // Sync back to host @@ -1784,7 +1784,7 @@ void p3_main_part2_f( qm_incld_d, nc_incld_d, nr_incld_d, ni_incld_d, bm_incld_d, mu_c_d, nu_d, lamc_d, cdist_d, cdist1_d, cdistr_d, mu_r_d, lamr_d, logn0r_d, qv2qi_depos_tend_d, precip_total_tend_d, nevapr_d, qr_evap_tend_d, vap_liq_exchange_d, - vap_ice_exchange_d, liq_ice_exchange_d, pratot_d, prctot_d, bools_d(0),nk); + vap_ice_exchange_d, liq_ice_exchange_d, pratot_d, prctot_d, bools_d(0),nk, physics::P3_Constants()); }); // Sync back to host. Skip intent in variables. @@ -1902,7 +1902,7 @@ void p3_main_part3_f( latent_heat_sublim_d, mu_c_d, nu_d, lamc_d, mu_r_d, lamr_d, vap_liq_exchange_d, ze_rain_d, ze_ice_d, diag_vm_qi_d, diag_eff_radius_qi_d, diag_diam_qi_d, rho_qi_d, - diag_equiv_reflectivity_d, diag_eff_radius_qc_d, diag_eff_radius_qr_d); + diag_equiv_reflectivity_d, diag_eff_radius_qc_d, diag_eff_radius_qr_d, physics::P3_Constants()); }); // Sync back to host @@ -2073,7 +2073,7 @@ Int p3_main_f( ekat::WorkspaceManager workspace_mgr(nk_pack, 52, policy); auto elapsed_microsec = P3F::p3_main(runtime_options, prog_state, diag_inputs, diag_outputs, infrastructure, - history_only, lookup_tables, workspace_mgr, nj, nk); + history_only, lookup_tables, workspace_mgr, nj, nk, physics::P3_Constants()); Kokkos::parallel_for(nj, KOKKOS_LAMBDA(const Int& i) { precip_liq_surf_temp_d(0, i / Spack::n)[i % Spack::n] = precip_liq_surf_d(i); diff --git a/components/eamxx/src/physics/p3/tests/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/CMakeLists.txt index fd864e427815..4e8901069e05 100644 --- a/components/eamxx/src/physics/p3/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/tests/CMakeLists.txt @@ -40,13 +40,13 @@ set(P3_TESTS_SRCS ) # P3_TESTS_SRCS if (SCREAM_DEBUG AND NOT SCREAM_TEST_SIZE STREQUAL "SHORT") - set (FORCE_RUN_DIFF_FAILS TRUE) + set (FORCE_RUN_DIFF_FAILS "WILL_FAIL") else () - set (FORCE_RUN_DIFF_FAILS FALSE) + set (FORCE_RUN_DIFF_FAILS "") endif() # NOTE: tests inside this if statement won't be built in a baselines-only build -if (NOT SCREAM_BASELINES_ONLY) +if (NOT SCREAM_ONLY_GENERATE_BASELINES) CreateUnitTest(p3_tests "${P3_TESTS_SRCS}" LIBS p3 THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} @@ -58,9 +58,9 @@ if (NOT SCREAM_BASELINES_ONLY) COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} LABELS "p3;physics;fail" - PROPERTIES WILL_FAIL ${FORCE_RUN_DIFF_FAILS}) + ${FORCE_RUN_DIFF_FAILS}) - if (NOT SCREAM_SMALL_KERNELS) + if (NOT SCREAM_SMALL_KERNELS) CreateUnitTest(p3_sk_tests "${P3_TESTS_SRCS}" LIBS p3_sk THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} @@ -72,12 +72,16 @@ if (NOT SCREAM_BASELINES_ONLY) COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} LABELS "p3_sk;physics;fail" - PROPERTIES WILL_FAIL ${FORCE_RUN_DIFF_FAILS}) + ${FORCE_RUN_DIFF_FAILS}) endif() endif() if (SCREAM_ENABLE_BASELINE_TESTS) - set(BASELINE_FILE_ARG "-b ${SCREAM_TEST_DATA_DIR}/p3_run_and_cmp.baseline") + if (SCREAM_ONLY_GENERATE_BASELINES) + set(BASELINE_FILE_ARG "-g -b ${SCREAM_BASELINES_DIR}/data/p3_run_and_cmp.baseline") + else() + set(BASELINE_FILE_ARG "-b ${SCREAM_BASELINES_DIR}/data/p3_run_and_cmp.baseline") + endif() CreateUnitTestExec(p3_run_and_cmp "p3_run_and_cmp.cpp" LIBS p3 @@ -101,44 +105,15 @@ if (SCREAM_ENABLE_BASELINE_TESTS) EXE_ARGS "${BASELINE_FILE_ARG}" LABELS "p3;physics;fail" EXCLUDE_MAIN_CPP - PROPERTIES WILL_FAIL ${FORCE_RUN_DIFF_FAILS}) - - # - # Use fake tests to generate shell commands to generate baselines - # - CreateUnitTestFromExec(p3_baseline_f90_fake p3_run_and_cmp - THREADS ${SCREAM_TEST_MAX_THREADS} - EXE_ARGS "-f -g ${BASELINE_FILE_ARG}" - PROPERTIES DISABLED True) - - CreateUnitTestFromExec(p3_baseline_cxx_fake p3_run_and_cmp - THREADS ${SCREAM_TEST_MAX_THREADS} - EXE_ARGS "-g ${BASELINE_FILE_ARG}" - PROPERTIES DISABLED True) + ${FORCE_RUN_DIFF_FAILS}) + # By default, baselines should be created using all fortran (ctest -L baseline_gen). If the user wants + # to use CXX to generate their baselines, they should use "ctest -L baseline_gen_cxx". + # Note: the baseline_gen label label is really only used if SCREAM_ONLY_GENERATE_BASELINES=ON, but no harm adding it if (SCREAM_TEST_MAX_THREADS GREATER 1) - get_test_property(p3_baseline_f90_fake_omp${SCREAM_TEST_MAX_THREADS} FULL_TEST_COMMAND P3_F90_GEN) - get_test_property(p3_baseline_cxx_fake_omp${SCREAM_TEST_MAX_THREADS} FULL_TEST_COMMAND P3_CXX_GEN) - else() - get_test_property(p3_baseline_f90_fake FULL_TEST_COMMAND P3_F90_GEN) - get_test_property(p3_baseline_cxx_fake FULL_TEST_COMMAND P3_CXX_GEN) + # ECUT only adds _ompX if we have more than one value of X, or if X>1 + set (TEST_SUFFIX _omp${SCREAM_TEST_MAX_THREADS}) endif() - - if (P3_F90_GEN STREQUAL "NOTFOUND") - message(FATAL_ERROR "Could not get FULL_TEST_COMMAND for p3_baseline fake test") - endif() - - separate_arguments(P3_F90_GEN_ARGS UNIX_COMMAND "${P3_F90_GEN}") - separate_arguments(P3_CXX_GEN_ARGS UNIX_COMMAND "${P3_CXX_GEN}") - - add_custom_target(p3_baseline_f90 - COMMAND ${CMAKE_COMMAND} -E env OMP_NUM_THREADS=${SCREAM_TEST_MAX_THREADS} ${P3_F90_GEN_ARGS}) - - add_custom_target(p3_baseline_cxx - COMMAND ${CMAKE_COMMAND} -E env OMP_NUM_THREADS=${SCREAM_TEST_MAX_THREADS} ${P3_CXX_GEN_ARGS}) - - # By default, baselines should be created using all fortran (make baseline). If the user wants - # to use CXX to generate their baselines, they should use "make baseline_cxx". - add_dependencies(baseline p3_baseline_f90) - add_dependencies(baseline_cxx p3_baseline_cxx) + set_tests_properties (p3_run_and_cmp_f90${TEST_SUFFIX} PROPERTIES LABELS "baseline_gen;baseline_cmp") + set_tests_properties (p3_run_and_cmp_cxx${TEST_SUFFIX} PROPERTIES LABELS "baseline_gen;cxx baseline_cmp") endif() diff --git a/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp index d9be4ed6b33e..aa9606e653e5 100644 --- a/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp @@ -81,7 +81,7 @@ static void cloud_water_autoconversion_unit_bfb_tests(){ } Functions::cloud_water_autoconversion(rho, qc_incld, nc_incld, - inv_qc_relvar, qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr); + inv_qc_relvar, qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr, physics::P3_Constants()); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { @@ -123,7 +123,8 @@ static void cloud_water_autoconversion_unit_bfb_tests(){ for(int si=0; si()); if((qc2qr_autoconv_tend < 0.0).any()){errors++;} } diff --git a/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp index 4e2ccfbf5004..a9267449541c 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp @@ -94,7 +94,7 @@ static void run_bfb() Spack nc2ni_immers_freeze_tend{0.0}; Functions::cldliq_immersion_freezing(T_atm, lamc, mu_c, cdist1, qc_incld, inv_qc_relvar, - qc2qi_hetero_freeze_tend, nc2ni_immers_freeze_tend); + qc2qi_hetero_freeze_tend, nc2ni_immers_freeze_tend, physics::P3_Constants()); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { diff --git a/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp index b494e108f056..6b9ed9f63181 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp @@ -97,7 +97,7 @@ static void run_bfb() Spack nc_accret_tend{0.0}; Functions::cloud_rain_accretion(rho, inv_rho, qc_incld, nc_incld, qr_incld, - inv_qc_relvar, qc2qr_accret_tend, nc_accret_tend); + inv_qc_relvar, qc2qr_accret_tend, nc_accret_tend, physics::P3_Constants()); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { diff --git a/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp index 9adcc8078f24..0e00cf5acddb 100644 --- a/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp @@ -161,7 +161,8 @@ struct UnitWrap::UnitTest::TestDsd2 { } Spack mu_r(0.0), lamr(0.0), cdistr(0.0), logn0r(0.0); - Functions::get_rain_dsd2(qr, nr, mu_r, lamr, cdistr, logn0r); + Functions::get_rain_dsd2(qr, nr, mu_r, lamr, physics::P3_Constants()); + Functions::get_cdistr_logn0r(qr, nr, mu_r, lamr, cdistr, logn0r); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp index d9a0c6b1c636..bfad594acfb1 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp @@ -92,7 +92,7 @@ struct UnitWrap::UnitTest::TestIceCollection { Functions::ice_cldliq_collection(rho, temp, rhofaci, table_val_qc2qi_collect, qi_incld, qc_incld, ni_incld, nc_incld, - qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc); + qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc, physics::P3_Constants()); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { @@ -181,7 +181,7 @@ struct UnitWrap::UnitTest::TestIceCollection { Spack qr2qi_collect_tend(0.0), nr_collect_tend(0.0); Functions::ice_rain_collection(rho, temp, rhofaci, logn0r, table_val_nr_collect, table_val_qr2qi_collect, qi_incld, ni_incld, qr_incld, - qr2qi_collect_tend, nr_collect_tend); + qr2qi_collect_tend, nr_collect_tend, physics::P3_Constants()); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp index ea30145ba001..cb26a7ee6c06 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp @@ -82,7 +82,7 @@ struct UnitWrap::UnitTest::TestIceNucleation { Spack qv2qi_nucleat_tend{0.0}; Spack ni_nucleat_tend{0.0}; Functions::ice_nucleation(temp, inv_rho, ni, ni_activated, qv_supersat_i, self_device(0).inv_dt, do_predict_nc, - do_prescribed_CCN, qv2qi_nucleat_tend, ni_nucleat_tend); + do_prescribed_CCN, qv2qi_nucleat_tend, ni_nucleat_tend, physics::P3_Constants()); for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { self_device(vs).qv2qi_nucleat_tend = qv2qi_nucleat_tend[s]; diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp index f5369cee2d9c..02167261bf86 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp @@ -98,7 +98,7 @@ static void run_bfb_calc_bulk_rhime() } Smask gt_small(qi_tot > qsmall); - Spack rho_rime = Functions::calc_bulk_rho_rime(qi_tot, qi_rim, bi_rim, gt_small); + Spack rho_rime = Functions::calc_bulk_rho_rime(qi_tot, qi_rim, bi_rim, physics::P3_Constants(), gt_small); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp index 51bd82ed3bd1..b74f734e87c4 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp @@ -92,7 +92,7 @@ static void run_bfb() Spack nr2ni_immers_freeze_tend{0.0}; Functions::rain_immersion_freezing(T_atm, lamr, mu_r, cdistr, qr_incld, - qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend); + qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend, physics::P3_Constants()); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp index 8a1fa4969bdf..dfbf5f3a2f1f 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp @@ -96,7 +96,7 @@ static void run_bfb_rain_vel() Spack mu_r(0), lamr(0), V_qr(0), V_nr(0); Functions::compute_rain_fall_velocity( - vn_table_vals, vm_table_vals, qr_incld, rhofacr, nr_incld, mu_r, lamr, V_qr, V_nr); + vn_table_vals, vm_table_vals, qr_incld, rhofacr, nr_incld, mu_r, lamr, V_qr, V_nr, physics::P3_Constants()); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp index 6902a1892775..5adc3ed71bab 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp @@ -74,7 +74,8 @@ struct UnitWrap::UnitTest::TestRainSelfCollection { nr_selfcollect_tend_local[s] = dc_device(vs).nr_selfcollect_tend; } - Functions::rain_self_collection(rho_local, qr_incld_local, nr_incld_local, nr_selfcollect_tend_local); + Functions::rain_self_collection(rho_local, qr_incld_local, nr_incld_local, nr_selfcollect_tend_local, + physics::P3_Constants()); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { diff --git a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt index cc4af5506d4a..5945189412c8 100644 --- a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt +++ b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt @@ -2,6 +2,50 @@ include(EkatUtils) include(EkatSetCompilerFlags) include(ScreamUtils) +# Copied from EKAT, YAKL is an interface target so requires special +# handling. Get rid of this once RRTMGP is using kokkos. +macro (SetCudaFlagsYakl targetName) + if (Kokkos_ENABLE_CUDA) + # We must find CUDA + find_package(CUDA REQUIRED) + + # Still check if CUDA_FOUND is true, since we don't know if the particular + # FindCUDA.cmake module being used is checking _FIND_REQUIRED + if (NOT CUDA_FOUND) + message (FATAL_ERROR "Error! Unable to find CUDA.") + endif() + + set(options CUDA_LANG) + set(args1v) + set(argsMv FLAGS) + cmake_parse_arguments(SCF "${options}" "${args1v}" "${argsMv}" ${ARGN}) + + if (SCF_FLAGS) + set (FLAGS ${SCF_FLAGS}) + else () + # We need host-device lambdas + set (FLAGS --expt-extended-lambda) + + IsDebugBuild (SCF_DEBUG) + if (SCF_DEBUG) + # Turn off fused multiply add for debug so we can stay BFB with host + list (APPEND FLAGS --fmad=false) + endif() + endif() + + # Set the flags on the target + if (SCF_CUDA_LANG) + # User is setting the src files language to CUDA + target_compile_options (${targetName} INTERFACE + "$<$:${FLAGS}>") + else() + # We assume the user is setting the src files lang to CXX + target_compile_options (${targetName} INTERFACE + "$<$:${FLAGS}>") + endif() + endif() +endmacro() + ################################## # YAKL # ################################## @@ -17,8 +61,9 @@ if (TARGET yakl) else () # Prepare CUDA/HIP flags for YAKL if (CUDA_BUILD) + string(REPLACE ";" " " KOKKOS_CUDA_OPTIONS_STR "${KOKKOS_CUDA_OPTIONS}") set(YAKL_ARCH "CUDA") - set(YAKL_CUDA_FLAGS "-DYAKL_ARCH_CUDA --expt-extended-lambda --expt-relaxed-constexpr -ccbin ${CMAKE_CXX_COMPILER}") + set(YAKL_CUDA_FLAGS "-DYAKL_ARCH_CUDA ${KOKKOS_CUDA_OPTIONS_STR} --expt-relaxed-constexpr -ccbin ${CMAKE_CXX_COMPILER}") string (REPLACE " " ";" YAKL_CUDA_FLAGS_LIST ${YAKL_CUDA_FLAGS}) endif() if (HIP_BUILD) @@ -36,21 +81,20 @@ else () # EAMxx *requires* MPI, so simply look for it, then link against it find_package(MPI REQUIRED COMPONENTS C) - target_link_libraries (yakl MPI::MPI_C) - EkatDisableAllWarning(yakl) + target_link_libraries (yakl INTERFACE MPI::MPI_C) # For debug builds, set -DYAKL_DEBUG if (CMAKE_BUILD_TYPE_ci STREQUAL "debug") - target_compile_definitions(yakl PUBLIC YAKL_DEBUG) + target_compile_definitions(yakl INTERFACE YAKL_DEBUG) endif() endif() # See eamxx/src/dynamics/homme/CMakeLists.txt for an explanation of this # workaround. if ((SCREAM_MACHINE STREQUAL "ascent" OR SCREAM_MACHINE STREQUAL "pm-gpu") AND CMAKE_BUILD_TYPE_ci STREQUAL "debug") - SetCudaFlags(yakl CUDA_LANG FLAGS -UNDEBUG) + SetCudaFlagsYakl(yakl CUDA_LANG FLAGS -UNDEBUG) else() - SetCudaFlags(yakl CUDA_LANG) + SetCudaFlagsYakl(yakl CUDA_LANG) endif() ################################## @@ -81,7 +125,7 @@ yakl_process_target(rrtmgp) # NOTE: cannot use 'PUBLIC' in target_link_libraries, # since yakl_process_target already used it # with the "plain" signature -target_link_libraries(rrtmgp yakl) +target_link_libraries(rrtmgp yakl Kokkos::kokkos) target_include_directories(rrtmgp PUBLIC ${SCREAM_BASE_DIR}/../../externals/YAKL ${EAM_RRTMGP_DIR}/external/cpp @@ -91,12 +135,6 @@ target_include_directories(rrtmgp PUBLIC ${EAM_RRTMGP_DIR}/external/cpp/rrtmgp/kernels ) -# The lines below are needed to ensure that kokkos_launch_compiler injects -# nvcc into compilations. rrtmgp uses YAKL, not kokkos, so the wrapper -# didn't know to add nvcc without these lines. -target_compile_definitions(rrtmgp PRIVATE KOKKOS_DEPENDENCE) -target_link_options(rrtmgp PRIVATE -DKOKKOS_DEPENDENCE) - # Build RRTMGP interface; note that we separate the SCREAM-specific RRTMGP interface # from the external core RRTMGP library because, ideally, the RRTMGP library has its # own build, and we would just use add_subdirectory() above to build it. Also, this @@ -125,7 +163,7 @@ yakl_process_target(scream_rrtmgp_yakl) # since yakl_process_target already used it # with the "plain" signature find_library(NETCDF_C netcdf HINTS ${NetCDF_C_PATH}/lib) -target_link_libraries(scream_rrtmgp_yakl ${NETCDF_C} rrtmgp scream_share) +target_link_libraries(scream_rrtmgp_yakl ${NETCDF_C} rrtmgp scream_share Kokkos::kokkos) target_include_directories(scream_rrtmgp_yakl PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_include_directories(scream_rrtmgp_yakl SYSTEM PUBLIC @@ -142,7 +180,7 @@ set(SCREAM_RRTMGP_SOURCES ) add_library(scream_rrtmgp ${SCREAM_RRTMGP_SOURCES}) -target_link_libraries(scream_rrtmgp PUBLIC scream_share physics_share csm_share scream_rrtmgp_yakl) +target_link_libraries(scream_rrtmgp PUBLIC scream_share physics_share csm_share scream_rrtmgp_yakl Kokkos::kokkos) set_target_properties(scream_rrtmgp PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/modules ) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 0d7a933ec3e0..16811885a3b9 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -159,6 +159,7 @@ void RRTMGPRadiation::set_grids(const std::shared_ptr grids_ add_field("dtau067" , scalar3d_layout_mid, nondim, grid_name); add_field("dtau105" , scalar3d_layout_mid, nondim, grid_name); add_field("sunlit" , scalar2d_layout , nondim, grid_name); + add_field("cldfrac_rad" , scalar3d_layout_mid, nondim, grid_name); // Cloud-top diagnostics following AeroCOM recommendation add_field("T_mid_at_cldtop", scalar2d_layout, K, grid_name); add_field("p_mid_at_cldtop", scalar2d_layout, Pa, grid_name); @@ -468,6 +469,7 @@ void RRTMGPRadiation::run_impl (const double dt) { auto d_surf_lw_flux_up = get_field_in("surf_lw_flux_up").get_view(); // Output fields auto d_tmid = get_field_out("T_mid").get_view(); + auto d_cldfrac_rad = get_field_out("cldfrac_rad").get_view(); // Aerosol optics only exist if m_do_aerosol_rad is true, so declare views and copy from FM if so using view_3d = Field::view_dev_t; @@ -868,6 +870,7 @@ void RRTMGPRadiation::run_impl (const double dt) { } else { cldfrac_tot(i+1,k+1) = 0; } + d_cldfrac_rad(icol,k) = cldfrac_tot(i+1,k+1); }); }); } else { @@ -877,6 +880,7 @@ void RRTMGPRadiation::run_impl (const double dt) { const int icol = i + beg; Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { cldfrac_tot(i+1,k+1) = d_cldfrac_tot(icol,k); + d_cldfrac_rad(icol,k) = d_cldfrac_tot(icol,k); }); }); } diff --git a/components/eamxx/src/physics/rrtmgp/tests/CMakeLists.txt b/components/eamxx/src/physics/rrtmgp/tests/CMakeLists.txt index 8a6a15948b4c..c6fcfae76f8d 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/rrtmgp/tests/CMakeLists.txt @@ -1,6 +1,4 @@ -# NOTE: tests inside this if statement won't be built in a baselines-only build -if (NOT SCREAM_BASELINES_ONLY) - +if (SCREAM_ONLY_GENERATE_BASELINES) # Build baseline code add_executable(generate_baseline generate_baseline.cpp) target_link_libraries(generate_baseline PUBLIC scream_rrtmgp rrtmgp_test_utils) @@ -8,23 +6,23 @@ if (NOT SCREAM_BASELINES_ONLY) # Generate allsky baseline with the usual cmake custom command-target pair pattern # Note: these "baselines" are not to compare scream with a previous version, but # rather to compare scream::rrtmgp with raw rrtmgp. - add_custom_command ( - OUTPUT ${SCREAM_TEST_DATA_DIR}/rrtmgp-allsky-baseline.nc - COMMAND ${CMAKE_COMMAND} -E env $ - ${SCREAM_DATA_DIR}/init/rrtmgp-allsky.nc - ${SCREAM_TEST_DATA_DIR}/rrtmgp-allsky-baseline.nc - ) - add_custom_target(rrtmgp_allsky_baseline.nc - DEPENDS ${SCREAM_TEST_DATA_DIR}/rrtmgp-allsky-baseline.nc + CreateUnitTestFromExec( + rrtmgp-allsky-baseline generate_baseline + LABELS baseline_gen rrtmgp + EXE_ARGS "${SCREAM_DATA_DIR}/init/rrtmgp-allsky.nc ${SCREAM_BASELINES_DIR}/data/rrtmgp-allsky-baseline.nc" ) - CreateUnitTest(rrtmgp_tests rrtmgp_tests.cpp - LIBS scream_rrtmgp rrtmgp_test_utils - LABELS "rrtmgp;physics" - EXE_ARGS "-i ${SCREAM_DATA_DIR}/init/rrtmgp-allsky.nc -b ${SCREAM_TEST_DATA_DIR}/rrtmgp-allsky-baseline.nc" - EXCLUDE_MAIN_CPP - ) - add_dependencies (rrtmgp_tests rrtmgp_allsky_baseline.nc) +else () + + if (SCREAM_ENABLE_BASELINE_TESTS) + # NOTE: tests inside this branch won't be built in a baselines-only build + CreateUnitTest(rrtmgp_tests rrtmgp_tests.cpp + LIBS scream_rrtmgp rrtmgp_test_utils + LABELS "rrtmgp;physics" + EXE_ARGS "-i ${SCREAM_DATA_DIR}/init/rrtmgp-allsky.nc -b ${SCREAM_BASELINES_DIR}/data/rrtmgp-allsky-baseline.nc" + EXCLUDE_MAIN_CPP + ) + endif() CreateUnitTest(rrtmgp_unit_tests rrtmgp_unit_tests.cpp LIBS scream_rrtmgp rrtmgp_test_utils diff --git a/components/eamxx/src/physics/share/physics_constants.hpp b/components/eamxx/src/physics/share/physics_constants.hpp index 16750a791e3e..fd78dff8c86a 100644 --- a/components/eamxx/src/physics/share/physics_constants.hpp +++ b/components/eamxx/src/physics/share/physics_constants.hpp @@ -5,6 +5,7 @@ #include "ekat/util/ekat_string_utils.hpp" #include "ekat/ekat_scalar_traits.hpp" +#include "ekat/logging/ekat_logger.hpp" #include @@ -52,7 +53,6 @@ struct Constants static constexpr Scalar SXTH = 1.0/6.0; static constexpr Scalar PIOV3 = Pi*THIRD; static constexpr Scalar PIOV6 = Pi*SXTH; - static constexpr Scalar AIMM = 0.65; static constexpr Scalar BIMM = 2.0; static constexpr Scalar CONS1 = PIOV6*RHOW; static constexpr Scalar CONS2 = 4.*PIOV3*RHOW; @@ -76,13 +76,8 @@ struct Constants static constexpr Scalar INV_CP = 1.0/CP; // static constexpr Scalar Tol = ekat::is_single_precision::value ? 2e-5 : 1e-14; static constexpr Scalar macheps = std::numeric_limits::epsilon(); - static constexpr Scalar mu_r_const = 1.0; static constexpr Scalar dt_left_tol = 1.e-4; static constexpr Scalar bcn = 2.; - static constexpr Scalar rho_rimeMin = 50.; - static constexpr Scalar rho_rimeMax = 900.; - static constexpr Scalar eci = 0.5; - static constexpr Scalar eri = 1.0; static constexpr Scalar dropmass = 5.2e-7; static constexpr Scalar NCCNST = 200.0e+6; static constexpr Scalar incloud_limit = 5.1e-3; @@ -126,6 +121,121 @@ struct Constants static constexpr Scalar earth_ellipsoid3 = 1.175; // third expansion coefficient for WGS84 ellipsoid }; +template +struct P3_Constants +{ + public: + Scalar p3_autoconversion_prefactor = 1350.0; + Scalar p3_mu_r_constant = 1.0; + Scalar p3_spa_to_nc = 1.0; + Scalar p3_k_accretion = 67.0; + Scalar p3_eci = 0.5; + Scalar p3_eri = 1.0; + Scalar p3_rho_rime_min = 50.0; + Scalar p3_rho_rime_max = 900.0; + Scalar p3_a_imm = 0.65; + Scalar p3_dep_nucleation_exponent = 0.304; + Scalar p3_ice_sed_knob = 1.0; + Scalar p3_d_breakup_cutoff = 0.00028; + + void set_p3_from_namelist(ekat::ParameterList ¶ms){ + + std::string nname = "p3_autoconversion_prefactor"; + if(params.isParameter(nname)) + p3_autoconversion_prefactor = params.get(nname); + + nname = "p3_mu_r_constant"; + if(params.isParameter(nname)) + p3_mu_r_constant = params.get(nname); + + nname = "p3_spa_to_nc"; + if(params.isParameter(nname)) + p3_spa_to_nc = params.get(nname); + + nname = "p3_k_accretion"; + if(params.isParameter(nname)) + p3_k_accretion = params.get(nname); + + nname = "p3_eci"; + if(params.isParameter(nname)) + p3_eci = params.get(nname); + + nname = "p3_eri"; + if(params.isParameter(nname)) + p3_eri = params.get(nname); + + nname = "p3_rho_rime_min"; + if(params.isParameter(nname)) + p3_rho_rime_min = params.get(nname); + + nname = "p3_rho_rime_max"; + if(params.isParameter(nname)) + p3_rho_rime_max = params.get(nname); + + nname = "p3_a_imm"; + if(params.isParameter(nname)) + p3_a_imm = params.get(nname); + + nname = "p3_dep_nucleation_exponent"; + if(params.isParameter(nname)) + p3_dep_nucleation_exponent = params.get(nname); + + nname = "p3_ice_sed_knob"; + if(params.isParameter(nname)) + p3_ice_sed_knob = params.get(nname); + + nname = "p3_d_breakup_cutoff"; + if(params.isParameter(nname)) + p3_d_breakup_cutoff = params.get(nname); + + }; + + void print_p3constants(std::shared_ptr logger){ + logger->info("P3 Constants:"); + + std::string nname = "p3_autoconversion_prefactor"; + logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_autoconversion_prefactor)); + + nname = "p3_mu_r_constant"; + logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_mu_r_constant)); + + nname = "p3_spa_to_nc"; + logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_spa_to_nc)); + + nname = "p3_k_accretion"; + logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_k_accretion)); + + nname = "p3_eci"; + logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_eci)); + + nname = "p3_eri"; + logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_eri)); + + nname = "p3_rho_rime_min"; + logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_rho_rime_min)); + + nname = "p3_rho_rime_max"; + logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_rho_rime_max)); + + nname = "p3_a_imm"; + logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_a_imm)); + + nname = "p3_dep_nucleation_exponent"; + logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_dep_nucleation_exponent)); + + nname = "p3_ice_sed_knob"; + logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_ice_sed_knob)); + + nname = "p3_d_breakup_cutoff"; + logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_d_breakup_cutoff)); + + logger->info(" "); + }; + + //one can implement a check here too, for acceptable ranges + +}; // P3_Constants + // Gases // Define the molecular weight for each gas, which can then be // used to determine the volume mixing ratio for each gas. diff --git a/components/eamxx/src/physics/share/tests/CMakeLists.txt b/components/eamxx/src/physics/share/tests/CMakeLists.txt index f3849b7571bf..a196c8470581 100644 --- a/components/eamxx/src/physics/share/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/share/tests/CMakeLists.txt @@ -1,14 +1,18 @@ include(ScreamUtils) # NOTE: tests inside this if statement won't be built in a baselines-only build -if (NOT SCREAM_BASELINES_ONLY) +if (NOT SCREAM_ONLY_GENERATE_BASELINES) CreateUnitTest(physics_test_data physics_test_data_unit_tests.cpp LIBS physics_share THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC}) endif() if (SCREAM_ENABLE_BASELINE_TESTS) - set(BASELINE_FILE_ARG "-b ${SCREAM_TEST_DATA_DIR}/physics_saturation.baseline") + if (SCREAM_ONLY_GENERATE_BASELINES) + set(BASELINE_FILE_ARG "-g -b ${SCREAM_BASELINES_DIR}/data/physics_saturation.baseline") + else() + set(BASELINE_FILE_ARG "-b ${SCREAM_BASELINES_DIR}/data/physics_saturation.baseline") + endif() # The comparison test. Expects baseline to exist. All thread configurations # will use the same baseline. @@ -17,23 +21,6 @@ if (SCREAM_ENABLE_BASELINE_TESTS) EXE_ARGS "${BASELINE_FILE_ARG}" LABELS "physics") - # - # Use fake tests to generate shell commands to generate baselines - # - CreateUnitTestFromExec(physics_saturation_baseline_fake physics_saturation_run_and_cmp - EXE_ARGS "-g ${BASELINE_FILE_ARG}" - PROPERTIES DISABLED True) - - get_test_property(physics_saturation_baseline_fake FULL_TEST_COMMAND PHYSICS_SATURATION_GEN) - - if (PHYSICS_SATURATION_GEN STREQUAL "NOTFOUND") - message(FATAL_ERROR "Could not get FULL_TEST_COMMAND for physics_saturation_baseline fake test") - endif() - - separate_arguments(PHYSICS_SATURATION_GEN_ARGS UNIX_COMMAND "${PHYSICS_SATURATION_GEN}") - - add_custom_target(physics_saturation_baseline - COMMAND ${CMAKE_COMMAND} -E env OMP_NUM_THREADS=${SCREAM_TEST_MAX_THREADS} ${PHYSICS_SATURATION_GEN_ARGS}) - - add_dependencies(baseline physics_saturation_baseline) + # Note: the baseline_gen label is really only used if SCREAM_ONLY_GENERATE_BASELINES=ON, but no harm adding it + set_tests_properties(physics_saturation_run_and_cmp PROPERTIES LABELS "baseline_gen;baseline_cmp") endif() diff --git a/components/eamxx/src/physics/share/tests/physics_saturation_run_and_cmp.cpp b/components/eamxx/src/physics/share/tests/physics_saturation_run_and_cmp.cpp index cbf9945d1445..2fcb68f6337c 100644 --- a/components/eamxx/src/physics/share/tests/physics_saturation_run_and_cmp.cpp +++ b/components/eamxx/src/physics/share/tests/physics_saturation_run_and_cmp.cpp @@ -50,7 +50,7 @@ struct UnitWrap::UnitTest::TestSaturation sat_ice_fp = physics::polysvp1(temps, true, Smask(true))[0]; sat_liq_fp = physics::polysvp1(temps, false, Smask(true))[0]; - //Functions::qv_sat_dry(const Spack& t_atm, const Spack& p_atm_dry, const bool ice, const Smask& range_mask, + //Functions::qv_sat_dry(const Spack& t_atm, const Spack& p_atm_dry, const bool ice, const Smask& range_mask, // const SaturationFcn func_idx, const char* caller) mix_ice_fr = physics::qv_sat_dry(temps, pres, true, Smask(true), physics::Polysvp1)[0]; mix_liq_fr = physics::qv_sat_dry(temps, pres, false,Smask(true), physics::Polysvp1)[0]; @@ -109,24 +109,27 @@ struct UnitWrap::UnitTest::TestSaturation EKAT_REQUIRE_MSG( fid, "generate_baseline can't write " << filename); for (auto p : params_) { - OutputData d; ParamSet ps = p; - Kokkos::parallel_reduce(1, - KOKKOS_LAMBDA(const size_t&, OutputData& output) { + Kokkos::View d_dev("",1); + Kokkos::parallel_for(1, + KOKKOS_LAMBDA(const size_t&) { TestSaturation::saturation_tests( ps.temperature, ps.pressure, - output.sat_ice_fp, - output.sat_liq_fp, - output.mix_ice_fr, - output.mix_liq_fr, - output.sat_ice_mkp, - output.sat_liq_mkp, - output.mix_ice_mkr, - output.mix_liq_mkr); - }, d); - + d_dev[0].sat_ice_fp, + d_dev[0].sat_liq_fp, + d_dev[0].mix_ice_fr, + d_dev[0].mix_liq_fr, + d_dev[0].sat_ice_mkp, + d_dev[0].sat_liq_mkp, + d_dev[0].mix_ice_mkr, + d_dev[0].mix_liq_mkr); + }); Kokkos::fence(); - write(fid, d); // Save the fields to the baseline file. + + auto d_host = Kokkos::create_mirror_view(d_dev); + Kokkos::deep_copy(d_host,d_dev); + + write(fid, d_host[0]); // Save the fields to the baseline file. } return 0; @@ -139,28 +142,31 @@ struct UnitWrap::UnitTest::TestSaturation int case_num = 0; for (auto p : params_) { ++case_num; - OutputData ref, d; + OutputData ref; ParamSet ps = p; std::cout << "--- checking physics saturation case # " << case_num << std::endl; read(fid, ref); - Kokkos::parallel_reduce(1, - KOKKOS_LAMBDA(const size_t&, OutputData& output) { + Kokkos::View d_dev("",1); + Kokkos::parallel_for(1, + KOKKOS_LAMBDA(const size_t&) { TestSaturation::saturation_tests( ps.temperature, ps.pressure, - output.sat_ice_fp, - output.sat_liq_fp, - output.mix_ice_fr, - output.mix_liq_fr, - output.sat_ice_mkp, - output.sat_liq_mkp, - output.mix_ice_mkr, - output.mix_liq_mkr); - }, d); - + d_dev[0].sat_ice_fp, + d_dev[0].sat_liq_fp, + d_dev[0].mix_ice_fr, + d_dev[0].mix_liq_fr, + d_dev[0].sat_ice_mkp, + d_dev[0].sat_liq_mkp, + d_dev[0].mix_ice_mkr, + d_dev[0].mix_liq_mkr); + }); Kokkos::fence(); - ne = compare(tol, ref, d); + auto d_host = Kokkos::create_mirror_view(d_dev); + Kokkos::deep_copy(d_host,d_dev); + + ne = compare(tol, ref, d_host[0]); if (ne) std::cout << "Ref impl failed.\n"; nerr += ne; } @@ -189,6 +195,7 @@ struct UnitWrap::UnitTest::TestSaturation Scalar mix_ice_mkr; Scalar mix_liq_mkr; + KOKKOS_INLINE_FUNCTION OutputData& operator+=(const OutputData& rhs) { sat_ice_fp += rhs.sat_ice_fp; diff --git a/components/eamxx/src/physics/shoc/CMakeLists.txt b/components/eamxx/src/physics/shoc/CMakeLists.txt index 051f9635d179..e37379095d00 100644 --- a/components/eamxx/src/physics/shoc/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/CMakeLists.txt @@ -108,7 +108,7 @@ if (SCREAM_SMALL_KERNELS) add_library(shoc ${SHOC_SRCS} ${SHOC_SK_SRCS}) else() add_library(shoc ${SHOC_SRCS}) - if (NOT SCREAM_LIBS_ONLY AND NOT SCREAM_BASELINES_ONLY) + if (NOT SCREAM_LIBS_ONLY AND NOT SCREAM_ONLY_GENERATE_BASELINES) add_library(shoc_sk ${SHOC_SRCS} ${SHOC_SK_SRCS}) # Always build shoc_sk with SCREAM_SMALL_KERNELS on target_compile_definitions(shoc_sk PUBLIC "SCREAM_SMALL_KERNELS") diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index c107e53b5b31..f0c368549400 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -432,7 +432,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) if (m_grid->has_geometry_data("dx_short")) { // We must be running with IntensiveObservationPeriod on, with a planar geometry auto dx = m_grid->get_geometry_data("dx_short").get_view()(); - Kokkos::deep_copy(cell_length, dx); + Kokkos::deep_copy(cell_length, dx*1000); // convert km -> m } else { const auto area = m_grid->get_geometry_data("area").get_view(); const auto lat = m_grid->get_geometry_data("lat").get_view(); diff --git a/components/eamxx/src/physics/shoc/tests/CMakeLists.txt b/components/eamxx/src/physics/shoc/tests/CMakeLists.txt index 485f625e1423..87e55959c528 100644 --- a/components/eamxx/src/physics/shoc/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/tests/CMakeLists.txt @@ -72,7 +72,7 @@ set(SHOC_TESTS_SRCS ) # SHOC_TESTS_SRCS # NOTE: tests inside this if statement won't be built in a baselines-only build -if (NOT SCREAM_BASELINES_ONLY) +if (NOT SCREAM_ONLY_GENERATE_BASELINES) CreateUnitTest(shoc_tests "${SHOC_TESTS_SRCS}" LIBS shoc THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} @@ -88,7 +88,11 @@ if (NOT SCREAM_BASELINES_ONLY) endif() if (SCREAM_ENABLE_BASELINE_TESTS) - set(BASELINE_FILE_ARG "-b ${SCREAM_TEST_DATA_DIR}/shoc_run_and_cmp.baseline") + if (SCREAM_ONLY_GENERATE_BASELINES) + set(BASELINE_FILE_ARG "-g -b ${SCREAM_BASELINES_DIR}/data/shoc_run_and_cmp.baseline") + else() + set(BASELINE_FILE_ARG "-b ${SCREAM_BASELINES_DIR}/data/shoc_run_and_cmp.baseline") + endif() CreateUnitTestExec(shoc_run_and_cmp "shoc_run_and_cmp.cpp" LIBS shoc @@ -104,42 +108,13 @@ if (SCREAM_ENABLE_BASELINE_TESTS) EXE_ARGS "-f ${BASELINE_FILE_ARG}" LABELS "shoc;physics") - # - # Use fake tests to generate shell commands to generate baselines - # - CreateUnitTestFromExec(shoc_baseline_f90_fake shoc_run_and_cmp - THREADS ${SCREAM_TEST_MAX_THREADS} - EXE_ARGS "-f -g ${BASELINE_FILE_ARG}" - PROPERTIES DISABLED True) - - CreateUnitTestFromExec(shoc_baseline_cxx_fake shoc_run_and_cmp - THREADS ${SCREAM_TEST_MAX_THREADS} - EXE_ARGS "-g ${BASELINE_FILE_ARG}" - PROPERTIES DISABLED True) - + # By default, baselines should be created using all fortran (ctest -L baseline_gen). If the user wants + # to use CXX to generate their baselines, they should use "ctest -L baseline_gen_cxx". + # Note: the baseline_gen label is really only used if SCREAM_ONLY_GENERATE_BASELINES=ON, but no harm adding it if (SCREAM_TEST_MAX_THREADS GREATER 1) - get_test_property(shoc_baseline_f90_fake_omp${SCREAM_TEST_MAX_THREADS} FULL_TEST_COMMAND SHOC_F90_GEN) - get_test_property(shoc_baseline_cxx_fake_omp${SCREAM_TEST_MAX_THREADS} FULL_TEST_COMMAND SHOC_CXX_GEN) - else() - get_test_property(shoc_baseline_f90_fake FULL_TEST_COMMAND SHOC_F90_GEN) - get_test_property(shoc_baseline_cxx_fake FULL_TEST_COMMAND SHOC_CXX_GEN) + # ECUT only adds _ompX if we have more than one value of X, or if X>1 + set (TEST_SUFFIX _omp${SCREAM_TEST_MAX_THREADS}) endif() - - if (SHOC_F90_GEN STREQUAL "NOTFOUND") - message(FATAL_ERROR "Could not get FULL_TEST_COMMAND for shoc_baseline fake test") - endif() - - separate_arguments(SHOC_F90_GEN_ARGS UNIX_COMMAND "${SHOC_F90_GEN}") - separate_arguments(SHOC_CXX_GEN_ARGS UNIX_COMMAND "${SHOC_CXX_GEN}") - - add_custom_target(shoc_baseline_f90 - COMMAND ${CMAKE_COMMAND} -E env OMP_NUM_THREADS=${SCREAM_TEST_MAX_THREADS} ${SHOC_F90_GEN_ARGS}) - - add_custom_target(shoc_baseline_cxx - COMMAND ${CMAKE_COMMAND} -E env OMP_NUM_THREADS=${SCREAM_TEST_MAX_THREADS} ${SHOC_CXX_GEN_ARGS}) - - # By default, baselines should be created using all fortran (make baseline). If the user wants - # to use CXX to generate their baselines, they should use "make baseline_cxx". - add_dependencies(baseline shoc_baseline_f90) - add_dependencies(baseline_cxx shoc_baseline_cxx) + set_tests_properties (shoc_run_and_cmp_f90${TEST_SUFFIX} PROPERTIES LABELS "baseline_gen;baseline_cmp") + set_tests_properties (shoc_run_and_cmp_cxx${TEST_SUFFIX} PROPERTIES LABELS "baseline_gen;cxx baseline_cmp") endif() diff --git a/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp b/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp index 6b59ce16a877..f244a2262f9c 100644 --- a/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp +++ b/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp @@ -3,7 +3,6 @@ #include "share/util/scream_time_stamp.hpp" #include "share/io/scream_scorpio_interface.hpp" #include "share/property_checks/field_within_interval_check.hpp" -#include "share/property_checks/field_lower_bound_check.hpp" #include "ekat/ekat_assert.hpp" #include "ekat/util/ekat_units.hpp" @@ -17,7 +16,8 @@ namespace scream SPA::SPA (const ekat::Comm& comm, const ekat::ParameterList& params) : AtmosphereProcess(comm, params) { - // Nothing to do here + EKAT_REQUIRE_MSG(m_params.isParameter("spa_data_file"), + "ERROR: spa_data_file is missing from SPA parameter list."); } // ========================================================================================= @@ -36,42 +36,109 @@ void SPA::set_grids(const std::shared_ptr grids_manager) const auto& grid_name = m_grid->name(); m_num_cols = m_grid->get_num_local_dofs(); // Number of columns on this rank m_num_levs = m_grid->get_num_vertical_levels(); // Number of levels per column - m_dofs_gids = m_grid->get_dofs_gids().get_view(); - m_min_global_dof = m_grid->get_global_min_dof_gid(); // Define the different field layouts that will be used for this process - // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and interfaces + // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and interfaces FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols, m_num_levs} }; FieldLayout scalar2d_layout { {COL}, {m_num_cols} }; FieldLayout scalar1d_layout_mid { {LEV}, {m_num_levs} }; // Use VAR field tag for gases for now; consider adding a tag? - FieldLayout scalar3d_swband_layout { {COL,SWBND, LEV}, {m_num_cols, m_nswbands, m_num_levs} }; - FieldLayout scalar3d_lwband_layout { {COL,LWBND, LEV}, {m_num_cols, m_nlwbands, m_num_levs} }; + FieldLayout scalar3d_swband_layout { {COL,SWBND, LEV}, {m_num_cols, m_nswbands, m_num_levs} }; + FieldLayout scalar3d_lwband_layout { {COL,LWBND, LEV}, {m_num_cols, m_nlwbands, m_num_levs} }; // Set of fields used strictly as input - constexpr int ps = Pack::n; + constexpr int ps = Spack::n; add_field("p_mid" , scalar3d_layout_mid, Pa, grid_name, ps); // Set of fields used strictly as output - add_field("nccn", scalar3d_layout_mid, 1/kg, grid_name,ps); - add_field("aero_g_sw", scalar3d_swband_layout, nondim, grid_name,ps); - add_field("aero_ssa_sw", scalar3d_swband_layout, nondim, grid_name,ps); - add_field("aero_tau_sw", scalar3d_swband_layout, nondim, grid_name,ps); - add_field("aero_tau_lw", scalar3d_lwband_layout, nondim, grid_name,ps); - - // Init output data structure + add_field("nccn", scalar3d_layout_mid, 1/kg, grid_name, ps); + add_field("aero_g_sw", scalar3d_swband_layout, nondim, grid_name, ps); + add_field("aero_ssa_sw", scalar3d_swband_layout, nondim, grid_name, ps); + add_field("aero_tau_sw", scalar3d_swband_layout, nondim, grid_name, ps); + add_field("aero_tau_lw", scalar3d_lwband_layout, nondim, grid_name, ps); + + // We can already create some of the spa structures + + // 1. Create SPAHorizInterp remapper + auto spa_data_file = m_params.get("spa_data_file"); + auto spa_map_file = m_params.get("spa_remap_file",""); + + // IOP cases cannot have a remap file. IOP file reader is itself a remaper, + // where a single column of data corresponding to the closest lat/lon pair to + // the IOP lat/lon parameters is read from file, and that column data is mapped + // to all columns of the IdentityRemapper source fields. + EKAT_REQUIRE_MSG(spa_map_file == "" or spa_map_file == "None" or not m_iop, + "Error! Cannot define spa_remap_file for cases with an Intensive Observation Period defined. " + "The IOP class defines it's own remap from file data -> model data.\n"); + + SPAHorizInterp = SPAFunc::create_horiz_remapper (m_grid,spa_data_file,spa_map_file, m_iop!=nullptr); + + // Grab a sw and lw field from the horiz interp, and check sw/lw dim against what we hardcoded in this class + auto nswbands_data = SPAHorizInterp->get_src_field(4).get_header().get_identifier().get_layout().dim(SWBND); + auto nlwbands_data = SPAHorizInterp->get_src_field(5).get_header().get_identifier().get_layout().dim(LWBND); + EKAT_REQUIRE_MSG (nswbands_data==m_nswbands, + "Error! Spa data file has a different number of sw bands than the model.\n" + " - spa data swbands: " + std::to_string(nswbands_data) + "\n" + " - model swbands : " + std::to_string(m_nswbands) + "\n"); + EKAT_REQUIRE_MSG (nlwbands_data==m_nlwbands, + "Error! Spa data file has a different number of lw bands than the model.\n" + " - spa data lwbands: " + std::to_string(nlwbands_data) + "\n" + " - model lwbands : " + std::to_string(m_nlwbands) + "\n"); + + const auto io_grid = SPAHorizInterp->get_src_grid(); + + // 2. Initialize the size of the SPAData structures. + // Note: add 2 to number of levels to allow extrapolation if model pressure is outside the data range + m_num_src_levs = io_grid->get_num_vertical_levels(); + SPAData_start = SPAFunc::SPAInput(m_num_cols, m_num_src_levs+2, m_nswbands, m_nlwbands); + SPAData_end = SPAFunc::SPAInput(m_num_cols, m_num_src_levs+2, m_nswbands, m_nlwbands); SPAData_out.init(m_num_cols,m_num_levs,m_nswbands,m_nlwbands,false); - // Note: only the number of levels associated with this data haven't been set. We can - // take this information directly from the spa data file. - m_spa_data_file = m_params.get("spa_data_file"); - scorpio::register_file(m_spa_data_file,scorpio::Read); - m_num_src_levs = scorpio::get_dimlen(m_spa_data_file,"lev"); - scorpio::eam_pio_closefile(m_spa_data_file); - SPAHorizInterp.m_comm = m_comm; + // 3 Read in hyam/hybm in start/end data, and pad them + Field hyam(FieldIdentifier("hyam",io_grid->get_vertical_layout(true),nondim,io_grid->name())); + Field hybm(FieldIdentifier("hybm",io_grid->get_vertical_layout(true),nondim,io_grid->name())); + hyam.allocate_view(); + hybm.allocate_view(); + + AtmosphereInput hvcoord_reader(spa_data_file,io_grid,{hyam,hybm},true); + hvcoord_reader.read_variables(); + hvcoord_reader.finalize(); + + // Do the copy on host, cause it's just easier, + // and this is just a cheap loop during init + auto hyam_h = hyam.get_view(); + auto hybm_h = hybm.get_view(); + auto nlevs = io_grid->get_num_vertical_levels(); + for (auto data : {SPAData_start, SPAData_end} ) { + auto spa_hyam = ekat::scalarize(data.hyam); + auto spa_hybm = ekat::scalarize(data.hybm); + auto spa_hyam_h = Kokkos::create_mirror_view(spa_hyam); + auto spa_hybm_h = Kokkos::create_mirror_view(spa_hybm); + for (int i=0; i(); - SPAData_out.AER_G_SW = get_field_out("aero_g_sw").get_view(); - SPAData_out.AER_SSA_SW = get_field_out("aero_ssa_sw").get_view(); - SPAData_out.AER_TAU_SW = get_field_out("aero_tau_sw").get_view(); - SPAData_out.AER_TAU_LW = get_field_out("aero_tau_lw").get_view(); - - // Retrieve the remap and data file locations from the parameter list: - EKAT_REQUIRE_MSG(m_params.isParameter("spa_remap_file"),"ERROR: spa_remap_file is missing from SPA parameter list."); - EKAT_REQUIRE_MSG(m_params.isParameter("spa_data_file"),"ERROR: spa_data_file is missing from SPA parameter list."); - m_spa_remap_file = m_params.get("spa_remap_file"); - - // Set the SPA remap weights. - // TODO: We may want to provide an option to calculate weights on-the-fly. - // If so, then the EKAT_REQUIRE_MSG above will need to be removed and - // we can have a default m_spa_data_file option that is online calculation. - using ci_string = ekat::CaseInsensitiveString; - ci_string no_filename = "none"; - if (m_spa_remap_file == no_filename) { - if (m_comm.am_i_root()) { - printf("WARNING: spa_remap_file has been set to 'NONE', assuming that SPA data and simulation are on the same grid - skipping horizontal interpolation\n"); - } - SPAFunc::set_remap_weights_one_to_one(m_min_global_dof,m_dofs_gids,SPAHorizInterp); - } else { - SPAFunc::get_remap_weights_from_file(m_spa_remap_file,m_min_global_dof,m_dofs_gids,SPAHorizInterp); - } - - // Initialize the size of the SPAData structures: add 2 to number of levels for padding - SPAData_start = SPAFunc::SPAInput(m_dofs_gids.size(), m_num_src_levs+2, m_nswbands, m_nlwbands); - SPAData_end = SPAFunc::SPAInput(m_dofs_gids.size(), m_num_src_levs+2, m_nswbands, m_nlwbands); - - // Update the local time state information and load the first set of SPA data for interpolation: - auto ts = timestamp(); - SPATimeState.inited = false; - SPATimeState.current_month = ts.get_month(); - SPAFunc::update_spa_timestate(m_spa_data_file,m_nswbands,m_nlwbands,ts,SPAHorizInterp,SPATimeState,SPAData_start,SPAData_end); - - // Set property checks for fields in this process + // Initialize SPAData_out with the views from the out fields + SPAData_out.CCN3 = get_field_out("nccn").get_view(); + SPAData_out.AER_G_SW = get_field_out("aero_g_sw").get_view(); + SPAData_out.AER_SSA_SW = get_field_out("aero_ssa_sw").get_view(); + SPAData_out.AER_TAU_SW = get_field_out("aero_tau_sw").get_view(); + SPAData_out.AER_TAU_LW = get_field_out("aero_tau_lw").get_view(); + + // Load the first month into spa_end. + // Note: At the first time step, the data will be moved into spa_beg, + // and spa_end will be reloaded from file with the new month. + const int curr_month = timestamp().get_month()-1; // 0-based + SPAFunc::update_spa_data_from_file(SPADataReader,SPAIOPDataReader,timestamp(),curr_month,*SPAHorizInterp,SPAData_end); + + // 6. Set property checks for fields in this process using Interval = FieldWithinIntervalCheck; const auto eps = std::numeric_limits::epsilon(); @@ -208,10 +250,10 @@ void SPA::run_impl (const double dt) /* Update the SPATimeState to reflect the current time, note the addition of dt */ SPATimeState.t_now = ts.frac_of_year_in_days(); /* Update time state and if the month has changed, update the data.*/ - SPAFunc::update_spa_timestate(m_spa_data_file,m_nswbands,m_nlwbands,ts,SPAHorizInterp,SPATimeState,SPAData_start,SPAData_end); + SPAFunc::update_spa_timestate(SPADataReader,SPAIOPDataReader,ts,*SPAHorizInterp,SPATimeState,SPAData_start,SPAData_end); // Call the main SPA routine to get interpolated aerosol forcings. - const auto& pmid_tgt = get_field_in("p_mid").get_view(); + const auto& pmid_tgt = get_field_in("p_mid").get_view(); SPAFunc::spa_main(SPATimeState, pmid_tgt, m_buffer.p_mid_src, SPAData_start,SPAData_end,m_buffer.spa_temp,SPAData_out); } diff --git a/components/eamxx/src/physics/spa/eamxx_spa_process_interface.hpp b/components/eamxx/src/physics/spa/eamxx_spa_process_interface.hpp index b2ca826d65b5..b2d9f2b36388 100644 --- a/components/eamxx/src/physics/spa/eamxx_spa_process_interface.hpp +++ b/components/eamxx/src/physics/spa/eamxx_spa_process_interface.hpp @@ -3,8 +3,9 @@ #include "physics/spa/spa_functions.hpp" #include "share/atm_process/atmosphere_process.hpp" -#include "ekat/ekat_parameter_list.hpp" -#include "ekat/util/ekat_lin_interp.hpp" +#include "share/io/scorpio_input.hpp" +#include "share/grid/remap/abstract_remapper.hpp" +#include #include @@ -21,20 +22,13 @@ namespace scream class SPA : public AtmosphereProcess { public: - using gid_type = AbstractGrid::gid_type; + using SPAFunc = spa::SPAFunctions; + using Spack = SPAFunc::Spack; + using KT = ekat::KokkosTypes; - using SPAFunc = spa::SPAFunctions; - using Spack = SPAFunc::Spack; - using Pack = ekat::Pack; - using KT = ekat::KokkosTypes; + using view_1d = typename SPAFunc::view_1d; + using view_2d = typename SPAFunc::view_2d; - using view_1d = typename SPAFunc::view_1d; - using view_2d = typename SPAFunc::view_2d; - using view_3d = typename SPAFunc::view_3d; - using view_1d_dof = typename SPAFunc::view_1d; - - template - using uview_1d = Unmanaged>; template using uview_2d = Unmanaged>; @@ -74,34 +68,30 @@ class SPA : public AtmosphereProcess void init_buffers(const ATMBufferManager &buffer_manager); // Keep track of field dimensions and the iteration count - int m_num_cols; + int m_num_cols; int m_num_levs; int m_num_src_levs; - int m_nk_pack; int m_nswbands = 14; int m_nlwbands = 16; - // DOF information - view_1d_dof m_dofs_gids; - int m_total_global_dofs; // Needed to make sure that remap data matches grid. - gid_type m_min_global_dof; - // Struct which contains temporary variables used during spa_main Buffer m_buffer; - // SPA specific files - std::string m_spa_remap_file; - std::string m_spa_data_file; + // IO structure to read in data for standard grids (keep it around to avoid re-creating PIO decomps) + std::shared_ptr SPADataReader; + // Similar to above, but stores info to read data for IOP grid + std::shared_ptr SPAIOPDataReader; // Structures to store the data used for interpolation + std::shared_ptr SPAHorizInterp; + SPAFunc::SPATimeState SPATimeState; - SPAFunc::SPAHorizInterp SPAHorizInterp; SPAFunc::SPAInput SPAData_start; SPAFunc::SPAInput SPAData_end; SPAFunc::SPAOutput SPAData_out; std::shared_ptr m_grid; -}; // class SPA +}; // class SPA } // namespace scream diff --git a/components/eamxx/src/physics/spa/spa_functions.hpp b/components/eamxx/src/physics/spa/spa_functions.hpp index 5a6efebe782e..631aae9eee78 100644 --- a/components/eamxx/src/physics/spa/spa_functions.hpp +++ b/components/eamxx/src/physics/spa/spa_functions.hpp @@ -1,15 +1,14 @@ #ifndef SPA_FUNCTIONS_HPP #define SPA_FUNCTIONS_HPP +#include "control/intensive_observation_period.hpp" #include "share/grid/abstract_grid.hpp" -#include "share/grid/remap/horizontal_remap_utility.hpp" -#include "share/scream_types.hpp" +#include "share/grid/remap/abstract_remapper.hpp" +#include "share/io/scorpio_input.hpp" #include "share/util/scream_time_stamp.hpp" +#include "share/scream_types.hpp" -#include "ekat/ekat_pack_kokkos.hpp" -#include "ekat/ekat_pack_utils.hpp" -#include "ekat/ekat_workspace.hpp" -#include "ekat/mpi/ekat_comm.hpp" +#include namespace scream { namespace spa { @@ -17,7 +16,6 @@ namespace spa { template struct SPAFunctions { - // // ------- Types -------- // @@ -25,22 +23,15 @@ struct SPAFunctions using Scalar = ScalarType; using Device = DeviceType; - template - using BigPack = ekat::Pack; - template - using SmallPack = ekat::Pack; - - using Pack = BigPack; - using Spack = SmallPack; + using Spack = ekat::Pack; using KT = KokkosTypes; using MemberType = typename KT::MemberType; - using WorkspaceManager = typename ekat::WorkspaceManager; - using Workspace = typename WorkspaceManager::Workspace; - using gid_type = AbstractGrid::gid_type; + using iop_ptr_type = std::shared_ptr; + template using view_1d = typename KT::template view_1d; template @@ -63,7 +54,6 @@ struct SPAFunctions struct SPATimeState { SPATimeState() = default; // Whether the timestate has been initialized. - bool inited = false; // The current month int current_month = -1; // Julian Date for the beginning of the month, as defined in @@ -137,31 +127,58 @@ struct SPAFunctions SPAData data; // All spa fields }; // SPAInput + struct IOPReader { + IOPReader (iop_ptr_type& iop_, + const std::string file_name_, + const std::vector& io_fields_, + const std::shared_ptr& io_grid_) + : iop(iop_), file_name(file_name_) + { + field_mgr = std::make_shared(io_grid_); + for (auto& f : io_fields_) { + field_mgr->add_field(f); + field_names.push_back(f.name()); + } + + // Set IO info for this grid and file in IOP object + iop->setup_io_info(file_name, io_grid_); + } + + void read_variables(const int time_index, const util::TimeStamp& ts) { + iop->read_fields_from_file_for_iop(file_name, field_names, ts, field_mgr, time_index); + } + + iop_ptr_type iop; + std::string file_name; + std::vector field_names; + std::shared_ptr field_mgr; + }; + // The output is really just SPAData, but for clarity it might // help to see a SPAOutput along a SPAInput in functions signatures using SPAOutput = SPAData; - struct SPAHorizInterp { - // This structure stores the information need by SPA to conduct horizontal - // interpolation from a set of source data to horizontal locations in the - // simulation grid. - // The source_grid_loc stores the column index in the source data, - // The target_grid_loc stores the column index in the target data that will be mapped to - // The weights stores the remapping weight to be applied to the source grid data for this location - // in the target data. - SPAHorizInterp() = default; - explicit SPAHorizInterp(const ekat::Comm& comm) - { - m_comm = comm; - } - // Horizontal Remap - HorizontalMap horiz_map; - // Comm group used for SPA - ekat::Comm m_comm; - - }; // SPAHorizInterp /* ------------------------------------------------------------------------------------------- */ // SPA routines + + static std::shared_ptr + create_horiz_remapper ( + const std::shared_ptr& model_grid, + const std::string& spa_data_file, + const std::string& map_file, + const bool use_iop = false); + + static std::shared_ptr + create_spa_data_reader ( + const std::shared_ptr& horiz_remapper, + const std::string& spa_data_file); + + static std::shared_ptr + create_spa_data_reader ( + iop_ptr_type& iop, + const std::shared_ptr& horiz_remapper, + const std::string& spa_data_file); + static void spa_main( const SPATimeState& time_state, const view_2d& p_tgt, @@ -171,34 +188,22 @@ struct SPAFunctions const SPAInput& data_tmp, // Temporary const SPAOutput& data_out); - static void get_remap_weights_from_file( - const std::string& remap_file_name, - const gid_type min_dof, - const view_1d& dofs_gids, - SPAHorizInterp& spa_horiz_interp); - - static void set_remap_weights_one_to_one( - gid_type min_dof, - const view_1d& dofs_gids, - SPAHorizInterp& spa_horiz_interp); - static void update_spa_data_from_file( - const std::string& spa_data_file_name, - const int time_index, - const int nswbands, - const int nlwbands, - SPAHorizInterp& spa_horiz_interp, - SPAInput& spa_data); + std::shared_ptr& scorpio_reader, + std::shared_ptr& iop_reader, + const util::TimeStamp& ts, + const int time_index, // zero-based + AbstractRemapper& spa_horiz_interp, + SPAInput& spa_input); static void update_spa_timestate( - const std::string& spa_data_file_name, - const int nswbands, - const int nlwbands, - const util::TimeStamp& ts, - SPAHorizInterp& spa_horiz_interp, - SPATimeState& time_state, - SPAInput& spa_beg, - SPAInput& spa_end); + std::shared_ptr& scorpio_reader, + std::shared_ptr& iop_reader, + const util::TimeStamp& ts, + AbstractRemapper& spa_horiz_interp, + SPATimeState& time_state, + SPAInput& spa_beg, + SPAInput& spa_end); // The following three are called during spa_main static void perform_time_interpolation ( diff --git a/components/eamxx/src/physics/spa/spa_functions_impl.hpp b/components/eamxx/src/physics/spa/spa_functions_impl.hpp index 665f4537083c..748983006a19 100644 --- a/components/eamxx/src/physics/spa/spa_functions_impl.hpp +++ b/components/eamxx/src/physics/spa/spa_functions_impl.hpp @@ -1,27 +1,27 @@ #ifndef SPA_FUNCTIONS_IMPL_HPP #define SPA_FUNCTIONS_IMPL_HPP -#include "share/scream_types.hpp" -#include "share/io/scream_scorpio_interface.hpp" -#include "share/io/scorpio_input.hpp" -#include "share/grid/point_grid.hpp" #include "physics/share/physics_constants.hpp" +#include "share/grid/remap/coarsening_remapper.hpp" +#include "share/grid/remap/refining_remapper_p2p.hpp" +#include "share/grid/remap/identity_remapper.hpp" +#include "share/io/scream_scorpio_interface.hpp" +#include "share/util/scream_timing.hpp" +#include "share/scream_types.hpp" -#include "ekat/kokkos/ekat_subview_utils.hpp" -#include "ekat/util/ekat_lin_interp.hpp" -#include "ekat/ekat_pack_utils.hpp" -#include "ekat/ekat_parse_yaml_file.hpp" - -#include +#include +#include +#include +#include +#include -#include "share/util/scream_timing.hpp" /*----------------------------------------------------------------- * The main SPA routines used to convert SPA data into a format that * is usable by the rest of the atmosphere processes. * * SPA or Simple Prescribed Aerosols provides a way to prescribe * aerosols for an atmospheric simulation using pre-computed data. - * + * * The data is typically provided at a frequency of monthly, and * does not necessarily have to be on the same horizontal or vertical * domain as the atmospheric simulation. @@ -61,12 +61,119 @@ * temporally interpolated in the last step and the set of hybrid coordinates (hyam and hybm) * that are used in EAM to construct the physics pressure profiles. * The SPA data is then projected onto the simulation pressure profile (pmid) - * using EKAT linear interpolation. + * using EKAT linear interpolation. -----------------------------------------------------------------*/ namespace scream { namespace spa { +template +std::shared_ptr +SPAFunctions:: +create_horiz_remapper ( + const std::shared_ptr& model_grid, + const std::string& spa_data_file, + const std::string& map_file, + const bool use_iop) +{ + using namespace ShortFieldTagsNames; + + scorpio::register_file(spa_data_file,scorpio::Read); + const int nlevs_data = scorpio::get_dimlen(spa_data_file,"lev"); + const int ncols_data = scorpio::get_dimlen(spa_data_file,"ncol"); + const int nswbands = scorpio::get_dimlen(spa_data_file,"swband"); + const int nlwbands = scorpio::get_dimlen(spa_data_file,"lwband"); + scorpio::eam_pio_closefile(spa_data_file); + + // We could use model_grid directly if using same num levels, + // but since shallow clones are cheap, we may as well do it (less lines of code) + auto horiz_interp_tgt_grid = model_grid->clone("spa_horiz_interp_tgt_grid",true); + horiz_interp_tgt_grid->reset_num_vertical_lev(nlevs_data); + + const int ncols_model = model_grid->get_num_global_dofs(); + std::shared_ptr remapper; + if (ncols_data==ncols_model + or + use_iop /*IOP class defines it's own remapper for file data*/) { + remapper = std::make_shared(horiz_interp_tgt_grid,IdentityRemapper::SrcAliasTgt); + } else { + EKAT_REQUIRE_MSG (ncols_data<=ncols_model, + "Error! We do not allow to coarsen spa data to fit the model. We only allow\n" + " spa data to be at the same or coarser resolution as the model.\n"); + // We must have a valid map file + EKAT_REQUIRE_MSG (map_file!="", + "ERROR: Spa data is on a different grid than the model one,\n" + " but spa_remap_file is missing from SPA parameter list."); + + remapper = std::make_shared(horiz_interp_tgt_grid,map_file); + } + + remapper->registration_begins(); + + const auto tgt_grid = remapper->get_tgt_grid(); + + const auto layout_2d = tgt_grid->get_2d_scalar_layout(); + const auto layout_ccn3 = tgt_grid->get_3d_scalar_layout(true); + const auto layout_sw = tgt_grid->get_3d_vector_layout(true,SWBND,nswbands); + const auto layout_lw = tgt_grid->get_3d_vector_layout(true,LWBND,nlwbands); + const auto nondim = ekat::units::Units::nondimensional(); + + Field ps (FieldIdentifier("PS", layout_2d, nondim,tgt_grid->name())); + Field ccn3 (FieldIdentifier("CCN3", layout_ccn3,nondim,tgt_grid->name())); + Field aero_g_sw (FieldIdentifier("AER_G_SW", layout_sw, nondim,tgt_grid->name())); + Field aero_ssa_sw (FieldIdentifier("AER_SSA_SW",layout_sw, nondim,tgt_grid->name())); + Field aero_tau_sw (FieldIdentifier("AER_TAU_SW",layout_sw, nondim,tgt_grid->name())); + Field aero_tau_lw (FieldIdentifier("AER_TAU_LW",layout_lw, nondim,tgt_grid->name())); + ps.allocate_view(); + ccn3.allocate_view(); + aero_g_sw.allocate_view(); + aero_ssa_sw.allocate_view(); + aero_tau_sw.allocate_view(); + aero_tau_lw.allocate_view(); + + remapper->register_field_from_tgt (ps); + remapper->register_field_from_tgt (ccn3); + remapper->register_field_from_tgt (aero_g_sw); + remapper->register_field_from_tgt (aero_ssa_sw); + remapper->register_field_from_tgt (aero_tau_sw); + remapper->register_field_from_tgt (aero_tau_lw); + + remapper->registration_ends(); + + return remapper; +} + +template +std::shared_ptr +SPAFunctions:: +create_spa_data_reader ( + const std::shared_ptr& horiz_remapper, + const std::string& spa_data_file) +{ + std::vector io_fields; + for (int i=0; iget_num_fields(); ++i) { + io_fields.push_back(horiz_remapper->get_src_field(i)); + } + const auto io_grid = horiz_remapper->get_src_grid(); + return std::make_shared(spa_data_file,io_grid,io_fields,true); +} + +template +std::shared_ptr::IOPReader> +SPAFunctions:: +create_spa_data_reader ( + iop_ptr_type& iop, + const std::shared_ptr& horiz_remapper, + const std::string& spa_data_file) +{ + std::vector io_fields; + for (int i=0; iget_num_fields(); ++i) { + io_fields.push_back(horiz_remapper->get_src_field(i)); + } + const auto io_grid = horiz_remapper->get_src_grid(); + return std::make_shared(iop, spa_data_file, io_fields, io_grid); +} + /*-----------------------------------------------------------------*/ // The main SPA routine which handles projecting SPA data onto the // horizontal columns and vertical pressure profiles of the atmospheric @@ -82,9 +189,9 @@ namespace spa { // set of SPA data projected onto the pressure profile of the current atmosphere // state. This is the data that will be passed to other processes. // ncols_tgt, nlevs_tgt: The number of columns and levels in the simulation grid. -// (not to be confused with the number of columns and levels used for the SPA data, +// (not to be confused with the number of columns and levels used for the SPA data, // which can be different.) -// nswbands, nlwbands: The number of shortwave (sw) and longwave (lw) aerosol bands +// nswbands, nlwbands: The number of shortwave (sw) and longwave (lw) aerosol bands // for the data that will be passed to radiation. template void SPAFunctions @@ -268,7 +375,7 @@ perform_vertical_interpolation( KOKKOS_LAMBDA(typename LIV::MemberType const& team) { const int icol = team.league_rank(); - + // Setup vert_interp.setup(team, ekat::subview(p_src,icol), ekat::subview(p_tgt,icol)); @@ -295,62 +402,6 @@ perform_vertical_interpolation( Kokkos::fence(); } -/*-----------------------------------------------------------------*/ -// Function to set the remap and weights for a one-to-one mapping. -// This is used when the SPA data and the simulation grid are the -// same and no remapping is needed. -// Note: This function should be called only once during SPA::init -template -void SPAFunctions -::set_remap_weights_one_to_one( - gid_type min_dof, - const view_1d& dofs_gids, - SPAHorizInterp& spa_horiz_interp - ) -{ - // There may be cases where the SPA data is defined on the same grid as the simulation - // and thus no remapping is required. This simple routine establishes a 1-1 horizontal - // mapping - const int num_local_cols = dofs_gids.size(); - auto& spa_horiz_map = spa_horiz_interp.horiz_map; - spa_horiz_map = HorizontalMap(spa_horiz_interp.m_comm,"SPA 1-1 Remap",dofs_gids,min_dof); - view_1d src_dofs("",1); - view_1d src_wgts("",1); - auto dofs_gids_h = Kokkos::create_mirror_view(dofs_gids); - Kokkos::deep_copy(dofs_gids_h,dofs_gids); - for (int ii=0;ii -void SPAFunctions -::get_remap_weights_from_file( - const std::string& remap_file_name, - const gid_type min_dof, - const view_1d& dofs_gids, - SPAHorizInterp& spa_horiz_interp - ) -{ - start_timer("EAMxx::SPA::get_remap_weights_from_file"); - auto& spa_horiz_map = spa_horiz_interp.horiz_map; - spa_horiz_map = HorizontalMap(spa_horiz_interp.m_comm,"SPA File Remap", dofs_gids, min_dof); - spa_horiz_map.set_remap_segments_from_file(remap_file_name); - spa_horiz_map.set_unique_source_dofs(); - stop_timer("EAMxx::SPA::get_remap_weights_from_file"); - -} // END get_remap_weights_from_file /*-----------------------------------------------------------------*/ /* Note: In this routine the SPA source data is padded in the vertical * to facilitate the proper behavior at the boundaries when doing the @@ -370,7 +421,7 @@ ::get_remap_weights_from_file( * lhs of the hybrid coordinate system to ensure that the lhs of the * source pressure levels is 0.0 and the rhs is big enough to be larger than * the highest pressure in the target pressure levels. - * + * * The padding sets the lhs in a vertical column of data to 0.0. * This has the effect of ramping the top-of-model target data down to 0.0 * when the top target pressure level is smaller than the top source pressure, @@ -393,264 +444,143 @@ ::get_remap_weights_from_file( * pressure profile that is constructed in spa_main is * a) the current length, and * b) ensures that whatever the target pressure profile is, it's within the - * the range of the source data. + * the range of the source data. */ template void SPAFunctions ::update_spa_data_from_file( - const std::string& spa_data_file_name, - const int time_index, // zero-based - const int nswbands, - const int nlwbands, - SPAHorizInterp& spa_horiz_interp, - SPAInput& spa_data) + std::shared_ptr& scorpio_reader, + std::shared_ptr& iop_reader, + const util::TimeStamp& ts, + const int time_index, // zero-based + AbstractRemapper& spa_horiz_interp, + SPAInput& spa_input) { + using namespace ShortFieldTagsNames; + using ESU = ekat::ExeSpaceUtils; + using Member = typename KokkosTypes::MemberType; + start_timer("EAMxx::SPA::update_spa_data_from_file"); - // Ensure all ranks are operating independently when reading the file, so there's a copy on all ranks - auto comm = spa_horiz_interp.m_comm; - - // Use HorizontalMap to define the set of source column data we need to load - auto& spa_horiz_map = spa_horiz_interp.horiz_map; - auto unique_src_dofs = spa_horiz_map.get_unique_source_dofs(); - const int num_local_cols = spa_horiz_map.get_num_unique_dofs(); - scorpio::register_file(spa_data_file_name,scorpio::Read); - const int source_data_nlevs = scorpio::get_dimlen(spa_data_file_name,"lev"); - - // Construct local arrays to read data into - // Note, all of the views being created here are meant to hold the source resolution - // data that will need to be horizontally interpolated to the simulation grid using the remap - // data. For example, - // We will first define the data surface pressure PS_v and read that from file. - // then we will use the horizontal interpolation structure, spa_horiz_interp, to - // interpolate PS_v onto the simulation grid: PS_v -> spa_data.PS - // and so on for the other variables. + + // 1. Read from file start_timer("EAMxx::SPA::update_spa_data_from_file::read_data"); - std::vector fnames = {"hyam","hybm","PS","CCN3","AER_G_SW","AER_SSA_SW","AER_TAU_SW","AER_TAU_LW"}; - ekat::ParameterList spa_data_in_params; - spa_data_in_params.set("Field Names",fnames); - spa_data_in_params.set("Filename",spa_data_file_name); - spa_data_in_params.set("Skip_Grid_Checks",true); // We need to skip grid checks because multiple ranks may want the same column of source data. - - // Retrieve number of cols on spa_data_file. - scorpio::register_file(spa_data_file_name,scorpio::Read); - int num_global_cols = scorpio::get_dimlen(spa_data_file_name,"ncol"); - scorpio::eam_pio_closefile(spa_data_file_name); - - // Construct the grid needed for input: - auto grid = std::make_shared("grid",num_local_cols,num_global_cols,source_data_nlevs,comm); - Kokkos::deep_copy(grid->get_dofs_gids().template get_view(),unique_src_dofs); - grid->get_dofs_gids().sync_to_host(); - - // Check that padding matches source size: - EKAT_REQUIRE(source_data_nlevs+2 == spa_data.data.nlevs); - EKAT_REQUIRE_MSG(nswbands==scorpio::get_dimlen(spa_data_file_name,"swband"), - "ERROR update_spa_data_from_file: Number of SW bands in simulation doesn't match the SPA data file"); - EKAT_REQUIRE_MSG(nlwbands==scorpio::get_dimlen(spa_data_file_name,"lwband"), - "ERROR update_spa_data_from_file: Number of LW bands in simulation doesn't match the SPA data file"); - - // Constuct views to read source data in from file - typename view_1d::HostMirror hyam_v_h("hyam",source_data_nlevs); - typename view_1d::HostMirror hybm_v_h("hybm",source_data_nlevs); - view_1d PS_v("PS",num_local_cols); - view_2d CCN3_v("CCN3",num_local_cols,source_data_nlevs); - view_3d AER_G_SW_v("AER_G_SW",num_local_cols,nswbands,source_data_nlevs); - view_3d AER_SSA_SW_v("AER_SSA_SW",num_local_cols,nswbands,source_data_nlevs); - view_3d AER_TAU_SW_v("AER_TAU_SW",num_local_cols,nswbands,source_data_nlevs); - view_3d AER_TAU_LW_v("AER_TAU_LW",num_local_cols,nlwbands,source_data_nlevs); - - auto PS_v_h = Kokkos::create_mirror_view(PS_v); - auto CCN3_v_h = Kokkos::create_mirror_view(CCN3_v); - auto AER_G_SW_v_h = Kokkos::create_mirror_view(AER_G_SW_v); - auto AER_SSA_SW_v_h = Kokkos::create_mirror_view(AER_SSA_SW_v); - auto AER_TAU_SW_v_h = Kokkos::create_mirror_view(AER_TAU_SW_v); - auto AER_TAU_LW_v_h = Kokkos::create_mirror_view(AER_TAU_LW_v); - - // Set up input structure to read data from file. - using namespace ShortFieldTagsNames; - FieldLayout scalar1d_layout { {LEV}, {source_data_nlevs} }; - FieldLayout scalar2d_layout_mid { {COL}, {num_local_cols} }; - FieldLayout scalar3d_layout_mid { {COL,LEV}, {num_local_cols, source_data_nlevs} }; - FieldLayout scalar3d_swband_layout { {COL,SWBND, LEV}, {num_local_cols, nswbands, source_data_nlevs} }; - FieldLayout scalar3d_lwband_layout { {COL,LWBND, LEV}, {num_local_cols, nlwbands, source_data_nlevs} }; - std::map> host_views; - std::map layouts; - // Define each input variable we need - host_views["hyam"] = view_1d_host(hyam_v_h.data(),hyam_v_h.size()); - layouts.emplace("hyam", scalar1d_layout); - host_views["hybm"] = view_1d_host(hybm_v_h.data(),hybm_v_h.size()); - layouts.emplace("hybm", scalar1d_layout); - // - host_views["PS"] = view_1d_host(PS_v_h.data(),PS_v_h.size()); - layouts.emplace("PS", scalar2d_layout_mid); - // - host_views["CCN3"] = view_1d_host(CCN3_v_h.data(),CCN3_v_h.size()); - layouts.emplace("CCN3",scalar3d_layout_mid); - // - host_views["AER_G_SW"] = view_1d_host(AER_G_SW_v_h.data(),AER_G_SW_v_h.size()); - layouts.emplace("AER_G_SW",scalar3d_swband_layout); - // - host_views["AER_SSA_SW"] = view_1d_host(AER_SSA_SW_v_h.data(),AER_SSA_SW_v_h.size()); - layouts.emplace("AER_SSA_SW",scalar3d_swband_layout); - // - host_views["AER_TAU_SW"] = view_1d_host(AER_TAU_SW_v_h.data(),AER_TAU_SW_v_h.size()); - layouts.emplace("AER_TAU_SW",scalar3d_swband_layout); - // - host_views["AER_TAU_LW"] = view_1d_host(AER_TAU_LW_v_h.data(),AER_TAU_LW_v_h.size()); - layouts.emplace("AER_TAU_LW",scalar3d_lwband_layout); - // - - // Now that we have all the variables defined we can use the scorpio_input class to grab the data. - AtmosphereInput spa_data_input(spa_data_in_params,grid,host_views,layouts); - spa_data_input.read_variables(time_index); - spa_data_input.finalize(); - scorpio::eam_pio_closefile(spa_data_file_name); + if (iop_reader) { + iop_reader->read_variables(time_index, ts); + } else { + scorpio_reader->read_variables(time_index); + } stop_timer("EAMxx::SPA::update_spa_data_from_file::read_data"); - start_timer("EAMxx::SPA::update_spa_data_from_file::apply_remap"); - // Copy data from host back to the device views. - Kokkos::deep_copy(PS_v,PS_v_h); - Kokkos::deep_copy(CCN3_v , CCN3_v_h); - Kokkos::deep_copy(AER_G_SW_v , AER_G_SW_v_h); - Kokkos::deep_copy(AER_SSA_SW_v, AER_SSA_SW_v_h); - Kokkos::deep_copy(AER_TAU_SW_v, AER_TAU_SW_v_h); - Kokkos::deep_copy(AER_TAU_LW_v, AER_TAU_LW_v_h); - - // Apply the remap to this data - spa_horiz_map.apply_remap(PS_v,spa_data.PS); // Note PS is not padded, so remap can be applied right away - // For padded data we need create temporary arrays to store the direct remapped data, then we can add - // padding. - int tgt_ncol = spa_data.data.ncols; - int tgt_nlev = spa_data.data.nlevs-2; // Note, the spa data already accounts for padding in the nlevs, so we subtract 2 - view_2d CCN3_unpad("",tgt_ncol,tgt_nlev); - view_3d AER_G_SW_unpad("",tgt_ncol,nswbands,tgt_nlev); - view_3d AER_SSA_SW_unpad("",tgt_ncol,nswbands,tgt_nlev); - view_3d AER_TAU_SW_unpad("",tgt_ncol,nswbands,tgt_nlev); - view_3d AER_TAU_LW_unpad("",tgt_ncol,nlwbands,tgt_nlev); - // Apply remap to "unpadded" data - spa_horiz_map.apply_remap(CCN3_v,CCN3_unpad); - spa_horiz_map.apply_remap(AER_G_SW_v, AER_G_SW_unpad); - spa_horiz_map.apply_remap(AER_SSA_SW_v, AER_SSA_SW_unpad); - spa_horiz_map.apply_remap(AER_TAU_SW_v, AER_TAU_SW_unpad); - spa_horiz_map.apply_remap(AER_TAU_LW_v, AER_TAU_LW_unpad); - stop_timer("EAMxx::SPA::update_spa_data_from_file::apply_remap"); + + // 2. Run the horiz remapper (it is a do-nothing op if spa data is on same grid as model) + start_timer("EAMxx::SPA::update_spa_data_from_file::horiz_remap"); + spa_horiz_interp.remap(/*forward = */ true); + stop_timer("EAMxx::SPA::update_spa_data_from_file::horiz_remap"); + + // 3. Copy from the tgt field of the remapper into the spa_data, padding data if necessary start_timer("EAMxx::SPA::update_spa_data_from_file::copy_and_pad"); - // Copy unpadded data to SPA data structure, add padding. - // Note, all variables we map to are packed, while all the data we just loaded as - // input are in real N-D views. So we need to set the pack and index of the actual - // data ahead by one value. - // Note, we want to pad the actual source data such that - // Y[0] = 0.0, note this is handled by the deep copy above - // Y[k+1] = y[k], k = 0,source_data_nlevs (y is the data from file) - // Y[N+2] = y[N-1], N = source_data_nlevs - Kokkos::deep_copy(spa_data.data.CCN3,0.0); - Kokkos::deep_copy(spa_data.data.AER_G_SW,0.0); - Kokkos::deep_copy(spa_data.data.AER_SSA_SW,0.0); - Kokkos::deep_copy(spa_data.data.AER_TAU_SW,0.0); - Kokkos::deep_copy(spa_data.data.AER_TAU_LW,0.0); - // 2D vars - CCN3 - Kokkos::parallel_for("", tgt_ncol*tgt_nlev, KOKKOS_LAMBDA (const int& idx) { - int icol = idx / tgt_nlev; - int klev1 = idx % tgt_nlev; - int kpack = (klev1+1) / Spack::n; - int klev2 = (klev1+1) % Spack::n; - spa_data.data.CCN3(icol,kpack)[klev2] = CCN3_unpad(icol,klev1); - if (klev1 == tgt_nlev-1) { - int kpack = (tgt_nlev+1) / Spack::n; - int klev2 = (tgt_nlev+1) % Spack::n; - spa_data.data.CCN3(icol,kpack)[klev2] = CCN3_unpad(icol,klev1); - } - }); + // Recall, the fields are registered in the order: ps, ccn3, g_sw, ssa_sw, tau_sw, tau_lw + auto ps = spa_horiz_interp.get_tgt_field (0).get_view(); + auto ccn3 = spa_horiz_interp.get_tgt_field (1).get_view(); + auto aero_g_sw = spa_horiz_interp.get_tgt_field (2).get_view(); + auto aero_ssa_sw = spa_horiz_interp.get_tgt_field (3).get_view(); + auto aero_tau_sw = spa_horiz_interp.get_tgt_field (4).get_view(); + auto aero_tau_lw = spa_horiz_interp.get_tgt_field (5).get_view(); + + const auto& sw_layout = spa_horiz_interp.get_tgt_field (2).get_header().get_identifier().get_layout(); + const auto& lw_layout = spa_horiz_interp.get_tgt_field (5).get_header().get_identifier().get_layout(); + + const int ncols = sw_layout.dim(COL); + const int nlevs = sw_layout.dim(LEV); + const int nswbands = sw_layout.dim(SWBND); + const int nlwbands = lw_layout.dim(LWBND); + + Kokkos::deep_copy(spa_input.PS,ps); + Kokkos::fence(); + auto spa_data_ccn3 = ekat::scalarize(spa_input.data.CCN3); + auto spa_data_aero_g_sw = ekat::scalarize(spa_input.data.AER_G_SW); + auto spa_data_aero_ssa_sw = ekat::scalarize(spa_input.data.AER_SSA_SW); + auto spa_data_aero_tau_sw = ekat::scalarize(spa_input.data.AER_TAU_SW); + auto spa_data_aero_tau_lw = ekat::scalarize(spa_input.data.AER_TAU_LW); + + auto copy_and_pad = KOKKOS_LAMBDA (const Member& team) { + int icol = team.league_rank(); + + auto copy_col = [&](const int k) { + spa_data_ccn3(icol,k+1) = ccn3(icol,k); + for (int isw=0; isw void SPAFunctions ::update_spa_timestate( - const std::string& spa_data_file_name, - const int nswbands, - const int nlwbands, - const util::TimeStamp& ts, - SPAHorizInterp& spa_horiz_interp, - SPATimeState& time_state, - SPAInput& spa_beg, - SPAInput& spa_end) + std::shared_ptr& scorpio_reader, + std::shared_ptr& iop_reader, + const util::TimeStamp& ts, + AbstractRemapper& spa_horiz_interp, + SPATimeState& time_state, + SPAInput& spa_beg, + SPAInput& spa_end) { - // Now we check if we have to update the data that changes monthly // NOTE: This means that SPA assumes monthly data to update. Not // any other frequency. - const auto month = ts.get_month(); - if (month != time_state.current_month or !time_state.inited) { - + const auto month = ts.get_month() - 1; // Make it 0-based + if (month != time_state.current_month) { // Update the SPA time state information time_state.current_month = month; - time_state.t_beg_month = util::TimeStamp({ts.get_year(),month,1}, {0,0,0}).frac_of_year_in_days(); - time_state.days_this_month = util::days_in_month(ts.get_year(),month); + time_state.t_beg_month = util::TimeStamp({ts.get_year(),month+1,1}, {0,0,0}).frac_of_year_in_days(); + time_state.days_this_month = util::days_in_month(ts.get_year(),month+1); + + // Copy spa_end'data into spa_beg'data, and read in the new spa_end + std::swap(spa_beg,spa_end); + // Update the SPA forcing data for this month and next month - // Start by copying next months data to this months data structure. + // Start by copying next months data to this months data structure. // NOTE: If the timestep is bigger than monthly this could cause the wrong values // to be assigned. A timestep greater than a month is very unlikely so we // will proceed. - // NOTE: we use zero-based time indexing here. - update_spa_data_from_file(spa_data_file_name,time_state.current_month-1,nswbands,nlwbands,spa_horiz_interp,spa_beg); - int next_month = time_state.current_month==12 ? 1 : time_state.current_month+1; - update_spa_data_from_file(spa_data_file_name,next_month-1,nswbands,nlwbands,spa_horiz_interp,spa_end); - // If time state was not initialized it is now: - time_state.inited = true; + int next_month = (time_state.current_month + 1) % 12; + update_spa_data_from_file(scorpio_reader,iop_reader,ts,next_month,spa_horiz_interp,spa_end); } } // END updata_spa_timestate @@ -659,12 +589,12 @@ template KOKKOS_INLINE_FUNCTION auto SPAFunctions:: get_var_column (const SPAData& data, const int icol, const int ivar) - -> view_1d + -> view_1d { if (ivar==0) { return ekat::subview(data.CCN3,icol); } else { - // NOTE: if we ever get 2+ long-wave vars, you will have to do + // NOTE: if we ever get 2+ long-wave vars, you will have to do // something like // if (jvar -#include -#include -#include +#include namespace { @@ -21,7 +15,6 @@ template using view_1d = typename KokkosTypes::template view_1d; using SPAFunc = spa::SPAFunctions; -using Spack = SPAFunc::Spack; using gid_type = SPAFunc::gid_type; Real ps_func (const int t, const gid_type icol); @@ -31,44 +24,34 @@ Real aer_func (const int t, const gid_type icol, const int bnd, const int klev, TEST_CASE("spa_one_to_one_remap","spa") { // Set up the mpi communicator and init the pio subsystem - ekat::Comm spa_comm(MPI_COMM_WORLD); // MPI communicator group used for I/O set as ekat object. - MPI_Fint fcomm = MPI_Comm_c2f(spa_comm.mpi_comm()); // MPI communicator group used for I/O. In our simple test we use MPI_COMM_WORLD, however a subset could be used. - scorpio::eam_init_pio_subsystem(fcomm); // Gather the initial PIO subsystem data creater by component coupler + ekat::Comm comm(MPI_COMM_WORLD); + scorpio::eam_init_pio_subsystem(comm); std::string spa_data_file = SCREAM_DATA_DIR "/init/spa_data_for_testing.nc"; - int max_time = 3; - int ncols = 20; - int nlevs = 4; - int nswbands = 2; - int nlwbands = 3; - - // Break the test set of columns into local degrees of freedom per mpi rank - auto comm_size = spa_comm.size(); - auto comm_rank = spa_comm.rank(); - - int my_ncols = ncols/comm_size + (comm_rank < ncols%comm_size ? 1 : 0); - view_1d dofs_gids("",my_ncols); - gid_type min_dof = 0; // We will set up the dof's to start from 0 - Kokkos::parallel_for("", my_ncols, KOKKOS_LAMBDA(const int& ii) { - dofs_gids(ii) = min_dof + static_cast(comm_rank + ii*comm_size); - }); - auto dofs_gids_h = Kokkos::create_mirror_view(dofs_gids); - Kokkos::deep_copy(dofs_gids_h,dofs_gids); - // Make sure that the total set of columns has been completely broken up. - int test_total_ncols = 0; - spa_comm.all_reduce(&my_ncols,&test_total_ncols,1,MPI_SUM); - REQUIRE(test_total_ncols == ncols); - - // Set up the set of SPA structures needed to run the test - SPAFunc::SPAHorizInterp spa_horiz_interp; - spa_horiz_interp.m_comm = spa_comm; - SPAFunc::set_remap_weights_one_to_one(min_dof,dofs_gids,spa_horiz_interp); - // Make sure one_to_one remap has the correct unique columns - auto spa_horiz_map = spa_horiz_interp.horiz_map; - REQUIRE(spa_horiz_map.get_num_unique_dofs()==my_ncols); + const int ncols_model = scorpio::get_dimlen(spa_data_file,"ncol"); + const int nlevs = scorpio::get_dimlen(spa_data_file,"lev"); + const int nswbands = scorpio::get_dimlen(spa_data_file,"swband"); + const int nlwbands = scorpio::get_dimlen(spa_data_file,"lwband"); + + auto grid_model = create_point_grid("model_grid",ncols_model,nlevs,comm); + + // Create horiz remapper + auto remapper = SPAFunc::create_horiz_remapper(grid_model,spa_data_file,"SHOULD_NOT_BE_NEEDED"); + const int ncols_data = remapper->get_src_grid()->get_num_global_dofs(); + + // This test should be tailored to have same number of columns as the spa data + REQUIRE (ncols_data==ncols_model); + REQUIRE (std::dynamic_pointer_cast(remapper)); + + // Create spa data reader + auto reader = SPAFunc::create_spa_data_reader(remapper,spa_data_file); + // TODO: We can add lat/lon to spa_data_file and test the impl for IOP + util::TimeStamp dummy_ts; + std::shared_ptr dummy_iop_reader; + // Recall, SPA data is padded, so we initialize with 2 more levels than the source data file. - SPAFunc::SPAInput spa_data(dofs_gids.size(), nlevs+2, nswbands, nlwbands); + SPAFunc::SPAInput spa_data(grid_model->get_num_local_dofs(), nlevs+2, nswbands, nlwbands); // Verify that the interpolated values match the algorithm for the data and the weights. // weights(i) = 1.0 @@ -79,42 +62,51 @@ TEST_CASE("spa_one_to_one_remap","spa") // aer_ssa_sw(t,i,b,k) = i // aer_tau_sw(t,i,b,k) = b // aer_tau_lw(t,i,b,k) = k - auto ps_h = Kokkos::create_mirror_view(spa_data.PS); - auto ccn3_h = Kokkos::create_mirror_view(spa_data.data.CCN3); - auto aer_g_sw_h = Kokkos::create_mirror_view(spa_data.data.AER_G_SW); - auto aer_ssa_sw_h = Kokkos::create_mirror_view(spa_data.data.AER_SSA_SW); - auto aer_tau_sw_h = Kokkos::create_mirror_view(spa_data.data.AER_TAU_SW); - auto aer_tau_lw_h = Kokkos::create_mirror_view(spa_data.data.AER_TAU_LW); + auto ps_d = spa_data.PS; + auto ccn3_d = ekat::scalarize(spa_data.data.CCN3); + auto aer_g_sw_d = ekat::scalarize(spa_data.data.AER_G_SW); + auto aer_ssa_sw_d = ekat::scalarize(spa_data.data.AER_SSA_SW); + auto aer_tau_sw_d = ekat::scalarize(spa_data.data.AER_TAU_SW); + auto aer_tau_lw_d = ekat::scalarize(spa_data.data.AER_TAU_LW); + + auto ps_h = Kokkos::create_mirror_view(ps_d); + auto ccn3_h = Kokkos::create_mirror_view(ccn3_d); + auto aer_g_sw_h = Kokkos::create_mirror_view(aer_g_sw_d); + auto aer_ssa_sw_h = Kokkos::create_mirror_view(aer_ssa_sw_d); + auto aer_tau_sw_h = Kokkos::create_mirror_view(aer_tau_sw_d); + auto aer_tau_lw_h = Kokkos::create_mirror_view(aer_tau_lw_d); + + auto dofs_gids_h = grid_model->get_dofs_gids().get_view(); + + const int max_time = 3; for (int time_index = 0;time_indexget_num_local_dofs(); ++idof) { + gid_type glob_i = dofs_gids_h(idof); + REQUIRE(ps_h(idof) == ps_func(time_index,glob_i)); + for (int kk=0; kk -#include -#include -#include +#include namespace { @@ -26,46 +20,36 @@ Real aer_func(const int t, const int bnd, const int klev, const int ncols, const TEST_CASE("spa_read_data","spa") { - // Set up the mpi communicator and init the pio subsystem - ekat::Comm spa_comm(MPI_COMM_WORLD); // MPI communicator group used for I/O set as ekat object. - MPI_Fint fcomm = MPI_Comm_c2f(spa_comm.mpi_comm()); // MPI communicator group used for I/O. In our simple test we use MPI_COMM_WORLD, however a subset could be used. - scorpio::eam_init_pio_subsystem(fcomm); // Gather the initial PIO subsystem data creater by component coupler - - // Establish the SPA function object using SPAFunc = spa::SPAFunctions; - using Spack = SPAFunc::Spack; - using gid_type = SPAFunc::gid_type; + constexpr Real tol = std::numeric_limits::epsilon()*1000; + // Set up the mpi communicator and init the pio subsystem + ekat::Comm comm(MPI_COMM_WORLD); + scorpio::eam_init_pio_subsystem(comm); + std::string spa_data_file = SCREAM_DATA_DIR "/init/spa_data_for_testing.nc"; std::string spa_remap_file = SCREAM_DATA_DIR "/init/spa_data_for_testing.nc"; - int max_time = 3; - int ncols = 48; - int ncols_src = 20; - int nlevs = 4; - int nswbands = 2; - int nlwbands = 3; - // Break the test set of columns into local degrees of freedom per mpi rank - auto comm_size = spa_comm.size(); - auto comm_rank = spa_comm.rank(); - - int my_ncols = ncols/comm_size + (comm_rank < ncols%comm_size ? 1 : 0); - view_1d dofs_gids("",my_ncols); - gid_type min_dof = 1; // Start global-ids from 1 - Kokkos::parallel_for("", my_ncols, KOKKOS_LAMBDA(const int& ii) { - dofs_gids(ii) = min_dof + static_cast(comm_rank + ii*comm_size); - }); - // Make sure that the total set of columns has been completely broken up. - int test_total_ncols = 0; - spa_comm.all_reduce(&my_ncols,&test_total_ncols,1,MPI_SUM); - REQUIRE(test_total_ncols == ncols); - - // Set up the set of SPA structures needed to run the test - SPAFunc::SPAHorizInterp spa_horiz_interp; - spa_horiz_interp.m_comm = spa_comm; - SPAFunc::get_remap_weights_from_file(spa_remap_file,min_dof,dofs_gids,spa_horiz_interp); + + const int ncols_model = 48; + const int nlevs = scorpio::get_dimlen(spa_data_file,"lev"); + const int nswbands = scorpio::get_dimlen(spa_data_file,"swband"); + const int nlwbands = scorpio::get_dimlen(spa_data_file,"lwband"); + + auto grid_model = create_point_grid("model_grid",ncols_model,nlevs,comm); + + // Create horiz remapper + auto remapper = SPAFunc::create_horiz_remapper(grid_model,spa_data_file,spa_remap_file); + const int ncols_data = remapper->get_src_grid()->get_num_global_dofs(); + + // Create spa data reader + auto reader = SPAFunc::create_spa_data_reader(remapper,spa_data_file); + // TODO: We can add lat/lon to spa_data_file and test the impl for IOP + std::shared_ptr dummy_iop_reader; + util::TimeStamp dummy_iop_ts; + // Recall, SPA data is padded, so we initialize with 2 more levels than the source data file. - SPAFunc::SPAInput spa_data(dofs_gids.size(), nlevs+2, nswbands, nlwbands); + SPAFunc::SPAInput spa_data(grid_model->get_num_local_dofs(), nlevs+2, nswbands, nlwbands); // Verify that the interpolated values match the algorithm for the data and the weights. // weights(i) = 1 / (2**i), weights(-1) = 1 / (2**(ncols-1)) such that sum(weights) = 1., for i=0,1,2 @@ -76,45 +60,52 @@ TEST_CASE("spa_read_data","spa") // aer_ssa_sw(t,i,b,k) = i // aer_tau_sw(t,i,b,k) = b // aer_tau_lw(t,i,b,k) = k - auto ps_h = Kokkos::create_mirror_view(spa_data.PS); - auto ccn3_h = Kokkos::create_mirror_view(spa_data.data.CCN3); - auto aer_g_sw_h = Kokkos::create_mirror_view(spa_data.data.AER_G_SW); - auto aer_ssa_sw_h = Kokkos::create_mirror_view(spa_data.data.AER_SSA_SW); - auto aer_tau_sw_h = Kokkos::create_mirror_view(spa_data.data.AER_TAU_SW); - auto aer_tau_lw_h = Kokkos::create_mirror_view(spa_data.data.AER_TAU_LW); - auto dofs_gids_h = Kokkos::create_mirror_view(dofs_gids); - Kokkos::deep_copy(dofs_gids_h,dofs_gids); + auto ps_d = spa_data.PS; + auto ccn3_d = ekat::scalarize(spa_data.data.CCN3); + auto aer_g_sw_d = ekat::scalarize(spa_data.data.AER_G_SW); + auto aer_ssa_sw_d = ekat::scalarize(spa_data.data.AER_SSA_SW); + auto aer_tau_sw_d = ekat::scalarize(spa_data.data.AER_TAU_SW); + auto aer_tau_lw_d = ekat::scalarize(spa_data.data.AER_TAU_LW); + + auto ps_h = Kokkos::create_mirror_view(ps_d); + auto ccn3_h = Kokkos::create_mirror_view(ccn3_d); + auto aer_g_sw_h = Kokkos::create_mirror_view(aer_g_sw_d); + auto aer_ssa_sw_h = Kokkos::create_mirror_view(aer_ssa_sw_d); + auto aer_tau_sw_h = Kokkos::create_mirror_view(aer_tau_sw_d); + auto aer_tau_lw_h = Kokkos::create_mirror_view(aer_tau_lw_d); + + const int max_time = 3; for (int time_index = 0;time_indexget_num_local_dofs(); ++idof) { + REQUIRE(std::abs(ps_h(idof) - ps_func(time_index,ncols_data)); + using iop_ptr = std::shared_ptr; + // Base constructor to set MPI communicator and params AtmosphereProcess (const ekat::Comm& comm, const ekat::ParameterList& params); @@ -276,6 +280,11 @@ class AtmosphereProcess : public ekat::enable_shared_from_thisset_iop(iop); + } + } + protected: // Adds fid to the list of required/computed fields of the group (as a whole). diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index 71a84fd6f7d9..c1d03f5905e4 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -122,7 +122,7 @@ get_component (const int i, const bool dynamic) { "Error! 'get_component' available only for vector fields.\n" " Layout of '" + fname + "': " + e2str(get_layout_type(layout.tags())) + "\n"); - const int idim = layout.get_vector_dim(); + const int idim = layout.get_vector_component_idx(); EKAT_REQUIRE_MSG (i>=0 && i FieldLayout::get_tensor_dims () const { + EKAT_REQUIRE_MSG (is_tensor_layout(), + "Error! 'get_tensor_dims' available only for tensor layouts.\n" + " Current layout: " + to_string(*this) + "\n" + " Layout type : " + e2str(get_layout_type(m_tags)) + "\n"); + + using namespace ShortFieldTagsNames; + std::vector cmp_tags = {CMP,NGAS,SWBND,LWBND,SWGPT,ISCCPTAU,ISCCPPRS}; + + std::vector idx; + auto it = m_tags.begin(); + do { + it = std::find_first_of (it,m_tags.cend(),cmp_tags.cbegin(),cmp_tags.cend()); + if (it!=m_tags.end()) { + idx.push_back(std::distance(m_tags.begin(),it)); + ++it; + } + } while (it!=m_tags.end()); + + EKAT_REQUIRE_MSG (idx.size()==2, + "Error! Could not find a two tensor tags in the layout.\n" + " - layout: " + to_string(*this) + "\n" + " - detected tags indices: " + ekat::join(idx,",") + "\n"); + + return idx; +} + +std::vector FieldLayout::get_tensor_tags () const { + auto idx = get_tensor_dims(); + return {m_tags[idx[0]], m_tags[idx[1]]}; } FieldLayout FieldLayout::strip_dim (const FieldTag tag) const { @@ -68,6 +117,14 @@ FieldLayout FieldLayout::strip_dim (const int idim) const { return FieldLayout (t,d); } +FieldLayout FieldLayout::clone_with_different_extent (const int idim, const int extent) const +{ + FieldLayout copy(m_tags,m_dims); + copy.set_dimension(idim,extent); + + return copy; +} + void FieldLayout::set_dimension (const int idim, const int dimension) { EKAT_REQUIRE_MSG(idim>=0 && idim=0, "Error! Dimensions must be non-negative."); @@ -95,6 +152,9 @@ LayoutType get_layout_type (const std::vector& field_tags) { // Start from undefined/invalid LayoutType result = LayoutType::Invalid; + // We don't care about TimeLevel + erase (tags,TL); + if (n_element==1 && ngp==2 && n_column==0) { // A Dynamics layout @@ -126,17 +186,17 @@ LayoutType get_layout_type (const std::vector& field_tags) { std::vector lev_tags = {LEV,ILEV}; return ekat::contains(lev_tags,t); }; - auto is_vec_tag = [](const FieldTag t) { - std::vector vec_tags = {CMP,NGAS,SWBND,LWBND,SWGPT,ISCCPTAU,ISCCPPRS}; - return ekat::contains(vec_tags,t); + auto is_cmp_tag = [](const FieldTag t) { + std::vector cmp_tags = {CMP,NGAS,SWBND,LWBND,SWGPT,ISCCPTAU,ISCCPPRS}; + return ekat::contains(cmp_tags,t); }; switch (size) { case 0: result = LayoutType::Scalar2D; break; case 1: - // The only tag left should be a vec tag, 'TL', or a lev tag - if (is_vec_tag(tags[0]) or tags[0]==TL) { + // The only tag left should be a cmp tag or a lev tag + if (is_cmp_tag(tags[0])) { result = LayoutType::Vector2D; } else if (is_lev_tag(tags[0])) { result = LayoutType::Scalar3D; @@ -144,18 +204,20 @@ LayoutType get_layout_type (const std::vector& field_tags) { break; case 2: // Possible supported scenarios: - // 1) - // 2) - if ( (is_vec_tag(tags[0]) or tags[0]==TL) and is_lev_tag(tags[1]) ) { + // 1) + // 3) + // where CMP,CMP1,CMP2 are any tag in cmp_tags + if ( is_cmp_tag(tags[0]) and is_lev_tag(tags[1]) ) { result = LayoutType::Vector3D; - } else if (tags[0]==TL && is_vec_tag(tags[1]) ) { + } else if (is_cmp_tag(tags[0]) and is_cmp_tag(tags[1])) { result = LayoutType::Tensor2D; } break; case 3: // The only supported scenario is: - // 1) - if ( tags[0]==TL && is_vec_tag(tags[1]) && is_lev_tag(tags[2]) ) { + // 1) + // where CMP1,CMP2 are any tag in cmp_tags + if (is_cmp_tag(tags[0]) and is_cmp_tag(tags[1]) and is_lev_tag(tags[2])) { result = LayoutType::Tensor3D; } } diff --git a/components/eamxx/src/share/field/field_layout.hpp b/components/eamxx/src/share/field/field_layout.hpp index d0ca9b76b565..d72f8649e9ef 100644 --- a/components/eamxx/src/share/field/field_layout.hpp +++ b/components/eamxx/src/share/field/field_layout.hpp @@ -13,7 +13,7 @@ namespace scream { -// The type of the layout, that is, the kind of field it represent. +// The type of the layout, that is, the kind of field it represents. enum class LayoutType { Invalid, Scalar0D, @@ -81,7 +81,7 @@ class FieldLayout { bool has_tags (const std::vector& tags) const; // The rank is the number of tags associated to this field. - int rank () const { return m_rank; } + int rank () const { return m_rank; } int dim (const FieldTag tag) const; int dim (const int idim) const; @@ -92,23 +92,35 @@ class FieldLayout { bool are_dimensions_set () const; - // Check if this layout is that of a vector field + // Check if this layout is that of a vector/tensor field bool is_vector_layout () const; + bool is_tensor_layout () const; - // If this is the layout of a vector field, get the idx of the vector dimension + // If this is the layout of a vector field, get the idx of the + // vector (CMP, Component) dimension // Note: throws if is_vector_layout()==false. + int get_vector_component_idx () const; + // get the dimension (extent) of the vector (CMP, Component) dimension + // calls get_vector_component_idx() int get_vector_dim () const; FieldTag get_vector_tag () const; + // If this is the layout of a tensor field, get the idx of the tensor dimensions + // Note: throws if is_tensor_layout()==false. + std::vector get_tensor_dims () const; + std::vector get_tensor_tags () const; + + // Returns a copy of this layout with a given dimension stripped FieldLayout strip_dim (const FieldTag tag) const; FieldLayout strip_dim (const int idim) const; + FieldLayout clone_with_different_extent (const int idim, const int extent) const; - // ----- Setters ----- // +protected: + // Only this class is allowed to change a layout. Customers can request + // a *slightly* different layout (via strip_dim or clone_with_different_extent) void set_dimension (const int idim, const int dimension); -protected: - int m_rank; std::vector m_tags; std::vector m_dims; @@ -121,6 +133,7 @@ std::string to_string (const FieldLayout& l); // ========================== IMPLEMENTATION ======================= // +// returns extent inline int FieldLayout::dim (const FieldTag t) const { auto it = ekat::find(m_tags,t); @@ -150,10 +163,10 @@ inline long long FieldLayout::size () const { return prod; } -inline FieldTag FieldLayout::tag (const int idim) const { +inline FieldTag FieldLayout::tag (const int idim) const { ekat::error::runtime_check(idim>=0 && idim& tags) const { bool b = true; @@ -181,4 +194,3 @@ inline bool operator== (const FieldLayout& fl1, const FieldLayout& fl2) { } // namespace scream #endif // SCREAM_FIELD_LAYOUT_HPP - diff --git a/components/eamxx/src/share/field/field_tag.hpp b/components/eamxx/src/share/field/field_tag.hpp index a96f781a465c..e685399a8bdc 100644 --- a/components/eamxx/src/share/field/field_tag.hpp +++ b/components/eamxx/src/share/field/field_tag.hpp @@ -40,7 +40,13 @@ enum class FieldTag { LongWaveBand, LongWaveGpoint, IsccpTau, - IsccpPrs + IsccpPrs, + // + MAM_NumModes, + MAM_NumRefIndexReal, + MAM_NumRefIndexImag, + MAM_NumCoefficients, + MAM_NumModesInFile }; // If using tags a lot, consider adding 'using namespace ShortFieldTagsNames' @@ -65,6 +71,13 @@ namespace ShortFieldTagsNames { constexpr auto LWGPT = FieldTag::LongWaveGpoint; constexpr auto ISCCPTAU = FieldTag::IsccpTau; constexpr auto ISCCPPRS = FieldTag::IsccpPrs; + constexpr auto NMODES = FieldTag::MAM_NumModes; + // + constexpr auto NREFINDEX_REAL = FieldTag::MAM_NumRefIndexReal; + constexpr auto NREFINDEX_IM = FieldTag::MAM_NumRefIndexImag; + + constexpr auto NCOEF_NUMBER = FieldTag::MAM_NumCoefficients; + constexpr auto MODE = FieldTag::MAM_NumModesInFile; } inline std::string e2str (const FieldTag ft) { @@ -117,6 +130,21 @@ inline std::string e2str (const FieldTag ft) { case FieldTag::IsccpPrs: name = "ISCCPPRS"; break; + case FieldTag::MAM_NumModes: + name = "num_modes"; + break; + case FieldTag::MAM_NumRefIndexReal: + name = "refindex_real"; + break; + case FieldTag::MAM_NumRefIndexImag: + name = "refindex_im"; + break; + case FieldTag::MAM_NumCoefficients: + name = "coef_number"; + break; + case FieldTag::MAM_NumModesInFile: + name = "mode"; + break; default: EKAT_ERROR_MSG("Error! Unrecognized field tag."); } diff --git a/components/eamxx/src/share/grid/abstract_grid.cpp b/components/eamxx/src/share/grid/abstract_grid.cpp index a345df2f7b19..ae304f18afd8 100644 --- a/components/eamxx/src/share/grid/abstract_grid.cpp +++ b/components/eamxx/src/share/grid/abstract_grid.cpp @@ -152,10 +152,9 @@ is_valid_layout (const FieldLayout& layout) const // Let's return true early to avoid segfautls below return true; } + const bool midpoints = layout.tags().back()==LEV; - const bool is_vec = layout.is_vector_layout(); - const int vec_dim = is_vec ? layout.dims()[layout.get_vector_dim()] : 0; - const auto vec_tag = is_vec ? layout.get_vector_tag() : INV; + const bool is3d = layout.tags().back()==LEV or layout.tags().back()==ILEV; switch (lt) { case LayoutType::Scalar1D: [[fallthrough]]; @@ -163,14 +162,29 @@ is_valid_layout (const FieldLayout& layout) const // 1d layouts need the right number of levels return layout.dims().back() == m_num_vert_levs or layout.dims().back() == (m_num_vert_levs+1); - case LayoutType::Scalar2D: - return layout==get_2d_scalar_layout(); - case LayoutType::Vector2D: - return layout==get_2d_vector_layout(vec_tag,vec_dim); + case LayoutType::Scalar2D: [[fallthrough]]; case LayoutType::Scalar3D: - return layout==get_3d_scalar_layout(midpoints); + return is3d ? layout==get_3d_scalar_layout(midpoints) + : layout==get_2d_scalar_layout(); + case LayoutType::Vector2D: [[fallthrough]]; case LayoutType::Vector3D: - return layout==get_3d_vector_layout(midpoints,vec_tag,vec_dim); + { + const auto vec_dim = layout.dims()[layout.get_vector_component_idx()]; + const auto vec_tag = layout.get_vector_tag(); + return is3d ? layout==get_3d_vector_layout(midpoints,vec_tag,vec_dim) + : layout==get_2d_vector_layout(vec_tag,vec_dim); + } + case LayoutType::Tensor2D: [[fallthrough]]; + case LayoutType::Tensor3D: + { + const auto ttags = layout.get_tensor_tags(); + std::vector tdims; + for (auto idx : layout.get_tensor_dims()) { + tdims.push_back(layout.dim(idx)); + } + return is3d ? layout==get_3d_tensor_layout(midpoints,ttags,tdims) + : layout==get_2d_tensor_layout(ttags,tdims); + } default: // Anything else is probably no return false; @@ -243,6 +257,17 @@ AbstractGrid::create_geometry_data (const FieldIdentifier& fid) return f; } +void +AbstractGrid::delete_geometry_data (const std::string& name) +{ + EKAT_REQUIRE_MSG (has_geometry_data(name), + "Error! Cannot delete geometry data, since it is does not exist.\n" + " - grid name: " + this->name() + "\n" + " - geo data name: " + name + "\n"); + + m_geo_fields.erase(name); +} + void AbstractGrid::set_geometry_data (const Field& f) { @@ -501,6 +526,9 @@ void AbstractGrid::copy_data (const AbstractGrid& src, const bool shallow) m_geo_fields[name] = src.m_geo_fields.at(name).clone(); } } + + m_global_max_dof_gid = src.m_global_max_dof_gid; + m_global_min_dof_gid = src.m_global_min_dof_gid; m_is_unique = src.m_is_unique; m_is_unique_computed = src.m_is_unique_computed; } diff --git a/components/eamxx/src/share/grid/abstract_grid.hpp b/components/eamxx/src/share/grid/abstract_grid.hpp index 46cf3a9446bf..10b1b2676540 100644 --- a/components/eamxx/src/share/grid/abstract_grid.hpp +++ b/components/eamxx/src/share/grid/abstract_grid.hpp @@ -74,8 +74,13 @@ class AbstractGrid : public ekat::enable_shared_from_this FieldLayout get_vertical_layout (const bool midpoints) const; virtual FieldLayout get_2d_scalar_layout () const = 0; virtual FieldLayout get_2d_vector_layout (const FieldTag vector_tag, const int vector_dim) const = 0; + virtual FieldLayout get_2d_tensor_layout (const std::vector& cmp_tags, + const std::vector& cmp_dims) const = 0; virtual FieldLayout get_3d_scalar_layout (const bool midpoints) const = 0; virtual FieldLayout get_3d_vector_layout (const bool midpoints, const FieldTag vector_tag, const int vector_dim) const = 0; + virtual FieldLayout get_3d_tensor_layout (const bool midpoints, + const std::vector& cmp_tags, + const std::vector& cmp_dims) const = 0; int get_num_vertical_levels () const { return m_num_vert_levs; } @@ -120,6 +125,7 @@ class AbstractGrid : public ekat::enable_shared_from_this // Sets pre-existing field as geometry data. void set_geometry_data (const Field& f); + void delete_geometry_data (const std::string& name); bool has_geometry_data (const std::string& name) const { return m_geo_fields.find(name)!=m_geo_fields.end(); diff --git a/components/eamxx/src/share/grid/point_grid.cpp b/components/eamxx/src/share/grid/point_grid.cpp index fb3236a4a01c..c1db5c795e34 100644 --- a/components/eamxx/src/share/grid/point_grid.cpp +++ b/components/eamxx/src/share/grid/point_grid.cpp @@ -55,6 +55,20 @@ PointGrid::get_2d_vector_layout (const FieldTag vector_tag, const int vector_dim return FieldLayout({COL,vector_tag},{get_num_local_dofs(),vector_dim}); } +FieldLayout +PointGrid::get_2d_tensor_layout (const std::vector& cmp_tags, + const std::vector& cmp_dims) const +{ + using namespace ShortFieldTagsNames; + + std::vector tags = {COL}; + std::vector dims = {get_num_local_dofs()}; + + tags.insert(tags.end(),cmp_tags.begin(),cmp_tags.end()); + dims.insert(dims.end(),cmp_dims.begin(),cmp_dims.end()); + return FieldLayout(tags,dims); +} + FieldLayout PointGrid::get_3d_scalar_layout (const bool midpoints) const { @@ -77,6 +91,26 @@ PointGrid::get_3d_vector_layout (const bool midpoints, const FieldTag vector_tag return FieldLayout({COL,vector_tag,VL},{get_num_local_dofs(),vector_dim,nvl}); } +FieldLayout +PointGrid::get_3d_tensor_layout (const bool midpoints, + const std::vector& cmp_tags, + const std::vector& cmp_dims) const +{ + using namespace ShortFieldTagsNames; + + int nvl = this->get_num_vertical_levels() + (midpoints ? 0 : 1); + auto VL = midpoints ? LEV : ILEV; + + std::vector tags = {COL}; + std::vector dims = {get_num_local_dofs()}; + + tags.insert(tags.end(),cmp_tags.begin(),cmp_tags.end()); + dims.insert(dims.end(),cmp_dims.begin(),cmp_dims.end()); + tags.push_back(VL); + dims.push_back(nvl); + return FieldLayout(tags,dims); +} + std::shared_ptr PointGrid::clone (const std::string& clone_name, const bool shallow) const diff --git a/components/eamxx/src/share/grid/point_grid.hpp b/components/eamxx/src/share/grid/point_grid.hpp index 3a3d426d1cf9..a66bf5d7d764 100644 --- a/components/eamxx/src/share/grid/point_grid.hpp +++ b/components/eamxx/src/share/grid/point_grid.hpp @@ -45,8 +45,13 @@ class PointGrid : public AbstractGrid // E.g., for a 2d structured grid, this could be a set of 2 indices. FieldLayout get_2d_scalar_layout () const override; FieldLayout get_2d_vector_layout (const FieldTag vector_tag, const int vector_dim) const override; + FieldLayout get_2d_tensor_layout (const std::vector& cmp_tags, + const std::vector& cmp_dims) const override; FieldLayout get_3d_scalar_layout (const bool midpoints) const override; FieldLayout get_3d_vector_layout (const bool midpoints, const FieldTag vector_tag, const int vector_dim) const override; + FieldLayout get_3d_tensor_layout (const bool midpoints, + const std::vector& cmp_tags, + const std::vector& cmp_dims) const override; FieldTag get_partitioned_dim_tag () const override { return FieldTag::Column; diff --git a/components/eamxx/src/share/grid/remap/abstract_remapper.cpp b/components/eamxx/src/share/grid/remap/abstract_remapper.cpp index 6d25787794fa..80df739aa4f4 100644 --- a/components/eamxx/src/share/grid/remap/abstract_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/abstract_remapper.cpp @@ -59,6 +59,34 @@ register_field (const field_type& src, const field_type& tgt) { bind_field(src,tgt); } +void AbstractRemapper:: +register_field_from_src (const field_type& src) { + const auto& src_fid = src.get_header().get_identifier(); + const auto& tgt_fid = create_tgt_fid(src_fid); + + Field tgt(tgt_fid); + const auto& src_ap = src.get_header().get_alloc_properties(); + auto& tgt_ap = tgt.get_header().get_alloc_properties(); + tgt_ap.request_allocation(src_ap.get_largest_pack_size()); + tgt.allocate_view(); + + register_field(src,tgt); +} + +void AbstractRemapper:: +register_field_from_tgt (const field_type& tgt) { + const auto& tgt_fid = tgt.get_header().get_identifier(); + const auto& src_fid = create_src_fid(tgt_fid); + + Field src(src_fid); + const auto& tgt_ap = tgt.get_header().get_alloc_properties(); + auto& src_ap = src.get_header().get_alloc_properties(); + src_ap.request_allocation(tgt_ap.get_largest_pack_size()); + src.allocate_view(); + + register_field(src,tgt); +} + void AbstractRemapper:: bind_field (const field_type& src, const field_type& tgt) { EKAT_REQUIRE_MSG(m_state!=RepoState::Clean, diff --git a/components/eamxx/src/share/grid/remap/abstract_remapper.hpp b/components/eamxx/src/share/grid/remap/abstract_remapper.hpp index afbe63b04a84..af80889add1b 100644 --- a/components/eamxx/src/share/grid/remap/abstract_remapper.hpp +++ b/components/eamxx/src/share/grid/remap/abstract_remapper.hpp @@ -56,6 +56,9 @@ class AbstractRemapper register_field(create_src_fid(tgt),tgt); } + virtual void register_field_from_src (const field_type& src); + virtual void register_field_from_tgt (const field_type& tgt); + // Call this to indicate that field registration is complete. void registration_ends (); diff --git a/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp b/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp index c05ea30efdcc..c798cd9a31f3 100644 --- a/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp @@ -15,45 +15,53 @@ namespace scream CoarseningRemapper:: CoarseningRemapper (const grid_ptr_type& src_grid, const std::string& map_file, - const bool track_mask) + const bool track_mask, + const bool populate_tgt_grid_geo_data) : HorizInterpRemapperBase (src_grid,map_file,InterpType::Coarsen) , m_track_mask (track_mask) { using namespace ShortFieldTagsNames; - // Replicate the src grid geo data in the tgt grid. We use this remapper to do - // the remapping (if needed), and clean it up afterwards. - const auto& src_geo_data_names = src_grid->get_geometry_data_names(); - registration_begins(); - for (const auto& name : src_geo_data_names) { - const auto& src_data = src_grid->get_geometry_data(name); - const auto& src_data_fid = src_data.get_header().get_identifier(); - const auto& layout = src_data_fid.get_layout(); - if (layout.tags()[0]!=COL) { - // Not a field to be coarsened (perhaps a vertical coordinate field). - // Simply copy it in the tgt grid, but we still need to assign the new grid name. - FieldIdentifier tgt_data_fid(src_data_fid.name(),src_data_fid.get_layout(),src_data_fid.get_units(),m_tgt_grid->name()); - auto tgt_data = m_coarse_grid->create_geometry_data(tgt_data_fid); - tgt_data.deep_copy(src_data); - } else { - // This field needs to be remapped - auto tgt_data_fid = create_tgt_fid(src_data_fid); - auto tgt_data = m_coarse_grid->create_geometry_data(tgt_data_fid); - register_field(src_data,tgt_data); + if (populate_tgt_grid_geo_data) { + // Replicate the src grid geo data in the tgt grid. We use this remapper to do + // the remapping (if needed), and clean it up afterwards. + const auto& src_geo_data_names = src_grid->get_geometry_data_names(); + registration_begins(); + for (const auto& name : src_geo_data_names) { + // Since different remappers may share the same data (if the map file is the same) + // the coarse grid may already have the geo data. + if (m_coarse_grid->has_geometry_data(name)) { + continue; + } + const auto& src_data = src_grid->get_geometry_data(name); + const auto& src_data_fid = src_data.get_header().get_identifier(); + const auto& layout = src_data_fid.get_layout(); + if (layout.tags()[0]!=COL) { + // Not a field to be coarsened (perhaps a vertical coordinate field). + // Simply copy it in the tgt grid, but we still need to assign the new grid name. + FieldIdentifier tgt_data_fid(src_data_fid.name(),src_data_fid.get_layout(),src_data_fid.get_units(),m_tgt_grid->name()); + auto tgt_data = m_coarse_grid->create_geometry_data(tgt_data_fid); + tgt_data.deep_copy(src_data); + } else { + // This field needs to be remapped + auto tgt_data_fid = create_tgt_fid(src_data_fid); + auto tgt_data = m_coarse_grid->create_geometry_data(tgt_data_fid); + register_field(src_data,tgt_data); + } } - } - registration_ends(); - if (get_num_fields()>0) { - remap(true); - - // The remap phase only alters the fields on device. - // We need to sync them to host as well - for (int i=0; i0) { + remap(true); + + // The remap phase only alters the fields on device. + // We need to sync them to host as well + for (int i=0; i(); + bool mask1d = mask.rank()==1; + view_1d mask_1d; + view_2d mask_2d; + // If the mask comes from FieldAtLevel, it's only defined on columns (rank=1) + // If the mask comes from vert interpolation remapper, it is defined on ncols x nlevs (rank=2) + if (mask.rank()==1) { + mask_1d = mask.get_view(); + } else { + mask_2d = mask.get_view(); + } + const int dim1 = layout.dim(1); + const int dim2 = layout.dim(2); + const int dim3 = PackInfo::num_packs(layout.dim(3)); + auto policy = ESU::get_default_team_policy(ncols,dim1*dim2*dim3); + Kokkos::parallel_for(policy, + KOKKOS_LAMBDA(const MemberType& team) { + const auto icol = team.league_rank(); + if (mask1d) { + auto mask = mask_1d(icol); + if (mask>mask_threshold) { + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,dim1*dim2*dim3), + [&](const int idx){ + const int j = (idx / dim3) / dim2; + const int k = (idx / dim3) % dim2; + const int l = idx % dim3; + auto x_sub = ekat::subview(x_view,icol,j,k); + x_sub(l) /= mask; + }); + } + } else { + auto m_sub = ekat::subview(mask_2d,icol); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,dim1*dim2*dim3), + [&](const int idx){ + const int j = (idx / dim3) / dim2; + const int k = (idx / dim3) % dim2; + const int l = idx % dim3; + auto x_sub = ekat::subview(x_view,icol,j,k); + auto masked = m_sub(l) > mask_threshold; + + if (masked.any()) { + x_sub(l).set(masked,x_sub(l)/m_sub(l)); + } + x_sub(l).set(!masked,mask_val); + }); + } + }); + break; + } } } @@ -458,6 +517,46 @@ local_mat_vec (const Field& x, const Field& y, const Field& mask) const }); break; } + case 4: + { + auto x_view = x.get_view(); + auto y_view = y.get_view< Pack****>(); + // Note, the mask is still assumed to be defined on COLxLEV so still only 2D for case 3. + view_1d mask_1d; + view_2d mask_2d; + bool mask1d = mask.rank()==1; + // If the mask comes from FieldAtLevel, it's only defined on columns (rank=1) + // If the mask comes from vert interpolation remapper, it is defined on ncols x nlevs (rank=2) + if (mask1d) { + mask_1d = mask.get_view(); + } else { + mask_2d = mask.get_view(); + } + const int dim1 = src_layout.dim(1); + const int dim2 = src_layout.dim(2); + const int dim3 = PackInfo::num_packs(src_layout.dim(3)); + auto policy = ESU::get_default_team_policy(nrows,dim1*dim2*dim3); + Kokkos::parallel_for(policy, + KOKKOS_LAMBDA(const MemberType& team) { + const auto row = team.league_rank(); + + const auto beg = row_offsets(row); + const auto end = row_offsets(row+1); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,dim1*dim2*dim3), + [&](const int idx){ + const int j = (idx / dim3) / dim2; + const int k = (idx / dim3) % dim2; + const int l = idx % dim3; + y_view(row,j,k,l) = weights(beg)*x_view(col_lids(beg),j,k,l) * + (mask1d ? mask_1d (col_lids(beg)) : mask_2d(col_lids(beg),l)); + for (int icol=beg+1; icol(); + const int dim1 = fl.dim(1); + const int dim2 = fl.dim(2); + const int dim3 = fl.dim(3); + auto policy = ESU::get_default_team_policy(num_send_gids,dim1*dim2*dim3); + Kokkos::parallel_for(policy, + KOKKOS_LAMBDA(const MemberType& team){ + const int i = team.league_rank(); + const int lid = lids_pids(i,0); + const int pid = lids_pids(i,1); + const int lidpos = i - pid_lid_start(pid); + const int offset = f_pid_offsets(pid); + + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,dim1*dim2*dim3), + [&](const int idx) { + const int idim = (idx / dim3) / dim2; + const int jdim = (idx / dim3) % dim2; + const int ilev = idx % dim3; + buf(offset + lidpos*dim1*dim2*dim3 + idim*dim2*dim3 + jdim*dim3 + ilev) = v(lid,idim,jdim,ilev); + }); + }); + } break; default: EKAT_ERROR_MSG ("Unexpected field rank in CoarseningRemapper::pack.\n" @@ -659,6 +782,34 @@ void CoarseningRemapper::recv_and_unpack () }); } break; + case 4: + { + auto v = f.get_view(); + const int dim1 = fl.dim(1); + const int dim2 = fl.dim(2); + const int dim3 = fl.dim(3); + auto policy = ESU::get_default_team_policy(num_tgt_dofs,dim1*dim2*dim3); + Kokkos::parallel_for(policy, + KOKKOS_LAMBDA(const MemberType& team){ + const int lid = team.league_rank(); + const int recv_beg = recv_lids_beg(lid); + const int recv_end = recv_lids_end(lid); + for (int irecv=recv_beg; irecv>& pid2gids_send) const void CoarseningRemapper::setup_mpi_data_structures () { using namespace ShortFieldTagsNames; + using gid_type = AbstractGrid::gid_type; const auto mpi_comm = m_comm.mpi_comm(); const auto mpi_real = ekat::get_mpi_type(); diff --git a/components/eamxx/src/share/grid/remap/coarsening_remapper.hpp b/components/eamxx/src/share/grid/remap/coarsening_remapper.hpp index 6c8088ead14b..8146fff8acf6 100644 --- a/components/eamxx/src/share/grid/remap/coarsening_remapper.hpp +++ b/components/eamxx/src/share/grid/remap/coarsening_remapper.hpp @@ -44,7 +44,8 @@ class CoarseningRemapper : public HorizInterpRemapperBase CoarseningRemapper (const grid_ptr_type& src_grid, const std::string& map_file, - const bool track_mask = false); + const bool track_mask = false, + const bool populate_tgt_grid_geo_data = true); ~CoarseningRemapper (); diff --git a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp index cb65bdda88d6..5fa1e46ba646 100644 --- a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp +++ b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp @@ -16,6 +16,7 @@ HorizInterpRemapperBase (const grid_ptr_type& fine_grid, const std::string& map_file, const InterpType type) : m_fine_grid(fine_grid) + , m_map_file (map_file) , m_type (type) , m_comm (fine_grid->get_comm()) { @@ -25,27 +26,66 @@ HorizInterpRemapperBase (const grid_ptr_type& fine_grid, " - fine grid name: " + fine_grid->name() + "\n" " - fine_grid_type: " + e2str(fine_grid->type()) + "\n"); EKAT_REQUIRE_MSG (fine_grid->is_unique(), - "Error! CoarseningRemapper requires a unique source grid.\n"); + "Error! HorizInterpRemapperBase requires a unique fine grid.\n"); // This is a special remapper. We only go in one direction m_bwd_allowed = false; - // Read the map file, loading the triplets this rank needs for the crs matrix - // in the map file that this rank has to read - auto my_triplets = get_my_triplets (map_file); + // Get the remap data (if not already present, it will be built) + auto& data = s_remapper_data[m_map_file]; + if (data.num_customers==0) { + data.build(m_map_file,m_fine_grid,m_comm,m_type); + } + ++data.num_customers; + + m_row_offsets = data.row_offsets; + m_col_lids = data.col_lids; + m_weights = data.weights; + + // The grids really only matter for the horiz part. We may have 2+ remappers with + // fine grids that only differ in terms of number of levs. Such remappers cannot + // store the same coarse grid. So we soft-clone the grid, and reset the number of levels + auto coarse_grid = data.coarse_grid->clone(data.coarse_grid->name(),true); + auto ov_coarse_grid = data.ov_coarse_grid->clone(data.ov_coarse_grid->name(),true); - // Create coarse/ov_coarse grids - create_coarse_grids (my_triplets); + // Reset num levs, and remove any geo data that depends on levs + using namespace ShortFieldTagsNames; + for (std::shared_ptr grid : {coarse_grid,ov_coarse_grid}) { + grid->reset_num_vertical_lev(fine_grid->get_num_vertical_levels()); + for (const auto& name : grid->get_geometry_data_names()) { + const auto& f = grid->get_geometry_data(name); + const auto& fl = f.get_header().get_identifier().get_layout(); + if (fl.has_tag(LEV) or fl.has_tag(ILEV)) { + grid->delete_geometry_data(name); + } + } + } + m_coarse_grid = coarse_grid; + m_ov_coarse_grid = ov_coarse_grid; - // Set src/tgt grid, based on interpolation type if (m_type==InterpType::Refine) { - set_grids (m_coarse_grid,m_fine_grid); + set_grids(m_coarse_grid,m_fine_grid); } else { - set_grids (m_fine_grid,m_coarse_grid); + set_grids(m_fine_grid,m_coarse_grid); + } +} + +HorizInterpRemapperBase:: +~HorizInterpRemapperBase () +{ + auto it = s_remapper_data.find(m_map_file); + if (it==s_remapper_data.end()) { + // This would be very suspicious. But since the error is "benign", + // and since we want to avoid throwing inside a destructor, just issue a warning. + std::cerr << "WARNING! Remapper data for this map file was already deleted!\n" + " - map file: " << m_map_file << "\n"; + return; } - // Create crs matrix - create_crs_matrix_structures (my_triplets); + --it->second.num_customers; + if (it->second.num_customers==0) { + s_remapper_data.erase(it); + } } FieldLayout HorizInterpRemapperBase:: @@ -58,28 +98,7 @@ create_src_layout (const FieldLayout& tgt_layout) const "[HorizInterpRemapperBase] Error! Input target layout is not valid for this remapper.\n" " - input layout: " + to_string(tgt_layout)); - using namespace ShortFieldTagsNames; - const auto lt = get_layout_type(tgt_layout.tags()); - const bool midpoints = tgt_layout.has_tag(LEV); - const int vec_dim = tgt_layout.is_vector_layout() ? tgt_layout.dim(CMP) : -1; - auto src = FieldLayout::invalid(); - switch (lt) { - case LayoutType::Scalar2D: - src = m_src_grid->get_2d_scalar_layout(); - break; - case LayoutType::Vector2D: - src = m_src_grid->get_2d_vector_layout(CMP,vec_dim); - break; - case LayoutType::Scalar3D: - src = m_src_grid->get_3d_scalar_layout(midpoints); - break; - case LayoutType::Vector3D: - src = m_src_grid->get_3d_vector_layout(midpoints,CMP,vec_dim); - break; - default: - EKAT_ERROR_MSG ("Layout not supported by CoarseningRemapper: " + e2str(lt) + "\n"); - } - return src; + return create_layout (tgt_layout, m_src_grid); } FieldLayout HorizInterpRemapperBase:: @@ -92,28 +111,55 @@ create_tgt_layout (const FieldLayout& src_layout) const "[HorizInterpRemapperBase] Error! Input source layout is not valid for this remapper.\n" " - input layout: " + to_string(src_layout)); + return create_layout (src_layout, m_tgt_grid); +} + +FieldLayout HorizInterpRemapperBase:: +create_layout (const FieldLayout& fl_in, + const grid_ptr_type& grid) const +{ using namespace ShortFieldTagsNames; - const auto lt = get_layout_type(src_layout.tags()); - auto tgt = FieldLayout::invalid(); - const bool midpoints = src_layout.has_tag(LEV); - const int vec_dim = src_layout.is_vector_layout() ? src_layout.dim(CMP) : -1; - switch (lt) { - case LayoutType::Scalar2D: - tgt = m_tgt_grid->get_2d_scalar_layout(); - break; - case LayoutType::Vector2D: - tgt = m_tgt_grid->get_2d_vector_layout(CMP,vec_dim); - break; + const auto type = get_layout_type(fl_in.tags()); + auto fl_out = FieldLayout::invalid(); + const bool midpoints = fl_in.has_tag(LEV); + const bool is3d = fl_in.has_tag(LEV) or fl_in.has_tag(ILEV); + switch (type) { + case LayoutType::Scalar2D: [[ fallthrough ]]; case LayoutType::Scalar3D: - tgt = m_tgt_grid->get_3d_scalar_layout(midpoints); + fl_out = is3d + ? grid->get_3d_scalar_layout(midpoints) + : grid->get_2d_scalar_layout(); break; + case LayoutType::Vector2D: [[ fallthrough ]]; case LayoutType::Vector3D: - tgt = m_tgt_grid->get_3d_vector_layout(midpoints,CMP,vec_dim); + { + auto vtag = fl_in.get_vector_tag(); + auto vdim = fl_in.dim(vtag); + fl_out = is3d + ? grid->get_3d_vector_layout(midpoints,vtag,vdim) + : grid->get_2d_vector_layout(vtag,vdim); break; + } + + case LayoutType::Tensor2D: [[ fallthrough ]]; + case LayoutType::Tensor3D: + { + auto ttags = fl_in.get_tensor_tags(); + std::vector tdims; + for (auto idx : fl_in.get_tensor_dims()) { + tdims.push_back(fl_in.dim(idx)); + } + fl_out = is3d + ? grid->get_3d_tensor_layout(midpoints,ttags,tdims) + : grid->get_2d_tensor_layout(ttags,tdims); + break; + } + default: - EKAT_ERROR_MSG ("Layout not supported by CoarseningRemapper: " + e2str(lt) + "\n"); + EKAT_ERROR_MSG ("Layout not supported by HorizInterpRemapperBase:\n" + " - layout: " + to_string(fl_in) + "\n"); } - return tgt; + return fl_out; } void HorizInterpRemapperBase::do_registration_ends () @@ -159,222 +205,6 @@ do_bind_field (const int ifield, const field_type& src, const field_type& tgt) } } -auto HorizInterpRemapperBase:: -get_my_triplets (const std::string& map_file) const - -> std::vector -{ - using gid_type = AbstractGrid::gid_type; - using namespace ShortFieldTagsNames; - - // 1. Load the map file chunking it evenly across all ranks - scorpio::register_file(map_file,scorpio::FileMode::Read); - - // 1.1 Create a "helper" grid, with as many dofs as the number - // of triplets in the map file, and divided linearly across ranks - const int ngweights = scorpio::get_dimlen(map_file,"n_s"); - int nlweights = ngweights / m_comm.size(); - if (m_comm.rank() < (ngweights % m_comm.size())) { - nlweights += 1; - } - - gid_type offset = nlweights; - m_comm.scan(&offset,1,MPI_SUM); - offset -= nlweights; // scan is inclusive, but we need exclusive - - // Create a unique decomp tag, which ensures all refining remappers have - // their own decomposition - static int tag_counter = 0; - const std::string int_decomp_tag = "RR::gmtg,int,grid-idx=" + std::to_string(tag_counter++); - const std::string real_decomp_tag = "RR::gmtg,real,grid-idx=" + std::to_string(tag_counter++); - - // 1.2 Read a chunk of triplets col indices - std::vector cols(nlweights); - std::vector rows(nlweights); - std::vector S(nlweights); - - scorpio::register_variable(map_file, "col", "col", {"n_s"}, "int", int_decomp_tag); - scorpio::register_variable(map_file, "row", "row", {"n_s"}, "int", int_decomp_tag); - scorpio::register_variable(map_file, "S", "S", {"n_s"}, "real", real_decomp_tag); - - std::vector dofs_offsets(nlweights); - std::iota(dofs_offsets.begin(),dofs_offsets.end(),offset); - scorpio::set_dof(map_file,"col",nlweights,dofs_offsets.data()); - scorpio::set_dof(map_file,"row",nlweights,dofs_offsets.data()); - scorpio::set_dof(map_file,"S" ,nlweights,dofs_offsets.data()); - scorpio::set_decomp(map_file); - - scorpio::grid_read_data_array(map_file,"col",-1,cols.data(),cols.size()); - scorpio::grid_read_data_array(map_file,"row",-1,rows.data(),rows.size()); - scorpio::grid_read_data_array(map_file,"S" ,-1,S.data(),S.size()); - - scorpio::eam_pio_closefile(map_file); - - // 1.3 Dofs in grid are likely 0-based, while row/col ids in map file - // are likely 1-based. To match dofs, we need to offset the row/cols - // ids we just read in. - int map_file_min_row = std::numeric_limits::max(); - int map_file_min_col = std::numeric_limits::max(); - for (int id=0; idget_global_min_dof_gid(); - } else { - col_offset -= m_fine_grid->get_global_min_dof_gid(); - } - for (auto& id : rows) { - id -= row_offset; - } - for (auto& id : cols) { - id -= col_offset; - } - - // Create a grid based on the row gids I read in (may be duplicated across ranks) - std::vector unique_gids; - const auto& gids = m_type==InterpType::Refine ? rows : cols; - for (auto gid : gids) { - if (not ekat::contains(unique_gids,gid)) { - unique_gids.push_back(gid); - } - } - auto io_grid = std::make_shared ("helper",unique_gids.size(),0,m_comm); - auto io_grid_gids_h = io_grid->get_dofs_gids().get_view(); - int k = 0; - for (auto gid : unique_gids) { - io_grid_gids_h(k++) = gid; - } - io_grid->get_dofs_gids().sync_to_dev(); - - // Create Triplets to export, sorted by gid - std::map> io_triplets; - auto io_grid_gid2lid = io_grid->get_gid2lid_map(); - for (int i=0; i(); - auto mpi_real_t = ekat::get_mpi_type(); - int lengths[3] = {1,1,1}; - MPI_Aint displacements[3] = {0, offsetof(Triplet,col), offsetof(Triplet,w)}; - MPI_Datatype types[3] = {mpi_gid_t,mpi_gid_t,mpi_real_t}; - MPI_Datatype mpi_triplet_t; - MPI_Type_create_struct (3,lengths,displacements,types,&mpi_triplet_t); - MPI_Type_commit(&mpi_triplet_t); - - // Create import-export - GridImportExport imp_exp (m_fine_grid,io_grid); - std::map> my_triplets_map; - imp_exp.gather(mpi_triplet_t,io_triplets,my_triplets_map); - MPI_Type_free(&mpi_triplet_t); - - std::vector my_triplets; - for (auto& it : my_triplets_map) { - my_triplets.reserve(my_triplets.size()+it.second.size()); - std::move(it.second.begin(),it.second.end(),std::back_inserter(my_triplets)); - } - - return my_triplets; -} - -void HorizInterpRemapperBase:: -create_coarse_grids (const std::vector& triplets) -{ - const int nlevs = m_fine_grid->get_num_vertical_levels(); - - // Gather overlapped coarse grid gids (rows or cols, depending on m_type) - std::map ov_gid2lid; - bool pickRow = m_type==InterpType::Coarsen; - for (const auto& t : triplets) { - ov_gid2lid.emplace(pickRow ? t.row : t.col,ov_gid2lid.size()); - } - int num_ov_gids = ov_gid2lid.size(); - - // Use a temp and then assing, b/c grid_ptr_type is a pointer to const, - // so you can't modify gids using that pointer - auto ov_coarse_grid = std::make_shared("ov_coarse_grid",num_ov_gids,nlevs,m_comm); - auto ov_coarse_gids_h = ov_coarse_grid->get_dofs_gids().get_view(); - for (const auto& it : ov_gid2lid) { - ov_coarse_gids_h[it.second] = it.first; - } - ov_coarse_grid->get_dofs_gids().sync_to_dev(); - m_ov_coarse_grid = ov_coarse_grid; - - // Create the unique coarse grid - auto coarse_gids = m_ov_coarse_grid->get_unique_gids(); - int num_gids = coarse_gids.size(); - m_coarse_grid = std::make_shared("coarse_grid",num_gids,nlevs,m_comm); - auto coarse_gids_h = m_coarse_grid->get_dofs_gids().get_view(); - std::copy(coarse_gids.begin(),coarse_gids.end(),coarse_gids_h.data()); - m_coarse_grid->get_dofs_gids().sync_to_dev(); -} - -void HorizInterpRemapperBase:: -create_crs_matrix_structures (std::vector& triplets) -{ - // Get row/col data depending on interp type - bool refine = m_type==InterpType::Refine; - auto row_grid = refine ? m_fine_grid : m_ov_coarse_grid; - auto col_grid = refine ? m_ov_coarse_grid : m_fine_grid; - const int num_rows = row_grid->get_num_local_dofs(); - - auto col_gid2lid = col_grid->get_gid2lid_map(); - auto row_gid2lid = row_grid->get_gid2lid_map(); - - // Sort triplets so that row GIDs appear in the same order as - // in the row grid. If two row GIDs are the same, use same logic - // with col - auto compare = [&] (const Triplet& lhs, const Triplet& rhs) { - auto lhs_lrow = row_gid2lid.at(lhs.row); - auto rhs_lrow = row_gid2lid.at(rhs.row); - auto lhs_lcol = col_gid2lid.at(lhs.col); - auto rhs_lcol = col_gid2lid.at(rhs.col); - return lhs_lrow("",num_rows+1); - m_col_lids = view_1d("",nnz); - m_weights = view_1d("",nnz); - - auto row_offsets_h = Kokkos::create_mirror_view(m_row_offsets); - auto col_lids_h = Kokkos::create_mirror_view(m_col_lids); - auto weights_h = Kokkos::create_mirror_view(m_weights); - - // Fill col ids and weights - for (int i=0; i row_counts(num_rows); - for (int i=0; i(); + auto y_view = y.get_view< Pack****>(); + const int dim1 = src_layout.dim(1); + const int dim2 = src_layout.dim(2); + const int dim3 = PackInfo::num_packs(src_layout.dim(3)); + auto policy = ESU::get_default_team_policy(nrows,dim1*dim2*dim3); + Kokkos::parallel_for(policy, + KOKKOS_LAMBDA(const MemberType& team) { + const auto row = team.league_rank(); + + const auto beg = row_offsets(row); + const auto end = row_offsets(row+1); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,dim1*dim2*dim3), + [&](const int idx){ + const int j = (idx / dim3) / dim2; + const int k = (idx / dim3) % dim2; + const int l = idx % dim3; + y_view(row,j,k,l) = weights(beg)*x_view(col_lids(beg),j,k,l); + for (int icol=beg+1; icol HorizInterpRemapperBase::s_remapper_data; + // ETI, so derived classes can call this method template void HorizInterpRemapperBase:: diff --git a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.hpp b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.hpp index 9a91220d9a5d..c604ea085c04 100644 --- a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.hpp +++ b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.hpp @@ -2,6 +2,7 @@ #define SCREAM_HORIZ_INTERP_REMAPPER_BASE_HPP #include "share/grid/remap/abstract_remapper.hpp" +#include "share/grid/remap/horiz_interp_remapper_data.hpp" namespace scream { @@ -16,18 +17,12 @@ namespace scream class HorizInterpRemapperBase : public AbstractRemapper { -protected: - enum class InterpType { - Refine, - Coarsen - }; - public: HorizInterpRemapperBase (const grid_ptr_type& fine_grid, const std::string& map_file, const InterpType type); - virtual ~HorizInterpRemapperBase () = default; + ~HorizInterpRemapperBase (); FieldLayout create_src_layout (const FieldLayout& tgt_layout) const override; FieldLayout create_tgt_layout (const FieldLayout& src_layout) const override; @@ -44,6 +39,9 @@ class HorizInterpRemapperBase : public AbstractRemapper protected: + FieldLayout create_layout (const FieldLayout& fl_in, + const grid_ptr_type& grid) const; + const identifier_type& do_get_src_field_id (const int ifield) const override { return m_src_fields[ifield].get_header().get_identifier(); } @@ -66,32 +64,11 @@ class HorizInterpRemapperBase : public AbstractRemapper EKAT_ERROR_MSG ("HorizInterpRemapperBase only supports fwd remapping.\n"); } - using gid_type = AbstractGrid::gid_type; using KT = KokkosTypes; template using view_1d = typename KT::template view_1d; - struct Triplet { - // Note: unfortunately, C++17 does not support emplace-ing POD - // types as aggregates unless a ctor is declared. C++20 does though. - Triplet () = default; - Triplet(const gid_type rr, const gid_type cc, const Real ww) - : row(rr), col(cc), w(ww) {} - gid_type row; - gid_type col; - Real w; - }; - - std::vector - get_my_triplets (const std::string& map_file) const; - - void create_coarse_grids (const std::vector& triplets); - - // Not a const ref, since we'll sort the triplets according to - // how row gids appear in the coarse grid - void create_crs_matrix_structures (std::vector& triplets); - void create_ov_fields (); void clean_up (); @@ -129,9 +106,15 @@ class HorizInterpRemapperBase : public AbstractRemapper view_1d m_col_lids; view_1d m_weights; + // Keep track of this, since we need to tell the remap data repo + // we are releasing the data for our map file. + std::string m_map_file; + InterpType m_type; ekat::Comm m_comm; + + static std::map s_remapper_data; }; } // namespace scream diff --git a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp new file mode 100644 index 000000000000..d44fc6aa2f0f --- /dev/null +++ b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp @@ -0,0 +1,271 @@ +#include "horiz_interp_remapper_data.hpp" + +#include "share/grid/point_grid.hpp" +#include "share/grid/grid_import_export.hpp" +#include "share/io/scream_scorpio_interface.hpp" + +#include + +namespace scream { + +// --------------- HorizRemapperData ---------------- // + +void HorizRemapperData:: +build (const std::string& map_file, + const std::shared_ptr& fine_grid_in, + const ekat::Comm& comm_in, + const InterpType type_in) +{ + comm = comm_in; + fine_grid = fine_grid_in; + type = type_in; + + // Gather sparse matrix triplets needed by this rank + auto my_triplets = get_my_triplets (map_file); + + // Create coarse/ov_coarse grids + create_coarse_grids (my_triplets); + + // Create crs matrix + create_crs_matrix_structures (my_triplets); +} + +auto HorizRemapperData:: +get_my_triplets (const std::string& map_file) const + -> std::vector +{ + using gid_type = AbstractGrid::gid_type; + using namespace ShortFieldTagsNames; + + // 1. Load the map file chunking it evenly across all ranks + scorpio::register_file(map_file,scorpio::FileMode::Read); + + // 1.1 Create a "helper" grid, with as many dofs as the number + // of triplets in the map file, and divided linearly across ranks + const int ngweights = scorpio::get_dimlen(map_file,"n_s"); + int nlweights = ngweights / comm.size(); + if (comm.rank() < (ngweights % comm.size())) { + nlweights += 1; + } + + gid_type offset = nlweights; + comm.scan(&offset,1,MPI_SUM); + offset -= nlweights; // scan is inclusive, but we need exclusive + + // Create a unique decomp tag, which ensures all refining remappers have + // their own decomposition + static int tag_counter = 0; + const std::string int_decomp_tag = "RR::gmtg,int,grid-idx=" + std::to_string(tag_counter++); + const std::string real_decomp_tag = "RR::gmtg,real,grid-idx=" + std::to_string(tag_counter++); + + // 1.2 Read a chunk of triplets col indices + std::vector cols(nlweights); + std::vector rows(nlweights); + std::vector S(nlweights); + + scorpio::register_variable(map_file, "col", "col", {"n_s"}, "int", int_decomp_tag); + scorpio::register_variable(map_file, "row", "row", {"n_s"}, "int", int_decomp_tag); + scorpio::register_variable(map_file, "S", "S", {"n_s"}, "real", real_decomp_tag); + + std::vector dofs_offsets(nlweights); + std::iota(dofs_offsets.begin(),dofs_offsets.end(),offset); + scorpio::set_dof(map_file,"col",nlweights,dofs_offsets.data()); + scorpio::set_dof(map_file,"row",nlweights,dofs_offsets.data()); + scorpio::set_dof(map_file,"S" ,nlweights,dofs_offsets.data()); + scorpio::set_decomp(map_file); + + // Figure out if we are reading the right map, that is: + // - n_a or n_b matches the fine grid ncols + // - the map "direction" (fine->coarse or coarse->fine) matches m_type + const int n_a = scorpio::get_dimlen(map_file,"n_a"); + const int n_b = scorpio::get_dimlen(map_file,"n_b"); + const int ncols_fine = fine_grid->get_num_global_dofs(); + EKAT_REQUIRE_MSG (n_a==ncols_fine or n_b==ncols_fine, + "Error! The input map seems incompatible with the remapper fine grid.\n" + " - map file: " + map_file + "\n" + " - map file n_a: " + std::to_string(n_a) + "\n" + " - map file n_b: " + std::to_string(n_b) + "\n" + " - fine grid ncols: " + std::to_string(ncols_fine) + "\n"); + const bool map_is_coarsening = n_a==ncols_fine; + EKAT_REQUIRE_MSG (map_is_coarsening==(type==InterpType::Coarsen), + "Error! The input map seems incompatible with the remapper type.\n" + " - map file: " + map_file + "\n" + " - map file n_a: " + std::to_string(n_a) + "\n" + " - map file n_b: " + std::to_string(n_b) + "\n" + " - fine grid ncols: " + std::to_string(ncols_fine) + "\n" + " - remapper type: " + std::string(type==InterpType::Refine ? "refine" : "coarsen") + "\n"); + + scorpio::grid_read_data_array(map_file,"col",-1,cols.data(),nlweights); + scorpio::grid_read_data_array(map_file,"row",-1,rows.data(),nlweights); + scorpio::grid_read_data_array(map_file,"S" ,-1,S.data(),nlweights); + + scorpio::eam_pio_closefile(map_file); + + // 1.3 Dofs in grid are likely 0-based, while row/col ids in map file + // are likely 1-based. To match dofs, we need to offset the row/cols + // ids we just read in. + int map_file_min_row = std::numeric_limits::max(); + int map_file_min_col = std::numeric_limits::max(); + for (int id=0; idget_global_min_dof_gid(); + } else { + col_offset -= fine_grid->get_global_min_dof_gid(); + } + for (auto& id : rows) { + id -= row_offset; + } + for (auto& id : cols) { + id -= col_offset; + } + + // Create a grid based on the row gids I read in (may be duplicated across ranks) + std::vector unique_gids; + const auto& gids = type==InterpType::Refine ? rows : cols; + for (auto gid : gids) { + if (not ekat::contains(unique_gids,gid)) { + unique_gids.push_back(gid); + } + } + auto io_grid = std::make_shared ("helper",unique_gids.size(),0,comm); + auto io_grid_gids_h = io_grid->get_dofs_gids().get_view(); + int k = 0; + for (auto gid : unique_gids) { + io_grid_gids_h(k++) = gid; + } + io_grid->get_dofs_gids().sync_to_dev(); + + // Create Triplets to export, sorted by gid + std::map> io_triplets; + auto io_grid_gid2lid = io_grid->get_gid2lid_map(); + for (int i=0; i(); + auto mpi_real_t = ekat::get_mpi_type(); + int lengths[3] = {1,1,1}; + MPI_Aint displacements[3] = {0, offsetof(Triplet,col), offsetof(Triplet,w)}; + MPI_Datatype types[3] = {mpi_gid_t,mpi_gid_t,mpi_real_t}; + MPI_Datatype mpi_triplet_t; + MPI_Type_create_struct (3,lengths,displacements,types,&mpi_triplet_t); + MPI_Type_commit(&mpi_triplet_t); + + // Create import-export + GridImportExport imp_exp (fine_grid,io_grid); + std::map> my_triplets_map; + imp_exp.gather(mpi_triplet_t,io_triplets,my_triplets_map); + MPI_Type_free(&mpi_triplet_t); + + std::vector my_triplets; + for (auto& it : my_triplets_map) { + my_triplets.reserve(my_triplets.size()+it.second.size()); + std::move(it.second.begin(),it.second.end(),std::back_inserter(my_triplets)); + } + + return my_triplets; +} + +void HorizRemapperData:: +create_coarse_grids (const std::vector& triplets) +{ + // Gather overlapped coarse grid gids (rows or cols, depending on type) + std::map ov_gid2lid; + bool pickRow = type==InterpType::Coarsen; + for (const auto& t : triplets) { + ov_gid2lid.emplace(pickRow ? t.row : t.col,ov_gid2lid.size()); + } + int num_ov_gids = ov_gid2lid.size(); + + // Use a temp and then assing, b/c grid_ptr_type is a pointer to const, + // so you can't modify gids using that pointer + ov_coarse_grid = std::make_shared("ov_coarse_grid",num_ov_gids,0,comm); + auto ov_coarse_gids_h = ov_coarse_grid->get_dofs_gids().get_view(); + for (const auto& it : ov_gid2lid) { + ov_coarse_gids_h[it.second] = it.first; + } + auto beg = ov_coarse_gids_h.data(); + auto end = beg+ov_coarse_gids_h.size(); + std::sort(beg,end); + + ov_coarse_grid->get_dofs_gids().sync_to_dev(); + + // Create the unique coarse grid + auto coarse_gids = ov_coarse_grid->get_unique_gids(); + int num_gids = coarse_gids.size(); + coarse_grid = std::make_shared("coarse_grid",num_gids,0,comm); + auto coarse_gids_h = coarse_grid->get_dofs_gids().get_view(); + std::copy(coarse_gids.begin(),coarse_gids.end(),coarse_gids_h.data()); + coarse_grid->get_dofs_gids().sync_to_dev(); +} + +void HorizRemapperData:: +create_crs_matrix_structures (std::vector& triplets) +{ + // Get row/col data depending on interp type + bool refine = type==InterpType::Refine; + auto row_grid = refine ? fine_grid : ov_coarse_grid; + auto col_grid = refine ? ov_coarse_grid : fine_grid; + const int num_rows = row_grid->get_num_local_dofs(); + + auto col_gid2lid = col_grid->get_gid2lid_map(); + auto row_gid2lid = row_grid->get_gid2lid_map(); + + // Sort triplets so that row GIDs appear in the same order as + // in the row grid. If two row GIDs are the same, use same logic + // with col + auto compare = [&] (const Triplet& lhs, const Triplet& rhs) { + auto lhs_lrow = row_gid2lid.at(lhs.row); + auto rhs_lrow = row_gid2lid.at(rhs.row); + auto lhs_lcol = col_gid2lid.at(lhs.col); + auto rhs_lcol = col_gid2lid.at(rhs.col); + return lhs_lrow("",num_rows+1); + col_lids = view_1d("",nnz); + weights = view_1d("",nnz); + + auto row_offsets_h = Kokkos::create_mirror_view(row_offsets); + auto col_lids_h = Kokkos::create_mirror_view(col_lids); + auto weights_h = Kokkos::create_mirror_view(weights); + + // Fill col ids and weights + for (int i=0; i row_counts(num_rows); + for (int i=0; i + +#include +#include +#include + +namespace scream { + +enum class InterpType { + Refine, + Coarsen +}; + +// A small struct to hold horiz remap data, which can +// be shared across multiple horiz remappers +struct HorizRemapperData { + using KT = KokkosTypes; + template + using view_1d = typename KT::template view_1d; + + void build (const std::string& map_file, + const std::shared_ptr& fine_grid, + const ekat::Comm& comm, + const InterpType type); + + // The coarse grid data + std::shared_ptr coarse_grid; + std::shared_ptr ov_coarse_grid; + + // The CRS matrix data for online interpolation + view_1d row_offsets; + view_1d col_lids; + view_1d weights; + + int num_customers = 0; +private: + using gid_type = AbstractGrid::gid_type; + + InterpType type; + std::shared_ptr fine_grid; + ekat::Comm comm; + + struct Triplet { + // Note: unfortunately, C++17 does not support emplace-ing POD + // types as aggregates unless a ctor is declared. C++20 does though. + Triplet () = default; + Triplet(const gid_type rr, const gid_type cc, const Real ww) + : row(rr), col(cc), w(ww) {} + gid_type row; + gid_type col; + Real w; + }; + + std::vector + get_my_triplets (const std::string& map_file) const; + + void create_coarse_grids (const std::vector& triplets); + + // Not a const ref, since we'll sort the triplets according to + // how row gids appear in the coarse grid + void create_crs_matrix_structures (std::vector& triplets); +}; + +} // namespace scream + +#endif // EAMXX_HORIZ_INTERP_REMAP_DATA_HPP diff --git a/components/eamxx/src/share/grid/remap/horizontal_remap_utility.cpp b/components/eamxx/src/share/grid/remap/horizontal_remap_utility.cpp deleted file mode 100644 index c9950f126cce..000000000000 --- a/components/eamxx/src/share/grid/remap/horizontal_remap_utility.cpp +++ /dev/null @@ -1,546 +0,0 @@ -#include "share/grid/remap/horizontal_remap_utility.hpp" -#include "share/util/scream_timing.hpp" - -namespace scream { - -/*-----------------------------------------------------------------------------------------------*/ -HorizontalMap::HorizontalMap(const ekat::Comm& comm) - : m_comm (comm) -{ - m_dofs_set = false; -} -/*-----------------------------------------------------------------------------------------------*/ -HorizontalMap::HorizontalMap(const ekat::Comm& comm, const std::string& map_name) - : m_name (map_name) - , m_comm (comm) -{ - m_dofs_set = false; -} -/*-----------------------------------------------------------------------------------------------*/ -HorizontalMap::HorizontalMap(const ekat::Comm& comm, const std::string& map_name, const view_1d& dofs_gids, const gid_type min_dof) - : m_name (map_name) - , m_comm (comm) -{ - set_dof_gids(dofs_gids,min_dof); -} -/*-----------------------------------------------------------------------------------------------*/ -/*-----------------------------------------------------------------------------------------------*/ -// This function sets the remap segments for this map given a remap file that was created offline. -// Note: This function assumes that the remap file follows the convention: -// col - This variable in the file represents the set of source dofs that map to a specific target. -// row - This variable represents the corresponding list of target columns mapped to -// S - This variable stores the corresponding weights for each col -> row pair -// n_s - Is the integer number of col -> row mappings. -// for reference there may also be -// n_a - Is the size of the source grid. -// n_b - Is the size of the target grid. -// Following these conventions we assume that -// col - will be used to populate a segment's "source_dofs" -// row - will be used to populate a segment's "m_dof" -// S - will be used to populate a segment's "weights" -void HorizontalMap::set_remap_segments_from_file(const std::string& remap_filename) -{ - // Ensure each horiz remap file gets a unique decomp tag - static std::map file2idx; - if (file2idx.find(remap_filename)==file2idx.end()) { - file2idx[remap_filename] = file2idx.size(); - } - - start_timer("EAMxx::HorizontalMap::set_remap_segments_from_file"); - // Open remap file and determine the amount of data to be read - scorpio::register_file(remap_filename,scorpio::Read); - const auto remap_size = scorpio::get_dimlen(remap_filename,"n_s"); // Note, here we assume a standard format of col, row, S - // Step 1: Read in the "row" data from the file to figure out which mpi ranks care about which - // chunk of the remap data. This step reduces the memory footprint of reading in the - // map data, which can be rather large. - // Distribute responsibility for reading remap data over all ranks - const int my_rank = m_comm.rank(); - const int num_ranks = m_comm.size(); - // my_chunk will represent the chunk of data this rank will read from file. - int my_chunk = remap_size/num_ranks; - int remainder = remap_size - (my_chunk*num_ranks); - if (remainder != 0) { - my_chunk += my_rank chunks_glob(num_ranks); - m_comm.all_gather(&my_chunk,chunks_glob.data(),1); - int my_start = 0; - for (int ii=0; ii tgt_col("row",my_chunk); - auto tgt_col_h = Kokkos::create_mirror_view(tgt_col); - std::vector vec_of_dims = {"n_s"}; - std::string i_decomp = "HR::srsff,phase1,dt=int,n_s=" + std::to_string(my_chunk) + ",file-idx=" + std::to_string(file2idx[remap_filename]); - scorpio::register_variable(remap_filename, "row", "row", vec_of_dims, "int", i_decomp); - std::vector var_dof(my_chunk); - std::iota(var_dof.begin(),var_dof.end(),my_start); - scorpio::set_dof(remap_filename,"row",var_dof.size(),var_dof.data()); - scorpio::set_decomp(remap_filename); - scorpio::grid_read_data_array(remap_filename,"row",0,tgt_col_h.data(),tgt_col_h.size()); - scorpio::eam_pio_closefile(remap_filename); - // Step 2: Now that we have the data distributed among all ranks we organize the data - // into sets of target column, start location in data and length of data. - // At the same time, determine the min_dof for remap column indices. - std::vector chunk_dof, chunk_start, chunk_len; - chunk_dof.push_back(tgt_col_h(0)); - chunk_start.push_back(my_start); - chunk_len.push_back(1); - int remap_min_dof = tgt_col_h(0); - for (int ii=1; ii num_chunks_per_rank(num_ranks), chunk_displacement(num_ranks); - int total_num_chunks; - int global_remap_min_dof; - m_comm.all_gather(&num_chunks, num_chunks_per_rank.data(),1); - m_comm.all_reduce(&remap_min_dof,&global_remap_min_dof,1,MPI_MIN); - chunk_displacement[0] = 0; - total_num_chunks = num_chunks_per_rank[0]; - for (int ii=1; ii buff_dof(total_num_chunks), buff_sta(total_num_chunks), buff_len(total_num_chunks); - MPI_Allgatherv(chunk_dof.data(), chunk_dof.size(),MPI_INT,buff_dof.data(),num_chunks_per_rank.data(),chunk_displacement.data(),MPI_INT,m_comm.mpi_comm()); - MPI_Allgatherv(chunk_start.data(),chunk_dof.size(),MPI_INT,buff_sta.data(),num_chunks_per_rank.data(),chunk_displacement.data(),MPI_INT,m_comm.mpi_comm()); - MPI_Allgatherv(chunk_len.data(), chunk_dof.size(),MPI_INT,buff_len.data(),num_chunks_per_rank.data(),chunk_displacement.data(),MPI_INT,m_comm.mpi_comm()); - // Step 3: Now that all of the ranks are aware of all of the "sets" of source -> target mappings we - // construct and add segments for just the DOF's this rank cares about. - std::vector seg_dof, seg_start, seg_length; - var_dof.clear(); - auto dofs_gids_h = Kokkos::create_mirror_view(m_dofs_gids); - Kokkos::deep_copy(dofs_gids_h,m_dofs_gids); - for (int ii=0; ii var_tmp(buff_len[ii]); - std::iota(var_tmp.begin(),var_tmp.end(),buff_sta[ii]); - seg_dof.push_back(buff_dof[ii]); - seg_start.push_back(var_dof.size()); - seg_length.push_back(buff_len[ii]); - var_dof.insert(var_dof.end(),var_tmp.begin(),var_tmp.end()); - } - } - } - // Now that we know which parts of the remap file this rank cares about we can construct segments - view_1d col("col",var_dof.size()); - view_1d S("S",var_dof.size()); - auto col_h = Kokkos::create_mirror_view(col); - auto S_h = Kokkos::create_mirror_view(S); - vec_of_dims = {"n_s"}; - i_decomp = "HR::srsff,phase2,dt=int,n_s=" + std::to_string(var_dof.size()) + ",file-idx=" + std::to_string(file2idx[remap_filename]); - std::string r_decomp = "HR::srsff,phase2,dt=real,n_s=" + std::to_string(var_dof.size()) + ",file-idx=" + std::to_string(file2idx[remap_filename]); - scorpio::register_file(remap_filename,scorpio::Read); - scorpio::register_variable(remap_filename, "col", "col", vec_of_dims, "int", i_decomp); - scorpio::register_variable(remap_filename, "S", "S", vec_of_dims, "real", r_decomp); - scorpio::set_dof(remap_filename,"col",var_dof.size(),var_dof.data()); - scorpio::set_dof(remap_filename,"S",var_dof.size(),var_dof.data()); - scorpio::set_decomp(remap_filename); - scorpio::grid_read_data_array(remap_filename,"col",0,col_h.data(),col_h.size()); - scorpio::grid_read_data_array(remap_filename,"S",0,S_h.data(),S_h.size()); - scorpio::eam_pio_closefile(remap_filename); - Kokkos::deep_copy(col,col_h); - Kokkos::deep_copy(S,S_h); - // Construct segments based on data just read from file - for (size_t ii=0; ii source_dofs("",seglength); - view_1d weights("",seglength); - Kokkos::parallel_for("", seglength, KOKKOS_LAMBDA (const int& jj) { - int idx = segstart + jj; - source_dofs(jj) = col(idx)-global_remap_min_dof; // Offset to zero based dofs - weights(jj) = S(idx); - }); - HorizontalMapSegment seg(seg_dof[ii]-global_remap_min_dof,seglength,source_dofs,weights); - add_remap_segment(seg); - } - stop_timer("EAMxx::HorizontalMap::set_remap_segments_from_file"); -} -/*-----------------------------------------------------------------------------------------------*/ -// This function is used to set the internal set of degrees of freedom (dof) this map is responsible for. -// We use the global dofs, offset by the minimum global dof to make everything zero based. Note, when -// gathering remap parameters from a file, depending on the algorithm that made the file the dof -// indices may be 1-based or 0-based. By offsetting everything to 0-based we avoid potential bugs. -void HorizontalMap::set_dof_gids(const view_1d& dofs_gids, const gid_type min_dof) -{ - start_timer("EAMxx::HorizontalMap::set_dof_gids"); - EKAT_REQUIRE(dofs_gids.size()>0); - m_dofs_gids = view_1d("",dofs_gids.size()); - const auto l_dofs_gids = m_dofs_gids; - m_num_dofs = m_dofs_gids.extent(0); - Kokkos::parallel_for("", m_num_dofs, KOKKOS_LAMBDA (const int& ii) { - l_dofs_gids(ii) = dofs_gids(ii)-min_dof; - }); - m_dofs_set = true; - stop_timer("EAMxx::HorizontalMap::set_dof_gids"); -} -/*-----------------------------------------------------------------------------------------------*/ -// This function adds a remap segment to a HorizontalMap, note, we want each segment to represent a full -// remapping. This function also checks if a segment already exists for the degree of freedom -// in question. If it does then instead of add the segment to the end, this function finds that -// segment and combines them into a new comprehensive segment. -void HorizontalMap::add_remap_segment(const HorizontalMapSegment& seg) -{ - // First determine if a segment already exists in this map for the seg_dof. - gid_type seg_dof = seg.get_dof(); - int match_loc = -999; - for (int iseg=0; iseg unique_dofs; - // Check all segments and add unique dofs. Done on HOST so we can use a vector, only done once - // per map so it's alright to not be optimized for performance. - int min_gid = INT_MAX; - int max_gid = INT_MIN; - for (int iseg=0; iseg("",m_num_unique_dofs); - auto temp_h = Kokkos::create_mirror_view(m_unique_dofs); - for (int ii=0; ii& source_data, const view_1d& remapped_data) { - start_timer("EAMxx::HorizontalMap::apply_remap_1d"); - if (m_num_dofs==0) { return; } // This HorizontalMap has nothing to do for this rank. - auto remapped_data_h = Kokkos::create_mirror_view(remapped_data); - auto source_data_h = Kokkos::create_mirror_view(source_data); - Kokkos::deep_copy(source_data_h,source_data); - Kokkos::deep_copy(remapped_data_h,0.0); - for (int iseg=0; iseg& source_data, const view_2d& remapped_data) { - start_timer("EAMxx::HorizontalMap::apply_remap_2d"); - if (m_num_dofs==0) { return; } // This HorizontalMap has nothing to do for this rank. - int num_levs = source_data.extent(1); - auto remapped_data_h = Kokkos::create_mirror_view(remapped_data); - auto source_data_h = Kokkos::create_mirror_view(source_data); - Kokkos::deep_copy(source_data_h,source_data); - Kokkos::deep_copy(remapped_data_h,0.0); - for (int iseg=0; iseg& source_data, const view_3d& remapped_data) { - start_timer("EAMxx::HorizontalMap::apply_remap_3d"); - if (m_num_dofs==0) { return; } // This HorizontalMap has nothing to do for this rank. - int num_levs = source_data.extent(2); - int num_bands = source_data.extent(1); - auto remapped_data_h = Kokkos::create_mirror_view(remapped_data); - auto source_data_h = Kokkos::create_mirror_view(source_data); - Kokkos::deep_copy(source_data_h,source_data); - Kokkos::deep_copy(remapped_data_h,0.0); - for (int iseg=0; iseg("",m_length); - m_source_idx = view_1d("",m_length); - m_weights = view_1d("",m_length); - m_source_idx_h = Kokkos::create_mirror_view(m_source_idx); - m_weights_h = Kokkos::create_mirror_view(m_weights); -} -/*-----------------------------------------------------------------------------------------------*/ -HorizontalMapSegment::HorizontalMapSegment(const gid_type dof_gid, const int length, const view_1d& source_dofs, const view_1d& weights) - : m_dof (dof_gid) - , m_length (length) -{ - m_source_dofs = view_1d("",m_length); - m_weights = view_1d("",m_length); - m_source_idx = view_1d("",m_length); - Kokkos::deep_copy(m_source_dofs,source_dofs); - Kokkos::deep_copy(m_weights,weights); - m_source_idx_h = Kokkos::create_mirror_view(m_source_idx); - m_weights_h = Kokkos::create_mirror_view(m_weights); -} -/*-----------------------------------------------------------------------------------------------*/ -void HorizontalMapSegment::sync_to_host() -{ - Kokkos::deep_copy(m_source_idx_h,m_source_idx); - Kokkos::deep_copy(m_weights_h,m_weights); -} -/*-----------------------------------------------------------------------------------------------*/ -bool HorizontalMapSegment::check() const -{ - // Basic check for bounds for arrays - EKAT_REQUIRE_MSG(m_source_dofs.extent_int(0)==m_length,"Error remap segment for DOF: " + std::to_string(m_dof) + ", source_dofs view not the correct length"); - EKAT_REQUIRE_MSG(m_weights.extent_int(0)==m_length,"Error remap segment for DOF: " + std::to_string(m_dof) + ", weightss view not the correct length"); - EKAT_REQUIRE_MSG(m_source_idx.extent_int(0)==m_length,"Error remap segment for DOF: " + std::to_string(m_dof) + ", source_idx view not the correct length"); - // Check that the segment weight adds up to 1.0 - Real wgt = 0.0; - const auto weights = m_weights; - Kokkos::parallel_reduce("", m_length, KOKKOS_LAMBDA (const int& ii, Real& lsum) { - lsum += weights(ii); - },wgt); - Real tol = std::numeric_limits::epsilon() * 100.0; - if (std::abs(wgt-1.0)>=tol) { - printf("ERROR: HorizontalMap: checking remap segment for DOF = %d, total weight = %e.\n",m_dof,wgt); - return false; - } - - // If we made it this far, things all passed - return true; -} -/*-----------------------------------------------------------------------------------------------*/ -void HorizontalMapSegment::print() const -{ - printf("\n--------------------\n"); - printf("Printing information for segment with DOF = %d, DOF_idx for local decomp = %d\n",m_dof,m_dof_idx); - printf(" length = %d\n",m_length); - - auto source_dofs_h = Kokkos::create_mirror_view(m_source_dofs); - auto source_idx_h = Kokkos::create_mirror_view(m_source_idx); - auto weights_h = Kokkos::create_mirror_view(m_weights); - Kokkos::deep_copy(source_dofs_h,m_source_dofs); - Kokkos::deep_copy(source_idx_h ,m_source_idx ); - Kokkos::deep_copy(weights_h ,m_weights ); - Real total_wgt = 0.0; - printf("%10s: %10s, %10s, %s\n","ii","source dof","source idx","weight"); - for (int ii=0; ii - -namespace scream { - -/*===============================================================================================*/ -/* HorizontalMapSegment - * Lightweight structure to represent a single remapping of source data to a single target column. - * Y_target = sum_(n=1)^N ( w_n * Y_source_n ) - * See description of HorizontalMap structure for more details on the mapping. - * - * This structure is used to organize the overall horizontal remapping stored in the HorizontalMap structure. - * Each segment represents a single target column in the HorizontalMap. - * -------------------------------------- - * A.S. Donahue (LLNL): 2022-09-07 - *===============================================================================================*/ -struct HorizontalMapSegment { - - using gid_type = AbstractGrid::gid_type; - using KT = KokkosTypes; - - template - using view_1d = typename KT::template view_1d; - - template - using view_1d_host = typename KT::template view_1d::HostMirror; - - // Constructors/Destructor - HorizontalMapSegment() {}; - HorizontalMapSegment(const gid_type dof_gid, const int length); - HorizontalMapSegment(const gid_type dof_gid, const int length, const view_1d& source_dofs, const view_1d& weights); - - // Helper Functions - bool check() const; // Check if this segment is valid - void print() const; // Useful for debugging, print the segment mapping info - - // Setter Functions - void set_dof_idx(const int idx) { m_dof_idx = idx; } - void sync_to_host(); - - // Getter Functions - gid_type get_dof() const { return m_dof; } - int get_dof_idx() const { return m_dof_idx; } - int get_length() const { return m_length; } - view_1d get_source_dofs() const { return m_source_dofs; } - view_1d get_source_idx() const { return m_source_idx; } - view_1d get_weights() const { return m_weights; } - view_1d_host get_source_idx_on_host() const { return m_source_idx_h; } - view_1d_host get_weights_on_host() const { return m_weights_h; } - - // TODO: Not sure why, but this can't be set as private, otherwise `set_dof_idx` doesn't work. - int m_dof_idx = -999; // The degree of freedom w.r.t. to the local index for this map -private: - - // Remap views - view_1d m_source_dofs; - view_1d m_source_idx; - view_1d m_weights; - view_1d_host m_source_idx_h; - view_1d_host m_weights_h; - - // Segment ID - gid_type m_dof; // The global degree of freedom this segment maps to - int m_length; -}; // HorizontalMapSegment - -/*===============================================================================================*/ -/* HorizontalMap - * Structure which can be used to setup and control a horizontal remapping. This structure - * follows the basic premise that there are a set of source columns >=1 that will map to a - * single target column with a specific set of weights. Mapping follows the expression: - * Y_target = sum_(n=1)^N ( w_n * Y_source_n ) - * where, - * Y_target: Is the remapped value on the target column. - * w_n: Is the n'th weight - * Y_source_n: Is the n'th column in the source data - * N: Is the total number of source columns mapping to the target column (N>=1) - * - * This structure follows the format used by the component coupler. - * -------------------------------------- - * A.S. Donahue (LLNL): 2022-09-07 - *===============================================================================================*/ - -class HorizontalMap { - // Note: The name used for mapping in the component coupler is HorizontalMap. We could adopt a different - // name if desired. - using gid_type = AbstractGrid::gid_type; - using KT = KokkosTypes; - - template - using view_1d = typename KT::template view_1d; - - template - using view_2d = typename KT::template view_2d; - - template - using view_3d = typename KT::template view_3d; - - -public: - // Constructors/Destructor - ~HorizontalMap() = default; - HorizontalMap() {}; - explicit HorizontalMap(const ekat::Comm& comm); - HorizontalMap(const ekat::Comm& comm, const std::string& map_name); - HorizontalMap(const ekat::Comm& comm, const std::string& map_name, const view_1d& dofs_gids, const gid_type min_dof); - - // Main remap functions - void apply_remap(const view_1d& source_data, const view_1d& remapped_data); - void apply_remap(const view_2d& source_data, const view_2d& remapped_data); - void apply_remap(const view_3d& source_data, const view_3d& remapped_data); - - // Helper functions - void check() const; // A check to make sure the map is valid - void print() const; // Useful for debugging - - // Builder functions - used to build the HorizontalMap - void set_dof_gids(const view_1d& dofs_gids, const gid_type min_dof); - void set_unique_source_dofs(); - void add_remap_segment(const HorizontalMapSegment& seg); - void set_remap_segments_from_file(const std::string& remap_filename); - - // Getter functions - view_1d get_unique_source_dofs() const { return m_unique_dofs; } - int get_num_unique_dofs() const { return m_num_unique_dofs; } - int get_num_of_dofs() const { return m_num_dofs; } - std::vector get_map_segments() const { return m_map_segments; } - int get_num_of_segments() const { return m_num_segments; } - -private: - - // Global degrees of freedom information on target grid - view_1d m_dofs_gids; - int m_num_dofs = 0; - // Global degrees of freedom information on source grid - view_1d m_unique_dofs; - int m_num_unique_dofs; - bool m_unique_set = false; - // HorizontalMap data - std::string m_name = ""; - ekat::Comm m_comm; - bool m_dofs_set = false; - std::vector m_map_segments; - int m_num_segments = 0; - -}; // struct HorizontalMap - -/*===============================================================================================*/ - -} //namespace scream - -#endif // EAMXX_HORIZONTAL_REMAP_UTILITY_HPP diff --git a/components/eamxx/src/share/grid/remap/identity_remapper.hpp b/components/eamxx/src/share/grid/remap/identity_remapper.hpp index 6b6659c3f60e..047bd90ef835 100644 --- a/components/eamxx/src/share/grid/remap/identity_remapper.hpp +++ b/components/eamxx/src/share/grid/remap/identity_remapper.hpp @@ -12,29 +12,40 @@ namespace scream * This remapper effectively does nothing, since its source and target * grids are the same. There is no *real* need for this routine, * but it makes it easier to 'generically' and 'agnostically' create - * remappers. If one hits the case of src_grid=tgt_grid, he/she can - * simply create an identity remapper. This remapper is guaranteed - * to do absolutely nothing (except for possibly some correctness - * checks, mostly through the base class interface) when the remap - * method is called. It *does* still store ids for source and target - * fields, mostly to allow queries via the base class interface. - * However, no field is actually stored (no views, that is), since - * there is no need to actually access the data. + * remappers. If one hits the case of src_grid=tgt_grid, they can + * simply create an identity remapper. When remap methods are called, + * this remapper will + * - call Field::deep_copy if m_aliasing = NoAliasing + * - do nothing if m_aliasing != NoAliasing */ class IdentityRemapper : public AbstractRemapper { public: - using base_type = AbstractRemapper; - - IdentityRemapper (const grid_ptr_type grid) - : base_type(grid,grid) + using base_type = AbstractRemapper; + + // If + enum Aliasing { + SrcAliasTgt, + TgtAliasSrc, + NoAliasing + }; + + IdentityRemapper (const grid_ptr_type grid, + const Aliasing aliasing = NoAliasing) + : base_type (grid,grid) { - // Nothing to do here + set_aliasing(aliasing); } ~IdentityRemapper () = default; + void set_aliasing (const Aliasing aliasing) { + EKAT_REQUIRE_MSG (get_state()!=RepoState::Closed, + "Error! Aliasing in IdentityRemapper must be set *before* registration ends.\n"); + m_aliasing = aliasing; + } + FieldLayout create_src_layout (const FieldLayout& tgt_layout) const override { EKAT_REQUIRE_MSG (is_valid_tgt_layout(tgt_layout), "[IdentityRemapper] Error! Input target layout is not valid for this remapper.\n" @@ -52,6 +63,24 @@ class IdentityRemapper : public AbstractRemapper return src_layout; } + void register_field_from_src (const field_type& src) { + EKAT_REQUIRE_MSG (m_aliasing!=SrcAliasTgt, + "Error! Makes no sense to register from src and ask that src alias tgt.\n"); + if (m_aliasing==TgtAliasSrc) { + register_field(src,src); + } else { + AbstractRemapper::register_field_from_src(src); + } + } + void register_field_from_tgt (const field_type& tgt) { + EKAT_REQUIRE_MSG (m_aliasing!=TgtAliasSrc, + "Error! Makes no sense to register from tgt and ask that tgt alias src.\n"); + if (m_aliasing==SrcAliasTgt) { + register_field(tgt,tgt); + } else { + AbstractRemapper::register_field_from_tgt(tgt); + } + } protected: const identifier_type& do_get_src_field_id (const int ifield) const override { @@ -81,17 +110,40 @@ class IdentityRemapper : public AbstractRemapper m_fields[ifield].second = tgt; } void do_registration_ends () override { - // Do nothing + // If src is an alias of tgt (or viceversa), make the pair of fields store the same field + if (m_aliasing==Aliasing::SrcAliasTgt) { + for (auto& it : m_fields) { + it.first = it.second; + } + } else if (m_aliasing==Aliasing::TgtAliasSrc) { + for (auto& it : m_fields) { + it.second = it.first; + } + } } void do_remap_fwd () override { - // Do nothing + // If src is an alias of tgt (or viceversa), no need to run the remapper. + // Otherwise, we can simply run Field::deep_copy. + if (m_aliasing==Aliasing::NoAliasing) { + for (auto& it : m_fields) { + it.second.deep_copy(it.first); + } + } } void do_remap_bwd () override { - // Do nothing + // If src is an alias of tgt (or viceversa), no need to run the remapper. + // Otherwise, we can simply run Field::deep_copy. + if (m_aliasing==Aliasing::NoAliasing) { + for (auto& it : m_fields) { + it.first.deep_copy(it.second); + } + } } std::vector> m_fields; + + Aliasing m_aliasing; }; } // namespace scream diff --git a/components/eamxx/src/share/grid/remap/refining_remapper_p2p.cpp b/components/eamxx/src/share/grid/remap/refining_remapper_p2p.cpp index b1c738fe9256..d4f878f77a85 100644 --- a/components/eamxx/src/share/grid/remap/refining_remapper_p2p.cpp +++ b/components/eamxx/src/share/grid/remap/refining_remapper_p2p.cpp @@ -242,6 +242,35 @@ void RefiningRemapperP2P::pack_and_send () Kokkos::parallel_for(policy,pack); break; } + case 4: + { + const auto v = f.get_view(); + const int dim1 = fl.dim(1); + const int dim2 = fl.dim(2); + const int dim3 = fl.dim(3); + const int f_col_size = dim1*dim2*dim3; + auto policy = ESU::get_default_team_policy(num_exports,dim1*dim2*dim3); + auto pack = KOKKOS_LAMBDA(const TeamMember& team) { + const int iexp = team.league_rank(); + const int icol = export_lids(iexp); + const int pid = export_pids(iexp); + auto pid_offset = pids_send_offsets(pid); + auto pos_within_pid = iexp - pid_offset; + auto offset = pid_offset*total_col_size + + ncols_send(pid)*f_col_sizes_scan_sum + + pos_within_pid*f_col_size; + auto col_pack = [&](const int& idx) { + const int j = (idx / dim3) / dim2; + const int k = (idx / dim3) % dim2; + const int l = idx % dim3; + send_buf(offset+idx) = v(icol,j,k,l); + }; + auto tvr = Kokkos::TeamVectorRange(team,f_col_size); + Kokkos::parallel_for(tvr,col_pack); + }; + Kokkos::parallel_for(policy,pack); + break; + } default: EKAT_ERROR_MSG ("Unexpected field rank in RefiningRemapperP2P::pack.\n" " - MPI rank : " + std::to_string(m_comm.rank()) + "\n" @@ -357,6 +386,35 @@ void RefiningRemapperP2P::recv_and_unpack () Kokkos::parallel_for(policy,unpack); break; } + case 4: + { + auto v = f.get_view(); + const int dim1 = fl.dim(1); + const int dim2 = fl.dim(2); + const int dim3 = fl.dim(3); + const int f_col_size = dim1*dim2*dim3; + auto policy = ESU::get_default_team_policy(num_imports,dim1*dim2*dim3); + auto unpack = KOKKOS_LAMBDA (const TeamMember& team) { + const int idx = team.league_rank(); + const int pid = import_pids(idx); + const int icol = import_lids(idx); + const auto pid_offset = pids_recv_offsets(pid); + const auto pos_within_pid = idx - pid_offset; + auto offset = pid_offset*total_col_size + + ncols_recv(pid)*f_col_sizes_scan_sum + + pos_within_pid*f_col_size; + auto col_unpack = [&](const int& idx) { + const int j = (idx / dim3) / dim2; + const int k = (idx / dim3) % dim2; + const int l = idx % dim3; + v(icol,j,k,l) = recv_buf(offset+idx); + }; + auto tvr = Kokkos::TeamVectorRange(team,f_col_size); + Kokkos::parallel_for(tvr,col_unpack); + }; + Kokkos::parallel_for(policy,unpack); + break; + } default: EKAT_ERROR_MSG ("Unexpected field rank in RefiningRemapperP2P::unpack.\n" " - MPI rank : " + std::to_string(m_comm.rank()) + "\n" diff --git a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp index beda1f5d7850..9abefed9eed7 100644 --- a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp @@ -5,6 +5,7 @@ #include "share/io/scorpio_input.hpp" #include "share/field/field_tag.hpp" #include "share/field/field_identifier.hpp" +#include "share/util/scream_universal_constants.hpp" #include "ekat/util/ekat_units.hpp" #include @@ -19,21 +20,21 @@ VerticalRemapper:: VerticalRemapper (const grid_ptr_type& src_grid, const std::string& map_file, const Field& lev_prof, - const Field& ilev_prof, - const Real mask_val) - : VerticalRemapper(src_grid,map_file,lev_prof,ilev_prof) + const Field& ilev_prof) + : VerticalRemapper(src_grid,map_file,lev_prof,ilev_prof,constants::DefaultFillValue::value) { - m_mask_val = mask_val; + // Nothing to do here } VerticalRemapper:: VerticalRemapper (const grid_ptr_type& src_grid, const std::string& map_file, const Field& lev_prof, - const Field& ilev_prof) + const Field& ilev_prof, + const Real mask_val) : AbstractRemapper() , m_comm (src_grid->get_comm()) - , m_mask_val(std::numeric_limits::max()/10.0) + , m_mask_val(mask_val) { using namespace ShortFieldTagsNames; @@ -67,6 +68,8 @@ VerticalRemapper (const grid_ptr_type& src_grid, // Gather the pressure level data for vertical remapping set_pressure_levels(map_file); + // Add tgt pressure levels to the tgt grid + tgt_grid->set_geometry_data(m_remap_pres); scorpio::eam_pio_closefile(map_file); } @@ -79,28 +82,9 @@ create_src_layout (const FieldLayout& tgt_layout) const "[VerticalRemapper] Error! Input target layout is not valid for this remapper.\n" " - input layout: " + to_string(tgt_layout)); - const auto lt = get_layout_type(tgt_layout.tags()); - auto src = FieldLayout::invalid(); - const bool midpoints = tgt_layout.has_tag(LEV); - const int vec_dim = tgt_layout.is_vector_layout() ? tgt_layout.dim(CMP) : -1; - switch (lt) { - case LayoutType::Scalar2D: - src = m_src_grid->get_2d_scalar_layout(); - break; - case LayoutType::Vector2D: - src = m_src_grid->get_2d_vector_layout(CMP,vec_dim); - break; - case LayoutType::Scalar3D: - src = m_src_grid->get_3d_scalar_layout(midpoints); - break; - case LayoutType::Vector3D: - src = m_src_grid->get_3d_vector_layout(midpoints,CMP,vec_dim); - break; - default: - EKAT_ERROR_MSG ("Layout not supported by VerticalRemapper: " + e2str(lt) + "\n"); - } - return src; + return create_layout(tgt_layout,m_src_grid); } + FieldLayout VerticalRemapper:: create_tgt_layout (const FieldLayout& src_layout) const { @@ -110,27 +94,51 @@ create_tgt_layout (const FieldLayout& src_layout) const "[VerticalRemapper] Error! Input source layout is not valid for this remapper.\n" " - input layout: " + to_string(src_layout)); - const auto lt = get_layout_type(src_layout.tags()); - auto tgt = FieldLayout::invalid(); - const bool midpoints = true; //src_layout.has_tag(LEV); - const int vec_dim = src_layout.is_vector_layout() ? src_layout.dim(CMP) : -1; + return create_layout(src_layout,m_tgt_grid); +} + +FieldLayout VerticalRemapper:: +create_layout (const FieldLayout& fl_in, + const grid_ptr_type& grid_out) const +{ + using namespace ShortFieldTagsNames; + + // NOTE: for the vert remapper, it doesn't really make sense to distinguish + // between midpoints and interfaces: we're simply asking for a quantity + // at a given set of pressure levels. So we choose to have fl_out + // to *always* have LEV as vertical tag. + const auto lt = get_layout_type(fl_in.tags()); + auto fl_out = FieldLayout::invalid(); switch (lt) { - case LayoutType::Scalar2D: - tgt = m_tgt_grid->get_2d_scalar_layout(); + case LayoutType::Scalar0D: [[ fallthrough ]]; + case LayoutType::Vector0D: [[ fallthrough ]]; + case LayoutType::Scalar2D: [[ fallthrough ]]; + case LayoutType::Vector2D: [[ fallthrough ]]; + case LayoutType::Tensor2D: + // These layouts do not have vertical dim tags, so no change + fl_out = fl_in; break; - case LayoutType::Vector2D: - tgt = m_tgt_grid->get_2d_vector_layout(CMP,vec_dim); + case LayoutType::Scalar1D: + fl_out = grid_out->get_vertical_layout(true); break; case LayoutType::Scalar3D: - tgt = m_tgt_grid->get_3d_scalar_layout(midpoints); + fl_out = grid_out->get_3d_scalar_layout(true); break; case LayoutType::Vector3D: - tgt = m_tgt_grid->get_3d_vector_layout(midpoints,CMP,vec_dim); + { + const auto vec_tag = fl_in.get_vector_tag(); + const auto vec_dim = fl_in.dim(vec_tag); + fl_out = grid_out->get_3d_vector_layout(true,vec_tag,vec_dim); break; + } default: - EKAT_ERROR_MSG ("Layout not supported by VerticalRemapper: " + e2str(lt) + "\n"); + // NOTE: this also include Tensor3D. We don't really have any atm proc + // that needs to handle a tensor3d quantity, so no need to add it + EKAT_ERROR_MSG ( + "Layout not supported by VerticalRemapper.\n" + " - input layout: " + to_string(fl_in) + "\n"); } - return tgt; + return fl_out; } void VerticalRemapper:: @@ -146,7 +154,7 @@ set_pressure_levels(const std::string& map_file) std::vector tags = {LEV}; std::vector dims = {m_num_remap_levs}; FieldLayout layout(tags,dims); - FieldIdentifier fid("p_remap",layout,ekat::units::Pa,m_tgt_grid->name()); + FieldIdentifier fid("p_levs",layout,ekat::units::Pa,m_tgt_grid->name()); m_remap_pres = Field(fid); m_remap_pres.get_header().get_alloc_properties().request_allocation(mPack::n); m_remap_pres.allocate_view(); @@ -312,7 +320,6 @@ void VerticalRemapper::do_remap_fwd () { using namespace ShortFieldTagsNames; // Loop over each field - constexpr auto can_pack = SCREAM_PACK_SIZE>1; const auto& tgt_pres_ap = m_remap_pres.get_header().get_alloc_properties(); for (int i=0; i>() && - tgt_ap.is_compatible>() && - src_pres_ap.is_compatible>() && - tgt_pres_ap.is_compatible>()) { + if (src_ap.is_compatible>() && + tgt_ap.is_compatible>() && + src_pres_ap.is_compatible>() && + tgt_pres_ap.is_compatible>()) { apply_vertical_interpolation(f_src,f_tgt); } else { apply_vertical_interpolation<1>(f_src,f_tgt); @@ -358,10 +365,10 @@ void VerticalRemapper::do_remap_fwd () const auto& src_ap = f_src.get_header().get_alloc_properties(); const auto& tgt_ap = f_tgt.get_header().get_alloc_properties(); const auto& src_pres_ap = src_tag == LEV ? m_src_mid.get_header().get_alloc_properties() : m_src_int.get_header().get_alloc_properties(); - if (can_pack && src_ap.is_compatible>() && - tgt_ap.is_compatible>() && - src_pres_ap.is_compatible>() && - tgt_pres_ap.is_compatible>()) { + if (src_ap.is_compatible>() && + tgt_ap.is_compatible>() && + src_pres_ap.is_compatible>() && + tgt_pres_ap.is_compatible>()) { apply_vertical_interpolation(f_src,f_tgt,true); } else { apply_vertical_interpolation<1>(f_src,f_tgt,true); @@ -378,44 +385,42 @@ template void VerticalRemapper:: apply_vertical_interpolation(const Field& f_src, const Field& f_tgt, const bool mask_interp) const { - - using Pack = ekat::Pack; - using namespace ShortFieldTagsNames; - using namespace scream::vinterp; - const auto& layout = f_src.get_header().get_identifier().get_layout(); - const auto rank = f_src.rank(); - const auto src_tag = layout.tags().back(); - const auto src_num_levs = layout.dims().back(); - // ARG mask_interp checks if this is a vertical interpolation of the mask array that tracks masked 0.0 or not 1.0 - Real mask_val = mask_interp ? 0.0 : m_mask_val; - - Field src_lev_f; - if (src_tag == ILEV) { - src_lev_f = m_src_int; - } else { - src_lev_f = m_src_mid; + using Pack = ekat::Pack; + using namespace ShortFieldTagsNames; + using namespace scream::vinterp; + const auto& layout = f_src.get_header().get_identifier().get_layout(); + const auto rank = f_src.rank(); + const auto src_tag = layout.tags().back(); + const auto src_num_levs = layout.dims().back(); + // ARG mask_interp checks if this is a vertical interpolation of the mask array that tracks masked 0.0 or not 1.0 + Real mask_val = mask_interp ? 0.0 : m_mask_val; + + Field src_lev_f; + if (src_tag == ILEV) { + src_lev_f = m_src_int; + } else { + src_lev_f = m_src_mid; + } + auto src_lev = src_lev_f.get_view(); + auto remap_pres_view = m_remap_pres.get_view(); + switch(rank) { + case 2: + { + auto src_view = f_src.get_view(); + auto tgt_view = f_tgt.get_view< Pack**>(); + perform_vertical_interpolation(src_lev,remap_pres_view,src_view,tgt_view,src_num_levs,m_num_remap_levs,mask_val); + break; } - auto src_lev = src_lev_f.get_view(); - auto remap_pres_view = m_remap_pres.get_view(); - switch(rank) { - case 2: - { - auto src_view = f_src.get_view(); - auto tgt_view = f_tgt.get_view< Pack**>(); - perform_vertical_interpolation(src_lev,remap_pres_view,src_view,tgt_view,src_num_levs,m_num_remap_levs,mask_val); - break; - } - case 3: - { - auto src_view = f_src.get_view(); - auto tgt_view = f_tgt.get_view< Pack***>(); - perform_vertical_interpolation(src_lev,remap_pres_view,src_view,tgt_view,src_num_levs,m_num_remap_levs,mask_val); - break; - } - default: - EKAT_ERROR_MSG ("Error! Field rank (" + std::to_string(rank) + ") not supported by VerticalRemapper.\n"); + case 3: + { + auto src_view = f_src.get_view(); + auto tgt_view = f_tgt.get_view< Pack***>(); + perform_vertical_interpolation(src_lev,remap_pres_view,src_view,tgt_view,src_num_levs,m_num_remap_levs,mask_val); + break; } - + default: + EKAT_ERROR_MSG ("Error! Field rank (" + std::to_string(rank) + ") not supported by VerticalRemapper.\n"); + } } } // namespace scream diff --git a/components/eamxx/src/share/grid/remap/vertical_remapper.hpp b/components/eamxx/src/share/grid/remap/vertical_remapper.hpp index 880d305b0638..f04baa933cc8 100644 --- a/components/eamxx/src/share/grid/remap/vertical_remapper.hpp +++ b/components/eamxx/src/share/grid/remap/vertical_remapper.hpp @@ -23,6 +23,7 @@ class VerticalRemapper : public AbstractRemapper const Field& ilev_prof, const Real mask_val); + // Calls the above one, with mask_val=max_float/10 VerticalRemapper (const grid_ptr_type& src_grid, const std::string& map_file, const Field& lev_prof, @@ -67,9 +68,21 @@ class VerticalRemapper : public AbstractRemapper src_col_size == tgt_col_size; } + // NOTE: for the vert remapper, it doesn't really make sense to distinguish + // between midpoints and interfaces: we're simply asking for a quantity + // at a given set of pressure levels. So we choose to NOT allow a tgt + // layout with ILEV tag. + bool is_valid_tgt_layout (const layout_type& layout) const override { + using namespace ShortFieldTagsNames; + return not layout.has_tag(ILEV) + and AbstractRemapper::is_valid_tgt_layout(layout); + } protected: + FieldLayout create_layout (const FieldLayout& fl_in, + const grid_ptr_type& grid_out) const; + void register_vertical_source_field(const Field& src); const identifier_type& do_get_src_field_id (const int ifield) const override { diff --git a/components/eamxx/src/share/grid/se_grid.cpp b/components/eamxx/src/share/grid/se_grid.cpp index 1e5556de6cad..1b941125e1cb 100644 --- a/components/eamxx/src/share/grid/se_grid.cpp +++ b/components/eamxx/src/share/grid/se_grid.cpp @@ -47,6 +47,24 @@ SEGrid::get_2d_vector_layout (const FieldTag vector_tag, const int vector_dim) c return FieldLayout({EL,vector_tag,GP,GP},{m_num_local_elem,vector_dim,m_num_gp,m_num_gp}); } +FieldLayout +SEGrid::get_2d_tensor_layout (const std::vector& cmp_tags, + const std::vector& cmp_dims) const +{ + using namespace ShortFieldTagsNames; + + std::vector tags = {EL}; + std::vector dims = {m_num_local_elem}; + + tags.insert(tags.end(),cmp_tags.begin(),cmp_tags.end()); + dims.insert(dims.end(),cmp_dims.begin(),cmp_dims.end()); + tags.push_back(GP); + tags.push_back(GP); + dims.push_back(m_num_gp); + dims.push_back(m_num_gp); + return FieldLayout(tags,dims); +} + FieldLayout SEGrid::get_3d_scalar_layout (const bool midpoints) const { @@ -69,6 +87,30 @@ SEGrid::get_3d_vector_layout (const bool midpoints, const FieldTag vector_tag, c return FieldLayout({EL,vector_tag,GP,GP,VL},{m_num_local_elem,vector_dim,m_num_gp,m_num_gp,nvl}); } +FieldLayout +SEGrid::get_3d_tensor_layout (const bool midpoints, + const std::vector& cmp_tags, + const std::vector& cmp_dims) const +{ + using namespace ShortFieldTagsNames; + + int nvl = this->get_num_vertical_levels() + (midpoints ? 0 : 1); + auto VL = midpoints ? LEV : ILEV; + + std::vector tags = {EL}; + std::vector dims = {m_num_local_elem}; + + tags.insert(tags.end(),cmp_tags.begin(),cmp_tags.end()); + dims.insert(dims.end(),cmp_dims.begin(),cmp_dims.end()); + tags.push_back(GP); + tags.push_back(GP); + tags.push_back(VL); + dims.push_back(m_num_gp); + dims.push_back(m_num_gp); + dims.push_back(nvl); + return FieldLayout(tags,dims); +} + Field SEGrid::get_cg_dofs_gids () { EKAT_REQUIRE_MSG (m_cg_dofs_gids.is_allocated(), diff --git a/components/eamxx/src/share/grid/se_grid.hpp b/components/eamxx/src/share/grid/se_grid.hpp index f31167b79a76..90af8de066da 100644 --- a/components/eamxx/src/share/grid/se_grid.hpp +++ b/components/eamxx/src/share/grid/se_grid.hpp @@ -22,8 +22,13 @@ class SEGrid : public AbstractGrid // Native layout of a dof. This is the natural way to index a dof in the grid. FieldLayout get_2d_scalar_layout () const override; FieldLayout get_2d_vector_layout (const FieldTag vector_tag, const int vector_dim) const override; + FieldLayout get_2d_tensor_layout (const std::vector& cmp_tags, + const std::vector& cmp_dims) const override; FieldLayout get_3d_scalar_layout (const bool midpoints) const override; FieldLayout get_3d_vector_layout (const bool midpoints, const FieldTag vector_tag, const int vector_dim) const override; + FieldLayout get_3d_tensor_layout (const bool midpoints, + const std::vector& cmp_tags, + const std::vector& cmp_dims) const override; FieldTag get_partitioned_dim_tag () const override { return FieldTag::Element; diff --git a/components/eamxx/src/share/io/scorpio_input.cpp b/components/eamxx/src/share/io/scorpio_input.cpp index 29a62763cec4..c4f023cc396f 100644 --- a/components/eamxx/src/share/io/scorpio_input.cpp +++ b/components/eamxx/src/share/io/scorpio_input.cpp @@ -12,7 +12,7 @@ namespace scream AtmosphereInput:: AtmosphereInput (const ekat::ParameterList& params, - const std::shared_ptr& field_mgr) + const std::shared_ptr& field_mgr) { init(params,field_mgr); } @@ -29,11 +29,13 @@ AtmosphereInput (const ekat::ParameterList& params, AtmosphereInput:: AtmosphereInput (const std::string& filename, const std::shared_ptr& grid, - const std::vector& fields) + const std::vector& fields, + const bool skip_grid_checks) { // Create param list and field manager on the fly ekat::ParameterList params; params.set("Filename",filename); + params.set("Skip_Grid_Checks",skip_grid_checks); auto& names = params.get>("Field Names",{}); auto fm = std::make_shared(grid); @@ -204,7 +206,12 @@ void AtmosphereInput::read_variables (const int time_index) { auto func_start = std::chrono::steady_clock::now(); if (m_atm_logger) { - m_atm_logger->info("[EAMxx::scorpio_input] Reading variables from file:\n\t " + m_filename + " ...\n"); + m_atm_logger->info("[EAMxx::scorpio_input] Reading variables from file"); + m_atm_logger->info(" file name: " + m_filename); + m_atm_logger->info(" var names: " + ekat::join(m_fields_names,", ")); + if (time_index!=-1) { + m_atm_logger->info(" time idx : " + std::to_string(time_index)); + } } EKAT_REQUIRE_MSG (m_inited_with_views || m_inited_with_fields, "Error! Scorpio structures not inited yet. Did you forget to call 'init(..)'?\n"); @@ -321,7 +328,7 @@ void AtmosphereInput::read_variables (const int time_index) auto func_finish = std::chrono::steady_clock::now(); if (m_atm_logger) { auto duration = std::chrono::duration_cast(func_finish - func_start)/1000.0; - m_atm_logger->info("[EAMxx::scorpio_input] Reading variables from file:\n\t " + m_filename + " ... done! (Elapsed time = " + std::to_string(duration.count()) +" seconds)\n"); + m_atm_logger->info(" Done! Elapsed time: " + std::to_string(duration.count()) +" seconds"); } } @@ -442,6 +449,15 @@ void AtmosphereInput::set_degrees_of_freedom() std::vector AtmosphereInput::get_var_dof_offsets(const FieldLayout& layout) { + using namespace ShortFieldTagsNames; + + // Precompute this *before* the early return, since it involves collectives. + // If one rank owns zero cols, and returns prematurely, the others will be left waiting. + AbstractGrid::gid_type min_gid; + if (layout.has_tag(COL) or layout.has_tag(EL)) { + min_gid = m_io_grid->get_global_min_dof_gid(); + } + // It may be that this MPI ranks owns no chunk of the field if (layout.size()==0) { return {}; @@ -461,17 +477,13 @@ AtmosphereInput::get_var_dof_offsets(const FieldLayout& layout) // of the MPI-local array w.r.t. the global array. So long as the offsets are in // the same order as the corresponding entry in the data to be read/written, we're good. auto dofs_h = m_io_grid->get_dofs_gids().get_view(); - if (layout.has_tag(ShortFieldTagsNames::COL)) { + if (layout.has_tag(COL)) { const int num_cols = m_io_grid->get_num_local_dofs(); // Note: col_size might be *larger* than the number of vertical levels, or even smaller. // E.g., (ncols,2,nlevs), or (ncols,2) respectively. scorpio::offset_t col_size = layout.size() / num_cols; - // Precompute this *before* the loop, since it involves expensive collectives. - // Besides, the loop might have different length on different ranks, so - // computing it inside might cause deadlocks. - auto min_gid = m_io_grid->get_global_min_dof_gid(); for (int icol=0; icolget_2d_scalar_layout(); const int num_my_elems = layout2d.dim(0); const int ngp = layout2d.dim(1); @@ -492,10 +504,6 @@ AtmosphereInput::get_var_dof_offsets(const FieldLayout& layout) // E.g., (ncols,2,nlevs), or (ncols,2) respectively. scorpio::offset_t col_size = layout.size() / num_cols; - // Precompute this *before* the loop, since it involves expensive collectives. - // Besides, the loop might have different length on different ranks, so - // computing it inside might cause deadlocks. - auto min_gid = m_io_grid->get_global_min_dof_gid(); for (int ie=0,icol=0; ie& layouts); AtmosphereInput (const std::string& filename, const std::shared_ptr& grid, - const std::vector& fields); + const std::vector& fields, + const bool skip_grid_checks = false); ~AtmosphereInput (); diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 08b4b83410d5..3943e59d5da1 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -41,9 +41,9 @@ void combine (const Real& new_val, Real& curr_val, const OutputAvgType avg_type) } // This one covers cases where a variable might be masked. KOKKOS_INLINE_FUNCTION -void combine_and_fill (const Real& new_val, Real& curr_val, Real& avg_coeff, const OutputAvgType avg_type, const Real fill_value) +void combine_and_fill (const Real& new_val, Real& curr_val, const OutputAvgType avg_type, const Real fill_value) { - const bool new_fill = (avg_coeff == 0.0); + const bool new_fill = new_val == fill_value; const bool curr_fill = curr_val == fill_value; if (curr_fill && new_fill) { // Then the value is already set to be filled and the new value doesn't change things. @@ -119,20 +119,6 @@ AtmosphereOutput (const ekat::Comm& comm, const ekat::ParameterList& params, { using vos_t = std::vector; - if (params.isParameter("fill_value")) { - m_fill_value = static_cast(params.get("fill_value")); - // If the fill_value is specified there is a good chance the user expects the average count to track filling. - m_track_avg_cnt = true; - } - if (params.isParameter("track_fill")) { - // Note, we do this after checking for fill_value to give users that opportunity to turn off fill tracking, even - // if they specify a specific fill value. - m_track_avg_cnt = params.get("track_fill"); - } - if (params.isParameter("fill_threshold")) { - m_avg_coeff_threshold = params.get("fill_threshold"); - } - // Figure out what kind of averaging is requested auto avg_type = params.get("Averaging Type"); m_avg_type = str2avg(avg_type); @@ -202,6 +188,29 @@ AtmosphereOutput (const ekat::Comm& comm, const ekat::ParameterList& params, // Register any diagnostics needed by this output stream set_diagnostics(); + // Avg count only makes sense if we have + // - non-instant output + // - we have one between: + // - vertically remapped output + // - field_at_XhPa diagnostic + // We already set m_track_avg_cnt to true if field_at_XhPa is found in set_diagnostics. + // Hence, here we only check if vert remap is active + + if (params.isParameter("track_avg_cnt")) { + // This is to be used for unit testing only, so that we can test avg cnt even + // if there is no vert remap and no field_at_XhPa diagnostic in the stream + m_track_avg_cnt = params.get("track_avg_cnt"); + } + if (use_vertical_remap_from_file) { + m_track_avg_cnt = true; + } + if (params.isParameter("fill_value")) { + m_fill_value = static_cast(params.get("fill_value")); + } + if (params.isParameter("fill_threshold")) { + m_avg_coeff_threshold = params.get("fill_threshold"); + } + // Helper lambda, to copy io string attributes. This will be used if any // remapper is created, to ensure atts set by atm_procs are not lost auto transfer_io_str_atts = [&] (const Field& src, Field& tgt) { @@ -216,8 +225,6 @@ AtmosphereOutput (const ekat::Comm& comm, const ekat::ParameterList& params, // Setup remappers - if needed if (use_vertical_remap_from_file) { - // When vertically remapping there is a chance that filled values will be present, so be sure to track these - m_track_avg_cnt = true; // We build a remapper, to remap fields from the fm grid to the io grid auto vert_remap_file = params.get("vertical_remap_file"); auto f_lev = get_field("p_mid","sim"); @@ -369,7 +376,8 @@ run (const std::string& filename, Real duration_write = 0.0; // Record of time spent writing output if (is_write_step) { if (m_atm_logger) { - m_atm_logger->info("[EAMxx::scorpio_output] Writing variables to file:\n\t " + filename + " ...\n"); + m_atm_logger->info("[EAMxx::scorpio_output] Writing variables to file"); + m_atm_logger->info(" file name: " + filename); } } @@ -418,37 +426,36 @@ run (const std::string& filename, // temporary views for each layout that are either 0 or 1 depending on if the // value is filled or unfilled. // We then use these values to update the overall average count views for that layout. - if (m_track_avg_cnt && m_add_time_dim) { + if (m_track_avg_cnt) { // Note, we assume that all fields that share a layout are also masked/filled in the same - // way. If, we need to handle a case where only a subset of output variables are expected to + // way. If we need to handle a case where only a subset of output variables are expected to // be masked/filled then the recommendation is to request those variables in a separate output // stream. - // We cycle through all fields and mark points that are filled/masked in the local views. First - // initialize them to 1 representing unfilled. - for (const auto& name : m_avg_cnt_names) { - auto& dev_view = m_local_tmp_avg_cnt_views_1d.at(name); - Kokkos::deep_copy(dev_view,1.0); - } - // Now we cycle through all the fields + // We cycle through all fields and we + // 1. Find the avg_cnt view for this field. + // 2. If we already processed the avg_cnt view, go to next field, and start from 1 again. + // 3. Add 1 to all entries of avg_cnt where field!=fill_value + std::set avg_updated; for (const auto& name : m_fields_names) { - auto field = get_field(name,"io"); - auto lookup = m_field_to_avg_cnt_map.at(name); - auto dev_view = m_local_tmp_avg_cnt_views_1d.at(lookup); - update_avg_cnt_view(field,dev_view); - } - // Finally, we update the overall avg_cnt_views - for (const auto& name : m_avg_cnt_names) { - auto track_view = m_dev_views_1d.at(name); - auto local_view = m_local_tmp_avg_cnt_views_1d.at(name); - const auto layout = m_layouts.at(name); - KT::RangePolicy policy(0,layout.size()); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int i) { - track_view(i) += local_view(i); - }); + auto avg_cnt_name = m_field_to_avg_cnt_map.at(name); + if (avg_updated.count(avg_cnt_name)==1) { + // We updated this avg_cnt by checking another field + continue; + } + auto field = get_field(name,"io"); + update_avg_cnt_view(field,m_dev_views_1d.at(avg_cnt_name)); + + // Make sure we don't double update this avg cnt + avg_updated.insert(avg_cnt_name); } } // Take care of updating and possibly writing fields. + // These are needed inside kernels, so crate local copies + auto do_avg_cnt = m_track_avg_cnt; + auto avg_type = m_avg_type; + auto fill_value = m_fill_value; + auto avg_coeff_threshold = m_avg_coeff_threshold; for (auto const& name : m_fields_names) { // Get all the info for this field. auto field = get_field(name,"io"); @@ -481,26 +488,6 @@ run (const std::string& filename, KT::RangePolicy policy(0,layout.size()); const auto extents = layout.extents(); - // Averaging count data - // If we are not tracking the average count then we don't need to build the - // views for the average count, so we leave them as essentially empty. - auto avg_cnt_dims = dims; - auto avg_cnt_data = data; - if (m_track_avg_cnt && m_add_time_dim) { - const auto lookup = m_field_to_avg_cnt_map.at(name); - avg_cnt_data = m_local_tmp_avg_cnt_views_1d.at(lookup).data(); - } else { - for (auto& dim : avg_cnt_dims) { - dim = 1; - } - avg_cnt_data = nullptr; - } - - auto avg_type = m_avg_type; - auto track_avg_cnt = m_track_avg_cnt; - auto add_time_dim = m_add_time_dim; - auto fill_value = m_fill_value; - auto avg_coeff_threshold = m_avg_coeff_threshold; // If the dev_view_1d is aliasing the field device view (must be Instant output), // then there's no point in copying from the field's view to dev_view if (not is_aliasing_field_view) { @@ -511,12 +498,11 @@ run (const std::string& filename, // handling a few more scenarios auto new_view_1d = field.get_strided_view(); auto avg_view_1d = view_Nd_dev<1>(data,dims[0]); - auto avg_coeff_1d = view_Nd_dev<1>(avg_cnt_data,avg_cnt_dims[0]); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int i) { - if (track_avg_cnt && add_time_dim) { - combine_and_fill(new_view_1d(i), avg_view_1d(i),avg_coeff_1d(i),avg_type,fill_value); + if (do_avg_cnt) { + combine_and_fill(new_view_1d(i),avg_view_1d(i),avg_type,fill_value); } else { - combine(new_view_1d(i), avg_view_1d(i),avg_type); + combine(new_view_1d(i),avg_view_1d(i),avg_type); } }); break; @@ -525,12 +511,11 @@ run (const std::string& filename, { auto new_view_2d = field.get_view(); auto avg_view_2d = view_Nd_dev<2>(data,dims[0],dims[1]); - auto avg_coeff_2d = view_Nd_dev<2>(avg_cnt_data,avg_cnt_dims[0],avg_cnt_dims[1]); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { int i,j; unflatten_idx(idx,extents,i,j); - if (track_avg_cnt && add_time_dim) { - combine_and_fill(new_view_2d(i,j), avg_view_2d(i,j),avg_coeff_2d(i,j),avg_type,fill_value); + if (do_avg_cnt) { + combine_and_fill(new_view_2d(i,j),avg_view_2d(i,j),avg_type,fill_value); } else { combine(new_view_2d(i,j), avg_view_2d(i,j),avg_type); } @@ -541,12 +526,11 @@ run (const std::string& filename, { auto new_view_3d = field.get_view(); auto avg_view_3d = view_Nd_dev<3>(data,dims[0],dims[1],dims[2]); - auto avg_coeff_3d = view_Nd_dev<3>(avg_cnt_data,dims[0],avg_cnt_dims[1],avg_cnt_dims[2]); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { int i,j,k; unflatten_idx(idx,extents,i,j,k); - if (track_avg_cnt && add_time_dim) { - combine_and_fill(new_view_3d(i,j,k), avg_view_3d(i,j,k),avg_coeff_3d(i,j,k),avg_type,fill_value); + if (do_avg_cnt) { + combine_and_fill(new_view_3d(i,j,k),avg_view_3d(i,j,k),avg_type,fill_value); } else { combine(new_view_3d(i,j,k), avg_view_3d(i,j,k),avg_type); } @@ -557,12 +541,11 @@ run (const std::string& filename, { auto new_view_4d = field.get_view(); auto avg_view_4d = view_Nd_dev<4>(data,dims[0],dims[1],dims[2],dims[3]); - auto avg_coeff_4d = view_Nd_dev<4>(avg_cnt_data,avg_cnt_dims[0],avg_cnt_dims[1],avg_cnt_dims[2],avg_cnt_dims[3]); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { int i,j,k,l; unflatten_idx(idx,extents,i,j,k,l); - if (track_avg_cnt && add_time_dim) { - combine_and_fill(new_view_4d(i,j,k,l), avg_view_4d(i,j,k,l),avg_coeff_4d(i,j,k,l),avg_type,fill_value); + if (do_avg_cnt) { + combine_and_fill(new_view_4d(i,j,k,l), avg_view_4d(i,j,k,l),avg_type,fill_value); } else { combine(new_view_4d(i,j,k,l), avg_view_4d(i,j,k,l),avg_type); } @@ -573,12 +556,11 @@ run (const std::string& filename, { auto new_view_5d = field.get_view(); auto avg_view_5d = view_Nd_dev<5>(data,dims[0],dims[1],dims[2],dims[3],dims[4]); - auto avg_coeff_5d = view_Nd_dev<5>(avg_cnt_data,avg_cnt_dims[0],avg_cnt_dims[1],avg_cnt_dims[2],avg_cnt_dims[3],avg_cnt_dims[4]); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { int i,j,k,l,m; unflatten_idx(idx,extents,i,j,k,l,m); - if (track_avg_cnt && add_time_dim) { - combine_and_fill(new_view_5d(i,j,k,l,m), avg_view_5d(i,j,k,l,m),avg_coeff_5d(i,j,k,l,m),avg_type,fill_value); + if (do_avg_cnt) { + combine_and_fill(new_view_5d(i,j,k,l,m), avg_view_5d(i,j,k,l,m),avg_type,fill_value); } else { combine(new_view_5d(i,j,k,l,m), avg_view_5d(i,j,k,l,m),avg_type); } @@ -589,12 +571,11 @@ run (const std::string& filename, { auto new_view_6d = field.get_view(); auto avg_view_6d = view_Nd_dev<6>(data,dims[0],dims[1],dims[2],dims[3],dims[4],dims[5]); - auto avg_coeff_6d = view_Nd_dev<6>(avg_cnt_data,avg_cnt_dims[0],avg_cnt_dims[1],avg_cnt_dims[2],avg_cnt_dims[3],avg_cnt_dims[4],avg_cnt_dims[5]); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { int i,j,k,l,m,n; unflatten_idx(idx,extents,i,j,k,l,m,n); - if (track_avg_cnt && add_time_dim) { - combine_and_fill(new_view_6d(i,j,k,l,m,n), avg_view_6d(i,j,k,l,m,n), avg_coeff_6d(i,j,k,l,m,n),avg_type,fill_value); + if (do_avg_cnt) { + combine_and_fill(new_view_6d(i,j,k,l,m,n), avg_view_6d(i,j,k,l,m,n), avg_type,fill_value); } else { combine(new_view_6d(i,j,k,l,m,n), avg_view_6d(i,j,k,l,m,n),avg_type); } @@ -608,7 +589,7 @@ run (const std::string& filename, if (is_write_step) { if (output_step and avg_type==OutputAvgType::Average) { - if (m_track_avg_cnt && m_add_time_dim) { + if (do_avg_cnt) { const auto avg_cnt_lookup = m_field_to_avg_cnt_map.at(name); const auto avg_cnt_view = m_dev_views_1d.at(avg_cnt_lookup); const auto avg_nsteps = avg_cnt_view.data(); @@ -654,7 +635,7 @@ run (const std::string& filename, } if (is_write_step) { if (m_atm_logger) { - m_atm_logger->info("[EAMxx::scorpio_output] Writing variables to file:\n\t " + filename + " ...done! (Elapsed time = " + std::to_string(duration_write/1000.0) +" seconds)\n"); + m_atm_logger->info(" Done! Elapsed time: " + std::to_string(duration_write/1000.0) +" seconds"); } } } // run @@ -837,18 +818,20 @@ void AtmosphereOutput::register_views() m_host_views_1d.emplace(name,Kokkos::create_mirror(m_dev_views_1d[name])); } - // Now create and store a dev view to track the averaging count for this layout (if we are tracking) - // We don't need to track average counts for files that are not tracking the time dim - set_avg_cnt_tracking(name,"",layout); + if (m_track_avg_cnt) { + // Now create and store a dev view to track the averaging count for this layout (if we are tracking) + // We don't need to track average counts for files that are not tracking the time dim + set_avg_cnt_tracking(name,layout); + } } // Initialize the local views reset_dev_views(); } /* ---------------------------------------------------------- */ -void AtmosphereOutput::set_avg_cnt_tracking(const std::string& name, const std::string& avg_cnt_suffix, const FieldLayout& layout) +void AtmosphereOutput::set_avg_cnt_tracking(const std::string& name, const FieldLayout& layout) { - // Make sure this field "name" hasn't already been regsitered with avg_cnt tracking. + // Make sure this field "name" hasn't already been registered with avg_cnt tracking. // Note, we check this because some diagnostics need to have their own tracking which // is created at the 'create_diagnostics' function. if (m_field_to_avg_cnt_map.count(name)>0) { @@ -866,9 +849,10 @@ void AtmosphereOutput::set_avg_cnt_tracking(const std::string& name, const std:: // Now create and store a dev view to track the averaging count for this layout (if we are tracking) // We don't need to track average counts for files that are not tracking the time dim + const auto& avg_cnt_suffix = m_field_to_avg_cnt_suffix[name]; const auto size = layout.size(); const auto tags = layout.tags(); - if (m_add_time_dim && m_track_avg_cnt) { + if (m_track_avg_cnt) { std::string avg_cnt_name = "avg_count" + avg_cnt_suffix; for (int ii=0; iiget_dim_name(layout.tag(ii)); @@ -879,7 +863,6 @@ void AtmosphereOutput::set_avg_cnt_tracking(const std::string& name, const std:: } m_field_to_avg_cnt_map.emplace(name,avg_cnt_name); m_dev_views_1d.emplace(avg_cnt_name,view_1d_dev("",size)); // Note, emplace will only add a new key if one isn't already there - m_local_tmp_avg_cnt_views_1d.emplace(avg_cnt_name,view_1d_dev("",size)); // Note, emplace will only add a new key if one isn't already there m_host_views_1d.emplace(avg_cnt_name,Kokkos::create_mirror(m_dev_views_1d[avg_cnt_name])); m_layouts.emplace(avg_cnt_name,layout); } @@ -890,7 +873,7 @@ reset_dev_views() { // Reset the local device views depending on the averaging type // Init dev view with an "identity" for avg_type - const Real fill_for_average = (m_track_avg_cnt && m_add_time_dim) ? m_fill_value : 0.0; + const Real fill_for_average = m_track_avg_cnt ? m_fill_value : 0.0; for (auto const& name : m_fields_names) { switch (m_avg_type) { case OutputAvgType::Instant: @@ -1019,7 +1002,7 @@ register_variables(const std::string& filename, } // If tracking average count variables then add the name of the tracking variable for this variable - if (m_track_avg_cnt && m_add_time_dim) { + if (m_track_avg_cnt) { const auto lookup = m_field_to_avg_cnt_map.at(name); set_variable_metadata(filename,name,"averaging_count_tracker",lookup); } @@ -1033,7 +1016,7 @@ register_variables(const std::string& filename, } } // Now register the average count variables - if (m_track_avg_cnt && m_add_time_dim) { + if (m_track_avg_cnt) { for (const auto& name : m_avg_cnt_names) { const auto layout = m_layouts.at(name); auto io_decomp_tag = set_decomp_tag(layout); @@ -1047,6 +1030,15 @@ register_variables(const std::string& filename, std::vector AtmosphereOutput::get_var_dof_offsets(const FieldLayout& layout) { + using namespace ShortFieldTagsNames; + + // Precompute this *before* the early return, since it involves collectives. + // If one rank owns zero cols, and returns prematurely, the others will be left waiting. + AbstractGrid::gid_type min_gid; + if (layout.has_tag(COL) or layout.has_tag(EL)) { + min_gid = m_io_grid->get_global_min_dof_gid(); + } + // It may be that this MPI rank owns no chunk of the field if (layout.size()==0) { return {}; @@ -1071,17 +1063,13 @@ AtmosphereOutput::get_var_dof_offsets(const FieldLayout& layout) // NOTE: In the case of regional output this rank may have 0 columns to write, thus, var_dof // should be empty, we check for this special case and return an empty var_dof. auto dofs_h = m_io_grid->get_dofs_gids().get_view(); - if (layout.has_tag(ShortFieldTagsNames::COL)) { + if (layout.has_tag(COL)) { const int num_cols = m_io_grid->get_num_local_dofs(); // Note: col_size might be *larger* than the number of vertical levels, or even smaller. // E.g., (ncols,2,nlevs), or (ncols,2) respectively. scorpio::offset_t col_size = layout.size() / num_cols; - // Precompute this *before* the loop, since it involves expensive collectives. - // Besides, the loop might have different length on different ranks, so - // computing it inside might cause deadlocks. - auto min_gid = m_io_grid->get_global_min_dof_gid(); for (int icol=0; icolget_2d_scalar_layout(); const int num_my_elems = layout2d.dim(0); const int ngp = layout2d.dim(1); @@ -1102,10 +1090,6 @@ AtmosphereOutput::get_var_dof_offsets(const FieldLayout& layout) // E.g., (ncols,2,nlevs), or (ncols,2) respectively. scorpio::offset_t col_size = layout.size() / num_cols; - // Precompute this *before* the loop, since it involves expensive collectives. - // Besides, the loop might have different length on different ranks, so - // computing it inside might cause deadlocks. - auto min_gid = m_io_grid->get_global_min_dof_gid(); for (int ie=0,icol=0; ieinitialize(util::TimeStamp(),RunType::Initial); // If specified, set avg_cnt tracking for this diagnostic. - if (m_add_time_dim && m_track_avg_cnt) { + if (m_track_avg_cnt) { const auto diag_field = diag->get_diagnostic(); const auto name = diag_field.name(); - const auto layout = diag_field.get_header().get_identifier().get_layout(); - set_avg_cnt_tracking(name,diag_avg_cnt_name,layout); + m_field_to_avg_cnt_suffix.emplace(name,diag_avg_cnt_name); } return diag; @@ -1368,105 +1371,98 @@ AtmosphereOutput::create_diagnostic (const std::string& diag_field_name) { // Helper function to mark filled points in a specific layout void AtmosphereOutput:: update_avg_cnt_view(const Field& field, view_1d_dev& dev_view) { - // If the dev_view_1d is aliasing the field device view (must be Instant output), - // then there's no point in copying from the field's view to dev_view const auto& name = field.name(); const auto& layout = m_layouts.at(name); const auto& dims = layout.dims(); - const auto rank = layout.rank(); auto data = dev_view.data(); - const bool is_diagnostic = (m_diagnostics.find(name) != m_diagnostics.end()); - const bool is_aliasing_field_view = - m_avg_type==OutputAvgType::Instant && - field.get_header().get_alloc_properties().get_padding()==0 && - field.get_header().get_parent().expired() && - not is_diagnostic; const auto fill_value = m_fill_value; - if (not is_aliasing_field_view) { - KT::RangePolicy policy(0,layout.size()); - const auto extents = layout.extents(); - switch (rank) { - case 1: - { - // For rank-1 views, we use strided layout, since it helps us - // handling a few more scenarios - auto src_view_1d = field.get_strided_view(); - auto tgt_view_1d = view_Nd_dev<1>(data,dims[0]); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int i) { - if (src_view_1d(i)==fill_value) { - tgt_view_1d(i) = 0.0; - } - }); - break; - } - case 2: - { - auto src_view_2d = field.get_view(); - auto tgt_view_2d = view_Nd_dev<2>(data,dims[0],dims[1]); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { - int i,j; - unflatten_idx(idx,extents,i,j); - if (src_view_2d(i,j)==fill_value) { - tgt_view_2d(i,j) = 0.0; - } - }); - break; - } - case 3: - { - auto src_view_3d = field.get_view(); - auto tgt_view_3d = view_Nd_dev<3>(data,dims[0],dims[1],dims[2]); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { - int i,j,k; - unflatten_idx(idx,extents,i,j,k); - if (src_view_3d(i,j,k)==fill_value) { - tgt_view_3d(i,j,k) = 0.0; - } - }); - break; - } - case 4: - { - auto src_view_4d = field.get_view(); - auto tgt_view_4d = view_Nd_dev<4>(data,dims[0],dims[1],dims[2],dims[3]); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { - int i,j,k,l; - unflatten_idx(idx,extents,i,j,k,l); - if (src_view_4d(i,j,k,l)==fill_value) { - tgt_view_4d(i,j,k,l) = 0.0; - } - }); - break; - } - case 5: - { - auto src_view_5d = field.get_view(); - auto tgt_view_5d = view_Nd_dev<5>(data,dims[0],dims[1],dims[2],dims[3],dims[4]); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { - int i,j,k,l,m; - unflatten_idx(idx,extents,i,j,k,l,m); - if (src_view_5d(i,j,k,l,m)==fill_value) { - tgt_view_5d(i,j,k,l,m) = 0.0; - } - }); - break; - } - case 6: - { - auto src_view_6d = field.get_view(); - auto tgt_view_6d = view_Nd_dev<6>(data,dims[0],dims[1],dims[2],dims[3],dims[4],dims[5]); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { - int i,j,k,l,m,n; - unflatten_idx(idx,extents,i,j,k,l,m,n); - if (src_view_6d(i,j,k,l,m,n)==fill_value) { - tgt_view_6d(i,j,k,l,m,n) = 0.0; - } - }); - break; - } - default: - EKAT_ERROR_MSG ("Error! Field rank (" + std::to_string(rank) + ") not supported by AtmosphereOutput.\n"); + + KT::RangePolicy policy(0,layout.size()); + const auto extents = layout.extents(); + switch (layout.rank()) { + case 1: + { + // For rank-1 views, we use strided layout, since it helps us + // handling a few more scenarios + auto src_view_1d = field.get_strided_view(); + auto tgt_view_1d = view_Nd_dev<1>(data,dims[0]); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int i) { + if (src_view_1d(i)!=fill_value) { + tgt_view_1d(i) += 1; + } + }); + break; } + case 2: + { + auto src_view_2d = field.get_view(); + auto tgt_view_2d = view_Nd_dev<2>(data,dims[0],dims[1]); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { + int i,j; + unflatten_idx(idx,extents,i,j); + if (src_view_2d(i,j)!=fill_value) { + tgt_view_2d(i,j) += 1; + } + }); + break; + } + case 3: + { + auto src_view_3d = field.get_view(); + auto tgt_view_3d = view_Nd_dev<3>(data,dims[0],dims[1],dims[2]); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { + int i,j,k; + unflatten_idx(idx,extents,i,j,k); + if (src_view_3d(i,j,k)!=fill_value) { + tgt_view_3d(i,j,k) += 1; + } + }); + break; + } + case 4: + { + auto src_view_4d = field.get_view(); + auto tgt_view_4d = view_Nd_dev<4>(data,dims[0],dims[1],dims[2],dims[3]); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { + int i,j,k,l; + unflatten_idx(idx,extents,i,j,k,l); + if (src_view_4d(i,j,k,l)!=fill_value) { + tgt_view_4d(i,j,k,l) += 1; + } + }); + break; + } + case 5: + { + auto src_view_5d = field.get_view(); + auto tgt_view_5d = view_Nd_dev<5>(data,dims[0],dims[1],dims[2],dims[3],dims[4]); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { + int i,j,k,l,m; + unflatten_idx(idx,extents,i,j,k,l,m); + if (src_view_5d(i,j,k,l,m)!=fill_value) { + tgt_view_5d(i,j,k,l,m) += 1; + } + }); + break; + } + case 6: + { + auto src_view_6d = field.get_view(); + auto tgt_view_6d = view_Nd_dev<6>(data,dims[0],dims[1],dims[2],dims[3],dims[4],dims[5]); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { + int i,j,k,l,m,n; + unflatten_idx(idx,extents,i,j,k,l,m,n); + if (src_view_6d(i,j,k,l,m,n)!=fill_value) { + tgt_view_6d(i,j,k,l,m,n) += 1; + } + }); + break; + } + default: + EKAT_ERROR_MSG ( + "Error! Field rank not not supported by AtmosphereOutput.\n" + " - field name: " + field.name() + "\n" + " - field layout: " + to_string(layout) + "\n"); } } diff --git a/components/eamxx/src/share/io/scorpio_output.hpp b/components/eamxx/src/share/io/scorpio_output.hpp index 25a3da0d170c..13877425f614 100644 --- a/components/eamxx/src/share/io/scorpio_output.hpp +++ b/components/eamxx/src/share/io/scorpio_output.hpp @@ -175,7 +175,7 @@ class AtmosphereOutput create_diagnostic (const std::string& diag_name); // Tracking the averaging of any filled values: - void set_avg_cnt_tracking(const std::string& name, const std::string& avg_cnt_suffix, const FieldLayout& layout); + void set_avg_cnt_tracking(const std::string& name, const FieldLayout& layout); // --- Internal variables --- // ekat::Comm m_comm; @@ -198,8 +198,8 @@ class AtmosphereOutput std::vector m_fields_names; std::vector m_avg_cnt_names; std::map m_field_to_avg_cnt_map; + std::map m_field_to_avg_cnt_suffix; std::map m_layouts; - std::map m_dofs; std::map> m_dims; std::map> m_diagnostics; std::map> m_diag_depends_on_diags; @@ -215,8 +215,6 @@ class AtmosphereOutput // Local views of each field to be used for "averaging" output and writing to file. std::map m_host_views_1d; std::map m_dev_views_1d; - std::map m_local_tmp_avg_cnt_views_1d; - std::map m_avg_coeff_views_1d; bool m_add_time_dim; bool m_track_avg_cnt = false; diff --git a/components/eamxx/src/share/io/scream_io_control.hpp b/components/eamxx/src/share/io/scream_io_control.hpp new file mode 100644 index 000000000000..63cef4111991 --- /dev/null +++ b/components/eamxx/src/share/io/scream_io_control.hpp @@ -0,0 +1,78 @@ +#ifndef SCREAM_IO_CONTROL_HPP +#define SCREAM_IO_CONTROL_HPP + +#include "share/util/scream_time_stamp.hpp" + +#include + +#include + +namespace scream +{ + +// Mini struct to hold IO frequency info +struct IOControl { + + // If frequency_units is not "none" or "never", frequency *must* be set to a positive number + int frequency = -1; + std::string frequency_units = "none"; + + int nsamples_since_last_write = 0; // Needed when updating output data, such as with the OAT::Average flag + + util::TimeStamp next_write_ts; + util::TimeStamp last_write_ts; + + bool output_enabled () const { + return frequency_units!="none" && frequency_units!="never"; + } + + bool is_write_step (const util::TimeStamp& ts) const { + if (not output_enabled()) return false; + return frequency_units=="nsteps" ? ts.get_num_steps()==next_write_ts.get_num_steps() + : ts==next_write_ts; + } + + // Computes next_write_ts from frequency and last_write_ts + void compute_next_write_ts () { + EKAT_REQUIRE_MSG (last_write_ts.is_valid(), + "Error! Cannot compute next_write_ts, since last_write_ts was never set.\n"); + if (frequency_units=="nsteps") { + // This avoids having an invalid date/time in the above check next time this fcn runs + next_write_ts = last_write_ts; + next_write_ts.set_num_steps(last_write_ts.get_num_steps()+frequency); + } else if (frequency_units=="nsecs") { + next_write_ts = last_write_ts; + next_write_ts += frequency; + } else if (frequency_units=="nmins") { + next_write_ts = last_write_ts; + next_write_ts += frequency*60; + } else if (frequency_units=="nhours") { + next_write_ts = last_write_ts; + next_write_ts += frequency*3600; + } else if (frequency_units=="ndays") { + next_write_ts = last_write_ts; + next_write_ts += frequency*86400; + } else if (frequency_units=="nmonths" or frequency_units=="nyears") { + auto date = last_write_ts.get_date(); + if (frequency_units=="nmonths") { + int temp = date[1] + frequency - 1; + date[1] = temp % 12 + 1; + date[0] += temp / 12; + } else { + date[0] += frequency; + } + + // Fix day, in case we moved to a month/year where current days. E.g., if last_write + // was on Mar 31st, and units='nmonths', next write is on Apr 30th. HOWEVER, this + // means we will *always* write on the 30th of each month after then, since we have + // no memory of the fact that we were writing on the 31st before. + date[2] = std::min(date[2],util::days_in_month(date[0],date[1])); + next_write_ts = util::TimeStamp(date,last_write_ts.get_time()); + } else { + EKAT_ERROR_MSG ("Error! Unrecognized/unsupported frequency unit '" + frequency_units + "'\n"); + } + } +}; + +} // namespace scream +#endif // SCREAM_IO_CONTROL_HPP diff --git a/components/eamxx/src/share/io/scream_io_file_specs.hpp b/components/eamxx/src/share/io/scream_io_file_specs.hpp new file mode 100644 index 000000000000..0e9ab2e042af --- /dev/null +++ b/components/eamxx/src/share/io/scream_io_file_specs.hpp @@ -0,0 +1,115 @@ +#ifndef SCREAM_IO_FILE_SPECS_HPP +#define SCREAM_IO_FILE_SPECS_HPP + +#include "share/io/scream_io_utils.hpp" +#include "share/util/scream_time_stamp.hpp" + +#include + +#include +#include + +namespace scream +{ + +// How the file capacity is specified +enum StorageType { + NumSnaps, // Fixed number of snaps per file + Monthly, // Each file contains output for one month + Yearly // Each file contains output for one year +}; + +inline std::string e2str (const StorageType st) { + switch (st) { + case NumSnaps: return "num_snapshots"; + case Yearly: return "one_year"; + case Monthly: return "one_month"; + default: return "unknown"; + } +} + +struct StorageSpecs { + + StorageType type = NumSnaps; + + // Current index ***in terms of this->type*** + // If type==NumSnaps, curr_idx=num_snapshots_in_file, + // otherwise it is the month/year index stored in this file + int curr_idx = -1; + + // A snapshot fits if + // - type=NumSnaps: the number of stored snaps is less than the max allowed per file. + // - otherwise: the snapshot month/year index match the one currently stored in the file + // or the file has no snapshot stored yet + bool snapshot_fits (const util::TimeStamp& t) { + const auto& idx = type==Monthly ? t.get_month() : t.get_year(); + switch (type) { + case Yearly: + case Monthly: + return curr_idx==-1 or curr_idx==idx; + case NumSnaps: + return num_snapshots_in_file::max(); +}; + +// Mini struct to hold some specs of an IO file +// To keep nc files small, we limit the number of snapshots in each nc file +// When the number of snapshots in a file reaches m_out_max_steps, it's time +// to close the out file, and open a new one. +struct IOFileSpecs { + + StorageSpecs storage = {}; + + bool is_open = false; + std::string filename; + + // If positive, flush the output file every these many snapshots + int flush_frequency = std::numeric_limits::max(); + + // bool file_is_full () const { return num_snapshots_in_file>=max_snapshots_in_file; } + bool file_needs_flush () const { + return storage.num_snapshots_in_file%flush_frequency==0; + } + + // Whether it is a model output, model restart, or history restart file + FileType ftype = FileType::Unset; + + bool is_restart_file () const { + return ftype==FileType::ModelRestart or ftype==FileType::HistoryRestart; + } + + std::string suffix () const { + if (ftype==FileType::HistoryRestart) + return ".rhist"; + else if (ftype==FileType::ModelRestart) + return ".r"; + else + return ""; + } + + void close () { + is_open = false; + storage.num_snapshots_in_file = 0; + storage.curr_idx = -1; + } +}; + +} // namespace scream +#endif // SCREAM_IO_FILE_SPECS_HPP diff --git a/components/eamxx/src/share/io/scream_io_utils.hpp b/components/eamxx/src/share/io/scream_io_utils.hpp index e4d803703128..8cbcd39a1a2c 100644 --- a/components/eamxx/src/share/io/scream_io_utils.hpp +++ b/components/eamxx/src/share/io/scream_io_utils.hpp @@ -3,14 +3,31 @@ #include "share/util/scream_time_stamp.hpp" -#include "ekat/util/ekat_string_utils.hpp" -#include "ekat/mpi/ekat_comm.hpp" +#include +#include #include namespace scream { +enum class FileType { + ModelOutput, + ModelRestart, + HistoryRestart, + Unset +}; + +inline std::string e2str(const FileType avg) { + using FT = FileType; + switch (avg) { + case FT::ModelOutput: return "model-output"; + case FT::ModelRestart: return "model-restart"; + case FT::HistoryRestart: return "history-restart"; + default: return "UNSET"; + } +} + enum class OutputAvgType { Instant, Max, @@ -42,95 +59,6 @@ inline OutputAvgType str2avg (const std::string& s) { return OAT::Invalid; } -// Mini struct to hold IO frequency info -struct IOControl { - // If units is not "none" or "never", freq *must* be set to a positive number - int frequency = -1; - int nsamples_since_last_write = 0; // Needed when updating output data, such as with the OAT::Average flag - util::TimeStamp timestamp_of_last_write; - std::string frequency_units = "none"; - - bool output_enabled () const { - return frequency_units!="none" && frequency_units!="never"; - } - - bool is_write_step (const util::TimeStamp& ts) { - // Mini-routine to determine if it is time to write output to file. - // The current allowable options are nsteps, nsecs, nmins, nhours, ndays, nmonths, nyears - // We query the frequency_units string value to determine which option it is. - bool ret = false; - if (frequency_units != "never" && frequency_units != "none") { - auto ts_diff = (ts-timestamp_of_last_write); - if (frequency_units == "nsteps") { - // Just use the num_steps from timestamps - return ((ts.get_num_steps()-timestamp_of_last_write.get_num_steps()) % frequency == 0); - // We will need to use timestamp information - } else if (frequency_units == "nsecs") { - ret = ((ts_diff > 0) && (ts_diff % frequency == 0)); - } else if (frequency_units == "nmins") { - ret = (ts_diff >= 60) && (ts_diff % (frequency*60) == 0); - } else if (frequency_units == "nhours") { - ret = (ts_diff >= 3600) && (ts_diff % (frequency*3600) == 0); - } else if (frequency_units == "ndays") { - ret = (ts_diff >= 86400) && (ts_diff % (frequency*86400) == 0); - } else if (frequency_units == "nmonths" || frequency_units == "nyears") { - // For months and years we need to be careful, can't just divide ts_diff by a set value. - // First we make sure that if we are the same day of the month and at the same time of day. - // TODO: Potential bug, if the day of last write >=29 there is a chance that we won't write - // in some subset of months, think Feb (28 days) and all of the months with only 30 days. - if (ts.get_day() == timestamp_of_last_write.get_day() && - ts.sec_of_day() == timestamp_of_last_write.sec_of_day()) { - auto diff = 0; - // Determine how many years have passed - diff += (ts.get_year() - timestamp_of_last_write.get_year()); - if (frequency_units == "nyears") { - ret = (diff>0) && (diff % frequency == 0); - } - // Determine number of months that have passed - diff *= 12; - diff += (ts.get_month() - timestamp_of_last_write.get_month()); - if (frequency_units == "nmonths") { - ret = (diff>0) && (diff % frequency == 0); - } - } - } else { - EKAT_REQUIRE_MSG(false,"Invalid frequency unit of [" + frequency_units + "] for output stream. Please check that all outputs have frequency_units of\n" - "none, never, nsteps, nsecs, nmins, nhours, ndays, nmonths, nyears"); - } - } - return ret; - } // End function is_write_step -}; - -// Mini struct to hold some specs of an IO file -// To keep nc files small, we limit the number of snapshots in each nc file -// When the number of snapshots in a file reaches m_out_max_steps, it's time -// to close the out file, and open a new one. -struct IOFileSpecs { - bool is_open = false; - std::string filename; - int num_snapshots_in_file = 0; - int max_snapshots_in_file; - - // If positive, flush the output file every these many snapshots - int flush_frequency = -1; - - bool file_is_full () const { return num_snapshots_in_file>=max_snapshots_in_file; } - bool file_needs_flush () const { return flush_frequency>0 and num_snapshots_in_file%flush_frequency==0; } - - // Adding number of MPI ranks to the filenamea is useful in testing, since we can run - // multiple instances of the same test in parallel (with different number of ranks), - // without the risk of them overwriting each other output. - // For production runs, this is not desirable. - bool filename_with_mpiranks = false; - - bool save_grid_data = true; - - // Whether this struct refers to a history restart file - bool hist_restart_file = false; - -}; - std::string find_filename_in_rpointer ( const std::string& casename, const bool model_restart, diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 263cb9e8803f..e9f8f5887025 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -55,36 +55,9 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, m_is_restarted_run = (case_t0("use_case_as_start_reference",!m_is_model_restart_output); - m_output_control.frequency_units = out_control_pl.get("frequency_units"); - // In case output is disabled, no point in doing anything else - if (m_output_control.frequency_units=="none" || m_output_control.frequency_units=="never") { - m_output_disabled = true; - return; - } - m_output_control.frequency = out_control_pl.get("Frequency"); - EKAT_REQUIRE_MSG (m_output_control.frequency>0, - "Error! Invalid frequency (" + std::to_string(m_output_control.frequency) + ") in Output Control. Please, use positive number.\n"); - - m_output_control.timestamp_of_last_write = start_ref ? m_case_t0 : m_run_t0; - - // File specs - constexpr auto large_int = 1000000; - m_output_file_specs.max_snapshots_in_file = m_params.get("Max Snapshots Per File",large_int); - m_output_file_specs.filename_with_mpiranks = out_control_pl.get("MPI Ranks in Filename",false); - m_output_file_specs.save_grid_data = out_control_pl.get("save_grid_data",!m_is_model_restart_output); - // Here, store if PG2 fields will be present in output streams. // Will be useful if multiple grids are defined (see below). bool pg2_grid_in_io_streams = false; @@ -136,7 +109,7 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, // For normal output, setup the geometry data streams, which we used to write the // geo data in the output file when we create it. - if (m_output_file_specs.save_grid_data) { + if (m_save_grid_data) { std::map> grids; for (const auto& it : m_output_streams) { grids[it->get_io_grid()->name()] = it->get_io_grid(); @@ -177,65 +150,38 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, } } - if (m_params.isSublist("Checkpoint Control")) { - // Output control - // TODO: It would be great if there was an option where, if Checkpoint Control was not a sublist, we - // could query the restart control information and just use that. - auto& pl = m_params.sublist("Checkpoint Control"); - m_checkpoint_control.frequency_units = pl.get("frequency_units"); - - if (m_checkpoint_control.output_enabled()) { - m_checkpoint_control.timestamp_of_last_write = run_t0; - m_checkpoint_control.frequency = pl.get("Frequency"); - EKAT_REQUIRE_MSG (m_output_control.frequency>0, - "Error! Invalid frequency (" + std::to_string(m_checkpoint_control.frequency) + ") in Checkpoint Control. Please, use positive number.\n"); - - // File specs - m_checkpoint_file_specs.max_snapshots_in_file = 1; - m_checkpoint_file_specs.flush_frequency = 1; - m_checkpoint_file_specs.filename_with_mpiranks = pl.get("MPI Ranks in Filename",false); - m_checkpoint_file_specs.save_grid_data = false; - m_checkpoint_file_specs.hist_restart_file = true; - } - } - - // If this is normal output (not the model restart output) and the output specs - // require it, we need to restart the output history. - // E.g., we might save 30-day avg value for field F, but due to job size - // break the run into three 10-day runs. We then need to save the state of - // our averaging in a "restart" file (e.g., the current avg). + // If this is model output (not model restart) we need to restart the output history. + // For instant output, this just entails restarting timestamps and counters, while + // for average output we also need to restore the accumulation state of the output fields // Note: the user might decide *not* to restart the output, so give the option // of disabling the restart. Also, the user might want to change the // filename_prefix, so allow to specify a different filename_prefix for the restart file. - if (m_is_restarted_run) { + if (m_is_restarted_run and not m_is_model_restart_output) { // Allow to skip history restart, or to specify a filename_prefix for the restart file // that is different from the filename_prefix of the current output. auto& restart_pl = m_params.sublist("Restart"); bool perform_history_restart = restart_pl.get("Perform Restart",true); auto hist_restart_filename_prefix = restart_pl.get("filename_prefix",m_filename_prefix); - if (m_is_model_restart_output) { - // For model restart output, the restart time (which is the start time of this run) is precisely - // when the last write happened, so we can quickly init the output control. - m_output_control.timestamp_of_last_write = m_run_t0; - m_output_control.nsamples_since_last_write = 0; - } else if (perform_history_restart) { + if (perform_history_restart) { using namespace scorpio; auto rhist_file = find_filename_in_rpointer(hist_restart_filename_prefix,false,m_io_comm,m_run_t0); + scorpio::register_file(rhist_file,scorpio::Read); // From restart file, get the time of last write, as well as the current size of the avg sample - m_output_control.timestamp_of_last_write = read_timestamp(rhist_file,"last_write"); + m_output_control.last_write_ts = read_timestamp(rhist_file,"last_write",true); + m_output_control.compute_next_write_ts(); m_output_control.nsamples_since_last_write = get_attribute(rhist_file,"num_snapshots_since_last_write"); if (m_avg_type!=OutputAvgType::Instant) { m_time_bnds.resize(2); - m_time_bnds[0] = m_output_control.timestamp_of_last_write.days_from(m_case_t0); + m_time_bnds[0] = m_output_control.last_write_ts.days_from(m_case_t0); } // If the type/freq of output needs restart data, we need to restart the streams - const bool output_every_step = m_output_control.frequency_units=="nsteps" && - m_output_control.frequency==1; - const bool has_checkpoint_data = m_avg_type!=OutputAvgType::Instant && not output_every_step; + const bool output_every_step = m_output_control.frequency_units=="nsteps" && + m_output_control.frequency==1; + const bool has_checkpoint_data = m_avg_type!=OutputAvgType::Instant && not output_every_step; if (has_checkpoint_data && m_output_control.nsamples_since_last_write>0) { for (auto stream : m_output_streams) { stream->restart(rhist_file); @@ -259,14 +205,26 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, "Error! Cannot change avg type when performing history restart.\n" " - old avg type: " << old_avg_type + "\n" " - new avg type: " << e2str(m_avg_type) << "\n"); - auto old_max_snaps = scorpio::get_attribute(rhist_file,"max_snapshots_per_file"); - EKAT_REQUIRE_MSG (old_max_snaps == m_output_file_specs.max_snapshots_in_file, - "Error! Cannot change max snapshots per file when performing history restart.\n" - " - old max snaps: " << old_max_snaps << "\n" - " - new max snaps: " << m_output_file_specs.max_snapshots_in_file << "\n" - "If you *really* want to change the file capacity, you need to force using a new file, setting\n" + + + auto old_storage_type = scorpio::get_attribute(rhist_file,"file_max_storage_type"); + EKAT_REQUIRE_MSG (old_storage_type == e2str(m_output_file_specs.storage.type), + "Error! Cannot change file storage type when performing history restart.\n" + " - old file_max_storage_type: " << old_storage_type << "\n" + " - new file_max_storage_type: " << e2str(m_output_file_specs.storage.type) << "\n" + "If you *really* want to change the file storage type, you need to force using a new file, setting\n" " Restart:\n" " force_new_file: true\n"); + if (old_storage_type=="num_snapshot") { + auto old_max_snaps = scorpio::get_attribute(rhist_file,"max_snapshots_per_file"); + EKAT_REQUIRE_MSG (old_max_snaps == m_output_file_specs.storage.max_snapshots_in_file, + "Error! Cannot change max snapshots per file when performing history restart.\n" + " - old max snaps: " << old_max_snaps << "\n" + " - new max snaps: " << m_output_file_specs.storage.max_snapshots_in_file << "\n" + "If you *really* want to change the file capacity, you need to force using a new file, setting\n" + " Restart:\n" + " force_new_file: true\n"); + } std::string fp_precision = m_params.get("Floating Point Precision"); auto old_fp_precision = scorpio::get_attribute(rhist_file,"fp_precision"); EKAT_REQUIRE_MSG (old_fp_precision == fp_precision, @@ -279,26 +237,20 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, const auto& last_output_filename = get_attribute(rhist_file,"last_output_filename"); m_resume_output_file = last_output_filename!="" and not restart_pl.get("force_new_file",false); if (m_resume_output_file) { + scorpio::register_file(last_output_filename,scorpio::Read); int num_snaps = scorpio::get_dimlen(last_output_filename,"time"); - - // End of checks. Close the file. scorpio::eam_pio_closefile(last_output_filename); - // If last output was full, we can no longer try to resume the file - if (num_snaps("skip_t0_output",false)) // This will be true for ERS/ERP tests + { + // In order to trigger a t0 write, we need to have next_write_ts matching run_t0 + m_output_control.next_write_ts = m_run_t0; this->run(m_run_t0); } @@ -328,7 +286,7 @@ add_global (const std::string& name, const ekat::any& global) { void OutputManager::run(const util::TimeStamp& timestamp) { // In case output is disabled, no point in doing anything else - if (m_output_disabled) { + if (not m_output_control.output_enabled()) { return; } @@ -340,6 +298,7 @@ void OutputManager::run(const util::TimeStamp& timestamp) std::string timer_root = m_is_model_restart_output ? "EAMxx::IO::restart" : "EAMxx::IO::standard"; start_timer(timer_root); + // Check if we need to open a new file ++m_output_control.nsamples_since_last_write; ++m_checkpoint_control.nsamples_since_last_write; @@ -365,17 +324,32 @@ void OutputManager::run(const util::TimeStamp& timestamp) // Create and setup output/checkpoint file(s), if necessary start_timer(timer_root+"::get_new_file"); - auto setup_output_file = [&](IOControl& control, IOFileSpecs& filespecs, - bool add_to_rpointer, const std::string& file_type) { + auto setup_output_file = [&](IOControl& control, IOFileSpecs& filespecs) { + // Check if the new snapshot fits, if not, close the file + // NOTE: if output is average/max/min AND we save one file per month/year, + // we don't want to check if *this* timestamp fits in the file, since + // it may be in the next month/year even though most of the time averaging + // window is in the right year. E.g., if the avg is over the whole month, + // you would end up saving Jan average in the Feb file, since the end + // of the window is Feb 1st 00:00:00. So instead, we use the *start* + // of the avg window. If the avg is such that it spans 2 months (e.g., + // a 7-day avg), then where we put the avg is arbitrary, so our choice + // is still fine. + util::TimeStamp snapshot_start; + if (m_avg_type==OutputAvgType::Instant or filespecs.storage.type==NumSnaps) { + snapshot_start = timestamp; + } else { + snapshot_start = m_case_t0; + snapshot_start += m_time_bnds[0]; + } + if (not filespecs.storage.snapshot_fits(snapshot_start)) { + eam_pio_closefile(filespecs.filename); + filespecs.close(); + } + // Check if we need to open a new file if (not filespecs.is_open) { - // If this is normal output, with some sort of average, then the timestamp should be - // the one of the last write, since that's when the current avg window started. - // For Instant output (includes model restart output) and history restart, use the current timestamp - auto file_ts = m_avg_type==OutputAvgType::Instant or filespecs.hist_restart_file - ? timestamp : control.timestamp_of_last_write; - - filespecs.filename = compute_filename (control,filespecs,file_ts); + filespecs.filename = compute_filename (control,filespecs,timestamp); // Register all dims/vars, write geometry data (e.g. lat/lon/hyam/hybm) setup_file(filespecs,control); } @@ -383,7 +357,7 @@ void OutputManager::run(const util::TimeStamp& timestamp) // If we are going to write an output checkpoint file, or a model restart file, // we need to append to the filename ".rhist" or ".r" respectively, and add // the filename to the rpointer.atm file. - if (add_to_rpointer && m_io_comm.am_i_root()) { + if (m_io_comm.am_i_root() and filespecs.is_restart_file()) { std::ofstream rpointer; if (m_is_model_restart_output) { rpointer.open("rpointer.atm"); // Open rpointer and nuke its content @@ -406,19 +380,19 @@ void OutputManager::run(const util::TimeStamp& timestamp) } if (m_atm_logger) { - m_atm_logger->info("[EAMxx::output_manager] - Writing " + file_type + ":"); + m_atm_logger->info("[EAMxx::output_manager] - Writing " + e2str(filespecs.ftype) + ":"); m_atm_logger->info("[EAMxx::output_manager] FILE: " + filespecs.filename); } }; if (is_output_step) { - setup_output_file(m_output_control,m_output_file_specs,m_is_model_restart_output,m_is_model_restart_output ? "model restart" : "model output"); + setup_output_file(m_output_control,m_output_file_specs); // Update time (must be done _before_ writing fields) pio_update_time(m_output_file_specs.filename,timestamp.days_from(m_case_t0)); } if (is_checkpoint_step) { - setup_output_file(m_checkpoint_control,m_checkpoint_file_specs,true,"history restart"); + setup_output_file(m_checkpoint_control,m_checkpoint_file_specs); if (is_full_checkpoint_step) { // Update time (must be done _before_ writing fields) @@ -455,13 +429,19 @@ void OutputManager::run(const util::TimeStamp& timestamp) if (m_atm_logger) { m_atm_logger->debug("[OutputManager]: writing globals...\n"); } + + // Since we wrote to file we need to reset the timestamps + control.last_write_ts = timestamp; + control.compute_next_write_ts(); + control.nsamples_since_last_write = 0; + if (m_is_model_restart_output) { // Only write nsteps on model restart set_attribute(filespecs.filename,"nsteps",timestamp.get_num_steps()); } else { - if (filespecs.hist_restart_file) { + if (filespecs.ftype==FileType::HistoryRestart) { // Update the date of last write and sample size - scorpio::write_timestamp (filespecs.filename,"last_write",m_output_control.timestamp_of_last_write); + scorpio::write_timestamp (filespecs.filename,"last_write",m_output_control.last_write_ts,true); scorpio::set_attribute (filespecs.filename,"last_output_filename",m_output_file_specs.filename); scorpio::set_attribute (filespecs.filename,"num_snapshots_since_last_write",m_output_control.nsamples_since_last_write); } @@ -470,7 +450,10 @@ void OutputManager::run(const util::TimeStamp& timestamp) set_attribute(filespecs.filename,"averaging_type",e2str(m_avg_type)); set_attribute(filespecs.filename,"averaging_frequency_units",m_output_control.frequency_units); set_attribute(filespecs.filename,"averaging_frequency",m_output_control.frequency); - set_attribute(filespecs.filename,"max_snapshots_per_file",m_output_file_specs.max_snapshots_in_file); + set_attribute(filespecs.filename,"file_max_storage_type",e2str(m_output_file_specs.storage.type)); + if (m_output_file_specs.storage.type==NumSnaps) { + set_attribute(filespecs.filename,"max_snapshots_per_file",m_output_file_specs.storage.max_snapshots_in_file); + } const auto& fp_precision = m_params.get("Floating Point Precision"); set_attribute(filespecs.filename,"fp_precision",fp_precision); } @@ -483,29 +466,21 @@ void OutputManager::run(const util::TimeStamp& timestamp) } // We're adding one snapshot to the file - ++filespecs.num_snapshots_in_file; + filespecs.storage.update_storage(timestamp); if (m_time_bnds.size()>0) { scorpio::grid_write_data_array(filespecs.filename, "time_bnds", m_time_bnds.data(), 2); } - // Since we wrote to file we need to reset the nsamples_since_last_write, the timestamp ... - control.nsamples_since_last_write = 0; - control.timestamp_of_last_write = timestamp; - - // Check if we need to close the output file - if (filespecs.file_is_full()) { - eam_pio_closefile(filespecs.filename); - filespecs.num_snapshots_in_file = 0; - filespecs.is_open = false; - } else if (filespecs.file_needs_flush()) { + // Check if we need to flush the output file + if (filespecs.file_needs_flush()) { eam_flush_file (filespecs.filename); } }; start_timer(timer_root+"::update_snapshot_tally"); // Important! Process output file first, and hist restart (if any) second. - // That's b/c write_global_data will update m_output_control.timestamp_of_last_write, + // That's b/c write_global_data will update m_output_control.last_write_ts, // which is later written as global data in the hist restart file if (is_output_step) { write_global_data(m_output_control,m_output_file_specs); @@ -551,25 +526,30 @@ compute_filename (const IOControl& control, const IOFileSpecs& file_specs, const util::TimeStamp& timestamp) const { - std::string suffix = - file_specs.hist_restart_file ? ".rhist" - : (m_is_model_restart_output ? ".r" : ""); - auto filename = m_filename_prefix + suffix; + auto filename = m_filename_prefix + file_specs.suffix(); // Always add avg type and frequency info filename += "." + e2str(m_avg_type); filename += "." + control.frequency_units+ "_x" + std::to_string(control.frequency); // Optionally, add number of mpi ranks (useful mostly in unit tests, to run multiple MPI configs in parallel) - if (file_specs.filename_with_mpiranks) { + if (m_params.get("MPI Ranks in Filename")) { filename += ".np" + std::to_string(m_io_comm.size()); } // Always add a time stamp - if (m_avg_type==OutputAvgType::Instant || file_specs.hist_restart_file) { - filename += "." + timestamp.to_string(); - } else { - filename += "." + control.timestamp_of_last_write.to_string(); + auto ts = (m_avg_type==OutputAvgType::Instant || file_specs.ftype==FileType::HistoryRestart) + ? timestamp : control.last_write_ts; + + switch (file_specs.storage.type) { + case NumSnaps: + filename += "." + ts.to_string(); break; + case Yearly: + filename += "." + std::to_string(ts.get_year()); break; + case Monthly: + filename += "." + std::to_string(ts.get_year()) + "-" + std::to_string(ts.get_month()); break; + default: + EKAT_ERROR_MSG ("Error! Unrecognized/unsupported file storage type.\n"); } return filename + ".nc"; @@ -585,21 +565,10 @@ set_params (const ekat::ParameterList& params, if (m_is_model_restart_output) { // We build some restart parameters internally - auto avg_type = m_params.get("Averaging Type","INSTANT"); - m_avg_type = str2avg(avg_type); - EKAT_REQUIRE_MSG (m_avg_type==OutputAvgType::Instant, - "Error! For restart output, the averaging type must be 'Instant'.\n" - " Note: you don't have to specify this parameter for restart output.\n"); - - m_output_file_specs.max_snapshots_in_file = m_params.get("Max Snapshots Per File",1); - EKAT_REQUIRE_MSG (m_output_file_specs.max_snapshots_in_file==1, - "Error! For restart output, max snapshots per file must be 1.\n" - " Note: you don't have to specify this parameter for restart output.\n"); - - m_output_file_specs.flush_frequency = m_params.get("flush_frequency",1); - EKAT_REQUIRE_MSG (m_output_file_specs.flush_frequency==1, - "Error! For restart output, file flush frequency must be 1.\n" - " Note: you don't have to specify this parameter for restart output.\n"); + m_avg_type = OutputAvgType::Instant; + m_output_file_specs.storage.type = NumSnaps; + m_output_file_specs.storage.max_snapshots_in_file = 1; + m_output_file_specs.flush_frequency = 1; auto& fields_pl = m_params.sublist("Fields"); for (const auto& it : field_mgrs) { @@ -617,7 +586,9 @@ set_params (const ekat::ParameterList& params, fields_pl.sublist(it.first).set("Field Names",fnames); } m_filename_prefix = m_params.get("filename_prefix"); - // Match precision of Fields + + // Hard code some parameters in case we access them later + m_params.set("MPI Ranks in Filename",false); m_params.set("Floating Point Precision","real"); } else { auto avg_type = m_params.get("Averaging Type"); @@ -626,10 +597,24 @@ set_params (const ekat::ParameterList& params, "Error! Unsupported averaging type '" + avg_type + "'.\n" " Valid options: Instant, Max, Min, Average. Case insensitive.\n"); - constexpr auto large_int = 1000000; - m_output_file_specs.max_snapshots_in_file = m_params.get("Max Snapshots Per File",large_int); + const auto& storage_type = m_params.get("file_max_storage_type","num_snapshots"); + auto& storage = m_output_file_specs.storage; + constexpr auto large_int = std::numeric_limits::max(); + if (storage_type=="num_snapshots") { + storage.type = NumSnaps; + storage.max_snapshots_in_file = m_params.get("Max Snapshots Per File",large_int); + EKAT_REQUIRE_MSG (storage.max_snapshots_in_file>0, + "Error! Value for 'Max Snapshots Per File' should be positive.\n" + " To request 'unlimited' storage, leave parameter unset.\n"); + } else if (storage_type=="one_year") { + storage.type = Yearly; + } else if (storage_type=="one_month") { + storage.type = Monthly; + } else { + EKAT_ERROR_MSG ("Error! Unrecognized/unsupported file storage type.\n"); + } m_filename_prefix = m_params.get("filename_prefix"); - m_output_file_specs.flush_frequency = m_params.get("flush_frequency",m_output_file_specs.max_snapshots_in_file); + m_output_file_specs.flush_frequency = m_params.get("flush_frequency",large_int); // Allow user to ask for higher precision for normal model output, // but default to single to save on storage @@ -639,8 +624,64 @@ set_params (const ekat::ParameterList& params, "Error! Invalid/unsupported value for 'Floating Point Precision'.\n" " - input value: " + prec + "\n" " - supported values: float, single, double, real\n"); + + // If not set, hard code to false for CIME cases, and true for standalone, + // since standalone may be running multiple versions of the same test at once + if (not m_params.isParameter("MPI Ranks in Filename")) { + m_params.set("MPI Ranks in Filename",is_scream_standalone()); + } + } + + // Output control + EKAT_REQUIRE_MSG(m_params.isSublist("output_control"), + "Error! The output control YAML file for " + m_filename_prefix + " is missing the sublist 'output_control'"); + auto& out_control_pl = m_params.sublist("output_control"); + m_output_control.frequency_units = out_control_pl.get("frequency_units"); + + // In case output is disabled, no point in doing anything else + if (m_output_control.frequency_units=="none" || m_output_control.frequency_units=="never") { + return; + } + m_output_control.frequency = out_control_pl.get("Frequency"); + EKAT_REQUIRE_MSG (m_output_control.frequency>0, + "Error! Invalid frequency (" + std::to_string(m_output_control.frequency) + ") in Output Control. Please, use positive number.\n"); + + // Determine which timestamp to use a reference for output frequency. Two options: + // 1. use_case_as_start_reference: TRUE - implies we want to calculate frequency from the beginning of the whole simulation, even if this is a restarted run. + // 2. use_case_as_start_reference: FALSE - implies we want to base the frequency of output on when this particular simulation started. + // Note, (2) is needed for restarts since the restart frequency in CIME assumes a reference of when this run began. + // NOTE: m_is_restarted_run and not m_is_model_restart_output, these timestamps will be corrected once we open the hist restart file + const bool use_case_as_ref = out_control_pl.get("use_case_as_start_reference",!m_is_model_restart_output); + const bool perform_history_restart = m_params.sublist("Restart").get("Perform Restart",true); + const auto& start_ref = use_case_as_ref and perform_history_restart ? m_case_t0 : m_run_t0; + m_output_control.last_write_ts = start_ref; + m_output_control.compute_next_write_ts(); + + // File specs + m_save_grid_data = out_control_pl.get("save_grid_data",!m_is_model_restart_output); + m_output_file_specs.ftype = m_is_model_restart_output ? FileType::ModelRestart : FileType::ModelOutput; + + if (m_params.isSublist("Checkpoint Control")) { + auto& pl = m_params.sublist("Checkpoint Control"); + m_checkpoint_control.frequency_units = pl.get("frequency_units"); + + if (m_checkpoint_control.output_enabled()) { + m_checkpoint_control.frequency = pl.get("Frequency"); + EKAT_REQUIRE_MSG (m_output_control.frequency>0, + "Error! Invalid frequency (" + std::to_string(m_checkpoint_control.frequency) + ") in Checkpoint Control. Please, use positive number.\n"); + + m_checkpoint_control.last_write_ts = m_run_t0; + m_checkpoint_control.compute_next_write_ts(); + + // File specs + m_checkpoint_file_specs.storage.type = NumSnaps; + m_checkpoint_file_specs.storage.max_snapshots_in_file = 1; + m_checkpoint_file_specs.flush_frequency = 1; + m_checkpoint_file_specs.ftype = FileType::HistoryRestart; + } } } + /*===============================================================================================*/ void OutputManager:: setup_file ( IOFileSpecs& filespecs, @@ -693,7 +734,10 @@ setup_file ( IOFileSpecs& filespecs, set_attribute(filename,"averaging_type",e2str(m_avg_type)); set_attribute(filename,"averaging_frequency_units",m_output_control.frequency_units); set_attribute(filename,"averaging_frequency",m_output_control.frequency); - set_attribute(filename,"max_snapshots_per_file",m_output_file_specs.max_snapshots_in_file); + set_attribute(filespecs.filename,"file_max_storage_type",e2str(m_output_file_specs.storage.type)); + if (m_output_file_specs.storage.type==NumSnaps) { + set_attribute(filename,"max_snapshots_per_file",m_output_file_specs.storage.max_snapshots_in_file); + } set_attribute(filename,"fp_precision",fp_precision); set_file_header(filespecs); } @@ -713,7 +757,7 @@ setup_file ( IOFileSpecs& filespecs, // If grid data is needed, also register geo data fields. Skip if file is resumed, // since grid data was written in the previous run - if (filespecs.save_grid_data and not m_resume_output_file) { + if (m_save_grid_data and not filespecs.is_restart_file() and not m_resume_output_file) { for (auto& it : m_geo_data_streams) { it->setup_output_file(filename,fp_precision,mode); } @@ -724,7 +768,7 @@ setup_file ( IOFileSpecs& filespecs, // the dims/vars are already in the file (we don't allow adding dims/vars) eam_pio_enddef (filename); - if (filespecs.save_grid_data and not m_resume_output_file) { + if (m_save_grid_data and not filespecs.is_restart_file() and not m_resume_output_file) { // Immediately run the geo data streams for (const auto& it : m_geo_data_streams) { it->run(filename,true,false,0); @@ -766,13 +810,7 @@ void OutputManager::set_file_header(const IOFileSpecs& file_specs) set_attribute(filename,"realm","atmos"); set_attribute(filename,"history",ts_str); set_attribute(filename,"Conventions","CF-1.8"); - if (m_is_model_restart_output) { - set_attribute(filename,"product","model-restart"); - } else if (file_specs.hist_restart_file) { - set_attribute(filename,"product","history-restart"); - } else { - set_attribute(filename,"product","model-output"); - } + set_attribute(filename,"product",e2str(file_specs.ftype)); } /*===============================================================================================*/ void OutputManager:: @@ -790,13 +828,28 @@ push_to_logger() m_atm_logger->info(" Filename prefix: " + m_filename_prefix); m_atm_logger->info(" Run t0: " + m_run_t0.to_string()); m_atm_logger->info(" Case t0: " + m_case_t0.to_string()); - m_atm_logger->info(" Reference t0: " + m_output_control.timestamp_of_last_write.to_string()); + m_atm_logger->info(" Reference t0: " + m_output_control.last_write_ts.to_string()); m_atm_logger->info(" Is Restart File ?: " + bool_to_string(m_is_model_restart_output)); m_atm_logger->info(" Is Restarted Run ?: " + bool_to_string(m_is_restarted_run)); m_atm_logger->info(" Averaging Type: " + e2str(m_avg_type)); m_atm_logger->info(" Output Frequency: " + std::to_string(m_output_control.frequency) + " " + m_output_control.frequency_units); - m_atm_logger->info(" Max snaps in file: " + std::to_string(m_output_file_specs.max_snapshots_in_file)); // TODO: add "not set" if the value is -1 - m_atm_logger->info(" Includes Grid Data ?: " + bool_to_string(m_output_file_specs.save_grid_data)); + switch (m_output_file_specs.storage.type) { + case NumSnaps: + { + auto ms = m_output_file_specs.storage.max_snapshots_in_file; + m_atm_logger->info(" File Capacity: " + (ms>0 ? std::to_string(ms) + "snapshots" : "UNLIMITED")); + break; + } + case Monthly: + m_atm_logger->info(" File Capacity: one month per file"); + break; + case Yearly: + m_atm_logger->info(" File Capacity: one year per file"); + break; + default: + EKAT_ERROR_MSG ("Error! Unrecognized/unsupported file storage type.\n"); + } + m_atm_logger->info(" Includes Grid Data ?: " + bool_to_string(m_save_grid_data)); // List each GRID - TODO // List all FIELDS - TODO } diff --git a/components/eamxx/src/share/io/scream_output_manager.hpp b/components/eamxx/src/share/io/scream_output_manager.hpp index e546922b4d6e..b707c61ee4c0 100644 --- a/components/eamxx/src/share/io/scream_output_manager.hpp +++ b/components/eamxx/src/share/io/scream_output_manager.hpp @@ -4,6 +4,8 @@ #include "share/io/scorpio_output.hpp" #include "share/io/scream_scorpio_interface.hpp" #include "share/io/scream_io_utils.hpp" +#include "share/io/scream_io_file_specs.hpp" +#include "share/io/scream_io_control.hpp" #include "share/field/field_manager.hpp" #include "share/grid/grids_manager.hpp" @@ -171,9 +173,6 @@ class OutputManager // Whether a restarted run can resume filling previous run output file (if not full) bool m_resume_output_file = false; - // If the user specifies freq units "none" or "never", output is disabled - bool m_output_disabled = false; - // The initial time stamp of the simulation and run. For initial runs, they coincide, // but for restarted runs, run_t0>case_t0, with the former being the time at which the // restart happens, and the latter being the start time of the *original* run. @@ -182,6 +181,9 @@ class OutputManager // The logger to be used throughout the ATM to log message std::shared_ptr m_atm_logger; + + // If true, we save grid data in output file + bool m_save_grid_data; }; } // namespace scream diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.F90 b/components/eamxx/src/share/io/scream_scorpio_interface.F90 index 6f8b83886547..50401f7f3051 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.F90 +++ b/components/eamxx/src/share/io/scream_scorpio_interface.F90 @@ -408,7 +408,7 @@ subroutine register_variable(filename,shortname,longname,units, & prev => curr curr => prev%next end do - + allocate(prev%next) curr => prev%next allocate(curr%var) @@ -562,7 +562,7 @@ function get_variable_metadata_float(filename, varname, metaname) result(metaval character(len=256), intent(in) :: varname character(len=256), intent(in) :: metaname real(kind=c_float) :: metaval - + ! Local variables type(pio_atm_file_t),pointer :: pio_file @@ -998,7 +998,7 @@ subroutine eam_pio_flush_file(fname) call lookup_pio_atm_file(trim(fname),pio_atm_file,found) if (found) then - if ( is_write(pio_atm_file%purpose) ) then + if ( is_write(pio_atm_file%purpose) .or. is_append(pio_atm_file%purpose) ) then call PIO_syncfile(pio_atm_file%pioFileDesc) else call errorHandle("PIO ERROR: unable to flush file: "//trim(fname)//", is not open in write mode",-999) @@ -1674,7 +1674,6 @@ subroutine grid_read_darray_double(filename, varname, buf, buf_size, time_index) call lookup_pio_atm_file(trim(filename),pio_atm_file,found) call get_var(pio_atm_file,varname,var) - ! Set the timesnap we are reading if (time_index .gt. 0) then ! The user has set a valid time index to read from diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 676b72f60aee..a74ceecd3930 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -190,7 +190,7 @@ bool has_variable (const std::string& filename, const std::string& varname) return false; } EKAT_REQUIRE_MSG (err==PIO_NOERR, - "Error! Something went wrong while retrieving dimension id.\n" + "Error! Something went wrong while retrieving variable id.\n" " - filename : " + filename + "\n" " - varname : " + varname + "\n" " - pio error: " + std::to_string(err) + "\n"); @@ -200,6 +200,54 @@ bool has_variable (const std::string& filename, const std::string& varname) return true; } + +bool has_attribute (const std::string& filename, const std::string& attname) +{ + return has_attribute(filename,"GLOBAL",attname); +} + +bool has_attribute (const std::string& filename, const std::string& varname, const std::string& attname) +{ + int ncid, varid, attid, err; + + bool was_open = is_file_open_c2f(filename.c_str(),-1); + if (not was_open) { + register_file(filename,Read); + } + + // Get file id + ncid = get_file_ncid_c2f (filename.c_str()); + + // Get var id + if (varname=="GLOBAL") { + varid = PIO_GLOBAL; + } else { + err = PIOc_inq_varid(ncid,varname.c_str(),&varid); + EKAT_REQUIRE_MSG (err==PIO_NOERR, + "Error! Something went wrong while retrieving variable id.\n" + " - filename : " + filename + "\n" + " - varname : " + varname + "\n" + " - pio error: " + std::to_string(err) + "\n"); + } + + // Get att id + err = PIOc_inq_attid(ncid,varid,attname.c_str(),&attid); + if (err==PIO_ENOTATT) { + return false; + } + EKAT_REQUIRE_MSG (err==PIO_NOERR, + "Error! Something went wrong while retrieving attribute id.\n" + " - filename : " + filename + "\n" + " - varname : " + varname + "\n" + " - attname : " + attname + "\n" + " - pio error: " + std::to_string(err) + "\n"); + + if (not was_open) { + eam_pio_closefile(filename); + } + + return true; +} /* ----------------------------------------------------------------- */ void set_dof(const std::string& filename, const std::string& varname, const Int dof_len, const std::int64_t* x_dof) { @@ -231,6 +279,7 @@ void register_dimension(const std::string &filename, const std::string& shortnam EKAT_REQUIRE_MSG (err==PIO_NOERR, "Error! Something went wrong querying for the unlimited dimension id.\n" " - filename: " + filename + "\n" + " - dimension : " + shortname + "\n" " - pio error: " + std::to_string(err) + "\n"); if (length==0) { EKAT_REQUIRE_MSG ( unlimid==dimid, @@ -292,7 +341,8 @@ void register_variable(const std::string &filename, const std::string& shortname if (mode==Write) { EKAT_REQUIRE_MSG ( units!="" and nc_dtype!="", "Error! Missing valid units and/or nc_dtype arguments for file open in Write mode.\n" - " - filename: " + filename + "\n"); + " - filename: " + filename + "\n" + " - varname : " + shortname + "\n"); } else { EKAT_REQUIRE_MSG ( has_var, "Error! Variable not found in file open in " + mode_str + " mode.\n" @@ -357,6 +407,7 @@ void register_variable(const std::string &filename, const std::string& shortname EKAT_REQUIRE_MSG(var_dimensions==dims_from_file, "Error! Input variable dimensions do not match the ones from the file.\n" " - filename : " + filename + "\n" + " - varname : " + shortname + "\n" " - input dims: (" + ekat::join(var_dimensions,",") + ")\n" " - file dims : (" + ekat::join(dims_from_file,",") + ")\n"); @@ -626,14 +677,24 @@ void grid_write_data_array(const std::string &filename, const std::strin grid_write_data_array_c2f_double(filename.c_str(),varname.c_str(),hbuf,buf_size); } /* ----------------------------------------------------------------- */ -void write_timestamp (const std::string& filename, const std::string& ts_name, const util::TimeStamp& ts) +void write_timestamp (const std::string& filename, const std::string& ts_name, + const util::TimeStamp& ts, const bool write_nsteps) { set_attribute(filename,ts_name,ts.to_string()); + if (write_nsteps) { + set_attribute(filename,ts_name+"_nsteps",ts.get_num_steps()); + } } /* ----------------------------------------------------------------- */ -util::TimeStamp read_timestamp (const std::string& filename, const std::string& ts_name) +util::TimeStamp read_timestamp (const std::string& filename, + const std::string& ts_name, + const bool read_nsteps) { - return util::str_to_time_stamp(get_attribute(filename,ts_name)); + auto ts = util::str_to_time_stamp(get_attribute(filename,ts_name)); + if (read_nsteps and has_attribute(filename,ts_name+"_nsteps")) { + ts.set_num_steps(get_attribute(filename,ts_name+"_nsteps")); + } + return ts; } /* ----------------------------------------------------------------- */ } // namespace scorpio diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.hpp b/components/eamxx/src/share/io/scream_scorpio_interface.hpp index 4289265d07aa..b64ca3ef25fd 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.hpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.hpp @@ -38,6 +38,8 @@ namespace scorpio { int get_dimlen(const std::string& filename, const std::string& dimname); bool has_dim(const std::string& filename, const std::string& dimname); bool has_variable (const std::string& filename, const std::string& varname); + bool has_attribute (const std::string& filename, const std::string& attname); + bool has_attribute (const std::string& filename, const std::string& varname, const std::string& attname); void set_decomp(const std::string& filename); /* Sets the degrees-of-freedom for a particular variable in a particular file. Called once for each variable, for each file. */ void set_dof(const std::string &filename, const std::string &varname, const Int dof_len, const offset_t* x_dof); @@ -93,8 +95,11 @@ namespace scorpio { } // Shortcut to write/read to/from YYYYMMDD/HHMMSS attributes in the NC file - void write_timestamp (const std::string& filename, const std::string& ts_name, const util::TimeStamp& ts); - util::TimeStamp read_timestamp (const std::string& filename, const std::string& ts_name); + void write_timestamp (const std::string& filename, const std::string& ts_name, + const util::TimeStamp& ts, const bool write_nsteps = false); + util::TimeStamp read_timestamp (const std::string& filename, + const std::string& ts_name, + const bool read_nsteps = false); extern "C" { /* Query whether the pio subsystem is inited or not */ diff --git a/components/eamxx/src/share/io/tests/CMakeLists.txt b/components/eamxx/src/share/io/tests/CMakeLists.txt index 21d293dbb15f..94bcee1790b0 100644 --- a/components/eamxx/src/share/io/tests/CMakeLists.txt +++ b/components/eamxx/src/share/io/tests/CMakeLists.txt @@ -17,6 +17,12 @@ CreateUnitTest(io_basic "io_basic.cpp" MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} ) +## Test output where we write one file per month +CreateUnitTest(io_monthly "io_monthly.cpp" + LIBS scream_io LABELS io + MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} +) + ## Test basic output (no packs, no diags, all avg types, all freq units) CreateUnitTest(io_filled "io_filled.cpp" LIBS scream_io LABELS io @@ -36,8 +42,7 @@ CreateUnitTest(io_diags "io_diags.cpp" ) # Test output on SE grid -configure_file(io_test_se_grid.yaml io_test_se_grid.yaml) -CreateUnitTest(io_test_se_grid "io_se_grid.cpp" +CreateUnitTest(io_se_grid "io_se_grid.cpp" LIBS scream_io LABELS io MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} ) diff --git a/components/eamxx/src/share/io/tests/io_basic.cpp b/components/eamxx/src/share/io/tests/io_basic.cpp index 4b7ee47bc5f5..378a956f8932 100644 --- a/components/eamxx/src/share/io/tests/io_basic.cpp +++ b/components/eamxx/src/share/io/tests/io_basic.cpp @@ -152,7 +152,6 @@ void write (const std::string& avg_type, const std::string& freq_units, auto& ctrl_pl = om_pl.sublist("output_control"); ctrl_pl.set("frequency_units",freq_units); ctrl_pl.set("Frequency",freq); - ctrl_pl.set("MPI Ranks in Filename",true); ctrl_pl.set("save_grid_data",false); // Create Output manager @@ -172,8 +171,8 @@ void write (const std::string& avg_type, const std::string& freq_units, t += dt; // Add 1 to all fields entries - for (const auto& n : fnames) { - auto f = fm->get_field(n); + for (const auto& name : fnames) { + auto f = fm->get_field(name); add(f,1.0); } @@ -231,6 +230,7 @@ void read (const std::string& avg_type, const std::string& freq_units, // The last one comes from // (a+1 + a+2 +..+a+freq)/freq = // a + sum(i)/freq = a + (freq(freq+1)/2)/freq + // = a + (freq+1)/2 double delta = (freq+1)/2.0; for (int n=0; n Averaging type: " + avg + " ", 40); write(avg,units,freq,seed,comm); - read(avg,units,freq,seed,comm); + read (avg,units,freq,seed,comm); print(" PASS\n"); } } diff --git a/components/eamxx/src/share/io/tests/io_diags.cpp b/components/eamxx/src/share/io/tests/io_diags.cpp index 90abb4c57775..b2fe7f994db6 100644 --- a/components/eamxx/src/share/io/tests/io_diags.cpp +++ b/components/eamxx/src/share/io/tests/io_diags.cpp @@ -186,7 +186,6 @@ void write (const int seed, const ekat::Comm& comm) auto& ctrl_pl = om_pl.sublist("output_control"); ctrl_pl.set("frequency_units",std::string("nsteps")); ctrl_pl.set("Frequency",1); - ctrl_pl.set("MPI Ranks in Filename",true); ctrl_pl.set("save_grid_data",false); // Create Output manager diff --git a/components/eamxx/src/share/io/tests/io_filled.cpp b/components/eamxx/src/share/io/tests/io_filled.cpp index 93e14589c44c..ba5eb7d871d8 100644 --- a/components/eamxx/src/share/io/tests/io_filled.cpp +++ b/components/eamxx/src/share/io/tests/io_filled.cpp @@ -133,12 +133,11 @@ void write (const std::string& avg_type, const std::string& freq_units, om_pl.set("Field Names",fnames); om_pl.set("Averaging Type", avg_type); om_pl.set("fill_value",FillValue); - om_pl.set("track_fill",true); om_pl.set("fill_threshold",fill_threshold); + om_pl.set("track_avg_cnt",true); auto& ctrl_pl = om_pl.sublist("output_control"); ctrl_pl.set("frequency_units",freq_units); ctrl_pl.set("Frequency",freq); - ctrl_pl.set("MPI Ranks in Filename",true); ctrl_pl.set("save_grid_data",false); // Create Output manager @@ -222,24 +221,24 @@ void read (const std::string& avg_type, const std::string& freq_units, auto f0 = fm0->get_field(fn).clone(); auto f = fm->get_field(fn); if (avg_type=="MIN") { - Real test_val = ((n+1)*freq%2==0) ? n*freq+1 : n*freq+2; + Real test_val = ((n+1)*freq%2==0) ? n*freq+1 : n*freq+2; set(f0,test_val); REQUIRE (views_are_equal(f,f0)); } else if (avg_type=="MAX") { - Real test_val = ((n+1)*freq%2==0) ? (n+1)*freq : (n+1)*freq-1; + Real test_val = ((n+1)*freq%2==0) ? (n+1)*freq : (n+1)*freq-1; set(f0,test_val); REQUIRE (views_are_equal(f,f0)); } else if (avg_type=="INSTANT") { - Real test_val = (n*freq%2==0) ? n*freq : FillValue; + Real test_val = (n*freq%2==0) ? n*freq : FillValue; set(f0,test_val); REQUIRE (views_are_equal(f,f0)); } else { // Is avg_type = AVERAGE - // Note, for AVERAGE type output with filling we need to check that the - // number of contributing fill steps surpasses the fill_threshold, if not - // then we know that the snap will reflect the fill value. - Real test_val; - Real M = freq/2 + (n%2==0 ? 0.0 : 1.0); - Real a = n*freq + (n%2==0 ? 0.0 : -1.0); + // Note, for AVERAGE type output with filling we need to check that the + // number of contributing fill steps surpasses the fill_threshold, if not + // then we know that the snap will reflect the fill value. + Real test_val; + Real M = freq/2 + (n%2==0 ? 0.0 : 1.0); + Real a = n*freq + (n%2==0 ? 0.0 : -1.0); test_val = (M/freq > fill_threshold) ? a + (M+1.0) : FillValue; set(f0,test_val); REQUIRE (views_are_equal(f,f0)); diff --git a/components/eamxx/src/share/io/tests/io_monthly.cpp b/components/eamxx/src/share/io/tests/io_monthly.cpp new file mode 100644 index 000000000000..4cf18fff1c71 --- /dev/null +++ b/components/eamxx/src/share/io/tests/io_monthly.cpp @@ -0,0 +1,230 @@ +#include + +#include "share/io/scream_output_manager.hpp" +#include "share/io/scorpio_input.hpp" + +#include "share/grid/mesh_free_grids_manager.hpp" + +#include "share/field/field_utils.hpp" +#include "share/field/field.hpp" +#include "share/field/field_manager.hpp" + +#include "share/util/scream_universal_constants.hpp" +#include "share/util/scream_setup_random_test.hpp" +#include "share/util/scream_time_stamp.hpp" +#include "share/scream_types.hpp" + +#include "ekat/util/ekat_units.hpp" +#include "ekat/ekat_parameter_list.hpp" +#include "ekat/ekat_assert.hpp" +#include "ekat/mpi/ekat_comm.hpp" +#include "ekat/util/ekat_test_utils.hpp" + +#include +#include + +namespace scream { + +void add (const Field& f, const double v) { + auto data = f.get_internal_view_data(); + auto nscalars = f.get_header().get_alloc_properties().get_num_scalars(); + for (int i=0; i +get_gm (const ekat::Comm& comm) +{ + // For 2+ ranks tests, this will check IO works correctly + // even if one rank owns 0 dofs + const int ngcols = std::max(comm.size()-1,1); + const int nlevs = 4; + auto gm = create_mesh_free_grids_manager(comm,0,0,nlevs,ngcols); + gm->build_grids(); + return gm; +} + +std::shared_ptr +get_fm (const std::shared_ptr& grid, + const util::TimeStamp& t0, const int seed) +{ + using FL = FieldLayout; + using FID = FieldIdentifier; + using namespace ShortFieldTagsNames; + + // Random number generation stuff + // NOTES + // - Use integers, so we can check answers without risk of + // non bfb diffs due to different order of sums. + // - Uniform_int_distribution returns an int, and the randomize + // util checks that return type matches the Field data type. + // So wrap the int pdf in a lambda, that does the cast. + std::mt19937_64 engine(seed); + auto my_pdf = [&](std::mt19937_64& engine) -> Real { + std::uniform_int_distribution pdf (0,100); + Real v = pdf(engine); + return v; + }; + + const int nlcols = grid->get_num_local_dofs(); + const int nlevs = grid->get_num_vertical_levels(); + + std::vector layouts = + { + FL({COL }, {nlcols }), + FL({COL, LEV}, {nlcols, nlevs}), + FL({COL,CMP,ILEV}, {nlcols,2,nlevs+1}) + }; + + auto fm = std::make_shared(grid); + + const auto units = ekat::units::Units::nondimensional(); + int count=0; + for (const auto& fl : layouts) { + FID fid("f_"+std::to_string(count),fl,units,grid->name()); + Field f(fid); + f.allocate_view(); + randomize (f,engine,my_pdf); + f.get_header().get_tracking().update_time_stamp(t0); + fm->add_field(f); + ++count; + } + + return fm; +} + +// Returns fields after initialization +void write (const int seed, const ekat::Comm& comm) +{ + // Create grid + auto gm = get_gm(comm); + auto grid = gm->get_grid("Point Grid"); + + // Time advance parameters + auto t0 = get_t0(); + const int dt = 86400*30; // 30 days + + // Create some fields + auto fm = get_fm(grid,t0,seed); + std::vector fnames; + for (auto it : *fm) { + fnames.push_back(it.second->name()); + } + + // Create output params + ekat::ParameterList om_pl; + om_pl.set("MPI Ranks in Filename",true); + om_pl.set("filename_prefix",std::string("io_monthly")); + om_pl.set("Field Names",fnames); + om_pl.set("Averaging Type", std::string("Instant")); + om_pl.set("file_max_storage_type",std::string("one_month")); + om_pl.set("Floating Point Precision",std::string("single")); + auto& ctrl_pl = om_pl.sublist("output_control"); + ctrl_pl.set("frequency_units",std::string("nsteps")); + ctrl_pl.set("Frequency",1); + ctrl_pl.set("save_grid_data",false); + + // Create Output manager + OutputManager om; + om.setup(comm,om_pl,fm,gm,t0,t0,false); + + // Time loop: do 11 steps, since we already did Jan output at t0 + const int nsteps = 11; + auto t = t0; + for (int n=0; nget_field(name); + add(f,1); + } + + // Run output manager + om.run (t); + } + + // Close file and cleanup + om.finalize(); +} + +void read (const int seed, const ekat::Comm& comm) +{ + // Time quantities + auto t0 = get_t0(); + int dt = 86400*30; + + // Get gm + auto gm = get_gm (comm); + auto grid = gm->get_grid("Point Grid"); + + // Get initial fields. Use wrong seed for fm, so fields are not + // inited with right data (avoid getting right answer without reading). + auto fm0 = get_fm(grid,t0,seed); + auto fm = get_fm(grid,t0,-seed-1); + std::vector fnames; + for (auto it : *fm) { + fnames.push_back(it.second->name()); + } + + // Get filename from timestamp + std::string casename = "io_monthly"; + auto get_filename = [&](const util::TimeStamp& t) { + std::string fname = casename + + ".INSTANT.nsteps_x1" + + ".np" + std::to_string(comm.size()) + + "." + std::to_string(t.get_year()) + + "-" + std::to_string(t.get_month()) + + ".nc"; + return fname; + }; + + // Create reader pl + ekat::ParameterList reader_pl; + reader_pl.set("Field Names",fnames); + + for (int n=0; n<12; ++n) { + auto t = t0 + n*dt; + auto filename = get_filename(t); + + // There should be just one time snapshot per file + REQUIRE(scorpio::get_dimlen(filename,"time")==1); + + reader_pl.set("Filename",filename); + AtmosphereInput reader(reader_pl,fm); + reader.read_variables(); + + for (const auto& fn : fnames) { + auto f0 = fm0->get_field(fn).clone(); + auto f = fm->get_field(fn); + add(f0,n); + REQUIRE (views_are_equal(f,f0)); + } + } +} + +TEST_CASE ("io_monthly") { + ekat::Comm comm(MPI_COMM_WORLD); + scorpio::eam_init_pio_subsystem(comm); + + auto seed = get_random_test_seed(&comm); + + if (comm.am_i_root()) { + std::cout << " -> Testing output with one file per month ...\n"; + } + write(seed,comm); + read (seed,comm); + if (comm.am_i_root()) { + std::cout << " -> Testing output with one file per month ... PASS\n"; + } + scorpio::eam_pio_finalize(); +} + +} // anonymous namespace diff --git a/components/eamxx/src/share/io/tests/io_packed.cpp b/components/eamxx/src/share/io/tests/io_packed.cpp index f4b467c619c1..585fc7b26f69 100644 --- a/components/eamxx/src/share/io/tests/io_packed.cpp +++ b/components/eamxx/src/share/io/tests/io_packed.cpp @@ -120,7 +120,6 @@ void write (const int freq, const int seed, const int ps, const ekat::Comm& comm auto& ctrl_pl = om_pl.sublist("output_control"); ctrl_pl.set("frequency_units",std::string("nsteps")); ctrl_pl.set("Frequency",freq); - ctrl_pl.set("MPI Ranks in Filename",true); ctrl_pl.set("save_grid_data",false); // Create Output manager diff --git a/components/eamxx/src/share/io/tests/io_remap_test.cpp b/components/eamxx/src/share/io/tests/io_remap_test.cpp index 91fa95b3fc29..bfed937636e2 100644 --- a/components/eamxx/src/share/io/tests/io_remap_test.cpp +++ b/components/eamxx/src/share/io/tests/io_remap_test.cpp @@ -243,6 +243,7 @@ TEST_CASE("io_remap_test","io_remap_test") om_source.setup(io_comm,source_remap_control,field_manager,gm,t0,t0,false); io_comm.barrier(); om_source.run(t0); + om_source.finalize(); print (" -> source data ... done\n",io_comm); print (" -> vertical remap ... \n",io_comm); @@ -250,6 +251,7 @@ TEST_CASE("io_remap_test","io_remap_test") om_vert.setup(io_comm,vert_remap_control,field_manager,gm,t0,t0,false); io_comm.barrier(); om_vert.run(t0); + om_vert.finalize(); print (" -> vertical remap ... done\n",io_comm); print (" -> horizontal remap ... \n",io_comm); @@ -257,6 +259,7 @@ TEST_CASE("io_remap_test","io_remap_test") om_horiz.setup(io_comm,horiz_remap_control,field_manager,gm,t0,t0,false); io_comm.barrier(); om_horiz.run(t0); + om_horiz.finalize(); print (" -> horizontal remap ... done\n",io_comm); print (" -> vertical-horizontal remap ... \n",io_comm); @@ -264,6 +267,7 @@ TEST_CASE("io_remap_test","io_remap_test") om_vert_horiz.setup(io_comm,vert_horiz_remap_control,field_manager,gm,t0,t0,false); io_comm.barrier(); om_vert_horiz.run(t0); + om_vert_horiz.finalize(); print (" -> vertical-horizontal remap ... done\n",io_comm); print (" -> Create output ... done\n",io_comm); @@ -640,7 +644,7 @@ std::shared_ptr get_test_fm(std::shared_ptr gr auto f_Vi = fm->get_field(fid_Vi); // Set some string to be written to file as attribute to the variables - for (const std::string& fname : {"Y_flat","Y_mid","Y_int","V_mid","V_int"}) { + for (const std::string fname : {"Y_flat","Y_mid","Y_int","V_mid","V_int"}) { auto& f = fm->get_field(fname); auto& str_atts = f.get_header().get_extra_data("io: string attributes"); str_atts["test"] = fname; @@ -669,14 +673,14 @@ std::shared_ptr get_test_fm(std::shared_ptr gr ekat::ParameterList set_output_params(const std::string& name, const std::string& remap_filename, const int p_ref, const bool vert_remap, const bool horiz_remap) { using vos_type = std::vector; - ekat::ParameterList output_yaml; - - output_yaml.set("filename_prefix",name); - output_yaml.set("Averaging Type","Instant"); - output_yaml.set("Max Snapshots Per File",1); - output_yaml.set("Floating Point Precision","real"); - auto& oc = output_yaml.sublist("output_control"); - oc.set("MPI Ranks in Filename",true); + ekat::ParameterList params; + + params.set("filename_prefix",name); + params.set("Averaging Type","Instant"); + params.set("Max Snapshots Per File",1); + params.set("Floating Point Precision","real"); + params.set("MPI Ranks in Filename",true); + auto& oc = params.sublist("output_control"); oc.set("Frequency",1); oc.set("frequency_units","nsteps"); @@ -689,16 +693,16 @@ ekat::ParameterList set_output_params(const std::string& name, const std::string fields_out.push_back("p_mid"); fields_out.push_back("p_int"); } - output_yaml.set("Field Names",fields_out); + params.set("Field Names",fields_out); if (vert_remap) { - output_yaml.set("vertical_remap_file",remap_filename); // TODO, make this work for general np=? + params.set("vertical_remap_file",remap_filename); // TODO, make this work for general np=? } if (horiz_remap) { - output_yaml.set("horiz_remap_file",remap_filename); // TODO, make this work for general np=? + params.set("horiz_remap_file",remap_filename); // TODO, make this work for general np=? } - return output_yaml; + return params; } /*==========================================================================================================*/ ekat::ParameterList set_input_params(const std::string& name, ekat::Comm& comm, const std::string& tstamp, const int p_ref) diff --git a/components/eamxx/src/share/io/tests/io_se_grid.cpp b/components/eamxx/src/share/io/tests/io_se_grid.cpp index 16872a7491f8..edf5f5dbbcd9 100644 --- a/components/eamxx/src/share/io/tests/io_se_grid.cpp +++ b/components/eamxx/src/share/io/tests/io_se_grid.cpp @@ -39,6 +39,7 @@ ekat::ParameterList get_in_params(const ekat::Comm& comm, TEST_CASE("se_grid_io") { + using strvec_t = std::vector; ekat::Comm io_comm(MPI_COMM_WORLD); int num_my_elems = 2; @@ -58,8 +59,15 @@ TEST_CASE("se_grid_io") auto fm0 = get_test_fm(grid,t0,true); ekat::ParameterList params; - ekat::parse_yaml_file("io_test_se_grid.yaml",params); + params.set("filename_prefix","io_se_grid"); + params.set("Averaging Type","Instant"); + params.set("Max Snapshots Per File",1); + params.set("Field Names",{"field_1","field_2","field_3","field_packed"}); params.set("Floating Point Precision","real"); + params.set("MPI Ranks in Filename",true); + auto& ctl_pl = params.sublist("output_control"); + ctl_pl.set("Frequency",1); + ctl_pl.set("frequency_units","nsteps"); OutputManager om; om.setup(io_comm,params,fm0,gm,t0,t0,false); @@ -164,7 +172,7 @@ ekat::ParameterList get_in_params(const ekat::Comm& comm, using vos_type = std::vector; ekat::ParameterList in_params("Input Parameters"); - std::string filename = "io_test_se_grid.INSTANT.nsteps_x1.np" + std::string filename = "io_se_grid.INSTANT.nsteps_x1.np" + std::to_string(comm.size()) + "." + t0.to_string() + ".nc"; diff --git a/components/eamxx/src/share/io/tests/io_test_se_grid.yaml b/components/eamxx/src/share/io/tests/io_test_se_grid.yaml deleted file mode 100644 index f4dd56e5cde6..000000000000 --- a/components/eamxx/src/share/io/tests/io_test_se_grid.yaml +++ /dev/null @@ -1,11 +0,0 @@ -%YAML 1.1 ---- -filename_prefix: io_test_se_grid -Averaging Type: Instant -Max Snapshots Per File: 1 -Field Names: [field_1, field_2, field_3, field_packed] -output_control: - MPI Ranks in Filename: true - Frequency: 1 - frequency_units: nsteps -... diff --git a/components/eamxx/src/share/io/tests/io_utils.cpp b/components/eamxx/src/share/io/tests/io_utils.cpp index 6064a29567a0..77779307a720 100644 --- a/components/eamxx/src/share/io/tests/io_utils.cpp +++ b/components/eamxx/src/share/io/tests/io_utils.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include @@ -40,22 +41,21 @@ TEST_CASE ("io_control") { IOControl control; control.frequency = 2; - control.timestamp_of_last_write = t0; + control.last_write_ts = t0; SECTION ("none") { control.frequency_units = "none"; REQUIRE (not control.output_enabled()); - REQUIRE (not control.is_write_step(t0)); } SECTION ("never") { control.frequency_units = "never"; REQUIRE (not control.output_enabled()); - REQUIRE (not control.is_write_step(t0)); } SECTION ("nsteps") { control.frequency_units = "nsteps"; + control.compute_next_write_ts(); auto t1 = t0 + 1; auto t2 = t1 + 1; REQUIRE (control.output_enabled()); @@ -65,6 +65,7 @@ TEST_CASE ("io_control") { SECTION ("nsecs") { control.frequency_units = "nsecs"; + control.compute_next_write_ts(); auto t1 = t0 + 1; auto t2 = t1 + 1; REQUIRE (control.output_enabled()); @@ -74,6 +75,7 @@ TEST_CASE ("io_control") { SECTION ("nmins") { control.frequency_units = "nmins"; + control.compute_next_write_ts(); auto t1 = t0 + 60; auto t2 = t1 + 60; REQUIRE (control.output_enabled()); @@ -83,6 +85,7 @@ TEST_CASE ("io_control") { SECTION ("nhours") { control.frequency_units = "nhours"; + control.compute_next_write_ts(); auto t1 = t0 + 3600; auto t2 = t1 + 3600; REQUIRE (control.output_enabled()); @@ -92,6 +95,7 @@ TEST_CASE ("io_control") { SECTION ("ndays") { control.frequency_units = "ndays"; + control.compute_next_write_ts(); auto t1 = t0 + 86400; auto t2 = t1 + 86400; REQUIRE (control.output_enabled()); @@ -101,6 +105,7 @@ TEST_CASE ("io_control") { SECTION ("nmonths") { control.frequency_units = "nmonths"; + control.compute_next_write_ts(); util::TimeStamp t1({2023,10,7},{12,0,0}); util::TimeStamp t2({2023,11,7},{12,0,0}); util::TimeStamp t3({2023,11,7},{13,0,0}); @@ -112,6 +117,7 @@ TEST_CASE ("io_control") { SECTION ("nyears") { control.frequency_units = "nyears"; + control.compute_next_write_ts(); util::TimeStamp t1({2024,9,7},{12,0,0}); util::TimeStamp t2({2025,9,7},{12,0,0}); util::TimeStamp t3({2025,9,7},{13,0,0}); diff --git a/components/eamxx/src/share/io/tests/output_restart.cpp b/components/eamxx/src/share/io/tests/output_restart.cpp index 118b9a84836a..62676450efee 100644 --- a/components/eamxx/src/share/io/tests/output_restart.cpp +++ b/components/eamxx/src/share/io/tests/output_restart.cpp @@ -79,14 +79,13 @@ TEST_CASE("output_restart","io") output_params.set("Floating Point Precision","real"); output_params.set>("Field Names",{"field_1", "field_2", "field_3", "field_4","field_5"}); output_params.set("fill_value",FillValue); - output_params.sublist("output_control").set("MPI Ranks in Filename","true"); + output_params.set("MPI Ranks in Filename","true"); + output_params.set("flush_frequency",1); output_params.sublist("output_control").set("frequency_units","nsteps"); output_params.sublist("output_control").set("Frequency",10); - output_params.sublist("Checkpoint Control").set("MPI Ranks in Filename","true"); output_params.sublist("Checkpoint Control").set("Frequency",5); // This skips a test that only matters for AD runs output_params.sublist("Checkpoint Control").set("is_unit_testing","true"); - output_params.sublist("Restart").set("MPI Ranks in Filename","true"); // Creates and runs an OM from output_params and given inputs auto run = [&](std::shared_ptr fm, @@ -118,7 +117,7 @@ TEST_CASE("output_restart","io") } }; // Run test for different avg type choices - for (const std::string& avg_type : {"INSTANT","AVERAGE"}) { + for (const std::string avg_type : {"INSTANT","AVERAGE"}) { { // In normal runs, the OM for the model restart takes care of nuking rpointer.atm, // and re-creating a new one. Here, we don't have that, so we must nuke it manually diff --git a/components/eamxx/src/share/scream_config.hpp b/components/eamxx/src/share/scream_config.hpp index fea495c5ae3f..391f0afdfbde 100644 --- a/components/eamxx/src/share/scream_config.hpp +++ b/components/eamxx/src/share/scream_config.hpp @@ -22,6 +22,15 @@ std::string scream_config_string(); bool use_leap_year (); void set_use_leap_year (const bool use_leap); +// Allow downstream code to avoid macros +bool constexpr is_scream_standalone () { +#ifdef SCREAM_CIME_BUILD + return false; +#else + return true; +#endif +} + } // namespace scream #endif // SCREAM_CONFIG_HPP diff --git a/components/eamxx/src/share/tests/CMakeLists.txt b/components/eamxx/src/share/tests/CMakeLists.txt index 0b2363fad2f6..bfdc0bba2b7e 100644 --- a/components/eamxx/src/share/tests/CMakeLists.txt +++ b/components/eamxx/src/share/tests/CMakeLists.txt @@ -1,5 +1,5 @@ # NOTE: tests inside this if statement won't be built in a baselines-only build -if (NOT ${SCREAM_BASELINES_ONLY}) +if (NOT SCREAM_ONLY_GENERATE_BASELINES) include(ScreamUtils) # Test vertical interpolation @@ -63,11 +63,4 @@ if (NOT ${SCREAM_BASELINES_ONLY}) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/atm_process_tests_named_procs.yaml ${CMAKE_CURRENT_BINARY_DIR}/atm_process_tests_named_procs.yaml COPYONLY) CreateUnitTest(atm_proc "atm_process_tests.cpp") - - # Test horizontal remapping utility - CreateUnitTest(horizontal_remap "horizontal_remap_test.cpp" - LIBS scream_io - LABELS horiz_remap - MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} - ) endif() diff --git a/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp b/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp index e56b43aab7ef..13832c490441 100644 --- a/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp +++ b/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp @@ -105,6 +105,8 @@ build_src_grid(const ekat::Comm& comm, const int ngdofs, Engine& engine) } constexpr int vec_dim = 2; +constexpr int tens_dim1 = 3; +constexpr int tens_dim2 = 4; Field create_field (const std::string& name, const LayoutType lt, const AbstractGrid& grid, const bool midpoints) { const auto u = ekat::units::Units::nondimensional(); @@ -116,6 +118,8 @@ Field create_field (const std::string& name, const LayoutType lt, const Abstract f = Field(FieldIdentifier(name,grid.get_2d_scalar_layout(),u,gn)); break; case LayoutType::Vector2D: f = Field(FieldIdentifier(name,grid.get_2d_vector_layout(CMP,vec_dim),u,gn)); break; + case LayoutType::Tensor2D: + f = Field(FieldIdentifier(name,grid.get_2d_tensor_layout({CMP,CMP},{tens_dim1,tens_dim2}),u,gn)); break; case LayoutType::Scalar3D: f = Field(FieldIdentifier(name,grid.get_3d_scalar_layout(midpoints),u,gn)); f.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); @@ -124,6 +128,10 @@ Field create_field (const std::string& name, const LayoutType lt, const Abstract f = Field(FieldIdentifier(name,grid.get_3d_vector_layout(midpoints,CMP,vec_dim),u,gn)); f.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); break; + case LayoutType::Tensor3D: + f = Field(FieldIdentifier(name,grid.get_3d_tensor_layout(midpoints,{CMP,CMP},{tens_dim1,tens_dim2}),u,gn)); + f.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); + break; default: EKAT_ERROR_MSG ("Invalid layout type for this unit test.\n"); } @@ -192,6 +200,13 @@ Field all_gather_field_impl (const Field& f, const ekat::Comm& comm) { data = data_vec.data(); } break; + case 4: + if (pid==comm.rank()) { + data = ekat::subview(f.get_view(),icol).data(); + } else { + data = data_vec.data(); + } + break; default: EKAT_ERROR_MSG ( "Unexpected rank in RefiningRemapperRMA unit test.\n" @@ -306,20 +321,26 @@ TEST_CASE("coarsening_remap") auto src_s2d = create_field("s2d", LayoutType::Scalar2D, *src_grid, false, engine); auto src_v2d = create_field("v2d", LayoutType::Vector2D, *src_grid, false, engine); + auto src_t2d = create_field("t2d", LayoutType::Tensor2D, *src_grid, false, engine); auto src_s3d_m = create_field("s3d_m",LayoutType::Scalar3D, *src_grid, true, engine); auto src_s3d_i = create_field("s3d_i",LayoutType::Scalar3D, *src_grid, false, engine); auto src_v3d_m = create_field("v3d_m",LayoutType::Vector3D, *src_grid, true, engine); auto src_v3d_i = create_field("v3d_i",LayoutType::Vector3D, *src_grid, false, engine); + auto src_t3d_m = create_field("t3d_m",LayoutType::Tensor3D, *src_grid, true, engine); + auto src_t3d_i = create_field("t3d_i",LayoutType::Tensor3D, *src_grid, false, engine); auto tgt_s2d = create_field("s2d", LayoutType::Scalar2D, *tgt_grid, false); auto tgt_v2d = create_field("v2d", LayoutType::Vector2D, *tgt_grid, false); + auto tgt_t2d = create_field("t2d", LayoutType::Tensor2D, *tgt_grid, false); auto tgt_s3d_m = create_field("s3d_m",LayoutType::Scalar3D, *tgt_grid, true ); auto tgt_s3d_i = create_field("s3d_i",LayoutType::Scalar3D, *tgt_grid, false); auto tgt_v3d_m = create_field("v3d_m",LayoutType::Vector3D, *tgt_grid, true ); auto tgt_v3d_i = create_field("v3d_i",LayoutType::Vector3D, *tgt_grid, false); + auto tgt_t3d_m = create_field("t3d_m",LayoutType::Tensor3D, *tgt_grid, true ); + auto tgt_t3d_i = create_field("t3d_i",LayoutType::Tensor3D, *tgt_grid, false); - std::vector src_f = {src_s2d,src_v2d,src_s3d_m,src_s3d_i,src_v3d_m,src_v3d_i}; - std::vector tgt_f = {tgt_s2d,tgt_v2d,tgt_s3d_m,tgt_s3d_i,tgt_v3d_m,tgt_v3d_i}; + std::vector src_f = {src_s2d,src_v2d,src_t2d,src_s3d_m,src_s3d_i,src_v3d_m,src_v3d_i,src_t3d_m,src_t3d_i}; + std::vector tgt_f = {tgt_s2d,tgt_v2d,tgt_t2d,tgt_s3d_m,tgt_s3d_i,tgt_v3d_m,tgt_v3d_i,tgt_t3d_m,tgt_t3d_i}; // -------------------------------------- // // Register fields in the remapper // @@ -357,7 +378,7 @@ TEST_CASE("coarsening_remap") const auto& l = gsrc.get_header().get_identifier().get_layout(); const auto ls = to_string(l); - std::string dots (25-ls.size(),'.'); + std::string dots (30-ls.size(),'.'); auto msg = " -> Checking field with layout " + to_string(l) + " " + dots; root_print (msg + "\n",comm); bool ok = true; @@ -396,6 +417,26 @@ TEST_CASE("coarsening_remap") } } } break; + case LayoutType::Tensor2D: + { + const auto v_src = gsrc.get_view(); + const auto v_tgt = gtgt.get_view(); + for (int idof=0; idof(); @@ -436,6 +477,29 @@ TEST_CASE("coarsening_remap") } } } break; + case LayoutType::Tensor3D: + { + const auto v_src = gsrc.get_view(); + const auto v_tgt = gtgt.get_view(); + auto f_nlevs = gsrc.get_header().get_identifier().get_layout().dims().back(); + for (int idof=0; idof; + using IVec = std::vector; + + FieldLayout fl1 ({COL},{1}); + FieldLayout fl2 ({COL,CMP},{1,1}); + FieldLayout fl3 ({COL,SWBND,LWBND},{1,1,1}); + FieldLayout fl4 ({COL,LEV},{1,1}); + FieldLayout fl5 ({COL,CMP,LEV},{1,1,1}); + FieldLayout fl6 ({COL,ISCCPTAU,ISCCPPRS,ILEV},{1,1,1,1}); + + REQUIRE (get_layout_type(fl1.tags())==LayoutType::Scalar2D); + REQUIRE (get_layout_type(fl2.tags())==LayoutType::Vector2D); + REQUIRE (get_layout_type(fl3.tags())==LayoutType::Tensor2D); + REQUIRE (get_layout_type(fl4.tags())==LayoutType::Scalar3D); + REQUIRE (get_layout_type(fl5.tags())==LayoutType::Vector3D); + REQUIRE (get_layout_type(fl6.tags())==LayoutType::Tensor3D); + + REQUIRE (not fl1.is_vector_layout()); + REQUIRE ( fl2.is_vector_layout()); + REQUIRE (not fl3.is_vector_layout()); + REQUIRE (not fl4.is_vector_layout()); + REQUIRE ( fl5.is_vector_layout()); + REQUIRE (not fl6.is_vector_layout()); + + REQUIRE (not fl1.is_tensor_layout()); + REQUIRE (not fl2.is_tensor_layout()); + REQUIRE ( fl3.is_tensor_layout()); + REQUIRE (not fl4.is_tensor_layout()); + REQUIRE (not fl5.is_tensor_layout()); + REQUIRE ( fl6.is_tensor_layout()); + + REQUIRE (fl2.get_vector_tag()==CMP); + REQUIRE (fl5.get_vector_tag()==CMP); + REQUIRE (fl2.get_vector_component_idx()==1); + REQUIRE (fl5.get_vector_component_idx()==1); + REQUIRE (fl2.get_vector_dim()==1); + REQUIRE (fl5.get_vector_dim()==1); + + REQUIRE (fl3.get_tensor_tags()==TVec{SWBND,LWBND}); + REQUIRE (fl6.get_tensor_tags()==TVec{ISCCPTAU,ISCCPPRS}); + REQUIRE (fl3.get_tensor_dims()==IVec{1,2}); + REQUIRE (fl6.get_tensor_dims()==IVec{1,2}); +} + TEST_CASE("field_identifier", "") { using namespace scream; using namespace ekat::units; diff --git a/components/eamxx/src/share/tests/horizontal_remap_test.cpp b/components/eamxx/src/share/tests/horizontal_remap_test.cpp deleted file mode 100644 index f9bdca47dae4..000000000000 --- a/components/eamxx/src/share/tests/horizontal_remap_test.cpp +++ /dev/null @@ -1,525 +0,0 @@ -#include - -#include "share/grid/remap/horizontal_remap_utility.hpp" - -#include "share/io/scorpio_input.hpp" -#include "share/io/scream_output_manager.hpp" - -#include "share/grid/mesh_free_grids_manager.hpp" -#include "share/grid/point_grid.hpp" - -#include "share/field/field_manager.hpp" - -#include "share/util/scream_setup_random_test.hpp" - -#include "ekat/util/ekat_units.hpp" -#include "ekat/util/ekat_test_utils.hpp" -#include "ekat/ekat_parse_yaml_file.hpp" -namespace { - -using namespace scream; - -using gid_type = AbstractGrid::gid_type; -using Pack = ekat::Pack; - -using KT = KokkosTypes; - -template -using view_1d = typename KT::template view_1d; - -template -using view_2d = typename KT::template view_2d; - -template -using view_3d = typename KT::template view_3d; - -template -using view_1d_host = typename KT::template view_1d::HostMirror; - -std::shared_ptr get_test_gm(const ekat::Comm& comm, const Int num_gcols, const Int num_levs); - -void run(std::mt19937_64& engine, const ekat::Comm& comm, const gid_type src_min_dof) -{ -/* Testing for the HorizontalMap structure and using it to apply a remap to data. - * This test has three main parts: - * 1. Construction of the remapping data, construction of the source data to be - * remapped, and the establishment of a baseline of remapped data. - * 2. The construction of a HorizontalMap where the remap data is gathered directly from - * the views derived in part 1. Apply the remap and compare against baseline. - * 3. After writing the remapping data and source data to file, we construct a - * HorizontalMap reampper with data gathered from the file. We then apply the remap - * and compare against the baseline. - * - * By breaking the test into these parts we are able to make sure that each test - * generate random data, i.e. we are not getting lucky with static numbers. We are - * also able to test the two main implementations of the horizontal remap structure; - * the generation of weights using a map constructed online, and the use of remapping - * weights constructed offline and save to file. - * - * INPUT ARGS - * - engine: This is just the random seed provided by the EAMxx testing setup - * - src_min_dof: This is the degree-of-freedom index for the first column. This - * simulates that some remapping algorithms may treat the first column - * with index=0 and others with index=1. Making this an input allows - * the test to check both cases. - */ - - using IPDF = std::uniform_int_distribution; - using RPDF = std::uniform_real_distribution; - constexpr Real tol = std::numeric_limits::epsilon()*10; - -//---------------------- Test configuration - int num_src_cols = 20; // Number of columns from source data - int num_tgt_cols = 10; // Number of columns on target grid - int num_levels = 2*SCREAM_PACK_SIZE+1; // Dummy variable needed for initialization of a grid. - - // Construct a target grid for remapped data - auto gm_tgt = get_test_gm(comm, num_tgt_cols, num_levels); - auto grid_tgt = gm_tgt->get_grid("Point Grid"); // Retrieve the actual grid from the grids manager. - auto num_loc_tgt_cols = grid_tgt->get_num_local_dofs(); // Number of columns (dofs) on this rank. - auto tgt_min_dof = grid_tgt->get_global_min_dof_gid(); // The starting index of the EAMxx grid. - auto dofs_gids = grid_tgt->get_dofs_gids().get_view(); // A view of the dofs on this rank with their global id. - auto dofs_gids_h = grid_tgt->get_dofs_gids().get_view(); - -//---------------------- -// Step 1: Generate a random remapping (source col, target col, weight) and random source data. -// Store these in local 1D views and create a remap file. -// Note, randomization is done entirely on ROOT so that we only have one version of the remap -// data. -// -// OUT - y_baseline, num_src_to_tgt_cols, n_s, map_src_cols, map_tgt_cols, map_wgts - - // When we create random source data we again only do this on root so that we - // only have one set of data. - std::vector vec_src_data(num_src_cols); - if (comm.am_i_root()) { - RPDF pdf_src_data(-1,1); // Randomizer for generating source data to be mapped from. - for (int ii=0; ii source_remap_data("",num_src_cols); - auto source_remap_data_h = Kokkos::create_mirror_view(source_remap_data); - for (int ii=0; iitgt pairs. - std::vector vec_src_col, vec_tgt_col; - std::vector vec_wgt; - std::map y_baseline; - std::map num_src_to_tgt_cols; // A record of how many source columns map to a specific target. Will be used later for testing segments. - IPDF pdf_map_src_num(2,num_src_cols); // Randomizer for number of source columns mapping to a target column - IPDF pdf_map_src(0,num_src_cols-1); // Randomizer for which source columns map to a target column - RPDF pdf_map_wgt(0,1); // Randomizer for mapping weights - // Cycle through all tgt_cols and set up a remap - for (int tcol=0; tcol temp_wgt; - std::vector temp_src; - Real wgt_sum = 0.0; - Real y_sum = 0.0; - for (int ii=0; ii map_src_cols_h(vec_src_col.data(),vec_wgt.size()); - view_1d_host map_tgt_cols_h(vec_tgt_col.data(),vec_wgt.size()); - view_1d_host map_wgts_h(vec_wgt.data(),vec_wgt.size()); - auto map_src_cols = Kokkos::create_mirror_view(map_src_cols_h); - auto map_tgt_cols = Kokkos::create_mirror_view(map_tgt_cols_h); - auto map_wgts = Kokkos::create_mirror_view(map_wgts_h ); - Kokkos::deep_copy(map_src_cols, map_src_cols_h); - Kokkos::deep_copy(map_tgt_cols, map_tgt_cols_h); - Kokkos::deep_copy(map_wgts , map_wgts_h ); - -//---------------------- - // Step 2: Use HorizontalMap with remap values and source data, and compare against baseline. - std::map y_from_views; - HorizontalMap remap_from_views(comm,"From Views", dofs_gids, tgt_min_dof); - for (int iseg=0; iseg source_dofs("",seg_len); - view_1d weights("",seg_len); - auto source_dofs_h = Kokkos::create_mirror_view(source_dofs); - auto weights_h = Kokkos::create_mirror_view(weights); - int idx = 0; - for (int ii=0; ii x_data_from_views("",num_unique_dofs_from_views); - Kokkos::parallel_for("", num_unique_dofs_from_views, KOKKOS_LAMBDA (const int& ii) { - const int idx = unique_dofs_from_views(ii); - x_data_from_views(ii) = source_remap_data(idx); - }); - // Apply the remap - view_1d y_data_from_views("",num_loc_tgt_cols); - remap_from_views.apply_remap(x_data_from_views,y_data_from_views); - // The remapped values should match the y_baseline - auto y_data_from_views_h = Kokkos::create_mirror_view(y_data_from_views); - Kokkos::deep_copy(y_data_from_views_h,y_data_from_views); - for (int ii=0; ii vec_of_remap_dims = {"n_s"}; - std::vector vec_of_data_dims = {"ncol"}; - std::string int_type = "int"; - scorpio::register_variable(filename,"col","col","unitless",vec_of_remap_dims,"int","int",remap_decomp_tag_i); - scorpio::register_variable(filename,"row","row","unitless",vec_of_remap_dims,"int","int",remap_decomp_tag_i); - scorpio::register_variable(filename,"S","S", "unitless",vec_of_remap_dims,"real","real",remap_decomp_tag_r); - scorpio::register_variable(filename,"src_data","src_data", "m",vec_of_data_dims,"real","real",data_decomp_tag_r); - // - DOFs - std::vector var_dof(n_s_local); - std::iota(var_dof.begin(),var_dof.end(),n_s_offset); - scorpio::set_dof(filename,"col",var_dof.size(),var_dof.data()); - scorpio::set_dof(filename,"row",var_dof.size(),var_dof.data()); - scorpio::set_dof(filename,"S",var_dof.size(),var_dof.data()); - var_dof.resize(unique_dofs_from_views.size()); - auto unique_dofs_from_views_h = Kokkos::create_mirror_view(unique_dofs_from_views); - Kokkos::deep_copy(unique_dofs_from_views_h,unique_dofs_from_views); - for (size_t ii=0; ii x_data_from_file("",var_dof.size()); - auto x_data_from_file_h = Kokkos::create_mirror_view(x_data_from_file); - scorpio::grid_read_data_array(filename,"src_data",0,x_data_from_file.data(),x_data_from_file.size()); - scorpio::eam_pio_closefile(filename); - Kokkos::deep_copy(x_data_from_file_h,x_data_from_file); - // Apply remap using the data - view_1d y_data_from_file("",num_loc_tgt_cols); - remap_from_file.apply_remap(x_data_from_file,y_data_from_file); - // The remapped values should match the y_baseline - auto y_data_from_file_h = Kokkos::create_mirror_view(y_data_from_file); - Kokkos::deep_copy(y_data_from_file_h,y_data_from_file); - for (int ii=0; ii x_2d_data("",unique_dofs_from_file.size(),num_levels); - view_2d y_2d_data("",num_loc_tgt_cols,num_levels); - view_3d x_3d_data("",unique_dofs_from_file.size(),num_bands,num_levels); - view_3d y_3d_data("",num_loc_tgt_cols,num_bands,num_levels); - auto x_2d_data_h = Kokkos::create_mirror_view(x_2d_data); - auto x_3d_data_h = Kokkos::create_mirror_view(x_3d_data); - for (size_t ii=0;ii Testing horizontal remapping for minimum source dof = 0..."); - } - run(engine, comm, 0); - if (comm.am_i_root()) { - printf("ok!\n"); - } - - if (comm.am_i_root()) { - printf(" -> Testing horizontal remapping for minimum source dof = 1..."); - } - run(engine, comm, 1); - if (comm.am_i_root()) { - printf("ok!\n"); - } - -} // TEST_CASE -//=============================================================================== - -TEST_CASE("horizontal_remap_units", "") { - - using namespace scream; - using IPDF = std::uniform_int_distribution; - using RPDF = std::uniform_real_distribution; - auto engine = scream::setup_random_test(); - ekat::Comm comm (MPI_COMM_WORLD); - - // Test the remap_segment structure - { - IPDF pdf_seg_len(2,100); - RPDF pdf_seg_wgt(0,1); - Int len = pdf_seg_len(engine); - HorizontalMapSegment test_segment(0,len); - auto seg_weights = test_segment.get_weights(); - ekat::genRandArray(seg_weights,engine,pdf_seg_wgt); - Real wgt_normalize = 0; - auto weights_h = Kokkos::create_mirror_view(seg_weights); - Kokkos::deep_copy(weights_h,seg_weights); - for (int ii=0; ii dof_gids("",num_dofs); - Kokkos::parallel_for("", num_dofs, KOKKOS_LAMBDA (const int& ii) { - dof_gids(ii) = ii; - }); - test_map.set_dof_gids(dof_gids,0); - // Create a remap segment - Int len = pdf_seg_len(engine); - HorizontalMapSegment test_seg(1,len); - auto test_weights = test_seg.get_weights(); - ekat::genRandArray(test_weights,engine,pdf_seg_wgt); - Real wgt_normalize = 0; - auto weights_h = Kokkos::create_mirror_view(test_weights); - Kokkos::deep_copy(weights_h,test_weights); - for (int ii=0; ii dof_gids("",num_dofs); - Kokkos::parallel_for("", num_dofs, KOKKOS_LAMBDA (const int& ii) { - dof_gids(ii) = ii; - }); - test_map.set_dof_gids(dof_gids,0); - // All of the segments combined should overlap by 2 on either side. The set of unique - // columns for all segments should be 0,1,...,seg_start+seg_len-1. So we check that. - test_map.set_unique_source_dofs(); - const auto& unique_dofs = test_map.get_unique_source_dofs(); - REQUIRE(unique_dofs.extent_int(0) == dof_end); - Kokkos::parallel_for("", dof_end, KOKKOS_LAMBDA (const int& ii) { - EKAT_KERNEL_REQUIRE(unique_dofs(ii) == ii); - }); - } - -} // TEST_CASE horizontal remap -/*===================================================================================================*/ -std::shared_ptr get_test_gm(const ekat::Comm& comm, const Int num_gcols, const Int num_levs) -{ -/* Simple routine to construct and return a grids manager given number of columns and levels */ - auto gm = create_mesh_free_grids_manager(comm,0,0,num_levs,num_gcols); - gm->build_grids(); - return gm; -} // end get_test_gm -/*==========================================================================================================*/ - -} // end namespace diff --git a/components/eamxx/src/share/tests/utils_tests.cpp b/components/eamxx/src/share/tests/utils_tests.cpp index e8a9c4f5c4ce..b3b18bf206b5 100644 --- a/components/eamxx/src/share/tests/utils_tests.cpp +++ b/components/eamxx/src/share/tests/utils_tests.cpp @@ -179,6 +179,16 @@ TEST_CASE ("time_stamp") { auto ts6 = ts1 + spd*1000; REQUIRE ( (ts6-ts1)==spd*1000 ); } + SECTION ("large_updates") { + TS base ({1850,1,1},{0,0,0},1); + for (int i=1; i<=500; ++i) { + TS curr ({1850+i,1,1},{0,0,0},0); + auto diff = curr.seconds_from(base); + TS time = base; + time += diff; + REQUIRE (time==curr); + } + } } TEST_CASE ("array_utils") { diff --git a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp index a3d132bf4f82..3515bcdaac91 100644 --- a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp +++ b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp @@ -161,6 +161,7 @@ void TimeInterpolation::initialize_data_from_files() input_params.set("Field Names",m_field_names); input_params.set("Filename",triplet_curr.filename); m_file_data_atm_input = AtmosphereInput(input_params,m_fm_time1); + m_file_data_atm_input.set_logger(m_logger); // Assign the mask value gathered from the FillValue found in the source file. // TODO: Should we make it possible to check if FillValue is in the metadata and only assign mask_value if it is? float var_fill_value; @@ -319,6 +320,7 @@ void TimeInterpolation::read_data() input_params.set("Field Names",m_field_names); input_params.set("Filename",triplet_curr.filename); m_file_data_atm_input = AtmosphereInput(input_params,m_fm_time1); + m_file_data_atm_input.set_logger(m_logger); // Also determine the FillValue, if used // TODO: Should we make it possible to check if FillValue is in the metadata and only assign mask_value if it is? float var_fill_value; @@ -328,6 +330,11 @@ void TimeInterpolation::read_data() field.get_header().set_extra_data("mask_value",var_fill_value); } } + + if (m_logger) { + m_logger->info(m_header); + m_logger->info("[EAMxx:time_interpolation] Reading data at time " + triplet_curr.timestamp.to_string()); + } m_file_data_atm_input.read_variables(triplet_curr.time_idx); m_time1 = triplet_curr.timestamp; } diff --git a/components/eamxx/src/share/util/eamxx_time_interpolation.hpp b/components/eamxx/src/share/util/eamxx_time_interpolation.hpp index 89d4c917d276..870dad330135 100644 --- a/components/eamxx/src/share/util/eamxx_time_interpolation.hpp +++ b/components/eamxx/src/share/util/eamxx_time_interpolation.hpp @@ -45,6 +45,13 @@ class TimeInterpolation { // Informational void print(); + // Option to add a logger + void set_logger(const std::shared_ptr& logger, + const std::string& header) { + m_logger = logger; + m_header = header; + } + protected: // Internal structure to store data source triplets (when using data from file) @@ -85,7 +92,8 @@ class TimeInterpolation { AtmosphereInput m_file_data_atm_input; bool m_is_data_from_file=false; - + std::shared_ptr m_logger; + std::string m_header; }; // class TimeInterpolation } // namespace util diff --git a/components/eamxx/src/share/util/scream_time_stamp.cpp b/components/eamxx/src/share/util/scream_time_stamp.cpp index fd0ef3785f10..d2ef8243e258 100644 --- a/components/eamxx/src/share/util/scream_time_stamp.cpp +++ b/components/eamxx/src/share/util/scream_time_stamp.cpp @@ -135,12 +135,6 @@ double TimeStamp::frac_of_year_in_days () const { return doy; } -void TimeStamp::set_num_steps (const int num_steps) { - EKAT_REQUIRE_MSG (m_num_steps==0, - "Error! Cannot reset m_num_steps once the count started.\n"); - m_num_steps = num_steps; -} - TimeStamp& TimeStamp::operator+=(const double seconds) { // Sanity checks // Note: (x-int(x)) only works for x small enough that can be stored in an int, @@ -161,30 +155,32 @@ TimeStamp& TimeStamp::operator+=(const double seconds) { auto& yy = m_date[0]; ++m_num_steps; - sec += seconds; + + constexpr auto spd = constants::seconds_per_day; + auto add_days = std::floor(seconds / spd); + auto add_secs = seconds - add_days*spd; + + sec += add_secs; + dd += add_days; // Carry over int carry; carry = sec / 60; - if (carry==0) { - return *this; - } - - sec = sec % 60; - min += carry; - carry = min / 60; - if (carry==0) { - return *this; - } - min = min % 60; - hour += carry; - carry = hour / 24; - - if (carry==0) { - return *this; + if (carry!=0) { + sec = sec % 60; + min += carry; + carry = min / 60; + if (carry!=0) { + min = min % 60; + hour += carry; + carry = hour / 24; + + if (carry!=0) { + hour = hour % 24; + dd += carry; + } + } } - hour = hour % 24; - dd += carry; while (dd>days_in_month(yy,mm)) { dd -= days_in_month(yy,mm); diff --git a/components/eamxx/src/share/util/scream_time_stamp.hpp b/components/eamxx/src/share/util/scream_time_stamp.hpp index fdfec5bb0ebf..259686eacf65 100644 --- a/components/eamxx/src/share/util/scream_time_stamp.hpp +++ b/components/eamxx/src/share/util/scream_time_stamp.hpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace scream { namespace util { @@ -46,9 +47,8 @@ class TimeStamp { // === Update method(s) === // - // Set the counter for the number of steps. Must be called while m_num_steps==0, - // for safety reasons (do not alter num steps while the count started). - void set_num_steps (const int num_steps); + // Set the counter for the number of steps. + void set_num_steps (const int num_steps) { m_num_steps = num_steps; } TimeStamp& operator= (const TimeStamp&) = default; @@ -63,7 +63,7 @@ class TimeStamp { std::vector m_date; // [year, month, day] std::vector m_time; // [hour, min, sec] - int m_num_steps = 0; // Number of steps since simulation started + int m_num_steps = std::numeric_limits::lowest(); // Number of steps since simulation started }; // Overload operators for TimeStamp diff --git a/components/eamxx/src/share/util/scream_universal_constants.hpp b/components/eamxx/src/share/util/scream_universal_constants.hpp index c133cd93cd39..9937bbf595fe 100644 --- a/components/eamxx/src/share/util/scream_universal_constants.hpp +++ b/components/eamxx/src/share/util/scream_universal_constants.hpp @@ -2,6 +2,7 @@ #define SCREAM_UNIVERSAL_CONSTANTS_HPP #include +#include namespace scream { @@ -14,9 +15,9 @@ constexpr int days_per_nonleap_year = 365; // TODO: When we switch to supporting C++17 we can use a simple `inline constexpr` rather than a struct template struct DefaultFillValue { - static const bool is_float = std::is_floating_point::value; - static const bool is_int = std::is_integral::value; - T value = is_int ? std::numeric_limits::max() / 2 : + static constexpr bool is_float = std::is_floating_point::value; + static constexpr bool is_int = std::is_integral::value; + static constexpr T value = is_int ? std::numeric_limits::max() / 2 : is_float ? std::numeric_limits::max() / 1e5 : std::numeric_limits::max(); }; diff --git a/components/eamxx/src/share/util/scream_utils.cpp b/components/eamxx/src/share/util/scream_utils.cpp index 91a412dfd8a4..2083f4d62b64 100644 --- a/components/eamxx/src/share/util/scream_utils.cpp +++ b/components/eamxx/src/share/util/scream_utils.cpp @@ -1,4 +1,5 @@ #include "share/util/scream_utils.hpp" +#include #if defined(SCREAM_ENABLE_STATM) #include @@ -43,4 +44,33 @@ long long get_mem_usage (const MemoryUnits u) { return mem; } +std::vector filename_glob(const std::vector& patterns) { + std::vector all_files; + for (const auto& pattern : patterns) { + auto files = globloc(pattern); + all_files.insert(all_files.end(), files.begin(), files.end()); + } + return all_files; +} + +std::vector globloc(const std::string& pattern) { + // glob struct resides on the stack + glob_t glob_result; + memset(&glob_result, 0, sizeof(glob_result)); + + int return_value = ::glob(pattern.c_str(), GLOB_TILDE, NULL, &glob_result); + if (return_value != 0) { + globfree(&glob_result); + EKAT_REQUIRE_MSG(return_value == 0, "glob() failed with return value " + std::to_string(return_value)); + } + + std::vector filenames; + for (size_t i = 0; i < glob_result.gl_pathc; ++i) { + filenames.push_back(std::string(glob_result.gl_pathv[i])); + } + + globfree(&glob_result); + return filenames; +} + } // namespace scream diff --git a/components/eamxx/src/share/util/scream_utils.hpp b/components/eamxx/src/share/util/scream_utils.hpp index 4dddebf75aa5..b1771f3a1aed 100644 --- a/components/eamxx/src/share/util/scream_utils.hpp +++ b/components/eamxx/src/share/util/scream_utils.hpp @@ -11,6 +11,7 @@ #include #include #include +#include namespace scream { @@ -347,6 +348,12 @@ check_mpi_call (int err, const std::string& context) { " - context: " + context + "\n"); } +// Find the full filename list from patterns +std::vector filename_glob(const std::vector& patterns); + +// Use globloc for each filename pattern +std::vector globloc(const std::string& pattern); + } // namespace scream #endif // SCREAM_UTILS_HPP diff --git a/components/eamxx/tests/CMakeLists.txt b/components/eamxx/tests/CMakeLists.txt index ab168e67e79f..7a7a9e821493 100644 --- a/components/eamxx/tests/CMakeLists.txt +++ b/components/eamxx/tests/CMakeLists.txt @@ -1,5 +1,55 @@ include (ScreamUtils) +# This function compares an output file created by an AD-driven test with its +# corresponding one in the baselines dir. The inputs (all REQUIRED) are +# +# - TEST_BASE_NAME: the base name of the test that generated the file; will be used also +# as the prefix for the baseline_cmp test name +# - GEN_TEST_NRANKS: the number of ranks used in the test that generated the file +# - OUT_FILE: the name of the output nc file +# - FIXTURES_BASE_NAME: base name of the FIXTURES_SETUP property of the test that generated the file. +# The string _np${GEN_TEST_NRANKS}_omp1 will be attached to this + +function (CreateBaselineTest TEST_BASE_NAME GEN_TEST_NRANKS OUT_FILE FIXTURES_BASE_NAME) + # Get names of src and tgt files + set (SRC_FILE ${CMAKE_CURRENT_BINARY_DIR}/${OUT_FILE}) + set (TGT_FILE ${SCREAM_BASELINES_DIR}/data/${OUT_FILE}) + + # Add comparison test using the CprncTest.cmake scritp shipped by EAMxx + add_test ( + NAME ${TEST_BASE_NAME}_baseline_cmp + COMMAND cmake -P ${CMAKE_BINARY_DIR}/bin/CprncTest.cmake ${SRC_FILE} ${TGT_FILE} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + set_tests_properties( + ${TEST_BASE_NAME}_baseline_cmp + PROPERTIES + LABELS baseline_cmp + FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_np${TEST_RANK_END}_omp1) + + # Add the test that generated the baseline to the baseline_gen label, so scripts/test-all-scream + # can run it when it has to generate baselines + if (TEST ${TEST_BASE_NAME}_np${GEN_TEST_NRANKS}) + set (GEN_TEST_FULL_NAME ${TEST_BASE_NAME}_np${GEN_TEST_NRANKS}) + elseif(TEST ${TEST_BASE_NAME}) + set (GEN_TEST_FULL_NAME ${TEST_BASE_NAME}) + else() + string (CONCAT msg + "Could not find the test that generated the output file\n" + " TEST_BASE_NAME: ${TEST_BASE_NAME}\n" + " OUT_FILE : ${OUT_FILE}\n") + message ("${msg}") + message (FATAL_ERROR "Aborting...") + endif() + set_tests_properties (${GEN_TEST_FULL_NAME} PROPERTIES LABELS baseline_gen) + + # test-all-scream will read this file to get the list of baseline nc files to + # copy into the baseline dir + file (APPEND ${SCREAM_TEST_OUTPUT_DIR}/baseline_list + "${SRC_FILE}\n" + ) + +endfunction() + # Some tests for checking that certain testing infrastructures work add_subdirectory(generic) @@ -10,13 +60,14 @@ if (NOT DEFINED ENV{SCREAM_FAKE_ONLY}) set(TEST_RANK_END ${SCREAM_TEST_MAX_RANKS}) # Initial condition files used in the tests - set(EAMxx_tests_IC_FILE_72lev "screami_unit_tests_ne2np4L72_20220822.nc") - set(EAMxx_tests_IC_FILE_128lev "screami_unit_tests_ne2np4L128_20220822.nc") - set(EAMxx_tests_TOPO_FILE "USGS-gtopo30_ne2np4pg2_x6t_20230331.nc") + set(EAMxx_tests_IC_FILE_72lev "screami_unit_tests_ne2np4L72_20220822.nc") + set(EAMxx_tests_IC_FILE_128lev "screami_unit_tests_ne2np4L128_20220822.nc") + set(EAMxx_tests_TOPO_FILE "USGS-gtopo30_ne2np4pg2_x6t_20230331.nc") + set(EAMxx_tests_IC_FILE_MAM4xx_72lev "scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc") # Testing individual atm processes - add_subdirectory(uncoupled) + add_subdirectory(single-process) # Testing multiple atm processes coupled together - add_subdirectory(coupled) + add_subdirectory(multi-process) endif() diff --git a/components/eamxx/tests/coupled/CMakeLists.txt b/components/eamxx/tests/coupled/CMakeLists.txt deleted file mode 100644 index b71893963fdb..000000000000 --- a/components/eamxx/tests/coupled/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -# NOTE: if you have baseline-type tests, add the subdirectory OUTSIDE the following if statement -if (NOT SCREAM_BASELINES_ONLY) - add_subdirectory(physics_only) - if (NOT "${SCREAM_DYNAMICS_DYCORE}" STREQUAL "NONE") - add_subdirectory(dynamics_physics) - endif() -endif() diff --git a/components/eamxx/tests/generic/fail_check/CMakeLists.txt b/components/eamxx/tests/generic/fail_check/CMakeLists.txt index 7c23f4fd52d5..8bbd96ccb67b 100644 --- a/components/eamxx/tests/generic/fail_check/CMakeLists.txt +++ b/components/eamxx/tests/generic/fail_check/CMakeLists.txt @@ -8,29 +8,25 @@ include(ScreamUtils) # - a REQUIRE clause that fails makes the test fail # - Our Create unit test logic does work for catching failures CreateUnitTest (fail "fail.cpp" - PROPERTIES WILL_FAIL TRUE - LABELS "fail") + WILL_FAIL LABELS "fail") if (Kokkos_ENABLE_DEBUG_BOUNDS_CHECK) # Ensure that Kokkos OOB are caught CreateUnitTest (kokkos_fail "kokkos_fail.cpp" - PROPERTIES WILL_FAIL TRUE - LABELS "fail") + WILL_FAIL LABELS "fail") endif() if (EKAT_ENABLE_VALGRIND) # Ensure that valgrind errors are caught EkatCreateUnitTest (valg_fail "valg_fail.cpp" - PROPERTIES WILL_FAIL TRUE - LABELS "fail") + WILL_FAIL LABELS "fail") endif() # Ensure that FPE *do* throw when we expect them to CreateUnitTestExec (scream_fpe_check "fpe_check.cpp") if (SCREAM_FPE) CreateUnitTestFromExec (scream_fpe_check scream_fpe_check - PROPERTIES WILL_FAIL TRUE - LABELS "check") + WILL_FAIL LABELS "check") else() CreateUnitTestFromExec (scream_fpe_check scream_fpe_check LABELS "check") diff --git a/components/eamxx/tests/meta-tests/CMakeLists.txt b/components/eamxx/tests/meta-tests/CMakeLists.txt index d5cf8f1b41e2..b583d5ef825e 100644 --- a/components/eamxx/tests/meta-tests/CMakeLists.txt +++ b/components/eamxx/tests/meta-tests/CMakeLists.txt @@ -11,24 +11,30 @@ include(ScreamUtils) # the test executable). # Test to ensure that a build failure is detected by our testing scripts -EkatCreateUnitTest(build_fail build_fail.cpp +EkatCreateUnitTest(build_fail build_fail.cpp EXCLUDE_MAIN_CPP EXCLUDE_TEST_SESSION LABELS "fail") # Test to ensure that a test failure is detected by our testing scripts -EkatCreateUnitTest(test_fail test_fail.cpp +EkatCreateUnitTest(test_fail test_fail.cpp EXCLUDE_MAIN_CPP EXCLUDE_TEST_SESSION LABELS "fail") # Tests to ensure the testing infrastructure is correctly spreading # concurrent tests across the available resources if (SCREAM_TEST_MAX_TOTAL_THREADS GREATER_EQUAL 16) - EkatCreateUnitTestExec(resource_spread resource_spread.cpp) + EkatCreateUnitTestExec(resource_spread resource_spread.cpp EXCLUDE_MAIN_CPP EXCLUDE_TEST_SESSION) - CreateUnitTestFromExec(resource_spread_thread resource_spread + # When scripts-tests builds this folder, they are not building Ekat (or even Kokkos). + # So we must add openmp to the compiler/linker flags manually + # NOTE: scripts-tests is already checking that this is an "OpenMP machine", so this is safe + find_package(OpenMP REQUIRED COMPONENTS CXX) + target_link_libraries(resource_spread OpenMP::OpenMP_CXX MPI::MPI_C) + + EkatCreateUnitTestFromExec(resource_spread_thread resource_spread PRINT_OMP_AFFINITY THREADS 1 4 1 MPI_RANKS 4) - CreateUnitTestFromExec(resource_spread_rank resource_spread + EkatCreateUnitTestFromExec(resource_spread_rank resource_spread PRINT_OMP_AFFINITY THREADS 4 MPI_RANKS 1 4 1) diff --git a/components/eamxx/tests/meta-tests/build_fail.cpp b/components/eamxx/tests/meta-tests/build_fail.cpp index 23f46a51e802..8ed7844235a6 100644 --- a/components/eamxx/tests/meta-tests/build_fail.cpp +++ b/components/eamxx/tests/meta-tests/build_fail.cpp @@ -1,16 +1,10 @@ -#include - #include -namespace scream { - #ifdef SCREAM_FORCE_BUILD_FAIL #error "Forcing failure to test test-all-scream" #endif -TEST_CASE("pass", "[fake_infra_test]") +int main(int,char**) { - REQUIRE(true); + return 0; } - -} // empty namespace diff --git a/components/eamxx/tests/meta-tests/resource_spread.cpp b/components/eamxx/tests/meta-tests/resource_spread.cpp index c3905cf25701..9b59297b84dd 100644 --- a/components/eamxx/tests/meta-tests/resource_spread.cpp +++ b/components/eamxx/tests/meta-tests/resource_spread.cpp @@ -1,19 +1,17 @@ -#include "ekat/kokkos/ekat_kokkos_types.hpp" - -#include - -#include #include #include -namespace scream { +#include -TEST_CASE("rank_and_thread_spread", "[fake_infra_test]") +int main(int argc, char** argv) { + MPI_Init(&argc,&argv); + // Nothing needs to be done here except sleeping to give a chance // for tests to run concurrently. const auto seconds_to_sleep = 5; std::this_thread::sleep_for(std::chrono::seconds(seconds_to_sleep)); -} -} // empty namespace + MPI_Finalize(); + return 0; +} diff --git a/components/eamxx/tests/meta-tests/test_fail.cpp b/components/eamxx/tests/meta-tests/test_fail.cpp index 7acfe34adc40..4e8acdb584a8 100644 --- a/components/eamxx/tests/meta-tests/test_fail.cpp +++ b/components/eamxx/tests/meta-tests/test_fail.cpp @@ -1,16 +1,10 @@ -#include - #include -namespace scream { - -TEST_CASE("pass", "[fake_infra_test]") +int main(int, char**) { #ifdef SCREAM_FORCE_RUN_FAIL - REQUIRE(false); + return 1; #else - REQUIRE(true); + return 0; #endif } - -} // empty namespace diff --git a/components/eamxx/tests/meta-tests/test_level_check/CMakeLists.txt b/components/eamxx/tests/meta-tests/test_level_check/CMakeLists.txt index 3c015db4f93e..18f94c286b10 100644 --- a/components/eamxx/tests/meta-tests/test_level_check/CMakeLists.txt +++ b/components/eamxx/tests/meta-tests/test_level_check/CMakeLists.txt @@ -1,8 +1,8 @@ include(ScreamUtils) -EkatCreateUnitTestExec(at_unit at_unit.cpp) -EkatCreateUnitTestExec(nightly_unit nightly_unit.cpp) -EkatCreateUnitTestExec(experimental_unit experimental_unit.cpp) +EkatCreateUnitTestExec(at_unit at_unit.cpp EXCLUDE_MAIN_CPP EXCLUDE_TEST_SESSION) +EkatCreateUnitTestExec(nightly_unit nightly_unit.cpp EXCLUDE_MAIN_CPP EXCLUDE_TEST_SESSION) +EkatCreateUnitTestExec(experimental_unit experimental_unit.cpp EXCLUDE_MAIN_CPP EXCLUDE_TEST_SESSION) CreateUnitTestFromExec(at_unit at_unit) CreateUnitTestFromExec(nightly_unit nightly_unit MINIMUM_TEST_LEVEL ${SCREAM_TEST_LEVEL_NIGHTLY}) diff --git a/components/eamxx/tests/meta-tests/test_level_check/at_unit.cpp b/components/eamxx/tests/meta-tests/test_level_check/at_unit.cpp index 5c2ef4bbbec0..280c9d343650 100644 --- a/components/eamxx/tests/meta-tests/test_level_check/at_unit.cpp +++ b/components/eamxx/tests/meta-tests/test_level_check/at_unit.cpp @@ -1,12 +1,7 @@ -#include - #include -namespace scream { - -TEST_CASE("at_unit") +int main(int,char**) { std::cout << "AT unit test running" << std::endl; + return 0; } - -} // empty namespace diff --git a/components/eamxx/tests/meta-tests/test_level_check/experimental_unit.cpp b/components/eamxx/tests/meta-tests/test_level_check/experimental_unit.cpp index 7598eb628ae0..012060d405b5 100644 --- a/components/eamxx/tests/meta-tests/test_level_check/experimental_unit.cpp +++ b/components/eamxx/tests/meta-tests/test_level_check/experimental_unit.cpp @@ -1,12 +1,7 @@ -#include - #include -namespace scream { - -TEST_CASE("experimental_unit") +int main(int,char**) { std::cout << "EXPERIMENTAL unit test running" << std::endl; + return 0; } - -} // empty namespace diff --git a/components/eamxx/tests/meta-tests/test_level_check/nightly_unit.cpp b/components/eamxx/tests/meta-tests/test_level_check/nightly_unit.cpp index e04ffccff98c..22c9fbcb7ce3 100644 --- a/components/eamxx/tests/meta-tests/test_level_check/nightly_unit.cpp +++ b/components/eamxx/tests/meta-tests/test_level_check/nightly_unit.cpp @@ -1,12 +1,7 @@ -#include - #include -namespace scream { - -TEST_CASE("nightly_unit") +int main(int,char**) { std::cout << "NIGHTLY unit test running" << std::endl; + return 0; } - -} // empty namespace diff --git a/components/eamxx/tests/multi-process/CMakeLists.txt b/components/eamxx/tests/multi-process/CMakeLists.txt new file mode 100644 index 000000000000..827a6167e13c --- /dev/null +++ b/components/eamxx/tests/multi-process/CMakeLists.txt @@ -0,0 +1,4 @@ +add_subdirectory(physics_only) +if (NOT "${SCREAM_DYNAMICS_DYCORE}" STREQUAL "NONE") + add_subdirectory(dynamics_physics) +endif() diff --git a/components/eamxx/tests/coupled/dynamics_physics/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt similarity index 57% rename from components/eamxx/tests/coupled/dynamics_physics/CMakeLists.txt rename to components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt index 5127f42969a2..5b263cbd66e5 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt @@ -2,13 +2,17 @@ if (SCREAM_DOUBLE_PRECISION) if ("${SCREAM_DYNAMICS_DYCORE}" STREQUAL "HOMME") add_subdirectory(homme_shoc_cld_p3_rrtmgp) + add_subdirectory(homme_shoc_cld_p3_rrtmgp_pg2) add_subdirectory(model_restart) add_subdirectory(homme_shoc_cld_spa_p3_rrtmgp) add_subdirectory(homme_shoc_cld_spa_p3_rrtmgp_128levels) - add_subdirectory(homme_shoc_cld_spa_p3_rrtmgp_pg2) add_subdirectory(homme_shoc_cld_spa_p3_rrtmgp_dp) if (SCREAM_ENABLE_MAM) - add_subdirectory(homme_mam4xx_pg2) + # Once the mam4xx aerosol microphysics AtmosphereProcess is running, the + # corresponding test here needs to be reworked with valid aerosol + # initial conditions. + #add_subdirectory(homme_mam4xx_pg2) + add_subdirectory(mam/homme_shoc_cld_p3_mam_optics_rrtmgp) endif() endif() endif() diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/homme_mam4xx_pg2/CMakeLists.txt similarity index 84% rename from components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/CMakeLists.txt rename to components/eamxx/tests/multi-process/dynamics_physics/homme_mam4xx_pg2/CMakeLists.txt index bb24e93b28d5..9463333ce39c 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_mam4xx_pg2/CMakeLists.txt @@ -1,5 +1,8 @@ include (ScreamUtils) +# This test needs to be reworked for microphysics -- currently it's still using +# input for nucleation. + set (TEST_BASE_NAME homme_mam4xx_pg2) set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) @@ -82,3 +85,10 @@ CompareNCFilesFamilyMpi ( LABELS dynamics physics mam4xx META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_mam4xx_pg2/input.yaml similarity index 100% rename from components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/input.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_mam4xx_pg2/input.yaml diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_mam4xx_pg2/output.yaml similarity index 96% rename from components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/output.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_mam4xx_pg2/output.yaml index 23beda243aea..c2e1a6bf17de 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_mam4xx_pg2/output.yaml @@ -40,5 +40,4 @@ Fields: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/CMakeLists.txt similarity index 87% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/CMakeLists.txt rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/CMakeLists.txt index d4618934c7c4..22acf3d88a4f 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/CMakeLists.txt @@ -79,3 +79,10 @@ CompareNCFilesFamilyMpi ( LABELS dynamics physics shoc cld p3 rrtmgp META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/input.yaml similarity index 95% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/input.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/input.yaml index 5eef64e73f5f..316b43ffdf73 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/input.yaml @@ -15,15 +15,10 @@ time_stepping: initial_conditions: Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} - get_topo_from_file: true surf_evap: 0.0 surf_sens_flux: 0.0 precip_liq_surf_mass: 0.0 precip_ice_surf_mass: 0.0 - aero_g_sw: 0.0 - aero_ssa_sw: 0.0 - aero_tau_sw: 0.0 - aero_tau_lw: 0.0 atmosphere_processes: atm_procs_list: [homme,physics] diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/output.yaml similarity index 86% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/output.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/output.yaml index 824d648f2b2e..b2f643933fcd 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/output.yaml @@ -2,7 +2,9 @@ --- filename_prefix: homme_shoc_cld_p3_rrtmgp_output Averaging Type: Instant -Max Snapshots Per File: 1 +Max Snapshots Per File: 744 # One output every 31 days +#remap_file: /g/g17/donahue5/Code/e3sm/scream-docs/regional_output_sites/20221123_ARM_sites_map.nc +remap_file: /usr/gdata/e3sm/ccsm3data/inputdata/atm/scream/maps/map_ne30np4_to_ne4pg2_mono.20220714.nc Fields: Physics GLL: Field Names: @@ -82,5 +84,4 @@ Fields: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/CMakeLists.txt similarity index 82% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2/CMakeLists.txt rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/CMakeLists.txt index 46d0513ae3f4..c3fa3c7bc2a9 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/CMakeLists.txt @@ -1,6 +1,6 @@ include (ScreamUtils) -set (TEST_BASE_NAME homme_shoc_cld_spa_p3_rrtmgp_pg2) +set (TEST_BASE_NAME homme_shoc_cld_p3_rrtmgp_pg2) set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) # Get or create the dynamics lib @@ -10,7 +10,7 @@ CreateDynamicsLib("theta-l_kokkos" 4 72 10) # Create the test CreateADUnitTest(${TEST_BASE_NAME} LABELS dynamics tms shoc cld p3 rrtmgp physics pg2 - LIBS cld_fraction nudging tms shoc spa p3 scream_rrtmgp ${dynLibName} diagnostics + LIBS cld_fraction nudging tms shoc p3 scream_rrtmgp ${dynLibName} diagnostics MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} ) @@ -67,8 +67,6 @@ configure_file(${SCREAM_SRC_DIR}/dynamics/homme/tests/theta.nl # Ensure test input files are present in the data dir set (TEST_INPUT_FILES - scream/maps/map_ne4np4_to_ne2np4_mono.nc - scream/init/spa_file_unified_and_complete_ne4_20220428.nc scream/init/${EAMxx_tests_IC_FILE_72lev} cam/topo/${EAMxx_tests_TOPO_FILE} ) @@ -83,6 +81,13 @@ CompareNCFilesFamilyMpi ( TEST_BASE_NAME ${TEST_BASE_NAME} FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} - LABELS dynamics physics tms shoc cld p3 rrtmgp spa pg2 + LABELS dynamics physics tms shoc cld p3 rrtmgp pg2 META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/input.yaml similarity index 86% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2/input.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/input.yaml index b778dbc2bbd8..8e463b8be24c 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/input.yaml @@ -15,6 +15,9 @@ initial_conditions: surf_sens_flux: 0.0 precip_liq_surf_mass: 0.0 precip_ice_surf_mass: 0.0 + surf_lw_flux_up: 400.0 + eddy_diff_mom: 0.02 + sgs_buoy_flux: -0.001 tke: 0.0 qm: 0.0 bm: 0.0 @@ -30,18 +33,16 @@ atmosphere_processes: homme: Moisture: moist physics: - atm_procs_list: [mac_aero_mic,rrtmgp] + atm_procs_list: [mac_mic,rrtmgp] schedule_type: Sequential Type: Group - mac_aero_mic: - atm_procs_list: [tms,shoc,CldFraction,spa,p3] + mac_mic: + atm_procs_list: [tms,shoc,CldFraction,p3] Type: Group schedule_type: Sequential number_of_subcycles: ${MAC_MIC_SUBCYCLES} - spa: - spa_remap_file: ${SCREAM_DATA_DIR}/maps/map_ne4np4_to_ne2np4_mono.nc - spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne4_20220428.nc p3: + do_prescribed_ccn: false max_total_ni: 740.0e3 shoc: lambda_low: 0.001 @@ -63,6 +64,7 @@ atmosphere_processes: rrtmgp_coefficients_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-lw-g128-210809.nc rrtmgp_cloud_optics_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-sw.nc rrtmgp_cloud_optics_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-lw.nc + do_aerosol_rad: false grids_manager: Type: Homme diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/output.yaml similarity index 93% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2/output.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/output.yaml index ca226588db27..d07cdf5a4d0b 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2/output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/output.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: homme_shoc_cld_spa_p3_rrtmgp_pg2_output +filename_prefix: homme_shoc_cld_p3_rrtmgp_pg2_output Averaging Type: Instant Max Snapshots Per File: 1 Fields: @@ -25,12 +25,6 @@ Fields: # CLD - cldfrac_ice - cldfrac_tot - # SPA - - aero_g_sw - - aero_ssa_sw - - aero_tau_lw - - aero_tau_sw - - nccn # P3 - bm - nc @@ -125,5 +119,4 @@ Fields: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/CMakeLists.txt similarity index 86% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/CMakeLists.txt rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/CMakeLists.txt index 03617b46bb0e..6308e2794112 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/CMakeLists.txt @@ -68,8 +68,7 @@ configure_file(${SCREAM_SRC_DIR}/dynamics/homme/tests/theta.nl # Ensure test input files are present in the data dir set (TEST_INPUT_FILES scream/init/spa_init_ne2np4.nc - scream/maps/map_ne4np4_to_ne2np4_mono.nc - scream/init/spa_file_unified_and_complete_ne4_20220428.nc + scream/init/spa_file_unified_and_complete_ne2np4L72_20231222.nc scream/init/${EAMxx_tests_IC_FILE_72lev} cam/topo/${EAMxx_tests_TOPO_FILE} ) @@ -87,3 +86,10 @@ CompareNCFilesFamilyMpi ( LABELS dynamics physics shoc cld p3 rrtmgp spa META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/input.yaml similarity index 94% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/input.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/input.yaml index fc2d6cab6c25..d2f51e9e1ccb 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/input.yaml @@ -31,8 +31,7 @@ atmosphere_processes: schedule_type: Sequential number_of_subcycles: ${MAC_MIC_SUBCYCLES} spa: - spa_remap_file: ${SCREAM_DATA_DIR}/maps/map_ne4np4_to_ne2np4_mono.nc - spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne4_20220428.nc + spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne2np4L72_20231222.nc p3: max_total_ni: 740.0e3 shoc: diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/output.yaml similarity index 98% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/output.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/output.yaml index 9728316223a9..21489de893ce 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/output.yaml @@ -104,5 +104,4 @@ Fields: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/CMakeLists.txt similarity index 88% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/CMakeLists.txt rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/CMakeLists.txt index 1daeade05d61..fe5282265857 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/CMakeLists.txt @@ -67,7 +67,7 @@ configure_file(${SCREAM_SRC_DIR}/dynamics/homme/tests/theta.nl # Ensure test input files are present in the data dir set (TEST_INPUT_FILES - scream/init/spa_file_unified_and_complete_ne4_20220428.nc + scream/init/spa_file_unified_and_complete_ne2np4L72_20231222.nc scream/init/${EAMxx_tests_IC_FILE_128lev} cam/topo/${EAMxx_tests_TOPO_FILE} ) @@ -98,3 +98,10 @@ foreach (NRANKS RANGE ${TEST_RANK_START} ${TEST_RANK_END}) LABELS "dynamics;shoc;cld;p3;rrtmgp;physics" FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_np${NRANKS}_omp1) endforeach() + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/input.yaml similarity index 94% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/input.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/input.yaml index 7c51e54dad0a..b92889cdcaa0 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/input.yaml @@ -32,8 +32,7 @@ atmosphere_processes: schedule_type: Sequential number_of_subcycles: ${MAC_MIC_SUBCYCLES} spa: - spa_remap_file: ${SCREAM_DATA_DIR}/maps/map_ne4np4_to_ne2np4_mono.nc - spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne4_20220428.nc + spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne2np4L72_20231222.nc p3: max_total_ni: 740.0e3 shoc: diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/output.yaml similarity index 85% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/output.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/output.yaml index 9e5249efe800..75c1d29a05ea 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/output.yaml @@ -68,9 +68,17 @@ Fields: - sfc_flux_sw_net - EAMxx_T_mid_tend - EAMxx_qv_tend + # Sliced Output + - T_mid_at_2m_above_surface + - T_mid_at_1000m_above_sealevel + - T_mid_at_500hPa + - T_mid_at_50000Pa + - T_mid_at_500mb + - T_mid_at_model_top + - T_mid_at_model_bot + - T_mid_at_lev_10 output_control: Frequency: 1 frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/CMakeLists.txt similarity index 84% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/CMakeLists.txt rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/CMakeLists.txt index 8a6bf81d4039..ef2494304f58 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/CMakeLists.txt @@ -8,7 +8,7 @@ set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) CreateDynamicsLib("theta-l_kokkos" 4 128 10) # Create the test -set (TEST_LABELS "dynamics;driver;shoc;cld;p3;rrtmgp;physics;dp") +set (TEST_LABELS "dynamics;driver;shoc;cld;spa;p3;rrtmgp;physics;dp") CreateUnitTest(homme_shoc_cld_spa_p3_rrtmgp_dp "homme_shoc_cld_spa_p3_rrtmgp_dp.cpp" LABELS ${TEST_LABELS} LIBS cld_fraction shoc spa p3 scream_rrtmgp ${dynLibName} scream_control diagnostics @@ -17,7 +17,7 @@ CreateUnitTest(homme_shoc_cld_spa_p3_rrtmgp_dp "homme_shoc_cld_spa_p3_rrtmgp_dp. ) # Set AD configurable options -set (ATM_TIME_STEP 50) +set (ATM_TIME_STEP 100) SetVarDependingOnTestSize(NUM_STEPS 2 4 48) # 1h 2h 24h set (RUN_T0 1999-07-10-00000) @@ -70,9 +70,6 @@ configure_file(${SCREAM_SRC_DIR}/dynamics/homme/tests/theta-dp.nl # Ensure test input files are present in the data dir set (TEST_INPUT_FILES - scream/init/spa_init_ne2np4.nc - scream/maps/map_ne4np4_to_ne2np4_mono.nc - scream/init/spa_file_unified_and_complete_ne4_20220428.nc scream/init/${EAMxx_tests_IC_FILE_128lev} cam/topo/${EAMxx_tests_TOPO_FILE} cam/scam/iop/DYCOMSrf01_iopfile_4scam.nc @@ -88,6 +85,13 @@ CompareNCFilesFamilyMpi ( TEST_BASE_NAME ${TEST_BASE_NAME} FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} - LABELS dynamics physics shoc cld p3 rrtmgp spa dp + LABELS dynamics physics shoc cld spa p3 rrtmgp dp META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/homme_shoc_cld_spa_p3_rrtmgp_dp.cpp b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/homme_shoc_cld_spa_p3_rrtmgp_dp.cpp similarity index 98% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/homme_shoc_cld_spa_p3_rrtmgp_dp.cpp rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/homme_shoc_cld_spa_p3_rrtmgp_dp.cpp index 5a8dc0fb29bf..235b7f22a16b 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/homme_shoc_cld_spa_p3_rrtmgp_dp.cpp +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/homme_shoc_cld_spa_p3_rrtmgp_dp.cpp @@ -56,7 +56,6 @@ TEST_CASE("scream_homme_physics", "scream_homme_physics") { ad.init_time_stamps (t0, t0); ad.create_atm_processes (); ad.create_grids (); - ad.setup_intensive_observation_period (); ad.create_fields (); // Setup surface coupler import to be NaNs for fields IOP should overwrite @@ -82,7 +81,6 @@ TEST_CASE("scream_homme_physics", "scream_homme_physics") { ad.setup_surface_coupling_data_manager(SurfaceCouplingTransferType::Import, 4, 4, ncols, import_data.data(), import_names[0], import_cpl_indices.data(), import_vec_comps.data(), import_constant_multiple.data(), do_import_during_init.data()); - ad.initialize_fields (); ad.initialize_output_managers (); ad.initialize_atm_procs (); diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/input.yaml similarity index 91% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/input.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/input.yaml index c545245de90c..950ef7e4db38 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/input.yaml @@ -2,13 +2,15 @@ --- driver_options: atmosphere_dag_verbosity_level: 5 - enable_intensive_observation_period: true + enable_iop: true -intensive_observation_period_options: +iop_options: doubly_periodic_mode: true iop_file: ${IOP_DATA_DIR}/DYCOMSrf01_iopfile_4scam.nc target_latitude: 31.5 target_longitude: 238.5 + iop_dosubsidence: true + iop_srf_prop: true time_stepping: time_step: ${ATM_TIME_STEP} @@ -25,6 +27,9 @@ initial_conditions: perturbed_fields: [T_mid] perturbation_limit: 0.001 perturbation_minimum_pressure: 900.0 # in millibar + surf_lw_flux_up: 400.0 + eddy_diff_mom: 0.02 + sgs_buoy_flux: -0.001 atmosphere_processes: atm_procs_list: [sc_import,homme,physics] @@ -41,8 +46,7 @@ atmosphere_processes: schedule_type: Sequential number_of_subcycles: ${MAC_MIC_SUBCYCLES} spa: - spa_remap_file: ${SCREAM_DATA_DIR}/maps/map_ne4np4_to_ne2np4_mono.nc - spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne4_20220428.nc + spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne2np4L72_20231222.nc p3: max_total_ni: 740.0e3 shoc: @@ -67,6 +71,7 @@ atmosphere_processes: rrtmgp_coefficients_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-lw-g128-210809.nc rrtmgp_cloud_optics_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-sw.nc rrtmgp_cloud_optics_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-lw.nc + do_aerosol_rad: false grids_manager: Type: Homme diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/output.yaml similarity index 98% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/output.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/output.yaml index 1c7f06085f81..ddec1a3b9fa6 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/output.yaml @@ -104,6 +104,5 @@ Fields: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/CMakeLists.txt new file mode 100644 index 000000000000..024aa833df39 --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/CMakeLists.txt @@ -0,0 +1,109 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME homme_shoc_cld_p3_mam_optics_rrtmgp) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Get or create the dynamics lib +# HOMME_TARGET NP PLEV QSIZE_D +CreateDynamicsLib("theta-l_kokkos" 4 72 41) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS cld_fraction ${dynLibName} shoc p3 scream_rrtmgp mam + LABELS dynamics shoc cld p3 rrtmgp physics mam4_optics + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 4 48) # 1h 2h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep shoc dt<=300s +set (SHOC_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_MAX_DT}") + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Set homme's test options, so that we can configure the namelist correctly +# Discretization/algorithm settings +set (HOMME_TEST_NE 2) +set (HOMME_TEST_LIM 9) +set (HOMME_TEST_REMAP_FACTOR 3) +set (HOMME_TEST_TRACERS_FACTOR 1) +set (HOMME_TEST_TIME_STEP 300) +set (HOMME_THETA_FORM 1) +set (HOMME_TTYPE 5) +set (HOMME_SE_FTYPE 0) +set (HOMME_TEST_TRANSPORT_ALG 0) +set (HOMME_TEST_CUBED_SPHERE_MAP 0) + +# Hyperviscosity settings +set (HOMME_TEST_HVSCALING 0) +set (HOMME_TEST_HVS 1) +set (HOMME_TEST_HVS_TOM 0) +set (HOMME_TEST_HVS_Q 1) + +set (HOMME_TEST_NU 7e15) +set (HOMME_TEST_NUDIV 1e15) +set (HOMME_TEST_NUTOP 2.5e5) + +# Testcase settings +set (HOMME_TEST_MOISTURE notdry) +set (HOMME_THETA_HY_MODE true) + +# Vert coord settings +set (HOMME_TEST_VCOORD_INT_FILE acme-72i.ascii) +set (HOMME_TEST_VCOORD_MID_FILE acme-72m.ascii) + +# Configure the namelist into the test directory +configure_file(${SCREAM_SRC_DIR}/dynamics/homme/tests/theta.nl + ${CMAKE_CURRENT_BINARY_DIR}/namelist.nl) + +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/mam4xx/physprops/mam4_mode1_rrtmg_aeronetdust_c20240206.nc + scream/mam4xx/physprops/mam4_mode2_rrtmg_c20240206.nc + scream/mam4xx/physprops/mam4_mode3_rrtmg_aeronetdust_c20240206.nc + scream/mam4xx/physprops/mam4_mode4_rrtmg_c20240206.nc + scream/mam4xx/physprops/water_refindex_rrtmg_c20240206.nc + scream/mam4xx/physprops/ocphi_rrtmg_c20240206.nc + scream/mam4xx/physprops/dust_aeronet_rrtmg_c20240206.nc + scream/mam4xx/physprops/ssam_rrtmg_c20240206.nc + scream/mam4xx/physprops/sulfate_rrtmg_c20240206.nc + scream/mam4xx/physprops/ocpho_rrtmg_c20240206.nc + scream/mam4xx/physprops/bcpho_rrtmg_c20240206.nc + scream/mam4xx/physprops/poly_rrtmg_c20240206.nc +) + +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_72lev}) +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS dynamics physics shoc cld p3 rrtmgp mam4_optics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/input.yaml new file mode 100644 index 000000000000..e6b942364886 --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/input.yaml @@ -0,0 +1,90 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + mass_column_conservation_error_tolerance: 1e-3 + energy_column_conservation_error_tolerance: 1e-4 + column_conservation_checks_fail_handling_type: Warning + property_check_data_fields: [phis] + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +initial_conditions: + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + surf_evap: 0.0 + surf_sens_flux: 0.0 + precip_liq_surf_mass: 0.0 + precip_ice_surf_mass: 0.0 + pbl_height : 1.0 + phis : 1.0 + +atmosphere_processes: + atm_procs_list: [homme,physics] + schedule_type: Sequential + homme: + Moisture: moist + physics: + atm_procs_list: [mac_mic,mam4_optics,rrtmgp] + schedule_type: Sequential + Type: Group + mac_mic: + atm_procs_list: [shoc,CldFraction,p3] + schedule_type: Sequential + Type: Group + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + p3: + do_prescribed_ccn: false + enable_column_conservation_checks: true + max_total_ni: 740.0e3 + shoc: + enable_column_conservation_checks: true + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 + + mam4_optics: + mam4_mode1_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode1_rrtmg_aeronetdust_c20240206.nc + mam4_mode2_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode2_rrtmg_c20240206.nc + mam4_mode3_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode3_rrtmg_aeronetdust_c20240206.nc + mam4_mode4_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode4_rrtmg_c20240206.nc + mam4_water_refindex_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/water_refindex_rrtmg_c20240206.nc + mam4_soa_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ocphi_rrtmg_c20240206.nc + mam4_dust_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/dust_aeronet_rrtmg_c20240206.nc + mam4_nacl_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ssam_rrtmg_c20240206.nc + mam4_so4_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/sulfate_rrtmg_c20240206.nc + mam4_pom_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ocpho_rrtmg_c20240206.nc + mam4_bc_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/bcpho_rrtmg_c20240206.nc + mam4_mom_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/poly_rrtmg_c20240206.nc + + rrtmgp: + column_chunk_size: 123 + active_gases: ["h2o", "co2", "o3", "n2o", "co" , "ch4", "o2", "n2"] + rrtmgp_coefficients_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-sw-g112-210809.nc + rrtmgp_coefficients_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-lw-g128-210809.nc + rrtmgp_cloud_optics_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-sw.nc + rrtmgp_cloud_optics_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-lw.nc + enable_column_conservation_checks: true + +grids_manager: + Type: Homme + physics_grid_type: GLL + dynamics_namelist_file_name: namelist.nl + vertical_coordinate_filename: IC_FILE + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/output.yaml new file mode 100644 index 000000000000..559f5158bb60 --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/output.yaml @@ -0,0 +1,121 @@ +%YAML 1.1 +--- +filename_prefix: homme_shoc_cld_p3_mam_optics_rrtmgp +Averaging Type: Instant +Max Snapshots Per File: 1 +Fields: + Physics GLL: + Field Names: + # HOMME + - ps + - pseudo_density + - omega + - p_int + - p_mid + - pseudo_density_dry + - p_dry_int + - p_dry_mid + # SHOC + - cldfrac_liq + - eddy_diff_mom + - sgs_buoy_flux + - tke + - inv_qc_relvar + - pbl_height + # CLD + - cldfrac_ice + - cldfrac_tot + # P3 + - bm + - nc + - ni + - nr + - qi + - qm + - qr + - T_prev_micro_step + - qv_prev_micro_step + - eff_radius_qc + - eff_radius_qi + - eff_radius_qr + - micro_liq_ice_exchange + - micro_vap_ice_exchange + - micro_vap_liq_exchange + - precip_ice_surf_mass + - precip_liq_surf_mass + - rainfrac + # SHOC + HOMME + - horiz_winds + # SHOC + P3 + - qc + - qv + # SHOC + P3 + RRTMGP + HOMME + - T_mid + #mam_optics + - aero_g_sw + - aero_ssa_sw + - aero_tau_lw + - aero_tau_sw + # RRTMGP + - sfc_alb_dif_nir + - sfc_alb_dif_vis + - sfc_alb_dir_nir + - sfc_alb_dir_vis + - LW_flux_dn + - LW_flux_up + - SW_flux_dn + - SW_flux_dn_dir + - SW_flux_up + - rad_heating_pdel + - sfc_flux_lw_dn + - sfc_flux_sw_net + # Diagnostics + - T_mid_at_lev_2 + - T_mid_at_model_top + - T_mid_at_model_bot + - T_mid_at_500mb + - T_mid_at_500hPa + - T_mid_at_50000Pa + - PotentialTemperature + - AtmosphereDensity + - Exner + - VirtualTemperature + - z_int + - geopotential_int_at_lev_2 + - z_mid_at_500mb + - geopotential_mid + - dz + - DryStaticEnergy + - SeaLevelPressure + - LiqWaterPath + - IceWaterPath + - VapWaterPath + - RainWaterPath + - RimeWaterPath + - ShortwaveCloudForcing + - LongwaveCloudForcing + - RelativeHumidity + - ZonalVapFlux + - MeridionalVapFlux + - PotentialTemperature_at_model_top + - PotentialTemperature_at_500mb + # GLL output for homme states. These + # represent all current possible homme + # states available. + Dynamics: + Field Names: + - v_dyn + - vtheta_dp_dyn + - dp3d_dyn + - phi_int_dyn + - ps_dyn + - phis_dyn + - omega_dyn + - Qdp_dyn + IO Grid Name: Physics GLL + +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps + MPI Ranks in Filename: true +... diff --git a/components/eamxx/tests/coupled/dynamics_physics/model_restart/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/CMakeLists.txt similarity index 100% rename from components/eamxx/tests/coupled/dynamics_physics/model_restart/CMakeLists.txt rename to components/eamxx/tests/multi-process/dynamics_physics/model_restart/CMakeLists.txt diff --git a/components/eamxx/tests/coupled/dynamics_physics/model_restart/input_baseline.yaml b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_baseline.yaml similarity index 100% rename from components/eamxx/tests/coupled/dynamics_physics/model_restart/input_baseline.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_baseline.yaml diff --git a/components/eamxx/tests/coupled/dynamics_physics/model_restart/input_initial.yaml b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_initial.yaml similarity index 100% rename from components/eamxx/tests/coupled/dynamics_physics/model_restart/input_initial.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_initial.yaml diff --git a/components/eamxx/tests/coupled/dynamics_physics/model_restart/input_restarted.yaml b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_restarted.yaml similarity index 100% rename from components/eamxx/tests/coupled/dynamics_physics/model_restart/input_restarted.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_restarted.yaml diff --git a/components/eamxx/tests/coupled/dynamics_physics/model_restart/model_output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/model_output.yaml similarity index 98% rename from components/eamxx/tests/coupled/dynamics_physics/model_restart/model_output.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/model_restart/model_output.yaml index 284b86f3a06f..671a10e57701 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/model_restart/model_output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/model_output.yaml @@ -80,7 +80,6 @@ Fields: - vtheta_dp_dyn - dp3d_dyn output_control: - MPI Ranks in Filename: true Frequency: 1 frequency_units: nmins ... diff --git a/components/eamxx/tests/coupled/dynamics_physics/model_restart/model_restart_output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/model_restart_output.yaml similarity index 96% rename from components/eamxx/tests/coupled/dynamics_physics/model_restart/model_restart_output.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/model_restart/model_restart_output.yaml index 13ca88309659..d4026069b167 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/model_restart/model_restart_output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/model_restart_output.yaml @@ -80,9 +80,7 @@ Fields: - vtheta_dp_dyn - dp3d_dyn output_control: - MPI Ranks in Filename: true Frequency: 1 frequency_units: nmins Checkpoint Control: - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/coupled/physics_only/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt similarity index 77% rename from components/eamxx/tests/coupled/physics_only/CMakeLists.txt rename to components/eamxx/tests/multi-process/physics_only/CMakeLists.txt index ed15610160ed..61b224f71ae6 100644 --- a/components/eamxx/tests/coupled/physics_only/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt @@ -2,6 +2,9 @@ if (SCREAM_DOUBLE_PRECISION) add_subdirectory(shoc_cld_p3_rrtmgp) add_subdirectory(shoc_cld_spa_p3_rrtmgp) + if (SCREAM_ENABLE_MAM) + add_subdirectory(mam/optics_rrtmgp) + endif() endif() add_subdirectory (atm_proc_subcycling) diff --git a/components/eamxx/tests/coupled/physics_only/atm_proc_subcycling/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/CMakeLists.txt similarity index 92% rename from components/eamxx/tests/coupled/physics_only/atm_proc_subcycling/CMakeLists.txt rename to components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/CMakeLists.txt index 8b16178ccd83..71fe2a5f43c2 100644 --- a/components/eamxx/tests/coupled/physics_only/atm_proc_subcycling/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/CMakeLists.txt @@ -45,8 +45,8 @@ CreateUnitTestFromExec (shoc_p3_monolithic shoc_p3 include (BuildCprnc) BuildCprnc() -set (SRC_FILE "shoc_p3_monolithic.INSTANT.nsteps_x3.${RUN_T0}.nc") -set (TGT_FILE "shoc_p3_subcycled.INSTANT.nsteps_x1.${RUN_T0}.nc") +set (SRC_FILE "shoc_p3_monolithic.INSTANT.nsteps_x3.np1.${RUN_T0}.nc") +set (TGT_FILE "shoc_p3_subcycled.INSTANT.nsteps_x1.np1.${RUN_T0}.nc") set (TEST_NAME check_subcycling) add_test (NAME ${TEST_NAME} COMMAND cmake -P ${CMAKE_BINARY_DIR}/bin/CprncTest.cmake ${SRC_FILE} ${TGT_FILE} @@ -57,7 +57,7 @@ set_tests_properties(${TEST_NAME} PROPERTIES # Check calculation of shoc tendencies when the parent group is subcycled set (script ${SCREAM_BASE_DIR}/scripts/check-tendencies) -set (fname shoc_p3_tend_subcycled.INSTANT.nsteps_x1.${RUN_T0}.nc) +set (fname shoc_p3_tend_subcycled.INSTANT.nsteps_x1.np1.${RUN_T0}.nc) add_test (NAME ${TEST_NAME}_tend_check COMMAND ${script} -f ${fname} -v tke -t shoc_tke_tend WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/components/eamxx/tests/coupled/physics_only/atm_proc_subcycling/input.yaml b/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/input.yaml similarity index 100% rename from components/eamxx/tests/coupled/physics_only/atm_proc_subcycling/input.yaml rename to components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/input.yaml diff --git a/components/eamxx/tests/coupled/physics_only/atm_proc_subcycling/output.yaml b/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/output.yaml similarity index 94% rename from components/eamxx/tests/coupled/physics_only/atm_proc_subcycling/output.yaml rename to components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/output.yaml index 2074321ad09a..2e9d18f64317 100644 --- a/components/eamxx/tests/coupled/physics_only/atm_proc_subcycling/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/output.yaml @@ -35,5 +35,4 @@ Field Names: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: false ... diff --git a/components/eamxx/tests/coupled/physics_only/atm_proc_subcycling/output_tend.yaml b/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/output_tend.yaml similarity index 86% rename from components/eamxx/tests/coupled/physics_only/atm_proc_subcycling/output_tend.yaml rename to components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/output_tend.yaml index 1b51444a0a3f..5103e182d60e 100644 --- a/components/eamxx/tests/coupled/physics_only/atm_proc_subcycling/output_tend.yaml +++ b/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/output_tend.yaml @@ -9,5 +9,4 @@ Field Names: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: false ... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/CMakeLists.txt new file mode 100644 index 000000000000..14e7a94e5c22 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/CMakeLists.txt @@ -0,0 +1,57 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME mam_optics_rrtmgp) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS shoc scream_rrtmgp mam + LABELS rrtmgp physics diagnostics mam4_optics + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +set (RUN_T0 2021-10-12-45000) + +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/mam4xx/physprops/mam4_mode1_rrtmg_aeronetdust_c20240206.nc + scream/mam4xx/physprops/mam4_mode2_rrtmg_c20240206.nc + scream/mam4xx/physprops/mam4_mode3_rrtmg_aeronetdust_c20240206.nc + scream/mam4xx/physprops/mam4_mode4_rrtmg_c20240206.nc + scream/mam4xx/physprops/water_refindex_rrtmg_c20240206.nc + scream/mam4xx/physprops/ocphi_rrtmg_c20240206.nc + scream/mam4xx/physprops/dust_aeronet_rrtmg_c20240206.nc + scream/mam4xx/physprops/ssam_rrtmg_c20240206.nc + scream/mam4xx/physprops/sulfate_rrtmg_c20240206.nc + scream/mam4xx/physprops/ocpho_rrtmg_c20240206.nc + scream/mam4xx/physprops/bcpho_rrtmg_c20240206.nc + scream/mam4xx/physprops/poly_rrtmg_c20240206.nc +) + +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS rrtmgp mam4_optics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/input.yaml new file mode 100644 index 000000000000..347054e5d9fd --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/input.yaml @@ -0,0 +1,55 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_optics, rrtmgp] + schedule_type: Sequential + mam4_optics: + mam4_mode1_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode1_rrtmg_aeronetdust_c20240206.nc + mam4_mode2_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode2_rrtmg_c20240206.nc + mam4_mode3_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode3_rrtmg_aeronetdust_c20240206.nc + mam4_mode4_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode4_rrtmg_c20240206.nc + mam4_water_refindex_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/water_refindex_rrtmg_c20240206.nc + mam4_soa_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ocphi_rrtmg_c20240206.nc + mam4_dust_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/dust_aeronet_rrtmg_c20240206.nc + mam4_nacl_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ssam_rrtmg_c20240206.nc + mam4_so4_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/sulfate_rrtmg_c20240206.nc + mam4_pom_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ocpho_rrtmg_c20240206.nc + mam4_bc_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/bcpho_rrtmg_c20240206.nc + mam4_mom_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/poly_rrtmg_c20240206.nc + rrtmgp: + column_chunk_size: 123 + active_gases: ["h2o", "co2", "o3", "n2o", "co" , "ch4", "o2", "n2"] + orbital_year: 1990 + rrtmgp_coefficients_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-sw-g112-210809.nc + rrtmgp_coefficients_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-lw-g128-210809.nc + rrtmgp_cloud_optics_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-sw.nc + rrtmgp_cloud_optics_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-lw.nc + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + pbl_height : 1.0 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/output.yaml new file mode 100644 index 000000000000..24e1dfbb3e7a --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/output.yaml @@ -0,0 +1,30 @@ +%YAML 1.1 +--- +filename_prefix: mam_optics_rrtmgp_output +Averaging Type: Instant +Max Snapshots Per File: 1 +Field Names: + # MAM_OPTICS + - aero_g_sw + - aero_ssa_sw + - aero_tau_lw + - aero_tau_sw + # RRTMGP + - sfc_alb_dif_nir + - sfc_alb_dif_vis + - sfc_alb_dir_nir + - sfc_alb_dir_vis + - LW_flux_dn + - LW_flux_up + - SW_flux_dn + - SW_flux_dn_dir + - SW_flux_up + - rad_heating_pdel + - sfc_flux_lw_dn + - sfc_flux_sw_net + +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps + MPI Ranks in Filename: true +... diff --git a/components/eamxx/tests/coupled/physics_only/shoc_cld_p3_rrtmgp/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/shoc_cld_p3_rrtmgp/CMakeLists.txt similarity index 100% rename from components/eamxx/tests/coupled/physics_only/shoc_cld_p3_rrtmgp/CMakeLists.txt rename to components/eamxx/tests/multi-process/physics_only/shoc_cld_p3_rrtmgp/CMakeLists.txt diff --git a/components/eamxx/tests/coupled/physics_only/shoc_cld_p3_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_cld_p3_rrtmgp/input.yaml similarity index 100% rename from components/eamxx/tests/coupled/physics_only/shoc_cld_p3_rrtmgp/input.yaml rename to components/eamxx/tests/multi-process/physics_only/shoc_cld_p3_rrtmgp/input.yaml diff --git a/components/eamxx/tests/coupled/physics_only/shoc_cld_p3_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_cld_p3_rrtmgp/output.yaml similarity index 96% rename from components/eamxx/tests/coupled/physics_only/shoc_cld_p3_rrtmgp/output.yaml rename to components/eamxx/tests/multi-process/physics_only/shoc_cld_p3_rrtmgp/output.yaml index 868970c7d5c9..a8b22c16ba09 100644 --- a/components/eamxx/tests/coupled/physics_only/shoc_cld_p3_rrtmgp/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_cld_p3_rrtmgp/output.yaml @@ -54,5 +54,4 @@ Field Names: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/coupled/physics_only/shoc_cld_spa_p3_rrtmgp/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/CMakeLists.txt similarity index 92% rename from components/eamxx/tests/coupled/physics_only/shoc_cld_spa_p3_rrtmgp/CMakeLists.txt rename to components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/CMakeLists.txt index 2cf18454eef3..799010ad10be 100644 --- a/components/eamxx/tests/coupled/physics_only/shoc_cld_spa_p3_rrtmgp/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/CMakeLists.txt @@ -23,9 +23,7 @@ math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_M # Ensure test input files are present in the data dir set (TEST_INPUT_FILES scream/init/${EAMxx_tests_IC_FILE_72lev} - scream/init/spa_init_ne2np4.nc - scream/maps/map_ne4np4_to_ne2np4_mono.nc - scream/init/spa_file_unified_and_complete_ne4_20220428.nc + scream/init/spa_file_unified_and_complete_ne2np4L72_20231222.nc cam/topo/${EAMxx_tests_TOPO_FILE} ) foreach (file IN ITEMS ${TEST_INPUT_FILES}) diff --git a/components/eamxx/tests/coupled/physics_only/shoc_cld_spa_p3_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/input.yaml similarity index 94% rename from components/eamxx/tests/coupled/physics_only/shoc_cld_spa_p3_rrtmgp/input.yaml rename to components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/input.yaml index aae97f2190e7..542625798f53 100644 --- a/components/eamxx/tests/coupled/physics_only/shoc_cld_spa_p3_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/input.yaml @@ -17,8 +17,7 @@ atmosphere_processes: schedule_type: Sequential number_of_subcycles: ${MAC_MIC_SUBCYCLES} spa: - spa_remap_file: ${SCREAM_DATA_DIR}/maps/map_ne4np4_to_ne2np4_mono.nc - spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne4_20220428.nc + spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne2np4L72_20231222.nc p3: max_total_ni: 740.0e3 shoc: diff --git a/components/eamxx/tests/coupled/physics_only/shoc_cld_spa_p3_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/output.yaml similarity index 97% rename from components/eamxx/tests/coupled/physics_only/shoc_cld_spa_p3_rrtmgp/output.yaml rename to components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/output.yaml index be396798eb49..1d2d91de183c 100644 --- a/components/eamxx/tests/coupled/physics_only/shoc_cld_spa_p3_rrtmgp/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/output.yaml @@ -63,5 +63,4 @@ Field Names: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt similarity index 77% rename from components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/CMakeLists.txt rename to components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt index 74283073c62c..973e58b7da53 100644 --- a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt @@ -19,6 +19,7 @@ CreateUnitTest(create_vert_remap_and_weights "create_vert_remap_and_weights.cpp" # Run a test to setup nudging source data: set (NUM_STEPS 5) set (POSTFIX source_data) +set (ADD_RANKS false) set (ATM_TIME_STEP 300) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_source_data.yaml ${CMAKE_CURRENT_BINARY_DIR}/input_source_data.yaml) @@ -35,6 +36,7 @@ CreateUnitTestFromExec (shoc_p3_source shoc_p3_nudging set (NUM_STEPS 5) set (ATM_TIME_STEP 300) set (POSTFIX nudged) +set (ADD_RANKS true) set (VERT_TYPE TIME_DEPENDENT_3D_PROFILE) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_nudging.yaml ${CMAKE_CURRENT_BINARY_DIR}/input_nudging.yaml) @@ -57,3 +59,16 @@ CreateUnitTestFromExec (shoc_p3_nudged_remapped shoc_p3_nudging EXE_ARGS "--use-colour no --ekat-test-params ifile=input_nudging_remapped.yaml" FIXTURES_REQUIRED shoc_p3_source_data) +# Run a test with nudging using data read in glob pattern and skip vertical interpolation: +set (NUM_STEPS 5) +set (ATM_TIME_STEP 300) +set (POSTFIX nudged_glob_novert) +set (VERT_TYPE TIME_DEPENDENT_3D_PROFILE) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_nudging_glob_novert.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input_nudging_glob_novert.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output_nudged_glob_novert.yaml) +CreateUnitTestFromExec (shoc_p3_nudging_glob_novert shoc_p3_nudging + EXE_ARGS "--use-colour no --ekat-test-params ifile=input_nudging_glob_novert.yaml" + FIXTURES_REQUIRED shoc_p3_source_data) + diff --git a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/create_vert_remap_and_weights.cpp b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/create_vert_remap_and_weights.cpp similarity index 98% rename from components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/create_vert_remap_and_weights.cpp rename to components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/create_vert_remap_and_weights.cpp index b9bdfc9fdb07..702d83f16d37 100644 --- a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/create_vert_remap_and_weights.cpp +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/create_vert_remap_and_weights.cpp @@ -80,5 +80,6 @@ TEST_CASE("create_vert_remap_and_weights","create_vert_remap_and_weights") { create_vert_remap(); create_nudging_weights_ncfile(1, 218, 72, "nudging_weights.nc"); + create_nudging_weights_ncfile(1, 218, 128, "nudging_weights_L128.nc"); } } // end namespace diff --git a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/input_nudging.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml similarity index 93% rename from components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/input_nudging.yaml rename to components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml index 2c9b58b678cc..2b33d15f9f75 100644 --- a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/input_nudging.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml @@ -14,7 +14,7 @@ atmosphere_processes: p3: max_total_ni: 720.0e3 nudging: - nudging_filename: [shoc_p3_source_data_${POSTFIX}.INSTANT.nsteps_x${NUM_STEPS}.${RUN_T0}.nc] + nudging_filenames_patterns: [shoc_p3_source_data_${POSTFIX}.INSTANT.nsteps_x${NUM_STEPS}.${RUN_T0}.nc] nudging_fields: ["T_mid", "qv"] nudging_timescale: 1000 use_nudging_weights: true diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging_glob_novert.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging_glob_novert.yaml new file mode 100644 index 000000000000..35d75015e2a1 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging_glob_novert.yaml @@ -0,0 +1,61 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + schedule_type: Sequential + atm_procs_list: [shoc,p3,nudging] + p3: + max_total_ni: 720.0e3 + nudging: + nudging_filenames_patterns: [./shoc_p3_source_data_nudged.INSTANT.nsteps_x*.nc] + nudging_fields: ["T_mid", "qv"] + nudging_timescale: 1000 + use_nudging_weights: true + nudging_weights_file: nudging_weights_L128.nc + skip_vert_interpolation: true + source_pressure_type: ${VERT_TYPE} + source_pressure_file: vertical_remap.nc ## Only used in the case of STATIC_1D_VERTICAL_PROFILE + shoc: + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 128 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_128lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + surf_evap: 0.0 + surf_sens_flux: 0.0 + precip_ice_surf_mass: 0.0 + precip_liq_surf_mass: 0.0 + +# The parameters for I/O control +Scorpio: + output_yaml_files: [output_${POSTFIX}.yaml] +... diff --git a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/input_source_data.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_source_data.yaml similarity index 100% rename from components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/input_source_data.yaml rename to components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_source_data.yaml diff --git a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/output.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output.yaml similarity index 94% rename from components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/output.yaml rename to components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output.yaml index d4779131bcbb..e740c5cdf1f1 100644 --- a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output.yaml @@ -3,6 +3,7 @@ filename_prefix: shoc_p3_${POSTFIX}_nudged Averaging Type: Instant Max Snapshots Per File: 2 +MPI Ranks in Filename: ${ADD_RANKS} Field Names: - p_mid - T_mid @@ -35,5 +36,4 @@ Field Names: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: false ... diff --git a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/output_remapped.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output_remapped.yaml similarity index 87% rename from components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/output_remapped.yaml rename to components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output_remapped.yaml index 4dde5dae2d5a..c75a6a862f71 100644 --- a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/output_remapped.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output_remapped.yaml @@ -4,6 +4,7 @@ filename_prefix: shoc_p3_${POSTFIX}_nudged_remapped Averaging Type: Instant Max Snapshots Per File: 2 vertical_remap_file: vertical_remap.nc +MPI Ranks in Filename: ${ADD_RANKS} Field Names: - T_mid - qv @@ -11,5 +12,4 @@ Field Names: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: false ... diff --git a/components/eamxx/tests/postrun/check_hashes_ers.py b/components/eamxx/tests/postrun/check_hashes_ers.py deleted file mode 100755 index be1da31ffff0..000000000000 --- a/components/eamxx/tests/postrun/check_hashes_ers.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python3 - -import os, sys, re, glob, subprocess - -def readall(fn): - with open(fn,'r') as f: - txt = f.read() - return txt - -def greptxt(pattern, txt): - return re.findall('(?:' + pattern + ').*', txt, flags=re.MULTILINE) - -def grep(pattern, fn): - txt = readall(fn) - return greptxt(pattern, txt) - -def get_log_glob_from_atm_modelio(case_dir): - filename = case_dir + os.path.sep + 'CaseDocs' + os.path.sep + 'atm_modelio.nml' - ln = grep('diro = ', filename)[0] - run_dir = ln.split()[2].split('"')[1] - ln = grep('logfile = ', filename)[0] - atm_log_fn = ln.split()[2].split('"')[1] - id = atm_log_fn.split('.')[2] - return run_dir + os.path.sep + 'e3sm.log.' + id + '*' - -def get_hash_lines(fn): - rlns = subprocess.run(['zgrep', 'exxhash', fn], capture_output=True) - rlns = rlns.stdout.decode().split('\n') - lns = [] - if len(rlns) == 0: return lns - for rln in rlns: - pos = rln.find('exxhash') - lns.append(rln[pos:]) - return lns - -def parse_time(hash_ln): - return hash_ln.split()[1:3] - -def all_equal(t1, t2): - if len(t1) != len(t2): return False - for i in range(len(t1)): - if t1[i] != t2[i]: return False - return True - -def find_first_index_at_time(lns, time): - for i, ln in enumerate(lns): - t = parse_time(ln) - if all_equal(time, t): return i - return None - -def diff(l1, l2): - diffs = [] - for i in range(len(l1)): - if l1[i] != l2[i]: - diffs.append((l1[i], l2[i])) - return diffs - -def main(case_dir): - # Look for the two e3sm.log files. - glob_pat = get_log_glob_from_atm_modelio(case_dir) - e3sm_fns = glob.glob(glob_pat) - if len(e3sm_fns) == 0: - print('Could not find e3sm.log files with glob string {}'.format(glob_pat)) - return False - e3sm_fns.sort() - if len(e3sm_fns) == 1: - # This is the first run. Exit and wait for the second - # run. (POSTRUN_SCRIPT is called after each of the two runs.) - print('Exiting on first run.') - return True - print('Diffing base {} and restart {}'.format(e3sm_fns[0], e3sm_fns[1])) - - # Because of the prefixed 1: and 2: on some systems, we can't just use - # zdiff. - lns = [] - for f in e3sm_fns: - lns.append(get_hash_lines(f)) - time = parse_time(lns[1][0]) - time_idx = find_first_index_at_time(lns[0], time) - if time_idx is None: - print('Could not find a start time.') - return False - lns[0] = lns[0][time_idx:] - if len(lns[0]) != len(lns[1]): - print('Number of hash lines starting at restart time do not agree.') - return False - diffs = diff(lns[0], lns[1]) - - # Flushed prints to e3sm.log can sometimes conflict with other - # output. Permit up to 'thr' diffs so we don't fail due to badly printed - # lines. This isn't a big loss in checking because an ERS_Ln22 second run - # writes > 1000 hash lines, and a true loss of BFBness is nearly certain to - # propagate to a large number of subsequent hashes. - thr = 5 - if len(lns[0]) < 100: thr = 0 - - ok = True - if len(diffs) > thr: - print('DIFF') - print(diffs[-10:]) - ok = False - else: - print('OK') - - return ok - -case_dir = sys.argv[1] -sys.exit(0 if main(case_dir) else 1) diff --git a/components/eamxx/tests/single-process/CMakeLists.txt b/components/eamxx/tests/single-process/CMakeLists.txt new file mode 100644 index 000000000000..d417a9920f46 --- /dev/null +++ b/components/eamxx/tests/single-process/CMakeLists.txt @@ -0,0 +1,26 @@ +if ("${SCREAM_DYNAMICS_DYCORE}" STREQUAL "HOMME") + add_subdirectory(homme) +endif() +add_subdirectory(p3) +add_subdirectory(shoc) +add_subdirectory(cld_fraction) +add_subdirectory(spa) +add_subdirectory(surface_coupling) +if (SCREAM_ENABLE_ML_CORRECTION ) + add_subdirectory(ml_correction) +endif() +if (SCREAM_DOUBLE_PRECISION) + add_subdirectory(rrtmgp) + add_subdirectory(cosp) +else() + message(STATUS "RRTMGP and COSP only supported for double precision builds; skipping") +endif() +if (SCREAM_ENABLE_MAM) + # Once the mam4xx aerosol microphysics AtmosphereProcess is running, the + # corresponding test here needs to be reworked with valid aerosol + # initial conditions. + add_subdirectory(mam/optics) +endif() +if (SCREAM_TEST_LEVEL GREATER_EQUAL SCREAM_TEST_LEVEL_EXPERIMENTAL) + add_subdirectory(zm) +endif() diff --git a/components/eamxx/tests/uncoupled/cld_fraction/CMakeLists.txt b/components/eamxx/tests/single-process/cld_fraction/CMakeLists.txt similarity index 100% rename from components/eamxx/tests/uncoupled/cld_fraction/CMakeLists.txt rename to components/eamxx/tests/single-process/cld_fraction/CMakeLists.txt diff --git a/components/eamxx/tests/uncoupled/cld_fraction/input.yaml b/components/eamxx/tests/single-process/cld_fraction/input.yaml similarity index 100% rename from components/eamxx/tests/uncoupled/cld_fraction/input.yaml rename to components/eamxx/tests/single-process/cld_fraction/input.yaml diff --git a/components/eamxx/tests/uncoupled/cosp/CMakeLists.txt b/components/eamxx/tests/single-process/cosp/CMakeLists.txt similarity index 50% rename from components/eamxx/tests/uncoupled/cosp/CMakeLists.txt rename to components/eamxx/tests/single-process/cosp/CMakeLists.txt index 769c6593aca8..6de9f09b2cd7 100644 --- a/components/eamxx/tests/uncoupled/cosp/CMakeLists.txt +++ b/components/eamxx/tests/single-process/cosp/CMakeLists.txt @@ -1,10 +1,14 @@ include (ScreamUtils) +set (TEST_BASE_NAME cosp_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + # Create the test -CreateADUnitTest(cosp_standalone +CreateADUnitTest(${TEST_BASE_NAME} LABELS cosp physics LIBS eamxx_cosp MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} ) # Set AD configurable options @@ -20,3 +24,11 @@ configure_file (${CMAKE_CURRENT_SOURCE_DIR}/input.yaml ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) configure_file (${CMAKE_CURRENT_SOURCE_DIR}/output.yaml ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: for other tests we do np1-vs-npX bfb tests, which is why one is enough. + # COSP, is not bfb w.r.t. num ranks though. Still, we only check 1 output files against baselines + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/uncoupled/cosp/input.yaml b/components/eamxx/tests/single-process/cosp/input.yaml similarity index 96% rename from components/eamxx/tests/uncoupled/cosp/input.yaml rename to components/eamxx/tests/single-process/cosp/input.yaml index 3336b0694db5..1447bcc654f1 100644 --- a/components/eamxx/tests/uncoupled/cosp/input.yaml +++ b/components/eamxx/tests/single-process/cosp/input.yaml @@ -28,7 +28,7 @@ initial_conditions: topography_filename: ${TOPO_DATA_DIR}/USGS-gtopo30_ne4np4pg2_16x_converted.c20200527.nc dtau067: 1.0 dtau105: 1.0 - cldfrac_tot_for_analysis: 0.5 + cldfrac_rad: 0.5 eff_radius_qc: 0.0 eff_radius_qi: 0.0 sunlit: 1.0 diff --git a/components/eamxx/tests/uncoupled/cosp/output.yaml b/components/eamxx/tests/single-process/cosp/output.yaml similarity index 86% rename from components/eamxx/tests/uncoupled/cosp/output.yaml rename to components/eamxx/tests/single-process/cosp/output.yaml index 59cc18aeddf9..7e942489c728 100644 --- a/components/eamxx/tests/uncoupled/cosp/output.yaml +++ b/components/eamxx/tests/single-process/cosp/output.yaml @@ -10,5 +10,4 @@ Fields: output_control: Frequency: 1 frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/uncoupled/homme/CMakeLists.txt b/components/eamxx/tests/single-process/homme/CMakeLists.txt similarity index 86% rename from components/eamxx/tests/uncoupled/homme/CMakeLists.txt rename to components/eamxx/tests/single-process/homme/CMakeLists.txt index 13dcdaea21b5..f1c28036782b 100644 --- a/components/eamxx/tests/uncoupled/homme/CMakeLists.txt +++ b/components/eamxx/tests/single-process/homme/CMakeLists.txt @@ -77,3 +77,10 @@ CompareNCFilesFamilyMpi ( LABELS dynamics META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/uncoupled/homme/input.yaml b/components/eamxx/tests/single-process/homme/input.yaml similarity index 100% rename from components/eamxx/tests/uncoupled/homme/input.yaml rename to components/eamxx/tests/single-process/homme/input.yaml diff --git a/components/eamxx/tests/uncoupled/homme/output.yaml b/components/eamxx/tests/single-process/homme/output.yaml similarity index 92% rename from components/eamxx/tests/uncoupled/homme/output.yaml rename to components/eamxx/tests/single-process/homme/output.yaml index d34812a6a834..4b88510d770e 100644 --- a/components/eamxx/tests/uncoupled/homme/output.yaml +++ b/components/eamxx/tests/single-process/homme/output.yaml @@ -20,5 +20,4 @@ Fields: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt b/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt new file mode 100644 index 000000000000..a3e6f64ec7d0 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt @@ -0,0 +1,66 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME mam4_optics_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LABELS mam4_optics physics + LIBS mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +SetVarDependingOnTestSize(NUM_STEPS 12 24 36) +set (ATM_TIME_STEP 1800) +set (RUN_T0 2021-10-12-45000) + + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/mam4xx/physprops/mam4_mode1_rrtmg_aeronetdust_c20240206.nc + scream/mam4xx/physprops/mam4_mode2_rrtmg_c20240206.nc + scream/mam4xx/physprops/mam4_mode3_rrtmg_aeronetdust_c20240206.nc + scream/mam4xx/physprops/mam4_mode4_rrtmg_c20240206.nc + scream/mam4xx/physprops/water_refindex_rrtmg_c20240206.nc + scream/mam4xx/physprops/ocphi_rrtmg_c20240206.nc + scream/mam4xx/physprops/dust_aeronet_rrtmg_c20240206.nc + scream/mam4xx/physprops/ssam_rrtmg_c20240206.nc + scream/mam4xx/physprops/sulfate_rrtmg_c20240206.nc + scream/mam4xx/physprops/ocpho_rrtmg_c20240206.nc + scream/mam4xx/physprops/bcpho_rrtmg_c20240206.nc + scream/mam4xx/physprops/poly_rrtmg_c20240206.nc + scream/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc +) + +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS mam4_optics physics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() \ No newline at end of file diff --git a/components/eamxx/tests/single-process/mam/optics/input.yaml b/components/eamxx/tests/single-process/mam/optics/input.yaml new file mode 100644 index 000000000000..1aa6f3f7134f --- /dev/null +++ b/components/eamxx/tests/single-process/mam/optics/input.yaml @@ -0,0 +1,47 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_optics] + mam4_optics: + mam4_mode1_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode1_rrtmg_aeronetdust_c20240206.nc + mam4_mode2_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode2_rrtmg_c20240206.nc + mam4_mode3_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode3_rrtmg_aeronetdust_c20240206.nc + mam4_mode4_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode4_rrtmg_c20240206.nc + mam4_water_refindex_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/water_refindex_rrtmg_c20240206.nc + mam4_soa_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ocphi_rrtmg_c20240206.nc + mam4_dust_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/dust_aeronet_rrtmg_c20240206.nc + mam4_nacl_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ssam_rrtmg_c20240206.nc + mam4_so4_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/sulfate_rrtmg_c20240206.nc + mam4_pom_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ocpho_rrtmg_c20240206.nc + mam4_bc_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/bcpho_rrtmg_c20240206.nc + mam4_mom_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/poly_rrtmg_c20240206.nc + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + type: point_grid + aliases: [Physics] + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + pbl_height : 1.0 + phis : 1.0 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/single-process/mam/optics/output.yaml b/components/eamxx/tests/single-process/mam/optics/output.yaml new file mode 100644 index 000000000000..8ea3d237933d --- /dev/null +++ b/components/eamxx/tests/single-process/mam/optics/output.yaml @@ -0,0 +1,17 @@ +%YAML 1.1 +--- +filename_prefix: mam4_optics_standalone_output +Averaging Type: Instant +Fields: + Physics: + Field Names: + - aero_g_sw + - aero_ssa_sw + - aero_tau_sw + - aero_tau_lw + +output_control: + Frequency: 2 + frequency_units: nsteps + MPI Ranks in Filename: true +... diff --git a/components/eamxx/tests/uncoupled/ml_correction/CMakeLists.txt b/components/eamxx/tests/single-process/ml_correction/CMakeLists.txt similarity index 100% rename from components/eamxx/tests/uncoupled/ml_correction/CMakeLists.txt rename to components/eamxx/tests/single-process/ml_correction/CMakeLists.txt diff --git a/components/eamxx/tests/uncoupled/ml_correction/input.yaml b/components/eamxx/tests/single-process/ml_correction/input.yaml similarity index 100% rename from components/eamxx/tests/uncoupled/ml_correction/input.yaml rename to components/eamxx/tests/single-process/ml_correction/input.yaml diff --git a/components/eamxx/tests/uncoupled/ml_correction/ml_correction_standalone.cpp b/components/eamxx/tests/single-process/ml_correction/ml_correction_standalone.cpp similarity index 100% rename from components/eamxx/tests/uncoupled/ml_correction/ml_correction_standalone.cpp rename to components/eamxx/tests/single-process/ml_correction/ml_correction_standalone.cpp diff --git a/components/eamxx/tests/uncoupled/ml_correction/test_correction.py b/components/eamxx/tests/single-process/ml_correction/test_correction.py similarity index 100% rename from components/eamxx/tests/uncoupled/ml_correction/test_correction.py rename to components/eamxx/tests/single-process/ml_correction/test_correction.py diff --git a/components/eamxx/tests/uncoupled/p3/CMakeLists.txt b/components/eamxx/tests/single-process/p3/CMakeLists.txt similarity index 83% rename from components/eamxx/tests/uncoupled/p3/CMakeLists.txt rename to components/eamxx/tests/single-process/p3/CMakeLists.txt index a155ebc4dc74..969ed3a9da71 100644 --- a/components/eamxx/tests/uncoupled/p3/CMakeLists.txt +++ b/components/eamxx/tests/single-process/p3/CMakeLists.txt @@ -48,3 +48,10 @@ foreach (NRANKS RANGE ${TEST_RANK_START} ${TEST_RANK_END}) LABELS "p3;physics" FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_np${NRANKS}_omp1) endforeach() + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/uncoupled/p3/input.yaml b/components/eamxx/tests/single-process/p3/input.yaml similarity index 100% rename from components/eamxx/tests/uncoupled/p3/input.yaml rename to components/eamxx/tests/single-process/p3/input.yaml diff --git a/components/eamxx/tests/uncoupled/p3/output.yaml b/components/eamxx/tests/single-process/p3/output.yaml similarity index 91% rename from components/eamxx/tests/uncoupled/p3/output.yaml rename to components/eamxx/tests/single-process/p3/output.yaml index a83da3a3848c..e52e02c4a19b 100644 --- a/components/eamxx/tests/uncoupled/p3/output.yaml +++ b/components/eamxx/tests/single-process/p3/output.yaml @@ -4,7 +4,7 @@ filename_prefix: p3_standalone_output Averaging Type: Instant Field Names: - T_mid - - T_mid_at_2m + - T_mid_at_2m_above_sealevel - T_prev_micro_step - qv - qc @@ -30,5 +30,4 @@ Field Names: output_control: Frequency: 1 frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/single-process/rrtmgp/CMakeLists.txt b/components/eamxx/tests/single-process/rrtmgp/CMakeLists.txt new file mode 100644 index 000000000000..792d3946a4c2 --- /dev/null +++ b/components/eamxx/tests/single-process/rrtmgp/CMakeLists.txt @@ -0,0 +1,92 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME rrtmgp_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_72lev}) + +if (SCREAM_ENABLE_BASELINE_TESTS AND NOT SCREAM_ONLY_GENERATE_BASELINES) + # Unit test to compare against raw rrtmgp output + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_unit.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input_unit.yaml) + CreateUnitTest(${TEST_BASE_NAME}_unit rrtmgp_standalone_unit.cpp + LABELS rrtmgp physics driver + LIBS scream_rrtmgp rrtmgp scream_control yakl diagnostics rrtmgp_test_utils + EXE_ARGS "--ekat-test-params rrtmgp_inputfile=${SCREAM_DATA_DIR}/init/rrtmgp-allsky.nc,rrtmgp_baseline=${SCREAM_BASELINES_DIR}/data/rrtmgp-allsky-baseline.nc" + ) +endif() + +## Create rrtmgp stand alone executable +CreateUnitTestExec(${TEST_BASE_NAME} "rrtmgp_standalone.cpp" + LIBS scream_rrtmgp rrtmgp scream_control yakl diagnostics +) + +# The RRTMGP stand-alone test that runs multi-step +# Set AD configurable options +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) +set (ATM_TIME_STEP 1800) +set (RUN_T0 2021-10-12-45000) + +# Test non-chunked version (sweep multiple ranks) +set (SUFFIX "_not_chunked") +set (COL_CHUNK_SIZE 1000) +configure_file (${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output_not_chunked.yaml) +configure_file (${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input_not_chunked.yaml) +CreateUnitTestFromExec( + ${TEST_BASE_NAME}_not_chunked ${TEST_BASE_NAME} + LABELS rrtmgp physics driver + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + EXE_ARGS "--ekat-test-params inputfile=input_not_chunked.yaml" + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME}_not_chunked +) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME}_not_chunked + FILE_META_NAME ${TEST_BASE_NAME}_output_not_chunked.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS rrtmgp physics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_not_chunked_npMPIRANKS_omp1 +) + +## Test chunked version (only for ${TEST_RANK_END}) and compare against non-chunked +set (SUFFIX "_chunked") +math (EXPR COL_PER_RANK "218 / ${TEST_RANK_END}") +math (EXPR COL_CHUNK_SIZE "${COL_PER_RANK} / 2") +if (COL_CHUNK_SIZE LESS 1) + message (FATAL_ERROR "Error! Chunk size for rrtmgp unit test is less than 1.") +endif() + +configure_file (${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input_chunked.yaml) +configure_file (${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output_chunked.yaml) +CreateUnitTestFromExec( + ${TEST_BASE_NAME}_chunked ${TEST_BASE_NAME} + LABELS rrtmgp physics driver + MPI_RANKS ${TEST_RANK_END} + EXE_ARGS "--ekat-test-params inputfile=input_chunked.yaml" + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME}_chunked + PROPERTIES PASS_REGULAR_EXPRESSION "(beg.end: 0, ${COL_CHUNK_SIZE})" +) + +CompareNCFiles( + TEST_NAME ${TEST_BASE_NAME}_chunked_vs_not_chunked + SRC_FILE ${TEST_BASE_NAME}_output_chunked.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc + TGT_FILE ${TEST_BASE_NAME}_output_not_chunked.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc + LABELS rrtmgp physics + FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_chunked_np${TEST_RANK_END}_omp1 + ${FIXTURES_BASE_NAME}_not_chunked_np${TEST_RANK_END}_omp1) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX, + # and that chunked is bfb with not_chunked + set (OUT_FILE ${TEST_BASE_NAME}_output_chunked.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME}_chunked ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}_chunked) +endif() diff --git a/components/eamxx/tests/uncoupled/rrtmgp/input.yaml b/components/eamxx/tests/single-process/rrtmgp/input.yaml similarity index 100% rename from components/eamxx/tests/uncoupled/rrtmgp/input.yaml rename to components/eamxx/tests/single-process/rrtmgp/input.yaml diff --git a/components/eamxx/tests/uncoupled/rrtmgp/input_unit.yaml b/components/eamxx/tests/single-process/rrtmgp/input_unit.yaml similarity index 100% rename from components/eamxx/tests/uncoupled/rrtmgp/input_unit.yaml rename to components/eamxx/tests/single-process/rrtmgp/input_unit.yaml diff --git a/components/eamxx/tests/uncoupled/rrtmgp/output.yaml b/components/eamxx/tests/single-process/rrtmgp/output.yaml similarity index 95% rename from components/eamxx/tests/uncoupled/rrtmgp/output.yaml rename to components/eamxx/tests/single-process/rrtmgp/output.yaml index 0baa5da841d2..b067d1412bc6 100644 --- a/components/eamxx/tests/uncoupled/rrtmgp/output.yaml +++ b/components/eamxx/tests/single-process/rrtmgp/output.yaml @@ -30,5 +30,4 @@ Field Names: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/uncoupled/rrtmgp/rrtmgp_standalone.cpp b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone.cpp similarity index 100% rename from components/eamxx/tests/uncoupled/rrtmgp/rrtmgp_standalone.cpp rename to components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone.cpp diff --git a/components/eamxx/tests/uncoupled/rrtmgp/rrtmgp_standalone_unit.cpp b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp similarity index 100% rename from components/eamxx/tests/uncoupled/rrtmgp/rrtmgp_standalone_unit.cpp rename to components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp diff --git a/components/eamxx/tests/uncoupled/shoc/CMakeLists.txt b/components/eamxx/tests/single-process/shoc/CMakeLists.txt similarity index 93% rename from components/eamxx/tests/uncoupled/shoc/CMakeLists.txt rename to components/eamxx/tests/single-process/shoc/CMakeLists.txt index bebc54448ca0..bae7f719856d 100644 --- a/components/eamxx/tests/uncoupled/shoc/CMakeLists.txt +++ b/components/eamxx/tests/single-process/shoc/CMakeLists.txt @@ -76,6 +76,13 @@ foreach (RANK RANGE ${TEST_RANK_START} ${TEST_RANK_END}) FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_np${RANK}_omp1) endforeach() +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() + ################################ # MUST FAIL tests # ################################ diff --git a/components/eamxx/tests/uncoupled/shoc/input.yaml b/components/eamxx/tests/single-process/shoc/input.yaml similarity index 100% rename from components/eamxx/tests/uncoupled/shoc/input.yaml rename to components/eamxx/tests/single-process/shoc/input.yaml diff --git a/components/eamxx/tests/uncoupled/shoc/output.yaml b/components/eamxx/tests/single-process/shoc/output.yaml similarity index 95% rename from components/eamxx/tests/uncoupled/shoc/output.yaml rename to components/eamxx/tests/single-process/shoc/output.yaml index 82ccde097b1d..1ad0d7d990c5 100644 --- a/components/eamxx/tests/uncoupled/shoc/output.yaml +++ b/components/eamxx/tests/single-process/shoc/output.yaml @@ -32,5 +32,4 @@ Fields: output_control: Frequency: 1 frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/uncoupled/spa/CMakeLists.txt b/components/eamxx/tests/single-process/spa/CMakeLists.txt similarity index 79% rename from components/eamxx/tests/uncoupled/spa/CMakeLists.txt rename to components/eamxx/tests/single-process/spa/CMakeLists.txt index 559f48af9e53..8501f48224cb 100644 --- a/components/eamxx/tests/uncoupled/spa/CMakeLists.txt +++ b/components/eamxx/tests/single-process/spa/CMakeLists.txt @@ -42,3 +42,10 @@ CompareNCFilesFamilyMpi ( LABELS spa physics META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/uncoupled/spa/input.yaml b/components/eamxx/tests/single-process/spa/input.yaml similarity index 88% rename from components/eamxx/tests/uncoupled/spa/input.yaml rename to components/eamxx/tests/single-process/spa/input.yaml index ffd8b4f50b8a..6682710df0c7 100644 --- a/components/eamxx/tests/uncoupled/spa/input.yaml +++ b/components/eamxx/tests/single-process/spa/input.yaml @@ -11,8 +11,7 @@ time_stepping: atmosphere_processes: atm_procs_list: [spa] spa: - spa_remap_file: ${SCREAM_DATA_DIR}/maps/map_ne4np4_to_ne2np4_mono.nc - spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne4_20220428.nc + spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne2np4L72_20231222.nc grids_manager: Type: Mesh Free diff --git a/components/eamxx/tests/uncoupled/spa/output.yaml b/components/eamxx/tests/single-process/spa/output.yaml similarity index 89% rename from components/eamxx/tests/uncoupled/spa/output.yaml rename to components/eamxx/tests/single-process/spa/output.yaml index 408d676c64df..89c6ef76bda4 100644 --- a/components/eamxx/tests/uncoupled/spa/output.yaml +++ b/components/eamxx/tests/single-process/spa/output.yaml @@ -12,5 +12,4 @@ Field Names: output_control: Frequency: 1 frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/uncoupled/surface_coupling/CMakeLists.txt b/components/eamxx/tests/single-process/surface_coupling/CMakeLists.txt similarity index 100% rename from components/eamxx/tests/uncoupled/surface_coupling/CMakeLists.txt rename to components/eamxx/tests/single-process/surface_coupling/CMakeLists.txt diff --git a/components/eamxx/tests/uncoupled/surface_coupling/input.yaml b/components/eamxx/tests/single-process/surface_coupling/input.yaml similarity index 100% rename from components/eamxx/tests/uncoupled/surface_coupling/input.yaml rename to components/eamxx/tests/single-process/surface_coupling/input.yaml diff --git a/components/eamxx/tests/uncoupled/surface_coupling/output.yaml b/components/eamxx/tests/single-process/surface_coupling/output.yaml similarity index 93% rename from components/eamxx/tests/uncoupled/surface_coupling/output.yaml rename to components/eamxx/tests/single-process/surface_coupling/output.yaml index de4da9b32b31..432079db9685 100644 --- a/components/eamxx/tests/uncoupled/surface_coupling/output.yaml +++ b/components/eamxx/tests/single-process/surface_coupling/output.yaml @@ -22,5 +22,4 @@ Field Names: output_control: Frequency: 1 frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/uncoupled/surface_coupling/surface_coupling.cpp b/components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp similarity index 99% rename from components/eamxx/tests/uncoupled/surface_coupling/surface_coupling.cpp rename to components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp index 106618b827ed..d34059f8559f 100644 --- a/components/eamxx/tests/uncoupled/surface_coupling/surface_coupling.cpp +++ b/components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp @@ -90,7 +90,6 @@ std::vector create_from_file_test_data(const ekat::Comm& comm, cons // Create output manager to handle the data scorpio::eam_init_pio_subsystem(comm); ekat::ParameterList om_pl; - om_pl.set("MPI Ranks in Filename",true); om_pl.set("filename_prefix",std::string("surface_coupling_forcing")); om_pl.set("Field Names",fnames); om_pl.set("Averaging Type", std::string("INSTANT")); @@ -99,7 +98,6 @@ std::vector create_from_file_test_data(const ekat::Comm& comm, cons auto& ctrl_pl = om_pl.sublist("output_control"); ctrl_pl.set("frequency_units",std::string("nsteps")); ctrl_pl.set("Frequency",1); - ctrl_pl.set("MPI Ranks in Filename",true); ctrl_pl.set("save_grid_data",false); OutputManager4Test om; om.setup(comm,om_pl,fm,gm,t0,false); diff --git a/components/eamxx/tests/uncoupled/zm/CMakeLists.txt b/components/eamxx/tests/single-process/zm/CMakeLists.txt similarity index 100% rename from components/eamxx/tests/uncoupled/zm/CMakeLists.txt rename to components/eamxx/tests/single-process/zm/CMakeLists.txt diff --git a/components/eamxx/tests/uncoupled/zm/input.yaml b/components/eamxx/tests/single-process/zm/input.yaml similarity index 100% rename from components/eamxx/tests/uncoupled/zm/input.yaml rename to components/eamxx/tests/single-process/zm/input.yaml diff --git a/components/eamxx/tests/uncoupled/zm/zm_standalone.cpp b/components/eamxx/tests/single-process/zm/zm_standalone.cpp similarity index 100% rename from components/eamxx/tests/uncoupled/zm/zm_standalone.cpp rename to components/eamxx/tests/single-process/zm/zm_standalone.cpp diff --git a/components/eamxx/tests/uncoupled/CMakeLists.txt b/components/eamxx/tests/uncoupled/CMakeLists.txt deleted file mode 100644 index dd0fd493e29a..000000000000 --- a/components/eamxx/tests/uncoupled/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ -# NOTE: if you have baseline-type tests, add the subdirectory OUTSIDE the following if statement -if (NOT SCREAM_BASELINES_ONLY) - if ("${SCREAM_DYNAMICS_DYCORE}" STREQUAL "HOMME") - add_subdirectory(homme) - endif() - add_subdirectory(p3) - add_subdirectory(shoc) - add_subdirectory(cld_fraction) - add_subdirectory(spa) - add_subdirectory(surface_coupling) - if (SCREAM_ENABLE_ML_CORRECTION ) - add_subdirectory(ml_correction) - endif() - if (SCREAM_DOUBLE_PRECISION) - add_subdirectory(rrtmgp) - add_subdirectory(cosp) - else() - message(STATUS "RRTMGP and COSP only supported for double precision builds; skipping") - endif() - if (SCREAM_ENABLE_MAM) - add_subdirectory(mam4) - endif() - if (SCREAM_TEST_LEVEL GREATER_EQUAL SCREAM_TEST_LEVEL_EXPERIMENTAL) - add_subdirectory(zm) - endif() -endif() diff --git a/components/eamxx/tests/uncoupled/mam4/CMakeLists.txt b/components/eamxx/tests/uncoupled/mam4/CMakeLists.txt deleted file mode 100644 index 66475af4cdd3..000000000000 --- a/components/eamxx/tests/uncoupled/mam4/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -include (ScreamUtils) - -# Create the test -CreateADUnitTest(mam4_nucleation_standalone - LABELS mam4 physics - LIBS mam - MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} -) - -# Set AD configurable options -set (ATM_TIME_STEP 1) -SetVarDependingOnTestSize(NUM_STEPS 12 24 36) -set (RUN_T0 2021-10-12-45000) - -## Copy (and configure) yaml files needed by tests -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml - ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml - ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) - -# Ensure test input files are present in the data dir -GetInputFile(scream/init/${EAMxx_tests_IC_FILE_72lev}) diff --git a/components/eamxx/tests/uncoupled/mam4/input.yaml b/components/eamxx/tests/uncoupled/mam4/input.yaml deleted file mode 100644 index 80571b614f0d..000000000000 --- a/components/eamxx/tests/uncoupled/mam4/input.yaml +++ /dev/null @@ -1,38 +0,0 @@ -%YAML 1.1 ---- -driver_options: - atmosphere_dag_verbosity_level: 5 - -time_stepping: - time_step: ${ATM_TIME_STEP} - run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX - number_of_steps: ${NUM_STEPS} - -atmosphere_processes: - atm_procs_list: [mam4_micro] - mam4_micro: - compute_tendencies: [q_aitken_so4, n_aitken, q_h2so4] - -grids_manager: - Type: Mesh Free - grids_names: [Physics] - Physics: - type: point_grid - number_of_global_columns: 218 - number_of_vertical_levels: 72 - -initial_conditions: - # The name of the file containing the initial conditions for this test. - Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} - q_aitken_so4: 0.0 - n_aitken: 0.0 - q_h2so4: 1.9186478479542893e-011 # 0.65e-10 is from namelist, but this is what gets to nucleation - pbl_height: 1100.0 - T_mid: 273.0 - p_mid: 1.e5 - qv: 0.0018908932854425809 # computed from relative humidity = 0.5 using Hardy formulae - -# The parameters for I/O control -Scorpio: - output_yaml_files: ["output.yaml"] -... diff --git a/components/eamxx/tests/uncoupled/mam4/output.yaml b/components/eamxx/tests/uncoupled/mam4/output.yaml deleted file mode 100644 index 4e6ad2ac9128..000000000000 --- a/components/eamxx/tests/uncoupled/mam4/output.yaml +++ /dev/null @@ -1,15 +0,0 @@ -%YAML 1.1 ---- -filename_prefix: mam4_nucleation_standalone_output -Averaging Type: Instant -Field Names: - - T_mid - - p_mid - - q_aitken_so4 - - n_aitken - - q_h2so4 -output_control: - Frequency: 1 - frequency_units: nsteps - MPI Ranks in Filename: true -... diff --git a/components/eamxx/tests/uncoupled/rrtmgp/CMakeLists.txt b/components/eamxx/tests/uncoupled/rrtmgp/CMakeLists.txt deleted file mode 100644 index 90a6eed8dcf7..000000000000 --- a/components/eamxx/tests/uncoupled/rrtmgp/CMakeLists.txt +++ /dev/null @@ -1,88 +0,0 @@ -INCLUDE (ScreamUtils) - -# Test atmosphere processes -if (NOT SCREAM_BASELINES_ONLY) - # Ensure test input files are present in the data dir - GetInputFile(scream/init/${EAMxx_tests_IC_FILE_72lev}) - - set (TEST_BASE_NAME rrtmgp_standalone) - set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) - - # Unit test to compare against raw rrtmgp output - configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_unit.yaml - ${CMAKE_CURRENT_BINARY_DIR}/input_unit.yaml) - CreateUnitTest(${TEST_BASE_NAME}_unit rrtmgp_standalone_unit.cpp - LABELS rrtmgp physics driver - LIBS scream_rrtmgp rrtmgp scream_control yakl diagnostics rrtmgp_test_utils - EXE_ARGS "--ekat-test-params rrtmgp_inputfile=${SCREAM_DATA_DIR}/init/rrtmgp-allsky.nc,rrtmgp_baseline=${SCREAM_TEST_DATA_DIR}/rrtmgp-allsky-baseline.nc" - ) - # This test needs the allsky baselines file - add_dependencies (rrtmgp_standalone_unit rrtmgp_allsky_baseline.nc) - - ## Create rrtmgp stand alone executable - CreateUnitTestExec(${TEST_BASE_NAME} "rrtmgp_standalone.cpp" - LIBS scream_rrtmgp rrtmgp scream_control yakl diagnostics - ) - - # The RRTMGP stand-alone test that runs multi-step - # Set AD configurable options - SetVarDependingOnTestSize(NUM_STEPS 2 5 48) - set (ATM_TIME_STEP 1800) - set (RUN_T0 2021-10-12-45000) - - # Test non-chunked version (sweep multiple ranks) - set (SUFFIX "_not_chunked") - set (COL_CHUNK_SIZE 1000) - configure_file (${CMAKE_CURRENT_SOURCE_DIR}/output.yaml - ${CMAKE_CURRENT_BINARY_DIR}/output_not_chunked.yaml) - configure_file (${CMAKE_CURRENT_SOURCE_DIR}/input.yaml - ${CMAKE_CURRENT_BINARY_DIR}/input_not_chunked.yaml) - CreateUnitTestFromExec( - ${TEST_BASE_NAME}_not_chunked ${TEST_BASE_NAME} - LABELS rrtmgp physics driver - MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} - EXE_ARGS "--ekat-test-params inputfile=input_not_chunked.yaml" - FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME}_not_chunked - ) - - # Compare output files produced by npX tests, to ensure they are bfb - include (CompareNCFiles) - - CompareNCFilesFamilyMpi ( - TEST_BASE_NAME ${TEST_BASE_NAME}_not_chunked - FILE_META_NAME ${TEST_BASE_NAME}_output_not_chunked.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc - MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} - LABELS rrtmgp physics - META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_not_chunked_npMPIRANKS_omp1 - ) - - - ## Test chunked version (only for ${TEST_RANK_END}) and compare against non-chunked - set (SUFFIX "_chunked") - math (EXPR COL_PER_RANK "218 / ${TEST_RANK_END}") - math (EXPR COL_CHUNK_SIZE "${COL_PER_RANK} / 2") - if (COL_CHUNK_SIZE LESS 1) - message (FATAL_ERROR "Error! Chunk size for rrtmgp unit test is less than 1.") - endif() - - configure_file (${CMAKE_CURRENT_SOURCE_DIR}/input.yaml - ${CMAKE_CURRENT_BINARY_DIR}/input_chunked.yaml) - configure_file (${CMAKE_CURRENT_SOURCE_DIR}/output.yaml - ${CMAKE_CURRENT_BINARY_DIR}/output_chunked.yaml) - CreateUnitTestFromExec( - ${TEST_BASE_NAME}_chunked ${TEST_BASE_NAME} - LABELS rrtmgp physics driver - MPI_RANKS ${TEST_RANK_END} - EXE_ARGS "--ekat-test-params inputfile=input_chunked.yaml" - FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME}_chunked - PROPERTIES PASS_REGULAR_EXPRESSION "(beg.end: 0, ${COL_CHUNK_SIZE})" - ) - - CompareNCFiles( - TEST_NAME ${TEST_BASE_NAME}_chunked_vs_not_chunked - SRC_FILE ${TEST_BASE_NAME}_output_chunked.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc - TGT_FILE ${TEST_BASE_NAME}_output_not_chunked.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc - LABELS rrtmgp physics - FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_chunked_np${TEST_RANK_END}_omp1 - ${FIXTURES_BASE_NAME}_not_chunked_np${TEST_RANK_END}_omp1) -endif() diff --git a/components/eamxx/tpls/CMakeLists.txt b/components/eamxx/tpls/CMakeLists.txt index d89f48d4cacd..ee5a6644b135 100644 --- a/components/eamxx/tpls/CMakeLists.txt +++ b/components/eamxx/tpls/CMakeLists.txt @@ -16,90 +16,3 @@ if (NOT SCREAM_LIB_ONLY) include(BuildCprnc) BuildCprnc() endif() - -# MAM aerosol support -if (SCREAM_ENABLE_MAM) - # We use CMake's ExternalProject capability to build and install Haero. - include(ExternalProject) - - if (SCREAM_CIME_BUILD) - # PROJECT_SOURCE_DIR is SCREAM_ROOT/components - set(EXTERNALS_DIR "${PROJECT_SOURCE_DIR}/../externals") - else() - # PROJECT_SOURCE_DIR is SCREAM_ROOT/components/eamxx - set(EXTERNALS_DIR "${PROJECT_SOURCE_DIR}/../../externals") - endif() - - # Normalize CMAKE_BUILD_TYPE. - string(TOLOWER "${CMAKE_BUILD_TYPE}" lc_build_type) - if(${lc_build_type} STREQUAL "release") - set(mam_build_type "Release") - else() - set(mam_build_type "Debug") # when in doubt... - endif() - - # Build and install the Haero aerosol package interface. - if (SCREAM_DOUBLE_PRECISION) - set(HAERO_PRECISION "double") - else() - set(HAERO_PRECISION "single") - endif() - set(HAERO_INSTALL_PREFIX "${PROJECT_BINARY_DIR}/externals/haero") - set(HAERO_CMAKE_OPTS - -DCMAKE_INSTALL_PREFIX=${HAERO_INSTALL_PREFIX} - -DCMAKE_BUILD_TYPE=${mam_build_type} - -DCMAKE_VERBOSE_MAKEFILE=${CMAKE_VERBOSE_MAKEFILE} - -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DHAERO_ENABLE_GPU=${Kokkos_ENABLE_CUDA} - -DHAERO_ENABLE_MPI=ON - -DHAERO_PRECISION=${HAERO_PRECISION} - -DEKAT_SOURCE_DIR=${EXTERNALS_DIR}/ekat - -DEKAT_BINARY_DIR=${PROJECT_BINARY_DIR}/externals/ekat - -DBUILD_SHARED_LIBS=OFF) - ExternalProject_Add(haero_proj - PREFIX ${PROJECT_BINARY_DIR}/externals/haero - SOURCE_DIR ${EXTERNALS_DIR}/haero - BINARY_DIR ${PROJECT_BINARY_DIR}/externals/haero - CMAKE_ARGS ${HAERO_CMAKE_OPTS} - DEPENDS ekat ekat_test_session ekat_test_main - LOG_CONFIGURE TRUE - BUILD_COMMAND make -j - LOG_BUILD TRUE - INSTALL_COMMAND make install - LOG_INSTALL TRUE - LOG_OUTPUT_ON_FAILURE TRUE - BUILD_ALWAYS TRUE) - add_library(haero STATIC IMPORTED GLOBAL) - set_target_properties(haero PROPERTIES IMPORTED_LOCATION ${HAERO_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libhaero.a) - - # Build and install MAM4xx. - set(MAM4XX_INSTALL_PREFIX "${PROJECT_BINARY_DIR}/externals/mam4xx") - set(MAM4XX_CMAKE_OPTS - -DCMAKE_INSTALL_PREFIX=${MAM4XX_INSTALL_PREFIX} - -DCMAKE_BUILD_TYPE=${mam_build_type} - -DCMAKE_VERBOSE_MAKEFILE=${CMAKE_VERBOSE_MAKEFILE} - -DMAM4XX_HAERO_DIR=${HAERO_INSTALL_PREFIX} - -DBUILD_SHARED_LIBS=OFF - ) - ExternalProject_Add(mam4xx_proj - PREFIX ${PROJECT_BINARY_DIR}/externals/mam4xx - SOURCE_DIR ${EXTERNALS_DIR}/mam4xx - BINARY_DIR ${PROJECT_BINARY_DIR}/externals/mam4xx - CMAKE_ARGS ${MAM4XX_CMAKE_OPTS} - DEPENDS haero_proj - LOG_CONFIGURE TRUE - BUILD_COMMAND make -j - LOG_BUILD TRUE - INSTALL_COMMAND make install - LOG_INSTALL TRUE - LOG_OUTPUT_ON_FAILURE TRUE - BUILD_ALWAYS TRUE) - add_library(mam4xx STATIC IMPORTED GLOBAL) - set_target_properties(mam4xx PROPERTIES IMPORTED_LOCATION ${MAM4XX_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libmam4xx.a) - # Bring in MAM4xx-related targets by including the generated mam4xx.cmake - # list(APPEND CMAKE_MODULE_PATH ${MAM4XX_INSTALL_PREFIX}/share) - # include(mam4xx) - -endif() diff --git a/components/elm/bld/ELMBuildNamelist.pm b/components/elm/bld/ELMBuildNamelist.pm index 4e5818e6963b..c6214323525b 100755 --- a/components/elm/bld/ELMBuildNamelist.pm +++ b/components/elm/bld/ELMBuildNamelist.pm @@ -797,7 +797,8 @@ sub setup_cmdl_fates_mode { "use_fates_planthydro", "use_fates_ed_st3", "use_fates_ed_prescribed_phys", "use_fates_inventory_init", "use_fates_fixed_biogeog", "use_fates_nocomp","use_fates_sp", "fates_inventory_ctrl_filename","use_fates_logging", "use_fates_tree_damage", - "use_fates_parteh_mode","use_fates_cohort_age_tracking","use_snicar_ad"); + "use_fates_parteh_mode","use_fates_cohort_age_tracking","use_snicar_ad", "use_fates_luh", + "fluh_timeseries"); foreach my $var ( @list ) { if ( defined($nl->get_value($var)) ) { $nl_flags->{$var} = $nl->get_value($var); @@ -857,6 +858,10 @@ sub setup_cmdl_fates_mode { if ( defined($nl->get_value($var)) ) { fatal_error("$var is being set, but can ONLY be set when -bgc fates option is used.\n"); } + $var = "use_fates_luh"; + if ( defined($nl->get_value($var)) ) { + fatal_error("$var is being set, but can ONLY be set when -bgc fates option is used.\n"); + } $var = "fates_inventory_ctrl_filename"; if ( defined($nl->get_value($var)) ) { fatal_error("$var is being set, but can ONLY be set when -bgc fates option is used.\n"); @@ -3292,8 +3297,43 @@ sub setup_logic_fates { add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'use_fates_nocomp', 'use_fates'=>$nl_flags->{'use_fates'}); add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'use_fates_tree_damage', 'use_fates'=>$nl_flags->{'use_fates'}); add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'fates_seeddisp_cadence', 'use_fates'=>$nl_flags->{'use_fates'}); + add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'use_fates_luh', 'use_fates'=>$nl_flags->{'use_fates'}); add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'fates_paramfile', 'phys'=>$nl_flags->{'phys'}); - + add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'fluh_timeseries', 'phys'=>$nl_flags->{'phys'}); + + # For FATES SP mode make sure no-competion, and fixed-biogeography are also set + # And also check for other settings that can't be trigged on as well + my $var = "use_fates_sp"; + if ( defined($nl->get_value($var)) ) { + if ( &value_is_true($nl->get_value($var)) ) { + my @list = ( "use_fates_nocomp", "use_fates_fixed_biogeog" ); + foreach my $var ( @list ) { + if ( ! &value_is_true($nl->get_value($var)) ) { + fatal_error("$var is required when FATES SP is on (use_fates_sp)" ); + } + } + # spit-fire can't be on with FATES SP mode is active + if ( $nl->get_value('fates_spitfire_mode') > 0 ) { + fatal_error('fates_spitfire_mode can NOT be set to greater than 0 when use_fates_sp is true'); + } + } + } + # check that fates landuse change mode has the necessary luh2 landuse timeseries data + my $var = "use_fates_luh"; + if ( defined($nl->get_value($var)) ) { + if ( &value_is_true($nl->get_value($var)) ) { + $var = "fluh_timeseries"; + add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, $var, + 'phys'=>$nl_flags->{'phys'}, 'hgrid'=>$nl_flags->{'res'}, + 'sim_year_range'=>$nl_flags->{'sim_year_range'}, nofail=>1 ); + my $fname = remove_leading_and_trailing_quotes( $nl->get_value($var) ); + if ( ! defined($nl->get_value($var)) ) { + fatal_error("$var is required when use_fates_luh is set" ); + } elsif ( ! -f "$fname" ) { + fatal_error("$fname does NOT point to a valid filename" ); + } + } + } } } diff --git a/components/elm/bld/configure b/components/elm/bld/configure index 01cc15f9d204..e0a13f30be7d 100755 --- a/components/elm/bld/configure +++ b/components/elm/bld/configure @@ -477,6 +477,7 @@ sub write_filepath_cesmbld "external_models/fates/biogeochem", "external_models/fates/fire", "external_models/fates/parteh", + "external_models/fates/radiation", "external_models/mpp/src/mpp/dtypes", "external_models/mpp/src/mpp/thermal", "external_models/mpp/src/mpp/util", diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index 4f02fc9df8ac..8294fdeeb5af 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -55,7 +55,14 @@ attributes from the config_cache.xml file (with keys converted to upper-case). NONE -NONE + + + + + +ALL +ALL + RD @@ -127,7 +134,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). -lnd/clm2/paramdata/fates_params_api25.5.0_12pft_c230710.nc +lnd/clm2/paramdata/fates_params_api.32.0.0_12pft_c231215.nc lnd/clm2/paramdata/CNP_parameters_c131108.nc @@ -267,10 +274,13 @@ ic_tod="0" sim_year="2000" glc_nec="0" use_crop=".true." irrigate=".true." nu_co lnd/clm2/initdata/20220928.I2010CRUELM.ne1024pg2_ICOS10.elm.r.2016-08-01-00000.nc lnd/clm2/initdata/20221218.F2010-CICE.ne30pg2_ne1024pg2.elm.r.2020-01-20-00000.nc +lnd/clm2/initdata/20231226.I2010CRUELM.ne1024pg2_ICOS10.elm.r.1994-10-01-00000.nc +lnd/clm2/initdata/20240104.I2010CRUELM.ne256pg2.elm.r.1994-10-01-00000.nc lnd/clm2/initdata/20220717.I2010CRUELM.ne1024pg2_ICOS10.elm.r.2013-01-01-00000.nc lnd/clm2/initdata/20220717.I2010CRUELM.ne1024pg2_ICOS10.elm.r.2013-04-01-00000.nc lnd/clm2/initdata/20220717.I2010CRUELM.ne1024pg2_ICOS10.elm.r.2013-07-01-00000.nc lnd/clm2/initdata/20220717.I2010CRUELM.ne1024pg2_ICOS10.elm.r.2013-10-01-00000.nc + lnd/clm2/initdata/ICRUCLM45-360x720cru.clm2.r.2016-08-01-00000.nc lnd/clm2/initdata/ICRUCLM45.r0125_oRRS15to5.clm2.r.2016-08-01-00000.nc lnd/clm2/initdata/ICRUCLM45.r0125_oRRS18to6v3.clm2.r.2016-08-01-00000.nc @@ -292,12 +302,16 @@ lnd/clm2/surfdata_map/surfdata_ne30pg2_simyr2010_c210402_with_TOP.nc lnd/clm2/surfdata_map/surfdata_ne120np4_simyr2010_c20181023.nc lnd/clm2/surfdata_map/surfdata_ne120pg2_simyr2010_c230119.nc + +lnd/clm2/surfdata_map/surfdata_ne256pg2_simyr1850_c240131.nc lnd/clm2/surfdata_map/surfdata_ne256pg2_simyr2010_c230207.nc lnd/clm2/surfdata_map/surfdata_ne1024pg2_simyr2010_c211021.nc lnd/clm2/surfdata_map/surfdata_ne1024np4_simyr2010_c220523.nc + +lnd/clm2/surfdata_map/surfdata_0.25x0.25_simyr2010_c240206_TOP.nc @@ -410,7 +424,7 @@ lnd/clm2/surfdata_map/surfdata_ne30np4_simyr1850_c180306.nc lnd/clm2/surfdata_map/surfdata_ne30pg2_simyr1850_c230417_with_TOP.nc -lnd/clm2/surfdata_map/surfdata_ne30pg2_simyr1950_c210729.nc +lnd/clm2/surfdata_map/surfdata_ne30pg2_simyr1950_c210729_with_TOP.nc lnd/clm2/surfdata_map/surfdata_ne16np4_simyr1850_c160108.nc @@ -434,6 +448,8 @@ lnd/clm2/surfdata_map/surfdata_0.5x0.5_simyr1850_c211019.nc lnd/clm2/surfdata_map/surfdata_0.125x0.125_simyr1850_c190730.nc lnd/clm2/surfdata_map/surfdata_0.125x0.125_simyr1950_c210924.nc + +lnd/clm2/surfdata_map/surfdata_0.25x0.25_simyr1850_c240125_TOP.nc lnd/clm2/surfdata_map/surfdata_conusx4v1_simyr1850_c160503.nc @@ -497,6 +513,10 @@ lnd/clm2/surfdata_map/surfdata_conusx4v1_simyr2000_c160503.nc use_crop=".false." >lnd/clm2/surfdata_map/surfdata.pftdyn_ne16np4_hist_simyr1850-2005_c160803.nc lnd/clm2/surfdata_map/surfdata.pftdyn_ne11np4_hist_simyr1850-2005_c160803.nc +lnd/clm2/surfdata_map/landuse.timeseries_ne1024pg2_historical_simyr1990-2014_c240109.nc +lnd/clm2/surfdata_map/landuse.timeseries_ne256pg2_hist_simyr1850-2015_c240131.nc lnd/clm2/surfdata_map/landuse.timeseries_0.125x0.125_hist_simyr1850-2015_c191004.nc @@ -565,6 +585,11 @@ lnd/clm2/surfdata_map/surfdata_conusx4v1_simyr2000_c160503.nc lnd/clm2/surfdata_map/surfdata.pftdyn_ne30np4_rcp2.6_simyr1850-2100_c130524.nc + +lnd/clm2/surfdata_map/fates-sci.1.68.3_api.31.0.0_tools.1.0.1/LUH2_states_transitions_management.timeseries_4x5_hist_simyr1850-2015_c231101.nc + + @@ -576,22 +601,10 @@ this mask will have smb calculated over the entire global land surface lnd/clm2/griddata/glcmaskdata_0.9x1.25_GIS_AIS.nc lnd/clm2/griddata/glcmaskdata_ne30np4_GIS_AIS_171002.nc - -glc/cism/griddata/glcmaskdata_48x96_Gland20km.nc -glc/cism/griddata/glcmaskdata_48x96_Gland10km.nc -glc/cism/griddata/glcmaskdata_48x96_Gland5km.nc -glc/cism/griddata/glcmaskdata_48x96_Gland5km.nc -glc/cism/griddata/glcmaskdata_0.9x1.25_Gland20km.nc -glc/cism/griddata/glcmaskdata_0.9x1.25_Gland10km.nc -glc/cism/griddata/glcmaskdata_0.9x1.25_Gland5km.nc -glc/cism/griddata/glcmaskdata_0.9x1.25_Gland5km.nc -glc/cism/griddata/glcmaskdata_1.9x2.5_Gland20km.nc -glc/cism/griddata/glcmaskdata_1.9x2.5_Gland10km.nc -glc/cism/griddata/glcmaskdata_1.9x2.5_Gland5km.nc -glc/cism/griddata/glcmaskdata_1.9x2.5_Gland5km.nc - + glc/cism/griddata/glcmaskdata_0.9x1.25_Gland5km.nc lnd/clm2/griddata/glcmaskdata_0.9x1.25_60S.nc +lnd/clm2/griddata/glcmaskdata_0.5x0.5_GIS_AIS.nc lnd/clm2/griddata/glcmaskdata_0.5x0.5_GIS_AIS.nc lnd/clm2/griddata/glcmaskdata_r0125_GIS_AIS.210407.nc lnd/clm2/griddata/glcmaskdata_ne30pg2_GIS_AIS.210407.nc @@ -2061,6 +2074,39 @@ this mask will have smb calculated over the entire global land surface +lnd/clm2/mappingdata/maps/0.25x0.25/map_0.5x0.5_AVHRR_to_0.25x0.25_nomask_aave_da_c240123.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_0.5x0.5_MODIS_to_0.25x0.25_nomask_aave_da_c240123.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_0.9x1.25_GRDC_to_0.25x0.25_nomask_aave_da_c240124.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_10x10min_IGBPmergeICESatGIS_to_0.25x0.25_nomask_aave_da_c240123.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_10x10min_nomask_to_0.25x0.25_nomask_aave_da_c240123.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_1km-merge-10min_HYDRO1K-merge-nomask_to_0.25x0.25_nomask_aave_da_c240123.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_360x720cru_cruncep_to_0.25x0.25_nomask_aave_da_c240124.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_GLOBE-Gardner-mergeGIS_to_0.25x0.25_nomask_aave_da_c240123.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_GLOBE-Gardner_to_0.25x0.25_nomask_aave_da_c240123.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_LandScan2004_to_0.25x0.25_nomask_aave_da_c240123.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_MODIS_to_0.25x0.25_nomask_aave_da_c240123.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_USGS_to_0.25x0.25_nomask_aave_da_c240123.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_5x5min_IGBP-GSDP_to_0.25x0.25_nomask_aave_da_c240123.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_5x5min_ISRIC-WISE_to_0.25x0.25_nomask_aave_da_c240123.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_5x5min_nomask_to_0.25x0.25_nomask_aave_da_c240123.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_0.5x0.5_GSDTG2000_to_0.25x0.25_nomask_aave_da_c240123.nc + lnd/clm2/mappingdata/maps/antarcticax4v1/map_0.5x0.5_AVHRR_to_antarcticax4v1_nomask_aave_da_c210130.nc .false. .false. .false. +.false. 1 0 .false. diff --git a/components/elm/bld/namelist_files/namelist_defaults_overall.xml b/components/elm/bld/namelist_files/namelist_defaults_overall.xml index 5bbe8211a643..daf8ae6b959b 100644 --- a/components/elm/bld/namelist_files/namelist_defaults_overall.xml +++ b/components/elm/bld/namelist_files/namelist_defaults_overall.xml @@ -90,7 +90,7 @@ determine default values for namelists. none -gland5UM +mpas.gis20km .false. .true. 0 diff --git a/components/elm/bld/namelist_files/namelist_defaults_tools.xml b/components/elm/bld/namelist_files/namelist_defaults_tools.xml index 760304107140..113f949ffe82 100644 --- a/components/elm/bld/namelist_files/namelist_defaults_tools.xml +++ b/components/elm/bld/namelist_files/namelist_defaults_tools.xml @@ -146,6 +146,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). ISRIC-WISE AVHRR AVHRR +ISRIC-WISE 3x3min @@ -174,7 +175,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). 5x5min 0.5x0.5 0.5x0.5 - +5x5min mksrf_flakwat @@ -201,7 +202,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). mksrf_fgrvl mksrf_fslp10 mksrf_fero - +mksrf_ffert lnd/clm2/rawdata/mksrf_navyoro_20min.c010129.nc @@ -209,7 +210,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). lnd/clm2/rawdata/pftlandusedyn.0.5x0.5.simyr1850-2005.c090630/mksrf_lai24pftnontrop_c120712.nc +>lnd/clm2/rawdata/mksrf_lai34cft_simyr2000_c200622.nc lnd/clm2/rawdata/pftlandusedyn.0.5x0.5.simyr1850-2005.c090630/mksrf_lai_global_c090506.nc @@ -308,6 +309,10 @@ attributes from the config_cache.xml file (with keys converted to upper-case). lnd/clm2/rawdata/mksrf_soilero_0.5x0.5.c220523.nc + +lnd/clm2/rawdata/mksrf_fert.c220309.nc + @@ -319,6 +324,10 @@ attributes from the config_cache.xml file (with keys converted to upper-case). lnd/clm2/rawdata/pftlanduse.3minx3min.simyr2000.c110913/mksrf_24pftNT_landuse_rc2000_c121207.nc +lnd/clm2/rawdata/mksrf_newcrop_c200813.nc +lnd/clm2/rawdata/mksrf_newcrop_c200813.nc + + lnd/clm2/rawdata/LUT_LUH2_HIST_LUH1f_07082020/LUT_LUH2_historical_1850_c07082020.nc diff --git a/components/elm/bld/namelist_files/namelist_definition.xml b/components/elm/bld/namelist_files/namelist_definition.xml index 1353f8b84b64..0f8b03dfec38 100644 --- a/components/elm/bld/namelist_files/namelist_definition.xml +++ b/components/elm/bld/namelist_files/namelist_definition.xml @@ -215,12 +215,11 @@ Normally this is ONLY used when running CESM with the active glacier model. + group="elm_inparm" valid_values="mpas.gis20km,mpas.gis1to10km,mpas.gis1to10kmR2" > Glacier model grid - gland20 = Greenland at 20km resolution - gland10 = Greenland at 10km resolution - gland5 = Greenland at 5km resolution - gland5UM = Greenland at 5km resolution, using new UMontana ice sheet data for present day Greenland + mpas.gis20km = Greenland at 20km resolution + mpas.gis1to10km = Greenland at 1-to-10km variable resolution (v1, kept for backwards compatibility) + mpas.gis1to10kmR2 = Greenland at 1-to-10km variable resolution (v2, improved init. cond., fewer total grid cells) + + +If TRUE, enable use of land use state and transition data from luh_timeseries file. +(Only valid for fates land use change runs, where there is a luh_timeseries file.) +(Also, only valid for use_fates = true and is incompatible with transient runs currently.) + + + + +Full pathname of unified land use harmonization data file. This causes the land-use +types to vary over time. + + Toggle to turn on if Kennedy et al plant hydraulics model is used. @@ -772,7 +786,7 @@ in its attributes. (Only used if scripgriddata_src_type = UGRID.) + valid_values="mksrf_fsoitex,mksrf_forganic,mksrf_flakwat,mksrf_fwetlnd,mksrf_fmax,mksrf_fmax,mksrf_fglacier,mksrf_fvocef,mksrf_furbtopo,mksrf_flndtopo,firrig,mksrf_furban,mksrf_fvegtyp,mksrf_fsoicol,mksrf_fsoiord,mksrf_flai,mksrf_fgdp,mksrf_fpeat,mksrf_fabm,mksrf_ftopostats,mksrf_fvic,mksrf_fch4,mksrf_fgrvl,mksrf_fslp10,mksrf_fero,mksrf_ffert" > Filename for mksurfdata_map to remap raw data into the output surface dataset @@ -930,6 +944,11 @@ Slope percentile dataset ELM-Erosion parameters dataset + +ELM-Fertilizer dataset + + If TRUE, output variables in double precision for mksurfdata @@ -1390,7 +1409,7 @@ CLM run type. +"512x1024,360x720cru,128x256,64x128,48x96,32x64,8x16,94x192,0.23x0.31,0.9x1.25,1.9x2.5,2.5x3.33,4x5,10x15,5x5_amazon,1x1_tropicAtl,1x1_camdenNJ,1x1_vancouverCAN,1x1_mexicocityMEX,1x1_asphaltjungleNJ,1x1_brazil,1x1_urbanc_alpha,1x1_numaIA,1x1_smallvilleIA,0.1x0.1,0.5x0.5,3x3min,5x5min,10x10min,0.33x0.33,ne4np4,ne4np4.pg2,ne11np4,ne16np4,ne30np4,ne30np4.pg2,ne60np4,ne120np4,ne120np4.pg2,ne240np4,ne256np4,ne256np4.pg2,ne1024np4,ne1024np4.pg2,1km-merge-10min,ne0np4_arm_x8v3_lowcon,ne0np4_conus_x4v1_lowcon,ne0np4_enax4v1,ne0np4_twpx4v1,r2,r05,r0125,NLDAS,ne0np4_northamericax4v1.pg2,ne0np4_arcticx4v1.pg2,r025"> Horizontal resolutions Note: 0.1x0.1, 0.5x0.5, 5x5min, 10x10min, 3x3min and 0.33x0.33 are only used for CLM tools @@ -1404,7 +1423,7 @@ Representative concentration pathway for future scenarios [radiative forcing at + valid_values="USGS,gx3v7,gx1v6,navy,test,tx0.1v2,tx1v1,T62,TL319,cruncep,oEC60to30v3,oEC60to30v3wLI,ECwISC30to60E1r2,EC30to60E2r2,WC14to60E2r3,WCAtl12to45E2r4,SOwISC12to60E2r4,ECwISC30to60E2r1,oRRS18to6,oRRS18to6v3,oRRS15to5,oARRM60to10,oARRM60to6,ARRM10to60E2r1,oQU480,oQU240,oQU240wLI,oQU120,oRRS30to10v3,oRRS30to10v3wLI,360x720cru,NLDASww3a,NLDAS,tx0.1v2,ICOS10,IcoswISC30E3r5,IcosXISC30E3r7"> Land mask description diff --git a/components/elm/bld/namelist_files/use_cases/2015-2100_SSP245_transient.xml b/components/elm/bld/namelist_files/use_cases/2015-2100_SSP245_transient.xml new file mode 100755 index 000000000000..e313dab6b16a --- /dev/null +++ b/components/elm/bld/namelist_files/use_cases/2015-2100_SSP245_transient.xml @@ -0,0 +1,45 @@ + + + + +Simulate transient land-use, and aerosol deposition changes from 2015 to 2100 +Simulate transient land-use, aerosol deposition, and Nitrogen deposition changes from 2015 to 2100 +Simulate transient land-use, aerosol deposition, and Nitrogen deposition changes from 2015 to 2100 +Simulate transient land-use, aerosol deposition, and Nitrogen deposition changes from 2015 to 2100 + + + +2015-2100 + +arb_ic + +flanduse_timeseries + + + +1850 +2005 +1850 + +2000 +2000 +2000 + +1850 +2010 +1850 + + + + +lnd/clm2/surfdata_map/surfdata_0.5x0.5_simyr1850_c200609_with_TOP.nc +lnd/clm2/surfdata_map/landuse.timeseries_0.5x0.5_ssp2_rcp45_simyr2015-2100_c240408.nc + + + +.false. +.true. +.true. +.false. + + diff --git a/components/elm/bld/namelist_files/use_cases/2015-2100_SSP370_transient.xml b/components/elm/bld/namelist_files/use_cases/2015-2100_SSP370_transient.xml index a693410774c3..0e8415d3c1ab 100755 --- a/components/elm/bld/namelist_files/use_cases/2015-2100_SSP370_transient.xml +++ b/components/elm/bld/namelist_files/use_cases/2015-2100_SSP370_transient.xml @@ -15,7 +15,8 @@ flanduse_timeseries - + 1850 2005 1850 @@ -27,13 +28,14 @@ 1850 2010 1850 ---> - -lnd/clm2/surfdata_map/surfdata_ne30np4.pg2_SSP3_RCP70_simyr2015_c220420_with_TOP.nc -lnd/clm2/surfdata_map/landuse.timeseries_ne30pg2_SSP3_RCP70_simyr2015-2100_c211015.nc +lnd/clm2/surfdata_map/surfdata_ne30np4.pg2_SSP3_RCP70_simyr2015_c220420_with_TOP.nc + + +lnd/clm2/surfdata_map/landuse.timeseries_ne30pg2_SSP3_RCP70_simyr2015-2100_c211015.nc +lnd/clm2/surfdata_map/landuse.timeseries_0.5x0.5_ssp3_rcp70_simyr2015-2100_c240308.nc diff --git a/components/elm/bld/namelist_files/use_cases/2015-2100_SSP585_transient.xml b/components/elm/bld/namelist_files/use_cases/2015-2100_SSP585_transient.xml index e93f28195646..2db3549ce5f0 100644 --- a/components/elm/bld/namelist_files/use_cases/2015-2100_SSP585_transient.xml +++ b/components/elm/bld/namelist_files/use_cases/2015-2100_SSP585_transient.xml @@ -15,7 +15,8 @@ flanduse_timeseries - + 1850 2005 1850 @@ -27,13 +28,14 @@ 1850 2010 1850 ---> - -lnd/clm2/surfdata_map/surfdata_ne30np4.pg2_SSP5_RCP85_simyr2015_c220420_with_TOP.nc -lnd/clm2/surfdata_map/landuse.timeseries_ne30pg2_SSP5_RCP85_simyr2015-2100_c220310.nc +lnd/clm2/surfdata_map/surfdata_ne30np4.pg2_SSP5_RCP85_simyr2015_c220420_with_TOP.nc + + +lnd/clm2/surfdata_map/landuse.timeseries_ne30pg2_SSP5_RCP85_simyr2015-2100_c220310.nc +lnd/clm2/surfdata_map/landuse.timeseries_0.5x0.5_ssp5_rcp85_simyr2015-2100_c240408.nc diff --git a/components/elm/bld/namelist_files/use_cases/20thC_CMIP6_transient.xml b/components/elm/bld/namelist_files/use_cases/20thC_CMIP6_transient.xml index 651a7d523835..c635abebc43f 100644 --- a/components/elm/bld/namelist_files/use_cases/20thC_CMIP6_transient.xml +++ b/components/elm/bld/namelist_files/use_cases/20thC_CMIP6_transient.xml @@ -45,6 +45,10 @@ lnd/clm2/surfdata_map/surfdata_0.5x0.5_simyr1850_c200609_with_TOP.nc + +lnd/clm2/surfdata_map/surfdata_ne1024pg2_simyr2010_c211021.nc +lnd/clm2/surfdata_map/landuse.timeseries_ne1024pg2_historical_simyr1990-2014_c240109.nc + .false. .false. diff --git a/components/elm/bld/namelist_files/use_cases/20thC_transient.xml b/components/elm/bld/namelist_files/use_cases/20thC_transient.xml index 283fb35605d1..86035ec6e081 100644 --- a/components/elm/bld/namelist_files/use_cases/20thC_transient.xml +++ b/components/elm/bld/namelist_files/use_cases/20thC_transient.xml @@ -30,5 +30,10 @@ 2010 1850 + +lnd/clm2/surfdata_map/surfdata_ne1024pg2_simyr2010_c211021.nc +lnd/clm2/surfdata_map/landuse.timeseries_ne1024pg2_historical_simyr1990-2014_c240109.nc +.false. +.false. diff --git a/components/elm/cime_config/config_component.xml b/components/elm/cime_config/config_component.xml index be8dae3b6d9f..94fac9ab9394 100755 --- a/components/elm/cime_config/config_component.xml +++ b/components/elm/cime_config/config_component.xml @@ -22,10 +22,10 @@ -phys elm -cppdefs '-DMODAL_AER -DAPPLY_POST_DECK_BUGFIXES' -phys elm -cppdefs '-DMODAL_AER -DAPPLY_POST_DECK_BUGFIXES' - build_component_clm + build_component_elm env_build.xml Provides option(s) for the ELM configure utility. - CLM_CONFIG_OPTS are normally set as compset variables (e.g., -bgc cn) + ELM_CONFIG_OPTS are normally set as compset variables (e.g., -bgc cn) and in general should not be modified for supported compsets. It is recommended that if you want to modify this value for your experiment, you should use your own user-defined component sets via using create_newcase @@ -83,6 +83,7 @@ 2015-2100_SSP585_transient 2015-2100_SSP585_CMIP6bgc_transient 2015-2100_SSP370_transient + 2015-2100_SSP245_transient 2010_CMIP6LR_control 2010_CMIP6HR_control @@ -95,7 +96,7 @@ 20thC_CMIP6_transient - run_component_clm + run_component_elm env_run.xml ELM namelist use_case. Determines the use-case that will be sent to the ELM build-namelist utility. @@ -103,7 +104,7 @@ used by expert users. - + char @@ -139,7 +140,7 @@ -bgc sp -bgc bgc - run_component_clm + run_component_elm env_run.xml ELM build-namelist options @@ -161,12 +162,12 @@ diagnostic constant - run_component_clm + run_component_elm env_run.xml Determines how ELM will determine where CO2 is set. If value is constant, it will be set to CCSM_CO2_PPMV, if value is either diagnostic or prognostic, the atmosphere model - MUST send it to ELM. CLM_CO2_TYPE is normally set by the specific + MUST send it to ELM. ELM_CO2_TYPE is normally set by the specific compset, since it HAS to be coordinated with settings for the atmospheric model. Do not modify this variable. If you want to modify for your experiment, use your own user-defined component set @@ -176,10 +177,10 @@ char - run_component_clm + run_component_elm env_run.xml ELM-specific namelist settings for -namelist option in the ELM - build-namelist. CLM_NAMELIST_OPTS is normally set as a compset variable + build-namelist. ELM_NAMELIST_OPTS is normally set as a compset variable and in general should not be modified for supported compsets. It is recommended that if you want to modify this value for your experiment, you should use your own user-defined component sets via using create_newcase @@ -191,7 +192,7 @@ char on,off off - run_component_clm + run_component_elm env_run.xml Turn on any settings for accellerating the model spinup. @@ -200,10 +201,10 @@ char UNSET - run_component_clm + run_component_elm env_run.xml Dataset name for user-created datasets. This is used as the argument - in Buildconf/clm.buildnml to build-namelist -clm_usr_name. An example of + in Buildconf/elm.buildnml to build-namelist -elm_usr_name. An example of such a dataset would be 1x1pt_boulderCO_c090722. The default value is UNSET. This is an advanced flag and should only be used by expert users. @@ -217,34 +218,34 @@ off on - run_component_clm + run_component_elm env_run.xml Flag to the ELM build-namelist command to force ELM to do a cold start (finidat will be set to blanks). A value of on forces the model to spin up from a cold-start (arbitrary initial conditions). Setting this value in the xml file will take - precedence over any settings for finidat in the $CASEROOT/user_clm_clm file. + precedence over any settings for finidat in the $CASEROOT/user_elm file. - clm4.5 physics: - clm4.5 Satellite phenology: - clm4.5 Satellite phenology with black carbon deposition: - clm4.5 cn: - clm4.5 cn with dynamic vegetation: - clm4.5 bgc (cn and methane): - clm4.5 prognostic crop: - clm4.5 vic hydrology: - clm4.5 FATES (Functionally Assembled Terr. Ecosystem Simulator): (experimental) - clm4.5 BGC (CN with vertically resolved soil BGC, based on Century with Methane) with dynamic veg - alm1.0 C only, nutirent competition via relative demand, ctc soil cascade with black carbon deposition: - alm1.0 C-N, nutirent competition via relative demand, ctc soil cascade with black carbon deposition: - alm1.0 C-N-P, nutirent competition via relative demand, ctc soil cascade with black carbon deposition: - alm1.0 C-N, nutirent competition via equilibrium chemistry approximation, ctc soil cascade with black carbon deposition: - alm1.0 C-N-P, nutirent competition via equilibrium chemistry approximation, ctc soil cascade with black carbon deposition: - clm5.0 physics: - clm5.0 Satellite phenology: - clm5.0 bgc (cn and methane): + ELM with : + Satellite phenology : + Satellite phenology with black carbon deposition : + CN : + CN with dynamic vegetation : + BGC (CN and methane) : + prognostic crop : + VIC hydrology : + FATES (Functionally Assembled Terr. Ecosystem Simulator) (experimental) : + BGC (CN with vertically resolved soil BGC, based on Century with Methane) with dynamic veg : + C only, nutrient competition via relative demand, ctc soil cascade with black carbon deposition : + C-N, nutrient competition via relative demand, ctc soil cascade with black carbon deposition : + C-N-P, nutrient competition via relative demand, ctc soil cascade with black carbon deposition : + C-N, nutrient competition via equilibrium chemistry approximation, ctc soil cascade with black carbon deposition : + C-N, nutrient competition via equilibrium chemistry approximation, cnt soil cascade black carbon deposition : + C-N-P, nutrient competition via equilibrium chemistry approximation, ctc soil cascade with black carbon deposition : + C-N-P, nutrient competition via equilibrium chemistry approximation, cnt soil cascade with black carbon deposition : + C-N-P, nutrient competition via relative demand, ctc soil cascade with black carbon deposition and subgrid solar radiation parameterization : diff --git a/components/elm/cime_config/config_compsets.xml b/components/elm/cime_config/config_compsets.xml index 01366aa71dd9..1e38dc81277a 100644 --- a/components/elm/cime_config/config_compsets.xml +++ b/components/elm/cime_config/config_compsets.xml @@ -24,6 +24,16 @@ 1850_DATM%QIA_ELM%SPBC_SICE_SOCN_MOSART_SGLC_SWAV + + IERA5ELM + 2000_DATM%ERA5_ELM%SP_SICE_SOCN_MOSART_SGLC_SWAV + + + + IERA56HRELM + 2000_DATM%ERA56HR_ELM%SP_SICE_SOCN_MOSART_SGLC_SWAV + + IELM 2000_DATM%QIA_ELM%SP_SICE_SOCN_MOSART_SGLC_SWAV @@ -129,6 +139,11 @@ 1850_DATM%QIA_ELM%CNPRDCTCBC_SICE_SOCN_MOSART_SGLC_SWAV + + I1850CNPRDCTCBCTOP + 1850_DATM%QIA_ELM%CNPRDCTCBCTOP_SICE_SOCN_MOSART_SGLC_SWAV + + ICB1850CRDCTCBC 1850_SATM_ELM%CRDCTCBC_SICE_SOCN_MOSART_SGLC_SWAV @@ -532,6 +547,11 @@ 2000_DATM%GSWP3v1_ELM%CNPRDCTCBC_SICE_SOCN_MOSART_SGLC_SWAV + + IGSWCNPRDCTCBCTOP + 2000_DATM%GSWP3v1_ELM%CNPRDCTCBCTOP_SICE_SOCN_MOSART_SGLC_SWAV + + IGSWCNECACTCBC 2000_DATM%GSWP3v1_ELM%CNECACTCBC_SICE_SOCN_MOSART_SGLC_SWAV @@ -627,6 +647,11 @@ 1850_DATM%GSWP3v1_ELM%CNPRDCTCBC_SICE_SOCN_MOSART_SGLC_SWAV + + I1850GSWCNPRDCTCBCTOP + 1850_DATM%GSWP3v1_ELM%CNPRDCTCBCTOP_SICE_SOCN_MOSART_SGLC_SWAV + + I20TRGSWCNRDCTCBC 20TR_DATM%GSWP3v1_ELM%CNRDCTCBC_SICE_SOCN_MOSART_SGLC_SWAV @@ -637,6 +662,11 @@ 20TR_DATM%GSWP3v1_ELM%CNPRDCTCBC_SICE_SOCN_MOSART_SGLC_SWAV + + I20TRGSWCNPRDCTCBCTOP + 20TR_DATM%GSWP3v1_ELM%CNPRDCTCBCTOP_SICE_SOCN_MOSART_SGLC_SWAV + + @@ -773,7 +803,7 @@ IGELM_MLI - 2000_DATM%QIA_ELM%SP_SICE_SOCN_MOSART_MALI%SIA_SWAV + 2000_DATM%QIA_ELM%SP_SICE_SOCN_MOSART_MALI_SWAV diff --git a/components/elm/cime_config/config_pes.xml b/components/elm/cime_config/config_pes.xml index 86acd12be937..51a0a7355256 100644 --- a/components/elm/cime_config/config_pes.xml +++ b/components/elm/cime_config/config_pes.xml @@ -218,9 +218,9 @@ - + - elm: ascent|summit PEs for grid l%1.9x2.5|l%0.9x1.25|l%360x720cru + elm: ascent|summit|improv PEs for grid l%1.9x2.5|l%0.9x1.25|l%360x720cru -2 -2 @@ -509,9 +509,9 @@ - + - elm: ascent|summit PEs for grid l%r05_*r%r05 + elm: ascent|summit|improv PEs for grid l%r05_*r%r05 -2 -2 @@ -523,4 +523,109 @@ + + + + elm on chrysalis: any compset on ne4 grid, 3x32x2 NODESxMPIxOMP + 32 + 64 + + 96 + 96 + 96 + 96 + 96 + 96 + 96 + 96 + + + 2 + 2 + 2 + 2 + 2 + 2 + 2 + 2 + + + + + + + + GIS 20km (low-res) testing config + 128 + 128 + + 128 + 128 + 128 + 128 + 128 + 128 + 128 + 128 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + + + + + + GIS 1-to-10km (high-res) config + 128 + 128 + + 512 + 512 + 512 + 512 + 512 + 512 + 512 + 512 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/disableDynpftCheck/user_nl_elm b/components/elm/cime_config/testdefs/testmods_dirs/elm/disableDynpftCheck/user_nl_elm new file mode 100644 index 000000000000..1ae7de276d44 --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/disableDynpftCheck/user_nl_elm @@ -0,0 +1 @@ +check_dynpft_consistency = .false. diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fates/user_nl_elm b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates/user_nl_elm index 9b07b504564b..cd078b90950c 100644 --- a/components/elm/cime_config/testdefs/testmods_dirs/elm/fates/user_nl_elm +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates/user_nl_elm @@ -16,10 +16,8 @@ hist_fincl1 = 'FATES_NCOHORTS', 'FATES_TRIMMING', 'FATES_AREA_PLANTS', 'FATES_SAPWOODC', 'FATES_LEAFC', 'FATES_FROOTC', 'FATES_REPROC', 'FATES_STRUCTC', 'FATES_NONSTRUCTC', 'FATES_VEGC_ABOVEGROUND', 'FATES_CANOPY_VEGC', 'FATES_USTORY_VEGC', 'FATES_PRIMARY_PATCHFUSION_ERR', -'FATES_DISTURBANCE_RATE_P2P', 'FATES_DISTURBANCE_RATE_P2S', -'FATES_DISTURBANCE_RATE_S2S', 'FATES_DISTURBANCE_RATE_FIRE', +'FATES_HARVEST_CARBON_FLUX', 'FATES_DISTURBANCE_RATE_FIRE', 'FATES_DISTURBANCE_RATE_LOGGING', 'FATES_DISTURBANCE_RATE_TREEFALL', -'FATES_DISTURBANCE_RATE_POTENTIAL', 'FATES_HARVEST_CARBON_FLUX', 'FATES_STOMATAL_COND', 'FATES_LBLAYER_COND', 'FATES_NPP', 'FATES_GPP', 'FATES_AUTORESP', 'FATES_GROWTH_RESP', 'FATES_MAINT_RESP', 'FATES_GPP_CANOPY', 'FATES_AUTORESP_CANOPY', 'FATES_GPP_USTORY', 'FATES_AUTORESP_USTORY', @@ -28,4 +26,5 @@ hist_fincl1 = 'FATES_NCOHORTS', 'FATES_TRIMMING', 'FATES_AREA_PLANTS', 'FATES_NEP', 'FATES_HET_RESP', 'FATES_FIRE_CLOSS', 'FATES_FIRE_FLUX_EL', 'FATES_CBALANCE_ERROR', 'FATES_ERROR_EL', 'FATES_LEAF_ALLOC', 'FATES_SEED_ALLOC', 'FATES_STEM_ALLOC', 'FATES_FROOT_ALLOC', -'FATES_CROOT_ALLOC', 'FATES_STORE_ALLOC' +'FATES_CROOT_ALLOC', 'FATES_STORE_ALLOC', +'FATES_PATCHAREA_LU', 'FATES_DISTURBANCE_RATE_MATRIX_LULU' diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold/shell_commands b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold/shell_commands index 5c5dc3a9118a..c2771ff61c4a 100644 --- a/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold/shell_commands +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold/shell_commands @@ -1,2 +1,8 @@ ./xmlchange TEST_MEMLEAK_TOLERANCE=0.75 ./xmlchange NTHRDS=1 + +# Change PIO settings as temporary fix for #6316 +if [ `./xmlquery --value LND_GRID` == 1.9x2.5 ]; then + ./xmlchange PIO_NUMTASKS=4 + ./xmlchange PIO_STRIDE=-999 +fi diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_allvars/user_nl_elm b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_allvars/user_nl_elm index 67f363bb0a3e..2aff9c0b3c23 100644 --- a/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_allvars/user_nl_elm +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_allvars/user_nl_elm @@ -14,8 +14,7 @@ hist_fincl1 = 'FATES_CROWNAREA_PF', 'FATES_CANOPYCROWNAREA_PF', 'FATES_LAISHA_TOP_CL', 'FATES_FABD_SUN_CLLLPF', 'FATES_FABD_SHA_CLLLPF', 'FATES_FABI_SUN_CLLLPF', 'FATES_FABI_SHA_CLLLPF', 'FATES_FABD_SUN_CLLL', 'FATES_FABD_SHA_CLLL', 'FATES_FABI_SUN_CLLL', 'FATES_FABI_SHA_CLLL', -'FATES_PARPROF_DIR_CLLLPF', 'FATES_PARPROF_DIF_CLLLPF', -'FATES_PARPROF_DIR_CLLL', 'FATES_PARPROF_DIF_CLLL', 'FATES_FABD_SUN_TOPLF_CL', +'FATES_PARPROF_DIR_CLLLPF', 'FATES_PARPROF_DIF_CLLLPF','FATES_FABD_SUN_TOPLF_CL', 'FATES_FABD_SHA_TOPLF_CL', 'FATES_FABI_SUN_TOPLF_CL', 'FATES_FABI_SHA_TOPLF_CL', 'FATES_NET_C_UPTAKE_CLLL', 'FATES_CROWNAREA_CLLL', 'FATES_NPLANT_CANOPY_SZAP', 'FATES_NPLANT_USTORY_SZAP', 'FATES_DDBH_CANOPY_SZAP', 'FATES_DDBH_USTORY_SZAP', diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_landuse/include_user_mods b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_landuse/include_user_mods new file mode 100644 index 000000000000..45e8af32d22a --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_landuse/include_user_mods @@ -0,0 +1 @@ +../fates_cold diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_landuse/user_nl_elm b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_landuse/user_nl_elm new file mode 100644 index 000000000000..098d4fd33a38 --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_landuse/user_nl_elm @@ -0,0 +1,2 @@ +flanduse_timeseries = '$DIN_LOC_ROOT/lnd/clm2/surfdata_map/landuse.timeseries_4x5_hist_simyr1850-2015_200311.nc' +do_harvest = .true. diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_luh2/include_user_mods b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_luh2/include_user_mods new file mode 100644 index 000000000000..45e8af32d22a --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_luh2/include_user_mods @@ -0,0 +1 @@ +../fates_cold diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_luh2/shell_commands b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_luh2/shell_commands new file mode 100644 index 000000000000..e02188057ce6 --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_luh2/shell_commands @@ -0,0 +1 @@ +./xmlchange TEST_MEMLEAK_TOLERANCE=0.2 diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_luh2/user_nl_elm b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_luh2/user_nl_elm new file mode 100644 index 000000000000..854c21407f1a --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_luh2/user_nl_elm @@ -0,0 +1 @@ +use_fates_luh = .true. diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_twostream/README b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_twostream/README new file mode 100644 index 000000000000..8211a863c35c --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_twostream/README @@ -0,0 +1,19 @@ +Testing FATES two-stream radiation scheme is activated by switching the fates_rad_model +parameter from 1 to 2. This is all that is needed, both radiation schemes +1) Norman and 2) two-stream use the same optical parameters. + +fates_rad_model + +Note that to avoid exceeding the filename string length maximum, the parameter +file generated on the fly is placed in the $SRCROOT/src/fates/parameter_files +directory. This may still run into problems is the $SRCROOT string is too long. + +Like the test with seed dispersal activation, the main downside of this method is +that this file will require a custom update for every fates parameter file API update. +Allowing the HLM to generate the file at runtime via buildnamelist step +will provide the capability to build the fates parameter file on +the fly which with the appropriate values for this test. + +Note that the test as currently designed is not machine agnostic as it requires +specific shell commands for enabling the workflow to have access to ncgen. Currently +this test is only usable on perlmutter. diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_twostream/include_user_mods b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_twostream/include_user_mods new file mode 100644 index 000000000000..45e8af32d22a --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_twostream/include_user_mods @@ -0,0 +1 @@ +../fates_cold diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_twostream/shell_commands b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_twostream/shell_commands new file mode 100644 index 000000000000..e3f2fa482d66 --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_twostream/shell_commands @@ -0,0 +1,15 @@ +module load e4s +spack env activate gcc +spack load nco + +SRCDIR=`./xmlquery SRCROOT --value` +CASEDIR=`./xmlquery CASEROOT --value` +FATESDIR=$SRCDIR/components/elm/src/external_models/fates +FATESPARAMFILE=$CASEDIR/fates_params_twostream.nc + +ncgen -o $FATESPARAMFILE $FATESDIR/parameter_files/fates_params_default.cdl + +$FATESDIR/tools/modify_fates_paramfile.py --O --fin $FATESPARAMFILE --fout $FATESPARAMFILE --var fates_rad_model --val 2 --allpfts + +spack unload nco +module unload e4s diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_twostream/user_nl_elm b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_twostream/user_nl_elm new file mode 100644 index 000000000000..cae5fc2112da --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_twostream/user_nl_elm @@ -0,0 +1 @@ +fates_paramfile = '$CASEROOT/fates_params_twostream.nc' diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_eca/user_nl_elm b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_eca/user_nl_elm index 407d0dcecf2a..8dfdff1ab8eb 100644 --- a/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_eca/user_nl_elm +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_eca/user_nl_elm @@ -6,6 +6,8 @@ fates_parteh_mode = 2 use_lch4 = .true. fates_spitfire_mode = 1 hist_empty_htapes = .true. +suplphos = 'NONE' +suplnitro = 'NONE' hist_fincl1 = 'FATES_NCOHORTS', 'FATES_TRIMMING', 'FATES_AREA_PLANTS', 'FATES_AREA_TREES', 'FATES_COLD_STATUS', 'FATES_GDD', 'FATES_NCHILLDAYS', 'FATES_NCOLDDAYS', 'FATES_DAYSINCE_COLDLEAFOFF', @@ -19,10 +21,8 @@ hist_fincl1 = 'FATES_NCOHORTS', 'FATES_TRIMMING', 'FATES_AREA_PLANTS', 'FATES_SAPWOODC', 'FATES_LEAFC', 'FATES_FROOTC', 'FATES_REPROC', 'FATES_STRUCTC', 'FATES_NONSTRUCTC', 'FATES_VEGC_ABOVEGROUND', 'FATES_CANOPY_VEGC', 'FATES_USTORY_VEGC', 'FATES_PRIMARY_PATCHFUSION_ERR', -'FATES_DISTURBANCE_RATE_P2P', 'FATES_DISTURBANCE_RATE_P2S', -'FATES_DISTURBANCE_RATE_S2S', 'FATES_DISTURBANCE_RATE_FIRE', +'FATES_HARVEST_CARBON_FLUX', 'FATES_DISTURBANCE_RATE_FIRE', 'FATES_DISTURBANCE_RATE_LOGGING', 'FATES_DISTURBANCE_RATE_TREEFALL', -'FATES_DISTURBANCE_RATE_POTENTIAL', 'FATES_HARVEST_CARBON_FLUX', 'FATES_STOMATAL_COND', 'FATES_LBLAYER_COND', 'FATES_NPP', 'FATES_GPP', 'FATES_AUTORESP', 'FATES_GROWTH_RESP', 'FATES_MAINT_RESP', 'FATES_GPP_CANOPY', 'FATES_AUTORESP_CANOPY', 'FATES_GPP_USTORY', 'FATES_AUTORESP_USTORY', diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_long/user_nl_elm b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_long/user_nl_elm index 9b07b504564b..4812969a278b 100644 --- a/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_long/user_nl_elm +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_long/user_nl_elm @@ -16,10 +16,8 @@ hist_fincl1 = 'FATES_NCOHORTS', 'FATES_TRIMMING', 'FATES_AREA_PLANTS', 'FATES_SAPWOODC', 'FATES_LEAFC', 'FATES_FROOTC', 'FATES_REPROC', 'FATES_STRUCTC', 'FATES_NONSTRUCTC', 'FATES_VEGC_ABOVEGROUND', 'FATES_CANOPY_VEGC', 'FATES_USTORY_VEGC', 'FATES_PRIMARY_PATCHFUSION_ERR', -'FATES_DISTURBANCE_RATE_P2P', 'FATES_DISTURBANCE_RATE_P2S', -'FATES_DISTURBANCE_RATE_S2S', 'FATES_DISTURBANCE_RATE_FIRE', +'FATES_HARVEST_CARBON_FLUX', 'FATES_DISTURBANCE_RATE_FIRE', 'FATES_DISTURBANCE_RATE_LOGGING', 'FATES_DISTURBANCE_RATE_TREEFALL', -'FATES_DISTURBANCE_RATE_POTENTIAL', 'FATES_HARVEST_CARBON_FLUX', 'FATES_STOMATAL_COND', 'FATES_LBLAYER_COND', 'FATES_NPP', 'FATES_GPP', 'FATES_AUTORESP', 'FATES_GROWTH_RESP', 'FATES_MAINT_RESP', 'FATES_GPP_CANOPY', 'FATES_AUTORESP_CANOPY', 'FATES_GPP_USTORY', 'FATES_AUTORESP_USTORY', diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_rd/user_nl_elm b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_rd/user_nl_elm index 30a0dbb88cb0..1e72d72d5712 100644 --- a/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_rd/user_nl_elm +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_rd/user_nl_elm @@ -6,6 +6,8 @@ fates_parteh_mode = 2 use_lch4 = .true. fates_spitfire_mode = 1 hist_empty_htapes = .true. +suplphos = 'NONE' +suplnitro = 'NONE' hist_fincl1 = 'FATES_NCOHORTS', 'FATES_TRIMMING', 'FATES_AREA_PLANTS', 'FATES_AREA_TREES', 'FATES_COLD_STATUS', 'FATES_GDD', 'FATES_NCHILLDAYS', 'FATES_NCOLDDAYS', 'FATES_DAYSINCE_COLDLEAFOFF', @@ -19,10 +21,8 @@ hist_fincl1 = 'FATES_NCOHORTS', 'FATES_TRIMMING', 'FATES_AREA_PLANTS', 'FATES_SAPWOODC', 'FATES_LEAFC', 'FATES_FROOTC', 'FATES_REPROC', 'FATES_STRUCTC', 'FATES_NONSTRUCTC', 'FATES_VEGC_ABOVEGROUND', 'FATES_CANOPY_VEGC', 'FATES_USTORY_VEGC', 'FATES_PRIMARY_PATCHFUSION_ERR', -'FATES_DISTURBANCE_RATE_P2P', 'FATES_DISTURBANCE_RATE_P2S', -'FATES_DISTURBANCE_RATE_S2S', 'FATES_DISTURBANCE_RATE_FIRE', +'FATES_HARVEST_CARBON_FLUX', 'FATES_DISTURBANCE_RATE_FIRE', 'FATES_DISTURBANCE_RATE_LOGGING', 'FATES_DISTURBANCE_RATE_TREEFALL', -'FATES_DISTURBANCE_RATE_POTENTIAL', 'FATES_HARVEST_CARBON_FLUX', 'FATES_STOMATAL_COND', 'FATES_LBLAYER_COND', 'FATES_NPP', 'FATES_GPP', 'FATES_AUTORESP', 'FATES_GROWTH_RESP', 'FATES_MAINT_RESP', 'FATES_GPP_CANOPY', 'FATES_AUTORESP_CANOPY', 'FATES_GPP_USTORY', 'FATES_AUTORESP_USTORY', diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/gis20kmERS/shell_commands b/components/elm/cime_config/testdefs/testmods_dirs/elm/gis20kmERS/shell_commands new file mode 100644 index 000000000000..4547393d8728 --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/gis20kmERS/shell_commands @@ -0,0 +1,10 @@ +./xmlchange STOP_OPTION=ndays +./xmlchange STOP_N=1 +./xmlchange REST_OPTION=ndays +./xmlchange REST_N=1 +./xmlchange NCPL_BASE_PERIOD=year +./xmlchange ATM_NCPL=17520 +./xmlchange LND_NCPL=17520 +./xmlchange OCN_NCPL=8760 +./xmlchange GLC_NCPL=365 +./xmlchange ROF_NCPL=17520 diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/gis20kmSMS/shell_commands b/components/elm/cime_config/testdefs/testmods_dirs/elm/gis20kmSMS/shell_commands new file mode 100644 index 000000000000..adec2de26653 --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/gis20kmSMS/shell_commands @@ -0,0 +1,11 @@ +./xmlchange STOP_OPTION=ndays +./xmlchange STOP_N=2 +./xmlchange REST_OPTION=ndays +./xmlchange REST_N=2 +./xmlchange NCPL_BASE_PERIOD=year +./xmlchange ATM_NCPL=17520 +./xmlchange LND_NCPL=17520 +./xmlchange OCN_NCPL=8760 +./xmlchange GLC_NCPL=365 +./xmlchange ROF_NCPL=17520 + diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/multi_inst/shell_commands b/components/elm/cime_config/testdefs/testmods_dirs/elm/multi_inst/shell_commands new file mode 100755 index 000000000000..0cb920194bea --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/multi_inst/shell_commands @@ -0,0 +1 @@ +./xmlchange NINST_LND=2,NINST_ROF=2 diff --git a/components/elm/docs/index.md b/components/elm/docs/index.md index 27c44c3cac1a..3c9b7edd07b0 100644 --- a/components/elm/docs/index.md +++ b/components/elm/docs/index.md @@ -2,6 +2,6 @@ Some introductory text here -* The [ELM User's Guide](user-guide/index.md) explains how to control ELM when its running within E3SM and how to run in Coupler-bypass mode -* The [ELM Developer's Guide](dev-guide/index.md) explains ELM data structures and how to develop new code. -* The [ELM Techincal Guide](tech-guide/index.md) explains the science behind ELM's code +* The [ELM User Guide](user-guide/index.md) explains how to control ELM when its running within E3SM and how to run in Coupler-bypass mode +* The [ELM Developer Guide](dev-guide/index.md) explains ELM data structures and how to develop new code. +* The [ELM Technical Guide](tech-guide/index.md) explains the science behind ELM's code diff --git a/components/elm/docs/user-guide/fates.md b/components/elm/docs/user-guide/fates.md new file mode 100644 index 000000000000..390f714ee0c2 --- /dev/null +++ b/components/elm/docs/user-guide/fates.md @@ -0,0 +1,21 @@ +# FATES + +FATES (Functionally Assembled Terrestrial Ecosystem Simulator) is a numerical terrestrial ecosystem model +hat can be utilized as a component with various host land models such as ELM. It is incorporated as an +external module, the development of which is primarily supported by the Department of Energy’s Office of +Science, through the [Next Generation Ecosystem Experiment - Tropics (NGEE-T) project](https://ngee-tropics.lbl.gov/). + +The FATES code is hosted online through a github repository at [https://github.com/NGEET/fates](https://github.com/NGEET/fates). + +## Technical Document + +The full FATES documentation detailing the scientific underpinnings of the model can be found in the +[FATES Technical Document](https://fates-users-guide.readthedocs.io/projects/tech-doc/en/stable/) +hosted on ReadTheDocs. + +## User's and Developer's Guide + +The FATES team also provides a [User's guide](https://fates-users-guide.readthedocs.io/en/latest/user/users-guide.html) +and a [Developer's guide](https://fates-users-guide.readthedocs.io/en/latest/developer/developer-guide.html) +also hosted on ReadTheDocs. Both guides and the technical documentation can be reached from the top level +[User's Guide](https://fates-users-guide.readthedocs.io/en/latest/index.html) diff --git a/components/elm/docs/user-guide/index.md b/components/elm/docs/user-guide/index.md index 0ec3b413c3fb..7995e54acb36 100644 --- a/components/elm/docs/user-guide/index.md +++ b/components/elm/docs/user-guide/index.md @@ -1 +1,78 @@ -start of the ELM User's Guide +# ELM User's Guide + +This User's Guide describes how to set up and run ELM. + +## Steps to build and run ELM + +A step-by-step instruction on how to run fully coupled E3SM can be found [here](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/2309226536). Here we describe running ELM driven by atmospheric forcings provided via the data atmosphere (DATM) model for configurations that are used in the E3SM water cycle v3 campaign. + +The water cycle campaigns of E3SM v1 and v2 used ELM's satellite phenology mode (SP-mode) in which a prescribed leaf area index is used in ELM. However, the E3SM v3 water cycle campaign uses an interactive phenology by including an active biogeochemistry (BGC) cycle in ELM. Additionally, a parameterization of sub-grid topographical effects on solar radiation is included within ELM. + +### Scientifically supported compsets + +The land-only compsets are referred to as "I"-compset and are supported for the following time periods: pre-industrial (1850) and historical transient (20TR). Additionally, multiple atmospheric forcing datasets can be used to drive the ELM simulations. The supported compsets are: + +1. `I1850GSWCNPRDCTCBCTOP`: Pre-industrial ELM simulation using atmospheric forcings from GSWP3 reanalysis dataset +2. `I20TRGSWCNPRDCTCBCTOP`: Historical ELM simulation using GSWP atmospheric forcings with time varying greenhouse gas forcing and land use, land cover dataset (year 1850-2014). + +The E3SM coupler output forcings are most commonly used as an offline land model spinup step, in preparation for a fully-coupled E3SM experiment. + +### Supported grid + +The `r05_r05` is the supported grid resolution for performing offline ELM simulation. + +## Customizing runs + +Few useful changes to `user_nl_elm` + +### Changing monthly output file + +ELM by default outputs monthly history file in `*elm.h0.**.nc` files +that include many variables (>200). At times, many of the default output +variables may not be of interest, thus one could remove all default variables +(via `hist_empty_htapes`) and only include select variables (via `hist_fincl1`) +to the monthly history files by + +```fortran + &elm_inparm + hist_empty_htapes = .true. + hist_fincl1 = 'TG', 'TV', 'FSA' + / +``` + +#### Saving additional output files + +ELM can output additional history files (such as `*elm.h1.*.nc`, `*elm.h2.*.nc`) +that have different temporal averaging (e.g. daily, hourly, every model timestep) via +`hist_nhtfrq` where + +- `-24` corresponds to daily average +- `-1` corresponds to hourly average +- `0` corresponds to monthly average +- `1` corresponds to each model time step + +The number of time slices in these additional files can be controlled +vai `hist_mfilt`. + +```fortran + &elm_inparm + hist_fincl2 = 'TG' + hist_fincl3 = 'TV' + hist_fincl4 = 'TG', 'TV', 'FSA' + hist_nhtfrq = 0, -24, -1, 1 + hist_mfilt = 12, 30, 24, 48 + / +``` + +Using the above-mentioned settings: + +- Each `*.elm.h1.*.nc` will include 30 daily average values of `TG` +- Each `*.elm.h2.*.nc` will include 24 hourly average values of `TV` +- Each `*.elm.h3.*.nc` will include 48 values of `TG`, `TV`, and `FSA` at + each model time step, which is typically is 30 min. + +## Running with FATES + +[FATES](fates.md) can be run in various modes with ELM through the use of namelist settings. +The [FATES User's Guide section on namelist options](https://fates-users-guide.readthedocs.io/en/latest/user/Namelist-Options-and-Run-Time-Modes.html) +provides guidance on enabling these different FATES run modes. diff --git a/components/elm/mkdocs.yml b/components/elm/mkdocs.yml index 900934e34947..55a1c82b2a4e 100644 --- a/components/elm/mkdocs.yml +++ b/components/elm/mkdocs.yml @@ -2,6 +2,6 @@ site_name: ELM nav: - Introduction: 'index.md' - - Users's Guide: user-guide/index.md - - Developers's Guide: dev-guide/index.md + - User Guide: user-guide/index.md + - Developer Guide: dev-guide/index.md - Technical Guide: tech-guide/index.md diff --git a/components/elm/src/biogeochem/CNPBudgetMod.F90 b/components/elm/src/biogeochem/CNPBudgetMod.F90 index aeb033a1c257..e4f705f1fe5c 100644 --- a/components/elm/src/biogeochem/CNPBudgetMod.F90 +++ b/components/elm/src/biogeochem/CNPBudgetMod.F90 @@ -376,7 +376,7 @@ subroutine Reset(mode, budg_fluxL, budg_fluxG, budg_fluxN, budg_stateL, budg_sta ! integer :: year, mon, day, sec integer :: ip - character(*),parameter :: subName = '(WaterBudget_Reset) ' + character(*),parameter :: subName = '(Reset) ' if (.not.present(mode)) then call get_curr_date(year, mon, day, sec) @@ -470,7 +470,7 @@ subroutine CNPBudget_Restart(bounds, ncid, flag) type(file_desc_t), intent(inout) :: ncid ! netcdf id character(len=*) , intent(in) :: flag ! 'read' or 'write' ! - character(len=*),parameter :: subname = 'WaterBudget_Restart' + character(len=*),parameter :: subname = 'CNPBudget_Restart' select case (trim(flag)) case ('define') @@ -718,8 +718,8 @@ subroutine Sum0(f_size, s_size, budg_fluxL, budg_fluxG, budg_stateL, budg_stateG budg_fluxGtmp = 0._r8 budg_stateGtmp = 0._r8 - call shr_mpi_sum(budg_fluxL, budg_fluxGtmp, mpicom, subName) - call shr_mpi_sum(budg_stateL, budg_stateGtmp, mpicom, subName) + call shr_mpi_sum(budg_fluxL, budg_fluxGtmp, mpicom, subName, all=.true.) + call shr_mpi_sum(budg_stateL, budg_stateGtmp, mpicom, subName, all=.true.) budg_fluxG = budg_fluxG + budg_fluxGtmp @@ -946,7 +946,8 @@ subroutine Restart_Define(bounds, ncid, name) call ncd_defvar(varname=trim(name)//'_budg_fluxG', xtype=ncd_double, & dim1name=trim(name)//'_budg_flux', & - long_name=trim(name)//'_budg_fluxG', units='mm', ncid=ncid) + long_name=trim(name)//'_budg_fluxG', & + units='g'//trim(name)//'/m2/s', ncid=ncid) call ncd_defvar(varname=trim(name)//'_budg_fluxN', xtype=ncd_double, & dim1name=trim(name)//'_budg_flux', & @@ -954,7 +955,8 @@ subroutine Restart_Define(bounds, ncid, name) call ncd_defvar(varname=trim(name)//'_budg_stateG', xtype=ncd_double, & dim1name=trim(name)//'_budg_state', & - long_name=trim(name)//'_budg_stateG', units='-', ncid=ncid) + long_name=trim(name)//'_budg_stateG', & + units='g'//trim(name)//'/m2', ncid=ncid) end subroutine Restart_Define @@ -990,8 +992,8 @@ subroutine Restart_Write(bounds, ncid, flag, name, f_size, s_size, & budg_fluxGtmp = 0._r8 budg_stateGtmp = 0._r8 - call shr_mpi_sum(budg_fluxL, budg_fluxGtmp, mpicom, subName) - call shr_mpi_sum(budg_stateL, budg_stateGtmp, mpicom, subName) + call shr_mpi_sum(budg_fluxL, budg_fluxGtmp, mpicom, subName, all=.true.) + call shr_mpi_sum(budg_stateL, budg_stateGtmp, mpicom, subName, all=.true.) ! Copy data from 2D into 1D array count = 0 @@ -1191,37 +1193,28 @@ subroutine CBudget_SetEndingMonthlyStates(bounds, col_cs, grc_cs) type(gridcell_carbon_state), intent(inout) :: grc_cs ! ! !LOCAL VARIABLES: - integer :: year_prev, month_prev, day_prev, sec_prev - integer :: year_curr, month_curr, day_curr, sec_curr + integer :: year, month, day, sec !----------------------------------------------------------------------- associate( & begcb => col_cs%begcb , & ! Input : [real(r8) (:) ] carbon mass begining of the time step endcb => col_cs%endcb , & ! Input : [real(r8) (:) ] carbon mass begining of the time step - tcs_month_end_grc => grc_cs%tcs_month_beg & ! Output: [real(r8) (:) ] grid-level carbon mass at the begining of a month + tcs_month_end_grc => grc_cs%tcs_month_end & ! Output: [real(r8) (:) ] grid-level carbon mass at the ending of a month ) - ! Get current and previous dates to determine if a new month started - call get_prev_date(year_curr, month_curr, day_curr, sec_curr); - call get_prev_date(year_prev, month_prev, day_prev, sec_prev) - - ! If at the beginning of a simulation, save grid-level TCS based on - ! 'begcb' from the current time step - if ( day_curr == 1 .and. sec_curr == 0 .and. get_nstep() <= 1 ) then - call c2g( bounds, & - begcb(bounds%begc:bounds%endc), & - tcs_month_end_grc(bounds%begg:bounds%endg), & - c2l_scale_type= 'unity', l2g_scale_type='unity' ) - endif + ! Get current dates to determine if a new month started + call get_curr_date(year, month, day, sec); ! If multiple steps into a simulation and the last time step was the ! end of a month, save grid-level TCS based on 'endcb' from the last ! time step - if (get_nstep() > 1 .and. day_prev == 1 .and. sec_prev == 0) then + if (get_nstep() > 1 .and. day == 1 .and. sec == 0) then call c2g( bounds, & endcb(bounds%begc:bounds%endc), & tcs_month_end_grc(bounds%begg:bounds%endg), & c2l_scale_type= 'unity', l2g_scale_type='unity' ) + else + tcs_month_end_grc(bounds%begg:bounds%endg) = spval endif end associate diff --git a/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 b/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 index aa318b8d93a1..940b34eb41f2 100644 --- a/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 +++ b/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 @@ -528,7 +528,7 @@ subroutine ColNBalanceCheck(bounds, & write(iulog,*)'n_to_plant = ',sminn_to_plant(c)*dt write(iulog,*)'plant_to_litter = ',plant_to_litter_nflux(c)*dt if(crop_prog) then - write(iulog,*)'fertm = ',fert_to_sminn(c)*dt + write(iulog,*)'fertn = ',fert_to_sminn(c)*dt write(iulog,*)'soyfx = ',soyfixn_to_sminn(c)*dt endif write(iulog,*)'fire = ',col_fire_nloss(c)*dt @@ -681,6 +681,9 @@ subroutine ColPBalanceCheck(bounds, & ! calculate total column-level inputs col_pinputs(c) = primp_to_labilep(c) + supplement_to_sminp(c) + if (crop_prog) col_pinputs(c) = col_pinputs(c) + & + fert_p_to_sminp(c) + if(use_fates) then col_poutputs(c) = secondp_to_occlp(c) + sminp_leached(c) + sminp_to_plant(c) + biochem_pmin_to_plant(c) @@ -756,6 +759,9 @@ subroutine ColPBalanceCheck(bounds, & write(iulog,*)'supplement_to_sminp = ',supplement_to_sminp(c)*dt write(iulog,*)'secondp_to_occlp = ',secondp_to_occlp(c)*dt write(iulog,*)'sminp_leached = ',sminp_leached(c)*dt + if(crop_prog) then + write(iulog,*)'fertp = ',fert_p_to_sminp(c)*dt + endif if(use_fates)then write(iulog,*) 'plant_to_litter_flux = ',plant_to_litter_pflux(c)*dt write(iulog,*) 'biochem_pmin_to_plant = ',biochem_pmin_to_plant(c)*dt diff --git a/components/elm/src/biogeochem/PhosphorusDynamicsMod.F90 b/components/elm/src/biogeochem/PhosphorusDynamicsMod.F90 index 077474e2a86e..5a8918236f4a 100644 --- a/components/elm/src/biogeochem/PhosphorusDynamicsMod.F90 +++ b/components/elm/src/biogeochem/PhosphorusDynamicsMod.F90 @@ -605,8 +605,8 @@ subroutine PhosphorusBiochemMin_balance(bounds,num_soilc, filter_soilc, & km_ptase => veg_vp%km_ptase , & alpha_ptase => veg_vp%alpha_ptase , & decomp_ppools_vr_col => col_ps%decomp_ppools_vr, & + lamda_ptase => veg_vp%lamda_ptase, & ! critical value of nitrogen cost of phosphatase activity induced phosphorus uptake - lamda_ptase => veg_vp%lamda_ptase , & cn_scalar => cnstate_vars%cn_scalar , & cp_scalar => cnstate_vars%cp_scalar , & is_soil => decomp_cascade_con%is_soil) @@ -639,28 +639,31 @@ subroutine PhosphorusBiochemMin_balance(bounds,num_soilc, filter_soilc, & if(use_fates) then do j = 1,nlevdecomp - do p = 1, alm_fates%fates(ci)%bc_out(s)%num_plant_comps - lamda_up = alm_fates%fates(ci)%bc_out(s)%cp_scalar(p)/ & - max(alm_fates%fates(ci)%bc_out(s)%cn_scalar(p),1e-20_r8) - lamda_up = min(max(lamda_up,0.0_r8), 150.0_r8) + do p = 1, alm_fates%fates(ci)%bc_out(s)%num_plant_comps + pft = alm_fates%fates(ci)%bc_out(s)%ft_index(p) + fr_frac = alm_fates%fates(ci)%bc_out(s)%veg_rootc(p,j) / & - sum(alm_fates%fates(ci)%bc_out(s)%veg_rootc(p,:)) + sum(alm_fates%fates(ci)%bc_out(s)%veg_rootc(:,:)) + + ! lamda is not used currently with fates, fates also does not + ! scale by the cn_ and cp_scalars + ! Also, we do not currently allow phosphatase released P + ! to go directly to the plants in FATES. This implies as an alpha of 0. + ! lamda_ptase = alm_fates%fates(ci)%bc_pconst%eca_lambda_ptase(pft) - pft = alm_fates%fates(ci)%bc_out(s)%ft_index(p) ptase_tmp = alm_fates%fates(ci)%bc_pconst%eca_vmax_ptase(pft) * & - fr_frac * max(lamda_up - lamda_ptase, 0.0_r8) / & - ( alm_fates%fates(ci)%bc_pconst%eca_km_ptase(pft) + & - max(lamda_up - alm_fates%fates(ci)%bc_pconst%eca_lambda_ptase(pft), 0.0_r8)) + fr_frac/dzsoi_decomp(j) / ( alm_fates%fates(ci)%bc_pconst%eca_km_ptase(pft) + 1._r8) - biochem_pmin_to_plant_vr_patch(p,j) = ptase_tmp * alm_fates%fates(ci)%bc_pconst%eca_alpha_ptase(pft) - biochem_pmin_vr(c,j) = biochem_pmin_vr(c,j) + ptase_tmp*(1._r8 - alm_fates%fates(ci)%bc_pconst%eca_alpha_ptase(pft)) + biochem_pmin_to_plant_vr_patch(p,j) = 0._r8 + biochem_pmin_vr(c,j) = biochem_pmin_vr(c,j) + ptase_tmp !*(1._r8 - alm_fates%fates(ci)%bc_pconst%eca_alpha_ptase(pft)) biochem_pmin_to_ecosysp_vr_col_pot(c,j) = biochem_pmin_to_ecosysp_vr_col_pot(c,j) + ptase_tmp end do end do else + do j = 1,nlevdecomp do p = col_pp%pfti(c), col_pp%pftf(c) if (veg_pp%active(p).and. (veg_pp%itype(p) .ne. noveg)) then diff --git a/components/elm/src/biogeophys/BareGroundFluxesMod.F90 b/components/elm/src/biogeophys/BareGroundFluxesMod.F90 index 7932887d5bc9..83662758eba8 100644 --- a/components/elm/src/biogeophys/BareGroundFluxesMod.F90 +++ b/components/elm/src/biogeophys/BareGroundFluxesMod.F90 @@ -191,6 +191,7 @@ subroutine BareGroundFluxes(bounds, num_nolakeurbanp, filter_nolakeurbanp, & qflx_ev_h2osfc => veg_wf%qflx_ev_h2osfc , & ! Output: [real(r8) (:) ] evaporation flux from h2osfc (W/m**2) [+ to atm] qflx_evap_soi => veg_wf%qflx_evap_soi , & ! Output: [real(r8) (:) ] soil evaporation (mm H2O/s) (+ = to atm) qflx_evap_tot => veg_wf%qflx_evap_tot , & ! Output: [real(r8) (:) ] qflx_evap_soi + qflx_evap_can + qflx_tran_veg + num_iter => frictionvel_vars%num_iter_patch , & ! Output: number of iterations required begp => bounds%begp , & endp => bounds%endp & ) @@ -252,7 +253,7 @@ subroutine BareGroundFluxes(bounds, num_nolakeurbanp, filter_nolakeurbanp, & ! Initialize Monin-Obukhov length and wind speed call MoninObukIni(ur(p), thv(c), dthv, zldis(p), z0mg_patch(p), um(p), obu(p)) - + num_iter(p) = 0._r8 end do ! Perform stability iteration @@ -322,6 +323,7 @@ subroutine BareGroundFluxes(bounds, num_nolakeurbanp, filter_nolakeurbanp, & fn = 0 do f = 1, fnold p = filterp(f) + num_iter(p) = real(iter,r8) if (.not. (abs(tau_diff(p)) < dtaumin)) then fn = fn + 1 filterp(fn) = p diff --git a/components/elm/src/biogeophys/CanopyFluxesMod.F90 b/components/elm/src/biogeophys/CanopyFluxesMod.F90 index 0732f48d733a..6c96e70e172d 100755 --- a/components/elm/src/biogeophys/CanopyFluxesMod.F90 +++ b/components/elm/src/biogeophys/CanopyFluxesMod.F90 @@ -379,6 +379,7 @@ subroutine CanopyFluxes(bounds, num_nolakeurbanp, filter_nolakeurbanp, & z0hv => frictionvel_vars%z0hv_patch , & ! Output: [real(r8) (:) ] roughness length over vegetation, sensible heat [m] z0qv => frictionvel_vars%z0qv_patch , & ! Output: [real(r8) (:) ] roughness length over vegetation, latent heat [m] rb1 => frictionvel_vars%rb1_patch , & ! Output: [real(r8) (:) ] boundary layer resistance (s/m) + num_iter => frictionvel_vars%num_iter_patch , & ! Output: number of iterations required t_h2osfc => col_es%t_h2osfc , & ! Input: [real(r8) (:) ] surface water temperature t_soisno => col_es%t_soisno , & ! Input: [real(r8) (:,:) ] soil temperature (Kelvin) @@ -749,7 +750,8 @@ subroutine CanopyFluxes(bounds, num_nolakeurbanp, filter_nolakeurbanp, & ! Initialize Monin-Obukhov length and wind speed call MoninObukIni(ur(p), thv(c), dthv(p), zldis(p), z0mv(p), um(p), obu(p)) - + num_iter(p) = 0._r8 + end do ! Set counter for leaf temperature iteration (itlef) @@ -1157,6 +1159,7 @@ subroutine CanopyFluxes(bounds, num_nolakeurbanp, filter_nolakeurbanp, & dele(p) = abs(efe(p)-efeb(p)) efeb(p) = efe(p) det(p) = max(del(p),del2(p)) + num_iter(p) = real(itlef,r8) end do fnold = fn fn = 0 diff --git a/components/elm/src/biogeophys/FrictionVelocityType.F90 b/components/elm/src/biogeophys/FrictionVelocityType.F90 index 9ee8d586cd94..bd2f4458dd5a 100644 --- a/components/elm/src/biogeophys/FrictionVelocityType.F90 +++ b/components/elm/src/biogeophys/FrictionVelocityType.F90 @@ -41,7 +41,9 @@ module FrictionVelocityType real(r8), pointer :: z0mg_col (:) ! col roughness length over ground, momentum [m] real(r8), pointer :: z0hg_col (:) ! col roughness length over ground, sensible heat [m] real(r8), pointer :: z0qg_col (:) ! col roughness length over ground, latent heat [m] - + real(r8), pointer :: num_iter_patch (:) ! number of iterations performed to find a solution + ! to the land-energy flux balance in CanopyFluxes() + contains procedure, public :: Init @@ -102,6 +104,7 @@ subroutine InitAllocate(this, bounds) allocate(this%z0mv_patch (begp:endp)) ; this%z0mv_patch (:) = spval allocate(this%z0hv_patch (begp:endp)) ; this%z0hv_patch (:) = spval allocate(this%z0qv_patch (begp:endp)) ; this%z0qv_patch (:) = spval + allocate(this%num_iter_patch (begp:endp)) ; this%num_iter_patch (:) = spval allocate(this%z0mg_col (begc:endc)) ; this%z0mg_col (:) = spval allocate(this%z0qg_col (begc:endc)) ; this%z0qg_col (:) = spval allocate(this%z0hg_col (begc:endc)) ; this%z0hg_col (:) = spval @@ -209,6 +212,11 @@ subroutine InitHistory(this, bounds) ptr_patch=this%z0qv_patch, default='inactive') end if + this%num_iter_patch(begp:endp) = spval + call hist_addfld1d(fname='ITER_LND_EBAL_AVG', units='count', & + avgflag='A', long_name='average number of iterations performed in land-energy balance', & + ptr_patch=this%num_iter_patch, default = 'inactive') + end subroutine InitHistory !----------------------------------------------------------------------- diff --git a/components/elm/src/biogeophys/SnowHydrologyMod.F90 b/components/elm/src/biogeophys/SnowHydrologyMod.F90 index 62f2ba24242a..6354dd598b0c 100644 --- a/components/elm/src/biogeophys/SnowHydrologyMod.F90 +++ b/components/elm/src/biogeophys/SnowHydrologyMod.F90 @@ -571,14 +571,14 @@ subroutine SnowCompaction(bounds, num_snowc, filter_snowc, & ! parameters real(r8), parameter :: c2 = 23.e-3_r8 ! [m3/kg] real(r8), parameter :: c3 = 2.777e-6_r8 ! [1/s] - real(r8), parameter :: c3_ams = 5.8e-7_r8 ! Schneider et al., 2020 [1/s] + real(r8), parameter :: c3_ams = 0.83e-6_r8 ! Schneider et al.,(2021),Table 2 [1/s] real(r8), parameter :: c4 = 0.04_r8 ! [1/K] real(r8), parameter :: c5 = 2.0_r8 ! real(r8), parameter :: dm = 100.0_r8 ! Upper Limit on Destructive Metamorphism Compaction [kg/m3] - real(r8), parameter :: rho_dm = 150.0_r8 ! Upper limit on destructive metamorphism compaction [kg/m3] (Anderson, 1976; Schneider et al., 2020) + real(r8), parameter :: rho_dm = 150.0_r8 ! Upper limit on destructive metamorphism compaction [kg/m3] (Anderson, 1976;Schneider et al., 2021) real(r8), parameter :: eta0 = 9.e+5_r8 ! The Viscosity Coefficient Eta0 [kg-s/m2] - real(r8), parameter :: k_creep_snow = 1.4e-9_r8 ! Creep coefficient for snow (bi < 550 kg / m3) [m3-s/kg] - real(r8), parameter :: k_creep_firn = 1.2e-9_r8 ! Creep coefficient for firn (bi > 550 kg / m3) + real(r8), parameter :: k_creep_snow = 9.2e-9_r8 ! Creep coefficient for snow (bi < 550 kg / m3) [m3-s/kg] + real(r8), parameter :: k_creep_firn = 3.7e-9_r8 ! Creep coefficient for firn (bi > 550 kg / m3) [m3-s/kg] ! real(r8) :: p_gls ! grain load stress [kg / m-s2] real(r8) :: burden(bounds%begc:bounds%endc) ! pressure of overlying snow [kg/m2] @@ -693,12 +693,12 @@ subroutine SnowCompaction(bounds, num_snowc, filter_snowc, & ddz2 = (-k_creep_snow * (max(denice / bi, 1._r8) - 1._r8) * & exp(-60.e6_r8 / (rgas * t_soisno(c,j))) * p_gls) / & (snw_rds(c,j) * 1.e-6_r8 * snw_rds(c,j) * 1.e-6_r8) - & - 2.02e-10_r8 + 1.0e-10_r8 else ! High density, i.e. firn ddz2 = (-k_creep_firn * (max(denice / bi, 1._r8) - 1._r8) * & exp(-60.e6_r8 / (rgas * t_soisno(c,j))) * p_gls) / & (snw_rds(c,j) * 1.e-6_r8 * snw_rds(c,j) * 1.e-6_r8) - & - 2.7e-11_r8 + 1.0e-10_r8 endif endif diff --git a/components/elm/src/biogeophys/SnowSnicarMod.F90 b/components/elm/src/biogeophys/SnowSnicarMod.F90 index fae56d8927d7..d6d6de75ffaa 100644 --- a/components/elm/src/biogeophys/SnowSnicarMod.F90 +++ b/components/elm/src/biogeophys/SnowSnicarMod.F90 @@ -80,7 +80,7 @@ module SnowSnicarMod integer, parameter :: snw_rds_max_tbl = 1500 ! maximum effective radius defined in Mie lookup table [microns] integer, parameter :: snw_rds_min_tbl = 30 ! minimium effective radius defined in Mie lookup table [microns] real(r8), parameter :: snw_rds_max = 1500._r8 ! maximum allowed snow effective radius [microns] - real(r8), parameter :: snw_rds_refrz = 1000._r8 ! effective radius of re-frozen snow [microns] + real(r8) :: snw_rds_refrz = 1000._r8 ! effective radius of re-frozen snow [microns] !$acc declare copyin(snw_rds_max_tbl) !$acc declare copyin(snw_rds_min_tbl) !$acc declare copyin(snw_rds_max ) @@ -1441,6 +1441,12 @@ subroutine SnowAge_grain(bounds, & newsnow = max(0._r8, (qflx_snow_grnd_col(c_idx)*dtime)) endif + if (use_extrasnowlayers) then + snw_rds_refrz = 1500._r8 + else + snw_rds_refrz = 1000._r8 + endif + ! snow that has re-frozen [kg/m2] refrzsnow = max(0._r8, (qflx_snofrz_lyr(c_idx,i)*dtime)) diff --git a/components/elm/src/biogeophys/SurfaceAlbedoMod.F90 b/components/elm/src/biogeophys/SurfaceAlbedoMod.F90 index 2b3666b5157f..073c836c19bc 100644 --- a/components/elm/src/biogeophys/SurfaceAlbedoMod.F90 +++ b/components/elm/src/biogeophys/SurfaceAlbedoMod.F90 @@ -754,13 +754,16 @@ subroutine SurfaceAlbedo(bounds, & ws(p) = esai(p) / max( elai(p)+esai(p), mpe ) end do - do ib = 1, numrad - do fp = 1,num_vegsol - p = filter_vegsol(fp) - rho(p,ib) = max( rhol(veg_pp%itype(p),ib)*wl(p) + rhos(veg_pp%itype(p),ib)*ws(p), mpe ) - tau(p,ib) = max( taul(veg_pp%itype(p),ib)*wl(p) + taus(veg_pp%itype(p),ib)*ws(p), mpe ) + ! rho and tau are not needed if fates is on + if (.not. use_fates) then + do ib = 1, numrad + do fp = 1,num_vegsol + p = filter_vegsol(fp) + rho(p,ib) = max( rhol(veg_pp%itype(p),ib)*wl(p) + rhos(veg_pp%itype(p),ib)*ws(p), mpe ) + tau(p,ib) = max( taul(veg_pp%itype(p),ib)*wl(p) + taus(veg_pp%itype(p),ib)*ws(p), mpe ) + end do end do - end do + end if ! Diagnose number of canopy layers for radiative transfer, in increments of dincmax. ! Add to number of layers so long as cumulative leaf+stem area does not exceed total diff --git a/components/elm/src/biogeophys/WaterBudgetMod.F90 b/components/elm/src/biogeophys/WaterBudgetMod.F90 index a1e70e6cd8e4..cf42f1e63085 100644 --- a/components/elm/src/biogeophys/WaterBudgetMod.F90 +++ b/components/elm/src/biogeophys/WaterBudgetMod.F90 @@ -374,8 +374,8 @@ subroutine WaterBudget_Sum0() budg_fluxGtmp = 0._r8 budg_stateGtmp = 0._r8 - call shr_mpi_sum(budg_fluxL, budg_fluxGtmp, mpicom, subName) - call shr_mpi_sum(budg_stateL, budg_stateGtmp, mpicom, subName) + call shr_mpi_sum(budg_fluxL, budg_fluxGtmp, mpicom, subName, all=.true. ) + call shr_mpi_sum(budg_stateL, budg_stateGtmp, mpicom, subName, all=.true. ) budg_fluxG = budg_fluxG + budg_fluxGtmp budg_stateG = budg_stateGtmp @@ -563,7 +563,7 @@ subroutine WaterBudget_Restart_Define(bounds, ncid) call ncd_defvar(varname='budg_fluxG', xtype=ncd_double, & dim1name='budg_flux', & - long_name='budg_fluxG', units='mm', ncid=ncid) + long_name='budg_fluxG', units='kg/m2/s', ncid=ncid) call ncd_defvar(varname='budg_fluxN', xtype=ncd_double, & dim1name='budg_flux', & @@ -571,7 +571,7 @@ subroutine WaterBudget_Restart_Define(bounds, ncid) call ncd_defvar(varname='budg_stateG', xtype=ncd_double, & dim1name='budg_state', & - long_name='budg_stateG', units='mm', ncid=ncid) + long_name='budg_stateG', units='kg/m2', ncid=ncid) end subroutine WaterBudget_Restart_Define @@ -601,8 +601,8 @@ subroutine WaterBudget_Restart_Write(bounds, ncid, flag) budg_fluxGtmp = 0._r8 budg_stateGtmp = 0._r8 - call shr_mpi_sum(budg_fluxL, budg_fluxGtmp, mpicom, subName) - call shr_mpi_sum(budg_stateL, budg_stateGtmp, mpicom, subName) + call shr_mpi_sum(budg_fluxL, budg_fluxGtmp, mpicom, subName, all=.true.) + call shr_mpi_sum(budg_stateL, budg_stateGtmp, mpicom, subName, all=.true. ) ! Copy data from 2D into 1D array count = 0 diff --git a/components/elm/src/data_types/CNStateType.F90 b/components/elm/src/data_types/CNStateType.F90 index 9af48a224a67..2f880568f30d 100644 --- a/components/elm/src/data_types/CNStateType.F90 +++ b/components/elm/src/data_types/CNStateType.F90 @@ -1368,6 +1368,11 @@ subroutine Restart(this, bounds, ncid, flag) interpinic_flag='interp', readvar=readvar, data=ptr2d) end if + call restartvar(ncid=ncid, flag=flag, varname='nfire', xtype=ncd_double, & + dim1name='column', & + long_name='', units='', & + interpinic_flag='interp', readvar=readvar, data=this%nfire_col) + call restartvar(ncid=ncid, flag=flag, varname='fpg', xtype=ncd_double, & dim1name='column', & long_name='', units='', & diff --git a/components/elm/src/data_types/ColumnDataType.F90 b/components/elm/src/data_types/ColumnDataType.F90 index 32ebf37af051..2d79412e568c 100644 --- a/components/elm/src/data_types/ColumnDataType.F90 +++ b/components/elm/src/data_types/ColumnDataType.F90 @@ -18,7 +18,7 @@ module ColumnDataType use elm_varpar , only : nlevdecomp_full, crop_prog, nlevdecomp use elm_varpar , only : i_met_lit, i_cel_lit, i_lig_lit, i_cwd use elm_varcon , only : spval, ispval, zlnd, snw_rds_min, denice, denh2o, tfrz, pondmx - use elm_varcon , only : watmin, bdsno, zsoi, zisoi, dzsoi_decomp + use elm_varcon , only : watmin, bdsno, bdfirn, zsoi, zisoi, dzsoi_decomp use elm_varcon , only : c13ratio, c14ratio, secspday use elm_varctl , only : use_fates, use_fates_planthydro, create_glacier_mec_landunit use elm_varctl , only : use_hydrstress, use_crop @@ -28,6 +28,7 @@ module ColumnDataType use elm_varctl , only : hist_wrtch4diag, use_century_decomp use elm_varctl , only : get_carbontag, override_bgc_restart_mismatch_dump use elm_varctl , only : pf_hmode, nu_com + use elm_varctl , only : use_extrasnowlayers use elm_varctl , only : use_fan use ch4varcon , only : allowlakeprod use pftvarcon , only : VMAX_MINSURF_P_vr, KM_MINSURF_P_vr, pinit_beta1, pinit_beta2 @@ -1742,8 +1743,20 @@ subroutine col_ws_init(this, begc, endc, h2osno_input, snow_depth_input, watsat_ end do do j = -nlevsno+1, 0 if (j > col_pp%snl(c)) then - this%h2osoi_ice(c,j) = col_pp%dz(c,j)*250._r8 - this%h2osoi_liq(c,j) = 0._r8 + if (use_extrasnowlayers) then + ! amschnei@uci.edu: Initialize "deep firn" on glacier columns + if (lun_pp%itype(l) == istice .or. lun_pp%itype(l) == istice_mec) then + this%h2osoi_ice(c,j) = col_pp%dz(c,j)*bdfirn + this%h2osoi_liq(c,j) = 0._r8 + else + this%h2osoi_ice(c,j) = col_pp%dz(c,j)*bdsno + this%h2osoi_liq(c,j) = 0._r8 + end if + else ! no firn model (default in v2) + ! Below, "250._r8" should instead be "bdsno", which is 250 kg m^3 by default + this%h2osoi_ice(c,j) = col_pp%dz(c,j)*250._r8 + this%h2osoi_liq(c,j) = 0._r8 + end if end if end do end if @@ -2513,6 +2526,8 @@ subroutine col_cs_restart ( this, bounds, ncid, flag, carbon_type, c12_carbonst ! !DESCRIPTION: ! Read/Write column carbon state information to/from restart file. ! + use elm_varctl, only : do_budgets + ! ! !ARGUMENTS: class(column_carbon_state) :: this type(bounds_type), intent(in) :: bounds @@ -2544,6 +2559,13 @@ subroutine col_cs_restart ( this, bounds, ncid, flag, carbon_type, c12_carbonst integer :: decomp_cascade_state, restart_file_decomp_cascade_state !----------------------------------------------------------------------- + if (do_budgets) then + call restartvar(ncid=ncid, flag=flag, varname='ENDCB', xtype=ncd_double, & + dim1name='column', & + long_name='carbon balance at end of timestep', units='gC/m2', & + interpinic_flag='interp', readvar=readvar, data=this%endcb) + endif + if (carbon_type == 'c13' .or. carbon_type == 'c14') then if (.not. present(c12_carbonstate_vars)) then call endrun(msg=' ERROR: for C14 must pass in c12_carbonstate_vars as argument' //& @@ -5821,26 +5843,20 @@ subroutine col_wf_init(this, begc, endc) avgflag='A', long_name='column-integrated snow freezing rate', & ptr_col=this%qflx_snofrz, set_lake=spval, c2l_scale_type='urbanf', default='inactive') - if (create_glacier_mec_landunit) then - this%qflx_glcice(begc:endc) = spval - call hist_addfld1d (fname='QICE', units='mm/s', & - avgflag='A', long_name='ice growth/melt', & - ptr_col=this%qflx_glcice, l2g_scale_type='ice') - end if - - if (create_glacier_mec_landunit) then - this%qflx_glcice_frz(begc:endc) = spval - call hist_addfld1d (fname='QICE_FRZ', units='mm/s', & - avgflag='A', long_name='ice growth', & - ptr_col=this%qflx_glcice_frz, l2g_scale_type='ice') - end if + this%qflx_glcice(begc:endc) = spval + call hist_addfld1d (fname='QICE', units='mm/s', & + avgflag='A', long_name='ice growth/melt', & + ptr_col=this%qflx_glcice, l2g_scale_type='ice') - if (create_glacier_mec_landunit) then - this%qflx_glcice_melt(begc:endc) = spval - call hist_addfld1d (fname='QICE_MELT', units='mm/s', & - avgflag='A', long_name='ice melt', & - ptr_col=this%qflx_glcice_melt, l2g_scale_type='ice') - end if + this%qflx_glcice_frz(begc:endc) = spval + call hist_addfld1d (fname='QICE_FRZ', units='mm/s', & + avgflag='A', long_name='ice growth', & + ptr_col=this%qflx_glcice_frz, l2g_scale_type='ice') + + this%qflx_glcice_melt(begc:endc) = spval + call hist_addfld1d (fname='QICE_MELT', units='mm/s', & + avgflag='A', long_name='ice melt', & + ptr_col=this%qflx_glcice_melt, l2g_scale_type='ice') ! As defined here, snow_sources - snow_sinks will equal the change in h2osno at any ! given time step but only if there is at least one snow layer (for all landunits @@ -7755,6 +7771,9 @@ subroutine col_cf_setvalues ( this, num_column, filter_column, value_column) this%prod1c_loss(i) = value_column this%prod10c_loss(i) = value_column this%prod100c_loss(i) = value_column + this%er(i) = value_column + this%som_c_leached(i) = value_column + this%somc_yield(i) = value_column this%somhr(i) = value_column ! REVISIT this%lithr(i) = value_column ! REVISIT this%hr(i) = value_column diff --git a/components/elm/src/data_types/GridcellDataType.F90 b/components/elm/src/data_types/GridcellDataType.F90 index 5c312cdd035d..3b52a7a47af0 100644 --- a/components/elm/src/data_types/GridcellDataType.F90 +++ b/components/elm/src/data_types/GridcellDataType.F90 @@ -443,7 +443,7 @@ subroutine grc_ws_restart(this, bounds, ncid, flag) !----------------------------------------------------------------------- call restartvar(ncid=ncid, flag=flag, varname='TWS_MONTH_BEGIN', xtype=ncd_double, & dim1name='gridcell', & - long_name='surface watertotal water storage at the beginning of a month', units='mm', & + long_name='total water storage at the beginning of a month', units='mm', & interpinic_flag='interp', readvar=readvar, data=this%tws_month_beg) end subroutine grc_ws_restart @@ -603,12 +603,12 @@ subroutine grc_cs_init(this, begg, endg,carbon_type) end if this%tcs_month_beg(begg:endg) = spval - call hist_addfld1d (fname='TCS_MONTH_BEGIN', units='mm', & + call hist_addfld1d (fname='TCS_MONTH_BEGIN', units='gC/m^2', & avgflag='I', long_name='total carbon storage at the beginning of a month', & ptr_lnd=this%tcs_month_beg) this%tcs_month_end(begg:endg) = spval - call hist_addfld1d (fname='TCS_MONTH_END', units='mm', & + call hist_addfld1d (fname='TCS_MONTH_END', units='gC/m^2', & avgflag='I', long_name='total carbon storage at the end of a month', & ptr_lnd=this%tcs_month_end) @@ -644,9 +644,13 @@ subroutine grc_cs_restart(this, bounds, ncid, flag) !----------------------------------------------------------------------- call restartvar(ncid=ncid, flag=flag, varname='TCS_MONTH_BEGIN', xtype=ncd_double, & dim1name='gridcell', & - long_name='surface watertotal carbon storage at the beginning of a month', units='mm', & + long_name='total carbon storage at the beginning of a month', units='gC/m^2', & interpinic_flag='interp', readvar=readvar, data=this%tcs_month_beg) + call restartvar(ncid=ncid, flag=flag, varname='TCS_MONTH_END', xtype=ncd_double, & + dim1name='gridcell', & + long_name='total carbon storage at the end of a month', units='gC/m^2', & + interpinic_flag='interp', readvar=readvar, data=this%tcs_month_end) end subroutine grc_cs_restart !------------------------------------------------------------------------ diff --git a/components/elm/src/dyn_subgrid/dynFATESLandUseChangeMod.F90 b/components/elm/src/dyn_subgrid/dynFATESLandUseChangeMod.F90 new file mode 100644 index 000000000000..53be7987182e --- /dev/null +++ b/components/elm/src/dyn_subgrid/dynFATESLandUseChangeMod.F90 @@ -0,0 +1,197 @@ +module dynFATESLandUseChangeMod + +#include "shr_assert.h" + + !--------------------------------------------------------------------------- + ! !DESCRIPTION: + ! Handle reading of the land use harmonization (LUH2) dataset + + ! !USES: + use shr_kind_mod , only : r8 => shr_kind_r8 + use shr_log_mod , only : errMsg => shr_log_errMsg + use decompMod , only : bounds_type, BOUNDS_LEVEL_PROC + use abortutils , only : endrun + use dynFileMod , only : dyn_file_type + use dynVarTimeUninterpMod , only : dyn_var_time_uninterp_type + use elm_varcon , only : grlnd + use elm_varctl , only : iulog + + implicit none + + private + + real(r8), allocatable, public :: landuse_transitions(:,:) + real(r8), allocatable, public :: landuse_states(:,:) + + integer, public, parameter :: num_landuse_transition_vars = 108 + integer, public, parameter :: num_landuse_state_vars = 12 + + type(dyn_file_type), target :: dynFatesLandUse_file + + ! Land use name arrays + character(len=5), public, parameter :: landuse_state_varnames(num_landuse_state_vars) = & + [character(len=5) :: 'primf', & ! forested primary land + 'primn', & ! non-forested primary land + 'secdf', & ! potentially forested secondary land + 'secdn', & ! potentially non-forested secondary land + 'pastr', & ! managed pasture + 'range', & ! rangeland + 'urban', & ! urban land + 'c3ann', & ! C3 annual crops + 'c4ann', & ! C4 annual crops + 'c3per', & ! C3 perennial crops + 'c4per', & ! C4 perennial crops + 'c3nfx'] ! C3 nitrogen-fixing crops + + character(len=14), public, parameter :: landuse_transition_varnames(num_landuse_transition_vars) = & + [character(len=14) :: 'primf_to_secdn','primf_to_pastr','primf_to_range','primf_to_urban', & + 'primf_to_c3ann','primf_to_c4ann','primf_to_c3per','primf_to_c4per','primf_to_c3nfx', & + 'primn_to_secdf','primn_to_pastr','primn_to_range','primn_to_urban', & + 'primn_to_c3ann','primn_to_c4ann','primn_to_c3per','primn_to_c4per','primn_to_c3nfx', & + 'secdf_to_secdn','secdf_to_pastr','secdf_to_range','secdf_to_urban', & + 'secdf_to_c3ann','secdf_to_c4ann','secdf_to_c3per','secdf_to_c4per','secdf_to_c3nfx', & + 'secdn_to_secdf','secdn_to_pastr','secdn_to_range','secdn_to_urban', & + 'secdn_to_c3ann','secdn_to_c4ann','secdn_to_c3per','secdn_to_c4per','secdn_to_c3nfx', & + 'pastr_to_secdf','pastr_to_secdn','pastr_to_range','pastr_to_urban', & + 'pastr_to_c3ann','pastr_to_c4ann','pastr_to_c3per','pastr_to_c4per','pastr_to_c3nfx', & + 'range_to_secdf','range_to_secdn','range_to_pastr','range_to_urban', & + 'range_to_c3ann','range_to_c4ann','range_to_c3per','range_to_c4per','range_to_c3nfx', & + 'urban_to_secdf','urban_to_secdn','urban_to_pastr','urban_to_range', & + 'urban_to_c3ann','urban_to_c4ann','urban_to_c3per','urban_to_c4per','urban_to_c3nfx', & + 'c3ann_to_c4ann','c3ann_to_c3per','c3ann_to_c4per','c3ann_to_c3nfx', & + 'c3ann_to_secdf','c3ann_to_secdn','c3ann_to_pastr','c3ann_to_range','c3ann_to_urban', & + 'c4ann_to_c3ann','c4ann_to_c3per','c4ann_to_c4per','c4ann_to_c3nfx', & + 'c4ann_to_secdf','c4ann_to_secdn','c4ann_to_pastr','c4ann_to_range','c4ann_to_urban', & + 'c3per_to_c3ann','c3per_to_c4ann','c3per_to_c4per','c3per_to_c3nfx', & + 'c3per_to_secdf','c3per_to_secdn','c3per_to_pastr','c3per_to_range','c3per_to_urban', & + 'c4per_to_c3ann','c4per_to_c4ann','c4per_to_c3per','c4per_to_c3nfx', & + 'c4per_to_secdf','c4per_to_secdn','c4per_to_pastr','c4per_to_range','c4per_to_urban', & + 'c3nfx_to_c3ann','c3nfx_to_c4ann','c3nfx_to_c3per','c3nfx_to_c4per', & + 'c3nfx_to_secdf','c3nfx_to_secdn','c3nfx_to_pastr','c3nfx_to_range','c3nfx_to_urban'] + + type(dyn_var_time_uninterp_type) :: landuse_transition_vars(num_landuse_transition_vars) ! value of each landuse variable + type(dyn_var_time_uninterp_type) :: landuse_state_vars(num_landuse_state_vars) ! value of each landuse variable + + public :: dynFatesLandUseInit + public :: dynFatesLandUseInterp + +contains + + !----------------------------------------------------------------------- + subroutine dynFatesLandUseInit(bounds, landuse_filename) + + ! !DESCRIPTION: + ! Initialize data structures for land use information. + + ! !USES: + use elm_varctl , only : use_fates_luh + use dynVarTimeUninterpMod , only : dyn_var_time_uninterp_type + use dynTimeInfoMod , only : YEAR_POSITION_START_OF_TIMESTEP + use dynTimeInfoMod , only : YEAR_POSITION_END_OF_TIMESTEP + + ! !ARGUMENTS: + type(bounds_type), intent(in) :: bounds ! proc-level bounds + character(len=*) , intent(in) :: landuse_filename ! name of file containing land use information + + ! !LOCAL VARIABLES + integer :: varnum, i ! counter for harvest variables + integer :: landuse_shape(1) ! land use shape + integer :: num_points ! number of spatial points + integer :: ier ! error code + real(r8), allocatable :: this_data(:) ! data for a single harvest variable + ! + character(len=*), parameter :: subname = 'dynFatesLandUseInit' + !----------------------------------------------------------------------- + + SHR_ASSERT_ALL(bounds%level == BOUNDS_LEVEL_PROC, subname // ': argument must be PROC-level bounds') + + ! Allocate and initialize the land use arrays + allocate(landuse_states(num_landuse_state_vars,bounds%begg:bounds%endg),stat=ier) + if (ier /= 0) then + call endrun(msg=' allocation error for landuse_states'//errMsg(__FILE__, __LINE__)) + end if + allocate(landuse_transitions(num_landuse_transition_vars,bounds%begg:bounds%endg),stat=ier) + if (ier /= 0) then + call endrun(msg=' allocation error for landuse_transitions'//errMsg(__FILE__, __LINE__)) + end if + + landuse_states = 0._r8 + landuse_transitions = 0._r8 + + if (use_fates_luh) then + + ! Generate the dyn_file_type object. Note that the land use data being read in is for the + ! transitions occuring within the current year + dynFatesLandUse_file = dyn_file_type(landuse_filename, YEAR_POSITION_END_OF_TIMESTEP) + + ! Get initial land use data + num_points = (bounds%endg - bounds%begg + 1) + landuse_shape(1) = num_points ! Does this need an explicit array shape to be passed to the constructor? + do varnum = 1, num_landuse_transition_vars + landuse_transition_vars(varnum) = dyn_var_time_uninterp_type( & + dyn_file=dynFatesLandUse_file, varname=landuse_transition_varnames(varnum), & + dim1name=grlnd, conversion_factor=1.0_r8, & + do_check_sums_equal_1=.false., data_shape=landuse_shape) + end do + do varnum = 1, num_landuse_state_vars + landuse_state_vars(varnum) = dyn_var_time_uninterp_type( & + dyn_file=dynFatesLandUse_file, varname=landuse_state_varnames(varnum), & + dim1name=grlnd, conversion_factor=1.0_r8, & + do_check_sums_equal_1=.false., data_shape=landuse_shape) + end do + end if + + ! Since fates needs state data during initialization, make sure to call + ! the interpolation routine at the start + call dynFatesLandUseInterp(bounds,init_state=.true.) + + end subroutine dynFatesLandUseInit + + + !----------------------------------------------------------------------- + subroutine dynFatesLandUseInterp(bounds, init_state) + + use dynTimeInfoMod , only : time_info_type + + ! !ARGUMENTS: + type(bounds_type), intent(in) :: bounds ! proc-level bounds + logical, optional, intent(in) :: init_state ! fates needs state for initialization + + ! !LOCAL VARIABLES: + integer :: varnum + integer :: i + logical :: init_flag + real(r8), allocatable :: this_data(:) + character(len=*), parameter :: subname = 'dynFatesLandUseInterp' + !----------------------------------------------------------------------- + SHR_ASSERT_ALL(bounds%level == BOUNDS_LEVEL_PROC, subname // ': argument must be PROC-level bounds') + + init_flag = .false. + if (present(init_state)) then + init_flag = init_state + end if + + ! input land use data for current year are stored in year+1 in the file + call dynFatesLandUse_file%time_info%set_current_year_get_year(1) + + if (dynFatesLandUse_file%time_info%is_before_time_series() .and. .not.(init_flag)) then + ! Reset the land use transitions to zero for safety + landuse_transitions(1:num_landuse_transition_vars,bounds%begg:bounds%endg) = 0._r8 + landuse_states(1:num_landuse_state_vars,bounds%begg:bounds%endg) = 0._r8 + else + ! Right now we don't account for the topounits + allocate(this_data(bounds%begg:bounds%endg)) + do varnum = 1, num_landuse_transition_vars + call landuse_transition_vars(varnum)%get_current_data(this_data) + landuse_transitions(varnum,bounds%begg:bounds%endg) = this_data(bounds%begg:bounds%endg) + end do + do varnum = 1, num_landuse_state_vars + call landuse_state_vars(varnum)%get_current_data(this_data) + landuse_states(varnum,bounds%begg:bounds%endg) = this_data(bounds%begg:bounds%endg) + end do + deallocate(this_data) + end if + + end subroutine dynFatesLandUseInterp + +end module dynFATESLandUseChangeMod diff --git a/components/elm/src/dyn_subgrid/dynSubgridDriverMod.F90 b/components/elm/src/dyn_subgrid/dynSubgridDriverMod.F90 index 479ee5ab5518..8ac0cf4c45e7 100644 --- a/components/elm/src/dyn_subgrid/dynSubgridDriverMod.F90 +++ b/components/elm/src/dyn_subgrid/dynSubgridDriverMod.F90 @@ -153,7 +153,8 @@ subroutine dynSubgrid_driver(bounds_proc, & ! OUTSIDE any loops over clumps in the driver. ! ! !USES: - use elm_varctl , only : use_cn, create_glacier_mec_landunit, use_fates + use elm_varctl , only : use_cn, create_glacier_mec_landunit + use elm_varctl , only : use_fates, use_fates_luh use decompMod , only : bounds_type, get_proc_clumps, get_clump_bounds use decompMod , only : BOUNDS_LEVEL_PROC use dynInitColumnsMod , only : initialize_new_columns @@ -161,6 +162,9 @@ subroutine dynSubgrid_driver(bounds_proc, & use dynConsBiogeochemMod , only : dyn_cnbal_patch, dyn_cnbal_column use dynpftFileMod , only : dynpft_interp use dynHarvestMod , only : dynHarvest_interp_harvest_types + + use dynFATESLandUseChangeMod, only : dynFatesLandUseInterp + use dynEDMod , only : dyn_ED use reweightMod , only : reweight_wrapup use subgridWeightsMod , only : compute_higher_order_weights, set_subgrid_diagnostic_fields @@ -244,6 +248,10 @@ subroutine dynSubgrid_driver(bounds_proc, & call dynHarvest_interp_harvest_types(bounds_proc) end if + if (use_fates_luh) then + call dynFatesLandUseInterp(bounds_proc) + end if + ! ========================================================================== ! Do everything else related to land cover change ! ========================================================================== diff --git a/components/elm/src/external_models/fates b/components/elm/src/external_models/fates index 0b105bec7a90..42d804ba54d0 160000 --- a/components/elm/src/external_models/fates +++ b/components/elm/src/external_models/fates @@ -1 +1 @@ -Subproject commit 0b105bec7a90c5e02d1098f9bbdcf255f358aaa4 +Subproject commit 42d804ba54d0cf013a9737018ff9920e0c9808ea diff --git a/components/elm/src/external_models/sbetr b/components/elm/src/external_models/sbetr index 1fd5a14f903e..66260f4991d6 160000 --- a/components/elm/src/external_models/sbetr +++ b/components/elm/src/external_models/sbetr @@ -1 +1 @@ -Subproject commit 1fd5a14f903e789e7b86030ff564ff9ea5e0b058 +Subproject commit 66260f4991d61439d4cba92eb633590b09f97920 diff --git a/components/elm/src/main/controlMod.F90 b/components/elm/src/main/controlMod.F90 index 51e0b2898417..3b8c08be31bd 100755 --- a/components/elm/src/main/controlMod.F90 +++ b/components/elm/src/main/controlMod.F90 @@ -259,6 +259,8 @@ subroutine control_init( ) use_fates_fixed_biogeog, & use_fates_nocomp, & use_fates_sp, & + use_fates_luh, & + fluh_timeseries, & fates_parteh_mode, & fates_seeddisp_cadence, & use_fates_tree_damage @@ -800,6 +802,7 @@ subroutine control_spmd() call mpi_bcast (fates_spitfire_mode, 1, MPI_INTEGER, 0, mpicom, ier) call mpi_bcast (fates_paramfile, len(fates_paramfile) , MPI_CHARACTER, 0, mpicom, ier) + call mpi_bcast (fluh_timeseries, len(fluh_timeseries) , MPI_CHARACTER, 0, mpicom, ier) call mpi_bcast (use_fates_logging, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_fates_planthydro, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_fates_cohort_age_tracking, 1, MPI_LOGICAL, 0, mpicom, ier) @@ -807,6 +810,7 @@ subroutine control_spmd() call mpi_bcast (use_fates_fixed_biogeog, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_fates_nocomp, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_fates_sp, 1, MPI_LOGICAL, 0, mpicom, ier) + call mpi_bcast (use_fates_luh, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_fates_ed_prescribed_phys, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_fates_inventory_init, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (fates_inventory_ctrl_filename, len(fates_inventory_ctrl_filename), & @@ -1213,6 +1217,7 @@ subroutine control_print () write(iulog, *) ' fates_spitfire_mode = ', fates_spitfire_mode write(iulog, *) ' use_fates_logging = ', use_fates_logging write(iulog, *) ' fates_paramfile = ', fates_paramfile + write(iulog, *) ' fluh_timeseries = ', fluh_timeseries write(iulog, *) ' use_fates_planthydro = ', use_fates_planthydro write(iulog, *) ' use_fates_tree_damage = ', use_fates_tree_damage write(iulog, *) ' use_fates_cohort_age_tracking = ',use_fates_cohort_age_tracking @@ -1223,6 +1228,7 @@ subroutine control_print () write(iulog, *) ' use_fates_fixed_biogeog = ', use_fates_fixed_biogeog write(iulog, *) ' use_fates_nocomp = ', use_fates_nocomp write(iulog, *) ' use_fates_sp = ', use_fates_sp + write(iulog, *) ' use_fates_luh = ', use_fates_luh write(iulog, *) ' fates_inventory_ctrl_filename = ',fates_inventory_ctrl_filename write(iulog, *) ' fates_seeddisp_cadence = ', fates_seeddisp_cadence write(iulog, *) ' fates_seeddisp_cadence: 0, 1, 2, 3 => off, daily, monthly, or yearly dispersal' diff --git a/components/elm/src/main/elm_initializeMod.F90 b/components/elm/src/main/elm_initializeMod.F90 index 2a8079692379..86e72030b946 100755 --- a/components/elm/src/main/elm_initializeMod.F90 +++ b/components/elm/src/main/elm_initializeMod.F90 @@ -12,8 +12,8 @@ module elm_initializeMod use elm_varctl , only : nsrest, nsrStartup, nsrContinue, nsrBranch use elm_varctl , only : create_glacier_mec_landunit, iulog use elm_varctl , only : use_lch4, use_cn, use_voc, use_c13, use_c14 - use elm_varctl , only : use_fates, use_betr, use_fates_sp, use_fan - use elm_varsur , only : wt_lunit, urban_valid, wt_nat_patch, wt_cft, wt_glc_mec, topo_glc_mec,firrig,f_surf,f_grd + use elm_varctl , only : use_fates, use_betr, use_fates_sp, use_fan, use_fates_luh + use elm_varsur , only : wt_lunit, urban_valid, wt_nat_patch, wt_cft, wt_glc_mec, topo_glc_mec,firrig,f_surf,f_grd use elm_varsur , only : fert_cft, fert_p_cft use elm_varsur , only : wt_tunit, elv_tunit, slp_tunit,asp_tunit,num_tunit_per_grd use perf_mod , only : t_startf, t_stopf @@ -468,6 +468,7 @@ subroutine initialize2( ) use elm_time_manager , only : get_curr_date, get_nstep, advance_timestep use elm_time_manager , only : timemgr_init, timemgr_restart_io, timemgr_restart use controlMod , only : nlfilename + use controlMod , only : fluh_timeseries use decompMod , only : get_proc_clumps, get_proc_bounds, get_clump_bounds, bounds_type use domainMod , only : ldomain use initInterpMod , only : initInterp @@ -508,6 +509,7 @@ subroutine initialize2( ) use ELMFatesInterfaceMod , only: ELMFatesTimesteps use FATESFireFactoryMod , only : scalar_lightning use FanStreamMod , only : fanstream_init, fanstream_interp + use dynFATESLandUseChangeMod, only : dynFatesLandUseInit ! ! !ARGUMENTS implicit none @@ -734,6 +736,11 @@ subroutine initialize2( ) call dynSubgrid_init(bounds_proc, glc2lnd_vars, crop_vars) call t_stopf('init_dyn_subgrid') + ! Initialize fates LUH2 usage + if (use_fates_luh) then + call dynFatesLandUseInit(bounds_proc, fluh_timeseries) + end if + ! ------------------------------------------------------------------------ ! Initialize modules (after time-manager initialization in most cases) ! ------------------------------------------------------------------------ diff --git a/components/elm/src/main/elm_instMod.F90 b/components/elm/src/main/elm_instMod.F90 index ce5fb9bda4e4..b8afca87e241 100644 --- a/components/elm/src/main/elm_instMod.F90 +++ b/components/elm/src/main/elm_instMod.F90 @@ -262,7 +262,7 @@ subroutine elm_inst_biogeophys(bounds_proc) ! use shr_scam_mod , only : shr_scam_getCloseLatLon use landunit_varcon , only : istice, istice_mec, istsoil - use elm_varcon , only : h2osno_max, bdsno + use elm_varcon , only : h2osno_max, bdsno, bdfirn use domainMod , only : ldomain use elm_varpar , only : nlevsno, numpft use elm_varctl , only : single_column, fsurdat, scmlat, scmlon, use_extrasnowlayers @@ -318,8 +318,18 @@ subroutine elm_inst_biogeophys(bounds_proc) else h2osno_col(c) = 0._r8 endif - else ! With a deeper firn model, we can no longer depend on "h2osno_max," because this will - ! potentially be large, resulting in a lot of artificial firn at initialization. + snow_depth_col(c) = h2osno_col(c) / bdsno + else! With a deeper firn model, we can no longer depend on "h2osno_max," because this will + ! potentially be large, resulting in a lot of artificial firn at initialization. ! + ! amschnei@uci.edu: UPDATE - By including a new parameter, bdfirn, we can + ! initialize a dense (e.g, 730 kd m^-3) "deep firn" layer for glacier + ! land units. This deep firn layer can be set to half of the total + ! maximum allotted snowpack mass (i.e., "h2osno_max"), which effectively + ! sets the Greenland and Antarctic ice sheets' (and mountain glaciers') + ! climatic (aka surface) mass balance (SMB) initial condition to 0. With this + ! cold start condition, a multi-decadal (100 to 300 years or more) + ! spin up is first required to prognose nonzero SMB. + ! ! However... (below docstring from CLMv5) ! In areas that should be snow-covered, it can be problematic to start with 0 snow ! cover, because this can affect the long-term state through soil heating, albedo @@ -328,14 +338,18 @@ subroutine elm_inst_biogeophys(bounds_proc) ! feedback may not activate on time (or at all). So, as a compromise, we start with ! a small amount of snow in places that are likely to be snow-covered for much or ! all of the year. + ! amschnei@uci.edu: Initializing "deep firn" for glacier columns if (lun_pp%itype(l)==istice .or. lun_pp%itype(l)==istice_mec) then ! land ice (including multiple elevation classes, i.e. glacier_mec) - h2osno_col(c) = 50._r8 + h2osno_col(c) = 0.5_r8*h2osno_max ! start with half full snow column, representing deep firn + snow_depth_col(c) = h2osno_col(c) / bdfirn else if (lun_pp%itype(l)==istsoil .and. grc_pp%latdeg(g) >= 44._r8) then ! Northern hemisphere seasonal snow h2osno_col(c) = 50._r8 + snow_depth_col(c) = h2osno_col(c) / bdsno else h2osno_col(c) = 0._r8 + snow_depth_col(c) = h2osno_col(c) / bdsno endif endif snow_depth_col(c) = h2osno_col(c) / bdsno diff --git a/components/elm/src/main/elm_varcon.F90 b/components/elm/src/main/elm_varcon.F90 index 3a95fc05c52e..01df6ce0afd2 100644 --- a/components/elm/src/main/elm_varcon.F90 +++ b/components/elm/src/main/elm_varcon.F90 @@ -67,6 +67,7 @@ module elm_varcon real(r8) :: oneatm = 1.01325e5_r8 ! one standard atmospheric pressure [Pa] real(r8) :: bdsno = 250._r8 ! bulk density snow (kg/m**3) + real(r8) :: bdfirn = 730._r8 ! bulk density of deep firn (kg/m**3) real(r8) :: alpha_aero = 1.0_r8 ! constant for aerodynamic parameter weighting real(r8) :: tlsai_crit = 2.0_r8 ! critical value of elai+esai for which aerodynamic parameters are maximum real(r8) :: watmin = 0.01_r8 ! minimum soil moisture (mm) @@ -248,7 +249,7 @@ subroutine elm_varcon_init() allocate( dzsoifl(1:nlevsoifl )) if (use_extrasnowlayers) then - h2osno_max = 10000._r8 + h2osno_max = 30000._r8 end if end subroutine elm_varcon_init diff --git a/components/elm/src/main/elm_varctl.F90 b/components/elm/src/main/elm_varctl.F90 index e1d26775a5dc..d5a61da8fdbf 100644 --- a/components/elm/src/main/elm_varctl.F90 +++ b/components/elm/src/main/elm_varctl.F90 @@ -229,8 +229,10 @@ module elm_varctl logical, public :: use_fates_ed_st3 = .false. ! true => static stand structure logical, public :: use_fates_ed_prescribed_phys = .false. ! true => prescribed physiology logical, public :: use_fates_inventory_init = .false. ! true => initialize fates from inventory - logical, public :: use_fates_nocomp = .false. ! true => use no comopetition mode - logical, public :: use_fates_sp = .false. ! true => use FATES satellite phenology mode + logical, public :: use_fates_nocomp = .false. ! true => no competition mode + logical, public :: use_fates_sp = .false. ! true => FATES satellite phenology mode + logical, public :: use_fates_luh = .false. ! true => FATES land use transitions mode + character(len=256), public :: fluh_timeseries = '' ! filename for land use harmonization data character(len=256), public :: fates_inventory_ctrl_filename = '' ! filename for inventory control integer, public :: fates_parteh_mode = -9 ! 1 => carbon only ! 2 => C+N+P (not enabled yet) diff --git a/components/elm/src/main/elmfates_interfaceMod.F90 b/components/elm/src/main/elmfates_interfaceMod.F90 index a7aec9e85933..9b9fbc6e3903 100644 --- a/components/elm/src/main/elmfates_interfaceMod.F90 +++ b/components/elm/src/main/elmfates_interfaceMod.F90 @@ -55,6 +55,7 @@ module ELMFatesInterfaceMod use elm_varctl , only : use_fates_fixed_biogeog use elm_varctl , only : use_fates_nocomp use elm_varctl , only : use_fates_sp + use elm_varctl , only : use_fates_luh use elm_varctl , only : use_fates_tree_damage use elm_varctl , only : nsrest, nsrBranch use elm_varctl , only : fates_inventory_ctrl_filename @@ -73,7 +74,8 @@ module ELMFatesInterfaceMod use elm_varpar , only : nlevdecomp_full use elm_varpar , only : i_met_lit, i_cel_lit, i_lig_lit use elm_varpar , only : surfpft_lb, surfpft_ub - use elm_varpar , only : natpft_size, cft_size + use elm_varpar , only : natpft_size, max_patch_per_col, maxpatch_urb + use elm_varpar , only : numcft, maxpatch_urb use PhotosynthesisType , only : photosyns_type Use TopounitDataType , only : topounit_atmospheric_flux, topounit_atmospheric_state use atm2lndType , only : atm2lnd_type @@ -134,6 +136,7 @@ module ELMFatesInterfaceMod use FatesInterfaceMod , only : DetermineGridCellNeighbors use FatesHistoryInterfaceMod, only : fates_hist use FatesRestartInterfaceMod, only : fates_restart_interface_type + use FatesInterfaceTypesMod, only : hlm_num_luh2_states use PRTGenericMod , only : num_elements use FatesPatchMod , only : fates_patch_type @@ -146,7 +149,8 @@ module ELMFatesInterfaceMod use EDInitMod , only : init_patches use EDInitMod , only : set_site_properties use EDPftVarcon , only : EDpftvarcon_inst - use EDSurfaceRadiationMod , only : ED_SunShadeFracs, ED_Norman_Radiation + use FatesRadiationDriveMod, only : FatesSunShadeFracs + use FatesRadiationDriveMod, only : FatesNormalizedCanopyRadiation use EDBtranMod , only : btran_ed, & get_active_suction_layers use EDCanopyStructureMod , only : canopy_summarization, update_hlm_dynamics @@ -166,14 +170,17 @@ module ELMFatesInterfaceMod anthro_ignitions, anthro_suppression use dynHarvestMod , only : num_harvest_vars, harvest_varnames, wood_harvest_units - use dynHarvestMod , only : harvest_rates ! these are dynamic in space and time - + use dynHarvestMod , only : harvest_rates ! these are dynamic in space and time + use dynSubgridControlMod , only : get_do_harvest ! this gets the namelist value use FatesConstantsMod , only : hlm_harvest_area_fraction use FatesConstantsMod , only : hlm_harvest_carbon - use dynSubgridControlMod, only : get_do_harvest ! this gets the namelist value + use dynFATESLandUseChangeMod, only : num_landuse_transition_vars, num_landuse_state_vars + use dynFATESLandUseChangeMod, only : landuse_transitions, landuse_states + use dynFATESLandUseChangeMod, only : landuse_transition_varnames, landuse_state_varnames + use dynFATESLandUseChangeMod, only : dynFatesLandUseInterp - use FatesInterfaceTypesMod , only : bc_in_type, bc_out_type + use FatesInterfaceTypesMod , only : bc_in_type, bc_out_type use ELMFatesParamInterfaceMod, only : fates_param_reader_ctsm_impl use FatesParametersInterface, only : fates_param_reader_type @@ -351,7 +358,7 @@ subroutine ELMFatesGlobals1() call SetFatesGlobalElements1(use_fates,natpft_size,0,var_reader) natpft_size = fates_maxPatchesPerSite - + max_patch_per_col= max(natpft_size, numcft, maxpatch_urb) return end subroutine ELMFatesGlobals1 @@ -382,7 +389,9 @@ subroutine ELMFatesGlobals2() integer :: pass_num_lu_harvest_types integer :: pass_lu_harvest integer :: pass_tree_damage - + integer :: pass_use_luh + integer :: pass_num_luh_states + integer :: pass_num_luh_transitions ! ---------------------------------------------------------------------------------- ! FATES lightning definitions ! 1 : use a global constant lightning rate found in fates_params. @@ -507,11 +516,23 @@ subroutine ELMFatesGlobals2() pass_lu_harvest = 0 pass_num_lu_harvest_types = 0 end if - call set_fates_ctrlparms('use_lu_harvest',ival=pass_lu_harvest) call set_fates_ctrlparms('num_lu_harvest_cats',ival=pass_num_lu_harvest_types) call set_fates_ctrlparms('use_logging',ival=pass_logging) + if (use_fates_luh) then + pass_use_luh = 1 + pass_num_luh_states = num_landuse_state_vars + pass_num_luh_transitions = num_landuse_transition_vars + else + pass_use_luh = 0 + pass_num_luh_states = 0 + pass_num_luh_transitions = 0 + end if + call set_fates_ctrlparms('use_luh2',ival=pass_use_luh) + call set_fates_ctrlparms('num_luh2_states',ival=pass_num_luh_states) + call set_fates_ctrlparms('num_luh2_transitions',ival=pass_num_luh_transitions) + if(use_fates_ed_st3) then pass_ed_st3 = 1 else @@ -747,7 +768,9 @@ subroutine init(this, bounds_proc ) ndecomp = 1 end if - call allocate_bcin(this%fates(nc)%bc_in(s),col_pp%nlevbed(c),ndecomp,num_harvest_vars,surfpft_lb,surfpft_ub) + call allocate_bcin(this%fates(nc)%bc_in(s), col_pp%nlevbed(c), ndecomp, & + num_harvest_vars, num_landuse_state_vars, num_landuse_transition_vars, & + surfpft_lb, surfpft_ub) call allocate_bcout(this%fates(nc)%bc_out(s),col_pp%nlevbed(c),ndecomp) call zero_bcs(this%fates(nc),s) @@ -870,7 +893,7 @@ subroutine dynamics_driv(this, bounds_clump, top_as_inst, & top_af_inst, atm2lnd_inst, soilstate_inst, & canopystate_inst, frictionvel_inst, soil_water_retention_curve ) - use FatesConstantsMod , only : m2_per_km2 + use FatesConstantsMod , only : m2_per_km2 ! This wrapper is called daily from clm_driver ! This wrapper calls ed_driver, which is the daily dynamics component of FATES @@ -889,7 +912,7 @@ subroutine dynamics_driv(this, bounds_clump, top_as_inst, & class(soil_water_retention_curve_type), intent(in) :: soil_water_retention_curve ! !LOCAL VARIABLES: - integer :: s ! site index + integer :: s, i ! site index integer :: c ! column index (HLM) integer :: j ! Soil layer index integer :: t ! topounit index (HLM) @@ -905,6 +928,8 @@ subroutine dynamics_driv(this, bounds_clump, top_as_inst, & real(r8), pointer :: lnfm24(:) ! 24-hour averaged lightning data real(r8), pointer :: gdp_lf_col(:) ! gdp data + logical :: do_landuse_update ! local flag to pass transitions update to fates + !----------------------------------------------------------------------- @@ -1048,6 +1073,13 @@ subroutine dynamics_driv(this, bounds_clump, top_as_inst, & end if this%fates(nc)%bc_in(s)%site_area=col_pp%wtgcell(c)*grc_pp%area(g)*m2_per_km2 + if (use_fates_luh) then + this%fates(nc)%bc_in(s)%hlm_luh_states = landuse_states(:,g) + this%fates(nc)%bc_in(s)%hlm_luh_state_names = landuse_state_varnames + this%fates(nc)%bc_in(s)%hlm_luh_transitions = landuse_transitions(:,g) + this%fates(nc)%bc_in(s)%hlm_luh_transition_names = landuse_transition_varnames + end if + end do ! Nutrient uptake fluxes have been accumulating with each short @@ -1805,11 +1837,12 @@ subroutine init_coldstart(this, canopystate_inst, soilstate_inst, frictionvel_in real(r8) :: vol_ice real(r8) :: eff_porosity integer :: nlevsoil - integer :: j + integer :: j, i integer :: s - integer :: c + integer :: c, g integer :: p ! HLM patch index integer :: ft ! plant functional type + logical :: do_landuse_update ! local flag to pass transitions update to fates ! Set the FATES global time and date variables call GetAndSetTime @@ -1859,7 +1892,6 @@ subroutine init_coldstart(this, canopystate_inst, soilstate_inst, frictionvel_in if (use_fates_planthydro) then do s = 1,this%fates(nc)%nsites - c = this%f2hmap(nc)%fcolumn(s) nlevsoil = this%fates(nc)%bc_in(s)%nlevsoil @@ -1893,9 +1925,23 @@ subroutine init_coldstart(this, canopystate_inst, soilstate_inst, frictionvel_in call HydrSiteColdStart(this%fates(nc)%sites,this%fates(nc)%bc_in) end if + do s = 1,this%fates(nc)%nsites + c = this%f2hmap(nc)%fcolumn(s) + g = col_pp%gridcell(c) + + if (use_fates_luh) then + this%fates(nc)%bc_in(s)%hlm_luh_states = landuse_states(:,g) + this%fates(nc)%bc_in(s)%hlm_luh_state_names = landuse_state_varnames + this%fates(nc)%bc_in(s)%hlm_luh_transitions = landuse_transitions(:,g) + this%fates(nc)%bc_in(s)%hlm_luh_transition_names = landuse_transition_varnames + end if + end do + + ! Initialize patches call init_patches(this%fates(nc)%nsites, this%fates(nc)%sites, & this%fates(nc)%bc_in) + do s = 1,this%fates(nc)%nsites c = this%f2hmap(nc)%fcolumn(s) @@ -2010,7 +2056,7 @@ subroutine wrap_sunfrac(this,bounds_clump,top_af_inst,canopystate_inst) ! as well as total patch sun/shade fraction output boundary condition ! ------------------------------------------------------------------------------- - call ED_SunShadeFracs(this%fates(nc)%nsites, & + call FatesSunShadeFracs(this%fates(nc)%nsites, & this%fates(nc)%sites, & this%fates(nc)%bc_in, & this%fates(nc)%bc_out) @@ -2518,7 +2564,7 @@ subroutine wrap_canopy_radiation(this, bounds_clump, & end do end do - call ED_Norman_Radiation(this%fates(nc)%nsites, & + call FatesNormalizedCanopyRadiation(this%fates(nc)%nsites, & this%fates(nc)%sites, & this%fates(nc)%bc_in, & this%fates(nc)%bc_out) @@ -2946,6 +2992,7 @@ subroutine init_history_io(this,bounds_proc) use FatesIOVariableKindMod, only : site_coage_r8, site_coage_pft_r8 use FatesIOVariableKindMod, only : site_can_r8, site_cnlf_r8, site_cnlfpft_r8 use FatesIOVariableKindMod, only : site_cdpf_r8, site_cdsc_r8, site_cdam_r8 + use FatesIOVariableKindMod, only : site_landuse_r8, site_lulu_r8 use FatesIODimensionsMod, only : fates_bounds_type @@ -3044,7 +3091,8 @@ subroutine init_history_io(this,bounds_proc) site_can_r8,site_cnlf_r8, site_cnlfpft_r8, site_scag_r8, & site_scagpft_r8, site_agepft_r8, site_elem_r8, site_elpft_r8, & site_elcwd_r8, site_elage_r8, site_coage_r8, site_coage_pft_r8, & - site_agefuel_r8,site_cdsc_r8, site_cdpf_r8, site_cdam_r8) + site_agefuel_r8,site_cdsc_r8, site_cdpf_r8, site_cdam_r8, & + site_landuse_r8, site_lulu_r8) d_index = fates_hist%dim_kinds(dk_index)%dim2_index dim2name = fates_hist%dim_bounds(d_index)%name @@ -3311,6 +3359,7 @@ subroutine hlm_bounds_to_fates_bounds(hlm, fates) use EDParamsMod, only : nclmax_fates => nclmax use FatesInterfaceTypesMod, only : numpft_fates => numpft use FatesInterfaceTypesMod, only : nlevcoage + use FatesConstantsMod, only : n_landuse_cats implicit none @@ -3395,6 +3444,12 @@ subroutine hlm_bounds_to_fates_bounds(hlm, fates) fates%agefuel_begin = 1 fates%agefuel_end = nlevage_fates * nfsc_fates + fates%landuse_begin = 1 + fates%landuse_end = n_landuse_cats + + fates%lulu_begin = 1 + fates%lulu_end = n_landuse_cats * n_landuse_cats + end subroutine hlm_bounds_to_fates_bounds ! ====================================================================================== diff --git a/components/elm/src/main/histFileMod.F90 b/components/elm/src/main/histFileMod.F90 index 6e89d15ff192..c11eafe96f45 100644 --- a/components/elm/src/main/histFileMod.F90 +++ b/components/elm/src/main/histFileMod.F90 @@ -28,12 +28,13 @@ module histFileMod use FatesInterfaceTypesMod , only : nlevheight_fates => nlevheight use FatesInterfaceTypesMod , only : nlevdamage_fates => nlevdamage use FatesInterfaceTypesMod , only : nlevcoage - use FatesLitterMod , only : nfsc_fates => nfsc - use FatesLitterMod , only : ncwd_fates => ncwd + use FatesLitterMod , only : nfsc_fates => nfsc + use FatesConstantsMod , only : n_landuse_cats + use FatesLitterMod , only : ncwd_fates => ncwd use FatesInterfaceTypesMod , only : numpft_fates => numpft use PRTGenericMod , only : nelements_fates => num_elements - use TopounitType , only : top_pp - use topounit_varcon , only: max_topounits, has_topounit + use TopounitType , only : top_pp + use topounit_varcon , only: max_topounits, has_topounit ! implicit none @@ -1951,6 +1952,8 @@ subroutine htape_create (t, histrest) call ncd_defdim(lnfid, 'fates_levelcwd', nelements_fates * ncwd_fates, dimid) call ncd_defdim(lnfid, 'fates_levelage', nelements_fates * nlevage_fates, dimid) call ncd_defdim(lnfid, 'fates_levagefuel', nlevage_fates * nfsc_fates, dimid) + call ncd_defdim(lnfid, 'fates_levlanduse', n_landuse_cats, dimid) + call ncd_defdim(lnfid, 'fates_levlulu', n_landuse_cats * n_landuse_cats, dimid) end if if ( .not. lhistrest )then @@ -2353,6 +2356,7 @@ subroutine htape_timeconst(t, mode) use FatesInterfaceTypesMod, only : fates_hdim_camap_levcapf use FatesInterfaceTypesMod, only : fates_hdim_levage use FatesInterfaceTypesMod, only : fates_hdim_levpft + use FatesInterfaceTypesMod, only : fates_hdim_levlanduse use FatesInterfaceTypesMod, only : fates_hdim_scmap_levscag use FatesInterfaceTypesMod, only : fates_hdim_agmap_levscag use FatesInterfaceTypesMod, only : fates_hdim_levfuel @@ -2463,6 +2467,8 @@ subroutine htape_timeconst(t, mode) long_name='FATES patch age (yr)', ncid=nfid(t)) call ncd_defvar(varname='fates_levpft',xtype=ncd_int, dim1name='fates_levpft', & long_name='FATES pft number', ncid=nfid(t)) + call ncd_defvar(varname='fates_levlanduse',xtype=ncd_int, dim1name='fates_levlanduse', & + long_name='FATES land use label', ncid=nfid(t)) call ncd_defvar(varname='fates_levfuel',xtype=ncd_int, dim1name='fates_levfuel', & long_name='FATES fuel index', ncid=nfid(t)) call ncd_defvar(varname='fates_levcwdsc',xtype=ncd_int, dim1name='fates_levcwdsc', & @@ -2548,6 +2554,7 @@ subroutine htape_timeconst(t, mode) call ncd_io(varname='fates_scmap_levscpf',data=fates_hdim_scmap_levscpf, ncid=nfid(t), flag='write') call ncd_io(varname='fates_levage',data=fates_hdim_levage, ncid=nfid(t), flag='write') call ncd_io(varname='fates_levpft',data=fates_hdim_levpft, ncid=nfid(t), flag='write') + call ncd_io(varname='fates_levlanduse',data=fates_hdim_levlanduse, ncid=nfid(t), flag='write') call ncd_io(varname='fates_levfuel',data=fates_hdim_levfuel, ncid=nfid(t), flag='write') call ncd_io(varname='fates_levcwdsc',data=fates_hdim_levcwdsc, ncid=nfid(t), flag='write') call ncd_io(varname='fates_levcan',data=fates_hdim_levcan, ncid=nfid(t), flag='write') @@ -4828,6 +4835,10 @@ subroutine hist_addfld2d (fname, type2d, units, avgflag, long_name, type1d_out, num2d = nlevcoage*numpft_fates case ('fates_levpft') num2d = numpft_fates + case ('fates_levlanduse') + num2d = n_landuse_cats + case ('fates_levlulu') + num2d = n_landuse_cats * n_landuse_cats case ('fates_levage') num2d = nlevage_fates case ('fates_levfuel') diff --git a/components/elm/src/main/restFileMod.F90 b/components/elm/src/main/restFileMod.F90 index 18f3c523de76..58554b50974e 100644 --- a/components/elm/src/main/restFileMod.F90 +++ b/components/elm/src/main/restFileMod.F90 @@ -404,6 +404,8 @@ subroutine restFile_write( bounds, file, & call veg_ps%Restart(bounds, ncid, flag='write') call veg_pf%Restart(bounds, ncid, flag='write') call crop_vars%Restart(bounds, ncid, flag='write') + + call grc_cs%Restart(bounds, ncid, flag='write') end if @@ -627,6 +629,8 @@ subroutine restFile_read( bounds, file, & call veg_ps%Restart(bounds, ncid, flag='read') call veg_pf%Restart(bounds, ncid, flag='read') call crop_vars%Restart(bounds, ncid, flag='read') + + call grc_cs%Restart(bounds, ncid, flag='read') end if if (use_fates) then diff --git a/components/elm/tools/mksurfdata_map/mksurfdata.pl b/components/elm/tools/mksurfdata_map/mksurfdata.pl index 11bb0b39fe30..222a9c51bd15 100755 --- a/components/elm/tools/mksurfdata_map/mksurfdata.pl +++ b/components/elm/tools/mksurfdata_map/mksurfdata.pl @@ -344,7 +344,7 @@ ($) $opts{'soil_override'} = 1; } # Check if pft set - if ( defined($opts{'crop'}) ) { $numpft = 24; } # First set numpft if crop is on + if ( defined($opts{'crop'}) ) { $numpft = 50; } # First set numpft if crop is on if ( defined($opts{'pft_frc'}) || defined($opts{'pft_idx'}) ) { &check_pft( ); $opts{'pft_override'} = 1; @@ -598,7 +598,7 @@ ($) my $crpdes = ""; if ( $mkcrop ne "" ) { $options = "-options $mkcrop"; - $crpdes = "mp24_"; + $crpdes = "mp50_"; } my $landuse_timeseries_text_file; if ( $sim_year ne $sim_yr0 ) { diff --git a/components/elm/tools/mksurfdata_map/src/Srcfiles b/components/elm/tools/mksurfdata_map/src/Srcfiles index f10103ae083e..f8bda5dcf61b 100644 --- a/components/elm/tools/mksurfdata_map/src/Srcfiles +++ b/components/elm/tools/mksurfdata_map/src/Srcfiles @@ -27,6 +27,7 @@ mktopostatsMod.F90 mkVICparamsMod.F90 mkCH4inversionMod.F90 mkSedMod.F90 +mkFertMod.F90 nanMod.F90 shr_file_mod.F90 shr_sys_mod.F90 diff --git a/components/elm/tools/mksurfdata_map/src/mksurfdat.F90 b/components/elm/tools/mksurfdata_map/src/mksurfdat.F90 index 6dbf5f8b5383..d062907cdccf 100644 --- a/components/elm/tools/mksurfdata_map/src/mksurfdat.F90 +++ b/components/elm/tools/mksurfdata_map/src/mksurfdat.F90 @@ -763,8 +763,10 @@ program mksurfdat ero_c1_o=ero_c1, ero_c2_o=ero_c2, ero_c3_o=ero_c3, tillage_o=tillage, & litho_o=litho) - call mkfert(ldomain, mapfname=map_ffert, datfname=mksrf_ffert, ndiag=ndiag, & - nfert_o=nfert, pfert_o=pfert) + if(num_cft > 0) then + call mkfert(ldomain, mapfname=map_ffert, datfname=mksrf_ffert, ndiag=ndiag, & + nfert_o=nfert, pfert_o=pfert) + end if do n = 1,ns_o diff --git a/components/elm/tools/mksurfdata_map/src/mkvarctl.F90 b/components/elm/tools/mksurfdata_map/src/mkvarctl.F90 index c02b22838cb4..48e745833749 100644 --- a/components/elm/tools/mksurfdata_map/src/mkvarctl.F90 +++ b/components/elm/tools/mksurfdata_map/src/mkvarctl.F90 @@ -52,7 +52,7 @@ module mkvarctl character(len=256), public :: mksrf_fslp10 = ' ' ! slope percentile file name character(len=256), public :: mksrf_fero = ' ' ! ELM-Erosion parameters data file name character(len=256), public :: mksrf_ffert = ' ' ! crop fertilizer data file name - integer , public :: numpft = 50 ! number of plant types + integer , public :: numpft = 16 ! number of plant types character(len=256), public :: map_fpft = ' ' ! Mapping file for PFT character(len=256), public :: map_flakwat = ' ' ! Mapping file for lake water diff --git a/components/homme/CMakeLists.txt b/components/homme/CMakeLists.txt index c016182e3bf0..fdb996fa17af 100644 --- a/components/homme/CMakeLists.txt +++ b/components/homme/CMakeLists.txt @@ -447,18 +447,22 @@ if(HOMME_BUILD_EXECS AND NOT BUILD_HOMME_WITHOUT_PIOLIBRARY) ENDIF () ENDIF () -# If we don't need kokkos we don't need EKAT, and if -# Homme is built in EAMxx EKAT is already built -IF (HOMME_USE_KOKKOS AND HOMME_STANDALONE) - # Add ekat's cmake/pkg_build folder to cmake path +IF (HOMME_USE_KOKKOS) + # Add ekat's cmake scripts folders to cmake path set (EKAT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../externals/ekat) set (EKAT_CMAKE_PATH ${EKAT_SOURCE_DIR}/cmake) list(APPEND CMAKE_MODULE_PATH - ${EKAT_CMAKE_PATH} - ${EKAT_CMAKE_PATH}/pkg_build - ${EKAT_CMAKE_PATH}/tpls + ${EKAT_CMAKE_PATH} + ${EKAT_CMAKE_PATH}/tpls ) - include (EkatBuildKokkos) + + # We first try to use find_package. If that doesn't work, build from EKAT's submodule + include (EkatFindKokkos) + if (NOT Kokkos_FOUND) + # The following script checks if Kokkos is already available as a target, and if so does nothing. + # For instance, if HOMME is built inside EAMxx, Kokkos will already be available + include (EkatBuildKokkos) + endif() ENDIF () # This folder contains the CMake macro used to build cxx unit tests diff --git a/components/homme/cmake/SetCompilerFlags.cmake b/components/homme/cmake/SetCompilerFlags.cmake index 49e79a463c41..9a37c6a36bd9 100644 --- a/components/homme/cmake/SetCompilerFlags.cmake +++ b/components/homme/cmake/SetCompilerFlags.cmake @@ -68,12 +68,6 @@ IF (${HOMME_USE_CXX}) SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g") INCLUDE(CheckCXXCompilerFlag) - CHECK_CXX_COMPILER_FLAG("-std=c++14" CXX14_SUPPORTED) - IF (CXX14_SUPPORTED) - SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14") - ELSEIF (${HOMME_USE_KOKKOS}) - MESSAGE (FATAL_ERROR "Kokkos needs C++14, but the C++ compiler does not support it.") - ENDIF () CHECK_CXX_COMPILER_FLAG("-cxxlib" CXXLIB_SUPPORTED) IF (CXXLIB_SUPPORTED) SET(CXXLIB_SUPPORTED_CACHE TRUE CACHE BOOL "") diff --git a/components/homme/src/preqx_kokkos/CMakeLists.txt b/components/homme/src/preqx_kokkos/CMakeLists.txt index 425e7c34da76..dff42bb97c66 100644 --- a/components/homme/src/preqx_kokkos/CMakeLists.txt +++ b/components/homme/src/preqx_kokkos/CMakeLists.txt @@ -168,6 +168,8 @@ MACRO(PREQX_KOKKOS_SETUP) ${SRC_SHARE_DIR}/cxx/EulerStepFunctor.cpp ${SRC_SHARE_DIR}/cxx/ExecSpaceDefs.cpp ${SRC_SHARE_DIR}/cxx/FunctorsBuffersManager.cpp + ${SRC_SHARE_DIR}/cxx/GllFvRemap.cpp + ${SRC_SHARE_DIR}/cxx/GllFvRemapImpl.cpp ${SRC_SHARE_DIR}/cxx/Hommexx_Session.cpp ${SRC_SHARE_DIR}/cxx/HybridVCoord.cpp ${SRC_SHARE_DIR}/cxx/HyperviscosityFunctor.cpp diff --git a/components/homme/src/preqx_kokkos/config.h.cmake.in b/components/homme/src/preqx_kokkos/config.h.cmake.in index c0bc4c311c39..119844f3def0 100644 --- a/components/homme/src/preqx_kokkos/config.h.cmake.in +++ b/components/homme/src/preqx_kokkos/config.h.cmake.in @@ -58,9 +58,8 @@ /* ZOLTAN2 SUBPACKAGE OF TRILINOS library */ #cmakedefine01 TRILINOS_HAVE_ZOLTAN2 -/* When doing BFB testing, we occasionally must use modified code. */ -/* Use this flag to protect such code. */ -#cmakedefine HOMMEXX_BFB_TESTING +/* Detect whether this is a kokkos target */ +#cmakedefine01 KOKKOS_TARGET /* Whether to use OpenMP4 */ #cmakedefine OMP4 diff --git a/components/homme/src/share/compose/cedr_util.hpp b/components/homme/src/share/compose/cedr_util.hpp index 52a1a4d77b5b..6d5ca7fecf57 100644 --- a/components/homme/src/share/compose/cedr_util.hpp +++ b/components/homme/src/share/compose/cedr_util.hpp @@ -4,11 +4,12 @@ #ifndef INCLUDE_CEDR_UTIL_HPP #define INCLUDE_CEDR_UTIL_HPP -#include - #include "cedr_kokkos.hpp" #include "cedr_mpi.hpp" +#include // For std::stringstream +#include // For std::cerr + namespace cedr { namespace util { diff --git a/components/homme/src/share/compose/compose_slmm.cpp b/components/homme/src/share/compose/compose_slmm.cpp index b8277b5423c6..27ab1e1a7d04 100644 --- a/components/homme/src/share/compose/compose_slmm.cpp +++ b/components/homme/src/share/compose/compose_slmm.cpp @@ -276,8 +276,6 @@ extern "C" { // Interface for Homme, through compose_mod.F90. void kokkos_init () { amb::dev_init_threads(); - Kokkos::InitArguments args; - args.disable_warnings = true; initialize_kokkos(); // Test these initialize correctly. Kokkos::View v("hi"); diff --git a/components/homme/src/share/compose/compose_slmm_siqk.cpp b/components/homme/src/share/compose/compose_slmm_siqk.cpp index 5f8c4474df17..628c023090cb 100644 --- a/components/homme/src/share/compose/compose_slmm_siqk.cpp +++ b/components/homme/src/share/compose/compose_slmm_siqk.cpp @@ -75,7 +75,7 @@ class TestSphereToRefKernel { } KOKKOS_INLINE_FUNCTION - void join (volatile value_type& dst, volatile value_type const& src) const { + void join (value_type& dst, value_type const& src) const { dst.max_nits = max(dst.max_nits, src.max_nits); dst.sum_nits += src.sum_nits; dst.nfails += src.nfails; diff --git a/components/homme/src/share/cxx/ExecSpaceDefs.cpp b/components/homme/src/share/cxx/ExecSpaceDefs.cpp index 784d37b65d20..8d496bff5d16 100644 --- a/components/homme/src/share/cxx/ExecSpaceDefs.cpp +++ b/components/homme/src/share/cxx/ExecSpaceDefs.cpp @@ -187,7 +187,7 @@ team_num_threads_vectors (const int num_parallel_iterations, #elif defined(KOKKOS_ENABLE_HIP) // Use 64 wavefronts per CU and 120 CUs. const int num_warps_device = 120*64; // no such thing Kokkos::Impl::hip_internal_maximum_warp_count(); - const int num_threads_warp = Kokkos::Experimental::Impl::HIPTraits::WarpSize; + const int num_threads_warp = Kokkos::Impl::HIPTraits::WarpSize; #else // I want thread-distribution rules to be unit-testable even when GPU spaces // are off. Thus, make up a GPU-like machine: diff --git a/components/homme/src/share/cxx/ExecSpaceDefs.hpp b/components/homme/src/share/cxx/ExecSpaceDefs.hpp index 8c18d8bcbb92..fbcd314cb2b9 100644 --- a/components/homme/src/share/cxx/ExecSpaceDefs.hpp +++ b/components/homme/src/share/cxx/ExecSpaceDefs.hpp @@ -143,11 +143,7 @@ struct DefaultThreadsDistribution { team_num_threads_vectors(const int num_parallel_iterations, const ThreadPreferences tp = ThreadPreferences()) { return Parallel::team_num_threads_vectors_from_pool( -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE - ExecSpaceType::thread_pool_size() -#else - ExecSpaceType::impl_thread_pool_size() -#endif + ExecSpaceType().impl_thread_pool_size() , num_parallel_iterations, tp); } }; diff --git a/components/homme/src/share/cxx/HybridVCoord.cpp b/components/homme/src/share/cxx/HybridVCoord.cpp index 569eef1d6849..199b6bc4f739 100644 --- a/components/homme/src/share/cxx/HybridVCoord.cpp +++ b/components/homme/src/share/cxx/HybridVCoord.cpp @@ -158,8 +158,8 @@ void HybridVCoord::random_init(int seed) { Errors::runtime_check(curr>prev,"Error! hybrid_a+hybrid_b is not increasing.\n", -1); - host_hybrid_am_real(i-1) = (host_hybrid_ai(i) + host_hybrid_ai(i))/2.0; - host_hybrid_bm_real(i-1) = (host_hybrid_bi(i) + host_hybrid_bi(i))/2.0; + host_hybrid_am_real(i-1) = (host_hybrid_ai(i) + host_hybrid_ai(i-1))/2.0; + host_hybrid_bm_real(i-1) = (host_hybrid_bi(i) + host_hybrid_bi(i-1))/2.0; } Kokkos::deep_copy(hybrid_ai, host_hybrid_ai); diff --git a/components/homme/src/share/prim_driver_base.F90 b/components/homme/src/share/prim_driver_base.F90 index 27df23dba4d4..df83e1cc386b 100644 --- a/components/homme/src/share/prim_driver_base.F90 +++ b/components/homme/src/share/prim_driver_base.F90 @@ -1613,16 +1613,17 @@ subroutine applyCAMforcing_tracers(elem,hvcoord,np1,np1_qdp,dt,adjustment) #ifdef MODEL_THETA_L if (dt_remap_factor==0) then - adjust_ps=.true. ! stay on reference levels for Eulerian case + adjust_ps=.true. ! stay on reference levels for Eulerian case else -#ifdef SCREAM - adjust_ps=.false. ! Lagrangian case can support adjusting dp3d or ps -#else - adjust_ps=.true. ! Lagrangian case can support adjusting dp3d or ps -#endif + adjust_ps=.false. ! Lagrangian case can support adjusting dp3d or ps endif #else - adjust_ps=.true. ! preqx requires forcing to stay on reference levels + adjust_ps=.true. ! preqx requires forcing to stay on reference levels +#endif + +#if defined(CAM) && !defined(SCREAM) + adjust_ps=.true. ! Special case when CAM is defined, and SCREAM is not defined, + ! require forcing to stay on reference levels no matter dt_remap_factor #endif dp=elem%state%dp3d(:,:,:,np1) diff --git a/components/eam/src/dynamics/se/semoab_mod.F90 b/components/homme/src/share/semoab_mod.F90 similarity index 92% rename from components/eam/src/dynamics/se/semoab_mod.F90 rename to components/homme/src/share/semoab_mod.F90 index 984b9793df40..2d8bb099ade3 100644 --- a/components/eam/src/dynamics/se/semoab_mod.F90 +++ b/components/homme/src/share/semoab_mod.F90 @@ -12,12 +12,10 @@ module semoab_mod use dimensions_mod, only: nelem, ne, np, nelemd, nlev use element_mod, only : element_t - use parallel_mod, only : parallel_t + use parallel_mod, only : parallel_t, abortmp use m_MergeSorts, only: IndexSet, IndexSort - use cam_grid_support, only: iMap - use cam_abortutils, only : endrun use edgetype_mod, only: edgedescriptor_t use gridgraph_mod, only: gridvertex_t @@ -209,8 +207,10 @@ subroutine create_moab_meshes(par, elem) if ( nelemd > 0 ) then allocate(moab_vert_coords(3*nverts) ) allocate(vdone(nverts)) - vdone = 0; + else + allocate(vdone(1)) ! will be passed to iMOAB_ResolveSharedEnts, and compilers are complaining about non-allocated arrays endif + vdone = 0; if ( nelemd > 0 ) currentval = gdofv( indx(1)) ! start over to identify coordinates of the vertices do ix=1,moab_dim_cquads @@ -251,7 +251,7 @@ subroutine create_moab_meshes(par, elem) if ( nelemd > 0 ) then ierr = iMOAB_CreateVertices(MHFID, dimcoord, dimen, moab_vert_coords) if (ierr > 0 ) & - call endrun('Error: fail to create MOAB vertices ') + call abortmp('Error: fail to create MOAB vertices ') endif !!num_el = nelemd2 mbtype = 3 ! quadrilateral @@ -261,7 +261,7 @@ subroutine create_moab_meshes(par, elem) if ( nelemd > 0 ) then ierr = iMOAB_CreateElements( MHFID, nelemd2, mbtype, nve, moabconn, block_ID ); if (ierr > 0 ) & - call endrun('Error: fail to create MOAB elements') + call abortmp('Error: fail to create MOAB elements') endif ! nverts: num vertices; vdone will store now the markers used in global resolve ! for this particular problem, markers are the global dofs at corner nodes @@ -272,18 +272,19 @@ subroutine create_moab_meshes(par, elem) numco = 1 ierr = iMOAB_DefineTagStorage(MHFID, tagname, tagtype, numco, tagindex ) if (ierr > 0 ) & - call endrun('Error: fail to retrieve global id tag') + call abortmp('Error: fail to retrieve global id tag') ! now set the values ent_type = 0 ! vertex type if ( nverts > 0 ) then ierr = iMOAB_SetIntTagStorage ( MHFID, tagname, nverts , ent_type, vdone) if (ierr > 0 ) & - call endrun('Error: fail to set marker id tag for vertices') + call abortmp('Error: fail to set marker id tag for vertices') endif + ! we need to call this even when no mesh locally, it involves a collective ierr = iMOAB_ResolveSharedEntities( MHFID, nverts, vdone ); if (ierr > 0 ) & - call endrun('Error: fail to resolve shared entities') + call abortmp('Error: fail to resolve shared entities') if ( nelemd > 0) then vdone = -1 ! reuse vdone for the new tag, GLOBAL_ID (actual tag that we want to store global dof ) @@ -294,7 +295,7 @@ subroutine create_moab_meshes(par, elem) newtagg='GLOBAL_ID'//C_NULL_CHAR ierr = iMOAB_DefineTagStorage(MHFID, newtagg, tagtype, numco, tagindex ) if (ierr > 0 ) & - call endrun('Error: fail to create new GDOF tag') + call abortmp('Error: fail to create new GDOF tag') do ie=1,nelemd do ii=1,elem(ie)%idxp%NumUniquePts i=elem(ie)%idxp%ia(ii) @@ -310,19 +311,19 @@ subroutine create_moab_meshes(par, elem) if ( nverts > 0 ) then ierr = iMOAB_SetIntTagStorage ( MHFID, newtagg, nverts , ent_type, vdone) if (ierr > 0 ) & - call endrun('Error: fail to set global dof tag for vertices') + call abortmp('Error: fail to set global dof tag for vertices') endif ierr = iMOAB_ReduceTagsMax ( MHFID, tagindex, ent_type) if (ierr > 0 ) & - call endrun('Error: fail to reduce max tag') + call abortmp('Error: fail to reduce max tag') ! set global id tag for elements ent_type = 1 ! now set the global id tag on elements if ( nelemd2 > 0 ) then ierr = iMOAB_SetIntTagStorage ( MHFID, newtagg, nelemd2 , ent_type, elemids) if (ierr > 0 ) & - call endrun('Error: fail to set global id tag for elements') + call abortmp('Error: fail to set global id tag for elements') endif ! now, after reduction, we can get the actual global ids for each vertex in the fine mesh @@ -333,18 +334,18 @@ subroutine create_moab_meshes(par, elem) allocate(vgids(nverts)) ierr = iMOAB_GetIntTagStorage ( MHFID, newtagg, nverts , ent_type, vgids) if (ierr > 0 ) & - call endrun('Error: fail to retrieve GLOBAL ID on each task') + call abortmp('Error: fail to retrieve GLOBAL ID on each task') endif ierr = iMOAB_UpdateMeshInfo(MHFID) if (ierr > 0 ) & - call endrun('Error: fail to update mesh info') + call abortmp('Error: fail to update mesh info') #ifdef MOABDEBUG ! write out the mesh file to disk, in parallel outfile = 'wholeFineATM.h5m'//C_NULL_CHAR wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR ierr = iMOAB_WriteMesh(MHFID, outfile, wopts) if (ierr > 0 ) & - call endrun('Error: fail to write the mesh file') + call abortmp('Error: fail to write the mesh file') #endif @@ -422,7 +423,7 @@ subroutine create_moab_meshes(par, elem) if ( nverts_c > 0 ) then ierr = iMOAB_CreateVertices(MHID, dimcoord, dimen, moab_vert_coords) if (ierr > 0 ) & - call endrun('Error: fail to create MOAB vertices ') + call abortmp('Error: fail to create MOAB vertices ') endif ! num_el = nelemd mbtype = 3 ! quadrilateral @@ -432,7 +433,7 @@ subroutine create_moab_meshes(par, elem) if ( nelemd > 0 ) then ierr = iMOAB_CreateElements( MHID, nelemd, mbtype, nve, moabconn_c, block_ID ); if (ierr > 0 ) & - call endrun('Error: fail to create MOAB elements') + call abortmp('Error: fail to create MOAB elements') endif ! idx: num vertices; vdone will store now the markers used in global resolve ! for this particular problem, markers are the global dofs at corner nodes @@ -443,28 +444,28 @@ subroutine create_moab_meshes(par, elem) numco = 1 ierr = iMOAB_DefineTagStorage(MHID, tagname, tagtype, numco, tagindex ) if (ierr > 0 ) & - call endrun('Error: fail to retrieve GDOFV id tag') + call abortmp('Error: fail to retrieve GDOFV id tag') ierr = iMOAB_DefineTagStorage(MHID, newtagg, tagtype, numco, tagindex ) if (ierr > 0 ) & - call endrun('Error: fail to retrieve GLOBAL_ID tag on coarse mesh') + call abortmp('Error: fail to retrieve GLOBAL_ID tag on coarse mesh') ! now set the values ent_type = 0 ! vertex type if ( nverts_c > 0 ) then ierr = iMOAB_SetIntTagStorage ( MHID, tagname, nverts_c , ent_type, vdone_c) if (ierr > 0 ) & - call endrun('Error: fail to set GDOFV tag for vertices') + call abortmp('Error: fail to set GDOFV tag for vertices') endif ! set global id tag for coarse elements, too; they will start at nets=1, end at nete=nelemd ent_type = 1 ! now set the global id tag on elements if ( nelemd > 0 ) then ierr = iMOAB_SetIntTagStorage ( MHID, newtagg, nelemd , ent_type, elemids) if (ierr > 0 ) & - call endrun('Error: fail to set global id tag for vertices') + call abortmp('Error: fail to set global id tag for vertices') endif ierr = iMOAB_ResolveSharedEntities( MHID, idx, vdone_c ); if (ierr > 0 ) & - call endrun('Error: fail to resolve shared entities') + call abortmp('Error: fail to resolve shared entities') ! global dofs are the GLL points are set for each element tagname='GLOBAL_DOFS'//C_NULL_CHAR @@ -472,7 +473,7 @@ subroutine create_moab_meshes(par, elem) numco = np*np ! usually, it is 16; each element will have the dofs in order ierr = iMOAB_DefineTagStorage(MHID, tagname, tagtype, numco, tagindex ) if (ierr > 0 ) & - call endrun('Error: fail to create global DOFS tag') + call abortmp('Error: fail to create global DOFS tag') ! now set the values ! set global dofs tag for coarse elements, too; they will start at nets=1, end at nete=nelemd ent_type = 1 ! now set the global id tag on elements @@ -499,7 +500,7 @@ subroutine create_moab_meshes(par, elem) if ( nelemd > 0 ) then ierr = iMOAB_SetIntTagStorage ( MHID, tagname, numvals, ent_type, gdofel) if (ierr > 0 ) & - call endrun('Error: fail to set globalDOFs tag for coarse elements') + call abortmp('Error: fail to set globalDOFs tag for coarse elements') endif ! set the global ids for coarse vertices the same as corresponding fine vertices @@ -507,19 +508,19 @@ subroutine create_moab_meshes(par, elem) if ( nverts_c > 0 ) then ierr = iMOAB_SetIntTagStorage ( MHID, newtagg, nverts_c , ent_type, vdone_c) if (ierr > 0 ) & - call endrun('Error: fail to set GLOBAL_DOFS tag values') + call abortmp('Error: fail to set GLOBAL_DOFS tag values') endif ierr = iMOAB_UpdateMeshInfo(MHID) if (ierr > 0 ) & - call endrun('Error: fail to update mesh info') + call abortmp('Error: fail to update mesh info') #ifdef MOABDEBUG ! write out the mesh file to disk, in parallel outfile = 'wholeATM.h5m'//C_NULL_CHAR wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR ierr = iMOAB_WriteMesh(MHID, outfile, wopts) if (ierr > 0 ) & - call endrun('Error: fail to write the mesh file') + call abortmp('Error: fail to write the mesh file') #endif if (fv_nphys > 0 ) then @@ -629,6 +630,8 @@ subroutine create_moab_meshes(par, elem) ! reuse moab_vert_coords for coordinates of pg mesh ! the first nverts_c coords are the same as coarse mesh; but we do create new allocate(vdone_pg(nverts_pg)) + else + allocate(vdone_pg(1)) endif do iv = 1, nverts_c vdone_pg(iv) = vdone_c(iv) ! also the coordinates will be the same !! @@ -737,7 +740,7 @@ subroutine create_moab_meshes(par, elem) if ( nverts_pg > 0 ) then ierr = iMOAB_CreateVertices(MHPGID, dimcoord, dimen, moab_vert_coords) if (ierr > 0 ) & - call endrun('Error: fail to create MOAB vertices ') + call abortmp('Error: fail to create MOAB vertices ') endif ! num_el = nelem_pg * mbtype = 3 ! quadrilateral @@ -747,44 +750,45 @@ subroutine create_moab_meshes(par, elem) if ( nelem_pg > 0 ) then ierr = iMOAB_CreateElements( MHPGID, nelem_pg, mbtype, nve, moabconn_pg, block_ID ); if (ierr > 0 ) & - call endrun('Error: fail to create MOAB elements') + call abortmp('Error: fail to create MOAB elements') endif tagname='GLOBAL_ID'//C_NULL_CHAR tagtype = 0 ! dense, integer numco = 1 ierr = iMOAB_DefineTagStorage(MHPGID, tagname, tagtype, numco, tagindex ) if (ierr > 0 ) & - call endrun('Error: fail to retrieve GLOBAL id tag') + call abortmp('Error: fail to retrieve GLOBAL id tag') ! now set the values ent_type = 0 ! vertex type if ( nverts_pg > 0 ) then ierr = iMOAB_SetIntTagStorage ( MHPGID, tagname, nverts_pg , ent_type, vdone_pg) if (ierr > 0 ) & - call endrun('Error: fail to set global id tag for vertices') + call abortmp('Error: fail to set global id tag for vertices') endif ! set global id tag for pg2 elements, too; they will start at nets=1, end at nete=nelemd*4 ent_type = 1 ! now set the global id tag on elements if ( nelem_pg > 0 ) then ierr = iMOAB_SetIntTagStorage ( MHPGID, tagname, nelem_pg , ent_type, elemids) if (ierr > 0 ) & - call endrun('Error: fail to set global id tag for edges') + call abortmp('Error: fail to set global id tag for edges') endif + ! this involves a collective, vdone_pg can be empty ierr = iMOAB_ResolveSharedEntities( MHPGID, nverts_pg, vdone_pg ); if (ierr > 0 ) & - call endrun('Error: fail to resolve shared ents for pg2 mesh') + call abortmp('Error: fail to resolve shared ents for pg2 mesh') ierr = iMOAB_UpdateMeshInfo(MHPGID) if (ierr > 0 ) & - call endrun('Error: fail to update mesh info for pg2 mesh') + call abortmp('Error: fail to update mesh info for pg2 mesh') #ifdef MOABDEBUG ! write out the mesh file to disk, in parallel outfile = 'wholeATM_PG2.h5m'//C_NULL_CHAR wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR ierr = iMOAB_WriteMesh(MHPGID, outfile, wopts) if (ierr > 0 ) & - call endrun('Error: fail to write the mesh file') + call abortmp('Error: fail to write the mesh file') #endif endif ! only valid for pg == 2 if ( nelemd > 0 ) then @@ -792,16 +796,14 @@ subroutine create_moab_meshes(par, elem) deallocate (indx_cell) deallocate (edge) ! deallocate(moabconn_pg) ! connectivity - deallocate(vdone_pg) endif - + deallocate(vdone_pg) ! this is now always allocated/deallocated, even for no mesh here endif ! deallocate if ( nelemd > 0 ) then deallocate(moabvh) deallocate(moabconn) ! do not keep it anymore, we are not setting another tag on fine mesh - deallocate(vdone) deallocate(gdofel) deallocate(indx) deallocate(elemids) @@ -810,6 +812,7 @@ subroutine create_moab_meshes(par, elem) deallocate(moabconn_c) deallocate(vdone_c) endif + deallocate(vdone) ! we are always allocating this now ! end copy end subroutine create_moab_meshes diff --git a/components/homme/src/theta-l_kokkos/cxx/ElementsState.cpp b/components/homme/src/theta-l_kokkos/cxx/ElementsState.cpp index f4ea57b71ca5..e7fe7f569c7d 100644 --- a/components/homme/src/theta-l_kokkos/cxx/ElementsState.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/ElementsState.cpp @@ -238,6 +238,80 @@ void ElementsState::randomize(const int seed, Kokkos::fence(); } + +void ElementsState::randomize(const int seed, + const HybridVCoord& hvcoord) { + // Check elements were inited + assert (m_num_elems>0); + + // Check data makes sense + assert (hvcoord.ps0>0); + assert (hvcoord.hybrid_ai0>=0); + + // Arbitrary minimum value to generate + constexpr const Real min_value = 0.015625; + + std::mt19937_64 engine(seed); + std::uniform_real_distribution random_dist(min_value, 1.0 / min_value); + std::uniform_real_distribution pdf_vtheta_dp(100.0, 1000.0); + + genRandArray(m_v, engine, random_dist); + genRandArray(m_w_i, engine, random_dist); + genRandArray(m_vtheta_dp, engine, pdf_vtheta_dp); + // Note: to avoid errors in the equation of state, we need phi to be increasing. + // Rather than using a constraint (which may call the function many times, + // we simply ask that there are no duplicates, then we sort it later. + auto sort_and_chek = [](const ExecViewManaged::HostMirror v)->bool { + Real* start = reinterpret_cast(v.data()); + Real* end = reinterpret_cast(v.data()) + NUM_LEV_P*VECTOR_SIZE; + std::sort(start,end); + std::reverse(start,end); + auto it = std::unique(start,end); + return it==end; + }; + for (int ie=0; ie pressure_pdf(800, 1200); + genRandArray(m_ps_v, engine, pressure_pdf); + + auto dp = m_dp3d; + auto ps = m_ps_v; + auto ps0 = hvcoord.ps0; + auto hyai = hvcoord.hybrid_ai_packed; + auto hybi = hvcoord.hybrid_bi_packed; + auto hyai_delta = hvcoord.hybrid_ai_delta; + auto hybi_delta = hvcoord.hybrid_bi_delta; + const auto tu = m_tu; + Kokkos::parallel_for(m_policy, KOKKOS_LAMBDA(const TeamMember& team) { + KernelVariables kv(team, tu); + const int ie = kv.ie / NUM_TIME_LEVELS; + const int tl = kv.ie % NUM_TIME_LEVELS; + + Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team,NP*NP), + [&](const int idx) { + const int igp = idx / NP; + const int jgp = idx % NP; + ColumnOps::compute_midpoint_delta(kv,hyai,hyai_delta); + ColumnOps::compute_midpoint_delta(kv,hybi,hybi_delta); + team.team_barrier(); + Kokkos::parallel_for(Kokkos::ThreadVectorRange(kv.team,NUM_LEV), + [&](const int ilev) { + dp(ie,tl,igp,jgp,ilev) = ps0*hyai_delta(ilev) + + ps(ie,tl,igp,jgp)*hybi_delta(ilev); + }); + }); + }); + Kokkos::fence(); +} + void ElementsState::pull_from_f90_pointers (CF90Ptr& state_v, CF90Ptr& state_w_i, CF90Ptr& state_vtheta_dp, CF90Ptr& state_phinh_i, CF90Ptr& state_dp3d, CF90Ptr& state_ps_v) { diff --git a/components/homme/src/theta-l_kokkos/cxx/ElementsState.hpp b/components/homme/src/theta-l_kokkos/cxx/ElementsState.hpp index 4fa353141402..aec7701661fa 100644 --- a/components/homme/src/theta-l_kokkos/cxx/ElementsState.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/ElementsState.hpp @@ -63,10 +63,10 @@ class ElementsState { void randomize(const int seed); void randomize(const int seed, const Real max_pressure); void randomize(const int seed, const Real max_pressure, const Real ps0, const Real hyai0); - void randomize(const int seed, const Real max_pressure, const Real ps0, const Real hyai0, const ExecViewUnmanaged& phis); - + void randomize(const int seed, const HybridVCoord& hvcoord); + KOKKOS_INLINE_FUNCTION int num_elems() const { return m_num_elems; } diff --git a/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp b/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp index 80993d1d0f1d..28a702c1d273 100644 --- a/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp @@ -99,8 +99,7 @@ class ForcingFunctor m_hydrostatic = p.theta_hydrostatic_mode; m_qsize = p.qsize; - // TODO: this may change, depending on the simulation params - m_adjust_ps = true; + m_adjust_ps = (p.dt_remap_factor == 0); m_eos.init(m_hydrostatic,m_hvcoord); m_elem_ops.init(m_hvcoord); @@ -153,7 +152,7 @@ class ForcingFunctor constexpr int int_size = NP*NP*NUM_LEV_P*VECTOR_SIZE; // 3 persistent midlayers, 2 non-persistent midlayer, and 1 non-persistent interface - return mid_size*(nelems*4+nslots) + (m_hydrostatic ? int_size*nslots : 0); + return mid_size*(nelems*4+nslots) + int_size*nslots; } void init_buffers (const FunctorsBuffersManager& fbm) { @@ -370,7 +369,7 @@ class ForcingFunctor }); if (!m_adjust_ps) { Kokkos::parallel_for(Kokkos::ThreadVectorRange(kv.team,NUM_LEV), - [&](const int ilev) { + [&](const int ilev) { dp_adj(ilev) = dp(ilev) + dp(ilev)*(fq(ilev)-q(ilev)); }); } @@ -386,7 +385,7 @@ class ForcingFunctor }); if (!m_adjust_ps) { Kokkos::parallel_for(Kokkos::ThreadVectorRange(kv.team,NUM_LEV), - [&](const int& ilev) { + [&](const int& ilev) { dp_adj(ilev) = dp(ilev) + compute_fqdt_pack(ilev,fq,qdp); }); } @@ -472,14 +471,14 @@ class ForcingFunctor } else { // Compute hydrostatic p from dp. Store in exner, then add to pnh auto p_i = Homme::subview(m_pi_i,kv.team_idx,igp,jgp); - m_elem_ops.compute_hydrostatic_p(kv,dp,p_i,exner); + m_elem_ops.compute_hydrostatic_p(kv,dp_adj,p_i,exner); Kokkos::parallel_for(Kokkos::ThreadVectorRange(kv.team,NUM_LEV), - [&](const int ilev) { + [&](const int ilev) { pnh(ilev) += exner(ilev); dp(ilev) = dp_adj(ilev); }); } - + Kokkos::parallel_for(Kokkos::ThreadVectorRange(kv.team,NUM_LEV), [&](const int ilev) { using namespace PhysicalConstants; diff --git a/components/homme/src/theta-l_kokkos/cxx/LimiterFunctor.hpp b/components/homme/src/theta-l_kokkos/cxx/LimiterFunctor.hpp index bf93be710e94..cd3bf7c32526 100644 --- a/components/homme/src/theta-l_kokkos/cxx/LimiterFunctor.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/LimiterFunctor.hpp @@ -67,10 +67,10 @@ struct LimiterFunctor { , m_hvcoord(hvcoord) , m_state(elements.m_state) , m_geometry(elements.m_geometry) - , m_policy_dp3d_lim (Homme::get_default_team_policy(m_num_elems)) - , m_tu(m_policy_dp3d_lim) , m_dp3d_thresh(params.dp3d_thresh) , m_vtheta_thresh(params.vtheta_thresh) + , m_policy_dp3d_lim (Homme::get_default_team_policy(m_num_elems)) + , m_tu(m_policy_dp3d_lim) { m_np1 = -1; } diff --git a/components/homme/test_execs/preqx_kokkos_ut/remap_preqx_ut.cpp b/components/homme/test_execs/preqx_kokkos_ut/remap_preqx_ut.cpp index 480c3782931c..771376e959fa 100644 --- a/components/homme/test_execs/preqx_kokkos_ut/remap_preqx_ut.cpp +++ b/components/homme/test_execs/preqx_kokkos_ut/remap_preqx_ut.cpp @@ -10,6 +10,7 @@ #include "utilities/TestUtils.hpp" #include +#include TEST_CASE("remap_interface", "vertical remap") { diff --git a/components/homme/test_execs/share_kokkos_ut/boundary_exchange_ut.cpp b/components/homme/test_execs/share_kokkos_ut/boundary_exchange_ut.cpp index 12a7e9a1673e..ad15a1c10bdd 100644 --- a/components/homme/test_execs/share_kokkos_ut/boundary_exchange_ut.cpp +++ b/components/homme/test_execs/share_kokkos_ut/boundary_exchange_ut.cpp @@ -11,6 +11,7 @@ #include #include +#include using namespace Homme; diff --git a/components/homme/test_execs/share_kokkos_ut/col_ops_ut.cpp b/components/homme/test_execs/share_kokkos_ut/col_ops_ut.cpp index b27b3f3281be..85368cd6ab0f 100644 --- a/components/homme/test_execs/share_kokkos_ut/col_ops_ut.cpp +++ b/components/homme/test_execs/share_kokkos_ut/col_ops_ut.cpp @@ -1,15 +1,17 @@ #include -#include - #include "ColumnOps.hpp" -#include "Types.hpp" #include "utilities/TestUtils.hpp" #include "utilities/SubviewUtils.hpp" #include "utilities/SyncUtils.hpp" #include "utilities/ViewUtils.hpp" +#include "Types.hpp" + +#include +#include + using namespace Homme; diff --git a/components/homme/test_execs/share_kokkos_ut/ppm_remap_ut.cpp b/components/homme/test_execs/share_kokkos_ut/ppm_remap_ut.cpp index 5fe4a9c21633..401a029607fd 100644 --- a/components/homme/test_execs/share_kokkos_ut/ppm_remap_ut.cpp +++ b/components/homme/test_execs/share_kokkos_ut/ppm_remap_ut.cpp @@ -8,6 +8,7 @@ #include "utilities/TestUtils.hpp" #include +#include using namespace Homme; diff --git a/components/homme/test_execs/share_kokkos_ut/sphere_op_ml.cpp b/components/homme/test_execs/share_kokkos_ut/sphere_op_ml.cpp index e4cbb4ec4594..e261b60b390d 100644 --- a/components/homme/test_execs/share_kokkos_ut/sphere_op_ml.cpp +++ b/components/homme/test_execs/share_kokkos_ut/sphere_op_ml.cpp @@ -13,6 +13,7 @@ #include #include #include +#include using namespace Homme; diff --git a/components/homme/test_execs/share_kokkos_ut/sphere_op_sl.cpp b/components/homme/test_execs/share_kokkos_ut/sphere_op_sl.cpp index 26cec000976d..fd2f79fabae5 100644 --- a/components/homme/test_execs/share_kokkos_ut/sphere_op_sl.cpp +++ b/components/homme/test_execs/share_kokkos_ut/sphere_op_sl.cpp @@ -13,6 +13,7 @@ #include #include #include +#include using namespace Homme; diff --git a/components/homme/test_execs/thetal_kokkos_ut/elem_ops_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/elem_ops_ut.cpp index 8cccddb5118d..9627b0d6d61c 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/elem_ops_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/elem_ops_ut.cpp @@ -1,14 +1,15 @@ #include -#include - #include "ElementOps.hpp" -#include "Types.hpp" #include "utilities/TestUtils.hpp" #include "utilities/SubviewUtils.hpp" #include "utilities/SyncUtils.hpp" #include "utilities/ViewUtils.hpp" +#include "Types.hpp" + +#include +#include using namespace Homme; diff --git a/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp index 7bfeeab0d748..5e4c51c7ca11 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp @@ -60,7 +60,8 @@ TEST_CASE("forcing", "forcing") { // Init everything through singleton, which is what happens in normal runs auto& c = Context::singleton(); auto& p = c.create(); - + p.dt_remap_factor = 1; + auto& hv = c.create(); hv.random_init(seed); @@ -165,7 +166,7 @@ TEST_CASE("forcing", "forcing") { std::cout << " -> adjustment: " << (adjustment ? "true" : "false") << "\n"; // Reset state, tracers, and forcing to the original random values - state.randomize(seed, 10*hv.ps0, hv.ps0, hv.hybrid_ai0); + state.randomize(seed, hv); tracers.randomize(seed,-1e-3,1e-3); forcing.randomize(seed); @@ -289,7 +290,7 @@ TEST_CASE("forcing", "forcing") { // Reset state and forcing to the original random values std::cout << "Testing dynamics forcing.\n"; - state.randomize(seed, 10*hv.ps0, hv.ps0, hv.hybrid_ai0); + state.randomize(seed, hv); forcing.randomize(seed); // Sync views diff --git a/components/mosart/bld/namelist_files/namelist_defaults_mosart.xml b/components/mosart/bld/namelist_files/namelist_defaults_mosart.xml index 9a7c5e4eeaed..1f7e44722127 100644 --- a/components/mosart/bld/namelist_files/namelist_defaults_mosart.xml +++ b/components/mosart/bld/namelist_files/namelist_defaults_mosart.xml @@ -42,12 +42,14 @@ for the CLM data in the CESM distribution rof/mosart/MOSART_global_8th_20180211c.nc +rof/mosart/MOSART_global_qd_20240212.v3.nc rof/mosart/MOSART_global_half_20180721a.nc rof/mosart/MOSART_Global_2deg_antarctica_flowing_to_north_c09162020.nc rof/mosart/MOSART_NLDAS_8th_20160426.nc share/meshes/rof/MOSART_global_8th.scrip.20180211c.nc +share/meshes/rof/SCRIPgrid_0.25x0.25_nomask_c200309.nc share/meshes/rof/SCRIPgrid_0.5x0.5_nomask_c110308.nc share/meshes/rof/SCRIPgrid_2x2_nomask_c210211.nc diff --git a/components/mosart/cime_config/buildnml b/components/mosart/cime_config/buildnml index ba015b01f6a0..90b3b3bb379a 100755 --- a/components/mosart/cime_config/buildnml +++ b/components/mosart/cime_config/buildnml @@ -46,8 +46,8 @@ def buildnml(case, caseroot, compname): # Verify rof grid is supported #-------------------------------------------------------------------- - rof_grid_supported = ("null", "r2", "r05", "r0125", "r01", "NLDAS", "ELM_USRDAT") - expect(rof_grid in rof_grid_supported, "ROF_GRID '{}' is not supported in mosart. Choose from: null, r2, r05, r0125, r01, NLDAS, ELM_USRDAT".format(rof_grid)) + rof_grid_supported = ("null", "r2", "r05", "r025", "r0125", "r01", "NLDAS", "ELM_USRDAT") + expect(rof_grid in rof_grid_supported, "ROF_GRID '{}' is not supported in mosart. Choose from: null, r2, r05, r025, r0125, r01, NLDAS, ELM_USRDAT".format(rof_grid)) #-------------------------------------------------------------------- # Invoke mosart build-namelist - output will go in $CASEBUILD/mosartconf diff --git a/components/mosart/docs/dev-guide/index.md b/components/mosart/docs/dev-guide/index.md index 82f7444f45f6..5d490290063c 100644 --- a/components/mosart/docs/dev-guide/index.md +++ b/components/mosart/docs/dev-guide/index.md @@ -1 +1,5 @@ -Please refer to the [ELM's Developer Guide](../../ELM/dev-guide/index.md) for MOSART development. +# MOSART Developer's Guide + +This guide provides MOSART data structures and how to develop new code for MOSART. + +Coming soon... diff --git a/components/mosart/docs/figures/mosart_concept.png b/components/mosart/docs/figures/mosart_concept.png new file mode 100644 index 000000000000..beeb742ca4a7 Binary files /dev/null and b/components/mosart/docs/figures/mosart_concept.png differ diff --git a/components/mosart/docs/index.md b/components/mosart/docs/index.md index c1ac0ac3b471..b784fbd5e35b 100644 --- a/components/mosart/docs/index.md +++ b/components/mosart/docs/index.md @@ -1,7 +1,7 @@ -#Model for Scale Adaptive River Transport (MOSART) +# Model for Scale Adaptive River Transport (MOSART) -Some introductory text here +MOSART is the river model of E3SM. It routes the runoff genrated by ELM and provides freshwater input to the ocean model. -* The [MOSART User's Guide](user-guide/index.md) explains how to control MOSART when its running within E3SM -* The [MOSART Developer's Guide](dev-guide/index.md) explains MOSART data structures and how to develop new code. -* The [MOSART Techincal Guide](tech-guide/index.md) explains the science behind MOSART's code +* The [MOSART User Guide](user-guide/index.md) explains how to control MOSART when its running within E3SM +* The [MOSART Developer Guide](dev-guide/index.md) explains MOSART data structures and how to develop new code. +* The [MOSART Technical Guide](tech-guide/index.md) explains the science behind MOSART's code diff --git a/components/mosart/docs/tech-guide/index.md b/components/mosart/docs/tech-guide/index.md index ff73bc070c0d..73903f405de8 100644 --- a/components/mosart/docs/tech-guide/index.md +++ b/components/mosart/docs/tech-guide/index.md @@ -1 +1,47 @@ -start of the MOSART Technical Guide +# MOSART Technical Guide + +This guide provides scientific and technical details about MOSART. + +## Physics + +MOSART is a one-dimension river transport model that is designed for river routing at local, regional, and global scales ([Li et al., 2013](https://doi.org/10.1175/JHM-D-12-015.1)). Its primary function is to supply freshwater inputs to ocean models within coupled Earth System Models. + +MOSART divides each spatial unit, such as a latitude/longitude grid or a sub-basin, into three hydrologic categories: hillslopes, tributaries, and a main channel (see figure below). The hillslopes receive runoff and send into tributaries, which then converge into a single main channel. This main channel connects adjacent upstream and downstream units through the river network. MOSART simplifies the multiple tributaries within a spatial unit into a single hypothetical sub-network channel, which has a transport capacity equivalent to all combined tributaries. + +![alt text](../figures/mosart_concept.png) + +- Hillslope Routing: Within each spatial unit, surface runoff is directed as overland flow to the sub-network channel, while subsurface runoff enters the sub-network channel directly. + +- Sub-network Channel Routing: This channel aggregates water from the hillslopes, routes it through the channel system, and discharges it into the main channel. + +- Main Channel Routing: The main channel collects water from the sub-network channel and any inflow from upstream spatial units, eventually discharging the accumulated water downstream to the next spatial unit or directly to the ocean. + +## Parameters + +### Main parameters required in the MOSART parameter file + +| Variable Name | Description [unit] | +|-------------------|--------------------------------------------------------------------------------| +| fdir | Flow direction [unitless] | +| lat | Latitude at cell center [degree] | +| lon | Longitude at cell center [degree] | +| frac | fraction of the unit draining to the outlet [0-1] | +| rslp | main channel slope [unitless] | +| rlen | main channel length [m] | +| tslp | mean tributary channel slope averaged through the unit [unitless] | +| area | local drainage area [m^2] | +| areaTotal | total upstream drainage area, local unit included; multi flow direction [m^2] | +| areaTotal2 | total upstream drainage area, local unit included; single flow direction [m^2] | +| rdep | bankfull depth of main channel [m] | +| rwid | bankfull width of main channel [m] | +| rwid0 | floodplain width linked to main channel [m] | +| gxr | drainage density [m^-1] | +| hslp | topographic slope [unitless] | +| twid | bankfull width of local tributaries [m] | +| nr | Manning''s roughness coefficient for main channel flow [unitless] | +| nt | Manning''s roughness coefficient for tributary channel flow [unitless] | +| nh | Manning''s roughness coefficient for overland flow [unitless] | + +### Parameters required by additional MOSART features + +Coming soon. diff --git a/components/mosart/docs/user-guide/index.md b/components/mosart/docs/user-guide/index.md index 858bd73b9a4d..f60983827288 100644 --- a/components/mosart/docs/user-guide/index.md +++ b/components/mosart/docs/user-guide/index.md @@ -1 +1,128 @@ -start of the MOSART User's Guide +# MOSART User's Guide + +This guide describes how to set up and run MOSART. + +## Steps to build and run MOSART + +A step-by-step instruction on how to run fully coupled E3SM can be found [here](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/2309226536). Here we describe running MOSART driven by runoff forcings provided via the data land mode(DLND). Although the default runoff forcing is predefined by the DLND compsets, the users is able to use other runoff forcing to drive MOSART. + +MOSART in the water cycle campaigns of E3SM v1, v2, and v3 was. + +### Scientifically supported compsets + +The mosart-only compsets are supported for multiple runoff forcing datasets that covers different domains and time periods: + +1. `RMOSQIAN`: A 57-year (1948-2004, no leap year) global Qian runoff forcings. The native resolutions are 192*288 at daily time scale. The path to the stream file is `$input_data_dir/lnd/dlnd7/hcru_hcru/QIAN.daily.1948-2004.nc` +2. `RMOSGPCC`: A one-year (1979) global GPCC runoff forcing. The native resoultions are 192*288 at daily time scale. The path to the stream file is `$input_data_dir/lnd/dlnd7/hcru_hcru/GPCC.daily.nc` +3. `RMOSNLDAS`: A one-year (1975) CLM runoff forcing for [NLDAS](https://ldas.gsfc.nasa.gov/nldas) domain (North America between 25N and 53N) and spatial resolution (1/8th degree). The time scale is 3-hourly. The path to the stream file is `$input_data_dir/lnd/dlnd7/NLDAS/mosart_nldas_hist_1975.clm2.h1.1975-01-01-00000.nc`. + +### Supported grid + +The `r05_r05`, `NLDAS_NLDAS`, and `MOS_USRDAT` are the supported grid resolutions for performing offline MOSART simulations. + +### User-defined runoff forcing + +Once the case is created based on the above compsets. The users can create a `user_dlnd.streams.txt` file in the `case_scripts` directory following the format below: + +```xml + + + + GENERIC + + + + time time + lon lon + lat lat + + + /path/to/forcing/file + + + Runoff_forcing.nc + + + + + QDRAI rofsub + QOVER rofsur + + + /path/to/forcing/file + + + Runoff_forcing.nc + + + 0 + + + +``` + +## Customizing runs + +For default river routing simulation in MOSART (i.e. natural flow routing), one [parameter file](../tech-guide/index.md#parameters) defined by `frivinp_rtm` is required. The geographic domain and spatial resolution must match the domain defined in the simulation case. Additional parameter/input files will be needed for some extra features. + +### Output variables and frequency + +MOSART by default outputs monthly history file in `*mosart.h0.**.nc` files that include all key variables. User could choose to output additional history files (such as `*mosart.h1.*.nc`, `*mosart.h2.*.nc`) that have different temporal averaging (e.g. daily, hourly, every model timestep) via `rtmhist_nhtfrq` where + +- `-24` corresponds to daily average +- `-1` corresponds to hourly average +- `0` corresponds to monthly average +- `1` corresponds to each model time step + +The number of time slices in these additional files can be controlled +vai `rtmhist_mfilt`. + +```fortran + &mosart_inparm + rtmhist_fincl2 = 'FLOODPLAIN_FRACTION' + rtmhist_fincl3 = 'RIVER_DISCHARGE_OVER_LAND_LIQ' + rtmhist_nhtfrq = 0, -24, -1 + rtmhist_mfilt = 12, 30, 24 + / +``` + +Using the above-mentioned settings: + +- Each `*.mosart.h1.*.nc` will include 30 daily average values of `FLOODPLAIN_FRACTION` +- Each `*.mosart.h2.*.nc` will include 24 hourly average values of `RIVER_DISCHARGE_OVER_LAND_LIQ` + +### Additional options + +The table below lists avaiable options defined by users through `user_nl_mosart`. + +| Flag Name | Description | +|-------------------|---------------------------------------------------------------------------------------| +| `routingmethod` | `1` for kenematic wave routing (default); `2` for diffusion wave routing | +| `wrmflag` | `.true.` for active water management; `.false.` for no water management (default) | +| `inundflag` | `.true.` for active flood inundation; `.false.` for no flood inundation (default) | +| `sediflag` | `.true.` for active sediment transport; `.false.` for no sediment transport (default) | +| `heatflag` | `.true.` for active heat transport; `.false.` for no heat transport (default) | + +## Additional MOSART features + +There are some options only made available for specific features. They can be defined through `user_nl_mosart`. + +### Water management + +- Parameter file: when water management is active, one additional parameter file `parafile` is required. This file defines the location and specifics for the dams/reservoirs simulated in this scheme. + +- `damconstructionflag`: `0` - dam always exist; `1` - dam construction year is considered. + +- `externaldemandflag`: `0` - no external water demand for the WM scheme; `1` - external water demand files are required. + + - Note if `externaldemandflag` is set to `1`, paths to monthly water demand files are requried in the `usr_nl_mosart` file through `demandpath = '/path/to/demand/files/`. + +### Flood inundation + +- `opt_eleprof`: `1` - use actural elevation profiles derived from DEM; `2` - use hypothetical elevation profile. + + - Note if `opt_eleprof` is set to `1`, the elevation profile data must be included in the MOSART parameter file. + +### Sediment transport + +- If sediment feature is activated, D50 data must be included in the MOSART parameter file. In addition, `rof_sed = .true.` has to be defined in `./user_nl_cpl` to allow sediment flux passing into the river model through the coupler. diff --git a/components/mosart/mkdocs.yml b/components/mosart/mkdocs.yml index 6d80ef7c93d7..cae190f03025 100644 --- a/components/mosart/mkdocs.yml +++ b/components/mosart/mkdocs.yml @@ -2,6 +2,6 @@ site_name: MOSART nav: - Introduction: 'index.md' - - Users's Guide: user-guide/index.md - - Developers's Guide: dev-guide/index.md + - User Guide: user-guide/index.md + - Developer Guide: dev-guide/index.md - Technical Guide: tech-guide/index.md diff --git a/components/mosart/src/riverroute/MOSART_physics_mod.F90 b/components/mosart/src/riverroute/MOSART_physics_mod.F90 index 912014904e93..cad978577cda 100644 --- a/components/mosart/src/riverroute/MOSART_physics_mod.F90 +++ b/components/mosart/src/riverroute/MOSART_physics_mod.F90 @@ -682,8 +682,8 @@ subroutine Euler ! check for negative channel storage if (negchan < -1.e-10) then - write(iulog,*) 'Warning: Negative channel storage found! ',negchan -! call shr_sys_abort('mosart: negative channel storage') + write(iulog,*) 'Error: Negative channel storage found! ',negchan + call shr_sys_abort('mosart: negative channel storage') endif TRunoff%flow = TRunoff%flow / Tctl%DLevelH2R TRunoff%erowm_regi(:,nt_nmud:nt_nsan) = TRunoff%erowm_regi(:,nt_nmud:nt_nsan) / Tctl%DLevelH2R diff --git a/components/mpas-albany-landice/bld/namelist_files/albany_input.yaml b/components/mpas-albany-landice/bld/namelist_files/albany_input.yaml index 89df18322ef7..478ed9dd6580 100644 --- a/components/mpas-albany-landice/bld/namelist_files/albany_input.yaml +++ b/components/mpas-albany-landice/bld/namelist_files/albany_input.yaml @@ -3,9 +3,25 @@ ANONYMOUS: Build Type: Tpetra + Problem: + LandIce Field Norm: + sliding_velocity_basalside: + Regularization Type: Given Value + Regularization Value: 1.0e-4 + LandIce BCs: + BC 0: + Cubature Degree: 4 + Basal Friction Coefficient: + Type: Power Law + Power Exponent: 1.0 + Mu Type: Field + Effective Pressure Type: Hydrostatic Computed At Nodes + Zero Effective Pressure On Floating Ice At Nodes: true + Zero Beta On Floating Ice: false + Use Pressurized Bed Above Sea Level: false # Discretization Description Discretization: -# Exodus Output File Name: albany_output.exo + #Exodus Output File Name: albany_output.exo Piro: # Nonlinear Solver Information @@ -22,13 +38,21 @@ ANONYMOUS: Combo Type: OR Number of Tests: 2 Test 0: - Test Type: NormF - Norm Type: Two Norm - Scale Type: Scaled - Tolerance: 1.0e-05 + Test Type: Combo + Combo Type: AND + Number of Tests: 2 + Test 0: + Test Type: NormF + Norm Type: Two Norm + Scale Type: Unscaled + Tolerance: 1.0e-03 + Test 1: + Test Type: RelativeNormF + Norm Type: Two Norm + Tolerance: 0.9999 Test 1: Test Type: MaxIters - Maximum Iterations: 50 + Maximum Iterations: 100 Printing: Output Precision: 3 Output Processor: 0 @@ -46,7 +70,7 @@ ANONYMOUS: Direction: Method: Newton Newton: - Forcing Term Method: Constant + Forcing Term Method: Type 2 Rescue Bad Newton Solve: true Linear Solver: Write Linear System: false @@ -230,4 +254,3 @@ ANONYMOUS: Coordinates: myRebalanceProlongatorFact Importer: myRepartitionFact - diff --git a/components/mpas-albany-landice/cime_config/buildnml b/components/mpas-albany-landice/cime_config/buildnml index edc88f09ffc8..0a3b35df93eb 100755 --- a/components/mpas-albany-landice/cime_config/buildnml +++ b/components/mpas-albany-landice/cime_config/buildnml @@ -77,6 +77,11 @@ def buildnml(case, caseroot, compname): grid_prefix += 'gis_1to10km_r01' decomp_date += '051920' decomp_prefix += 'mpasli.graph.info.' + elif glc_grid == 'mpas.gis1to10kmR2': + grid_date += '20230202' + grid_prefix += 'gis_1to10km_r02' + decomp_date += '020223' + decomp_prefix += 'mpasli.graph.info.' else: expect(False, "ERROR: mali buildnml encountered unknown GLC_GRID: " + glc_grid) @@ -205,7 +210,7 @@ def buildnml(case, caseroot, compname): lines.append(' filename_template="{}.mali.hist.$Y-$M-$D_$S.nc"'.format(casename)) lines.append(' filename_interval="0001-00-00_00:00:00"') lines.append(' clobber_mode="truncate"') - lines.append(' output_interval="0001-00-00_00:00:00">') + lines.append(' output_interval="0000-00-01_00:00:00">') # Note: if output_interval is less than dt, then multiples of that interval will be checked. # There is some performance hit for making this too small. For now making it 1 day. lines.append('') diff --git a/components/mpas-albany-landice/cime_config/config_compsets.xml b/components/mpas-albany-landice/cime_config/config_compsets.xml index 8aa3687893cd..f3124b784945 100644 --- a/components/mpas-albany-landice/cime_config/config_compsets.xml +++ b/components/mpas-albany-landice/cime_config/config_compsets.xml @@ -3,7 +3,7 @@ - Compset format is explained in cime/config/e3sm/allactive/config_compsets.xml + Compset format is explained in cime_config/allactive/config_compsets.xml or in the CIME documenation diff --git a/components/mpas-albany-landice/driver/glc_comp_mct.F b/components/mpas-albany-landice/driver/glc_comp_mct.F index 957422fc8595..ac33d17b1f31 100644 --- a/components/mpas-albany-landice/driver/glc_comp_mct.F +++ b/components/mpas-albany-landice/driver/glc_comp_mct.F @@ -886,12 +886,23 @@ subroutine glc_run_mct( EClock, cdata_g, x2g_g, g2x_g)!{{{ ! will allow that. ! Finally, set whence to latest_before so we have piecewise-constant forcing. ! Could add, e.g., linear interpolation later. -! In coupled mode, we don't want to use forcing from a file - that should come from the coupler! -! call mpas_stream_mgr_read(stream_manager, whence=MPAS_STREAM_LATEST_BEFORE, ierr=err_tmp) -! err = ior(err, err_tmp) -! call mpas_stream_mgr_reset_alarms(stream_manager, direction=MPAS_STREAM_INPUT, ierr=err_tmp) -! err = ior(err, err_tmp) + ! NOTE: If any input ("forcing") files here contain fields that the coupler also passes, + ! the input file versions here will clobber the fields passed from the coupler. + + call mpas_stream_mgr_read(domain % streamManager, whence=MPAS_STREAM_LATEST_BEFORE, saveActualWhen = .true., ierr=err_tmp) + err = ior(err, err_tmp) + call mpas_stream_mgr_reset_alarms(domain % streamManager, direction=MPAS_STREAM_INPUT, ierr=err_tmp) + err = ior(err, err_tmp) + + ! call analysis driver compute, restart, write subroutines + ! (note: alarms and timers are handled by the analysis member code) + call li_analysis_compute(domain, err_tmp) + err = ior(err, err_tmp) + call li_analysis_restart(domain, err_tmp) + err = ior(err, err_tmp) + call li_analysis_write(domain, err_tmp) + err = ior(err, err_tmp) ! === ! === Write Output and/or Restart, if needed diff --git a/components/mpas-framework/src/CMakeLists.txt b/components/mpas-framework/src/CMakeLists.txt index b6e8e63a19e8..5f1ae35a63a1 100644 --- a/components/mpas-framework/src/CMakeLists.txt +++ b/components/mpas-framework/src/CMakeLists.txt @@ -71,8 +71,7 @@ set(COMMON_RAW_SOURCES external/ezxml/ezxml.c) include(${CMAKE_CURRENT_SOURCE_DIR}/framework/framework.cmake) if (COMP_INTERFACE STREQUAL "moab") list(APPEND COMMON_RAW_SOURCES framework/mpas_moabmesh.F) - include(${MOAB_PATH}/lib/cmake/MOAB/MOABConfig.cmake) - list(APPEND INCLUDES "${MOAB_PATH}/include") + list(APPEND INCLUDES ${MOAB_INCLUDE_DIRS}) list(APPEND CPPDEFS "-DHAVE_MOAB") endif() include(${CMAKE_CURRENT_SOURCE_DIR}/operators/operators.cmake) diff --git a/components/mpas-framework/src/framework/mpas_pool_routines.F b/components/mpas-framework/src/framework/mpas_pool_routines.F index 9611129c8eae..7c01125bf7f2 100644 --- a/components/mpas-framework/src/framework/mpas_pool_routines.F +++ b/components/mpas-framework/src/framework/mpas_pool_routines.F @@ -930,22 +930,36 @@ end subroutine mpas_pool_clone_pool!}}} !> \details !> This routine assumes srcPool and destPool have identical members. It will !> copy the data from the members of srcPool into the members of destPool. +!> The overrideTimeLevels argument enables a given time level of a srcPool +!> with >1 time level to be copied into a destPool with a single time level. ! !----------------------------------------------------------------------- - recursive subroutine mpas_pool_copy_pool(srcPool, destPool)!{{{ + recursive subroutine mpas_pool_copy_pool(srcPool, destPool, overrideTimeLevels)!{{{ implicit none type (mpas_pool_type), pointer :: srcPool type (mpas_pool_type), pointer :: destPool + integer, intent(in), optional :: overrideTimeLevels integer :: i, j, threadNum + integer :: timeLevels type (mpas_pool_member_type), pointer :: ptr type (mpas_pool_data_type), pointer :: dptr type (mpas_pool_data_type), pointer :: mem threadNum = mpas_threading_get_thread_num() + timeLevels = -1 + + if (present(overrideTimeLevels)) then + timeLevels = overrideTimeLevels + + if (timeLevels < 1) then + call mpas_pool_set_error_level(MPAS_POOL_FATAL) + call pool_mesg('ERROR in mpas_pool_copy_pool: Input time levels cannot be less than 1.') + end if + end if if ( threadNum == 0 ) then do i=1,srcPool % size @@ -985,8 +999,14 @@ recursive subroutine mpas_pool_copy_pool(srcPool, destPool)!{{{ dptr => ptr % data - ! Do this through brute force... mem => pool_get_member(destPool, ptr % key, MPAS_POOL_FIELD) + + ! Allow for overrideTimeLevels + if (timeLevels == -1) then + timeLevels = mem % contentsTimeLevs + endif + + ! Do this through brute force... if (associated(dptr % r0)) then call mpas_duplicate_field(dptr % r0, mem % r0, copy_array_only=.true.) else if (associated(dptr % r1)) then @@ -1014,83 +1034,135 @@ recursive subroutine mpas_pool_copy_pool(srcPool, destPool)!{{{ else if (associated(dptr % l0)) then call mpas_duplicate_field(dptr % l0, mem % l0, copy_array_only=.true.) else if (associated(dptr % r0a)) then - do j=1,mem % contentsTimeLevs - mem % r0 => mem % r0a(j) - call mpas_duplicate_field(dptr % r0a(j), mem % r0, copy_array_only=.true.) - nullify(mem % r0) - end do + if (timeLevels > 1) then + do j=1,timeLevels + mem % r0 => mem % r0a(j) + call mpas_duplicate_field(dptr % r0a(j), mem % r0, copy_array_only=.true.) + nullify(mem % r0) + end do + else + call mpas_duplicate_field(dptr % r0a(1), mem % r0, copy_array_only=.true.) + end if else if (associated(dptr % r1a)) then - do j=1,mem % contentsTimeLevs - mem % r1 => mem % r1a(j) - call mpas_duplicate_field(dptr % r1a(j), mem % r1, copy_array_only=.true.) - nullify(mem % r1) - end do + if (timeLevels > 1) then + do j=1,timeLevels + mem % r1 => mem % r1a(j) + call mpas_duplicate_field(dptr % r1a(j), mem % r1, copy_array_only=.true.) + nullify(mem % r1) + end do + else + call mpas_duplicate_field(dptr % r1a(1), mem % r1, copy_array_only=.true.) + end if else if (associated(dptr % r2a)) then - do j=1,mem % contentsTimeLevs - mem % r2 => mem % r2a(j) - call mpas_duplicate_field(dptr % r2a(j), mem % r2, copy_array_only=.true.) - nullify(mem % r2) - end do + if (timeLevels > 1) then + do j=1,timeLevels + mem % r2 => mem % r2a(j) + call mpas_duplicate_field(dptr % r2a(j), mem % r2, copy_array_only=.true.) + nullify(mem % r2) + end do + else + call mpas_duplicate_field(dptr % r2a(1), mem % r2, copy_array_only=.true.) + end if else if (associated(dptr % r3a)) then - do j=1,mem % contentsTimeLevs - mem % r3 => mem % r3a(j) - call mpas_duplicate_field(dptr % r3a(j), mem % r3, copy_array_only=.true.) - nullify(mem % r3) - end do + if (timeLevels > 1) then + do j=1,timeLevels + mem % r3 => mem % r3a(j) + call mpas_duplicate_field(dptr % r3a(j), mem % r3, copy_array_only=.true.) + nullify(mem % r3) + end do + else + call mpas_duplicate_field(dptr % r3a(1), mem % r3, copy_array_only=.true.) + end if else if (associated(dptr % r4a)) then - do j=1,mem % contentsTimeLevs - mem % r4 => mem % r4a(j) - call mpas_duplicate_field(dptr % r4a(j), mem % r4, copy_array_only=.true.) - nullify(mem % r4) - end do + if (timeLevels > 1) then + do j=1,timeLevels + mem % r4 => mem % r4a(j) + call mpas_duplicate_field(dptr % r4a(j), mem % r4, copy_array_only=.true.) + nullify(mem % r4) + end do + else + call mpas_duplicate_field(dptr % r4a(1), mem % r4, copy_array_only=.true.) + end if else if (associated(dptr % r5a)) then - do j=1,mem % contentsTimeLevs - mem % r5 => mem % r5a(j) - call mpas_duplicate_field(dptr % r5a(j), mem % r5, copy_array_only=.true.) - nullify(mem % r5) - end do + if (timeLevels > 1) then + do j=1,timeLevels + mem % r5 => mem % r5a(j) + call mpas_duplicate_field(dptr % r5a(j), mem % r5, copy_array_only=.true.) + nullify(mem % r5) + end do + else + call mpas_duplicate_field(dptr % r5a(1), mem % r5, copy_array_only=.true.) + end if else if (associated(dptr % i0a)) then - do j=1,mem % contentsTimeLevs - mem % i0 => mem % i0a(j) - call mpas_duplicate_field(dptr % i0a(j), mem % i0, copy_array_only=.true.) - nullify(mem % i0) - end do + if (timeLevels > 1) then + do j=1,timeLevels + mem % i0 => mem % i0a(j) + call mpas_duplicate_field(dptr % i0a(j), mem % i0, copy_array_only=.true.) + nullify(mem % i0) + end do + else + call mpas_duplicate_field(dptr % i0a(1), mem % i0, copy_array_only=.true.) + end if else if (associated(dptr % i1a)) then - do j=1,mem % contentsTimeLevs - mem % i1 => mem % i1a(j) - call mpas_duplicate_field(dptr % i1a(j), mem % i1, copy_array_only=.true.) - nullify(mem % i1) - end do + if (timeLevels > 1) then + do j=1,timeLevels + mem % i1 => mem % i1a(j) + call mpas_duplicate_field(dptr % i1a(j), mem % i1, copy_array_only=.true.) + nullify(mem % i1) + end do + else + call mpas_duplicate_field(dptr % i1a(1), mem % i1, copy_array_only=.true.) + end if else if (associated(dptr % i2a)) then - do j=1,mem % contentsTimeLevs - mem % i2 => mem % i2a(j) - call mpas_duplicate_field(dptr % i2a(j), mem % i2, copy_array_only=.true.) - nullify(mem % i2) - end do + if (timeLevels > 1) then + do j=1,timeLevels + mem % i2 => mem % i2a(j) + call mpas_duplicate_field(dptr % i2a(j), mem % i2, copy_array_only=.true.) + nullify(mem % i2) + end do + else + call mpas_duplicate_field(dptr % i2a(1), mem % i2, copy_array_only=.true.) + end if else if (associated(dptr % i3a)) then - do j=1,mem % contentsTimeLevs - mem % i3 => mem % i3a(j) - call mpas_duplicate_field(dptr % i3a(j), mem % i3, copy_array_only=.true.) - nullify(mem % i3) - end do + if (timeLevels > 1) then + do j=1,timeLevels + mem % i3 => mem % i3a(j) + call mpas_duplicate_field(dptr % i3a(j), mem % i3, copy_array_only=.true.) + nullify(mem % i3) + end do + else + call mpas_duplicate_field(dptr % i3a(1), mem % i3, copy_array_only=.true.) + end if else if (associated(dptr % c0a)) then - do j=1,mem % contentsTimeLevs - mem % c0 => mem % c0a(j) - call mpas_duplicate_field(dptr % c0a(j), mem % c0, copy_array_only=.true.) - nullify(mem % c0) - end do + if (timeLevels > 1) then + do j=1,timeLevels + mem % c0 => mem % c0a(j) + call mpas_duplicate_field(dptr % c0a(j), mem % c0, copy_array_only=.true.) + nullify(mem % c0) + end do + else + call mpas_duplicate_field(dptr % c0a(1), mem % c0, copy_array_only=.true.) + end if else if (associated(dptr % c1a)) then - do j=1,mem % contentsTimeLevs - mem % c1 => mem % c1a(j) - call mpas_duplicate_field(dptr % c1a(j), mem % c1, copy_array_only=.true.) - nullify(mem % c1) - end do + if (timeLevels > 1) then + do j=1,timeLevels + mem % c1 => mem % c1a(j) + call mpas_duplicate_field(dptr % c1a(j), mem % c1, copy_array_only=.true.) + nullify(mem % c1) + end do + else + call mpas_duplicate_field(dptr % c1a(1), mem % c1, copy_array_only=.true.) + end if else if (associated(dptr % l0a)) then - do j=1,mem % contentsTimeLevs - mem % l0 => mem % l0a(j) - call mpas_duplicate_field(dptr % l0a(j), mem % l0, copy_array_only=.true.) - nullify(mem % l0) - end do + if (timeLevels > 1) then + do j=1,timeLevels + mem % l0 => mem % l0a(j) + call mpas_duplicate_field(dptr % l0a(j), mem % l0, copy_array_only=.true.) + nullify(mem % l0) + end do + else + call mpas_duplicate_field(dptr % l0a(1), mem % l0, copy_array_only=.true.) + end if else call pool_mesg('While copying pool, member '//trim(ptr % key)//' has no valid field pointers.') end if diff --git a/components/mpas-ocean/bld/build-namelist b/components/mpas-ocean/bld/build-namelist index 0b8d3c782fa7..c8e1c1e3a7a4 100755 --- a/components/mpas-ocean/bld/build-namelist +++ b/components/mpas-ocean/bld/build-namelist @@ -557,9 +557,9 @@ add_default($nl, 'config_Redi_constant_kappa'); add_default($nl, 'config_Redi_maximum_slope'); add_default($nl, 'config_Redi_use_slope_taper'); add_default($nl, 'config_Redi_use_surface_taper'); +add_default($nl, 'config_Redi_limit_term1'); add_default($nl, 'config_Redi_use_quasi_monotone_limiter'); add_default($nl, 'config_Redi_quasi_monotone_safety_factor'); -add_default($nl, 'config_Redi_limit_term1'); add_default($nl, 'config_Redi_min_layers_diag_terms'); add_default($nl, 'config_Redi_horizontal_taper'); add_default($nl, 'config_Redi_horizontal_ramp_min'); @@ -616,6 +616,8 @@ add_default($nl, 'config_use_cvmix'); add_default($nl, 'config_cvmix_prandtl_number'); add_default($nl, 'config_cvmix_background_scheme'); add_default($nl, 'config_cvmix_background_diffusion'); +add_default($nl, 'config_cvmix_background_diffusion_passive'); +add_default($nl, 'config_cvmix_background_diffusion_passive_enable'); add_default($nl, 'config_cvmix_background_viscosity'); add_default($nl, 'config_cvmix_BryanLewis_bl1'); add_default($nl, 'config_cvmix_BryanLewis_bl2'); @@ -1005,6 +1007,12 @@ add_default($nl, 'config_manufactured_solution_wavelength_x'); add_default($nl, 'config_manufactured_solution_wavelength_y'); add_default($nl, 'config_manufactured_solution_amplitude'); +##################################### +# Namelist group: init_mode_subgrid # +##################################### + +add_default($nl, 'config_subgrid_table_levels'); + ################################################ # Namelist group: tracer_forcing_activeTracers # ################################################ @@ -1775,6 +1783,7 @@ my @groups = qw(run_modes transport_tests init_mode_vert_levels manufactured_solution + init_mode_subgrid tracer_forcing_activetracers tracer_forcing_debugtracers tracer_forcing_ecosystracers diff --git a/components/mpas-ocean/bld/build-namelist-group-list b/components/mpas-ocean/bld/build-namelist-group-list index 11864939025d..c109ba9990e3 100644 --- a/components/mpas-ocean/bld/build-namelist-group-list +++ b/components/mpas-ocean/bld/build-namelist-group-list @@ -40,6 +40,7 @@ my @groups = qw(run_modes transport_tests init_mode_vert_levels manufactured_solution + init_mode_subgrid tracer_forcing_activetracers tracer_forcing_debugtracers tracer_forcing_ecosystracers diff --git a/components/mpas-ocean/bld/build-namelist-section b/components/mpas-ocean/bld/build-namelist-section index 28030392442d..b202284f3005 100644 --- a/components/mpas-ocean/bld/build-namelist-section +++ b/components/mpas-ocean/bld/build-namelist-section @@ -96,9 +96,9 @@ add_default($nl, 'config_Redi_constant_kappa'); add_default($nl, 'config_Redi_maximum_slope'); add_default($nl, 'config_Redi_use_slope_taper'); add_default($nl, 'config_Redi_use_surface_taper'); +add_default($nl, 'config_Redi_limit_term1'); add_default($nl, 'config_Redi_use_quasi_monotone_limiter'); add_default($nl, 'config_Redi_quasi_monotone_safety_factor'); -add_default($nl, 'config_Redi_limit_term1'); add_default($nl, 'config_Redi_min_layers_diag_terms'); add_default($nl, 'config_Redi_horizontal_taper'); add_default($nl, 'config_Redi_horizontal_ramp_min'); @@ -155,6 +155,8 @@ add_default($nl, 'config_use_cvmix'); add_default($nl, 'config_cvmix_prandtl_number'); add_default($nl, 'config_cvmix_background_scheme'); add_default($nl, 'config_cvmix_background_diffusion'); +add_default($nl, 'config_cvmix_background_diffusion_passive'); +add_default($nl, 'config_cvmix_background_diffusion_passive_enable'); add_default($nl, 'config_cvmix_background_viscosity'); add_default($nl, 'config_cvmix_BryanLewis_bl1'); add_default($nl, 'config_cvmix_BryanLewis_bl2'); @@ -522,6 +524,12 @@ add_default($nl, 'config_manufactured_solution_wavelength_x'); add_default($nl, 'config_manufactured_solution_wavelength_y'); add_default($nl, 'config_manufactured_solution_amplitude'); +##################################### +# Namelist group: init_mode_subgrid # +##################################### + +add_default($nl, 'config_subgrid_table_levels'); + ################################################ # Namelist group: tracer_forcing_activeTracers # ################################################ diff --git a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml index 4069eb144cff..248654ef10f5 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml @@ -49,6 +49,11 @@ '00:10:00' '00:30:00' '00:30:00' +'00:30:00' +'00:08:00' +'00:04:00' +'00:02:00' +'00:01:00' 'split_explicit_ab2' 2 @@ -73,6 +78,11 @@ .true. .true. .true. +.true. +.true. +.true. +.true. +.true. -1.0 .false. 30.0e3 @@ -89,6 +99,11 @@ .true. .true. .true. +.true. +.true. +.true. +.true. +.true. 10.0 1000.0 1000.0 @@ -99,6 +114,11 @@ 462.0 1000.0 1000.0 +1000.0 +308.0 +154.0 +77.0 +38.5 .false. 10.0 @@ -124,6 +144,11 @@ 1.18e10 1.2e11 1.2e11 +1.2e11 +3.50e09 +4.37e08 +5.46e07 +6.83e06 1.0 .false. 0.0 @@ -146,16 +171,20 @@ 0.01 .true. .true. +.true. .true. 0.9 -.true. 0 15 'ramp' 'RossbyRadius' 'RossbyRadius' - 'ramp' +'ramp' +'RossbyRadius' +'RossbyRadius' +'RossbyRadius' +'RossbyRadius' 20e3 30e3 30e3 @@ -180,8 +209,12 @@ 'constant' 'N2_dependent' 'N2_dependent' - 'constant' +'constant' +'N2_dependent' +'N2_dependent' +'N2_dependent' +'N2_dependent' 900.0 600.0 600.0 @@ -191,6 +224,11 @@ 600.0 600.0 600.0 +600.0 +600.0 +600.0 +600.0 +600.0 0.3 'constant' 300.0 @@ -198,8 +236,12 @@ 3.0 1.0 1.0 - 3.0 +3.0 +1.0 +1.0 +1.0 +1.0 0.13 1000.0 200.0 @@ -209,8 +251,12 @@ 'ramp' 'RossbyRadius' 'RossbyRadius' - 'ramp' +'ramp' +'RossbyRadius' +'RossbyRadius' +'RossbyRadius' +'RossbyRadius' 20e3 30e3 30e3 @@ -229,6 +275,8 @@ 1.0 'constant' 0.0 +0.0 +.false. 1.0e-4 8.0e-5 1.05E-4 @@ -347,6 +395,11 @@ 'pressure_only' 'pressure_only' 'pressure_only' +'pressure_only' +'pressure_only' +'pressure_only' +'pressure_only' +'pressure_only' 'Jenkins' .false. 10.0 @@ -360,6 +413,11 @@ 4.48e-3 4.48e-3 4.48e-3 +4.48e-3 +4.48e-3 +4.48e-3 +4.48e-3 +4.48e-3 1e-4 5e-2 0.011 @@ -368,12 +426,22 @@ 0.00295 0.00295 0.00295 +0.00295 +0.00295 +0.00295 +0.00295 +0.00295 3.1e-4 8.42e-5 8.42e-5 8.42e-5 8.42e-5 8.42e-5 +8.42e-5 +8.42e-5 +8.42e-5 +8.42e-5 +8.42e-5 'flux-form' @@ -397,6 +465,11 @@ 4.48e-3 4.48e-3 4.48e-3 +4.48e-3 +4.48e-3 +4.48e-3 +4.48e-3 +4.48e-3 1.0e-3 10.0 2.5e-3 @@ -475,6 +548,11 @@ '0000_00:00:15' '0000_00:01:15' '0000_00:01:00' +'0000_00:01:00' +'0000_00:00:10' +'0000_00:00:05' +'0000_00:00:02.5' +'0000_00:00:01.25' 2 .true. 2 @@ -516,6 +594,11 @@ .false. .false. .false. +.false. +.false. +.false. +.false. +.false. .false. .false. .false. @@ -565,6 +648,9 @@ 2000000.0 1 + +-1 + .true. .true. @@ -1033,6 +1119,11 @@ .true. .true. .true. +.true. +.true. +.true. +.true. +.true. '0000-00-00_01:00:00' 'mocStreamfunctionOutput' .true. @@ -1115,16 +1206,31 @@ .true. .true. .true. +.true. +.true. +.true. +.true. +.true. 'dt' 'conservationCheckOutput' .false. .true. .true. .true. +.true. +.true. +.true. +.true. +.true. .false. .true. .true. .true. +.true. +.true. +.true. +.true. +.true. .true. 'conservationCheckRestart' diff --git a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml index 01456cd6583a..cd8535a41587 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml @@ -202,9 +202,9 @@ Default: Defined in namelist_defaults.xml -Time integration method. +Time integration method. These options are only supported in standalone, not E3SM: 'LTS', 'FB_LTS'. -Valid values: 'split_explicit', 'RK4', 'unsplit_explicit', 'split_implicit', 'LTS', 'split_explicit_ab2' +Valid values: 'split_explicit', 'RK4', 'unsplit_explicit', 'split_implicit', 'split_explicit_ab2', 'LTS', 'FB_LTS' Default: Defined in namelist_defaults.xml @@ -245,7 +245,7 @@ Default: Defined in namelist_defaults.xml -Reference cell width. If config_hmix_use_ref_cell_width = .true., then hmix coefficients are set to be nu_{2h} = config_mom_del2*(cellWidth / config_hmix_use_ref_cell_width) and nu_{4h} = config_mom_del4*(cellWidth / config_hmix_use_ref_cell_width)^3 where cellWidth is the effective cell width computed as 2*sqrt(areaCell/pi). See Hoch et al 2020 JAMES eq 1,2. This relation applies within a simulation, but also generally among multiple simulations, so the parameters config_mom_del2, config_mom_del4, and config_hmix_use_ref_cell_width can generally remain at their standard values, and just be adjusted for fine tuning. +Reference cell width. If config_hmix_use_ref_cell_width = .true., then hmix coefficients are set to be $nu_{2h}$ = config_mom_del2*(cellWidth / config_hmix_use_ref_cell_width) and $nu_{4h}$ = config_mom_del4*(cellWidth / config_hmix_use_ref_cell_width)$^3$ where cellWidth is the effective cell width computed as 2*sqrt(areaCell/pi). See Hoch et al 2020 JAMES eq 1,2. This relation applies within a simulation, but also generally among multiple simulations, so the parameters config_mom_del2, config_mom_del4, and config_hmix_use_ref_cell_width can generally remain at their standard values, and just be adjusted for fine tuning. Valid values: Any positive real number, but typically a resolution number such as 30km. Default: Defined in namelist_defaults.xml @@ -307,7 +307,7 @@ Default: Defined in namelist_defaults.xml -Coefficient for horizontal biharmonic operator on momentum. If config_hmix_use_ref_cell_width = .true. then $\nu_{4h}$ = config_mom_del4*(cellWidth / config_hmix_use_ref_cell_width)^3. If config_hmix_use_ref_cell_width = .false. then it is referenced to the smallest cell. +Coefficient for horizontal biharmonic operator on momentum. If config_hmix_use_ref_cell_width = .true. then $\nu_{4h}$ = config_mom_del4*(cellWidth / config_hmix_use_ref_cell_width)$^3$. If config_hmix_use_ref_cell_width = .false. then it is referenced to the smallest cell. Valid values: any positive real Default: Defined in namelist_defaults.xml @@ -331,7 +331,7 @@ Default: Defined in namelist_defaults.xml -Coefficient for horizontal biharmonic operator on tracers. If config_hmix_use_ref_cell_width = .true. then $\nu_{4h}$ = config_tracer_del4*(cellWidth / config_hmix_use_ref_cell_width)^3. If config_hmix_use_ref_cell_width = .false. then it is referenced to the smallest cell. +Coefficient for horizontal biharmonic operator on tracers. If config_hmix_use_ref_cell_width = .true. then $\nu_{4h}$ = config_tracer_del4*(cellWidth / config_hmix_use_ref_cell_width)$^3$. If config_hmix_use_ref_cell_width = .false. then it is referenced to the smallest cell. Valid values: any positive real Default: Defined in namelist_defaults.xml @@ -747,6 +747,22 @@ Valid values: Any positive real value. Default: Defined in namelist_defaults.xml + +Background vertical diffusion applied to passive tracer quantities + +Valid values: Any positive real value. +Default: Defined in namelist_defaults.xml + + + +flag to enable using a different background vertical diffusion for passive tracers + +Valid values: .true. or .false. +Default: Defined in namelist_defaults.xml + + Background vertical viscosity applied to horizontal velocity @@ -1188,7 +1204,7 @@ Default: Defined in namelist_defaults.xml -Name of shortwave absorption type used in simulation. +Name of shortwave absorption type used in simulation. Valid values: 'jerlov' or 'ohlmann00' or 'none' Default: Defined in namelist_defaults.xml @@ -1809,7 +1825,7 @@ Default: Defined in namelist_defaults.xml -If true applies r h^-1 instead of just r. +If true applies r h$^-1$ instead of just r. Valid values: .true. or .false. Default: Defined in namelist_defaults.xml @@ -2038,7 +2054,7 @@ Default: Defined in namelist_defaults.xml -number of large iterations over stages 1-3 +number of large iterations over stages 1-3; For the split_explicit_ab2 time integrator, this value only affects the first time step when it is not a restart run. For restart runs, this value has no effect on the split_explicit_ab2 time integrator. Valid values: any positive integer, but typically 1, 2, or 3 Default: Defined in namelist_defaults.xml @@ -2640,6 +2656,17 @@ Default: Defined in namelist_defaults.xml + + + +Number of levels in subgrid lookup tables + +Valid values: Any positive non-zero integer. A value of -1 causes this to be overwritten with the configurations subgrid table levels definition. +Default: Defined in namelist_defaults.xml + + + 0: + # This is here because the comparison is run for each submission + # and we only want to compare once the whole run is finished. We + # need to return a pass here to continue the submission process. + self._test_status.set_status( + CIME.test_status.BASELINE_PHASE, CIME.test_status.TEST_PEND_STATUS + ) + return + + self._test_status.set_status( + CIME.test_status.BASELINE_PHASE, CIME.test_status.TEST_FAIL_STATUS + ) + + run_dir = self._case.get_value("RUNDIR") + case_name = self._case.get_value("CASE") + base_dir = os.path.join( + self._case.get_value("BASELINE_ROOT"), + self._case.get_value("BASECMP_CASE"), + ) + + test_name = str(case_name.split(".")[-1]) + evv_config = { + test_name: { + "module": os.path.join(evv_lib_dir, "extensions", "kso.py"), + "test-case": "Test", + "test-dir": run_dir, + "ref-case": "Baseline", + "ref-dir": base_dir, + "var-set": "default", + "ninst": NINST, + "critical": 0, + "component": self.ocn_component, + "alpha": 0.05, + "hist-name": "hist.am.timeSeriesStatsClimatology", + } + } + + json_file = os.path.join(run_dir, ".".join([case_name, "json"])) + with open(json_file, "w", encoding="utf-8") as config_file: + json.dump(evv_config, config_file, indent=4) + + evv_out_dir = os.path.join(run_dir, ".".join([case_name, "evv"])) + evv(["-e", json_file, "-o", evv_out_dir]) + + with open( + os.path.join(evv_out_dir, "index.json"), encoding="utf-8" + ) as evv_f: + evv_status = json.load(evv_f) + + comments = "" + for evv_ele in evv_status["Page"]["elements"]: + if "Table" in evv_ele: + comments = "; ".join( + f"{key}: {val[0]}" + for key, val in evv_ele["Table"]["data"].items() + ) + if evv_ele["Table"]["data"]["Test status"][0].lower() == "pass": + self._test_status.set_status( + CIME.test_status.BASELINE_PHASE, + CIME.test_status.TEST_PASS_STATUS, + ) + break + + status = self._test_status.get_status(CIME.test_status.BASELINE_PHASE) + mach_name = self._case.get_value("MACH") + mach_obj = Machines(machine=mach_name) + htmlroot = CIME.utils.get_htmlroot(mach_obj) + urlroot = CIME.utils.get_urlroot(mach_obj) + if htmlroot is not None: + with CIME.utils.SharedArea(): + dir_util.copy_tree( + evv_out_dir, + os.path.join(htmlroot, "evv", case_name), + preserve_mode=False, + ) + if urlroot is None: + urlroot = f"[{mach_name.capitalize()}_URL]" + viewing = f"{urlroot}/evv/{case_name}/index.html" + else: + viewing = ( + f"{evv_out_dir}\n" + " EVV viewing instructions can be found at: " + " https://github.com/E3SM-Project/E3SM/blob/master/cime/scripts/" + "climate_reproducibility/README.md#test-passfail-and-extended-output" + ) + comments = ( + f"{CIME.test_status.BASELINE_PHASE} {status} for test '{test_name}'.\n" + f" {comments}\n" + " EVV results can be viewed at:\n" + f" {viewing}" + ) + + CIME.utils.append_testlog(comments, self._orig_caseroot) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index a363c27883a3..47076dee726a 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -134,6 +134,8 @@ def buildnml(case, caseroot, compname): if ocn_ic_mode == 'spunup': logger.warning("WARNING: The specified compset is requesting ocean ICs spunup from a G-case") logger.warning(" But no file available for this grid.") + if ocn_ismf == 'data': + data_ismf_file = 'prescribed_ismf_paolo2023.oQU240wLI.20240404.nc' elif ocn_grid == 'oQU120': decomp_date = '230424' @@ -270,6 +272,58 @@ def buildnml(case, caseroot, compname): if ocn_ismf == 'data': data_ismf_file = 'prescribed_ismf_adusumilli2020.SOwISC12to60E2r4.230516.nc' + elif ocn_grid == 'FRISwISC08to60E3r1': + decomp_date = '20230913' # changed to date of partiotions in ../files_for_e3sm/assembled_files/inputdata/ocn/mpas-o/FRISwISC08to60E3r1/partitions + decomp_prefix = 'partitions/mpas-o.graph.info.' + restoring_file = 'sss.PHC2_monthlyClimatology.FRISwISC08to60E3r1.20230913.nc' + analysis_mask_file = 'FRISwISC08to60E3r1_mocBasinsAndTransects20210623.nc' + ic_date = '20230913' + ic_prefix = 'mpaso.FRISwISC08to60E3r1' # from ../files_for_e3sm/assembled_files/inputdata/ocn/mpas-o/FRISwISC08to60E3r1/mpaso.FRISwISC08to60E3r1.20230913.nc + if ocn_ic_mode == 'spunup': + ic_date = '20230913' # changed to same as decomp_date, but the spun up file does not yet exist + ic_prefix = 'mpaso.FRISwISC08to60E3r1.rstFromG-anvil' # the spun up file does not yet exist + if ocn_ismf == 'data': + data_ismf_file = 'prescribed_ismf_adusumilli2020.FRISwISC08to60E3r1.20230913.nc' + + elif ocn_grid == 'FRISwISC04to60E3r1': + decomp_date = '20230913' # changed to date of partiotions in ../files_for_e3sm/assembled_files/inputdata/ocn/mpas-o/FRISwISC04to60E3r1/partitions + decomp_prefix = 'partitions/mpas-o.graph.info.' + restoring_file = 'sss.PHC2_monthlyClimatology.FRISwISC04to60E3r1.20230913.nc' + analysis_mask_file = 'FRISwISC04to60E3r1_mocBasinsAndTransects20210623.nc' + ic_date = '20230913' + ic_prefix = 'mpaso.FRISwISC04to60E3r1' # from ../files_for_e3sm/assembled_files/inputdata/ocn/mpas-o/FRISwISC04to60E3r1/mpaso.FRISwISC04to60E3r1.20230913.nc + if ocn_ic_mode == 'spunup': + ic_date = '20230913' # changed to same as decomp_date, but the spun up file does not yet exist + ic_prefix = 'mpaso.FRISwISC04to60E3r1.rstFromG-anvil' # the spun up file does not yet exist + if ocn_ismf == 'data': + data_ismf_file = 'prescribed_ismf_adusumilli2020.FRISwISC04to60E3r1.20230913.nc' + + elif ocn_grid == 'FRISwISC02to60E3r1': + decomp_date = '20230914' # changed to date of partiotions in ../files_for_e3sm/assembled_files/inputdata/ocn/mpas-o/FRISwISC02to60E3r1/partitions + decomp_prefix = 'partitions/mpas-o.graph.info.' + restoring_file = 'sss.PHC2_monthlyClimatology.FRISwISC02to60E3r1.20230914.nc' + analysis_mask_file = 'FRISwISC02to60E3r1_mocBasinsAndTransects20210623.nc' + ic_date = '20230914' + ic_prefix = 'mpaso.FRISwISC02to60E3r1' # from ../files_for_e3sm/assembled_files/inputdata/ocn/mpas-o/FRISwISC02to60E3r1/mpaso.FRISwISC02to60E3r1.20230914.nc + if ocn_ic_mode == 'spunup': + ic_date = '20230914' # changed to same as decomp_date, but the spun up file does not yet exist + ic_prefix = 'mpaso.FRISwISC02to60E3r1.rstFromG-anvil' # the spun up file does not yet exist + if ocn_ismf == 'data': + data_ismf_file = 'prescribed_ismf_adusumilli2020.FRISwISC02to60E3r1.20230914.nc' + + elif ocn_grid == 'FRISwISC01to60E3r1': + decomp_date = '20230915' # changed to date of partiotions in ../files_for_e3sm/assembled_files/inputdata/ocn/mpas-o/FRISwISC01to60E3r1/partitions + decomp_prefix = 'partitions/mpas-o.graph.info.' + restoring_file = 'sss.PHC2_monthlyClimatology.FRISwISC01to60E3r1.20230915.nc' + analysis_mask_file = 'FRISwISC01to60E3r1_mocBasinsAndTransects20210623.nc' + ic_date = '20230915' + ic_prefix = 'mpaso.FRISwISC01to60E3r1' # from ../files_for_e3sm/assembled_files/inputdata/ocn/mpas-o/FRISwISC01to60E3r1/mpaso.FRISwISC01to60E3r1.20230915.nc + if ocn_ic_mode == 'spunup': + ic_date = '20230915' # changed to same as decomp_date, but the spun up file does not yet exist + ic_prefix = 'mpaso.FRISwISC01to60E3r1.rstFromG-anvil' # the spun up file does not yet exist + if ocn_ismf == 'data': + data_ismf_file = 'prescribed_ismf_adusumilli2020.FRISwISC01to60E3r1.20230915.nc' + elif ocn_grid == 'ECwISC30to60E2r1': decomp_date = '200915' decomp_prefix = 'mpas-o.graph.info.' @@ -293,8 +347,23 @@ def buildnml(case, caseroot, compname): if ocn_ic_mode == 'spunup': ic_date = '20231121' ic_prefix = 'mpaso.IcoswISC30E3r5.rstFromG-chrysalis' + if ocn_bgc in ['eco_only', 'eco_and_dms', 'eco_and_macromolecules', 'eco_and_dms_and_macromolecules']: + ic_date = '20231215' + ic_prefix = 'mpaso.IcoswISC30E3r5.20231120+MARBL_ICfromOMIP_64levels' + eco_forcing_file = 'ecoForcingSurfaceMonthly.IcoswISC30E3r5.20231215.nc' if ocn_ismf == 'data': - data_ismf_file = 'prescribed_ismf_adusumilli2020.IcoswISC30E3r5.20231120.nc' + data_ismf_file = 'prescribed_ismf_paolo2023.IcoswISC30E3r5.20240227.nc' + + elif ocn_grid == 'IcosXISC30E3r7': + decomp_date = '20240314' + decomp_prefix = 'partitions/mpas-o.graph.info.' + restoring_file = 'sss.PHC2_monthlyClimatology.IcosXISC30E3r7.20240314.nc' + analysis_mask_file = 'IcosXISC30E3r7_mocBasinsAndTransects20210623.nc' + ic_date = '20240314' + ic_prefix = 'mpaso.IcosXISC30E3r7' + if ocn_ic_mode == 'spunup': + ic_date = '20240314' + ic_prefix = 'mpaso.IcosXISC30E3r7.rstFromPiControlSpinup-chrysalis' #-------------------------------------------------------------------- # Set OCN_FORCING = datm_forced_restoring if restoring file is available @@ -422,7 +491,7 @@ def buildnml(case, caseroot, compname): lines.append('') lines.append('') lines.append('') lines.append('') lines.append(' ') lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') if ocn_bgc in ['eco_only', 'eco_and_dms', 'eco_and_macromolecules', 'eco_and_dms_and_macromolecules']: @@ -755,7 +826,7 @@ def buildnml(case, caseroot, compname): lines.append('') lines.append(' ') lines.append(' ') - lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') if ocn_bgc in ['eco_only', 'eco_and_dms', 'eco_and_macromolecules', 'eco_and_dms_and_macromolecules']: @@ -1006,7 +1077,7 @@ def buildnml(case, caseroot, compname): lines.append(' filename_interval="00-01-00_00:00:00"') lines.append(' reference_time="01-01-01_00:00:00"') lines.append(' output_interval="00-01-00_00:00:00"') - lines.append(' clobber_mode="append"') + lines.append(' clobber_mode="truncate"') lines.append(' packages="conservationCheckAMPKG">') lines.append('') lines.append('') @@ -1135,7 +1206,7 @@ def buildnml(case, caseroot, compname): lines.append('') lines.append(' ') + if ocn_grid.startswith("FRIS"): + lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') @@ -1227,6 +1301,8 @@ def buildnml(case, caseroot, compname): lines.append(' ') lines.append(' ') lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') @@ -1362,7 +1438,7 @@ def buildnml(case, caseroot, compname): lines.append('') lines.append(' ') lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') @@ -1424,7 +1502,7 @@ def buildnml(case, caseroot, compname): lines.append('') lines.append(' ') lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') @@ -1527,7 +1607,7 @@ def buildnml(case, caseroot, compname): lines.append('') lines.append('2000_DATM%JRA-1p5_SLND_MPASSI_MPASO%DATMFORCED_DROF%JRA-1p5_SGLC_SWAV + + GMPAS-OECO-JRA1p4 + 2000_DATM%JRA-1p4-2018_SLND_MPASSI_MPASO%OECODATMFORCED_DROF%JRA-1p4-2018_SGLC_SWAV + + + + GMPAS-OIECO-JRA1p4 + 2000_DATM%JRA-1p4-2018_SLND_MPASSI%BGC_MPASO%OIECODATMFORCED_DROF%JRA-1p4-2018_SGLC_SWAV + + + + GPMPAS-OECO-JRA1p4 + 2000_DATM%JRA-1p4-2018_ELM%SPBC_MPASSI_MPASO%OECODATMFORCED_MOSART_SGLC_SWAV + + + + GPMPAS-OIECO-JRA1p4 + 2000_DATM%JRA-1p4-2018_ELM%SPBC_MPASSI%BGC_MPASO%OIECODATMFORCED_MOSART_SGLC_SWAV + + GMPAS-JRA1p5-DIB-PISMF 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV diff --git a/components/mpas-ocean/cime_config/config_tests.xml b/components/mpas-ocean/cime_config/config_tests.xml new file mode 100644 index 000000000000..a89986257e86 --- /dev/null +++ b/components/mpas-ocean/cime_config/config_tests.xml @@ -0,0 +1,23 @@ + + + + + + + + climate reproducibility test using the multivariate K-S test + 1 + FALSE + FALSE + nmonths + 24 + $STOP_OPTION + $STOP_N + $STOP_OPTION + $STOP_N + 0 + + + diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/jra_1958/shell_commands b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/jra_1958/shell_commands new file mode 100644 index 000000000000..1d43ad8c5baf --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/jra_1958/shell_commands @@ -0,0 +1,4 @@ +./xmlchange DATM_CLMNCEP_YR_START=1958 +./xmlchange DATM_CLMNCEP_YR_END=1958 +./xmlchange DROF_STRM_YR_START=1958 +./xmlchange DROF_STRM_YR_END=1958 diff --git a/components/mpas-ocean/docs/ccase.template.v3LR.anvil.sh b/components/mpas-ocean/docs/ccase.template.v3LR.anvil.sh new file mode 100755 index 000000000000..ed9af7110589 --- /dev/null +++ b/components/mpas-ocean/docs/ccase.template.v3LR.anvil.sh @@ -0,0 +1,498 @@ +#!/bin/bash -fe + +# E3SM Coupled Model Group run_e3sm script template. +# +# Bash coding style inspired by: +# http://kfirlavi.herokuapp.com/blog/2012/11/14/defensive-bash-programming + +main() { + +# For debugging, uncomment libe below +#set -x + +# --- Configuration flags ---- + +# Machine and project +readonly MACHINE=anvil +readonly PROJECT="condo" + +# Simulation +readonly COMPSET="CMPASO-JRA1p4" +readonly RESOLUTION="TL319_IcoswISC30E3r5" +readonly CASE_NAME="test.CMPAS-JRA.TL319_IcoswISC30E3r5.anvil" # MODIFY +#readonly CASE_GROUP="" + +# Code and compilation +readonly CHECKOUT="20240304" +readonly BRANCH="399d430" # master as of 2024-03-04 E3Sm v3.0.0 tag +#readonly CHERRY=() +readonly DEBUG_COMPILE=false + +# Run options +readonly MODEL_START_TYPE="initial" # 'initial', 'continue', 'branch', 'hybrid' +readonly START_DATE="0001-01-01" + +# Additional options for 'branch' and 'hybrid' +# readonly GET_REFCASE=TRUE +# readonly RUN_REFDIR="" +# readonly RUN_REFCASE="" +# readonly RUN_REFDATE="0301-01-01" + +# Set paths +readonly CODE_ROOT="/lcrc/group/e3sm/${USER}/E3SM/code/${CHECKOUT}" +readonly CASE_ROOT="/lcrc/group/e3sm/${USER}/e3sm_scratch/${MACHINE}/${CASE_NAME}" + +# Sub-directories +readonly CASE_BUILD_DIR=${CASE_ROOT}/build +readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/archive + +# Define type of run +# short tests: 'S_2x5_ndays', 'M_1x10_ndays', 'M80_1x10_ndays' +# or 'production' for full simulation + +#readonly run='S_1x1_nmonth' +readonly run='production' +if [ "${run}" != "production" ]; then + + # Short test simulations + tmp=($(echo $run | tr "_" " ")) + layout=${tmp[0]} + units=${tmp[2]} + resubmit=$(( ${tmp[1]%%x*} -1 )) + length=${tmp[1]##*x} + + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/tests/${run}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/tests/${run}/run + readonly PELAYOUT=${layout} + readonly WALLTIME="2:00:00" + readonly STOP_OPTION=${units} + readonly STOP_N=${length} + readonly REST_OPTION=${STOP_OPTION} + readonly REST_N=${STOP_N} + readonly RESUBMIT=${resubmit} + readonly DO_SHORT_TERM_ARCHIVING=false + +else + + # Production simulation + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/run + readonly PELAYOUT="L" + readonly WALLTIME="02:00:00" + readonly STOP_OPTION="nmonths" + readonly STOP_N="1" + readonly REST_OPTION="nmonths" + readonly REST_N="1" + readonly RESUBMIT="0" + readonly DO_SHORT_TERM_ARCHIVING=false +fi + +# Coupler history +readonly HIST_OPTION="nmonths" +readonly HIST_N="1" + +# Leave empty (unless you understand what it does) +readonly OLD_EXECUTABLE="" + +# --- Toggle flags for what to do ---- +do_fetch_code=false +do_create_newcase=true +do_modify_pelayout=false +do_case_setup=true +do_case_build=true +do_case_submit=true + +# --- Now, do the work --- + +# Make directories created by this script world-readable +umask 022 + +# Fetch code from Github +fetch_code + +# Create case +create_newcase + +# Custom PE layout +modify_pelayout + +# Setup +case_setup + +# Build +case_build + +# Configure runtime options +runtime_options + +# Copy script into case_script directory for provenance +copy_script + +# Submit +case_submit + +# All done +echo $'\n----- All done -----\n' + +} + +# ======================= +# Custom user_nl settings +# ======================= + +user_nl() { + +cat << EOF >> user_nl_eam +EOF + +cat << EOF >> user_nl_elm +EOF + +cat << EOF >> user_nl_mpaso +EOF + +cat << EOF >> user_nl_mpassi +EOF + +} + +# ===================================== +# Customize MPAS stream files if needed +# ===================================== + +patch_mpas_streams() { + +echo + +} + +# ===================================================== +# Custom PE layout: +# ===================================================== + +modify_pelayout() { + + if [ "${do_modify_pelayout,,}" != "true" ]; then + echo $'\n----- Skipping changing PE-layout -----\n' + return + fi + + echo $'\n----- changing PE layout to custom -----\n' + pushd ${CASE_SCRIPTS_DIR} + + ./xmlchange MAX_MPITASKS_PER_NODE=32 + ./xmlchange MAX_TASKS_PER_NODE=256 + ./xmlchange NTASKS_ATM=1024 + ./xmlchange NTASKS_CPL=1024 + ./xmlchange NTASKS_OCN=1024 + ./xmlchange NTASKS_WAV=1024 + ./xmlchange NTASKS_GLC=1024 + ./xmlchange NTASKS_ICE=1024 + ./xmlchange NTASKS_ROF=1024 + ./xmlchange NTASKS_LND=1024 + ./xmlchange NTASKS_ESP=1 + ./xmlchange NTASKS_IAC=1 + + ./xmlchange NTHRDS_ATM=8 + ./xmlchange NTHRDS_CPL=8 + ./xmlchange NTHRDS_OCN=8 + ./xmlchange NTHRDS_WAV=8 + ./xmlchange NTHRDS_GLC=1 + ./xmlchange NTHRDS_ICE=8 + ./xmlchange NTHRDS_ROF=8 + ./xmlchange NTHRDS_LND=8 + ./xmlchange NTHRDS_ESP=1 + ./xmlchange NTHRDS_IAC=1 + + ./xmlchange ROOTPE_ATM=0 + ./xmlchange ROOTPE_CPL=0 + ./xmlchange ROOTPE_OCN=0 + ./xmlchange ROOTPE_WAV=0 + ./xmlchange ROOTPE_GLC=0 + ./xmlchange ROOTPE_ICE=0 + ./xmlchange ROOTPE_ROF=0 + ./xmlchange ROOTPE_LND=0 + ./xmlchange ROOTPE_ESP=0 + ./xmlchange ROOTPE_IAC=0 + + popd + + } +###################################################### +### Most users won't need to change anything below ### +###################################################### + +#----------------------------------------------------- +fetch_code() { + + if [ "${do_fetch_code,,}" != "true" ]; then + echo $'\n----- Skipping fetch_code -----\n' + return + fi + + echo $'\n----- Starting fetch_code -----\n' + local path=${CODE_ROOT} + local repo=E3SM + + echo "Cloning $repo repository branch $BRANCH under $path" + if [ -d "${path}" ]; then + echo "ERROR: Directory already exists. Not overwriting" + exit 20 + fi + mkdir -p ${path} + pushd ${path} + + # This will put repository, with all code + git clone git@github.com:E3SM-Project/${repo}.git . + + # Check out desired branch + git checkout ${BRANCH} + + # Custom addition + if [ "${CHERRY}" != "" ]; then + echo ----- WARNING: adding git cherry-pick ----- + for commit in "${CHERRY[@]}" + do + echo ${commit} + git cherry-pick ${commit} + done + echo ------------------------------------------- + fi + + # Bring in all submodule components + git submodule update --init --recursive + + popd +} + +#----------------------------------------------------- +create_newcase() { + + if [ "${do_create_newcase,,}" != "true" ]; then + echo $'\n----- Skipping create_newcase -----\n' + return + fi + + echo $'\n----- Starting create_newcase -----\n' + + if [[ ${PELAYOUT} == custom-* ]]; + then + layout="M" # temporary placeholder for create_newcase + else + layout=${PELAYOUT} + + fi + ${CODE_ROOT}/cime/scripts/create_newcase \ + --case ${CASE_NAME} \ + --output-root ${CASE_ROOT} \ + --script-root ${CASE_SCRIPTS_DIR} \ + --handle-preexisting-dirs u \ + --compset ${COMPSET} \ + --res ${RESOLUTION} \ + --machine ${MACHINE} ${COMPILER} \ + --project ${PROJECT} \ + --walltime ${WALLTIME} \ + --pecount ${layout} \ + + if [ $? != 0 ]; then + echo $'\nNote: if create_newcase failed because sub-directory already exists:' + echo $' * delete old case_script sub-directory' + echo $' * or set do_newcase=false\n' + exit 35 + fi + +} + +#----------------------------------------------------- +case_setup() { + + if [ "${do_case_setup,,}" != "true" ]; then + echo $'\n----- Skipping case_setup -----\n' + return + fi + + echo $'\n----- Starting case_setup -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Setup some CIME directories + ./xmlchange EXEROOT=${CASE_BUILD_DIR} + ./xmlchange RUNDIR=${CASE_RUN_DIR} + + # Short term archiving + ./xmlchange DOUT_S=${DO_SHORT_TERM_ARCHIVING^^} + ./xmlchange DOUT_S_ROOT=${CASE_ARCHIVE_DIR} + + # Build with COSP, except for a data atmosphere (datm) + if [ `./xmlquery --value COMP_ATM` == "datm" ]; then + echo $'\nThe specified configuration uses a data atmosphere, so cannot activate COSP simulator\n' + else + echo $'\nConfiguring E3SM to use the COSP simulator\n' + ./xmlchange --id CAM_CONFIG_OPTS --append --val='-cosp' + fi + + # Extracts input_data_dir in case it is needed for user edits to the namelist later + local input_data_dir=`./xmlquery DIN_LOC_ROOT --value` + + # Custom user_nl + user_nl + + # Finally, run CIME case.setup + ./case.setup --reset + + popd +} + +#----------------------------------------------------- +case_build() { + + pushd ${CASE_SCRIPTS_DIR} + + # do_case_build = false + if [ "${do_case_build,,}" != "true" ]; then + + echo $'\n----- case_build -----\n' + + if [ "${OLD_EXECUTABLE}" == "" ]; then + # Ues previously built executable, make sure it exists + if [ -x ${CASE_BUILD_DIR}/e3sm.exe ]; then + echo 'Skipping build because $do_case_build = '${do_case_build} + else + echo 'ERROR: $do_case_build = '${do_case_build}' but no executable exists for this case.' + exit 297 + fi + else + # If absolute pathname exists and is executable, reuse pre-exiting executable + if [ -x ${OLD_EXECUTABLE} ]; then + echo 'Using $OLD_EXECUTABLE = '${OLD_EXECUTABLE} + cp -fp ${OLD_EXECUTABLE} ${CASE_BUILD_DIR}/ + else + echo 'ERROR: $OLD_EXECUTABLE = '$OLD_EXECUTABLE' does not exist or is not an executable file.' + exit 297 + fi + fi + echo 'WARNING: Setting BUILD_COMPLETE = TRUE. This is a little risky, but trusting the user.' + ./xmlchange BUILD_COMPLETE=TRUE + + # do_case_build = true + else + + echo $'\n----- Starting case_build -----\n' + + # Turn on debug compilation option if requested + if [ "${DEBUG_COMPILE^^}" == "TRUE" ]; then + ./xmlchange DEBUG=${DEBUG_COMPILE^^} + fi + + # Run CIME case.build + ./case.build + + fi + + # Some user_nl settings won't be updated to *_in files under the run directory + # Call preview_namelists to make sure *_in and user_nl files are consistent. + echo $'\n----- Preview namelists -----\n' + ./preview_namelists + + popd +} + +#----------------------------------------------------- +runtime_options() { + + echo $'\n----- Starting runtime_options -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Set simulation start date + ./xmlchange RUN_STARTDATE=${START_DATE} + + # Segment length + ./xmlchange STOP_OPTION=${STOP_OPTION,,},STOP_N=${STOP_N} + + # Restart frequency + ./xmlchange REST_OPTION=${REST_OPTION,,},REST_N=${REST_N} + + # Coupler history + ./xmlchange HIST_OPTION=${HIST_OPTION,,},HIST_N=${HIST_N} + + # Coupler budgets (always on) + ./xmlchange BUDGETS=TRUE + + # Set resubmissions + if (( RESUBMIT > 0 )); then + ./xmlchange RESUBMIT=${RESUBMIT} + fi + + # Run type + # Start from default of user-specified initial conditions + if [ "${MODEL_START_TYPE,,}" == "initial" ]; then + ./xmlchange RUN_TYPE="startup" + ./xmlchange CONTINUE_RUN="FALSE" + + # Continue existing run + elif [ "${MODEL_START_TYPE,,}" == "continue" ]; then + ./xmlchange CONTINUE_RUN="TRUE" + + elif [ "${MODEL_START_TYPE,,}" == "branch" ] || [ "${MODEL_START_TYPE,,}" == "hybrid" ]; then + ./xmlchange RUN_TYPE=${MODEL_START_TYPE,,} + ./xmlchange GET_REFCASE=${GET_REFCASE} + ./xmlchange RUN_REFDIR=${RUN_REFDIR} + ./xmlchange RUN_REFCASE=${RUN_REFCASE} + ./xmlchange RUN_REFDATE=${RUN_REFDATE} + echo 'Warning: $MODEL_START_TYPE = '${MODEL_START_TYPE} + echo '$RUN_REFDIR = '${RUN_REFDIR} + echo '$RUN_REFCASE = '${RUN_REFCASE} + echo '$RUN_REFDATE = '${START_DATE} + else + echo 'ERROR: $MODEL_START_TYPE = '${MODEL_START_TYPE}' is unrecognized. Exiting.' + exit 380 + fi + + # Patch mpas streams files + patch_mpas_streams + + popd +} + +#----------------------------------------------------- +case_submit() { + + if [ "${do_case_submit,,}" != "true" ]; then + echo $'\n----- Skipping case_submit -----\n' + return + fi + + echo $'\n----- Starting case_submit -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Run CIME case.submit + ./case.submit + + popd +} + +#----------------------------------------------------- +copy_script() { + + echo $'\n----- Saving run script for provenance -----\n' + + local script_provenance_dir=${CASE_SCRIPTS_DIR}/run_script_provenance + mkdir -p ${script_provenance_dir} + local this_script_name=`basename $0` + local script_provenance_name=${this_script_name}.`date +%Y%m%d-%H%M%S` + cp -vp ${this_script_name} ${script_provenance_dir}/${script_provenance_name} + +} + +#----------------------------------------------------- +# Silent versions of popd and pushd +pushd() { + command pushd "$@" > /dev/null +} +popd() { + command popd "$@" > /dev/null +} + +# Now, actually run the script +#----------------------------------------------------- +main + diff --git a/components/mpas-ocean/docs/dev-guide/index.md b/components/mpas-ocean/docs/dev-guide/index.md new file mode 100644 index 000000000000..c7665da944cf --- /dev/null +++ b/components/mpas-ocean/docs/dev-guide/index.md @@ -0,0 +1,5 @@ +# MPAS-Ocean Developer Guide + +The E3SM developer guide may be found at [https://e3sm.org/model/running-e3sm/developing-e3sm/](https://e3sm.org/model/running-e3sm/developing-e3sm/). + +The MPAS developers guide is at [https://mpas-dev.github.io/files/documents/MPAS-DevelopersGuide.pdf](https://mpas-dev.github.io/files/documents/MPAS-DevelopersGuide.pdf). diff --git a/components/mpas-ocean/docs/gcase.template.v3LR.chrysalis.sh b/components/mpas-ocean/docs/gcase.template.v3LR.chrysalis.sh new file mode 100755 index 000000000000..b9fff6197f85 --- /dev/null +++ b/components/mpas-ocean/docs/gcase.template.v3LR.chrysalis.sh @@ -0,0 +1,498 @@ +#!/bin/bash -fe + +# E3SM Coupled Model Group run_e3sm script template. +# +# Bash coding style inspired by: +# http://kfirlavi.herokuapp.com/blog/2012/11/14/defensive-bash-programming + +main() { + +# For debugging, uncomment libe below +#set -x + +# --- Configuration flags ---- + +# Machine and project +readonly MACHINE=chrysalis +readonly PROJECT="e3sm" + +# Simulation +readonly COMPSET="GMPAS-JRA1p5" +readonly RESOLUTION="TL319_IcoswISC30E3r5" +readonly CASE_NAME="test4.GMPAS-JRA.TL319_IcoswISC30E3r5.chrysalis" # MODIFY +#readonly CASE_GROUP="" + +# Code and compilation +readonly CHECKOUT="20240304" +readonly BRANCH="399d430" # master as of 2024-03-04 E3Sm v3.0.0 tag +#readonly CHERRY=() +readonly DEBUG_COMPILE=false + +# Run options +readonly MODEL_START_TYPE="initial" # 'initial', 'continue', 'branch', 'hybrid' +readonly START_DATE="0001-01-01" + +# Additional options for 'branch' and 'hybrid' +# readonly GET_REFCASE=TRUE +# readonly RUN_REFDIR="" +# readonly RUN_REFCASE="" +# readonly RUN_REFDATE="0301-01-01" + +# Set paths +readonly CODE_ROOT="/lcrc/group/e3sm/${USER}/E3SM/code/${CHECKOUT}" +readonly CASE_ROOT="/lcrc/group/e3sm/${USER}/e3sm_scratch/${MACHINE}/${CASE_NAME}" + +# Sub-directories +readonly CASE_BUILD_DIR=${CASE_ROOT}/build +readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/archive + +# Define type of run +# short tests: 'S_2x5_ndays', 'M_1x10_ndays', 'M80_1x10_ndays' +# or 'production' for full simulation + +#readonly run='S_1x1_nmonth' +readonly run='production' +if [ "${run}" != "production" ]; then + + # Short test simulations + tmp=($(echo $run | tr "_" " ")) + layout=${tmp[0]} + units=${tmp[2]} + resubmit=$(( ${tmp[1]%%x*} -1 )) + length=${tmp[1]##*x} + + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/tests/${run}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/tests/${run}/run + readonly PELAYOUT=${layout} + readonly WALLTIME="2:00:00" + readonly STOP_OPTION=${units} + readonly STOP_N=${length} + readonly REST_OPTION=${STOP_OPTION} + readonly REST_N=${STOP_N} + readonly RESUBMIT=${resubmit} + readonly DO_SHORT_TERM_ARCHIVING=false + +else + + # Production simulation + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/run + readonly PELAYOUT="L" + readonly WALLTIME="02:00:00" + readonly STOP_OPTION="nmonths" + readonly STOP_N="1" + readonly REST_OPTION="nmonths" + readonly REST_N="1" + readonly RESUBMIT="0" + readonly DO_SHORT_TERM_ARCHIVING=false +fi + +# Coupler history +readonly HIST_OPTION="nmonths" +readonly HIST_N="1" + +# Leave empty (unless you understand what it does) +readonly OLD_EXECUTABLE="" + +# --- Toggle flags for what to do ---- +do_fetch_code=false +do_create_newcase=true +do_modify_pelayout=false +do_case_setup=true +do_case_build=true +do_case_submit=true + +# --- Now, do the work --- + +# Make directories created by this script world-readable +umask 022 + +# Fetch code from Github +fetch_code + +# Create case +create_newcase + +# Custom PE layout +modify_pelayout + +# Setup +case_setup + +# Build +case_build + +# Configure runtime options +runtime_options + +# Copy script into case_script directory for provenance +copy_script + +# Submit +case_submit + +# All done +echo $'\n----- All done -----\n' + +} + +# ======================= +# Custom user_nl settings +# ======================= + +user_nl() { + +cat << EOF >> user_nl_eam +EOF + +cat << EOF >> user_nl_elm +EOF + +cat << EOF >> user_nl_mpaso +EOF + +cat << EOF >> user_nl_mpassi +EOF + +} + +# ===================================== +# Customize MPAS stream files if needed +# ===================================== + +patch_mpas_streams() { + +echo + +} + +# ===================================================== +# Custom PE layout: +# ===================================================== + +modify_pelayout() { + + if [ "${do_modify_pelayout,,}" != "true" ]; then + echo $'\n----- Skipping changing PE-layout -----\n' + return + fi + + echo $'\n----- changing PE layout to custom -----\n' + pushd ${CASE_SCRIPTS_DIR} + + ./xmlchange MAX_MPITASKS_PER_NODE=32 + ./xmlchange MAX_TASKS_PER_NODE=256 + ./xmlchange NTASKS_ATM=1024 + ./xmlchange NTASKS_CPL=1024 + ./xmlchange NTASKS_OCN=1024 + ./xmlchange NTASKS_WAV=1024 + ./xmlchange NTASKS_GLC=1024 + ./xmlchange NTASKS_ICE=1024 + ./xmlchange NTASKS_ROF=1024 + ./xmlchange NTASKS_LND=1024 + ./xmlchange NTASKS_ESP=1 + ./xmlchange NTASKS_IAC=1 + + ./xmlchange NTHRDS_ATM=8 + ./xmlchange NTHRDS_CPL=8 + ./xmlchange NTHRDS_OCN=8 + ./xmlchange NTHRDS_WAV=8 + ./xmlchange NTHRDS_GLC=1 + ./xmlchange NTHRDS_ICE=8 + ./xmlchange NTHRDS_ROF=8 + ./xmlchange NTHRDS_LND=8 + ./xmlchange NTHRDS_ESP=1 + ./xmlchange NTHRDS_IAC=1 + + ./xmlchange ROOTPE_ATM=0 + ./xmlchange ROOTPE_CPL=0 + ./xmlchange ROOTPE_OCN=0 + ./xmlchange ROOTPE_WAV=0 + ./xmlchange ROOTPE_GLC=0 + ./xmlchange ROOTPE_ICE=0 + ./xmlchange ROOTPE_ROF=0 + ./xmlchange ROOTPE_LND=0 + ./xmlchange ROOTPE_ESP=0 + ./xmlchange ROOTPE_IAC=0 + + popd + + } +###################################################### +### Most users won't need to change anything below ### +###################################################### + +#----------------------------------------------------- +fetch_code() { + + if [ "${do_fetch_code,,}" != "true" ]; then + echo $'\n----- Skipping fetch_code -----\n' + return + fi + + echo $'\n----- Starting fetch_code -----\n' + local path=${CODE_ROOT} + local repo=E3SM + + echo "Cloning $repo repository branch $BRANCH under $path" + if [ -d "${path}" ]; then + echo "ERROR: Directory already exists. Not overwriting" + exit 20 + fi + mkdir -p ${path} + pushd ${path} + + # This will put repository, with all code + git clone git@github.com:E3SM-Project/${repo}.git . + + # Check out desired branch + git checkout ${BRANCH} + + # Custom addition + if [ "${CHERRY}" != "" ]; then + echo ----- WARNING: adding git cherry-pick ----- + for commit in "${CHERRY[@]}" + do + echo ${commit} + git cherry-pick ${commit} + done + echo ------------------------------------------- + fi + + # Bring in all submodule components + git submodule update --init --recursive + + popd +} + +#----------------------------------------------------- +create_newcase() { + + if [ "${do_create_newcase,,}" != "true" ]; then + echo $'\n----- Skipping create_newcase -----\n' + return + fi + + echo $'\n----- Starting create_newcase -----\n' + + if [[ ${PELAYOUT} == custom-* ]]; + then + layout="M" # temporary placeholder for create_newcase + else + layout=${PELAYOUT} + + fi + ${CODE_ROOT}/cime/scripts/create_newcase \ + --case ${CASE_NAME} \ + --output-root ${CASE_ROOT} \ + --script-root ${CASE_SCRIPTS_DIR} \ + --handle-preexisting-dirs u \ + --compset ${COMPSET} \ + --res ${RESOLUTION} \ + --machine ${MACHINE} ${COMPILER} \ + --project ${PROJECT} \ + --walltime ${WALLTIME} \ + --pecount ${layout} \ + + if [ $? != 0 ]; then + echo $'\nNote: if create_newcase failed because sub-directory already exists:' + echo $' * delete old case_script sub-directory' + echo $' * or set do_newcase=false\n' + exit 35 + fi + +} + +#----------------------------------------------------- +case_setup() { + + if [ "${do_case_setup,,}" != "true" ]; then + echo $'\n----- Skipping case_setup -----\n' + return + fi + + echo $'\n----- Starting case_setup -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Setup some CIME directories + ./xmlchange EXEROOT=${CASE_BUILD_DIR} + ./xmlchange RUNDIR=${CASE_RUN_DIR} + + # Short term archiving + ./xmlchange DOUT_S=${DO_SHORT_TERM_ARCHIVING^^} + ./xmlchange DOUT_S_ROOT=${CASE_ARCHIVE_DIR} + + # Build with COSP, except for a data atmosphere (datm) + if [ `./xmlquery --value COMP_ATM` == "datm" ]; then + echo $'\nThe specified configuration uses a data atmosphere, so cannot activate COSP simulator\n' + else + echo $'\nConfiguring E3SM to use the COSP simulator\n' + ./xmlchange --id CAM_CONFIG_OPTS --append --val='-cosp' + fi + + # Extracts input_data_dir in case it is needed for user edits to the namelist later + local input_data_dir=`./xmlquery DIN_LOC_ROOT --value` + + # Custom user_nl + user_nl + + # Finally, run CIME case.setup + ./case.setup --reset + + popd +} + +#----------------------------------------------------- +case_build() { + + pushd ${CASE_SCRIPTS_DIR} + + # do_case_build = false + if [ "${do_case_build,,}" != "true" ]; then + + echo $'\n----- case_build -----\n' + + if [ "${OLD_EXECUTABLE}" == "" ]; then + # Ues previously built executable, make sure it exists + if [ -x ${CASE_BUILD_DIR}/e3sm.exe ]; then + echo 'Skipping build because $do_case_build = '${do_case_build} + else + echo 'ERROR: $do_case_build = '${do_case_build}' but no executable exists for this case.' + exit 297 + fi + else + # If absolute pathname exists and is executable, reuse pre-exiting executable + if [ -x ${OLD_EXECUTABLE} ]; then + echo 'Using $OLD_EXECUTABLE = '${OLD_EXECUTABLE} + cp -fp ${OLD_EXECUTABLE} ${CASE_BUILD_DIR}/ + else + echo 'ERROR: $OLD_EXECUTABLE = '$OLD_EXECUTABLE' does not exist or is not an executable file.' + exit 297 + fi + fi + echo 'WARNING: Setting BUILD_COMPLETE = TRUE. This is a little risky, but trusting the user.' + ./xmlchange BUILD_COMPLETE=TRUE + + # do_case_build = true + else + + echo $'\n----- Starting case_build -----\n' + + # Turn on debug compilation option if requested + if [ "${DEBUG_COMPILE^^}" == "TRUE" ]; then + ./xmlchange DEBUG=${DEBUG_COMPILE^^} + fi + + # Run CIME case.build + ./case.build + + fi + + # Some user_nl settings won't be updated to *_in files under the run directory + # Call preview_namelists to make sure *_in and user_nl files are consistent. + echo $'\n----- Preview namelists -----\n' + ./preview_namelists + + popd +} + +#----------------------------------------------------- +runtime_options() { + + echo $'\n----- Starting runtime_options -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Set simulation start date + ./xmlchange RUN_STARTDATE=${START_DATE} + + # Segment length + ./xmlchange STOP_OPTION=${STOP_OPTION,,},STOP_N=${STOP_N} + + # Restart frequency + ./xmlchange REST_OPTION=${REST_OPTION,,},REST_N=${REST_N} + + # Coupler history + ./xmlchange HIST_OPTION=${HIST_OPTION,,},HIST_N=${HIST_N} + + # Coupler budgets (always on) + ./xmlchange BUDGETS=TRUE + + # Set resubmissions + if (( RESUBMIT > 0 )); then + ./xmlchange RESUBMIT=${RESUBMIT} + fi + + # Run type + # Start from default of user-specified initial conditions + if [ "${MODEL_START_TYPE,,}" == "initial" ]; then + ./xmlchange RUN_TYPE="startup" + ./xmlchange CONTINUE_RUN="FALSE" + + # Continue existing run + elif [ "${MODEL_START_TYPE,,}" == "continue" ]; then + ./xmlchange CONTINUE_RUN="TRUE" + + elif [ "${MODEL_START_TYPE,,}" == "branch" ] || [ "${MODEL_START_TYPE,,}" == "hybrid" ]; then + ./xmlchange RUN_TYPE=${MODEL_START_TYPE,,} + ./xmlchange GET_REFCASE=${GET_REFCASE} + ./xmlchange RUN_REFDIR=${RUN_REFDIR} + ./xmlchange RUN_REFCASE=${RUN_REFCASE} + ./xmlchange RUN_REFDATE=${RUN_REFDATE} + echo 'Warning: $MODEL_START_TYPE = '${MODEL_START_TYPE} + echo '$RUN_REFDIR = '${RUN_REFDIR} + echo '$RUN_REFCASE = '${RUN_REFCASE} + echo '$RUN_REFDATE = '${START_DATE} + else + echo 'ERROR: $MODEL_START_TYPE = '${MODEL_START_TYPE}' is unrecognized. Exiting.' + exit 380 + fi + + # Patch mpas streams files + patch_mpas_streams + + popd +} + +#----------------------------------------------------- +case_submit() { + + if [ "${do_case_submit,,}" != "true" ]; then + echo $'\n----- Skipping case_submit -----\n' + return + fi + + echo $'\n----- Starting case_submit -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Run CIME case.submit + ./case.submit + + popd +} + +#----------------------------------------------------- +copy_script() { + + echo $'\n----- Saving run script for provenance -----\n' + + local script_provenance_dir=${CASE_SCRIPTS_DIR}/run_script_provenance + mkdir -p ${script_provenance_dir} + local this_script_name=`basename $0` + local script_provenance_name=${this_script_name}.`date +%Y%m%d-%H%M%S` + cp -vp ${this_script_name} ${script_provenance_dir}/${script_provenance_name} + +} + +#----------------------------------------------------- +# Silent versions of popd and pushd +pushd() { + command pushd "$@" > /dev/null +} +popd() { + command popd "$@" > /dev/null +} + +# Now, actually run the script +#----------------------------------------------------- +main + diff --git a/components/mpas-ocean/docs/index.md b/components/mpas-ocean/docs/index.md new file mode 100644 index 000000000000..4e75ec0f8e8b --- /dev/null +++ b/components/mpas-ocean/docs/index.md @@ -0,0 +1,7 @@ +# Model for Prediction Across Scales-Ocean + +The Model for Prediction Across Scales-Ocean (MPAS-Ocean) is the ocean component of the Energy Exascale Earth System Model (E3SM), developed by the U.S. Department of Energy. + +* The [MPAS-Ocean User Guide](user-guide/index.md) explains how to run and customize MPAS-Ocean within E3SM. +* The [MPAS-Ocean Developer Guide](dev-guide/index.md) explains MPAS-Ocean data structures and how to write new code. +* The [MPAS-Ocean Technical Guide](tech-guide/index.md) explains the science behind MPAS-Ocean. diff --git a/components/mpas-ocean/docs/tech-guide/index.md b/components/mpas-ocean/docs/tech-guide/index.md new file mode 100644 index 000000000000..beb4d1e9e7ad --- /dev/null +++ b/components/mpas-ocean/docs/tech-guide/index.md @@ -0,0 +1,29 @@ +# MPAS-Ocean Technical Guide + +This Technical Guide describes the governing equations, physics, and numerical discretizations of MPAS-Ocean. + +## Guides, Design Documents, and documentation + +MPAS-Ocean may be run as either a stand-alone model, or within the E3SM coupled climate model. +The MPAS-Ocean code for both stand-alone and coupled is housed in the repository [https://github.com/E3SM-Project/E3SM](https://github.com/E3SM-Project/E3SM) within the directory `components/mpas-ocean`. The stand-alone executable may be built within that directory using the make command with the required libraries, as described in Chapter 1 of the [User's Guide](https://zenodo.org/records/11098080). + +The [MPAS-Ocean User's Guide](https://zenodo.org/records/11098080) provides a description of the MPAS Framework in Part I, the Governing equations for MPAS-Ocean in Chapter 8, and describes the physics behind each term and parameterization in chapter 11. + +All new features are created with design documents. The location of these documents have moved over the years, but can still be found in these locations: + +1. [MPAS Documents repository](https://github.com/MPAS-Dev/MPAS-Documents/tree/master/ocean) +2. [E3SM repository MPAS-Ocean docs](https://github.com/E3SM-Project/E3SM/tree/master/components/mpas-ocean/docs/design_docs) +3. Documents for the new Omega model are similar to MPAS-Ocean and may be found in the [Omega documentation](https://docs.e3sm.org/Omega/develop/index.html). + +All test cases are housed in the [Compass repository](https://github.com/MPAS-Dev/compass) and, more recently, the [Polaris repository](https://github.com/E3SM-Project/polaris). The corresponding documentation is housed in the [Compass docs](https://mpas-dev.github.io/compass/latest/) and [Polaris docs](http://docs.e3sm.org/polaris/main/) pages. + +## Publications + +Beyond the documentation, there are many publications that describe the inner workings of MPAS-Ocean: + +Ringler, T., Petersen, M., Higdon, R.L., Jacobsen, D., Jones, P.W., Maltrud, M., 2013. +[A multi-resolution approach to global ocean modeling](https://doi.org/10.1016/j.ocemod.2013.04.010). Ocean Modelling 69, 211-232. + +Petersen, M.R., D.W. Jacobsen, T.D. Ringler, M.W. Hecht, M.E. Maltrud, [Evaluation of the arbitrary Lagrangian–Eulerian vertical coordinate method in the MPAS-Ocean model](http://dx.doi.org/10.1016/j.ocemod.2014.12.004), Ocean Modelling, Volume 86, February 2015, Pages 93-113, ISSN 1463-5003. + +Petersen, M. R., Asay‐Davis, X. S., Berres, A. S., Chen, Q., Feige, N., Hoffman, M. J., et al. (2019). [An evaluation of the ocean and sea ice climate of E3SM using MPAS and interannual CORE‐II forcing](https://doi.org/10.1029/2018MS001373). Journal of Advances in Modeling Earth Systems, 11, 1438– 1458. diff --git a/components/mpas-ocean/docs/user-guide/index.md b/components/mpas-ocean/docs/user-guide/index.md new file mode 100644 index 000000000000..0bc14046db0c --- /dev/null +++ b/components/mpas-ocean/docs/user-guide/index.md @@ -0,0 +1,200 @@ + +# MPAS-Ocean Quick Start + +This MPAS-Ocean Quick Start Guide describes how to set up and run MPAS-Ocean within E3SM. More details can be found in the [MPAS-Ocean User's Guide](https://zenodo.org/records/11098080), as well as instructions on running the stand-alone ocean model. + +## Steps to build and run MPAS-Ocean + +Step-by-step instructions on how to run E3SM can be found at [https://docs.e3sm.org/running-e3sm-guide](https://docs.e3sm.org/running-e3sm-guide). + +This MPAS-Ocean Quick Start guide provides the additional information to run configurations that are not fully-coupled (e.g. C-case: active ocean only; G-case: active ocean and sea ice) within E3SM. This is done by changing the compset. Certain parameters, including the mesh, namelist parameters, input data files, and output file specifcations can also be modified. These are described below as ways to customize runs. + +Templates of 1-month example E3SM run-scripts for a [G-case](../gcase.template.v3LR.chrysalis.sh) and [C-case](../ccase.template.v3LR.anvil.sh) are provided. Key information for the user to modify includes: + +- `MACHINE` and `PROJECT` if applicable +- Paths for code repository in `CODE_ROOT` and case directory in `CASE_ROOT`, which includes the run directory for output. +- Simulation compsets, resolution and name (see below) +- Wallclock duration in `WALLTIME` and simulation duration with `STOP_OPTION` and `STOP_N`. + +Additional runscript examples can be found [here](https://github.com/E3SM-Project/SimulationScripts/tree/master/archive/CoupledGroup/v3.LR) for v3.LR. + +## Scientifically supported compsets and meshes + +### Compsets + +The compsets below are typical ocean and sea ice-focused compsets supported by E3SM: + +`GMPAS-JRA1p5` - Active ocean-sea ice configuration forced by data atmosphere based on JRA55 v1.5 (covers 63 years, 1958-2020) + +`GMPAS-IAF` - Active ocean-sea ice configuration forced by data atmosphere based on CORE-II (covers 62 years, 1948-2009) + +`GMPAS-JRA1p5-DIB-DISMF` - Active ocean-sea ice configuration forced by JRA55 v1.5 atmosphere (as above), with data iceberg and data ice-shelf melt + +`GMPAS-JRA1p5-DIB-PISMF` - Active ocean-sea ice configuration forced by JRA55 v1.5 atmosphere (as above), with data iceberg and prognostic ice-shelf melt + +`CMPASO-JRA1p4` - Active ocean configuration forced by data atmosphere based on JRA55 v1.4 (covers 61 years, 1958-2018) + +`CMPASO-IAF` - Active ocean configuration forced by data atmosphere based on CORE-II (covers 62 years, 1948-2009) + +Additional compsets can be found in the [mpas-ocean `config_compsets.xml`](https://github.com/E3SM-Project/E3SM/blob/master/components/mpas-ocean/cime_config/config_compsets.xml). Note that the fully coupled compsets and their aliases can be found in the [cime allactive `config_compsets.xml`](https://github.com/E3SM-Project/E3SM/blob/master/cime_config/allactive/config_compsets.xml). +For more information on the schemes used within MPAS-Ocean, refer to the [MPAS-Ocean User's Guide](https://zenodo.org/records/11098080). + +A full list of Compsets in the current repository can be listed using + +```text +cd cime/scripts +./query_config --compsets +``` + +### Meshes + +Some supported meshes for G- and C-cases include: + +`TL319_IcoswISC30E3r5` - Icosahedral 30 km mesh with ice shelves cavities (wISC), E3SMv3 (E3) revision r5, TL319 is the grid for JRA. + +`T62_IcoswISC30E3r5` - Icosahedral 30 km mesh with ice shelves cavities (wISC), E3SMv3 (E3) revision r5, T62 is the grid for CORE-II. + +`T62_oQU240` - Quasi uniform 2-degree ocean mesh, to be used with CORE-II only. Good for rapid testing, used in nightly testing **not** production runs. This grid is not scientifically validated . + +Note: the mesh should be consistent with the compset (e.g. JRA vs CORE). Additional mesh information can be found [here](https://github.com/E3SM-Project/E3SM/blob/master/cime_config/config_grids.xml). + +A full list of Meshes in the current repository can be listed using + +```text +cd cime/scripts +./query_config --grids +``` + +## Customizing runs + +### Namelist changes + +Without additional input, E3SM will generate the namelist file `mpaso_in` in the run directory using the default values for the compset and mesh requested. +Namelist parameters can be changed from default values by modifying the `user_nl_mpaso` file, found in the `case_scripts` directory. This is done by entering ``[namelist option] = [changed value]`` as separate lines in the ``user_nl_mpaso`` file. All other options will remain defaults. These changes can be added at run time and will take effect in the next submission. + +Refer to the [MPAS-Ocean User's Guide](https://zenodo.org/records/11098080) (Chapter 10) for a comprehensive description of the namelist parameters and the options that they correspond to. Namelist options may also be found in the code repository in the file `components/mpas-ocean/src/Registry.xml` for general flags, `components/mpas-ocean/src/tracer_groups/Registry_*.xml` for specific tracer group flags, and `components/mpas-ocean/src/analysis_members/Registry_*.xml` for analysis member flags. + +#### Example of a namelist change via `user_nl_mpaso` + +```text + config_GM_closure = 'constant' + config_gm_constant_kappa = 900 + config_time_integrator = 'split_explicit' + config_am_timeseriesstatsmonthly_compute_interval = '00-00-01_00:00:00' +``` + +In this example, the namelist changes include changing the eddy closure (first 2 options for the type of closure and a parameter value), switching the time integration scheme, and modifying an analysis member (in this case, the interval from which the monthly analysis member is computed). + +Reminder: `user_nl_mpaso` can be empty. All options not specified are defaults (given the compset and mesh). Some options (like interior restoring) require extra fields to be present in the input file. + +### Configuring input and output for MPAS-Ocean + +The reading and writing of model fields in MPAS is handled by user-configurable streams. A stream +represents a fixed set of model fields, together with dimensions and attributes, that are all written +or read together to or from the same file or set of files. They are used for reading initial conditions, for writing and reading restart fields, and for writing additional model history fields. +Streams are defined in XML configuration files that are created at build time for each model core. The name of this XML file for the ocean core is `streams.ocean` (the sea ice has a similar `streams.seaice`). Importantly, the stream file is generated in the run directory during the ``./case.setup`` step, but **changes made into the run directory will not take effect**. To make changes to the output fields, **copy the `streams.ocean` file from the run directory into the``case_scripts/SourceMods/src.mpaso/`` directory**. Changes to the stream file made into the ``SourceMods`` sub-directory will take effect on the next case submission (there is no need to re-compile after making modifications to the XML file). Alternatively, changes to the streams file can be made directly in the code in ``components/mpas-ocean/cime_config/buildnml``. + +#### Checking initial conditions + +Key information to check regarding the input data typically include: + +```text + + +``` + +The `mesh` filename points to the mesh file used. The `input` filename points to the file containing the ocean initial conditions (if the run type is `initial`). +The streams file can be large, it is often useful to rely on the search function to navigate it. For larger meshes (millions of horizontal cells) the flag `io_type="pnetcdf"` must be changed to `io_type="pnetcdf,cdf5"`. + +#### Checking and modifying the output data + +By default, MPAS-Ocean will output a set of monthly-averaged variables. The streams file can be modified to include additional variables in the existing output files, produce additional output files, or change the output frequency (e.g. high frequency files shifting between daily or 5-daily frequencies). + +The XML file is organized into blocks describing each stream. Typical streams for output include: + +`timeSeriesStatsMonthlyOutput` - monthly averaged output + +`highFrequencyOutput` - high frequency snapshots (not averaged) output with frequency `output_interval` + +Under each block header is the list of variables (individual variables, variable structure, or variable arrays) that will be output within the relevant stream. + +##### Example workflow for modifying the output fields + +- copy the `streams.ocean` file from the run directory to the `SourceMods` directory (see above) +- identify the variable name for the variable of interest. You can find the variable name by searching the ocean Registry.xml (in the `src` directory) or Registry_package.xml in the `tracer` and `analysis_member` sub-directories. Note whether the variable of interest is included within a `var_array` or a `var_struct`. +- identify the output stream of interest (e.g. monthly averages, high frequency, others). You can search for a stream name, known output filename, or output interval. +- check whether the variable of interest is included in the `streams.ocean` file. Search for the `var name`, or the `var_array` and `var_struct` if applicable. If it is, copy the variable line from other streams into the stream of interest. If it is not included, copy it from the Registry.xml. +- check whether the relevant stream is turned on. This includes checking that `output_interval` in the stream header is not `None`. +- make further modifications: e.g. you can modify the `output_interval` for the high-frequency stream. If you are turning on a new stream, remove unnecessary variables from the stream. + +##### Excerpts from a `streams.ocean` file + +```text + + + + + + + + + + +... + +``` + +```text + + + + + + + + + + + + + + + + + + + + + +``` + +A more comprehensive description of the streams options can be found in Chapter 6 of the [MPAS-Ocean User's Guide](https://zenodo.org/records/11098080). diff --git a/components/mpas-ocean/driver/mpaso_cpl_indices.F b/components/mpas-ocean/driver/mpaso_cpl_indices.F index 968a4090fe4a..f099cf8ea46a 100644 --- a/components/mpas-ocean/driver/mpaso_cpl_indices.F +++ b/components/mpas-ocean/driver/mpaso_cpl_indices.F @@ -17,12 +17,18 @@ module mpaso_cpl_indices integer :: index_o2x_So_dhdx integer :: index_o2x_So_dhdy integer :: index_o2x_Fioo_q + integer :: index_o2x_Foxo_q_li integer :: index_o2x_Fioo_frazil + integer :: index_o2x_Foxo_frazil_li integer :: index_o2x_Faoo_h2otemp integer :: index_o2x_Faoo_fco2_ocn integer :: index_o2x_Faoo_fdms_ocn integer :: index_o2x_So_ssh - + integer :: index_o2x_Foxo_ismw + integer :: index_o2x_Foxo_rrofl + integer :: index_o2x_Foxo_rrofi + integer :: index_o2x_Foxo_ismh + integer :: index_o2x_Foxo_rrofih ! ocn -> drv for calculation of ocean-ice sheet interactions @@ -182,12 +188,21 @@ subroutine mpaso_cpl_indices_set( ) index_o2x_So_dhdx = mct_avect_indexra(o2x,'So_dhdx') index_o2x_So_dhdy = mct_avect_indexra(o2x,'So_dhdy') index_o2x_Fioo_q = mct_avect_indexra(o2x,'Fioo_q',perrWith='quiet') + index_o2x_Foxo_q_li = mct_avect_indexra(o2x,'Foxo_q_li',perrWith='quiet') index_o2x_Fioo_frazil = mct_avect_indexra(o2x,'Fioo_frazil',perrWith='quiet') + index_o2x_Foxo_frazil_li= mct_avect_indexra(o2x,'Foxo_frazil_li',perrWith='quiet') index_o2x_Faoo_h2otemp = mct_avect_indexra(o2x,'Faoo_h2otemp',perrWith='quiet') index_o2x_Faoo_fco2_ocn = mct_avect_indexra(o2x,'Faoo_fco2_ocn',perrWith='quiet') index_o2x_Faoo_fdms_ocn = mct_avect_indexra(o2x,'Faoo_fdms_ocn',perrWith='quiet') index_o2x_So_ssh = mct_avect_indexra(o2x,'So_ssh') + index_o2x_Foxo_ismw = mct_avect_indexra(o2x,'Foxo_ismw',perrWith='quiet') + index_o2x_Foxo_rrofl = mct_avect_indexra(o2x,'Foxo_rrofl',perrWith='quiet') + index_o2x_Foxo_rrofi = mct_avect_indexra(o2x,'Foxo_rrofi',perrWith='quiet') + + index_o2x_Foxo_ismh = mct_avect_indexra(o2x,'Foxo_ismh',perrWith='quiet') + index_o2x_Foxo_rrofih = mct_avect_indexra(o2x,'Foxo_rrofih',perrWith='quiet') + index_o2x_So_blt = mct_avect_indexra(o2x,'So_blt') index_o2x_So_bls = mct_avect_indexra(o2x,'So_bls') index_o2x_So_htv = mct_avect_indexra(o2x,'So_htv') diff --git a/components/mpas-ocean/driver/ocn_comp_mct.F b/components/mpas-ocean/driver/ocn_comp_mct.F index ec02127f7cab..be40de0622a9 100644 --- a/components/mpas-ocean/driver/ocn_comp_mct.F +++ b/components/mpas-ocean/driver/ocn_comp_mct.F @@ -753,13 +753,12 @@ end subroutine xml_stream_get_attributes call mpas_pool_get_subpool(block_ptr % structs, 'scratch', scratchPool) call ocn_forcing_build_fraction_absorbed_array(meshPool, statePool, forcingPool, ierr, 1) + call ocn_frazil_forcing_build_arrays(domain, meshPool, forcingPool, statePool, ierr) call mpas_timer_start("land_ice_build_arrays", .false.) call ocn_surface_land_ice_fluxes_build_arrays(meshPool, forcingPool, scratchPool, & statePool, err=ierr) call mpas_timer_stop("land_ice_build_arrays") - call ocn_frazil_forcing_build_arrays(domain, meshPool, forcingPool, statePool, ierr) - block_ptr => block_ptr % next end do @@ -1095,7 +1094,7 @@ subroutine ocn_run_mct( EClock, cdata_o, x2o_o, o2x_o)!{{{ call mpas_pool_get_subpool(block_ptr % structs, 'scratch', scratchPool) call ocn_forcing_build_fraction_absorbed_array(meshPool, statePool, forcingPool, ierr, 1) - + call ocn_frazil_forcing_build_arrays(domain, meshPool, forcingPool, statePool, ierr) call mpas_timer_start("land_ice_build_arrays", .false.) call ocn_surface_land_ice_fluxes_build_arrays(meshPool, & forcingPool, scratchPool, statePool, ierr) @@ -1103,8 +1102,6 @@ subroutine ocn_run_mct( EClock, cdata_o, x2o_o, o2x_o)!{{{ call ocn_surface_land_ice_fluxes_accumulate_fluxes(meshPool, forcingPool, & statePool, dt, ierr) - call ocn_frazil_forcing_build_arrays(domain, meshPool, forcingPool, statePool, ierr) - call ocn_eddy_compute_mixed_layer_depth(statePool, forcingPool) if (config_use_GM .or. config_submesoscale_enable) then call ocn_eddy_compute_buoyancy_gradient() @@ -2700,7 +2697,12 @@ subroutine ocn_export_mct(o2x_o, errorCode) !{{{ avgOceanSurfaceDOCSemiLabile, & avgOceanSurfaceFeParticulate, & avgOceanSurfaceFeDissolved, & - ssh + ssh, & + avgLandIceFreshwaterFlux, & + avgRemovedRiverRunoffFlux, & + avgRemovedIceRunoffFlux, & + avgLandIceHeatFlux, & + avgRemovedIceRunoffHeatFlux real (kind=RKIND), dimension(:,:), pointer :: avgTracersSurfaceValue, avgSurfaceVelocity, & avgSSHGradient, avgOceanSurfacePhytoC, & @@ -2709,6 +2711,7 @@ subroutine ocn_export_mct(o2x_o, errorCode) !{{{ real (kind=RKIND) :: surfaceFreezingTemp logical, pointer :: frazilIceActive, & + config_remove_AIS_coupler_runoff, & config_use_ecosysTracers, & config_use_DMSTracers, & config_use_MacroMoleculesTracers, & @@ -2725,6 +2728,7 @@ subroutine ocn_export_mct(o2x_o, errorCode) !{{{ call mpas_pool_get_package(domain % packages, 'frazilIceActive', frazilIceActive) call mpas_pool_get_config(domain % configs, 'config_use_ecosysTracers', config_use_ecosysTracers) call mpas_pool_get_config(domain % configs, 'config_land_ice_flux_mode', config_land_ice_flux_mode) + call mpas_pool_get_config(domain % configs, 'config_remove_AIS_coupler_runoff', config_remove_AIS_coupler_runoff) call mpas_pool_get_config(domain % configs, 'config_use_DMSTracers', config_use_DMSTracers) call mpas_pool_get_config(domain % configs, 'config_use_MacroMoleculesTracers', config_use_MacroMoleculesTracers) call mpas_pool_get_config(domain % configs, 'config_use_ecosysTracers_sea_ice_coupling', & @@ -2767,6 +2771,17 @@ subroutine ocn_export_mct(o2x_o, errorCode) !{{{ call mpas_pool_get_array(statePool, 'accumulatedFrazilIceMass', accumulatedFrazilIceMass, 1) end if + ! Cryo fields + if (trim(config_land_ice_flux_mode) == 'standalone' .or. trim(config_land_ice_flux_mode) == 'data') then + call mpas_pool_get_array(forcingPool, 'avgLandIceFreshwaterFlux', avgLandIceFreshwaterFlux) + call mpas_pool_get_array(forcingPool, 'avgLandIceHeatFlux', avgLandIceHeatFlux) + endif + if (config_remove_AIS_coupler_runoff) then + call mpas_pool_get_array(forcingPool, 'avgRemovedRiverRunoffFlux', avgRemovedRiverRunoffFlux) + call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoffFlux', avgRemovedIceRunoffFlux) + call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoffHeatFlux', avgRemovedIceRunoffHeatFlux) + endif + ! BGC fields if (config_use_ecosysTracers) then @@ -2818,6 +2833,17 @@ subroutine ocn_export_mct(o2x_o, errorCode) !{{{ o2x_o % rAttr(index_o2x_Faoo_h2otemp, n) = avgTotalFreshWaterTemperatureFlux(i) * rho_sw * cp_sw + ! Cryo fields + if (trim(config_land_ice_flux_mode) == 'standalone' .or. trim(config_land_ice_flux_mode) == 'data') then + o2x_o % rAttr(index_o2x_Foxo_ismw, n) = avgLandIceFreshwaterFlux(i) + o2x_o % rAttr(index_o2x_Foxo_ismh, n) = avgLandIceHeatFlux(i) + endif + if (config_remove_AIS_coupler_runoff) then + o2x_o % rAttr(index_o2x_Foxo_rrofl, n) = avgRemovedRiverRunoffFlux(i) + o2x_o % rAttr(index_o2x_Foxo_rrofi, n) = avgRemovedIceRunoffFlux(i) + o2x_o % rAttr(index_o2x_Foxo_rrofih, n) = avgRemovedIceRunoffHeatFlux(i) + endif + if ( frazilIceActive ) then ! negative when frazil ice can be melted keepFrazil = .true. @@ -2853,6 +2879,10 @@ subroutine ocn_export_mct(o2x_o, errorCode) !{{{ o2x_o % rAttr(index_o2x_Fioo_q, n) = 0.0_RKIND o2x_o % rAttr(index_o2x_Fioo_frazil, n) = 0.0_RKIND + if (trim(config_land_ice_flux_mode) == 'standalone' .or. trim(config_land_ice_flux_mode) == 'data') then + o2x_o % rAttr(index_o2x_Foxo_q_li, n) = accumulatedFrazilIceMass(i) * config_frazil_heat_of_fusion / ocn_cpl_dt + o2x_o % rAttr(index_o2x_Foxo_frazil_li, n) = accumulatedFrazilIceMass(i) / ocn_cpl_dt + endif end if @@ -3981,7 +4011,12 @@ subroutine ocn_export_moab(EClock) !{{{ avgOceanSurfaceDOCSemiLabile, & avgOceanSurfaceFeParticulate, & avgOceanSurfaceFeDissolved, & - ssh + ssh, & + avgLandIceFreshwaterFlux, & + avgRemovedRiverRunoffFlux, & + avgRemovedIceRunoffFlux, & + avgLandIceHeatFlux, & + avgRemovedIceRunoffHeatFlux real (kind=RKIND), dimension(:,:), pointer :: avgTracersSurfaceValue, avgSurfaceVelocity, & avgSSHGradient, avgOceanSurfacePhytoC, & @@ -3990,6 +4025,7 @@ subroutine ocn_export_moab(EClock) !{{{ real (kind=RKIND) :: surfaceFreezingTemp logical, pointer :: frazilIceActive, & + config_remove_AIS_coupler_runoff, & config_use_ecosysTracers, & config_use_DMSTracers, & config_use_MacroMoleculesTracers, & @@ -4006,6 +4042,7 @@ subroutine ocn_export_moab(EClock) !{{{ call mpas_pool_get_package(domain % packages, 'frazilIceActive', frazilIceActive) call mpas_pool_get_config(domain % configs, 'config_use_ecosysTracers', config_use_ecosysTracers) call mpas_pool_get_config(domain % configs, 'config_land_ice_flux_mode', config_land_ice_flux_mode) + call mpas_pool_get_config(domain % configs, 'config_remove_AIS_coupler_runoff', config_remove_AIS_coupler_runoff) call mpas_pool_get_config(domain % configs, 'config_use_DMSTracers', config_use_DMSTracers) call mpas_pool_get_config(domain % configs, 'config_use_MacroMoleculesTracers', config_use_MacroMoleculesTracers) call mpas_pool_get_config(domain % configs, 'config_use_ecosysTracers_sea_ice_coupling', & @@ -4047,6 +4084,16 @@ subroutine ocn_export_moab(EClock) !{{{ call mpas_pool_get_array(forcingPool, 'frazilSurfacePressure', frazilSurfacePressure) call mpas_pool_get_array(statePool, 'accumulatedFrazilIceMass', accumulatedFrazilIceMass, 1) end if + + if (trim(config_land_ice_flux_mode) == 'standalone' .or. trim(config_land_ice_flux_mode) == 'data') then + call mpas_pool_get_array(forcingPool, 'avgLandIceFreshwaterFlux', avgLandIceFreshwaterFlux) + call mpas_pool_get_array(forcingPool, 'avgLandIceHeatFlux', avgLandIceHeatFlux) + endif + if (config_remove_AIS_coupler_runoff) then + call mpas_pool_get_array(forcingPool, 'avgRemovedRiverRunoffFlux', avgRemovedRiverRunoffFlux) + call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoffFlux', avgRemovedIceRunoffFlux) + call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoffHeatFlux', avgRemovedIceRunoffHeatFlux) + endif ! BGC fields if (config_use_ecosysTracers) then @@ -4098,7 +4145,17 @@ subroutine ocn_export_moab(EClock) !{{{ o2x_om(n, index_o2x_So_dhdy) = avgSSHGradient(index_avgMeridionalSSHGradient, i) o2x_om(n, index_o2x_Faoo_h2otemp) = avgTotalFreshWaterTemperatureFlux(i) * rho_sw * cp_sw - + + if (trim(config_land_ice_flux_mode) == 'standalone' .or. trim(config_land_ice_flux_mode) == 'data') then + o2x_om(n, index_o2x_Foxo_ismw) = avgLandIceFreshwaterFlux(i) + o2x_om(n, index_o2x_Foxo_ismh) = avgLandIceHeatFlux(i) + endif + if (config_remove_AIS_coupler_runoff) then + o2x_om(n, index_o2x_Foxo_rrofl) = avgRemovedRiverRunoffFlux(i) + o2x_om(n, index_o2x_Foxo_rrofi) = avgRemovedIceRunoffFlux(i) + o2x_om(n, index_o2x_Foxo_rrofih) = avgRemovedIceRunoffHeatFlux(i) + endif + if ( frazilIceActive ) then ! negative when frazil ice can be melted keepFrazil = .true. @@ -4134,7 +4191,10 @@ subroutine ocn_export_moab(EClock) !{{{ o2x_om(n, index_o2x_Fioo_q) = 0.0_RKIND o2x_om(n, index_o2x_Fioo_frazil) = 0.0_RKIND - + if (trim(config_land_ice_flux_mode) == 'standalone' .or. trim(config_land_ice_flux_mode) == 'data') then + o2x_om(n, index_o2x_Foxo_q_li) = accumulatedFrazilIceMass(i) * config_frazil_heat_of_fusion / ocn_cpl_dt + o2x_om(n, index_o2x_Foxo_frazil_li) = accumulatedFrazilIceMass(i) / ocn_cpl_dt + endif end if ! Reset SeaIce Energy and Accumulated Frazil Ice diff --git a/components/mpas-ocean/mkdocs.yml b/components/mpas-ocean/mkdocs.yml new file mode 100644 index 000000000000..3e2500579293 --- /dev/null +++ b/components/mpas-ocean/mkdocs.yml @@ -0,0 +1,7 @@ +site_name: MPAS-Ocean + +nav: + - Introduction: 'index.md' + - Users's Guide: user-guide/index.md + - Developers's Guide: dev-guide/index.md + - Technical Guide: tech-guide/index.md diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index f8b56c083d44..475a2751def9 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -110,6 +110,9 @@ description="Number of wavenumbers used to reconstruct Stokes drift depth profile" definition="6" /> + @@ -201,8 +204,8 @@ possible_values="Any time stamp in 'YYYY-MM-DD_hh:mm:ss' format. Items can be removed from the left if they are unused." /> @@ -477,6 +480,14 @@ description="Background vertical diffusion applied to tracer quantities" possible_values="Any positive real value." /> + + + + + + + + + + + + @@ -1623,6 +1662,7 @@ + @@ -2598,6 +2638,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - @@ -192,10 +192,7 @@ description="Sea ice salinity flux from coupler. Positive into the ocean." /> - @@ -305,7 +302,6 @@ - @@ -362,7 +358,6 @@ - diff --git a/components/mpas-ocean/src/analysis_members/Registry_global_stats.xml b/components/mpas-ocean/src/analysis_members/Registry_global_stats.xml index 1f76e701fd29..480ad08ef3f3 100644 --- a/components/mpas-ocean/src/analysis_members/Registry_global_stats.xml +++ b/components/mpas-ocean/src/analysis_members/Registry_global_stats.xml @@ -148,6 +148,9 @@ + @@ -157,6 +160,9 @@ + @@ -247,6 +253,9 @@ + @@ -256,6 +265,9 @@ + @@ -346,6 +358,9 @@ + @@ -355,6 +370,9 @@ + @@ -445,6 +463,9 @@ + @@ -454,6 +475,9 @@ + @@ -544,6 +568,9 @@ + @@ -553,6 +580,9 @@ + @@ -643,6 +673,9 @@ + @@ -652,6 +685,9 @@ + @@ -742,6 +778,9 @@ + @@ -751,6 +790,9 @@ + + + diff --git a/components/mpas-ocean/src/analysis_members/Registry_time_series_stats_monthly_mean.xml b/components/mpas-ocean/src/analysis_members/Registry_time_series_stats_monthly_mean.xml index 58ac7e95af4c..2858e01961d2 100644 --- a/components/mpas-ocean/src/analysis_members/Registry_time_series_stats_monthly_mean.xml +++ b/components/mpas-ocean/src/analysis_members/Registry_time_series_stats_monthly_mean.xml @@ -187,6 +187,8 @@ + + diff --git a/components/mpas-ocean/src/analysis_members/Registry_time_series_stats_monthly_min.xml b/components/mpas-ocean/src/analysis_members/Registry_time_series_stats_monthly_min.xml index 5ee051585d71..30121568ad46 100644 --- a/components/mpas-ocean/src/analysis_members/Registry_time_series_stats_monthly_min.xml +++ b/components/mpas-ocean/src/analysis_members/Registry_time_series_stats_monthly_min.xml @@ -126,6 +126,8 @@ + + diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F index a4f2a0b7ab5d..bdd07e799b5d 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F @@ -598,16 +598,14 @@ subroutine energy_conservation(domain, err) enddo end if - if (landIceFreshwaterFluxesOn & - .and.config_use_frazil_ice_formation & - .and.config_frazil_under_land_ice) then + if (config_use_frazil_ice_formation .and. config_frazil_under_land_ice) then call mpas_pool_get_array(statePool, 'accumulatedLandIceFrazilMass', accumulatedLandIceFrazilMassOld, 1) call mpas_pool_get_array(statePool, 'accumulatedLandIceFrazilMass', accumulatedLandIceFrazilMassNew, 2) do iCell = 1, nCellsSolve ! Frazil ice mass is negative. Negative coefficient makes heat ! flux positive, because freezing ice releases heat. sumArray(18) = sumArray(18) - areaCell(iCell) * config_frazil_heat_of_fusion & - * (accumulatedLandIceFrazilMassNew(iCell) - accumulatedLandIceFrazilMassOld(iCell)) + * (accumulatedLandIceFrazilMassNew(iCell) - accumulatedLandIceFrazilMassOld(iCell))/dt enddo end if @@ -748,9 +746,7 @@ subroutine energy_conservation(domain, err) if (landIceFreshwaterFluxesOn) then v=accumulatedLandIceHeatFlux ; write(m,"('landIceHeatFlux ',es16.8,' ',f16.8)") v,v/A; call mpas_log_write(m); s=s+v end if - if (landIceFreshwaterFluxesOn & - .and.config_use_frazil_ice_formation & - .and.config_frazil_under_land_ice) then + if (config_use_frazil_ice_formation .and. config_frazil_under_land_ice) then v=accumulatedLandIceFrazilHeatFlux ; write(m,"(' landIceFrazilHeatFlux ',es16.8,' (already in hfreeze, do not sum )',f16.8)") v,v/A; call mpas_log_write(m); ! no sum: s=s+v end if write(m,"('SUM EXPLICIT HEAT FLUXES ',es16.8,' ',f16.8)") s, s/A; call mpas_log_write(m) @@ -976,9 +972,7 @@ subroutine mass_conservation(domain, err) enddo end if - if (landIceFreshwaterFluxesOn & - .and.config_use_frazil_ice_formation & - .and.config_frazil_under_land_ice) then + if (config_use_frazil_ice_formation .and. config_frazil_under_land_ice) then call mpas_pool_get_array(statePool, 'accumulatedLandIceFrazilMass', accumulatedLandIceFrazilMassOld, 1) call mpas_pool_get_array(statePool, 'accumulatedLandIceFrazilMass', accumulatedLandIceFrazilMassNew, 2) do iCell = 1, nCellsSolve @@ -1101,9 +1095,7 @@ subroutine mass_conservation(domain, err) write(m,"(' SUM: ice runoff ',es16.8,' x2o_Foxx_rofi wfrzrof SUM ',f16.8)") v,v*c; call mpas_log_write(m) v=accumulatedLandIceFlux ; write(m,"('landIceFreshwaterFlux ',es16.8,' ',f16.8)") v,v*c; call mpas_log_write(m); s=s+v endif - if (landIceFreshwaterFluxesOn & - .and.config_use_frazil_ice_formation & - .and.config_frazil_under_land_ice) then + if (config_use_frazil_ice_formation .and. config_frazil_under_land_ice) then v=accumulatedLandIceFrazilFlux ; write(m,"(' landIceFrazilFlux ',es16.8,' (already in wfreeze, do not sum )',f16.8)") v,v*c; call mpas_log_write(m); ! no sum: s=s+v endif write(m,"('SUM VOLUME FLUXES ',es16.8,' ',f16.8,es16.8)") s, s*c; call mpas_log_write(m) @@ -1170,8 +1162,8 @@ subroutine salt_conservation(domain, err) real(kind=RKIND), pointer :: & accumulatedSeaIceSalinityFlux, & - accumulatedFrazilSalinityFlux, & - accumulatedLandIceFrazilSalinityFlux + accumulatedFrazilSalinityFlux + ! accumulatedLandIceFrazilSalinityFlux is not present because it is always 0 real(kind=RKIND), dimension(:), allocatable :: & sumArray, & @@ -1217,7 +1209,6 @@ subroutine salt_conservation(domain, err) call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedSeaIceSalinityFlux", accumulatedSeaIceSalinityFlux) call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedFrazilSalinityFlux", accumulatedFrazilSalinityFlux) - call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedLandIceFrazilSalinityFlux", accumulatedLandIceFrazilSalinityFlux) !------------------------------------------------------------- ! Net salt flux to ice @@ -1258,14 +1249,10 @@ subroutine salt_conservation(domain, err) enddo end if - if (landIceFreshwaterFluxesOn & - .and.config_use_frazil_ice_formation & - .and.config_frazil_under_land_ice) then - call mpas_pool_get_array(statePool, 'accumulatedLandIceFrazilMass', accumulatedLandIceFrazilMassOld, 1) - call mpas_pool_get_array(statePool, 'accumulatedLandIceFrazilMass', accumulatedLandIceFrazilMassNew, 2) + if (config_use_frazil_ice_formation .and. config_frazil_under_land_ice) then + ! Land ice frazil salinity is always 0 do iCell = 1, nCellsSolve - sumArray(3) = sumArray(3) + areaCell(iCell) & - * (accumulatedLandIceFrazilMassNew(iCell) - accumulatedLandIceFrazilMassOld(iCell))/dt + sumArray(3) = sumArray(3) + 0.0_RKIND enddo end if @@ -1278,7 +1265,6 @@ subroutine salt_conservation(domain, err) ! accumulate fluxes accumulatedSeaIceSalinityFlux = accumulatedSeaIceSalinityFlux + sumArrayOut(1) accumulatedFrazilSalinityFlux = accumulatedFrazilSalinityFlux + sumArrayOut(2) - accumulatedLandIceFrazilSalinityFlux = accumulatedLandIceFrazilSalinityFlux + sumArrayOut(3) ! cleanup deallocate(sumArray) @@ -1295,7 +1281,6 @@ subroutine salt_conservation(domain, err) ! Average the fluxes accumulatedSeaIceSalinityFlux = accumulatedSeaIceSalinityFlux /accumulatedFluxCounter accumulatedFrazilSalinityFlux = accumulatedFrazilSalinityFlux /accumulatedFluxCounter - accumulatedLandIceFrazilSalinityFlux = accumulatedLandIceFrazilSalinityFlux /accumulatedFluxCounter ! get initial salt content call MPAS_pool_get_array(conservationCheckSaltAMPool, "initialSalt", initialSalt) @@ -1312,8 +1297,7 @@ subroutine salt_conservation(domain, err) call MPAS_pool_get_array(conservationCheckSaltAMPool, "netSaltFlux", netSaltFlux) netSaltFlux = accumulatedSeaIceSalinityFlux & - + accumulatedFrazilSalinityFlux & - + accumulatedLandIceFrazilSalinityFlux + + accumulatedFrazilSalinityFlux ! compute the final salt error call MPAS_pool_get_array(conservationCheckSaltAMPool, "absoluteSaltError", absoluteSaltError) @@ -1341,7 +1325,7 @@ subroutine salt_conservation(domain, err) if (landIceFreshwaterFluxesOn & .and.config_use_frazil_ice_formation & .and.config_frazil_under_land_ice) then -v=accumulatedLandIceFrazilSalinityFlux; write(m,"('LandIceFrazilSalinityFlux',es16.8,' (already in wmelt, do not sum) ',f16.8)") v,v*c; call mpas_log_write(m); !no sum: s=s+v +v=0; write(m,"('LandIceFrazilSalinityFlux',es16.8,' (already in wmelt, do not sum) ',f16.8)") v,v*c; call mpas_log_write(m); !no sum: s=s+v end if write(m,"('SUM VOLUME FLUXES ',es16.8,' ',f16.8,es16.8)") s, s*c; call mpas_log_write(m) @@ -2266,8 +2250,7 @@ subroutine reset_accumulated_variables(domain) real(kind=RKIND), pointer :: & accumulatedSeaIceSalinityFlux, & - accumulatedFrazilSalinityFlux, & - accumulatedLandIceFrazilSalinityFlux + accumulatedFrazilSalinityFlux real(kind=RKIND), pointer :: & accumulatedCarbonSourceSink, & @@ -2353,11 +2336,9 @@ subroutine reset_accumulated_variables(domain) call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedSeaIceSalinityFlux", accumulatedSeaIceSalinityFlux) call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedFrazilSalinityFlux", accumulatedFrazilSalinityFlux) - call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedLandIceFrazilSalinityFlux", accumulatedLandIceFrazilSalinityFlux) accumulatedSeaIceSalinityFlux = 0.0_RKIND accumulatedFrazilSalinityFlux = 0.0_RKIND - accumulatedLandIceFrazilSalinityFlux = 0.0_RKIND call MPAS_pool_get_subpool(domain % blocklist % structs, "conservationCheckCarbonAM", conservationCheckCarbonAMPool) @@ -2432,7 +2413,8 @@ subroutine ocn_restart_conservation_check(domain, err)!{{{ err = 0 if ( trim(config_land_ice_flux_mode) == 'standalone' .or. & - trim(config_land_ice_flux_mode) == 'coupled' ) then + trim(config_land_ice_flux_mode) == 'coupled' .or. & + trim(config_land_ice_flux_mode) == 'data' ) then landIceFreshwaterFluxesOn = .true. end if diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_global_stats.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_global_stats.F index c21d1d4b5b50..203665999a86 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_global_stats.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_global_stats.F @@ -254,9 +254,9 @@ subroutine ocn_compute_global_stats(domain, timeLevel, err)!{{{ real (kind=RKIND), dimension(:), pointer :: evaporationFlux, snowFlux real (kind=RKIND), dimension(:), pointer :: seaIceFreshWaterFlux, icebergFreshWaterFlux, riverRunoffFlux, iceRunoffFlux - real (kind=RKIND), dimension(:), pointer :: rainFlux, landIceFreshwaterFlux + real (kind=RKIND), dimension(:), pointer :: rainFlux, landIceFreshwaterFlux, landIceFreshwaterFluxTotal real (kind=RKIND), dimension(:), pointer :: accumulatedLandIceMass, accumulatedLandIceHeat, & - accumulatedLandIceFrazilMass + accumulatedLandIceFrazilMass, frazilIceFreshwaterFlux real (kind=RKIND), dimension(:,:), pointer :: frazilLayerThicknessTendency @@ -364,6 +364,8 @@ subroutine ocn_compute_global_stats(domain, timeLevel, err)!{{{ call mpas_pool_get_array(forcingPool, 'rainFlux', rainFlux) call mpas_pool_get_array(forcingPool, 'frazilLayerThicknessTendency', frazilLayerThicknessTendency) call mpas_pool_get_array(forcingPool, 'landIceFreshwaterFlux', landIceFreshwaterFlux) + call mpas_pool_get_array(forcingPool, 'landIceFreshwaterFluxTotal', landIceFreshwaterFluxTotal) + call mpas_pool_get_array(forcingPool, 'frazilIceFreshwaterFlux', frazilIceFreshwaterFlux) call mpas_pool_get_array(forcingPool, 'landIceFloatingMask', landIceFloatingMask) call mpas_pool_get_array(tracersSurfaceFluxPool, 'activeTracersSurfaceFlux', activeTracersSurfaceFlux) @@ -377,6 +379,7 @@ subroutine ocn_compute_global_stats(domain, timeLevel, err)!{{{ call mpas_pool_get_array(statePool, 'accumulatedLandIceHeat', accumulatedLandIceHeat, 1) call mpas_pool_get_array(statePool, 'accumulatedLandIceFrazilMass', accumulatedLandIceFrazilMass, 1) + allocate(areaEdge(1:nEdgesSolve)) areaEdge = dcEdge(1:nEdgesSolve)*dvEdge(1:nEdgesSolve) @@ -958,6 +961,21 @@ subroutine ocn_compute_global_stats(domain, timeLevel, err)!{{{ maxes_tmp(variableIndex) = 0.0_RKIND end if + ! landIceFreshwaterFluxTotal + variableIndex = variableIndex + 1 + if ( associated(landIceFreshwaterFluxTotal) ) then + call ocn_compute_field_area_weighted_local_stats_surface(dminfo, nCellsSolve, & + landIceFloatingArea, & + landIceFreshwaterFluxTotal(1:nCellsSolve), sums_tmp(variableIndex), & + sumSquares_tmp(variableIndex), mins_tmp(variableIndex), & + maxes_tmp(variableIndex)) + else + sums_tmp(variableIndex) = 0.0_RKIND + sumSquares_tmp(variableIndex) = 0.0_RKIND + mins_tmp(variableIndex) = 0.0_RKIND + maxes_tmp(variableIndex) = 0.0_RKIND + end if + sums(variableIndex) = sums(variableIndex) + sums_tmp(variableIndex) sumSquares(variableIndex) = sumSquares(variableIndex) + sumSquares_tmp(variableIndex) mins(variableIndex) = min(mins(variableIndex), mins_tmp(variableIndex)) @@ -1024,6 +1042,21 @@ subroutine ocn_compute_global_stats(domain, timeLevel, err)!{{{ maxes_tmp(variableIndex) = 0.0_RKIND end if + ! frazilIceFreshwaterFlux + variableIndex = variableIndex + 1 + if ( associated(frazilIceFreshwaterFlux) ) then + call ocn_compute_field_area_weighted_local_stats_surface(dminfo, nCellsSolve, & + areaCell(1:nCellsSolve), & + frazilIceFreshwaterFlux(1:nCellsSolve), sums_tmp(variableIndex), & + sumSquares_tmp(variableIndex), mins_tmp(variableIndex), & + maxes_tmp(variableIndex)) + else + sums_tmp(variableIndex) = 0.0_RKIND + sumSquares_tmp(variableIndex) = 0.0_RKIND + mins_tmp(variableIndex) = 0.0_RKIND + maxes_tmp(variableIndex) = 0.0_RKIND + end if + sums(variableIndex) = sums(variableIndex) + sums_tmp(variableIndex) sumSquares(variableIndex) = sumSquares(variableIndex) + sumSquares_tmp(variableIndex) mins(variableIndex) = min(mins(variableIndex), mins_tmp(variableIndex)) @@ -1299,6 +1332,16 @@ subroutine ocn_compute_global_stats(domain, timeLevel, err)!{{{ rms(variableIndex) = 0.0_RKIND end if + ! landIceFreshwaterFluxTotal + variableIndex = variableIndex + 1 + if (associated(landIceFreshwaterFluxTotal) .and. landIceFloatingAreaSum > 0.0_RKIND) then + averages(variableIndex) = sums(variableIndex)/(landIceFloatingAreaSum) + rms(variableIndex) = sqrt(sumSquares(variableIndex)/(landIceFloatingAreaSum)) + else + averages(variableIndex) = 0.0_RKIND + rms(variableIndex) = 0.0_RKIND + end if + ! continue accumulating fresh water inputs netFreshwaterInput = netFreshwaterInput + sums(variableIndex) * dt/rho_sw @@ -1332,6 +1375,16 @@ subroutine ocn_compute_global_stats(domain, timeLevel, err)!{{{ rms(variableIndex) = 0.0_RKIND end if + ! frazilIceFreshwaterFlux + variableIndex = variableIndex + 1 + if (frazilIcePkgActive) then + averages(variableIndex) = sums(variableIndex)/(areaCellGlobal) + rms(variableIndex) = sqrt(sumSquares(variableIndex)/(areaCellGlobal)) + else + averages(variableIndex) = 0.0_RKIND + rms(variableIndex) = 0.0_RKIND + end if + ! calculate fresh water conservation check quantities absoluteFreshWaterConservation = totalVolumeChange - netFreshwaterInput if (abs(totalVolumeChange) < 1e-12_RKIND) then diff --git a/components/mpas-ocean/src/driver/mpas_ocn_core_interface.F b/components/mpas-ocean/src/driver/mpas_ocn_core_interface.F index ec6a19c81d11..7d9eb083f907 100644 --- a/components/mpas-ocean/src/driver/mpas_ocn_core_interface.F +++ b/components/mpas-ocean/src/driver/mpas_ocn_core_interface.F @@ -138,6 +138,7 @@ function ocn_setup_packages(configPool, packagePool, iocontext) result(ierr)!{{{ logical, pointer :: gotmPKGActive logical, pointer :: verticalRemapPKGActive logical, pointer :: activeWavePKGActive + logical, pointer :: subgridWetDryPKGActive type (mpas_pool_iterator_type) :: pkgItr logical, pointer :: packageActive @@ -172,6 +173,7 @@ function ocn_setup_packages(configPool, packagePool, iocontext) result(ierr)!{{{ logical, pointer :: config_use_time_varying_land_ice_forcing logical, pointer :: config_use_gotm logical, pointer :: config_use_active_wave + logical, pointer :: config_use_subgrid_wetting_drying character (len=StrKIND), pointer :: config_time_integrator character (len=StrKIND), pointer :: config_ocean_run_mode @@ -427,6 +429,15 @@ function ocn_setup_packages(configPool, packagePool, iocontext) result(ierr)!{{{ activeWavePKGActive = .true. endif + ! + ! test for use of subgrid wetting and drying + ! + call mpas_pool_get_package(packagePool, 'subgridWetDryPKGActive', subgridWetDryPKGActive) + call mpas_pool_get_config(configPool, 'config_use_subgrid_wetting_drying', config_use_subgrid_wetting_drying) + if (config_use_subgrid_wetting_drying) then + subgridWetDryPKGActive = .true. + end if + ! ! call into analysis member driver to set analysis member packages ! diff --git a/components/mpas-ocean/src/mode_forward/Makefile b/components/mpas-ocean/src/mode_forward/Makefile index fb95d0236e89..3761921d7433 100644 --- a/components/mpas-ocean/src/mode_forward/Makefile +++ b/components/mpas-ocean/src/mode_forward/Makefile @@ -6,13 +6,19 @@ OBJS = mpas_ocn_forward_mode.o \ mpas_ocn_time_integration_si.o \ mpas_ocn_time_integration_split.o \ mpas_ocn_time_integration_lts.o \ + mpas_ocn_time_integration_fblts.o \ mpas_ocn_time_integration_split_ab2.o all: forward_mode forward_mode: $(OBJS) -mpas_ocn_time_integration.o: mpas_ocn_time_integration_rk4.o mpas_ocn_time_integration_si.o mpas_ocn_time_integration_split.o mpas_ocn_time_integration_lts.o mpas_ocn_time_integration_split_ab2.o +mpas_ocn_time_integration.o: mpas_ocn_time_integration_rk4.o \ + mpas_ocn_time_integration_si.o \ + mpas_ocn_time_integration_split.o \ + mpas_ocn_time_integration_lts.o \ + mpas_ocn_time_integration_fblts.o \ + mpas_ocn_time_integration_split_ab2.o mpas_ocn_time_integration_rk4.o: @@ -27,6 +33,7 @@ mpas_ocn_forward_mode.o: mpas_ocn_time_integration.o \ mpas_ocn_time_integration_si.o \ mpas_ocn_time_integration_split.o \ mpas_ocn_time_integration_lts.o \ + mpas_ocn_time_integration_fblts.o \ mpas_ocn_time_integration_split_ab2.o clean: diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F index af69a63e46cf..d5ba6b08eb6b 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F @@ -80,6 +80,7 @@ module ocn_forward_mode use ocn_submesoscale_eddies use ocn_stokes_drift use ocn_manufactured_solution + use ocn_subgrid use ocn_high_freq_thickness_hmix_del2 @@ -300,6 +301,8 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ ierr = ior(ierr, err_tmp) call ocn_diagnostics_init(domain, err_tmp) ierr = ior(ierr,err_tmp) + call ocn_subgrid_init(domain, err_tmp) + ierr = ior(ierr,err_tmp) if(ierr.eq.1) then call mpas_log_write('An error was encountered while initializing the MPAS-Ocean forward mode', MPAS_LOG_CRIT) @@ -700,6 +703,7 @@ function ocn_forward_mode_run(domain) result(ierr)!{{{ call mpas_pool_get_subpool(block_ptr % structs, 'forcing', forcingPool) call mpas_pool_get_subpool(block_ptr % structs, 'scratch', scratchPool) call ocn_forcing_build_fraction_absorbed_array(meshPool, statePool, forcingPool, ierr, 1) + call ocn_frazil_forcing_build_arrays(domain, meshPool, forcingPool, statePool, err) call mpas_timer_start("land_ice_build_arrays") call ocn_surface_land_ice_fluxes_build_arrays(meshPool, & forcingPool, scratchPool, statePool, err) @@ -707,7 +711,6 @@ function ocn_forward_mode_run(domain) result(ierr)!{{{ call ocn_surface_land_ice_fluxes_accumulate_fluxes(meshPool, forcingPool, & statePool, dt, err) - call ocn_frazil_forcing_build_arrays(domain, meshPool, forcingPool, statePool, err) call ocn_tidal_forcing_build_array(domain, meshPool, forcingPool, statePool, err) ! Compute normalGMBolusVelocity, relativeSlope and RediDiffVertCoef if respective flags are turned on diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration.F index 4faa2dde0fed..68edc9335359 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration.F @@ -34,6 +34,7 @@ module ocn_time_integration use ocn_time_integration_split use ocn_time_integration_si use ocn_time_integration_lts + use ocn_time_integration_fblts use ocn_time_integration_split_ab2 @@ -72,7 +73,8 @@ module ocn_time_integration timeIntSemiImplicit = 3, &! Semi-implicit timeIntRK4 = 4, &! 4th-order Runge-Kutta timeIntLTS = 5, &! local time-stepping - timeIntSplitExplicitAB2 = 6 ! split-explicit AB2 baroclinic + timeIntFBLTS = 6, &! forward-backward lts + timeIntSplitExplicitAB2 = 7 ! split-explicit AB2 baroclinic !*********************************************************************** @@ -136,6 +138,8 @@ subroutine ocn_timestep(domain, dt, timeStamp)!{{{ call ocn_time_integrator_rk4(domain, dt) case (timeIntLTS) call ocn_time_integrator_lts(domain, dt) + case (timeIntFBLTS) + call ocn_time_integrator_fblts(domain, dt) case (timeIntSplitExplicitAB2) call ocn_time_integrator_split_ab2(domain, dt) end select @@ -232,6 +236,10 @@ subroutine ocn_timestep_init(domain, dt, err)!{{{ case ('LTS') timeIntegratorChoice = timeIntLTS call ocn_time_integration_lts_init(domain) + + case ('FB_LTS') + timeIntegratorChoice = timeIntFBLTS + call ocn_time_integration_fblts_init(domain) case ('split_explicit_ab2') timeIntegratorChoice = timeIntSplitExplicitAB2 diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_fblts.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_fblts.F new file mode 100644 index 000000000000..5b0db3848cb4 --- /dev/null +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_fblts.F @@ -0,0 +1,1976 @@ +! Copyright (c) 2021, Los Alamos National Security, LLC (LANS) +! and the University Corporation for Atmospheric Research (UCAR). +! +! Unless noted otherwise source code is licensed under the BSD license. +! Additional copyright and license information can be found in the LICENSE file +! distributed with this code, or at http://mpas-dev.github.com/license.html +! +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! ocn_time_integration_fblts +! +!> \brief MPAS barotropic ocean LTS Time integration scheme +!> \author Jeremy Lilly +!> \date October 2023 +!> \details +!> This module contains the FB_LTS init routine and the FB_LTS +!> barotropic ocean time integration scheme with splitting +!> on the fast and slow tendency terms. +! +!----------------------------------------------------------------------- + +module ocn_time_integration_fblts + + use mpas_pool_routines + use mpas_dmpar + use mpas_threading + use mpas_vector_reconstruction + use mpas_timer + + use ocn_tendency + use ocn_diagnostics + use ocn_mesh + use ocn_vmix + use ocn_config + use ocn_time_average_coupled + + implicit none + private + save + + !-------------------------------------------------------------------- + ! + ! Public member functions + ! + !-------------------------------------------------------------------- + public :: ocn_time_integrator_fblts, & + ocn_time_integration_fblts_init + + contains + + +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! ocn_time_integrator_fblts +! +!> \brief MPAS barotropic ocean FB_LTS time integration scheme +!> \author Jeremy Lilly +!> \date October 2023 +!> \details +!> This routine integrates one timestep (dt) using an FB_LTS time +!> integrator with a splitting of the fast and slow tendency terms +! +!----------------------------------------------------------------------- + + subroutine ocn_time_integrator_fblts(domain, dt)!{{{ + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! Advance model state forward in time by the specified time step + ! using a local time stepping scheme with splitting of the fast and + ! slow tendencies + ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + implicit none + + !----------------------------------------------------------------- + ! Input variables + !----------------------------------------------------------------- + + real (kind=RKIND), intent(in) :: & + dt !< [in] time step (sec) to move forward + + !----------------------------------------------------------------- + ! Input/output variables + !----------------------------------------------------------------- + + type (domain_type), intent(inout) :: & + domain !< [inout] model state to advance forward + + !----------------------------------------------------------------- + ! Local variables + !----------------------------------------------------------------- + + integer :: & + iCell, iEdge, iRegion, k, ic, ie, im, & ! iterators + M, & ! M = dtCoarse / dtFine + nRegions ! number of interface regions (two) + + type (block_type), pointer :: & + block ! structure with subdomain data + + type (mpas_pool_type), pointer :: & + tendPool, & ! structure holding tendencies + statePool, & ! structure holding state variables + meshPool, & ! structure holding mesh variables + verticalMeshPool, & ! structure holding mesh variables + forcingPool, & ! structure holding forcing variables + scratchPool, & ! structure holding temporary variables + tracersPool ! structure holding tracers variables + + ! LTS Pools + type (mpas_pool_type), pointer :: & + LTSPool, & ! structure holding LTS variables + tendSlowPool, & ! structure holding the slow tendency variables + tendSum3rdPool, & ! structure holding one of the correction terms for the interface + prevTendSlowPool, nextTendSlowPool, & ! structures containing intermediate data + prevTendSum3rdPool, nextTendSum3rdPool ! structures containing intermediate data + + ! Tend Array Pointers + real (kind=RKIND), dimension(:,:), pointer :: & + normalVelocityTend, & ! normal velocity fast tendency + layerThicknessTend, & ! layer thickness tendency + normalVelocityTendSlow, & ! normal velocity slow tendency + normalVelocityTendSum3rd, & ! one of the normal velocity correction terms for the interface + layerThicknessTendSum3rd ! one of the layer thickness correction terms for the interface + + ! State Array Pointers + real (kind=RKIND), dimension(:,:), pointer :: & + normalVelocityCur, & ! normal velocity at time n + normalVelocityNew, & ! normal velocity at time n+1 + normalVelocityFirstStage, & ! normal velocity at first stage of LTS + normalVelocitySecondStage, & ! normal velocity at second stage of LTS + normalVelocityForTend, & ! extra variable to store averages of data for tends calculations + layerThicknessCur, & ! layer thickness at time n + layerThicknessNew, & ! layer thickness at time n+1 + layerThicknessFirstStage, & ! layer thickness at first stage of LTS + layerThicknessSecondStage, & ! layer thickness at second stage of LTS + layerThicknessForTend ! extra variable to store averages of data for tends calculations + + ! Local pointer for ssh + real (kind=RKIND), dimension(:), pointer :: & + ssh + + ! LTS objects + real (kind=RKIND) :: & + dtFine ! fine dt, defined as dt / M + integer, dimension(:,:), pointer :: & + nCellsInLTSRegion, & ! number of cells in a given LTS region + nEdgesInLTSRegion ! number of edges in a given LTS region + integer, dimension(:,:,:), pointer :: & + cellsInLTSRegion, & ! list of cells in a given LTS region + edgesInLTSRegion ! list of edges in a given LTS region + real (kind=RKIND) :: & + weight1st, weight2nd ! coefficients for each RK stage + real (kind=RKIND) :: & + weightTendSum3rd ! coefficients for the interface correction + + integer err + err = 0 + + !----------------------------------------------------------------- + ! Begin routine + !----------------------------------------------------------------- + + if (.not. config_disable_tr_all_tend) then + call mpas_log_write("ERROR: tracers are not currently implemented for local time-stepping") + call mpas_log_write("config_disable_tr_all_tend should be true in the namelist file") + call abort + end if + + + call mpas_timer_start("FB_LTS time-step prep") + + ! LTS parameters + M = config_dt_scaling_LTS + nRegions = 2 + dtFine = dt / M + + ! Weights for RK stages, weight for third stage is 1 + weight1st = 1.0_RKIND / 3.0_RKIND + weight2nd = 1.0_RKIND / 2.0_RKIND + + block => domain % blocklist + + ! Retrieve model state, pools + call mpas_pool_get_subpool(block%structs, 'mesh', meshPool) + call mpas_pool_get_subpool(block%structs, 'verticalMesh', verticalMeshPool) + call mpas_pool_get_subpool(block%structs, 'state', statePool) + call mpas_pool_get_subpool(block%structs, 'forcing', forcingPool) + call mpas_pool_get_subpool(block%structs, 'LTS', LTSPool) + call mpas_pool_get_subpool(block%structs, 'tend', tendPool) + call mpas_pool_get_subpool(block%structs, 'scratch', scratchPool) + call mpas_pool_get_subpool(statePool, 'tracers', tracersPool) + + ! Retrieve state variables at necessary time levels + call mpas_pool_get_array(statePool, 'normalVelocity', & + normalVelocityCur, 1) + call mpas_pool_get_array(statePool, 'normalVelocity', & + normalVelocityNew, 2) + call mpas_pool_get_array(statePool, 'normalVelocity', & + normalVelocityFirstStage, 3) + call mpas_pool_get_array(statePool, 'normalVelocity', & + normalVelocitySecondStage, 4) + call mpas_pool_get_array(statePool, 'normalVelocity', & + normalVelocityForTend, 5) + call mpas_pool_get_array(statePool, 'layerThickness', & + layerThicknessCur, 1) + call mpas_pool_get_array(statePool, 'layerThickness', & + layerThicknessNew, 2) + call mpas_pool_get_array(statePool, 'layerThickness', & + layerThicknessFirstStage, 3) + call mpas_pool_get_array(statePool, 'layerThickness', & + layerThicknessSecondStage, 4) + call mpas_pool_get_array(statePool, 'layerThickness', & + layerThicknessForTend, 5) + + ! Retrieve tendency variables + call mpas_pool_get_array(tendPool, 'normalVelocity', normalVelocityTend) + call mpas_pool_get_array(tendPool, 'layerThickness', layerThicknessTend) + + ! Retrieve LTS arrays + call mpas_pool_get_array(LTSPool, 'cellsInLTSRegion', cellsInLTSRegion) + call mpas_pool_get_array(LTSPool, 'nCellsInLTSRegion', nCellsInLTSRegion) + call mpas_pool_get_array(LTSPool, 'edgesInLTSRegion', edgesInLTSRegion) + call mpas_pool_get_array(LTSPool, 'nEdgesInLTSRegion', nEdgesInLTSRegion) + + ! Create and retrieve additional pools for LTS + call mpas_pool_get_subpool(block % structs, 'tend_sum_3rd', tendSum3rdPool) + call mpas_pool_get_subpool(block % structs, 'tend_slow', tendSlowPool) + + call mpas_pool_copy_pool(tendPool, tendSum3rdPool, 1) + call mpas_pool_copy_pool(tendPool, tendSlowPool, 1) + + + call mpas_pool_get_array(tendSlowPool, 'normalVelocity', & + normalVelocityTendSlow) + call mpas_pool_get_array(tendSum3rdPool, 'normalVelocity', & + normalVelocityTendSum3rd) + call mpas_pool_get_array(tendSum3rdPool, 'layerThickness', & + layerThicknessTendSum3rd) + + ! Init variables + do iEdge = 1, nEdgesAll ! do we need this? + do k = 1, maxLevelEdgeTop(iEdge) + normalVelocityNew(k, iEdge) = normalVelocityCur(k, iEdge) + normalVelocityFirstStage(k, iEdge) = normalVelocityCur(k, iEdge) + normalVelocitySecondStage(k, iEdge) = normalVelocityCur(k, iEdge) + end do + end do + + do iCell = 1, nCellsAll ! do we need this? + do k = 1, maxLevelCell(iCell) + layerThicknessNew(k, iCell) = layerThicknessCur(k, iCell) + layerThicknessFirstStage(k, iCell) = layerThicknessCur(k, iCell) + layerThicknessSecondStage(k, iCell) = layerThicknessCur(k, iCell) + end do + end do + + normalVelocityTendSum3rd(:,:) = 0.0_RKIND + layerThicknessTendSum3rd(:,:) = 0.0_RKIND + + + call mpas_timer_stop("FB_LTS time-step prep") + + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! + ! BEGIN FB_LTS SCHEME + ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call mpas_timer_start("FB_LTS main loop") + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! BEGIN: Slow tendency calculation + ! Calculate the slow tendencies on all LTS regions (only the + ! momentum equation contains slow terms) + call mpas_timer_start("FB_LTS compute slow tendencies") + + call ocn_tend_vel(domain, tendSlowPool, statePool, forcingPool, 1, & + domain % dminfo, dt) + + call mpas_timer_stop("FB_LTS compute slow tendencies") + ! END: Slow tendency calculation + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! BEGIN COARSE ADVANCEMENT + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! BEGIN: Thickness stage 1 + ! Compute the first stage of the thickness on fine*, interface 1, + ! interface 2, and coarse + + ! Compute fast tendencies for thickness + call mpas_timer_start("FB_LTS compute fast tendencies") + call mpas_timer_start("FB_LTS compute fast thickness tendencies") + + call ocn_lts_thick_tend(LTSPool, tendPool, & + normalVelocityCur, layerThicknessCur, & + 0, 1, 1, 1, 1) + + call mpas_timer_stop("FB_LTS compute fast thickness tendencies") + call mpas_timer_stop("FB_LTS compute fast tendencies") + + ! Advance thickness solution with first stage + call mpas_timer_start("FB_LTS advance solution") + call mpas_timer_start("FB_LTS advance thickness solution") + + ! Interface layers + do iRegion = 1,nRegions + do ic = 1, nCellsInLTSRegion(iRegion,2) + iCell = cellsInLTSRegion(iRegion,2,ic) + layerThicknessFirstStage(:,iCell) = layerThicknessCur(:,iCell) & + + weight1st * dt * layerThicknessTend(:,iCell) + end do + end do + ! Coarse + do ic = 1, nCellsInLTSRegion(2,1) + iCell = cellsInLTSRegion(2,1,ic) + layerThicknessFirstStage(:,iCell) = layerThicknessCur(:,iCell) & + + weight1st * dt * layerThicknessTend(:,iCell) + end do + ! Fine layers close to interface layers + do ic = 1, nCellsInLTSRegion(1,3) + iCell = cellsInLTSRegion(1,3,ic) + layerThicknessFirstStage(:,iCell) = layerThicknessCur(:,iCell) & + + weight1st * dt * layerThicknessTend(:,iCell) + end do + + call mpas_timer_stop("FB_LTS advance thickness solution") + call mpas_timer_stop("FB_LTS advance solution") + + ! END: Thickness stage 1 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! BEGIN: Normal velocity stage 1 + ! Compute the first stage of the normal velocity on fine*, + ! interface 1, interface 2, and coarse + + ! Perform forward-backward average of thickness for stage one of + ! FB-RK(3,2): + ! h = weight1*stage1 + (1-weight1)*current + layerThicknessForTend(:,:) = config_fb_weight_1 * layerThicknessFirstStage(:,:) & + + (1.0_RKIND - config_fb_weight_1) * layerThicknessCur(:,:) + + ! Halo update before tend calculation + ! Might be able to remove this? + call mpas_timer_start("FB_LTS prognostic halo update") + + ! Don't need to update normalVelocityCur, already updated + + call mpas_timer_start("FB_LTS thickness halo update") + call mpas_dmpar_field_halo_exch(domain, 'layerThickness', timeLevel=5) + call mpas_timer_stop("FB_LTS thickness halo update") + + call mpas_timer_stop("FB_LTS prognostic halo update") + + ! Compute fast tendencies for normal velocity + call mpas_timer_start("FB_LTS compute fast tendencies") + call mpas_timer_start("FB_LTS compute fast velocity tendencies") + + call ocn_lts_vel_tend(LTSPool, tendPool, & + normalVelocityCur, layerThicknessForTend, & + 0, 1, 1, 1, 1) + + normalVelocityTend(:,:) = normalVelocityTend(:,:) & + + normalVelocityTendSlow(:,:) + + call mpas_timer_stop("FB_LTS compute fast velocity tendencies") + call mpas_timer_stop("FB_LTS compute fast tendencies") + + ! Advance normal velocity solution with first stage + call mpas_timer_start("FB_LTS advance solution") + call mpas_timer_start("FB_LTS advance velocity solution") + + ! Interface regions + do iRegion = 1,nRegions + do ie = 1, nEdgesInLTSRegion(iRegion,2) + iEdge = edgesInLTSRegion(iRegion,2,ie) + normalVelocityFirstStage(:,iEdge) = normalVelocityCur(:,iEdge) & + + weight1st * dt * normalVelocityTend(:,iEdge) + end do + end do + ! Coarse + do ie = 1, nEdgesInLTSRegion(2,1) + iEdge = edgesInLTSRegion(2,1,ie) + normalVelocityFirstStage(:,iEdge) = normalVelocityCur(:,iEdge) & + + weight1st * dt * normalVelocityTend(:,iEdge) + end do + ! Fine layers close to interface layers + do ie = 1, nEdgesInLTSRegion(1,3) + iEdge = edgesInLTSRegion(1,3,ie) + normalVelocityFirstStage(:,iEdge) = normalVelocityCur(:,iEdge) & + + weight1st * dt * normalVelocityTend(:,iEdge) + end do + + call mpas_timer_stop("FB_LTS advance velocity solution") + call mpas_timer_stop("FB_LTS advance solution") + + ! END: Normal velocity stage 1 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! BEGIN: Thickness stage 2 + ! Compute the second stage of the thickness on fine*, interface 1, + ! interface 2, and coarse + + ! Halo update before tend calculation + call mpas_timer_start("FB_LTS prognostic halo update") + + call mpas_timer_start("FB_LTS velocity halo update") + call mpas_dmpar_field_halo_exch(domain, 'normalVelocity', timeLevel=3) + call mpas_timer_stop("FB_LTS velocity halo update") + + call mpas_timer_start("FB_LTS thickness halo update") + call mpas_dmpar_field_halo_exch(domain, 'layerThickness', timeLevel=3) + call mpas_timer_stop("FB_LTS thickness halo update") + + call mpas_timer_stop("FB_LTS prognostic halo update") + + ! Compute fast tendencies for thickness + call mpas_timer_start("FB_LTS compute fast tendencies") + call mpas_timer_start("FB_LTS compute fast thickness tendencies") + + call ocn_lts_thick_tend(LTSPool, tendPool, & + normalVelocityFirstStage, layerThicknessFirstStage, & + 0, 1, 1, 1, 1) + + call mpas_timer_stop("FB_LTS compute fast thickness tendencies") + call mpas_timer_stop("FB_LTS compute fast tendencies") + + ! Advance thickness solution with first stage + call mpas_timer_start("FB_LTS advance solution") + call mpas_timer_start("FB_LTS advance thickness solution") + + ! Interface layers + do iRegion = 1,nRegions + do ic = 1, nCellsInLTSRegion(iRegion,2) + iCell = cellsInLTSRegion(iRegion,2,ic) + layerThicknessSecondStage(:,iCell) = layerThicknessCur(:,iCell) & + + weight2nd * dt * layerThicknessTend(:,iCell) + end do + end do + ! Coarse + do ic = 1, nCellsInLTSRegion(2,1) + iCell = cellsInLTSRegion(2,1,ic) + layerThicknessSecondStage(:,iCell) = layerThicknessCur(:,iCell) & + + weight2nd * dt * layerThicknessTend(:,iCell) + end do + ! Fine layers close to interface layers + do ic = 1, nCellsInLTSRegion(1,3) + iCell = cellsInLTSRegion(1,3,ic) + layerThicknessSecondStage(:,iCell) = layerThicknessCur(:,iCell) & + + weight2nd * dt * layerThicknessTend(:,iCell) + end do + + call mpas_timer_stop("FB_LTS advance thickness solution") + call mpas_timer_stop("FB_LTS advance solution") + + ! END: Thickness stage 2 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! BEGIN: Normal velocity stage 2 + ! Compute the second stage of the normal velocity on fine*, + ! interface 1, interface 2, and coarse + + ! Perform forward-backward average of thickness for stage two of + ! FB-RK(3,2): + ! h = weight2*stage2 + (1-weight2)*current + layerThicknessForTend(:,:) = config_fb_weight_2 * layerThicknessSecondStage(:,:) & + + (1.0_RKIND - config_fb_weight_2) * layerThicknessCur(:,:) + + ! Halo update before tend calculation + ! Might be able to remove this? + call mpas_timer_start("FB_LTS prognostic halo update") + + ! Don't need to update normalVelocityFirstStage, previously updated + + call mpas_timer_start("FB_LTS thickness halo update") + call mpas_dmpar_field_halo_exch(domain, 'layerThickness', timeLevel=5) + call mpas_timer_stop("FB_LTS thickness halo update") + + call mpas_timer_stop("FB_LTS prognostic halo update") + + ! Compute fast tendencies for normal velocity + call mpas_timer_start("FB_LTS compute fast tendencies") + call mpas_timer_start("FB_LTS compute fast velocity tendencies") + + call ocn_lts_vel_tend(LTSPool, tendPool, & + normalVelocityFirstStage, layerThicknessForTend, & + 0, 1, 1, 1, 1) + + normalVelocityTend(:,:) = normalVelocityTend(:,:) & + + normalVelocityTendSlow(:,:) + + call mpas_timer_stop("FB_LTS compute fast velocity tendencies") + call mpas_timer_stop("FB_LTS compute fast tendencies") + + ! Advance normal velocity solution with first stage + call mpas_timer_start("FB_LTS advance solution") + call mpas_timer_start("FB_LTS advance velocity solution") + + ! Interface regions + do iRegion = 1,nRegions + do ie = 1, nEdgesInLTSRegion(iRegion,2) + iEdge = edgesInLTSRegion(iRegion,2,ie) + normalVelocitySecondStage(:,iEdge) = normalVelocityCur(:,iEdge) & + + weight2nd * dt * normalVelocityTend(:,iEdge) + end do + end do + ! Coarse + do ie = 1, nEdgesInLTSRegion(2,1) + iEdge = edgesInLTSRegion(2,1,ie) + normalVelocitySecondStage(:,iEdge) = normalVelocityCur(:,iEdge) & + + weight2nd * dt * normalVelocityTend(:,iEdge) + end do + ! Fine layers close to interface layers + do ie = 1, nEdgesInLTSRegion(1,3) + iEdge = edgesInLTSRegion(1,3,ie) + normalVelocitySecondStage(:,iEdge) = normalVelocityCur(:,iEdge) & + + weight2nd * dt * normalVelocityTend(:,iEdge) + end do + + call mpas_timer_stop("FB_LTS advance velocity solution") + call mpas_timer_stop("FB_LTS advance solution") + + ! END: Normal velocity stage 2 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! BEGIN: Thickness stage 3 + ! Compute the third stage of the thickness on fine*, interface 1, + ! interface 2, and coarse + + ! Halo update before tends calculation + call mpas_timer_start("FB_LTS prognostic halo update") + + call mpas_timer_start("FB_LTS velocity halo update") + call mpas_dmpar_field_halo_exch(domain, 'normalVelocity', timeLevel=4) + call mpas_timer_stop("FB_LTS velocity halo update") + + call mpas_timer_start("FB_LTS thickness halo update") + call mpas_dmpar_field_halo_exch(domain, 'layerThickness', timeLevel=4) + call mpas_timer_stop("FB_LTS thickness halo update") + + call mpas_timer_stop("FB_LTS prognostic halo update") + + ! Compute fast tendencies for thickness + call mpas_timer_start("FB_LTS compute fast tendencies") + call mpas_timer_start("FB_LTS compute fast thickness tendencies") + + call ocn_lts_thick_tend(LTSPool, tendPool, & + normalVelocitySecondStage, layerThicknessSecondStage, & + 0, 1, 1, 1, 1) + + call mpas_timer_stop("FB_LTS compute fast thickness tendencies") + call mpas_timer_stop("FB_LTS compute fast tendencies") + + ! Advance thickness solution with first stage + call mpas_timer_start("FB_LTS advance solution") + call mpas_timer_start("FB_LTS advance thickness solution") + + ! Interface layers + do iRegion = 1,nRegions + do ic = 1, nCellsInLTSRegion(iRegion,2) + iCell = cellsInLTSRegion(iRegion,2,ic) + layerThicknessNew(:,iCell) = layerThicknessCur(:,iCell) & + + dt * layerThicknessTend(:,iCell) + end do + end do + ! Coarse + do ic = 1, nCellsInLTSRegion(2,1) + iCell = cellsInLTSRegion(2,1,ic) + layerThicknessNew(:,iCell) = layerThicknessCur(:,iCell) & + + dt * layerThicknessTend(:,iCell) + end do + ! Fine layers close to interface layers + do ic = 1, nCellsInLTSRegion(1,3) + iCell = cellsInLTSRegion(1,3,ic) + layerThicknessNew(:,iCell) = layerThicknessCur(:,iCell) & + + dt * layerThicknessTend(:,iCell) + end do + + call mpas_timer_stop("FB_LTS advance thickness solution") + call mpas_timer_stop("FB_LTS advance solution") + + ! END: Thickness stage 3 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! BEGIN: Normal velocity stage 3 + ! Compute the third stage of the normal velocity on, interface 1, + ! and coarse + + ! Perform forward-backward average of thickness for stage three of + ! FB-RK(3,2): + ! h = weight3*new + (1-2*weight3)*secondStage + weight3*current + layerThicknessForTend(:,:) = config_fb_weight_3 * layerThicknessNew(:,:) & + + (1.0_RKIND - 2.0_RKIND*config_fb_weight_3) * layerThicknessSecondStage(:,:) & + + config_fb_weight_3 * layerThicknessCur(:,:) + + ! Halo update before tend calculation + ! Might be able to remove this? + call mpas_timer_start("FB_LTS prognostic halo update") + + ! Don't need to update normalVelocitySecondStage, previously updated + + call mpas_timer_start("FB_LTS thickness halo update") + call mpas_dmpar_field_halo_exch(domain, 'layerThickness', timeLevel=5) + call mpas_timer_stop("FB_LTS thickness halo update") + + call mpas_timer_stop("FB_LTS prognostic halo update") + + ! Compute fast tendencies for normal velocity + call mpas_timer_start("FB_LTS compute fast tendencies") + call mpas_timer_start("FB_LTS compute fast velocity tendencies") + + call ocn_lts_vel_tend(LTSPool, tendPool, & + normalVelocitySecondStage, layerThicknessForTend, & + 0, 0, 1, 0, 1) + + normalVelocityTend(:,:) = normalVelocityTend(:,:) & + + normalVelocityTendSlow(:,:) + + call mpas_timer_stop("FB_LTS compute fast velocity tendencies") + call mpas_timer_stop("FB_LTS compute fast tendencies") + + ! Advance normal velocity solution with first stage + call mpas_timer_start("FB_LTS advance solution") + call mpas_timer_start("FB_LTS advance velocity solution") + + ! Interface 1 + do ie = 1, nEdgesInLTSRegion(1,2) + iEdge = edgesInLTSRegion(1,2,ie) + normalVelocityNew(:,iEdge) = normalVelocityCur(:,iEdge) & + + dt * normalVelocityTend(:,iEdge) + end do + ! Coarse + do ie = 1, nEdgesInLTSRegion(2,1) + iEdge = edgesInLTSRegion(2,1,ie) + normalVelocityNew(:,iEdge) = normalVelocityCur(:,iEdge) & + + dt * normalVelocityTend(:,iEdge) + end do + + call mpas_timer_stop("FB_LTS advance velocity solution") + call mpas_timer_stop("FB_LTS advance solution") + + ! END: Normal velocity stage 3 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! END COARSE ADVANCEMENT + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! BEGIN FINE ADVANCEMENT + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + do im = 0, M-1 + + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! BEGIN: Thickness stage 1 + ! Compute the first stage of the thickness on fine + + ! Calculate predicted 'current' data at intermediate + ! time-level on interface 1: + ! u,h = (im/M)*new + (1- (im/M))*current + call mpas_timer_start('FB_LTS interface prediction') + + normalVelocityForTend(:,:) = normalVelocityCur(:,:) + do ie = 1, nEdgesInLTSRegion(1,2) + iEdge = edgesInLTSRegion(1,2,ie) + normalVelocityForTend(:,iEdge) = (REAL(im)/M) * normalVelocityNew(:,iEdge) & + + (1.0_RKIND - REAL(im)/M) * normalVelocityCur(:,iEdge) + end do + + layerThicknessForTend(:,:) = layerThicknessCur(:,:) + do ic = 1, nCellsInLTSRegion(1,2) + iCell = cellsInLTSRegion(1,2,ic) + layerThicknessForTend(:,iCell) = (REAL(im)/M) * layerThicknessNew(:,iCell) & + + (1.0_RKIND - REAL(im)/M) * layerThicknessCur(:,iCell) + end do + + call mpas_timer_stop('FB_LTS interface prediction') + + ! Halo update before tend calculation + call mpas_timer_start("FB_LTS prognostic halo update") + + call mpas_timer_start("FB_LTS velocity halo update") + call mpas_dmpar_field_halo_exch(domain, 'normalVelocity', timeLevel=5) + call mpas_timer_stop("FB_LTS velocity halo update") + + call mpas_timer_start("FB_LTS thickness halo update") + call mpas_dmpar_field_halo_exch(domain, 'layerThickness', timeLevel=5) + call mpas_timer_stop("FB_LTS thickness halo update") + + call mpas_timer_stop("FB_LTS prognostic halo update") + + ! Compute fast tendencies for thickness + call mpas_timer_start("FB_LTS compute fast tendencies") + call mpas_timer_start("FB_LTS compute fast thickness tendencies") + + call ocn_lts_thick_tend(LTSPool, tendPool, & + normalVelocityForTend, layerThicknessForTend, & + 1, 1, 0, 0, 0) + + call mpas_timer_stop("FB_LTS compute fast thickness tendencies") + call mpas_timer_stop("FB_LTS compute fast tendencies") + + ! Advance thickness solution with first stage + call mpas_timer_start("FB_LTS advance solution") + call mpas_timer_start("FB_LTS advance thickness solution") + + ! Fine cells adjacent to interface 1 + do ic = 1, nCellsInLTSRegion(1,3) + iCell = cellsInLTSRegion(1,3,ic) + layerThicknessFirstStage(:,iCell) = layerThicknessCur(:,iCell) & + + weight1st * dtFine * layerThicknessTend(:,iCell) + end do + ! Fine in the interior, away from interface 1 + do ic = 1, nCellsInLTSRegion(1,1) + iCell = cellsInLTSRegion(1,1,ic) + layerThicknessFirstStage(:,iCell) = layerThicknessCur(:,iCell) & + + weight1st * dtFine * layerThicknessTend(:,iCell) + end do + + call mpas_timer_stop("FB_LTS advance thickness solution") + call mpas_timer_stop("FB_LTS advance solution") + + ! END: Thickness stage 1 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! BEGIN: Normal velocity stage 1 + ! Compute the first stage of the normal velocity on fine + + ! Perform forward-backward average of thickness for stage one of + ! FB-RK(3,2): + ! h = weight1*stage1 + (1-weight1)*current + layerThicknessForTend(:,:) = config_fb_weight_1 * layerThicknessFirstStage(:,:) & + + (1.0_RKIND - config_fb_weight_1) * layerThicknessCur(:,:) + + ! Calculate predicted FB averaged thickness data at + ! intermediate time-level on interface 1: + ! h = weight1*'firstStage' + (1-weight1)*'current' + ! = weight1*( (im/M)*new + (1/M)*firstStage + (1-(im+1)/M)*current ) + ! + (1-weight1)*( (im/M)*new + (1-im/M)*current ) + call mpas_timer_start('FB_LTS interface prediction') + + do ic = 1, nCellsInLTSRegion(1,2) + iCell = cellsInLTSRegion(1,2,ic) + layerThicknessForTend(:,iCell) = config_fb_weight_1 & + * ( (REAL(im)/M) * layerThicknessNew(:,iCell) & + + (1.0_RKIND/M) * layerThicknessFirstStage(:,iCell) & + + (1.0_RKIND - (REAL(im)+1.0_RKIND)/M) * layerThicknessCur(:,iCell) ) & + + (1.0_RKIND - config_fb_weight_1) & + * ( (REAL(im)/M) * layerThicknessNew(:,iCell) & + + (1.0_RKIND - REAL(im)/M) * layerThicknessCur(:,iCell) ) + end do + + call mpas_timer_stop('FB_LTS interface prediction') + + ! Halo update before tend calculation + call mpas_timer_start("FB_LTS prognostic halo update") + + ! Don't need to update normalVelocityForTend, previously updated + + call mpas_timer_start("FB_LTS thickness halo update") + call mpas_dmpar_field_halo_exch(domain, 'layerThickness', timeLevel=5) + call mpas_timer_stop("FB_LTS thickness halo update") + + call mpas_timer_stop("FB_LTS prognostic halo update") + + ! Compute fast tendencies for normal velocity + call mpas_timer_start("FB_LTS compute fast tendencies") + call mpas_timer_start("FB_LTS compute fast velocity tendencies") + + call ocn_lts_vel_tend(LTSPool, tendPool, & + normalVelocityForTend, layerThicknessForTend, & + 1, 1, 0, 0, 0) + + normalVelocityTend(:,:) = normalVelocityTend(:,:) & + + normalVelocityTendSlow(:,:) + + call mpas_timer_stop("FB_LTS compute fast velocity tendencies") + call mpas_timer_stop("FB_LTS compute fast tendencies") + + ! Advance normal velocity solution with first stage + call mpas_timer_start("FB_LTS advance solution") + call mpas_timer_start("FB_LTS advance velocity solution") + + ! Fine cells adjacent to interface 1 + do ie = 1, nEdgesInLTSRegion(1,3) + iEdge = edgesInLTSRegion(1,3,ie) + normalVelocityFirstStage(:,iEdge) = normalVelocityCur(:,iEdge) & + + weight1st * dtFine * normalVelocityTend(:,iEdge) + end do + ! Fine in the interior, away from interface 1 + do ie = 1, nEdgesInLTSRegion(1,1) + iEdge = edgesInLTSRegion(1,1,ie) + normalVelocityFirstStage(:,iEdge) = normalVelocityCur(:,iEdge) & + + weight1st * dtFine * normalVelocityTend(:,iEdge) + end do + + call mpas_timer_stop("FB_LTS advance velocity solution") + call mpas_timer_stop("FB_LTS advance solution") + + ! END: Normal velocity stage 1 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! BEGIN: Thickness stage 2 + ! Compute the second stage of the thickness on fine + + ! Calculate predicted stage 1 data at intermediate + ! time-level on interface 1: + ! u,h = (im/M)*new + (1/M)*firstStage + (1-(im+1)/M)*current + call mpas_timer_start('FB_LTS interface prediction') + + normalVelocityForTend(:,:) = normalVelocityFirstStage(:,:) + do ie = 1, nEdgesInLTSRegion(1,2) + iEdge = edgesInLTSRegion(1,2,ie) + normalVelocityForTend(:,iEdge) = (REAL(im)/M) * normalVelocityNew(:,iEdge) & + + (1.0_RKIND/M) * normalVelocityFirstStage(:,iEdge) & + + (1.0_RKIND - (REAL(im)+1.0_RKIND)/M) * normalVelocityCur(:,iEdge) + end do + + layerThicknessForTend(:,:) = layerThicknessFirstStage(:,:) + do ic = 1, nCellsInLTSRegion(1,2) + iCell = cellsInLTSRegion(1,2,ic) + layerThicknessForTend(:,iCell) = (REAL(im)/M) * layerThicknessNew(:,iCell) & + + (1.0_RKIND/M) * layerThicknessFirstStage(:,iCell) & + + (1.0_RKIND - (REAL(im)+1.0_RKIND)/M) * layerThicknessCur(:,iCell) + end do + + call mpas_timer_stop('FB_LTS interface prediction') + + ! Halo update before tend calculation + call mpas_timer_start("FB_LTS prognostic halo update") + + call mpas_timer_start("FB_LTS velocity halo update") + call mpas_dmpar_field_halo_exch(domain, 'normalVelocity', timeLevel=5) + call mpas_timer_stop("FB_LTS velocity halo update") + + call mpas_timer_start("FB_LTS thickness halo update") + call mpas_dmpar_field_halo_exch(domain, 'layerThickness', timeLevel=5) + call mpas_timer_stop("FB_LTS thickness halo update") + + call mpas_timer_stop("FB_LTS prognostic halo update") + + ! Compute fast tendencies for thickness + call mpas_timer_start("FB_LTS compute fast tendencies") + call mpas_timer_start("FB_LTS compute fast thickness tendencies") + + call ocn_lts_thick_tend(LTSPool, tendPool, & + normalVelocityForTend, layerThicknessForTend, & + 1, 1, 0, 0, 0) + + call mpas_timer_stop("FB_LTS compute fast thickness tendencies") + call mpas_timer_stop("FB_LTS compute fast tendencies") + + ! Advance thickness solution with second stage + call mpas_timer_start("FB_LTS advance solution") + call mpas_timer_start("FB_LTS advance thickness solution") + + ! Fine cells adjacent to interface 1 + do ic = 1, nCellsInLTSRegion(1,3) + iCell = cellsInLTSRegion(1,3,ic) + layerThicknessSecondStage(:,iCell) = layerThicknessCur(:,iCell) & + + weight2nd * dtFine * layerThicknessTend(:,iCell) + end do + ! Fine in the interior, away from interface 1 + do ic = 1, nCellsInLTSRegion(1,1) + iCell = cellsInLTSRegion(1,1,ic) + layerThicknessSecondStage(:,iCell) = layerThicknessCur(:,iCell) & + + weight2nd * dtFine * layerThicknessTend(:,iCell) + end do + + call mpas_timer_stop("FB_LTS advance thickness solution") + call mpas_timer_stop("FB_LTS advance solution") + + ! END: Thickness stage 2 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! BEGIN: Normal velocity stage 2 + ! Compute the second stage of the normal velocity on fine + + ! Perform forward-backward average of thickness for stage two of + ! FB-RK(3,2): + ! h = weight2*stage2 + (1-weight2)*current + layerThicknessForTend(:,:) = config_fb_weight_2 * layerThicknessSecondStage(:,:) & + + (1.0_RKIND - config_fb_weight_2) * layerThicknessCur(:,:) + + ! Calculate predicted FB averaged thickness data at + ! intermediate time-level on interface 1: + ! h = weight2*'secondStage' + (1-weight2)*'current' + ! = weight2*( (im/M)*new + (1/M)*secondStage + (1-(im+1)/M)*current ) + ! + (1-weight2)*( (im/M)*new + (1-im/M)*current ) + call mpas_timer_start('FB_LTS interface prediction') + + do ic = 1, nCellsInLTSRegion(1,2) + iCell = cellsInLTSRegion(1,2,ic) + layerThicknessForTend(:,iCell) = config_fb_weight_2 & + * ( (REAL(im)/M) * layerThicknessNew(:,iCell) & + + (1.0_RKIND/M) * layerThicknessSecondStage(:,iCell) & + + (1.0_RKIND - (REAL(im)+1.0_RKIND)/M) * layerThicknessCur(:,iCell) ) & + + (1.0_RKIND - config_fb_weight_2) & + * ( (REAL(im)/M) * layerThicknessNew(:,iCell) & + + (1.0_RKIND - REAL(im)/M) * layerThicknessCur(:,iCell) ) + end do + + call mpas_timer_stop('FB_LTS interface prediction') + + ! Halo update before tend calculation + call mpas_timer_start("FB_LTS prognostic halo update") + + ! Don't need to update normalVelocityForTend, previously updated + + call mpas_timer_start("FB_LTS thickness halo update") + call mpas_dmpar_field_halo_exch(domain, 'layerThickness', timeLevel=5) + call mpas_timer_stop("FB_LTS thickness halo update") + + call mpas_timer_stop("FB_LTS prognostic halo update") + + ! Compute fast tendencies for normal velocity + call mpas_timer_start("FB_LTS compute fast tendencies") + call mpas_timer_start("FB_LTS compute fast velocity tendencies") + + call ocn_lts_vel_tend(LTSPool, tendPool, & + normalVelocityForTend, layerThicknessForTend, & + 1, 1, 0, 0, 0) + + normalVelocityTend(:,:) = normalVelocityTend(:,:) & + + normalVelocityTendSlow(:,:) + + call mpas_timer_stop("FB_LTS compute fast velocity tendencies") + call mpas_timer_stop("FB_LTS compute fast tendencies") + + ! Advance normal velocity solution with second stage + call mpas_timer_start("FB_LTS advance solution") + call mpas_timer_start("FB_LTS advance velocity solution") + + ! Fine cells adjacent to interface 1 + do ie = 1, nEdgesInLTSRegion(1,3) + iEdge = edgesInLTSRegion(1,3,ie) + normalVelocitySecondStage(:,iEdge) = normalVelocityCur(:,iEdge) & + + weight2nd * dtFine * normalVelocityTend(:,iEdge) + end do + ! Fine in the interior, away from interface 1 + do ie = 1, nEdgesInLTSRegion(1,1) + iEdge = edgesInLTSRegion(1,1,ie) + normalVelocitySecondStage(:,iEdge) = normalVelocityCur(:,iEdge) & + + weight2nd * dtFine * normalVelocityTend(:,iEdge) + end do + + call mpas_timer_stop("FB_LTS advance velocity solution") + call mpas_timer_stop("FB_LTS advance solution") + + ! END: Normal velocity stage 2 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! BEGIN: Thickness stage 3 + ! Compute the third stage of the thickness on fine and + ! increment the third stage interface correction terms + + ! Calculate predicted stage 2 data at intermediate + ! time-level on interface 1: + ! u,h = (im/M)*new + (1/M)*secondStage + (1-(im+1)/M)*current + call mpas_timer_start('FB_LTS interface prediction') + + normalVelocityForTend(:,:) = normalVelocitySecondStage(:,:) + do ie = 1, nEdgesInLTSRegion(1,2) + iEdge = edgesInLTSRegion(1,2,ie) + normalVelocityForTend(:,iEdge) = (REAL(im)/M) * normalVelocityNew(:,iEdge) & + + (1.0_RKIND/M) * normalVelocitySecondStage(:,iEdge) & + + (1.0_RKIND - (REAL(im)+1.0_RKIND)/M) * normalVelocityCur(:,iEdge) + end do + + layerThicknessForTend(:,:) = layerThicknessSecondStage(:,:) + do ic = 1, nCellsInLTSRegion(1,2) + iCell = cellsInLTSRegion(1,2,ic) + layerThicknessForTend(:,iCell) = (REAL(im)/M) * layerThicknessNew(:,iCell) & + + (1.0_RKIND/M) * layerThicknessSecondStage(:,iCell) & + + (1.0_RKIND - (REAL(im)+1.0_RKIND)/M) * layerThicknessCur(:,iCell) + end do + + call mpas_timer_stop('FB_LTS interface prediction') + + ! Halo update before tend calculation + call mpas_timer_start("FB_LTS prognostic halo update") + + call mpas_timer_start("FB_LTS velocity halo update") + call mpas_dmpar_field_halo_exch(domain, 'normalVelocity', timeLevel=5) + call mpas_timer_stop("FB_LTS velocity halo update") + + call mpas_timer_start("FB_LTS thickness halo update") + call mpas_dmpar_field_halo_exch(domain, 'layerThickness', timeLevel=5) + call mpas_timer_stop("FB_LTS thickness halo update") + + call mpas_timer_stop("FB_LTS prognostic halo update") + + ! Compute fast tendencies for thickness + call mpas_timer_start("FB_LTS compute fast tendencies") + call mpas_timer_start("FB_LTS compute fast thickness tendencies") + + call ocn_lts_thick_tend(LTSPool, tendPool, & + normalVelocityForTend, layerThicknessForTend, & + 1, 1, 1, 1, 0) + + call mpas_timer_stop("FB_LTS compute fast thickness tendencies") + call mpas_timer_stop("FB_LTS compute fast tendencies") + + ! Advance thickness solution with third stage + call mpas_timer_start("FB_LTS advance solution") + call mpas_timer_start("FB_LTS advance thickness solution") + + ! Increment interface correction terms + do iRegion = 1,nRegions + do ic = 1, nCellsInLTSRegion(iRegion,2) + iCell = cellsInLTSRegion(iRegion,2,ic) + layerThicknessTendSum3rd(:,iCell) = layerThicknessTendSum3rd(:,iCell) & + + layerThicknessTend(:,iCell) + end do + end do + ! Fine cells adjacent to interface 1 + do ic = 1, nCellsInLTSRegion(1,3) + iCell = cellsInLTSRegion(1,3,ic) + layerThicknessNew(:,iCell) = layerThicknessCur(:,iCell) & + + dtFine * layerThicknessTend(:,iCell) + end do + ! Fine in the interior, away from interface 1 + do ic = 1, nCellsInLTSRegion(1,1) + iCell = cellsInLTSRegion(1,1,ic) + layerThicknessNew(:,iCell) = layerThicknessCur(:,iCell) & + + dtFine * layerThicknessTend(:,iCell) + end do + + call mpas_timer_stop("FB_LTS advance thickness solution") + call mpas_timer_stop("FB_LTS advance solution") + + ! END: Thickness stage 3 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! BEGIN: Normal velocity stage 3 + ! Compute the third stage of the normal velocity on fine and + ! increment the third stage interface correction terms + + ! Perform forward-backward average of thickness for stage three of + ! FB-RK(3,2): + ! h = weight3*new + (1-2*weight3)*stageTwo + weight3*current + layerThicknessForTend(:,:) = config_fb_weight_3 * layerThicknessNew(:,:) & + + (1.0_RKIND - 2.0_RKIND*config_fb_weight_3) * layerThicknessSecondStage(:,:) & + + config_fb_weight_3 * layerThicknessCur(:,:) + + ! Calculate predicted FB averaged thickness data at + ! intermediate time-level on interface 1: + ! h = weight3*new + (1-2*weight3)*'secondStage' + weight3*'current' + ! = weight3*( ((im+1)/M)*new + (1-(im+1)/M)*current ) + ! + (1-2*weight3)*( (im/M)*new + (1/M)*secondStage + (1-(im+1)/M)*current ) + ! + weight3*( (im/M)*new + (1-im/M)*current ) + call mpas_timer_start('FB_LTS interface prediction') + + do ic = 1, nCellsInLTSRegion(1,2) + iCell = cellsInLTSRegion(1,2,ic) + layerThicknessForTend(:,iCell) = config_fb_weight_3 & + * ( ((REAL(im)+1.0_RKIND)/M) * layerThicknessNew(:,iCell) & + + (1.0_RKIND - (REAL(im)+1.0_RKIND)/M) * layerThicknessCur(:,iCell) ) & + + (1.0_RKIND - 2.0_RKIND*config_fb_weight_3) & + * ( (REAL(im)/M) * layerThicknessNew(:,iCell) & + + (1.0_RKIND/M) * layerThicknessSecondStage(:,iCell) & + + (1.0_RKIND - (REAL(im)+1.0_RKIND)/M) * layerThicknessCur(:,iCell) ) & + + config_fb_weight_3 & + * ( (REAL(im)/M) * layerThicknessNew(:,iCell) & + + (1.0_RKIND - REAL(im)/M) * layerThicknessCur(:,iCell) ) + end do + + call mpas_timer_stop('FB_LTS interface prediction') + + ! Halo update before tend calculation + call mpas_timer_start("FB_LTS prognostic halo update") + + ! Don't need to update normalVelocityForTend, previously updated + + call mpas_timer_start("FB_LTS thickness halo update") + call mpas_dmpar_field_halo_exch(domain, 'layerThickness', timeLevel=5) + call mpas_timer_stop("FB_LTS thickness halo update") + + call mpas_timer_stop("FB_LTS prognostic halo update") + + ! Compute fast tendencies for normal velocity + call mpas_timer_start("FB_LTS compute fast tendencies") + call mpas_timer_start("FB_LTS compute fast velocity tendencies") + + call ocn_lts_vel_tend(LTSPool, tendPool, & + normalVelocityForTend, layerThicknessForTend, & + 1, 1, 1, 1, 0) + + normalVelocityTend(:,:) = normalVelocityTend(:,:) & + + normalVelocityTendSlow(:,:) + + call mpas_timer_stop("FB_LTS compute fast velocity tendencies") + call mpas_timer_stop("FB_LTS compute fast tendencies") + + ! Advance normal velocity solution with third stage + ! Here, we write into normalVelocityCur, rather than + ! normalVelocityNew, so that the New data is stored + ! in normalVelocityCur to be used at the beginning of the + ! next subcycled time-step + call mpas_timer_start("FB_LTS advance solution") + call mpas_timer_start("FB_LTS advance velocity solution") + + ! Increment interface correction terms + do iRegion = 1,nRegions + do ie = 1, nEdgesInLTSRegion(iRegion,2) + iEdge = edgesInLTSRegion(iRegion,2,ie) + normalVelocityTendSum3rd(:,iEdge) = normalVelocityTendSum3rd(:,iEdge) & + + normalVelocityTend(:,iEdge) + end do + end do + ! Fine cells adjacent to interface 1 + do ie = 1, nEdgesInLTSRegion(1,3) + iEdge = edgesInLTSRegion(1,3,ie) + normalVelocityCur(:,iEdge) = normalVelocityCur(:,iEdge) & + + dtFine * normalVelocityTend(:,iEdge) + end do + ! Fine in the interior, away from interface 1 + do ie = 1, nEdgesInLTSRegion(1,1) + iEdge = edgesInLTSRegion(1,1,ie) + normalVelocityCur(:,iEdge) = normalVelocityCur(:,iEdge) & + + dtFine * normalVelocityTend(:,iEdge) + end do + + call mpas_timer_stop("FB_LTS advance velocity solution") + call mpas_timer_stop("FB_LTS advance solution") + + ! END: Normal velocity stage 3 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + + ! Copy new solution into cur solution for use at the beginning + ! of the next substep + ! We only do this for layerThickness here because we write + ! into normalVelocityCur rather than normalVelocityNew above + ! for stage 3 of the subcycled normal velocity + call mpas_timer_start("FB_LTS copy soln") + + ! Fine adjacent to interface 1 + do ic = 1, nCellsInLTSRegion(1,3) + iCell = cellsInLTSRegion(1,3,ic) + layerThicknessCur(:,iCell) = layerThicknessNew(:,iCell) + end do + ! Fine in the interior, away from interface 1 + do ic = 1, nCellsInLTSRegion(1,1) + iCell = cellsInLTSRegion(1,1,ic) + layerThicknessCur(:,iCell) = layerThicknessNew(:,iCell) + end do + + call mpas_timer_stop("FB_LTS copy soln") + + end do ! end of fine time-step subcycling loop + + + ! Copy data from normalVelocityCur into normalVelocityNew + ! which currently holds the New data. + call mpas_timer_start("FB_LTS copy soln") + + ! Fine adjacent to interface 1 + do ie = 1, nEdgesInLTSRegion(1,3) + iEdge = edgesInLTSRegion(1,3,ie) + normalVelocityNew(:,iEdge) = normalVelocityCur(:,iEdge) + end do + ! Fine in the interior, away from interface 1 + do ie = 1, nEdgesInLTSRegion(1,1) + iEdge = edgesInLTSRegion(1,1,ie) + normalVelocityNew(:,iEdge) = normalVelocityCur(:,iEdge) + end do + + call mpas_timer_stop("FB_LTS copy soln") + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! END FINE ADVANCEMENT + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! BEGIN INTERFACE CORRECTION + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + ! Calculate the corrected solution in the interface layers + call mpas_timer_start("FB_LTS interface correction") + + ! Perform the correction on interface 1 and 2 + do iRegion = 1, nRegions + ! Normal velocity correction + do ie = 1, nEdgesInLTSRegion(iRegion,2) + iEdge = edgesInLTSRegion(iRegion,2,ie) + normalVelocityNew(:,iEdge) = normalVelocityCur(:,iEdge) & + + dtFine * normalVelocityTendSum3rd(:,iEdge) + end do + ! Layer thickness correction + do ic = 1, nCellsInLTSRegion(iRegion,2) + iCell = cellsInLTSRegion(iRegion,2,ic) + layerThicknessNew(:,iCell) = layerThicknessCur(:,iCell) & + + dtFine * layerThicknessTendSum3rd(:,iCell) + end do + end do + + call mpas_timer_stop("FB_LTS interface correction") + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! END INTERFACE CORRECTION + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + + ! Final prognostic halo update, is this needed? + call mpas_timer_start("FB_LTS prognostic halo update") + + call mpas_timer_start("FB_LTS velocity halo update") + call mpas_dmpar_field_halo_exch(domain, 'normalVelocity', timeLevel=2) + call mpas_timer_stop("FB_LTS velocity halo update") + + call mpas_timer_start("FB_LTS thickness halo update") + call mpas_dmpar_field_halo_exch(domain, 'layerThickness', timeLevel=2) + call mpas_timer_stop("FB_LTS thickness halo update") + + call mpas_timer_stop("FB_LTS prognostic halo update") + + ! Implicit vmix, DO WE NEED THIS? + call mpas_timer_start("FB_LTS implicit vmix") + if (.not. config_disable_vel_vmix) then + call ocn_diagnostic_solve(dt, statePool, forcingPool, meshPool, & + verticalMeshPool, scratchPool, tracersPool, 2) + call ocn_vmix_implicit(dt, meshPool, statePool, forcingPool, scratchPool, err, 2) + + ! Halo update + call mpas_timer_start("FB_LTS prognostic halo update") + + call mpas_timer_start("FB_LTS velocity halo update") + call mpas_dmpar_field_halo_exch(domain, 'normalVelocity', timeLevel=2) + call mpas_timer_stop("FB_LTS velocity halo update") + + call mpas_timer_start("FB_LTS thickness halo update") + call mpas_dmpar_field_halo_exch(domain, 'layerThickness', timeLevel=2) + call mpas_timer_stop("FB_LTS thickness halo update") + + call mpas_timer_stop("FB_LTS prognostic halo update") + end if + call mpas_timer_stop("FB_LTS implicit vmix") + + call mpas_timer_stop("FB_LTS main loop") + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! + ! END FB_LTS SCHEME + ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + + call mpas_timer_start("FB_LTS cleanup") + + if (config_prescribe_velocity) then + do iEdge = 1, nEdgesAll + normalVelocityNew(:,iEdge) = normalVelocityCur(:,iEdge) + end do + end if + + if (config_prescribe_thickness) then + do iCell = 1, nCellsAll + layerThicknessNew(:,iCell) = layerThicknessCur(:,iCell) + end do + end if + + do iEdge = 1, nEdgesAll + normalTransportVelocity(:,iEdge) = normalVelocityNew(:,iEdge) + end do + + call mpas_reconstruct(meshPool, normalVelocityNew, & + velocityX, velocityY, velocityZ, & + velocityZonal, velocityMeridional, & + includeHalos = .true.) + + call mpas_reconstruct(meshPool, gradSSH, & + gradSSHX, gradSSHY, gradSSHZ, & + gradSSHZonal, gradSSHMeridional, & + includeHalos = .true.) + + do iCell = 1, nCellsAll + surfaceVelocity(indexSurfaceVelocityZonal, iCell) = velocityZonal(1, iCell) + surfaceVelocity(indexSurfaceVelocityMeridional, iCell) = velocityMeridional(1, iCell) + + SSHGradient(indexSSHGradientZonal, iCell) = gradSSHZonal(iCell) + SSHGradient(indexSSHGradientMeridional, iCell) = gradSSHMeridional(iCell) + end do + + call ocn_time_average_coupled_accumulate(statePool, forcingPool, 2) + + do iCell = 1, nCellsAll + surfaceVelocity(indexSurfaceVelocityZonal, iCell) = velocityZonal(1, iCell) + surfaceVelocity(indexSurfaceVelocityMeridional, iCell) = velocityMeridional(1, iCell) + + SSHGradient(indexSSHGradientZonal, iCell) = gradSSHZonal(iCell) + SSHGradient(indexSSHGradientMeridional, iCell) = gradSSHMeridional(iCell) + end do + + ! Update diagnostics + call ocn_diagnostic_solve(dt, statePool, forcingPool, meshPool, & + verticalMeshPool, scratchPool, tracersPool, 2) + + call mpas_timer_stop("FB_LTS cleanup") + + end subroutine ocn_time_integrator_fblts!}}} + + +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! ocn_time_integration_fblts_init +! +!> \brief Initialization for MPAS ocean FB_LTS time integration scheme +!> \author Giacomo Capodaglio +!> \date November 2022 +!> \details +!> This routine computes the LTS arrays +! +!----------------------------------------------------------------------- + + subroutine ocn_time_integration_fblts_init(domain)!{{{ + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! Sets the LTS arrays + ! + ! Output: LTS arrays are written + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + implicit none + + !----------------------------------------------------------------- + ! Input/output variables + !----------------------------------------------------------------- + + type (domain_type), intent(inout) :: & + domain !< Input/output: model state + + !----------------------------------------------------------------- + ! Local variables + !----------------------------------------------------------------- + + type (block_type), pointer :: & + block + + type (mpas_pool_type), pointer :: & + LTSPool + + type (mpas_pool_type), pointer :: & + tendSlowPool, & + tendSum3rdPool, & + prevTendSlowPool, nextTendSlowPool, & + prevTendSum3rdPool, nextTendSum3rdPool + + type (mpas_pool_type), pointer :: & + tendPool + + integer, dimension(:), allocatable :: & + isLTSRegionEdgeAssigned + + integer, dimension(:), pointer :: & + LTSRegion + + integer, dimension(:,:), pointer :: & + nCellsInLTSRegion, & + nEdgesInLTSRegion + + integer, dimension(:,:,:), pointer :: & + cellsInLTSRegion, & + edgesInLTSRegion + + integer, dimension(2) :: & + minMaxLTSRegion + + integer :: & + i, iCell, iEdge, iRegion, coarseRegions, & + fineRegions, fineRegionsM1 + + !----------------------------------------------------------------- + ! Begin routine + !----------------------------------------------------------------- + + minMaxLTSRegion(1) = 1 + minMaxLTSRegion(2) = 2 + + block => domain % blocklist + call mpas_pool_get_subpool(block%structs, 'tend', tendPool) + + call mpas_pool_create_pool(tendSum3rdPool) + call mpas_pool_clone_pool(tendPool, tendSum3rdPool, 1) + call mpas_pool_create_pool(tendSlowPool) + call mpas_pool_clone_pool(tendPool, tendSlowPool, 1) + + call mpas_pool_add_subpool(block % structs, 'tend_sum_3rd', tendSum3rdPool) + call mpas_pool_add_subpool(block % structs, 'tend_slow', tendSlowPool) + + if (associated(block % prev)) then + call mpas_pool_get_subpool(block % prev % structs, 'tend_sum_3rd', tendSum3rdPool) + call mpas_pool_get_subpool(block % prev % structs, 'tend_slow', tendSlowPool) + else + nullify(prevTendSum3rdPool) + nullify(prevTendSlowPool) + end if + + if (associated(block % next)) then + call mpas_pool_get_subpool(block % next % structs, 'tend_sum_3rd', nextTendSum3rdPool) + call mpas_pool_get_subpool(block % next % structs, 'tend_slow', nextTendSlowPool) + else + nullify(nextTendSum3rdPool) + nullify(nextTendSlowPool) + end if + + call mpas_pool_get_subpool(block % structs, 'tend_sum_3rd', tendSum3rdPool) + call mpas_pool_get_subpool(block % structs, 'tend_slow', tendSlowPool) + + if (associated(prevTendSum3rdPool) .and. associated(nextTendSum3rdPool)) then + call mpas_pool_link_pools(tendSum3rdPool, prevTendSum3rdPool, nextTendSum3rdPool) + else if (associated(prevTendSum3rdPool)) then + call mpas_pool_link_pools(tendSum3rdPool, prevTendSum3rdPool) + else if (associated(nextTendSum3rdPool)) then + call mpas_pool_link_pools(tendSum3rdPool,nextPool=nextTendSum3rdPool) + else + call mpas_pool_link_pools(tendSum3rdPool) + end if + + if (associated(prevTendSlowPool) .and. associated(nextTendSlowPool)) then + call mpas_pool_link_pools(tendSlowPool, prevTendSlowPool, nextTendSlowPool) + else if (associated(prevTendSlowPool)) then + call mpas_pool_link_pools(tendSlowPool, prevTendSlowPool) + else if (associated(nextTendSlowPool)) then + call mpas_pool_link_pools(tendSlowPool,nextPool=nextTendSlowPool) + else + call mpas_pool_link_pools(tendSlowPool) + end if + + call mpas_pool_link_parinfo(block, tendSum3rdPool) + call mpas_pool_link_parinfo(block, tendSlowPool) + + call mpas_pool_get_subpool(block % structs, 'LTS', LTSPool) + + call mpas_pool_get_array(LTSPool, 'LTSRegion', LTSRegion) + call mpas_pool_get_array(LTSPool, 'cellsInLTSRegion', cellsInLTSRegion) + call mpas_pool_get_array(LTSPool, 'nCellsInLTSRegion', nCellsInLTSRegion) + call mpas_pool_get_array(LTSPool, 'edgesInLTSRegion', edgesInLTSRegion) + call mpas_pool_get_array(LTSPool, 'nEdgesInLTSRegion', nEdgesInLTSRegion) + + ! LTS Regions code: + ! 1 = fine + ! 2 = coarse + ! 3 = interface layer 1 + ! 4 = interface layer 2 + ! 5 = fine (to advance when doing 1st, 2nd, 3rd stage on interface) + + nCellsInLTSRegion(:,:) = 0 + nEdgesInLTSRegion(:,:) = 0 + + ! This is a loop to build the lists of elements in the fine, coarse, + ! and interface regions. Only loops up to nCellsOwned because in the + ! time-stepping we only want to advance the cells owned by the MPI process + do iCell = 1, nCellsOwned + do iRegion = 1,2 + if (iRegion == minMaxLTSRegion(iRegion)) then + if(LTSRegion(iCell) == minMaxLTSRegion(iRegion)) then + nCellsInLTSRegion(iRegion,1) = nCellsInLTSRegion(iRegion,1) + 1 + cellsInLTSRegion(iRegion,1,nCellsInLTSRegion(iRegion,1)) = iCell + end if + if(LTSRegion(iCell) == (minMaxLTSRegion(iRegion) + 2) ) then + nCellsInLTSRegion(iRegion,2) = nCellsInLTSRegion(iRegion,2) + 1 + cellsInLTSRegion(iRegion,2,nCellsInLTSRegion(iRegion,2)) = iCell + end if + end if + end do + if (LTSRegion(iCell) == 5) then + nCellsInLTSRegion(1,3) = nCellsInLTSRegion(1,3) + 1 + cellsInLTSRegion(1,3,nCellsInLTSRegion(1,3)) = iCell + end if + end do + + ! Below we fill out the lists for the edges, according to the LTSRegion + ! that have been assigned to the cells. We move from the fine to the + ! coarse (i.e. from the fine to the nearest LTS region in the direction + ! of the coarse). Note that edges shared between cells of different LTS + ! regions are owned by the cell in the LTS region closest to the fine + ! region, see Figure 3 in "Conservative explicit local time-stepping + ! schemes for the shallow water equations" by Hoang et al. (halo edges + ! however are owned by whatever processor they are initially assigned to). + + allocate(isLTSRegionEdgeAssigned(nEdgesOwned)) + isLTSRegionEdgeAssigned(:) = 0 + + do iCell = 1, nCellsInLTSRegion(1,1) + do i = 1, nEdgesOnCell(cellsInLTSRegion(1,1,iCell)) + iEdge = edgesOnCell(i,cellsInLTSRegion(1,1,iCell)) + if (iEdge .le. nEdgesOwned) then + if (isLTSRegionEdgeAssigned(iEdge) == 0) then + nEdgesInLTSRegion(1,1) = nEdgesInLTSRegion(1,1) + 1 + edgesInLTSRegion(1,1, nEdgesInLTSRegion(1,1)) = iEdge + isLTSRegionEdgeAssigned(iEdge) = 1 + end if + end if + end do + end do + + fineRegions = 3 + fineRegionsM1 = 2 + do iRegion = 1, fineRegionsM1 + do iCell = 1, nCellsInLTSRegion(1, fineRegions - iRegion + 1) + do i = 1, nEdgesOnCell(cellsInLTSRegion(1, fineRegions - iRegion + 1, iCell)) + iEdge = edgesOnCell(i,cellsInLTSRegion(1, fineRegions - iRegion + 1, iCell)) + if (iEdge .le. nEdgesOwned) then + if (isLTSRegionEdgeAssigned(iEdge) == 0) then + nEdgesInLTSRegion(1, fineRegions - iRegion + 1) & + = nEdgesInLTSRegion(1, fineRegions - iRegion + 1) + 1 + edgesInLTSRegion(1, fineRegions - iRegion + 1, & + nEdgesInLTSRegion(1, fineRegions - iRegion + 1)) & + = iEdge + isLTSRegionEdgeAssigned(iEdge) = 1 + end if + end if + end do + end do + end do + + coarseRegions = 2 + do iRegion = 1, coarseRegions + do iCell = 1, nCellsInLTSRegion(2, coarseRegions - iRegion + 1) + do i = 1, nEdgesOnCell(cellsInLTSRegion(2,coarseRegions - iRegion + 1,iCell)) + iEdge = edgesOnCell(i,cellsInLTSRegion(2,coarseRegions - iRegion + 1,iCell)) + if (iEdge .le. nEdgesOwned) then + if (isLTSRegionEdgeAssigned(iEdge) == 0) then + nEdgesInLTSRegion(2, coarseRegions - iRegion + 1) & + = nEdgesInLTSRegion(2, coarseRegions - iRegion + 1) + 1 + edgesInLTSRegion(2, coarseRegions - iRegion + 1, & + nEdgesInLTSRegion(2, coarseRegions - iRegion + 1)) & + = iEdge + isLTSRegionEdgeAssigned(iEdge) = 1 + end if + end if + end do + end do + end do + + deallocate(isLTSRegionEdgeAssigned) + + end subroutine ocn_time_integration_fblts_init!}}} + + +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! ocn_lts_thick_tend +! +!> \brief Calculate thickness tendencies for FB_LTS +!> \author Jeremy Lilly +!> \date October 2023 +!> \details +!> This routine calculates the thickness tendency on different +!> LTS regions +! +!----------------------------------------------------------------------- + + subroutine ocn_lts_thick_tend(LTSPool, tendPool, & + normalVelocity, layerThickness, & + computeOnFineInterior, computeOnFineInterfaceAdjacent, & + computeOnInterfaceOne, computeOnInterfaceTwo, & + computeOnCoarse)!{{{ + + !----------------------------------------------------------------- + ! Input variables + !----------------------------------------------------------------- + + integer, intent(in) :: & + computeOnFineInterior, & + computeOnFineInterfaceAdjacent, & + computeOnInterfaceOne, & + computeOnInterfaceTwo, & + computeOnCoarse + + real (kind=RKIND), dimension(:,:), pointer, intent(in) :: & + layerThickness, & + normalVelocity + + type (mpas_pool_type), intent(in) :: & + LTSPool !< Input: LTS data + + !----------------------------------------------------------------- + ! Input/output variables + !----------------------------------------------------------------- + + type (mpas_pool_type), intent(inout) :: & + tendPool !< Input/output: tendency variables + + !----------------------------------------------------------------- + ! Local variables + !----------------------------------------------------------------- + + integer, dimension(:,:), pointer :: & + nCellsInLTSRegion + + integer, dimension(:,:,:), pointer :: & + cellsInLTSRegion + + real (kind=RKIND), dimension(:,:), pointer :: & + layerThicknessTend + + integer :: & + iEdge, cell1, cell2, k, ie, & + ic, i, iCell, kmin, kmax + real (kind=RKIND) :: & + invdcEdge, flux + + !----------------------------------------------------------------- + ! Begin routine + !----------------------------------------------------------------- + + call mpas_pool_get_array(LTSPool, 'cellsInLTSRegion', cellsInLTSRegion) + call mpas_pool_get_array(LTSPool, 'nCellsInLTSRegion', nCellsInLTSRegion) + + call mpas_pool_get_array(tendPool, 'layerThickness', layerThicknessTend) + + ! calculate layerThickEdgeFlux + if (config_thickness_flux_type == 'centered') then + + do iEdge = 1, nEdgesAll + kmin = minLevelEdgeBot(iEdge) + kmax = maxLevelEdgeTop(iEdge) + cell1 = cellsOnEdge(1,iEdge) + cell2 = cellsOnEdge(2,iEdge) + do k = 1,nVertLevels + ! initialize layerThicknessEdgeMean to avoid divide by + ! zero and NaN problems. + layerThickEdgeFlux(k,iEdge) = -1.0e34_RKIND + end do + do k = kmin,kmax + ! central differenced + layerThickEdgeFlux(k,iEdge) = 0.5_RKIND * & + ( layerThickness(k,cell1) + & + layerThickness(k,cell2) ) + end do + end do + + else if (config_thickness_flux_type == 'upwind') then + + do iEdge = 1, nEdgesAll + kmin = minLevelEdgeBot(iEdge) + kmax = maxLevelEdgeTop(iEdge) + cell1 = cellsOnEdge(1,iEdge) + cell2 = cellsOnEdge(2,iEdge) + do k=1,nVertLevels + ! initialize layerThicknessEdgeFlux to avoid divide by + ! zero and NaN problems. + layerThickEdgeFlux(k,iEdge) = -1.0e34_RKIND + end do + do k = kmin,kmax + if (normalVelocity(k,iEdge) > 0.0_RKIND) then + layerThickEdgeFlux(k,iEdge) = layerThickness(k,cell1) + elseif (normalVelocity(k,iEdge) < 0.0_RKIND) then + layerThickEdgeFlux(k,iEdge) = layerThickness(k,cell2) + else + layerThickEdgeFlux(k,iEdge) = max(layerThickness(k,cell1), & + layerThickness(k,cell2)) + end if + end do + end do + + else + + call mpas_log_write('ERROR: config_thickness_flux_type selected not implemented for FB_LTS', & + messageType=MPAS_LOG_CRIT) + call abort + + end if + + layerThicknessTend(:, :) = 0.0_RKIND + + ! Fine in the interior, away from interface 1 + if (computeOnFineInterior == 1) then + do ic = 1, nCellsInLTSRegion(1,1) + iCell = cellsInLTSRegion(1,1,ic) + do i = 1, nEdgesOnCell(iCell) + iEdge = edgesOnCell(i, iCell) + do k = minLevelEdgeBot(iEdge), maxLevelEdgeTop(iEdge) + flux = normalVelocity(k, iEdge) * dvEdge(iEdge) * layerThickEdgeFlux(k, iEdge) + layerThicknessTend(k, iCell) = layerThicknessTend(k,iCell) & + + edgeSignOnCell(i, iCell) * flux * invAreaCell(iCell) + end do + end do + end do + end if + + ! Fine adjacent to interface 1 + if (computeOnFineInterfaceAdjacent == 1) then + do ic = 1, nCellsInLTSRegion(1,3) + iCell = cellsInLTSRegion(1,3,ic) + do i = 1, nEdgesOnCell(iCell) + iEdge = edgesOnCell(i, iCell) + do k = minLevelEdgeBot(iEdge), maxLevelEdgeTop(iEdge) + flux = normalVelocity(k, iEdge) * dvEdge(iEdge) * layerThickEdgeFlux(k, iEdge) + layerThicknessTend(k, iCell) = layerThicknessTend(k,iCell) & + + edgeSignOnCell(i, iCell) * flux * invAreaCell(iCell) + end do + end do + end do + end if + + ! Interface 1 + if (computeOnInterfaceOne == 1) then + do ic = 1, nCellsInLTSRegion(1,2) + iCell = cellsInLTSRegion(1,2,ic) + do i = 1, nEdgesOnCell(iCell) + iEdge = edgesOnCell(i, iCell) + do k = minLevelEdgeBot(iEdge), maxLevelEdgeTop(iEdge) + flux = normalVelocity(k, iEdge) * dvEdge(iEdge) * layerThickEdgeFlux(k, iEdge) + layerThicknessTend(k, iCell) = layerThicknessTend(k, iCell) & + + edgeSignOnCell(i, iCell) * flux * invAreaCell(iCell) + end do + end do + end do + end if + + ! Interface 2 + if (computeOnInterfaceTwo == 1) then + do ic = 1, nCellsInLTSRegion(2,2) + iCell = cellsInLTSRegion(2,2,ic) + do i = 1, nEdgesOnCell(iCell) + iEdge = edgesOnCell(i, iCell) + do k = minLevelEdgeBot(iEdge), maxLevelEdgeTop(iEdge) + flux = normalVelocity(k, iEdge) * dvEdge(iEdge) * layerThickEdgeFlux(k, iEdge) + layerThicknessTend(k, iCell) = layerThicknessTend(k, iCell) & + + edgeSignOnCell(i, iCell) * flux * invAreaCell(iCell) + end do + end do + end do + end if + + ! Coarse + if (computeOnCoarse == 1) then + do ic = 1, nCellsInLTSRegion(2,1) + iCell = cellsInLTSRegion(2,1,ic) + do i = 1, nEdgesOnCell(iCell) + iEdge = edgesOnCell(i, iCell) + do k = minLevelEdgeBot(iEdge), maxLevelEdgeTop(iEdge) + flux = normalVelocity(k, iEdge) * dvEdge(iEdge) * layerThickEdgeFlux(k, iEdge) + layerThicknessTend(k, iCell) = layerThicknessTend(k,iCell) & + + edgeSignOnCell(i, iCell) * flux * invAreaCell(iCell) + end do + end do + end do + end if + + end subroutine ocn_lts_thick_tend!}}} + + +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! ocn_lts_vel_tend +! +!> \brief Calculate velocity tendencies for FB_LTS +!> \author Jeremy Lilly +!> \date October 2023 +!> \details +!> This routine calculates the velocity tendency on different +!> LTS regions +! +!----------------------------------------------------------------------- + + subroutine ocn_lts_vel_tend(LTSPool, tendPool, & + normalVelocity, layerThickness, & + computeOnFineInterior, computeOnFineInterfaceAdjacent, & + computeOnInterfaceOne, computeOnInterfaceTwo, & + computeOnCoarse)!{{{ + + !----------------------------------------------------------------- + ! Input variables + !----------------------------------------------------------------- + + integer, intent(in) :: & + computeOnFineInterior, & + computeOnFineInterfaceAdjacent, & + computeOnInterfaceOne, & + computeOnInterfaceTwo, & + computeOnCoarse + + real (kind=RKIND), dimension(:,:), pointer, intent(in) :: & + layerThickness, & + normalVelocity + + type (mpas_pool_type), intent(in) :: & + LTSPool !< Input: LTS data + + !----------------------------------------------------------------- + ! Input/output variables + !----------------------------------------------------------------- + + type (mpas_pool_type), intent(inout) :: & + tendPool !< Input/output: tendency variables + + !----------------------------------------------------------------- + ! Local variables + !----------------------------------------------------------------- + + integer, dimension(:,:), pointer :: & + nEdgesInLTSRegion + + integer, dimension(:,:,:), pointer :: & + edgesInLTSRegion + + real (kind=RKIND), dimension(nCellsAll) :: & + ssh + + real (kind=RKIND), dimension(:,:), pointer :: & + normalVelocityTend + + integer :: & + iEdge, cell1, cell2, k, ie, & + ic, i, iCell, kmin, kmax + + real (kind=RKIND) :: & + invdcEdge, betaSelfAttrLoad, flux, ssh_sal_on, tidal_pot_for_on + + !----------------------------------------------------------------- + ! Begin routine + !----------------------------------------------------------------- + + betaSelfAttrLoad = config_self_attraction_and_loading_beta + + call mpas_pool_get_array(LTSPool, 'edgesInLTSRegion', edgesInLTSRegion) + call mpas_pool_get_array(LTSPool, 'nEdgesInLTSRegion', nEdgesInLTSRegion) + + call mpas_pool_get_array(tendPool, 'normalVelocity', normalVelocityTend) + + if (config_use_self_attraction_loading) then + ssh_sal_on = 1.0_RKIND + else + ssh_sal_on = 0.0_RKIND + endif + if (config_use_tidal_potential_forcing) then + tidal_pot_for_on = 1.0_RKIND + else + tidal_pot_for_on = 0.0_RKIND + end if + + ! ssh + do iCell = 1, nCellsAll + k = maxLevelCell(iCell) + zTop(k:nVertLevels,iCell) = -bottomDepth(iCell) + layerThickness(k,iCell) + do k = maxLevelCell(iCell)-1, minLevelCell(iCell), -1 + zTop(k,iCell) = zTop(k+1,iCell) + layerThickness(k,iCell) + end do + ! copy zTop(1,iCell) into sea-surface height array + ssh(iCell) = zTop(minLevelCell(iCell),iCell) + end do + + normalVelocityTend(:,:) = 0.0_RKIND + + ! Fine in the interior, away from interface 1 + if (computeOnFineInterior == 1) then + do ie = 1, nEdgesInLTSRegion(1,1) + iEdge = edgesInLTSRegion(1,1,ie) + cell1 = cellsOnEdge(1,iEdge) + cell2 = cellsOnEdge(2,iEdge) + invdcEdge = 1.0_RKIND / dcEdge(iEdge) + kMin = minLevelEdgeBot(iEdge) + kMax = maxLevelEdgeTop(iEdge) + do k=kMin,kMax + normalVelocityTend(k,iEdge) = normalVelocityTend(k,iEdge) & + - edgeMask(k,iEdge) * invdcEdge & + * ( gravity * ( (ssh(cell2) - ssh(cell1)) & + - tidal_pot_for_on * (1.0_RKIND - ssh_sal_on) & + * betaSelfAttrLoad * (ssh(cell2) - ssh(cell1)) ) ) + end do + end do + end if + + ! Fine adjacent to interface 1 + if (computeOnFineInterfaceAdjacent == 1) then + do ie = 1, nEdgesInLTSRegion(1,3) + iEdge = edgesInLTSRegion(1,3,ie) + cell1 = cellsOnEdge(1,iEdge) + cell2 = cellsOnEdge(2,iEdge) + invdcEdge = 1.0_RKIND / dcEdge(iEdge) + kMin = minLevelEdgeBot(iEdge) + kMax = maxLevelEdgeTop(iEdge) + do k=kMin,kMax + normalVelocityTend(k,iEdge) = normalVelocityTend(k,iEdge) & + - edgeMask(k,iEdge) * invdcEdge & + * ( gravity * ( (ssh(cell2) - ssh(cell1)) & + - tidal_pot_for_on * (1.0_RKIND - ssh_sal_on) & + * betaSelfAttrLoad * (ssh(cell2) - ssh(cell1)) ) ) + end do + end do + end if + + ! Interface 1 + if (computeOnInterfaceOne == 1) then + do ie = 1, nEdgesInLTSRegion(1,2) + iEdge = edgesInLTSRegion(1,2,ie) + cell1 = cellsOnEdge(1,iEdge) + cell2 = cellsOnEdge(2,iEdge) + invdcEdge = 1.0_RKIND / dcEdge(iEdge) + kMin = minLevelEdgeBot(iEdge) + kMax = maxLevelEdgeTop(iEdge) + do k=kMin,kMax + normalVelocityTend(k,iEdge) = normalVelocityTend(k,iEdge) & + - edgeMask(k,iEdge) * invdcEdge & + * ( gravity * ( (ssh(cell2) - ssh(cell1)) & + - tidal_pot_for_on * (1.0_RKIND - ssh_sal_on) & + * betaSelfAttrLoad * (ssh(cell2) - ssh(cell1)) ) ) + end do + end do + end if + + ! Interface 2 + if (computeOnInterfaceTwo == 1) then + do ie = 1, nEdgesInLTSRegion(2,2) + iEdge = edgesInLTSRegion(2,2,ie) + cell1 = cellsOnEdge(1,iEdge) + cell2 = cellsOnEdge(2,iEdge) + invdcEdge = 1.0_RKIND / dcEdge(iEdge) + kMin = minLevelEdgeBot(iEdge) + kMax = maxLevelEdgeTop(iEdge) + do k=kMin,kMax + normalVelocityTend(k,iEdge) = normalVelocityTend(k,iEdge) & + - edgeMask(k,iEdge) * invdcEdge & + * ( gravity * ( (ssh(cell2) - ssh(cell1)) & + - tidal_pot_for_on * (1.0_RKIND - ssh_sal_on) & + * betaSelfAttrLoad * (ssh(cell2) - ssh(cell1)) ) ) + end do + end do + end if + + ! Coarse + if (computeOnCoarse == 1) then + do ie = 1, nEdgesInLTSRegion(2,1) + iEdge = edgesInLTSRegion(2,1,ie) + cell1 = cellsOnEdge(1,iEdge) + cell2 = cellsOnEdge(2,iEdge) + invdcEdge = 1.0_RKIND / dcEdge(iEdge) + kMin = minLevelEdgeBot(iEdge) + kMax = maxLevelEdgeTop(iEdge) + do k=kMin,kMax + normalVelocityTend(k,iEdge) = normalVelocityTend(k,iEdge) & + - edgeMask(k,iEdge) * invdcEdge & + * ( gravity * ( (ssh(cell2) - ssh(cell1)) & + - tidal_pot_for_on * (1.0_RKIND - ssh_sal_on) & + * betaSelfAttrLoad * (ssh(cell2) - ssh(cell1)) ) ) + end do + end do + end if + + end subroutine ocn_lts_vel_tend!}}} + + +end module ocn_time_integration_fblts + diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_lts.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_lts.F index d123c69aecb5..24f9f208ab3b 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_lts.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_lts.F @@ -254,20 +254,18 @@ subroutine ocn_time_integrator_lts(domain,dt)!{{{ call mpas_pool_get_array(LTSPool, 'nEdgesInLTSRegion', & nEdgesInLTSRegion) - !--- Create additional pools for LTS - call mpas_pool_create_pool(tendSum1stPool) - call mpas_pool_clone_pool(tendPool, tendSum1stPool, 1) - call mpas_pool_create_pool(tendSum2ndPool) - call mpas_pool_clone_pool(tendPool, tendSum2ndPool, 1) - call mpas_pool_create_pool(tendSum3rdPool) - call mpas_pool_clone_pool(tendPool, tendSum3rdPool, 1) - call mpas_pool_create_pool(tendSlowPool) - call mpas_pool_clone_pool(tendPool, tendSlowPool, 1) + !--- Update additional pools for LTS + call mpas_pool_get_subpool(block % structs, 'tend_sum_1st', tendSum1stPool) + call mpas_pool_get_subpool(block % structs, 'tend_sum_2nd', tendSum2ndPool) + call mpas_pool_get_subpool(block % structs, 'tend_sum_3rd', tendSum3rdPool) + call mpas_pool_get_subpool(block % structs, 'tend_slow', tendSlowPool) + + + call mpas_pool_copy_pool(tendPool, tendSum1stPool, 1) + call mpas_pool_copy_pool(tendPool, tendSum2ndPool, 1) + call mpas_pool_copy_pool(tendPool, tendSum3rdPool, 1) + call mpas_pool_copy_pool(tendPool, tendSlowPool, 1) - call mpas_pool_add_subpool(block % structs, 'tend_sum_1st', tendSum1stPool) - call mpas_pool_add_subpool(block % structs, 'tend_sum_2nd', tendSum2ndPool) - call mpas_pool_add_subpool(block % structs, 'tend_sum_3rd', tendSum3rdPool) - call mpas_pool_add_subpool(block % structs, 'tend_slow', tendSlowPool) call mpas_pool_get_array(tendSlowPool, 'normalVelocity', normalVelocityTendSlow) @@ -307,79 +305,6 @@ subroutine ocn_time_integrator_lts(domain,dt)!{{{ normalVelocityTendSum3rd(:,:) = 0.0_RKIND layerThicknessTendSum3rd(:,:) = 0.0_RKIND - if (associated(block % prev)) then - call mpas_pool_get_subpool(block % prev % structs, 'tend_sum_1st', tendSum1stPool) - call mpas_pool_get_subpool(block % prev % structs, 'tend_sum_2nd', tendSum2ndPool) - call mpas_pool_get_subpool(block % prev % structs, 'tend_sum_3rd', tendSum3rdPool) - call mpas_pool_get_subpool(block % prev % structs, 'tend_slow', tendSlowPool) - else - nullify(prevTendSum1stPool) - nullify(prevTendSum2ndPool) - nullify(prevTendSum3rdPool) - nullify(prevTendSlowPool) - end if - - if (associated(block % next)) then - call mpas_pool_get_subpool(block % next % structs, 'tend_sum_1st', nextTendSum1stPool) - call mpas_pool_get_subpool(block % next % structs, 'tend_sum_2nd', nextTendSum2ndPool) - call mpas_pool_get_subpool(block % next % structs, 'tend_sum_3rd', nextTendSum3rdPool) - call mpas_pool_get_subpool(block % next % structs, 'tend_slow', nextTendSlowPool) - else - nullify(nextTendSum1stPool) - nullify(nextTendSum2ndPool) - nullify(nextTendSum3rdPool) - nullify(nextTendSlowPool) - end if - - call mpas_pool_get_subpool(block % structs, 'tend_sum_1st', tendSum1stPool) - call mpas_pool_get_subpool(block % structs, 'tend_sum_2nd', tendSum2ndPool) - call mpas_pool_get_subpool(block % structs, 'tend_sum_3rd', tendSum3rdPool) - call mpas_pool_get_subpool(block % structs, 'tend_slow', tendSlowPool) - - if (associated(prevTendSum1stPool) .and. associated(nextTendSum1stPool)) then - call mpas_pool_link_pools(tendSum1stPool, prevTendSum1stPool, nextTendSum1stPool) - else if (associated(prevTendSum1stPool)) then - call mpas_pool_link_pools(tendSum1stPool, prevTendSum1stPool) - else if (associated(nextTendSum1stPool)) then - call mpas_pool_link_pools(tendSum1stPool,nextPool=nextTendSum1stPool) - else - call mpas_pool_link_pools(tendSum1stPool) - end if - - if (associated(prevTendSum2ndPool) .and. associated(nextTendSum2ndPool)) then - call mpas_pool_link_pools(tendSum2ndPool, prevTendSum2ndPool, nextTendSum2ndPool) - else if (associated(prevTendSum2ndPool)) then - call mpas_pool_link_pools(tendSum2ndPool, prevTendSum2ndPool) - else if (associated(nextTendSum2ndPool)) then - call mpas_pool_link_pools(tendSum2ndPool,nextPool=nextTendSum2ndPool) - else - call mpas_pool_link_pools(tendSum2ndPool) - end if - - if (associated(prevTendSum3rdPool) .and. associated(nextTendSum3rdPool)) then - call mpas_pool_link_pools(tendSum3rdPool, prevTendSum3rdPool, nextTendSum3rdPool) - else if (associated(prevTendSum3rdPool)) then - call mpas_pool_link_pools(tendSum3rdPool, prevTendSum3rdPool) - else if (associated(nextTendSum3rdPool)) then - call mpas_pool_link_pools(tendSum3rdPool,nextPool=nextTendSum3rdPool) - else - call mpas_pool_link_pools(tendSum3rdPool) - end if - - if (associated(prevTendSlowPool) .and. associated(nextTendSlowPool)) then - call mpas_pool_link_pools(tendSlowPool, prevTendSlowPool, nextTendSlowPool) - else if (associated(prevTendSlowPool)) then - call mpas_pool_link_pools(tendSlowPool, prevTendSlowPool) - else if (associated(nextTendSlowPool)) then - call mpas_pool_link_pools(tendSlowPool,nextPool=nextTendSlowPool) - else - call mpas_pool_link_pools(tendSlowPool) - end if - - call mpas_pool_link_parinfo(block, tendSum1stPool) - call mpas_pool_link_parinfo(block, tendSum2ndPool) - call mpas_pool_link_parinfo(block, tendSum3rdPool) - call mpas_pool_link_parinfo(block, tendSlowPool) call mpas_timer_stop("lts time-step prep") @@ -1084,16 +1009,6 @@ subroutine ocn_time_integrator_lts(domain,dt)!{{{ ! DIAGNOSTICS UPDATE --- call ocn_diagnostic_solve(dt, statePool, forcingPool, meshPool, verticalMeshPool, scratchPool, tracersPool, 2) - call mpas_pool_destroy_pool(tendSum1stPool) - call mpas_pool_destroy_pool(tendSum2ndPool) - call mpas_pool_destroy_pool(tendSum3rdPool) - call mpas_pool_destroy_pool(tendSlowPool) - - call mpas_pool_remove_subpool(block % structs, 'tend_sum_1st') - call mpas_pool_remove_subpool(block % structs, 'tend_sum_2nd') - call mpas_pool_remove_subpool(block % structs, 'tend_sum_3rd') - call mpas_pool_remove_subpool(block % structs, 'tend_slow') - call mpas_timer_stop("lts cleanup phase") @@ -1123,6 +1038,16 @@ subroutine ocn_time_integration_lts_init(domain)!{{{ type (block_type), pointer :: block type (mpas_pool_type), pointer :: LTSPool + type (mpas_pool_type), pointer :: tendPool + type (mpas_pool_type), pointer :: & + tendSlowPool, & + tendSum1stPool, & + tendSum2ndPool, & + tendSum3rdPool, & + prevTendSlowPool, nextTendSlowPool, & + prevTendSum1stPool, nextTendSum1stPool, & + prevTendSum2ndPool, nextTendSum2ndPool, & + prevTendSum3rdPool, nextTendSum3rdPool integer, dimension(:), allocatable :: isLTSRegionEdgeAssigned integer :: i, iCell, iEdge, iRegion, coarseRegions, fineRegions, fineRegionsM1 integer, dimension(:), pointer :: LTSRegion @@ -1134,6 +1059,93 @@ subroutine ocn_time_integration_lts_init(domain)!{{{ minMaxLTSRegion(2) = 2 block => domain % blocklist + + ! Create additional pools + call mpas_pool_get_subpool(block%structs, 'tend', tendPool) + + call mpas_pool_create_pool(tendSum1stPool) + call mpas_pool_clone_pool(tendPool, tendSum1stPool, 1) + call mpas_pool_create_pool(tendSum2ndPool) + call mpas_pool_clone_pool(tendPool, tendSum2ndPool, 1) + call mpas_pool_create_pool(tendSum3rdPool) + call mpas_pool_clone_pool(tendPool, tendSum3rdPool, 1) + call mpas_pool_create_pool(tendSlowPool) + call mpas_pool_clone_pool(tendPool, tendSlowPool, 1) + + call mpas_pool_add_subpool(block % structs, 'tend_sum_1st', tendSum1stPool) + call mpas_pool_add_subpool(block % structs, 'tend_sum_2nd', tendSum2ndPool) + call mpas_pool_add_subpool(block % structs, 'tend_sum_3rd', tendSum3rdPool) + call mpas_pool_add_subpool(block % structs, 'tend_slow', tendSlowPool) + + if (associated(block % prev)) then + call mpas_pool_get_subpool(block % prev % structs, 'tend_sum_1st', tendSum1stPool) + call mpas_pool_get_subpool(block % prev % structs, 'tend_sum_2nd', tendSum2ndPool) + call mpas_pool_get_subpool(block % prev % structs, 'tend_sum_3rd', tendSum3rdPool) + call mpas_pool_get_subpool(block % prev % structs, 'tend_slow', tendSlowPool) + else + nullify(prevTendSum1stPool) + nullify(prevTendSum2ndPool) + nullify(prevTendSum3rdPool) + nullify(prevTendSlowPool) + end if + + if (associated(block % next)) then + call mpas_pool_get_subpool(block % next % structs, 'tend_sum_1st', nextTendSum1stPool) + call mpas_pool_get_subpool(block % next % structs, 'tend_sum_2nd', nextTendSum2ndPool) + call mpas_pool_get_subpool(block % next % structs, 'tend_sum_3rd', nextTendSum3rdPool) + call mpas_pool_get_subpool(block % next % structs, 'tend_slow', nextTendSlowPool) + else + nullify(nextTendSum1stPool) + nullify(nextTendSum2ndPool) + nullify(nextTendSum3rdPool) + nullify(nextTendSlowPool) + end if + + if (associated(prevTendSum1stPool) .and. associated(nextTendSum1stPool)) then + call mpas_pool_link_pools(tendSum1stPool, prevTendSum1stPool, nextTendSum1stPool) + else if (associated(prevTendSum1stPool)) then + call mpas_pool_link_pools(tendSum1stPool, prevTendSum1stPool) + else if (associated(nextTendSum1stPool)) then + call mpas_pool_link_pools(tendSum1stPool,nextPool=nextTendSum1stPool) + else + call mpas_pool_link_pools(tendSum1stPool) + end if + + if (associated(prevTendSum2ndPool) .and. associated(nextTendSum2ndPool)) then + call mpas_pool_link_pools(tendSum2ndPool, prevTendSum2ndPool, nextTendSum2ndPool) + else if (associated(prevTendSum2ndPool)) then + call mpas_pool_link_pools(tendSum2ndPool, prevTendSum2ndPool) + else if (associated(nextTendSum2ndPool)) then + call mpas_pool_link_pools(tendSum2ndPool,nextPool=nextTendSum2ndPool) + else + call mpas_pool_link_pools(tendSum2ndPool) + end if + + if (associated(prevTendSum3rdPool) .and. associated(nextTendSum3rdPool)) then + call mpas_pool_link_pools(tendSum3rdPool, prevTendSum3rdPool, nextTendSum3rdPool) + else if (associated(prevTendSum3rdPool)) then + call mpas_pool_link_pools(tendSum3rdPool, prevTendSum3rdPool) + else if (associated(nextTendSum3rdPool)) then + call mpas_pool_link_pools(tendSum3rdPool,nextPool=nextTendSum3rdPool) + else + call mpas_pool_link_pools(tendSum3rdPool) + end if + + if (associated(prevTendSlowPool) .and. associated(nextTendSlowPool)) then + call mpas_pool_link_pools(tendSlowPool, prevTendSlowPool, nextTendSlowPool) + else if (associated(prevTendSlowPool)) then + call mpas_pool_link_pools(tendSlowPool, prevTendSlowPool) + else if (associated(nextTendSlowPool)) then + call mpas_pool_link_pools(tendSlowPool,nextPool=nextTendSlowPool) + else + call mpas_pool_link_pools(tendSlowPool) + end if + + call mpas_pool_link_parinfo(block, tendSum1stPool) + call mpas_pool_link_parinfo(block, tendSum2ndPool) + call mpas_pool_link_parinfo(block, tendSum3rdPool) + call mpas_pool_link_parinfo(block, tendSlowPool) + call mpas_pool_get_subpool(block % structs, 'LTS', LTSPool) call mpas_pool_get_array(LTSPool, 'LTSRegion', LTSRegion) diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_rk4.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_rk4.F index af583448157c..5fbb56e6ec1f 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_rk4.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_rk4.F @@ -44,6 +44,9 @@ module ocn_time_integration_rk4 use ocn_surface_land_ice_fluxes use ocn_transport_tests + use ocn_subgrid + + implicit none private save @@ -195,6 +198,9 @@ subroutine ocn_time_integrator_rk4(domain, dt)!{{{ real (kind=RKIND), dimension(:,:,:), pointer :: activeTracersCur, activeTracersNew + real(kind=RKIND), dimension(:,:), pointer :: subgridSshCellTableRange, subgridWetVolumeCellTable + + ! Get config options call mpas_pool_get_config(domain % configs, 'config_mom_del4', config_mom_del4) call mpas_pool_get_config(domain % configs, 'config_filter_btr_mode', config_filter_btr_mode) @@ -228,10 +234,9 @@ subroutine ocn_time_integrator_rk4(domain, dt)!{{{ call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) call mpas_pool_get_subpool(block % structs, 'diagnostics', diagnosticsPool) - call mpas_pool_create_pool(provisStatePool) + call mpas_pool_get_subpool(block % structs, 'provis_state', provisStatePool) - call mpas_pool_clone_pool(statePool, provisStatePool, 1) - call mpas_pool_add_subpool(block % structs, 'provis_state', provisStatePool) + call mpas_pool_copy_pool(statePool, provisStatePool, 1) call mpas_pool_get_dimension(block % dimensions, 'nCells', nCells) call mpas_pool_get_dimension(block % dimensions, 'nEdges', nEdges) @@ -249,6 +254,13 @@ subroutine ocn_time_integrator_rk4(domain, dt)!{{{ call mpas_pool_get_array(meshPool, 'maxLevelCell', maxLevelCell) call mpas_pool_get_array(meshPool, 'maxLevelEdgeTop', maxLevelEdgeTop) + call mpas_pool_get_array(meshPool, 'subgridWetVolumeCellTable', & + subgridWetVolumeCellTable) + + call mpas_pool_get_array(meshPool, 'subgridSshCellTableRange', & + subgridSshCellTableRange) + + ! Lower k-loop limit of 1 rather than minLevel* needed in *New = *Cur ! assignments below are needed to maintain bit-for-bit results @@ -315,37 +327,6 @@ subroutine ocn_time_integrator_rk4(domain, dt)!{{{ block => block % next end do - block => domain % blocklist - do while(associated(block)) - if (associated(block % prev)) then - call mpas_pool_get_subpool(block % prev % structs, 'provis_state', prevProvisPool) - else - nullify(prevProvisPool) - end if - - if (associated(block % next)) then - call mpas_pool_get_subpool(block % next % structs, 'provis_state', nextProvisPool) - else - nullify(nextProvisPool) - end if - - call mpas_pool_get_subpool(block % structs, 'provis_state', provisStatePool) - - if (associated(prevProvisPool) .and. associated(nextProvisPool)) then - call mpas_pool_link_pools(provisStatePool, prevProvisPool, nextProvisPool) - else if (associated(prevProvisPool)) then - call mpas_pool_link_pools(provisStatePool, prevProvisPool) - else if (associated(nextProvisPool)) then - call mpas_pool_link_pools(provisStatePool, nextPool=nextProvisPool) - else - call mpas_pool_link_pools(provisStatePool) - end if - - call mpas_pool_link_parinfo(block, provisStatePool) - - block => block % next - end do - ! Fourth-order Runge-Kutta, solving dy/dt = f(t,y) is typically written as follows ! where h = delta t is the large time step. Here f(t,y) is the right hand side, ! called the tendencies in the code below. @@ -481,7 +462,7 @@ subroutine ocn_time_integrator_rk4(domain, dt)!{{{ if (config_prevent_drying) then block => domain % blocklist do while (associated(block)) - call ocn_prevent_drying_rk4(block, dt, rk_substep_weights(rk_step), config_zero_drying_velocity, err) + call ocn_prevent_drying_rk4(domain, block, dt, rk_substep_weights(rk_step), config_zero_drying_velocity, err) block => block % next end do ! exchange fields for parallelization @@ -708,6 +689,15 @@ subroutine ocn_time_integrator_rk4(domain, dt)!{{{ layerThicknessNew(k, iCell) = tidalInputMask(iCell)*(tidalBCValue(iCell) + bottomDepth(iCell))*(restingThickness(k,iCell)/totalDepth) !(1.0_RKIND - tidalInputMask(iCell))*layerThicknessNew(k, iCell) ! generalized tappered assumption code end do + + if ( config_use_subgrid_wetting_drying ) then + call ocn_subgrid_layer_thickness_lookup( tidalBCValue(iCell), & + subgridWetVolumeCellTable(:,iCell), & + subgridSshCellTableRange(:,iCell),& + bottomDepth(iCell), & + layerThicknessNew(1,iCell) ) + end if + end if end do end if @@ -853,16 +843,6 @@ subroutine ocn_time_integrator_rk4(domain, dt)!{{{ call mpas_timer_stop("RK4-cleanup phase") - block => domain % blocklist - do while(associated(block)) - call mpas_pool_get_subpool(block % structs, 'provis_state', provisStatePool) - - call mpas_pool_destroy_pool(provisStatePool) - - call mpas_pool_remove_subpool(block % structs, 'provis_state') - block => block % next - end do - end subroutine ocn_time_integrator_rk4!}}} subroutine ocn_time_integrator_rk4_compute_vel_tends(domain, block, dt, & @@ -1731,7 +1711,8 @@ subroutine ocn_time_integration_rk4_init(domain)!{{{ ! local variables !----------------------------------------------------------------- type (block_type), pointer :: block - type (mpas_pool_type), pointer :: meshPool + type (mpas_pool_type), pointer :: meshPool, statePool, provisStatePool + type (mpas_pool_type), pointer :: nextProvisPool, prevProvisPool logical, pointer :: config_use_debugTracers integer, pointer :: nVertLevels ! End preamble @@ -1742,12 +1723,43 @@ subroutine ocn_time_integration_rk4_init(domain)!{{{ block => domain % blocklist do while (associated(block)) call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) + call mpas_pool_get_subpool(block % structs, 'state', statePool) call mpas_pool_get_dimension(meshPool, 'nVertLevels', nVertLevels) if (config_use_debugTracers .and. nVertLevels == 1) then call mpas_log_write('Debug tracers may cause failures in a ' & // 'single layer case. Consider setting ' & // 'config_use_debugTracers to .false.', MPAS_LOG_WARN) endif + + call mpas_pool_create_pool(provisStatePool) + call mpas_pool_clone_pool(statePool, provisStatePool, 1) + call mpas_pool_add_subpool(block % structs, 'provis_state', provisStatePool) + + if (associated(block % prev)) then + call mpas_pool_get_subpool(block % prev % structs, 'provis_state', prevProvisPool) + else + nullify(prevProvisPool) + end if + + if (associated(block % next)) then + call mpas_pool_get_subpool(block % next % structs, 'provis_state', nextProvisPool) + else + nullify(nextProvisPool) + end if + + call mpas_pool_get_subpool(block % structs, 'provis_state', provisStatePool) + + if (associated(prevProvisPool) .and. associated(nextProvisPool)) then + call mpas_pool_link_pools(provisStatePool, prevProvisPool, nextProvisPool) + else if (associated(prevProvisPool)) then + call mpas_pool_link_pools(provisStatePool, prevProvisPool) + else if (associated(nextProvisPool)) then + call mpas_pool_link_pools(provisStatePool, nextPool=nextProvisPool) + else + call mpas_pool_link_pools(provisStatePool) + end if + + call mpas_pool_link_parinfo(block, provisStatePool) block => block % next end do diff --git a/components/mpas-ocean/src/mode_init/Makefile b/components/mpas-ocean/src/mode_init/Makefile index d1aba81e14f5..2f16f1973183 100644 --- a/components/mpas-ocean/src/mode_init/Makefile +++ b/components/mpas-ocean/src/mode_init/Makefile @@ -7,7 +7,8 @@ UTILS = mpas_ocn_init_spherical_utils.o \ mpas_ocn_init_cell_markers.o \ mpas_ocn_init_interpolation.o \ mpas_ocn_init_ssh_and_landIcePressure.o \ - mpas_ocn_init_smoothing.o + mpas_ocn_init_smoothing.o \ + mpas_ocn_init_subgrid.o TEST_CASES = mpas_ocn_init_baroclinic_channel.o \ mpas_ocn_init_lock_exchange.o \ @@ -31,7 +32,8 @@ TEST_CASES = mpas_ocn_init_baroclinic_channel.o \ mpas_ocn_init_mixed_layer_eddy.o \ mpas_ocn_init_transport_tests.o \ mpas_ocn_init_test_sht.o \ - mpas_ocn_init_parabolic_bowl.o + mpas_ocn_init_parabolic_bowl.o \ + mpas_ocn_init_buttermilk_bay.o #mpas_ocn_init_TEMPLATE.o all: init_mode @@ -54,6 +56,8 @@ mpas_ocn_init_vertical_grids.o: mpas_ocn_init_seaSurfaceHeightAndPressure.o: +mpas_ocn_init_subgrid.o: + mpas_ocn_init_baroclinic_channel.o: $(UTILS) mpas_ocn_init_iso.o: $(UTILS) @@ -100,6 +104,7 @@ mpas_ocn_init_test_sht.o: $(UTILS) mpas_ocn_init_parabolic_bowl.o: $(UTILS) +mpas_ocn_init_buttermilk_bay.o: $(UTILS) #mpas_ocn_init_TEMPLATE.o: $(UTILS) diff --git a/components/mpas-ocean/src/mode_init/Registry.xml b/components/mpas-ocean/src/mode_init/Registry.xml index 3c05635a9c1b..1ef457fe5328 100644 --- a/components/mpas-ocean/src/mode_init/Registry.xml +++ b/components/mpas-ocean/src/mode_init/Registry.xml @@ -20,6 +20,7 @@ #include "Registry_mixed_layer_eddy.xml" #include "Registry_test_sht.xml" #include "Registry_parabolic_bowl.xml" +#include "Registry_buttermilk_bay.xml" // #include "Registry_TEMPLATE.xml" diff --git a/components/mpas-ocean/src/mode_init/Registry_buttermilk_bay.xml b/components/mpas-ocean/src/mode_init/Registry_buttermilk_bay.xml new file mode 100644 index 000000000000..09754d2a7d4b --- /dev/null +++ b/components/mpas-ocean/src/mode_init/Registry_buttermilk_bay.xml @@ -0,0 +1,58 @@ + + + + + + + + + + + + + + + + diff --git a/components/mpas-ocean/src/mode_init/Registry_parabolic_bowl.xml b/components/mpas-ocean/src/mode_init/Registry_parabolic_bowl.xml index b2188231dc4e..bd01e21e1be7 100644 --- a/components/mpas-ocean/src/mode_init/Registry_parabolic_bowl.xml +++ b/components/mpas-ocean/src/mode_init/Registry_parabolic_bowl.xml @@ -2,16 +2,16 @@ - + - - @@ -23,13 +23,24 @@ description="Gravitational accerlation" possible_values="Any real number" /> - + + + + - - - - - diff --git a/components/mpas-ocean/src/mode_init/mpas_ocn_init_buttermilk_bay.F b/components/mpas-ocean/src/mode_init/mpas_ocn_init_buttermilk_bay.F new file mode 100644 index 000000000000..0b9f4cbf512b --- /dev/null +++ b/components/mpas-ocean/src/mode_init/mpas_ocn_init_buttermilk_bay.F @@ -0,0 +1,906 @@ +! Copyright (c) 2013, Los Alamos National Security, LLC (LANS) +! and the University Corporation for Atmospheric Research (UCAR). +! +! Unless noted otherwise source code is licensed under the BSD license. +! Additional copyright and license information can be found in the LICENSE file +! distributed with this code, or at http://mpas-dev.github.com/license.html +! +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! ocn_init_buttermilk_bay +! +!> \brief initialize the Buttermilk Bay case +!> \author D. Wirasaet, S. Brus +!> \date May-June 2022 +!> \details +!> This module contains the routines for initializing the Buttermilk +!> Bay test case with or without subgrid corrections +!> +!----------------------------------------------------------------------- + +module ocn_init_Buttermilk_bay + + use mpas_kind_types + use mpas_io_units + use mpas_derived_types + use mpas_pool_routines + use mpas_constants + use mpas_dmpar + + use ocn_constants + use ocn_config + use ocn_init_vertical_grids + use ocn_init_cell_markers + use ocn_subgrid + use ocn_init_subgrid + + use mpas_constants + use mpas_io + use mpas_io_streams + use mpas_stream_manager + + + implicit none + private + save + + !-------------------------------------------------------------------- + ! + ! Public parameters + ! + !-------------------------------------------------------------------- + + !-------------------------------------------------------------------- + ! + ! Public member functions + ! + !-------------------------------------------------------------------- + + public :: ocn_init_setup_Buttermilk_bay, & + ocn_init_validate_Buttermilk_bay + + !-------------------------------------------------------------------- + ! + ! Private module variables + ! + !-------------------------------------------------------------------- + + ! For netcdf topobathy input variables + integer :: nLatTopo, nLonTopo + + type (field1DReal) :: topoLat, topoLon + type (field2DReal) :: topoIC + + real(kind=RKIND), parameter:: eps = 1.0e-10_RKIND ; +!*********************************************************************** + +contains + +!*********************************************************************** +! +! routine ocn_init_setup_Buttermilk_bay +! +!> \brief Setup for this initial condition +!> \author D. Wirasaet and S. Brus +!> \date May-June 2022 +!> \details +!> This routine sets up the initial conditions for this case. +!> To be run in sigma vertical coordinates and single-layer +! +!----------------------------------------------------------------------- + + subroutine ocn_init_setup_Buttermilk_bay(domain, iErr)!{{{ + use mpas_vector_operations ! To calcutate edgeNormalVector + + implicit none + !-------------------------------------------------------------------- + + type (domain_type), intent(inout) :: domain + integer, intent(out) :: iErr + + type (block_type), pointer :: block_ptr + type (mpas_pool_type), pointer :: meshPool + type (mpas_pool_type), pointer :: statePool + type (mpas_pool_type), pointer :: tracersPool + type (mpas_pool_type), pointer :: verticalMeshPool + type (mpas_pool_type), pointer :: forcingPool + + ! local variables + integer :: iCell, iEdge, iVertex, k, idx + real (kind=RKIND) :: yMin, yMax, xMin, xMax, dcEdgeMin, dcEdgeMinGlobal + real (kind=RKIND) :: yMinGlobal, yMaxGlobal, yMidGlobal, xMinGlobal, xMaxGlobal + real (kind=RKIND) :: localVar1, localVar2 + real (kind=RKIND), dimension(:), pointer :: interfaceLocations + + ! Define dimension pointers + integer, pointer :: nCellsSolve, nEdgesSolve, nVerticesSolve, nVertLevels, nVertLevelsP1 + integer, pointer :: index_temperature, index_salinity + integer, pointer :: maxEdges + + ! Define variable pointers + logical, pointer :: on_a_sphere + integer, dimension(:), pointer :: minLevelCell, maxLevelCell + integer, dimension(:), pointer :: nEdgesOnCell + integer, dimension(:,:), pointer :: verticesOnCell, verticesOnEdge + integer, dimension(:,:), pointer :: cellsOnEdge, cellsOnVertex + real (kind=RKIND), dimension(:), pointer :: xCell, yCell, refBottomDepth, refZMid, & + vertCoordMovementWeights, bottomDepth, fCell, fEdge, fVertex, dcEdge + real (kind=RKIND), dimension(:,:), pointer:: zMid + + real (kind=RKIND), dimension(:), pointer:: xEdge, yEdge, xVertex, yVertex + real (kind=RKIND) :: minBottomDepth, maxBottomDepth, globalMaxBottomDepth, globalMinBottomDepth + real (kind=RKIND), dimension(:,:), pointer :: layerThickness, restingThickness + real (kind=RKIND), dimension(:,:,:), pointer :: activeTracers + + real (kind=RKIND), dimension(:), pointer :: ssh + real (kind=RKIND), dimension(:), pointer :: areaCell + real (kind=RKIND), dimension(:,:), pointer :: edgeNormalVectors + real (kind=RKIND), dimension(:,:), pointer :: normalVelocity + ! Elevation Bcs + real (kind=RKIND), dimension(:), pointer :: tidalInputMask + + + real (kind=RKIND):: HH, uu, vv + real (kind=RKIND):: RR, num, den + real (kind=RKIND):: xshift = 0.0, yshift = 0.0 + real (kind=RKIND) :: layerThicknessEdgeAverage + real (kind=RKIND), dimension(:,:), allocatable :: rSubgridPoints, sSubgridPoints + real (kind=RKIND), dimension(:), allocatable :: subgridBathymetryValues, subgridAreas + real (kind=RKIND), dimension(:), allocatable :: subgridSshValues + real (kind=RKIND), dimension(:), allocatable :: subgridUValues, subgridVValues + real (kind=RKIND), dimension(:), allocatable :: uVelocityAverage, vVelocityAverage + integer :: nSubgridCell, nSubgridEdge, nSubgridVertex + integer :: nSubgridTriPerSlice + integer :: v1, v2 + integer :: c1, c2 + real (kind=RKIND) :: x(3), y(3) + integer :: slice, nSlice + real (kind=RKIND) :: deltaZ + + + integer:: i, j, jj + integer:: nsubgridCellEdge, iEdgeSegment + real (kind=RKIND):: pi + real (kind=RKIND), dimension(:,:), allocatable :: cellEdgeBathymetryValues + real (kind=RKIND), dimension(:), allocatable:: dsEdge + real (kind=RKIND), dimension(:), allocatable:: xSubgridCell, ySubgridCell + real (kind=RKIND):: bathymetryMin, bathymetryMax + iErr = 0 + + if(config_init_configuration .ne. trim('buttermilk_bay')) return + + ! Determine vertical grid for configuration + call mpas_pool_get_subpool(domain % blocklist % structs, 'mesh', meshPool) + call mpas_pool_get_dimension(meshPool, 'nVertLevels', nVertLevels) + call mpas_pool_get_dimension(meshPool, 'nVertLevelsP1', nVertLevelsP1) + call mpas_pool_get_config(meshPool, 'on_a_sphere', on_a_sphere) + + nVertLevels = config_Buttermilk_bay_vert_levels ; + nVertLevelsP1 = nVertLevels + 1 + + allocate(interfaceLocations(nVertLevelsP1)) + call ocn_generate_vertical_grid( config_vertical_grid, interfaceLocations, ocnConfigs ) ; + !! Mental note: interfaceLocatons = (k-1)/N ; + + ! Initalize min/max values to large positive and negative values + yMin = 1.0E10_RKIND + yMax = -1.0E10_RKIND + xMin = 1.0E10_RKIND + xMax = -1.0E10_RKIND + dcEdgeMin = 1.0E10_RKIND + + ! Determine local min and max values. + block_ptr => domain % blocklist + do while(associated(block_ptr)) + call mpas_pool_get_subpool(block_ptr % structs, 'mesh', meshPool) + + call mpas_pool_get_dimension( meshPool, 'nCellsSolve', nCellsSolve ) + call mpas_pool_get_dimension( meshPool, 'nEdgesSolve', nEdgesSolve ) + + call mpas_pool_get_array(meshPool, 'xCell', xCell) + call mpas_pool_get_array(meshPool, 'yCell', yCell) + call mpas_pool_get_array(meshPool, 'dcEdge', dcEdge) + + yMin = min( yMin, minval(yCell(1:nCellsSolve))) + yMax = max( yMax, maxval(yCell(1:nCellsSolve))) + xMin = min( xMin, minval(xCell(1:nCellsSolve))) + xMax = max( xMax, maxval(xCell(1:nCellsSolve))) + dcEdgeMin = min( dcEdgeMin, minval(dcEdge(1:nEdgesSolve))) + + block_ptr => block_ptr % next + end do + + ! Determine global min and max values. + call mpas_dmpar_min_real(domain % dminfo, yMin, yMinGlobal) + call mpas_dmpar_max_real(domain % dminfo, yMax, yMaxGlobal) + call mpas_dmpar_min_real(domain % dminfo, xMin, xMinGlobal) + call mpas_dmpar_max_real(domain % dminfo, xMax, xMaxGlobal) + call mpas_dmpar_min_real(domain % dminfo, dcEdgeMin, dcEdgeMinGlobal) + + pi = acos(-1.0_RKIND) + + xshift = xMin + yshift = (3.0_RKIND*yMin + dcEdgeMin*sin(pi/3.0_RKIND))/3.0_RKIND + ! print*, "xMin, yMin = ", xMin, yMin, dcEdgeMin + ! print*, "xshift, yshift = ", xshift, yshift + + !*********************************************************************** + ! + ! Topography + ! + !*********************************************************************** + + call mpas_log_write( 'Reading bathymetry from a NetCDF file') + + if (config_Buttermilk_bay_topography_source == 'latlon_file' .or. & + config_Buttermilk_bay_topography_source == 'xy_file' ) then + call mpas_log_write( 'Reading topography data from file.') + call ocn_init_setup_Buttermilk_bay_read_topo(domain, iErr) + endif + + !-------------------------------------------------------------------- + ! Use this section to set initial values + !-------------------------------------------------------------------- + + block_ptr => domain % blocklist + call mpas_pool_get_subpool(block_ptr % structs, 'mesh', meshPool) + call mpas_pool_get_subpool(block_ptr % structs, 'state', statePool) + call mpas_pool_get_subpool(block_ptr % structs, 'verticalMesh', verticalMeshPool) + call mpas_pool_get_subpool(statePool, 'tracers', tracersPool) + + call mpas_pool_get_subpool(block_ptr % structs, 'forcing', forcingPool) + call mpas_pool_get_array(forcingPool, 'tidalInputMask', tidalInputMask) + + call mpas_pool_get_dimension(meshPool, 'nVertLevels', nVertLevels) ; + call mpas_pool_get_dimension(meshPool, 'nCellsSolve', nCellsSolve) ; + call mpas_pool_get_dimension(meshPool, 'nEdgesSolve', nEdgesSolve) ; + call mpas_pool_get_dimension(meshPool, 'nVerticesSolve', nVerticesSolve) ; + call mpas_pool_get_dimension(meshPool, 'maxEdges', maxEdges) + + call mpas_pool_get_dimension(tracersPool, 'index_temperature', index_temperature) + call mpas_pool_get_dimension(tracersPool, 'index_salinity', index_salinity) + + call mpas_pool_get_array(meshPool, 'xCell', xCell) + call mpas_pool_get_array(meshPool, 'yCell', yCell) + call mpas_pool_get_array(meshPool, 'refBottomDepth', refBottomDepth) + call mpas_pool_get_array(meshPool, 'vertCoordMovementWeights', vertCoordMovementWeights) + call mpas_pool_get_array(meshPool, 'bottomDepth', bottomDepth) + call mpas_pool_get_array(meshPool, 'minLevelCell', minLevelCell) + call mpas_pool_get_array(meshPool, 'maxLevelCell', maxLevelCell) + call mpas_pool_get_array(meshPool, 'nEdgesOnCell', nEdgesOnCell) + call mpas_pool_get_array(meshPool, 'areaCell', areaCell) + + call mpas_pool_get_array(meshPool, 'verticesOnCell', verticesOnCell) + call mpas_pool_get_array(meshPool, 'verticesOnEdge', verticesOnEdge) + call mpas_pool_get_array(meshPool, 'cellsOnEdge', cellsOnEdge) + call mpas_pool_get_array(meshPool, 'cellsOnVertex', cellsOnVertex) + + call mpas_pool_get_array(meshPool, 'fCell', fCell) + call mpas_pool_get_array(meshPool, 'fEdge', fEdge) + call mpas_pool_get_array(meshPool, 'fVertex', fVertex) + + call mpas_pool_get_array(meshPool, 'xEdge', xEdge ) + call mpas_pool_get_array(meshPool, 'yEdge', yEdge ) + call mpas_pool_get_array(meshPool, 'xVertex', xVertex ) + call mpas_pool_get_array(meshPool, 'yVertex', yVertex ) + + call mpas_pool_get_array(statePool, 'zMid', zMid, 1) ; + + call mpas_pool_get_array(statePool, 'ssh', ssh, 1) + call mpas_pool_get_array(meshPool, 'edgeNormalVectors', edgeNormalVectors ) ; + call mpas_pool_get_array(statePool, 'normalVelocity', normalVelocity ) ; + + call mpas_pool_get_array(tracersPool, 'activeTracers', activeTracers, 1) + call mpas_pool_get_array(statePool, 'layerThickness', layerThickness, 1) + + call mpas_pool_get_array(verticalMeshPool, 'refZMid', refZMid) + call mpas_pool_get_array(verticalMeshPool, 'restingThickness', restingThickness) + + call mpas_pool_get_array(meshPool, 'subgridWetVolumeCellTable', & + subgridWetVolumeCellTable) + call mpas_pool_get_array(meshPool, 'subgridWetVolumeEdgeTable', & + subgridWetVolumeEdgeTable) + call mpas_pool_get_array(meshPool, 'subgridWetVolumeVertexTable', & + subgridWetVolumeVertexTable) + call mpas_pool_get_array(meshPool, 'subgridWetFractionCellTable', & + subgridWetFractionCellTable) + call mpas_pool_get_array(meshPool, 'subgridWetFractionEdgeTable', & + subgridWetFractionEdgeTable) + call mpas_pool_get_array(meshPool,'subgridWetFractionVertexTable',& + subgridWetFractionVertexTable) + call mpas_pool_get_array(meshPool, 'subgridSshCellTableRange', & + subgridSshCellTableRange) + call mpas_pool_get_array(meshPool, 'subgridSshEdgeTableRange', & + subgridSshEdgeTableRange) + call mpas_pool_get_array(meshPool, 'subgridSshVertexTableRange', & + subgridSshVertexTableRange) + call mpas_pool_get_array(meshPool, 'subgridEdgeBathymetryMean', & + subgridEdgeBathymetryMean) + call mpas_pool_get_array(meshPool, 'subgridVertexBathymetryMean', & + subgridVertexBathymetryMean) + call mpas_pool_get_array(meshPool, 'subgridCellBathymetryMin', & + subgridCellBathymetryMin) + call mpas_pool_get_array(meshPool, 'subgridEdgeBathymetryMin', & + subgridEdgeBathymetryMin) + call mpas_pool_get_array(meshPool, 'subgridVertexBathymetryMin', & + subgridVertexBathymetryMin) + call mpas_pool_get_dimension(meshPool, 'nSubgridTableLevels', & + nSubgridTableLevels) + call mpas_pool_get_array(meshPool, 'subgridLayerThicknessDebug', & + subgridLayerThicknessDebug) + + ! if config_buttermilk_bay_adjust_domain_center == .true., + ! Adjust center of the mesh so that its center is located at (0,0) + if ( config_Buttermilk_bay_adjust_domain ) then + xCell = xCell - xshift ; + yCell = yCell - yshift ; + + xEdge = xEdge - xshift ; + yEdge = yEdge - yshift ; + + xVertex = xVertex - xshift ; + yVertex = yVertex - yshift ; + + ! get min, max coordinates of model domain ! + ! after adjusting the coordinates ! + yMin = min( yMin, minval(yCell(1:nCellsSolve))) + yMax = max( yMax, maxval(yCell(1:nCellsSolve))) + xMin = min( xMin, minval(xCell(1:nCellsSolve))) + xMax = max( xMax, maxval(xCell(1:nCellsSolve))) + + ! Determine global min and max values. + call mpas_dmpar_min_real(domain % dminfo, yMin, yMinGlobal) + call mpas_dmpar_max_real(domain % dminfo, yMax, yMaxGlobal) + call mpas_dmpar_min_real(domain % dminfo, xMin, xMinGlobal) + call mpas_dmpar_max_real(domain % dminfo, xMax, xMaxGlobal) + end if + + ! Initlialze vector + call mpas_initialize_vectors(meshPool) ; + + minLevelCell(:) = 1 + do iCell = 1, nCellsSolve + ! Set up vertical grid + maxLevelCell(iCell) = nVertLevels ; ! sigma coordinates + end do + + + do iCell = 1, nCellsSolve + ! Set temperature + activeTracers(index_temperature, :, iCell) = 10.0_RKIND + + ! Set salinity + activeTracers(index_salinity, :, iCell) = 30.0_RKIND + + ! Set Coriolis parameters, if other than zero + fCell(iCell) = 0.0_RKIND; + end do + + do iEdge = 1, nEdgesSolve + fEdge(iEdge) = 0.0_RKIND; + end do + + do iVertex = 1, nVerticesSolve + fVertex(iVertex) = 0.0_RKIND; + end do + + allocate(uVelocityAverage(nEdgesSolve)) + allocate(vVelocityAverage(nEdgesSolve)) + + if (config_use_subgrid_wetting_drying) then + + call ocn_subgrid_init(domain,iErr) + call ocn_init_subgrid_calculations(domain, & + ocn_init_Buttermilk_bay_bathymetry, & + ocn_init_Buttermilk_bay_velocity, & + ocn_init_Buttermilk_bay_ssh, & + config_Buttermilk_bay_subgrid_refinement_level, & + config_Buttermilk_bay_subgrid_edge_bathymetry_max_pixel, & + config_Buttermilk_bay_subgrid_use_thin_layer, & + uVelocityAverage, & + vVelocityAverage, & + iErr) + end if + + ! Find max bottom depth + maxBottomDepth = maxval( bottomDepth ) ; + minBottomDepth = minval( bottomDepth ) ; + call mpas_dmpar_max_real( domain % dminfo, maxBottomDepth, globalMaxBottomDepth ) ; + call mpas_dmpar_min_real( domain % dminfo, minBottomDepth, globalMinBottomDepth ) ; + + ! Set refBottomDepth and refZMid + do k = 1, nVertLevels + refBottomDepth(k) = globalMaxBottomDepth*interfaceLocations(k+1) ; + refZMid(k) = -0.5_RKIND*( interfaceLocations(k+1) + interfaceLocations(k))*globalMaxBottomDepth ; + end do + + ! Set vertCoordMovementWeights + vertCoordMovementWeights(:) = 1.0_RKIND + + ! Set velocity + do iEdge = 1, nEdgesSolve + + if (config_use_subgrid_wetting_drying) then + do k = 1, nVertLevels + normalVelocity(k,iEdge) = uVelocityAverage(iEdge)*edgeNormalVectors(1,iEdge) & + + vVelocityAverage(iEdge)*edgeNormalVectors(2,iEdge) ; + end do + else + call ocn_init_Buttermilk_bay_velocity(xEdge(iEdge), yEdge(iEdge), uu, vv) + do k = 1, nVertLevels + normalVelocity(k,iEdge) = uu*edgeNormalVectors(1,iEdge) + vv*edgeNormalVectors(2,iEdge) ; + end do + end if + end do + + ! Set layer thickness and ssh + if (config_use_wetting_drying) then + + do iCell = 1, nCellsSolve + ! Set up vertical grid + maxLevelCell(iCell) = nVertLevels ; ! sigma coordinates + end do + + do iCell = 1, nCellsSolve + ! + ! make sure depth is thick enough via ssh = TOTAL_DEPTH - bottomDepth + ! add a thin layer of nlayer*config_drying_min_cellhight + ! + + if (config_use_subgrid_wetting_drying) then + ! Initial contion for a subgrid run + call ocn_subgrid_ssh_lookup(layerThickness(1,iCell),& + subgridWetVolumeCellTable(:,iCell),& + subgridSshCellTableRange(:,iCell),& + bottomDepth(iCell),& + subgridCellBathymetryMin(iCell),& + ssh(iCell)) + !call ocn_subgrid_layer_thickness_lookup(ssh(iCell), & + ! subgridWetVolumeCellTable(:,iCell), & + ! subgridSshCellTableRange(:,iCell),& + ! bottomDepth(iCell),& + ! LayerThickness(1,iCell)) + + else + ! Initial condition for a standard run + call ocn_init_Buttermilk_bay_bathymetry(xCell(iCell), yCell(iCell), bottomDepth(iCell)) + + ssh(iCell) = -bottomDepth(iCell) ; + do k = 1, maxLevelCell(iCell) + ! + layerThickness(k,iCell) = max( config_drying_min_cell_height, & + bottomDepth(iCell)/real(maxLevelCell(iCell),RKIND) ) + + + if (layerThickness(k,iCell) < config_drying_min_cell_height) then + call mpas_log_write('layerThickness($i,$i)=$r', MPAS_LOG_CRIT, & + intArgs=(/k,iCell/), & + realArgs=(/layerThickness(k,iCell)/)) + end if + + ssh(iCell) = ssh(iCell) + layerThickness(k,iCell) ; + end do + ! + endif + + do k = 1, maxLevelCell(iCell) + restingThickness(k,iCell) = bottomDepth(iCell)/maxLevelCell(iCell) + end do + + end do + + end if + + + if (config_use_subgrid_wetting_drying) then + do iCell = 1,nCellsSolve + call ocn_subgrid_layer_thickness_lookup(ssh(iCell), & + subgridWetVolumeCellTable(:,iCell), & + subgridSshCellTableRange(:,iCell),& + bottomDepth(iCell),& + subgridLayerThicknessDebug(iCell)) + enddo + end if + + ! Set tidal boundary mask + do iCell = 1, nCellsSolve + tidalInputMask(iCell) = 0.0_RKIND + if ( yCell(iCell) < (yMin+(dcEdgeMin*sin(pi/3.0_RKIND)/2.0_RKIND)) & + .and. yCell(iCell) > (yMin-(dcEDgeMin*sin(pi/3.0_RKIND)/2.0_RKIND)) ) then + + if ( (xCell(iCell) - dcEdgeMin/2.0_RKIND) > 2048.0_RKIND & + .and. (xCell(iCell) + dcEdgeMin/2.0_RKIND) < 3072.0_RKIND ) then + tidalInputMask(iCell) = 1.0_RKIND + end if + ! spread it over multiple cells + ! if (yCell(iCell) > (25.0e3 - 3*dcEdgeMinGlobal)) then + ! tidalInputMask(iCell) = exp(-((yCell(iCell)-25.0e3)/dcEdgeMinGlobal)**2.0) + end if + end do + + deallocate(interfaceLocations) + if (config_global_ocean_topography_source == 'latlon_file') then + call mpas_log_write( 'Cleaning up topography IC fields') + call ocn_init_Buttermilk_bay_destroy_topo_fields() + endif + !-------------------------------------------------------------------- + + print*, "****** End Butter milk bay init *****" ; + + return ; + end subroutine ocn_init_setup_Buttermilk_bay!}}} + +!*********************************************************************** +! +! routine ocn_init_Buttermilk_bay_bathymetry +! +!> \brief Interpolate bathymetry +!> \author Steven Brus, D. Wirasaet +!> \date November 2022 +!> \details Return the value of the bathymetry at a given x,y point +!> +! +!----------------------------------------------------------------------- + + subroutine ocn_init_Buttermilk_bay_bathymetry(x, y, depth)!{{{ + + implicit none + + real (kind=RKIND), intent(in) :: x, y + real (kind=RKIND), intent(out) :: depth + + integer:: ix(2) + real (kind=RKIND) :: xc(2) + real (kind=RKIND) :: dx(2), x0(2), xN(2), bath(4), f(4), den + real (kind=RKIND) :: xv(2), yv(2) + + !-------------------------------------------------------------------- + ! + ! DEM must be on a uniform grid + x0 = (/ topoLon%array(1), topoLat%array(1) /) ; + xN = (/ topoLon%array(nLonTopo), topoLat%array(nLatTopo) /) ; + dx = (/ topoLon%array(2) - topoLon%array(1), & + topoLat%array(2) - topoLat%array(1) /) ; + + ! Bilienar interpolation + xc = (/ x, y /) ; + ix = floor( (xc - x0)/dx ) + 1 ; + + if ( ( ix(1) >= 1 .and. ix(1) < nLonTopo ) .and. & + ( ix(2) >= 1 .and. ix(2) < nLatTopo ) ) then + ! include the west and soutth bourdaries of + ! a given uniform raster DEM + bath(1) = TopoIC%array(ix(1),ix(2)) ; + bath(2) = TopoIC%array(ix(1)+1,ix(2)) ; + bath(3) = TopoIC%array(ix(1)+1,ix(2)+1) ; + bath(4) = TopoIC%array(ix(1),ix(2)) ; + + xv(1) = topoLon%array(ix(1)) ; + xv(2) = topoLon%array(ix(1)+1) ; + + yv(1) = topoLat%array(ix(2)) ; + yv(2) = topoLat%array(ix(2)+1) ; + + den = dx(1)*dx(2) ; + + f(1) = ( xc(1) - xv(2) )*( xc(2) - yv(2) )/den ; + f(2) = ( xc(1) - xv(1) )*( xc(2) - yv(2) )/(-den) ; + f(3) = ( xc(1) - xv(1) )*( xc(2) - yv(1) )/den ; + f(4) = ( xc(1) - xv(2) )*( xc(2) - yv(1) )/(-den) ; + + depth = sum( bath*f ) ; + else + ! nearest extrapolation ! + ix(1) = merge( 1, ix(1), ix(1) < 1 ) ; + ix(1) = merge( nLonTopo, ix(1), ix(1) >= nLonTopo ) ; + ix(2) = merge( 1, ix(2), ix(2) < 1 ) ; + ix(2) = merge( nLatTopo, ix(2), ix(2) >= nLatTopo ) ; + + depth = TopoIC%array(ix(1),ix(2)) ; + endif + + return ; + end subroutine ocn_init_Buttermilk_bay_bathymetry!}}} + +!*********************************************************************** +! +! routine ocn_init_Buttermilk_bay_ssh +! +!> \brief Compute initial ssh field +!> \author Steven Brus, D. Wirasaet +!> \date November 2022 +!> \details Use exact solution to compute ssh field for initial conditions +!> +! +!----------------------------------------------------------------------- + + subroutine ocn_init_Buttermilk_bay_ssh(x, y, bottomDepth, ssh)!{{{ + + implicit none + + real (kind=RKIND), intent(in) :: x, y + real (kind=RKIND), intent(in) :: bottomDepth + real (kind=RKIND), intent(out) :: ssh + real (kind=RKIND) :: RR + + !-------------------------------------------------------------------- + + ssh = 0.0_RKIND ; + ssh = - bottomDepth + max( ssh + bottomDepth, 0.0_RKIND ) ; + + return + !-------------------------------------------------------------------- + + end subroutine ocn_init_Buttermilk_bay_ssh!}}} + +!*********************************************************************** +! +! routine ocn_init_Buttermilk_bay_velocity +! +!> \brief Compute initial velocity field +!> \author Steven Brus, D. Wirasaet +!> \date November 2022 +!> \details Use exact solution to comupte velocity field for initial conditions +!> +! +!----------------------------------------------------------------------- + + subroutine ocn_init_Buttermilk_bay_velocity(x, y, u, v)!{{{ + + implicit none + + real (kind=RKIND), intent(in) :: x, y + real (kind=RKIND), intent(out) :: u, v + real (kind=RKIND) :: RR, HH + + !-------------------------------------------------------------------- + + u = 0.0_RKIND ; + v = 0.0_RKIND ; + !-------------------------------------------------------------------- + + end subroutine ocn_init_Buttermilk_bay_velocity!}}} + +!*********************************************************************** +! +! routine ocn_init_setup_global_ocean_read_topo +! +!> \brief Read the topography IC file +!> \author D. Wirasaet, S. Brus +!> \date Sep 2023 +!> \details +!> This routine reads the topography IC file, including latitude and longitude +!> information for topography data. +!> +!> Adapted from ocn_init_setup_global_ocean_read_topo +!----------------------------------------------------------------------- + subroutine ocn_init_setup_Buttermilk_bay_read_topo(domain, iErr)!{{{ + type (domain_type), intent(inout) :: domain + integer, intent(out) :: iErr + + type (MPAS_Stream_type) :: topographyStream + + iErr = 0 + + ! Define stream for depth levels + call MPAS_createStream(topographyStream, domain % iocontext, & + config_Buttermilk_bay_topography_file, MPAS_IO_NETCDF, & + MPAS_IO_READ, ierr=iErr) + + ! Setup topoLat, topoLon, and topoIC fields for stream to be read in + topoLat % fieldName = trim(config_Buttermilk_bay_topography_lat_varname) + topoLat % dimSizes(1) = nLatTopo + topoLat % dimNames(1) = trim(config_Buttermilk_bay_topography_nlat_dimname) + topoLat % isVarArray = .false. + topoLat % isPersistent = .true. + topoLat % isActive = .true. + topoLat % hasTimeDimension = .false. + topoLat % block => domain % blocklist + allocate(topoLat % attLists(1)) + allocate(topoLat % array(nLatTopo)) + + topoLon % fieldName = trim(config_Buttermilk_bay_topography_lon_varname) + topoLon % dimSizes(1) = nLonTopo + topoLon % dimNames(1) = trim(config_Buttermilk_bay_topography_nlon_dimname) + topoLon % isVarArray = .false. + topoLon % isPersistent = .true. + topoLon % isActive = .true. + topoLon % hasTimeDimension = .false. + topoLon % block => domain % blocklist + allocate(topoLon % attLists(1)) + allocate(topoLon % array(nLonTopo)) + + topoIC % fieldName = trim(config_Buttermilk_bay_topography_varname) + topoIC % dimSizes(1) = nLonTopo + topoIC % dimSizes(2) = nLatTopo + topoIC % dimNames(1) = trim(config_Buttermilk_bay_topography_nlon_dimname) + topoIC % dimNames(2) = trim(config_Buttermilk_bay_topography_nlat_dimname) + topoIC % isVarArray = .false. + topoIC % isPersistent = .true. + topoIC % isActive = .true. + topoIC % hasTimeDimension = .false. + topoIC % block => domain % blocklist + allocate(topoIC % attLists(1)) + allocate(topoIC % array(nLonTopo, nLatTopo)) + + + ! Add topoLat, topoLon, and topoIC fields to stream + call MPAS_streamAddField(topographyStream, topoLat, iErr) + call MPAS_streamAddField(topographyStream, topoLon, iErr) + call MPAS_streamAddField(topographyStream, topoIC, iErr) + + ! Read stream + call MPAS_readStream(topographyStream, 1, iErr) + topoIC%array = -topoIC%array ; + + + ! Close stream + call MPAS_closeStream(topographyStream) + + if ( config_Buttermilk_bay_topography_latlon_degrees .and. & + config_Buttermilk_bay_topography_source == 'latlon_file' ) then + topoLat % array(:) = topoLat % array(:) * pii / 180.0_RKIND + topoLon % array(:) = topoLon % array(:) * pii / 180.0_RKIND + end if + + end subroutine ocn_init_setup_Buttermilk_bay_read_topo!}}} + + +!*********************************************************************** +! +! routine ocn_init_Buttermilk_bay_destroy_topo_fields +! +!> \brief Topography field cleanup routine +!> \author D. Wirasaet and S. Brus +!> \date Sep 2023 +!> \details +!> This routine destroys the fields that were created to hold topography +!> initial condition information +!> +!> NOTE: adapteed from ocn_init_global_ocaen_destroy_topo_fileds +!----------------------------------------------------------------------- + + subroutine ocn_init_Buttermilk_bay_destroy_topo_fields()!{{{ + implicit none + + deallocate(topoIC % array) + deallocate(topoLat % array) + deallocate(topoLon % array) + end subroutine ocn_init_Buttermilk_bay_destroy_topo_fields!}}} + + +!*********************************************************************** +! +! routine ocn_init_Buttermilk_bay +! +!> \brief Validation for this initial condition +!> \author D. Wirasaet and Steven Brus +!> \date Sep 2022 +!> \details +!> This routine validates the configuration options for this case. +!> +!----------------------------------------------------------------------- + subroutine ocn_init_validate_Buttermilk_bay(configPool, packagePool, iocontext, iErr)!{{{ + implicit none + + !-------------------------------------------------------------------- + type (mpas_pool_type), intent(inout) :: configPool, packagePool + type (mpas_io_context_type), intent(inout), target :: iocontext + + integer, intent(out) :: iErr + + ! character (len=StrKIND), pointer :: config_init_configuration + integer, pointer :: config_vert_levels, config_Buttermilk_bay_vert_levels + integer, pointer :: config_subgrid_table_levels, config_Buttermilk_bay_subgrid_table_levels + + + type (mpas_io_context_type), pointer :: iocontext_ptr + type (MPAS_IO_Handle_type) :: inputFile + character (len=StrKIND), pointer :: config_init_configuration, & + config_Buttermilk_bay_topography_source, & + config_Buttermilk_bay_topography_file, & + config_Buttermilk_bay_topography_nlat_dimname, & + config_Buttermilk_bay_topography_nlon_dimname + + iErr = 0 + + call mpas_pool_get_config(configPool, 'config_init_configuration', config_init_configuration) + + if(config_init_configuration .ne. trim('buttermilk_bay')) return + + iocontext_ptr => iocontext + + call mpas_pool_get_config(configPool, 'config_vert_levels', config_vert_levels) + call mpas_pool_get_config(configPool, & + 'config_Buttermilk_bay_vert_levels', & + config_Buttermilk_bay_vert_levels) + + if(config_vert_levels <= 0 .and. config_Buttermilk_bay_vert_levels > 0) then + config_vert_levels = config_Buttermilk_bay_vert_levels + else if (config_vert_levels <= 0) then + call mpas_log_write( 'Validation failed for Buttermilk bay.'// & + 'Not given a usable value for vertical levels.', MPAS_LOG_CRIT) + iErr = 1 + end if + + call mpas_pool_get_config(configPool, 'config_subgrid_table_levels', config_subgrid_table_levels) + call mpas_pool_get_config(configPool, & + 'config_Buttermilk_bay_subgrid_table_levels', & + config_Buttermilk_bay_subgrid_table_levels) + if (config_subgrid_table_levels <=0 .and. config_Buttermilk_bay_subgrid_table_levels >0) then + config_subgrid_table_levels = config_Buttermilk_bay_subgrid_table_levels + else if (config_subgrid_table_levels <=0) then + call mpas_log_write( 'Validation failed for Buttermilk bay.'// & + 'Not given a usable value for subgrid table levels.', MPAS_LOG_CRIT) + iErr = 1 + end if + + + !---------- adapted from ocn_init_validate_global_ocean + call mpas_pool_get_config(configPool, 'config_Buttermilk_bay_topography_source', & + config_Buttermilk_bay_topography_source) + call mpas_pool_get_config(configPool, 'config_Buttermilk_bay_topography_file', & + config_Buttermilk_bay_topography_file) + call mpas_pool_get_config(configPool, 'config_Buttermilk_bay_topography_nlat_dimname', & + config_Buttermilk_bay_topography_nlat_dimname) + call mpas_pool_get_config(configPool, 'config_Buttermilk_bay_topography_nlon_dimname', & + config_Buttermilk_bay_topography_nlon_dimname) + + + call mpas_log_write( config_Buttermilk_bay_topography_source ) + call mpas_log_write( config_Buttermilk_bay_topography_file ) + call mpas_log_write( config_Buttermilk_bay_topography_nlat_dimname ) + call mpas_log_write( config_Buttermilk_bay_topography_nlon_dimname ) + + if (config_Buttermilk_bay_topography_source /= 'latlon_file' .and. & + config_Buttermilk_bay_topography_source /= 'xy_file') then + call mpas_log_write( 'Unexpected value for & + config_Buttermilk_bay_topography_source: ' & + // trim(config_Buttermilk_bay_topography_source), MPAS_LOG_CRIT) + iErr = 1 + return + end if + + if (config_Buttermilk_bay_topography_file == 'none' .and. & + (config_Buttermilk_bay_topography_source == 'latlon_file' .or. & + config_Buttermilk_bay_topography_source == 'xy_file') ) then + call mpas_log_write( 'Validation failed for Buttermilk bay test case. ' & + // 'Invalid filename for config_Buttermilk_bay_topography_file ' & + // config_Buttermilk_bay_topography_file // ' ' & + // config_Buttermilk_bay_topography_source , MPAS_LOG_CRIT) + iErr = 1 + return + end if + + call mpas_log_write( ' in ocn_init_validate_buttermilk_bay '//config_Buttermilk_bay_topography_source ) + + if (config_Buttermilk_bay_topography_source == 'latlon_file' .or. & + config_Buttermilk_bay_topography_source == 'xy_file' ) then + + inputFile = MPAS_io_open(trim(config_Buttermilk_bay_topography_file), & + MPAS_IO_READ, MPAS_IO_NETCDF, iocontext_ptr, ierr=iErr) + if (iErr /= 0) then + call mpas_log_write( 'could not open file '// & + trim(config_Buttermilk_bay_topography_file), MPAS_LOG_CRIT) + return + end if + + call MPAS_io_inq_dim(inputFile, config_Buttermilk_bay_topography_nlat_dimname, nLatTopo, iErr) + call MPAS_io_inq_dim(inputFile, config_Buttermilk_bay_topography_nlon_dimname, nLonTopo, iErr) + + call MPAS_io_close(inputFile, iErr) + + end if + !---------- + + call mpas_log_write( ' Done ocn_init_validate_buttermilk_bay ' ) + !-------------------------------------------------------------------- + + end subroutine ocn_init_validate_ButterMilk_bay!}}} + +!*********************************************************************** + +end module ocn_init_Buttermilk_bay + +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! vim: foldmethod=marker diff --git a/components/mpas-ocean/src/mode_init/mpas_ocn_init_mode.F b/components/mpas-ocean/src/mode_init/mpas_ocn_init_mode.F index 2ffa19a66465..0c73a0f83012 100644 --- a/components/mpas-ocean/src/mode_init/mpas_ocn_init_mode.F +++ b/components/mpas-ocean/src/mode_init/mpas_ocn_init_mode.F @@ -63,8 +63,8 @@ module ocn_init_mode use ocn_init_mixed_layer_eddy use ocn_init_transport_tests use ocn_init_test_sht - use ocn_init_parabolic_bowl + use ocn_init_Buttermilk_bay implicit none private @@ -310,6 +310,7 @@ function ocn_init_mode_run(domain) result(iErr)!{{{ call ocn_init_setup_transport_tests(domain, ierr) call ocn_init_setup_test_sht(domain, ierr) call ocn_init_setup_parabolic_bowl(domain, iErr) + call ocn_init_setup_Buttermilk_bay(domain, iErr) !call ocn_init_setup_TEMPLATE(domain, ierr) call mpas_log_write( ' Completed setup of: ' // trim(config_init_configuration)) @@ -429,6 +430,8 @@ subroutine ocn_init_mode_validate_configuration(configPool, packagePool, ioconte call ocn_init_validate_parabolic_bowl(configPool, packagePool, iocontext, iErr=err_tmp) iErr = ior(iErr, err_tmp) + call ocn_init_validate_Buttermilk_bay(configPool, packagePool, iocontext, iErr=err_tmp) + iErr = ior(iErr, err_tmp) ! call ocn_init_validate_TEMPLATE(configPool, packagePool, iocontext, iErr=err_tmp) ! iErr = ior(iErr, err_tmp) diff --git a/components/mpas-ocean/src/mode_init/mpas_ocn_init_parabolic_bowl.F b/components/mpas-ocean/src/mode_init/mpas_ocn_init_parabolic_bowl.F index 43fb215ed230..a67da7e52acf 100644 --- a/components/mpas-ocean/src/mode_init/mpas_ocn_init_parabolic_bowl.F +++ b/components/mpas-ocean/src/mode_init/mpas_ocn_init_parabolic_bowl.F @@ -33,6 +33,8 @@ module ocn_init_parabolic_bowl use ocn_config use ocn_init_vertical_grids use ocn_init_cell_markers + use ocn_subgrid + use ocn_init_subgrid implicit none private @@ -132,6 +134,9 @@ subroutine ocn_init_setup_parabolic_bowl(domain, iErr)!{{{ real (kind=RKIND):: xshift = 0.0, yshift = 0.0 real (kind=RKIND) :: layerThicknessEdgeAverage real (kind=RKIND), dimension(:,:), allocatable :: rSubgridPoints, sSubgridPoints + real (kind=RKIND), dimension(:), allocatable :: subgridBathymetryValues, subgridAreas + real (kind=RKIND), dimension(:), allocatable :: subgridSshValues + real (kind=RKIND), dimension(:), allocatable :: subgridUValues, subgridVValues real (kind=RKIND), dimension(:), allocatable :: uVelocityAverage, vVelocityAverage integer :: nSubgridCell, nSubgridEdge, nSubgridVertex integer :: nSubgridTriPerSlice @@ -142,6 +147,12 @@ subroutine ocn_init_setup_parabolic_bowl(domain, iErr)!{{{ integer :: i,j real (kind=RKIND) :: deltaZ + integer:: nsubgridCellEdge, iEdgeSegment + real (kind=RKIND), dimension(:,:), allocatable :: cellEdgeBathymetryValues + real (kind=RKIND), dimension(:), allocatable:: dsEdge + integer:: jj + real (kind=RKIND), dimension(:), allocatable:: xSubgridCell, ySubgridCell + iErr = 0 if(config_init_configuration .ne. trim('parabolic_bowl')) return @@ -212,137 +223,180 @@ subroutine ocn_init_setup_parabolic_bowl(domain, iErr)!{{{ block_ptr => domain % blocklist - do while(associated(block_ptr)) - call mpas_pool_get_subpool(block_ptr % structs, 'mesh', meshPool) - call mpas_pool_get_subpool(block_ptr % structs, 'state', statePool) - call mpas_pool_get_subpool(block_ptr % structs, 'verticalMesh', verticalMeshPool) - call mpas_pool_get_subpool(statePool, 'tracers', tracersPool) - - call mpas_pool_get_dimension(meshPool, 'nVertLevels', nVertLevels) ; - call mpas_pool_get_dimension(meshPool, 'nCellsSolve', nCellsSolve) ; - call mpas_pool_get_dimension(meshPool, 'nEdgesSolve', nEdgesSolve) ; - call mpas_pool_get_dimension(meshPool, 'nVerticesSolve', nVerticesSolve) ; - call mpas_pool_get_dimension(meshPool, 'maxEdges', maxEdges) - - call mpas_pool_get_dimension(tracersPool, 'index_temperature', index_temperature) - call mpas_pool_get_dimension(tracersPool, 'index_salinity', index_salinity) + call mpas_pool_get_subpool(block_ptr % structs, 'mesh', meshPool) + call mpas_pool_get_subpool(block_ptr % structs, 'state', statePool) + call mpas_pool_get_subpool(block_ptr % structs, 'verticalMesh', verticalMeshPool) + call mpas_pool_get_subpool(statePool, 'tracers', tracersPool) + + call mpas_pool_get_dimension(meshPool, 'nVertLevels', nVertLevels) ; + call mpas_pool_get_dimension(meshPool, 'nCellsSolve', nCellsSolve) ; + call mpas_pool_get_dimension(meshPool, 'nEdgesSolve', nEdgesSolve) ; + call mpas_pool_get_dimension(meshPool, 'nVerticesSolve', nVerticesSolve) ; + call mpas_pool_get_dimension(meshPool, 'maxEdges', maxEdges) + + call mpas_pool_get_dimension(tracersPool, 'index_temperature', index_temperature) + call mpas_pool_get_dimension(tracersPool, 'index_salinity', index_salinity) + + call mpas_pool_get_array(meshPool, 'xCell', xCell) + call mpas_pool_get_array(meshPool, 'yCell', yCell) + call mpas_pool_get_array(meshPool, 'refBottomDepth', refBottomDepth) + call mpas_pool_get_array(meshPool, 'vertCoordMovementWeights', vertCoordMovementWeights) + call mpas_pool_get_array(meshPool, 'bottomDepth', bottomDepth) + call mpas_pool_get_array(meshPool, 'minLevelCell', minLevelCell) + call mpas_pool_get_array(meshPool, 'maxLevelCell', maxLevelCell) + call mpas_pool_get_array(meshPool, 'nEdgesOnCell', nEdgesOnCell) + call mpas_pool_get_array(meshPool, 'areaCell', areaCell) + + call mpas_pool_get_array(meshPool, 'verticesOnCell', verticesOnCell) + call mpas_pool_get_array(meshPool, 'verticesOnEdge', verticesOnEdge) + call mpas_pool_get_array(meshPool, 'cellsOnEdge', cellsOnEdge) + call mpas_pool_get_array(meshPool, 'cellsOnVertex', cellsOnVertex) + + call mpas_pool_get_array(meshPool, 'fCell', fCell) + call mpas_pool_get_array(meshPool, 'fEdge', fEdge) + call mpas_pool_get_array(meshPool, 'fVertex', fVertex) + + call mpas_pool_get_array(meshPool, 'xEdge', xEdge ) + call mpas_pool_get_array(meshPool, 'yEdge', yEdge ) + call mpas_pool_get_array(meshPool, 'xVertex', xVertex ) + call mpas_pool_get_array(meshPool, 'yVertex', yVertex ) - call mpas_pool_get_array(meshPool, 'xCell', xCell) - call mpas_pool_get_array(meshPool, 'yCell', yCell) - call mpas_pool_get_array(meshPool, 'refBottomDepth', refBottomDepth) - call mpas_pool_get_array(meshPool, 'vertCoordMovementWeights', vertCoordMovementWeights) - call mpas_pool_get_array(meshPool, 'bottomDepth', bottomDepth) - call mpas_pool_get_array(meshPool, 'minLevelCell', minLevelCell) - call mpas_pool_get_array(meshPool, 'maxLevelCell', maxLevelCell) - call mpas_pool_get_array(meshPool, 'nEdgesOnCell', nEdgesOnCell) - call mpas_pool_get_array(meshPool, 'areaCell', areaCell) - - call mpas_pool_get_array(meshPool, 'verticesOnCell', verticesOnCell) - call mpas_pool_get_array(meshPool, 'verticesOnEdge', verticesOnEdge) - call mpas_pool_get_array(meshPool, 'cellsOnEdge', cellsOnEdge) - call mpas_pool_get_array(meshPool, 'cellsOnVertex', cellsOnVertex) - - call mpas_pool_get_array(meshPool, 'fCell', fCell) - call mpas_pool_get_array(meshPool, 'fEdge', fEdge) - call mpas_pool_get_array(meshPool, 'fVertex', fVertex) - - call mpas_pool_get_array(meshPool, 'xEdge', xEdge ) - call mpas_pool_get_array(meshPool, 'yEdge', yEdge ) - call mpas_pool_get_array(meshPool, 'xVertex', xVertex ) - call mpas_pool_get_array(meshPool, 'yVertex', yVertex ) + call mpas_pool_get_array(statePool, 'zMid', zMid, 1) ; - call mpas_pool_get_array(statePool, 'zMid', zMid, 1) ; + call mpas_pool_get_array(statePool, 'ssh', ssh, 1) + call mpas_pool_get_array(meshPool, 'edgeNormalVectors', edgeNormalVectors ) ; + call mpas_pool_get_array(statePool, 'normalVelocity', normalVelocity ) ; - call mpas_pool_get_array(statePool, 'ssh', ssh, 1) - call mpas_pool_get_array(meshPool, 'edgeNormalVectors', edgeNormalVectors ) ; - call mpas_pool_get_array(statePool, 'normalVelocity', normalVelocity ) ; + call mpas_pool_get_array(tracersPool, 'activeTracers', activeTracers, 1) + call mpas_pool_get_array(statePool, 'layerThickness', layerThickness, 1) - call mpas_pool_get_array(tracersPool, 'activeTracers', activeTracers, 1) - call mpas_pool_get_array(statePool, 'layerThickness', layerThickness, 1) + call mpas_pool_get_array(verticalMeshPool, 'refZMid', refZMid) + call mpas_pool_get_array(verticalMeshPool, 'restingThickness', restingThickness) - call mpas_pool_get_array(verticalMeshPool, 'refZMid', refZMid) - call mpas_pool_get_array(verticalMeshPool, 'restingThickness', restingThickness) + ! if config_parabolic_bowl_adjust_domain_center == .true., + ! Adjust center of the mesh so that its center is located at (0,0) + if ( config_parabolic_bowl_adjust_domain_center ) then + xCell = xCell - xshift ; + yCell = yCell - yshift ; - ! if config_parabolic_bowl_adjust_domain_center == .true., - ! Adjust center of the mesh so that its center is located at (0,0) - if ( config_parabolic_bowl_adjust_domain_center ) then - xCell = xCell - xshift ; - yCell = yCell - yshift ; + xEdge = xEdge - xshift ; + yEdge = yEdge - yshift ; - xEdge = xEdge - xshift ; - yEdge = yEdge - yshift ; + xVertex = xVertex - xshift ; + yVertex = yVertex - yshift ; + end if - xVertex = xVertex - xshift ; - yVertex = yVertex - yshift ; - end if + ! Initlialze vector + call mpas_initialize_vectors(meshPool) ; - ! Initlialze edge normal vectors - call mpas_initialize_vectors(meshPool) ; + minLevelCell(:) = 1 + do iCell = 1, nCellsSolve + ! Set up vertical grid + maxLevelCell(iCell) = nVertLevels ; ! sigma coordinates + end do - minLevelCell(:) = 1 - do iCell = 1, nCellsSolve - ! Set up vertical grid - maxLevelCell(iCell) = nVertLevels ; ! sigma coordinates - end do + do iCell = 1, nCellsSolve + + ! Set temperature + activeTracers(index_temperature, :, iCell) = 10.0_RKIND + + ! Set salinity + activeTracers(index_salinity, :, iCell) = 30.0_RKIND + + ! Set Coriolis parameters, if other than zero + fCell(iCell) = config_parabolic_bowl_coriolis_parameter ; + end do + + do iEdge = 1, nEdgesSolve + fEdge(iEdge) = config_parabolic_bowl_coriolis_parameter ; + end do + + do iVertex = 1, nVerticesSolve + fVertex(iVertex) = config_parabolic_bowl_coriolis_parameter ; + end do + + allocate(uVelocityAverage(nEdgesSolve)) + allocate(vVelocityAverage(nEdgesSolve)) + + if (config_use_subgrid_wetting_drying) then + + call ocn_subgrid_init(domain,iErr) + call ocn_init_subgrid_calculations(domain, & + ocn_init_parabolic_bowl_bathymetry, & + ocn_init_parabolic_bowl_velocity, & + ocn_init_parabolic_bowl_ssh, & + config_parabolic_bowl_subgrid_refinement_level, & + config_parabolic_bowl_subgrid_edge_bathymetry_max_pixel, & + config_parabolic_bowl_subgrid_use_thin_layer, & + uVelocityAverage, & + vVelocityAverage, & + iErr) + end if + + ! Find max bottom depth + maxBottomDepth = maxval( bottomDepth ) ; + minBottomDepth = minval( bottomDepth ) ; + call mpas_dmpar_max_real( domain % dminfo, maxBottomDepth, globalMaxBottomDepth ) ; + call mpas_dmpar_min_real( domain % dminfo, minBottomDepth, globalMinBottomDepth ) ; + + ! Set refBottomDepth and refZMid + do k = 1, nVertLevels + refBottomDepth(k) = globalMaxBottomDepth*interfaceLocations(k+1) ; + refZMid(k) = -0.5_RKIND*( interfaceLocations(k+1) + interfaceLocations(k))*globalMaxBottomDepth ; + end do + + ! Set vertCoordMovementWeights + vertCoordMovementWeights(:) = 1.0_RKIND + + ! Set velocity + do iEdge = 1, nEdgesSolve + + if (config_use_subgrid_wetting_drying) then + do k = 1, nVertLevels + normalVelocity(k,iEdge) = uVelocityAverage(iEdge)*edgeNormalVectors(1,iEdge) & + + vVelocityAverage(iEdge)*edgeNormalVectors(2,iEdge) ; + end do + else + call ocn_init_parabolic_bowl_velocity(xEdge(iEdge), yEdge(iEdge), uu, vv) + do k = 1, nVertLevels + normalVelocity(k,iEdge) = uu*edgeNormalVectors(1,iEdge) + vv*edgeNormalVectors(2,iEdge) ; + end do + end if + end do + + ! Set layer thickness and ssh + if (config_use_wetting_drying) then + + do iCell = 1, nCellsSolve + ! Set up vertical grid + maxLevelCell(iCell) = nVertLevels ; ! sigma coordinates + end do do iCell = 1, nCellsSolve - - ! Set temperature - activeTracers(index_temperature, :, iCell) = 10.0_RKIND - - ! Set salinity - activeTracers(index_salinity, :, iCell) = 30.0_RKIND - - ! Set Coriolis parameters - fCell(iCell) = config_parabolic_bowl_coriolis_parameter ; - end do - - do iEdge = 1, nEdgesSolve - fEdge(iEdge) = config_parabolic_bowl_coriolis_parameter ; - end do - - do iVertex = 1, nVerticesSolve - fVertex(iVertex) = config_parabolic_bowl_coriolis_parameter ; - end do - - - ! Find max bottom depth - maxBottomDepth = maxval( bottomDepth ) ; - minBottomDepth = minval( bottomDepth ) ; - call mpas_dmpar_max_real( domain % dminfo, maxBottomDepth, globalMaxBottomDepth ) ; - call mpas_dmpar_min_real( domain % dminfo, minBottomDepth, globalMinBottomDepth ) ; - - ! Set refBottomDepth and refZMid - do k = 1, nVertLevels - refBottomDepth(k) = globalMaxBottomDepth*interfaceLocations(k+1) ; - refZMid(k) = -0.5_RKIND*( interfaceLocations(k+1) + interfaceLocations(k))*globalMaxBottomDepth ; - end do - - ! Set vertCoordMovementWeights - vertCoordMovementWeights(:) = 1.0_RKIND - - ! Set velocity - do iEdge = 1, nEdgesSolve - - call ocn_init_parabolic_bowl_velocity(xEdge(iEdge), yEdge(iEdge), uu, vv) - do k = 1, nVertLevels - normalVelocity(k,iEdge) = uu*edgeNormalVectors(1,iEdge) + vv*edgeNormalVectors(2,iEdge) ; - end do - end do - - ! Set layer thickness and ssh - if (config_use_wetting_drying) then - - do iCell = 1, nCellsSolve - ! - ! make sure depth is thick enough via ssh = TOTAL_DEPTH - bottomDepth - ! add a thin layer of nlayer*config_drying_min_cellhight - ! + ! + ! make sure depth is thick enough via ssh = TOTAL_DEPTH - bottomDepth + ! add a thin layer of nlayer*config_drying_min_cellhight + ! + if (config_use_subgrid_wetting_drying) then + + call ocn_subgrid_ssh_lookup(layerThickness(1,iCell),& + subgridWetVolumeCellTable(:,iCell),& + subgridSshCellTableRange(:,iCell),& + bottomDepth(iCell),& + subgridCellBathymetryMin(iCell),& + ssh(iCell)) + !call ocn_subgrid_layer_thickness_lookup(ssh(iCell), & + ! subgridWetVolumeCellTable(:,iCell), & + ! subgridSshCellTableRange(:,iCell),& + ! bottomDepth(iCell),& + ! LayerThickness(1,iCell)) + + else call ocn_init_parabolic_bowl_bathymetry(xCell(iCell),yCell(iCell),bottomDepth(iCell)) call ocn_init_parabolic_bowl_ssh(xCell(iCell),yCell(iCell),bottomDepth(iCell),ssh(iCell)) ssh(iCell) = - bottomDepth(iCell) + & @@ -359,17 +413,25 @@ subroutine ocn_init_setup_parabolic_bowl(domain, iErr)!{{{ realArgs=(/layerThickness(k,iCell)/)) end if end do - - - do k = 1, maxLevelCell(iCell) - restingThickness(k,iCell) = bottomDepth(iCell)/maxLevelCell(iCell) - end do + endif + + + do k = 1, maxLevelCell(iCell) + restingThickness(k,iCell) = bottomDepth(iCell)/maxLevelCell(iCell) end do - - end if - - block_ptr => block_ptr % next - end do + end do + + end if + + if (config_use_subgrid_wetting_drying) then + do iCell = 1,nCellsSolve + call ocn_subgrid_layer_thickness_lookup(ssh(iCell), & + subgridWetVolumeCellTable(:,iCell), & + subgridSshCellTableRange(:,iCell),& + bottomDepth(iCell),& + subgridLayerThicknessDebug(iCell)) + enddo + endif deallocate(interfaceLocations) !-------------------------------------------------------------------- @@ -436,6 +498,7 @@ subroutine ocn_init_parabolic_bowl_ssh(x, y, bottomDepth, ssh)!{{{ ssh = (sqrtOneMC2/oneMC) - 1.0_RKIND - ((RR**2)/(LL**2))*( (oneMC2/(oneMC**2)) - 1.0_RKIND ) ; ssh = config_parabolic_bowl_b0*ssh ; + !ssh = - bottomDepth + max(ssh + bottomDepth, config_drying_min_cell_height + eps) !-------------------------------------------------------------------- @@ -506,6 +569,7 @@ subroutine ocn_init_validate_parabolic_bowl(configPool, packagePool, iocontext, character (len=StrKIND), pointer :: config_init_configuration integer, pointer :: config_vert_levels, config_parabolic_bowl_vert_levels + integer, pointer :: config_subgrid_table_levels, config_parabolic_bowl_subgrid_table_levels iErr = 0 @@ -519,7 +583,17 @@ subroutine ocn_init_validate_parabolic_bowl(configPool, packagePool, iocontext, if(config_vert_levels <= 0 .and. config_parabolic_bowl_vert_levels > 0) then config_vert_levels = config_parabolic_bowl_vert_levels else if (config_vert_levels <= 0) then - call mpas_log_write( 'Validation failed for para_bowl. Not given a usable value for vertical levels.', MPAS_LOG_CRIT) + call mpas_log_write( 'Validation failed for parabolic_bowl. Not given a usable value for vertical levels.', MPAS_LOG_CRIT) + iErr = 1 + end if + + call mpas_pool_get_config(configPool, 'config_subgrid_table_levels', config_subgrid_table_levels) + call mpas_pool_get_config(configPool, 'config_parabolic_bowl_subgrid_table_levels', config_parabolic_bowl_subgrid_table_levels) + + if (config_subgrid_table_levels <= 0 .and. config_parabolic_bowl_subgrid_table_levels > 0) then + config_subgrid_table_levels = config_parabolic_bowl_subgrid_table_levels + else if (config_subgrid_table_levels <= 0) then + call mpas_log_write( 'Validation failed for parabolic_bowl. Not given a usable value for subgrid table levels.', MPAS_LOG_CRIT) iErr = 1 end if diff --git a/components/mpas-ocean/src/mode_init/mpas_ocn_init_subgrid.F b/components/mpas-ocean/src/mode_init/mpas_ocn_init_subgrid.F new file mode 100644 index 000000000000..37c3e2a273d4 --- /dev/null +++ b/components/mpas-ocean/src/mode_init/mpas_ocn_init_subgrid.F @@ -0,0 +1,941 @@ +! Copyright (c) 2013, Los Alamos National Security, LLC (LANS) +! and the University Corporation for Atmospheric Research (UCAR). +! +! Unless noted otherwise source code is licensed under the BSD license. +! Additional copyright and license information can be found in the LICENSE file +! distributed with this code, or at http://mpas-dev.github.com/license.html +! +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! ocn_init_subgrid +! +!> \brief Calculate the subgrid information +!> \author D. Wirasaet, S. Brus +!> \date May-June 2022 +!> \details +!> This module contains the routines for calculating the information +!> needed to use subgrid corrctions within a forward model run. +!> +!----------------------------------------------------------------------- + +module ocn_init_subgrid + + use mpas_kind_types + use mpas_io_units + use mpas_derived_types + use mpas_pool_routines + use mpas_constants + use mpas_dmpar + + use ocn_constants + use ocn_config + use ocn_subgrid + + use mpas_constants + use mpas_io + use mpas_io_streams + use mpas_stream_manager + + + implicit none + private + save + + !-------------------------------------------------------------------- + ! + ! Public parameters + ! + !-------------------------------------------------------------------- + + !-------------------------------------------------------------------- + ! + ! Public member functions + ! + !-------------------------------------------------------------------- + + public :: ocn_init_subgrid_calculations + + !-------------------------------------------------------------------- + ! + ! Private module variables + ! + !-------------------------------------------------------------------- + + real(kind=RKIND), parameter:: eps = 1.0e-10_RKIND ; + real(kind=RKIND) :: subgrid_thin_layer + + abstract interface + subroutine bathymetry_function(x, y, b) + use mpas_kind_types, only: rkind + implicit none + real (kind=RKIND), intent(in) :: x + real (kind=RKIND), intent(in) :: y + real (kind=RKIND), intent(out) :: b + end subroutine + subroutine velocity_function(x, y, u, v) + use mpas_kind_types, only: rkind + implicit none + real (kind=RKIND), intent(in) :: x + real (kind=RKIND), intent(in) :: y + real (kind=RKIND), intent(out) :: u + real (kind=RKIND), intent(out) :: v + end subroutine + subroutine ssh_function(x, y, b, z) + use mpas_kind_types, only: rkind + implicit none + real (kind=RKIND), intent(in) :: x + real (kind=RKIND), intent(in) :: y + real (kind=RKIND), intent(in) :: b + real (kind=RKIND), intent(out) :: z + end subroutine + end interface + +!*********************************************************************** + +contains + +!*********************************************************************** +! +! routine ocn_init_subgrid_calculations +! +!> \brief Calculations for subgrid look-up tables +!> \author D. Wirasaet and S. Brus +!> \date May-June 2022 +!> \details +!> This routine performs the vertical integration of the wet fraction +!> over the cell, edge, and vertex control volumes to create +!> lookup tables for the relationship between ssh and wet volume per +!> unit area. +! +!----------------------------------------------------------------------- + + subroutine ocn_init_subgrid_calculations(domain, problem_bathymetry, problem_velocity, problem_ssh, & + subgrid_refinement_level, & + subgrid_edge_bathymetry_max_pixel, & + subgrid_use_thin_layer, & + uVelocityAverage, vVelocityAverage, iErr)!{{{ + + implicit none + !-------------------------------------------------------------------- + + type (domain_type), intent(inout) :: domain + integer, intent(out) :: iErr + procedure(bathymetry_function) :: problem_bathymetry + procedure(velocity_function) :: problem_velocity + procedure(ssh_function) :: problem_ssh + integer, intent(in) :: subgrid_refinement_level + logical, intent(in) :: subgrid_edge_bathymetry_max_pixel + logical, intent(in) :: subgrid_use_thin_layer + + real (kind=RKIND), dimension(:), intent(out) :: uVelocityAverage, vVelocityAverage + + type (block_type), pointer :: block_ptr + type (mpas_pool_type), pointer :: meshPool + type (mpas_pool_type), pointer :: statePool + + ! local variables + integer :: iCell, iEdge, iVertex, k, idx + + ! Define dimension pointers + integer, pointer :: nCellsSolve, nEdgesSolve, nVerticesSolve + integer, pointer :: maxEdges + + ! Define variable pointers + integer, dimension(:), pointer :: nEdgesOnCell + integer, dimension(:,:), pointer :: verticesOnCell, verticesOnEdge + integer, dimension(:,:), pointer :: cellsOnEdge, cellsOnVertex + real (kind=RKIND), dimension(:), pointer :: xCell, yCell, bottomDepth + real (kind=RKIND), dimension(:,:), pointer:: edgeNormalVectors + real (kind=RKIND), dimension(:), pointer:: xEdge, yEdge, xVertex, yVertex + + real (kind=RKIND), dimension(:), pointer :: ssh + real (kind=RKIND), dimension(:,:), pointer :: normalVelocity + real (kind=RKIND), dimension(:,:), pointer :: layerThickness + real (kind=RKIND), dimension(:), pointer :: areaCell + + real (kind=RKIND) :: layerThicknessEdgeAverage + real (kind=RKIND), dimension(:,:), allocatable :: rSubgridPoints, sSubgridPoints + real (kind=RKIND), dimension(:), allocatable :: subgridBathymetryValues, subgridAreas + real (kind=RKIND), dimension(:), allocatable :: subgridSshValues + real (kind=RKIND), dimension(:), allocatable :: subgridUValues, subgridVValues + integer :: nSubgridCell, nSubgridEdge, nSubgridVertex + integer :: nSubgridTriPerSlice + integer :: v1, v2 + integer :: c1, c2 + real (kind=RKIND) :: x(3), y(3) + integer :: slice, nSlice + real (kind=RKIND) :: deltaZ + + + + integer:: i, j, jj + integer:: nsubgridCellEdge, iEdgeSegment + real (kind=RKIND):: pi + real (kind=RKIND), dimension(:,:), allocatable :: cellEdgeBathymetryValues + real (kind=RKIND), dimension(:), allocatable:: dsEdge + real (kind=RKIND), dimension(:), allocatable:: xSubgridCell, ySubgridCell + real (kind=RKIND):: bathymetryMin, bathymetryMax + iErr = 0 + + + block_ptr => domain % blocklist + call mpas_pool_get_subpool(block_ptr % structs, 'mesh', meshPool) + call mpas_pool_get_subpool(block_ptr % structs, 'state', statePool) + + call mpas_pool_get_dimension(meshPool, 'nCellsSolve', nCellsSolve) ; + call mpas_pool_get_dimension(meshPool, 'nEdgesSolve', nEdgesSolve) ; + call mpas_pool_get_dimension(meshPool, 'nVerticesSolve', nVerticesSolve) ; + call mpas_pool_get_dimension(meshPool, 'maxEdges', maxEdges) + + call mpas_pool_get_array(meshPool, 'xCell', xCell) + call mpas_pool_get_array(meshPool, 'yCell', yCell) + call mpas_pool_get_array(meshPool, 'bottomDepth', bottomDepth) + call mpas_pool_get_array(meshPool, 'nEdgesOnCell', nEdgesOnCell) + call mpas_pool_get_array(meshPool, 'areaCell', areaCell) + + call mpas_pool_get_array(meshPool, 'verticesOnCell', verticesOnCell) + call mpas_pool_get_array(meshPool, 'verticesOnEdge', verticesOnEdge) + call mpas_pool_get_array(meshPool, 'cellsOnEdge', cellsOnEdge) + call mpas_pool_get_array(meshPool, 'cellsOnVertex', cellsOnVertex) + + call mpas_pool_get_array(meshPool, 'xEdge', xEdge ) + call mpas_pool_get_array(meshPool, 'yEdge', yEdge ) + call mpas_pool_get_array(meshPool, 'xVertex', xVertex ) + call mpas_pool_get_array(meshPool, 'yVertex', yVertex ) + + call mpas_pool_get_array(statePool, 'ssh', ssh, 1) + call mpas_pool_get_array(meshPool, 'edgeNormalVectors', edgeNormalVectors ) ; + call mpas_pool_get_array(statePool, 'normalVelocity', normalVelocity ) ; + call mpas_pool_get_array(statePool, 'layerThickness', layerThickness) ; + + if (subgrid_use_thin_layer) then + subgrid_thin_layer = config_drying_min_cell_height + eps + else + subgrid_thin_layer = 0.0_RKIND + end if + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! Cells + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + print*, "Begin cells", nCellsSolve + + nSubgridTriPerSlice = subgrid_refinement_level**2 + allocate(rSubgridPoints(3,maxEdges*nSubgridTriPerSlice), sSubgridPoints(3,maxEdges*nSubgridTriPerSlice)) + call ocn_init_define_subgrid_points(subgrid_refinement_level, rSubgridPoints, sSubgridPoints) + allocate(subgridBathymetryValues(maxEdges*nSubgridTriPerSlice), subgridAreas(maxEdges*nSubgridTriPerSlice)) + allocate(subgridSshValues(maxEdges*nSubgridTriPerSlice)) + allocate(subgridUValues(maxEdges*nSubgridTriPerSlice)) + allocate(subgridVValues(maxEdges*nSubgridTriPerSlice)) + + allocate(xSubgridCell(maxEdges*nSubgridTriPerSlice)) + allocate(ySubgridCell(maxEdges*nSubgridTriPErSlice)) + xSubgridCell = 0.0_RKIND + ySubgridCell = 0.0_RKIND + + do iCell = 1,nCellsSolve + ! + ! Evaluate subgrid bathymetry at centers of sub-triangles for cell slices + ! (all subdivided triangles for each cell slice are gathered into + ! subgridBathymetryValues and subgridAreas) + !--------------------------------------------------------------- + + nSubgridCell = 0 ! Counter for all subgrid triangles over cell slices + do slice = 1,nEdgesOnCell(iCell) ! Loop over cell slices + + v1 = verticesOnCell(slice,iCell) + if (slice+1 <= nEdgesOnCell(iCell)) then + v2 = verticesOnCell(slice+1,iCell) + else + v2 = verticesOnCell(1,iCell) + endif + + ! Cell slice coordinates + x(1) = xCell(iCell) + y(1) = yCell(iCell) + + x(2) = xVertex(v1) + y(2) = yVertex(v1) + + x(3) = xVertex(v2) + y(3) = yVertex(v2) + + call ocn_init_evaluate_subgrid_data(x, y, nSubgridTriPerSlice, nSubgridCell, rSubgridPoints, sSubgridPoints, & + problem_bathymetry, problem_velocity, problem_ssh, & + subgridBathymetryValues, subgridAreas, subgridSshValues, & + subgridUValues, subgridVValues, xSubgridCell, ysubgridCell) !{{{ + enddo + + ! Evaluate bounds of look-up table range + !--------------------------------------------------------------- + bathymetryMin = maxval( subgridBathymetryValues(1:nSubgridCell) ) + bathymetryMax = minval( subgridBathymetryValues(1:nSubgridCell) ) + + if ( abs(bathymetryMin - bathymetryMax) > 100.0*eps ) then + subgridSshCellTableRange(1,iCell) = -maxval(subgridBathymetryValues(1:nSubgridCell)) + subgrid_thin_layer + subgridSshCellTableRange(2,iCell) = -minval(subgridBathymetryValues(1:nSubgridCell)) + else + ! flat bathy ! + subgridSshCellTableRange(1,iCell) = -bathymetryMin + subgrid_thin_layer + subgridSshCellTableRange(2,iCell) = -bathymetryMin + 2.0*config_drying_min_cell_height + endif + + ! Evaluate subgrid bathymetry + !--------------------------------------------------------------- + bottomDepth(iCell) = sum(subgridBathymetryValues(1:nSubgridCell)*subgridAreas(1:nSubgridCell))/sum(subgridAreas(1:nSubgridCell)) + subgridCellBathymetryMin(iCell) = maxval(subgridBathymetryValues(1:nSubgridCell)) + + ! Vertical integration of wet fraction + !--------------------------------------------------------------- + + call ocn_init_vertical_integration(iCell,subgridSshCellTableRange, nSubgridCell, subgridBathymetryValues, subgridAreas, & + subgridWetVolumeCellTable, subgridWetFractionCellTable) + + ! Evaluate wet layerThickness average + !--------------------------------------------------------------- + call ocn_init_grid_average(nSubgridCell, subgridBathymetryValues, subgridSshValues, subgridAreas, layerThickness(1,iCell)) +! call ocn_init_wet_average_ssh(nSubgridCell, subgridBathymetryValues, subgridSshValues, subgridAreas, ssh(iCell))!{{{ + enddo + + do iCell = 1, nCellsSolve + call ocn_subgrid_ssh_lookup( config_drying_min_cell_height, & + subgridWetVolumeCellTable(:,iCell), & + subgridSshCellTableRange(:,iCell),& + bottomDepth(iCell),& + subgridCellBathymetryMin(iCell),& + subgridSShCellTableRange(3,iCell) ) + + if ( ssh(iCell) < subgridSshCellTableRange(3,iCell) ) then + ssh(iCell) = subgridSshCellTableRange(3,iCell) ; + end if + end do + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! Edges + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + print*, "Begin edges", nEdgesSolve + + nSubgridCellEdge=subgrid_refinement_level + allocate( cellEdgeBathymetryValues(3,nSubgridCellEdge) ) + allocate( dsEdge(nSubgridCellEdge) ) + cellEdgeBathymetryValues = -99999 ; + + + do iEdge = 1,nEdgesSolve + !-------------------------------------------------------------- + ! Evaluate subgrid bathymetry at centers of sub-triangles for edge slices + ! (all subdivided triangles for each edge slice are gathered into + ! subgridBathymetryValues and subgridAreas) + !--------------------------------------------------------------- + + nSlice = 0 + do slice = 1,2 + if (cellsOnEdge(slice,iEdge) <= nCellsSolve) then + nSlice = nSlice + 1 + endif + enddo + + nSubgridEdge = 0 ! Counter for all subgrid triangles over edge slices + do slice = 1,nSlice ! Loop over edge slices + + ! Edge slice coordinates + x(1) = xVertex(verticesOnEdge(1,iEdge)) + y(1) = yVertex(verticesOnEdge(1,iEdge)) + + x(2) = xVertex(verticesOnEdge(2,iEdge)) + y(2) = yVertex(verticesOnEdge(2,iEdge)) + + x(3) = xCell(cellsOnEdge(slice,iEdge)) + y(3) = yCell(cellsOnEdge(slice,iEdge)) + + + call ocn_init_evaluate_subgrid_data(x, y, nSubgridTriPerSlice, nSubgridEdge, rSubgridPoints, sSubgridPoints, & + problem_bathymetry, problem_velocity, problem_ssh, & + subgridBathymetryValues, subgridAreas, subgridSshValues, subgridUValues, subgridVValues, & + xSubgridCell, ySubgridCell) + + enddo + + ! Evaluate velocity average + !--------------------------------------------------------------- + call ocn_init_grid_average(nSubgridCell, subgridBathymetryValues, subgridSshValues, subgridAreas, layerThicknessEdgeAverage, & + subgridUValues, subgridVValues, uVelocityAverage(iEdge), vVelocityAverage(iEdge)) + + if ( .NOT. subgrid_edge_bathymetry_max_pixel) then + ! Evaluate bounds of look-up table range + !--------------------------------------------------------------- + subgridSshEdgeTableRange(1,iEdge) = -maxval(subgridBathymetryValues(1:nSubgridEdge)) + subgrid_thin_layer + subgridSshEdgeTableRange(2,iEdge) = -minval(subgridBathymetryValues(1:nSubgridEdge)) + + ! Evaluate subgrid bathymetry + !--------------------------------------------------------------- + subgridEdgeBathymetryMean(iEdge) = sum(subgridBathymetryValues(1:nSubgridEdge))/real(nSubgridEdge,RKIND) + subgridEdgeBathymetryMin(iEdge) = maxval(subgridBathymetryValues(1:nSubgridEdge)) + + ! Vertical integration of wet fraction + !--------------------------------------------------------------- + call ocn_init_vertical_integration(iEdge,subgridSshEdgeTableRange, nSubgridEdge, subgridBathymetryValues, subgridAreas, & + subgridWetVolumeEdgeTable, subgridWetFractionEdgeTable) + else + ! DW: Use the higher values of the pair of subcells along the + ! cell edge + cellEdgeBathymetryValues(1,:) = subgridBathymetryValues(1:2*nSubgridCellEdge - 1:2) + if ( nslice > 1 ) then + cellEdgeBathymetryValues(2,:) = & + subgridBathymetryValues(nSubgridTriPerSlice+2*nsubgridCellEdge - 1:nSubgridTriPerSlice+1:-2) + else + cellEdgeBathymetryValues(2,:) = cellEdgeBathymetryValues(1,:) + endif + + do iEdgeSegment = 1, nSubgridCellEdge + cellEdgeBathymetryValues(3,iEdgeSegment) = minval(cellEdgeBathymetryValues(1:2,iEdgeSegment) ) + end do + dsEdge(:) = sqrt( (x(2) - x(1))*(x(2) - x(1)) + (y(2) - y(1))*(y(2)- y(1)) )/nSubgridCellEdge + + + ! Evaluate bounds of look-up table range + !--------------------------------------------------------------- + subgridSshEdgeTableRange(1,iEdge) = -maxval(CellEdgeBathymetryValues(3,1:nSubgridCellEdge)) + subgrid_thin_layer + subgridSshEdgeTableRange(2,iEdge) = -minval(CellEdgeBathymetryValues(3,1:nSubgridCellEdge)) + + ! Evaluate bounds of look-up table range + !--------------------------------------------------------------- + subgridEdgeBathymetryMean(iEdge) = sum(cellEdgeBathymetryValues(3,1:nSubgridCellEdge))/real(nSubgridCellEdge,RKIND) + subgridEdgeBathymetryMin(iEdge) = maxval(cellEdgeBathymetryValues(3,1:nSubgridCellEdge)) + + ! Vertical integration of wet fraction + !--------------------------------------------------------------- + call ocn_init_vertical_integration( iEdge, subgridSshEdgeTableRange, & + nSubgridCellEdge, cellEdgeBathymetryValues(3,:), dsEdge, subgridWetVolumeEdgeTable, subgridWetFractionEdgeTable ) + endif + + end do + + ! find an ssh value corresponding to drying_min_cell-height + ! of each edge + do iEdge = 1,nEdgesSolve + call ocn_subgrid_ssh_lookup( config_drying_min_cell_height, & + subgridWetVolumeEdgeTable(:,iEdge),& + subgridSshEdgeTableRange(:,iEdge),& + subgridEdgeBathymetryMean(iEdge),& + subgridEdgeBathymetryMin(iEdge),& + subgridSshEdgeTableRange(3,iEdge) ) + end do + + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! Vertex + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + print*, "Begin vertex", nVerticesSolve + +vertex: do iVertex = 1,nVerticesSolve + + ! Evaluate subgrid bathymetry at centers of sub-triangles for vertex triangle + !--------------------------------------------------------------- + + nSlice = 0 + do slice = 1,3 + if (cellsOnVertex(slice,iVertex) <= nCellsSolve) then + nSlice = nSlice + 1 + endif + enddo + + if (nSlice < 3) then + cycle vertex + endif + + nSubgridVertex = 0 ! Counter for all subgrid triangles over edge slices + do slice = 1,nSlice + + c1 = cellsOnVertex(slice,iVertex) + if (slice < 3) then + c2 = cellsOnVertex(slice+1,iVertex) + else + c2 = cellsOnVertex(1,iVertex) + endif + + ! Vertex slice coordinates + x(1) = xCell(c1) + y(1) = yCell(c1) + + x(2) = xCell(c2) + y(2) = yCell(c2) + + x(3) = xVertex(iVertex) + y(3) = yVertex(iVertex) + + call ocn_init_evaluate_subgrid_data(x, y, nSubgridTriPerSlice, nSubgridVertex, rSubgridPoints, sSubgridPoints, & + problem_bathymetry, problem_velocity, problem_ssh, & + subgridBathymetryValues, subgridAreas) + enddo + + ! Evaluate bounds of look-up table range + !--------------------------------------------------------------- + subgridSshVertexTableRange(1,iVertex) = -maxval(subgridBathymetryValues(1:nSubgridVertex)) + subgrid_thin_layer + subgridSshVertexTableRange(2,iVertex) = -minval(subgridBathymetryValues(1:nSubgridVertex)) + + ! Evaluate subgrid bathymetry + !--------------------------------------------------------------- + subgridVertexBathymetryMean(iVertex) = sum(subgridBathymetryValues(1:nSubgridVertex)*subgridAreas(1:nSubgridVertex))/sum(subgridAreas(1:nSubgridVertex)) + subgridVertexBathymetryMin(iVertex) = maxval(subgridBathymetryValues(1:nSubgridVertex)) + + ! Vertical integration of wet fraction + !--------------------------------------------------------------- + call ocn_init_vertical_integration(iVertex,subgridSshVertexTableRange, nSubgridVertex, subgridBathymetryValues, subgridAreas, & + subgridWetVolumeVertexTable, subgridWetFractionVertexTable) + + enddo vertex + + ! find an ssh value corresponding to drying_min_cell-height + ! of each edge + do iVertex = 1,nVerticesSolve + + call ocn_subgrid_ssh_lookup( config_drying_min_cell_height, & + subgridWetVolumeVertexTable(:,iVertex),& + subgridSshVertexTableRange(:,iVertex),& + subgridVertexBathymetryMean(iVertex),& + subgridVertexBathymetryMin(iVertex),& + subgridSshVertexTableRange(3,iVertex) ) + end do + + return ; + end subroutine ocn_init_subgrid_calculations!}}} + +!*********************************************************************** +! +! routine ocn_init_define_subgrid_points +! +!> \brief Define subgrid points on reference triangle +!> \author Steven Brus +!> \date November 2022 +!> \details Gives the r and s coordinates of the subgrid triangle vertices +!> +! +!----------------------------------------------------------------------- + + subroutine ocn_init_define_subgrid_points(nSubgridLevel, rSubgridPoints, sSubgridPoints)!{{{ + + implicit none + + integer, intent(in) :: nSubgridLevel + real (kind=RKIND), dimension(:,:), intent(inout) :: rSubgridPoints, sSubgridPoints + + integer :: i,j,k + integer :: nSubgridTri + real (kind=RKIND) :: dx + real (kind=RKIND), dimension(:), allocatable :: xPoints + + !-------------------------------------------------------------------- + + ! Equi-spaced nodes on -1,+1 + allocate(xPoints(nSubgridLevel+1)) + dx = 2.0_RKIND/real(nSubgridLevel,RKIND) + xPoints(1) = -1.0_RKIND + do i = 2,nSubgridLevel+1 + xPoints(i) = xPoints(i-1)+dx + enddo + + ! * + ! |\ + ! | \ + ! | \ + ! | \ + ! s *----* + ! |\ u |\ + ! | \ | \ + ! | \ | \ + ! | l \| l \ + ! *----*----* + ! r + + ! Trianglulate tensor product of equi-spaced nodes on reference triangle + ! r - horizontal coordinate (i index), s - vertical coordinate (j index) + nSubgridTri = 0 + do j = 1,nSubgridLevel + do i = 1,nSubgridLevel+1 - j + + nSubgridTri = nSubgridTri+1 + + ! lower triangle in pair (see l in triangle above) + rSubgridPoints(1,nSubgridTri) = xPoints(i) + sSubgridPoints(1,nSubgridTri) = xPoints(j) + + rSubgridPoints(2,nSubgridTri) = xPoints(i+1) + sSubgridPoints(2,nSubgridTri) = xPoints(j) + + rSubgridPoints(3,nSubgridTri) = xPoints(i) + sSubgridPoints(3,nSubgridTri) = xPoints(j+1) + + ! upper triangle in pair (see u in triangle above, doesn't occur next to hypotenuse) + if (i < nSubgridLevel+1 - j) then + + nSubgridTri = nSubgridTri + 1 + + rSubgridPoints(1,nSubgridTri) = xPoints(i+1) + sSubgridPoints(1,nSubgridTri) = xPoints(j) + + rSubgridPoints(2,nSubgridTri) = xPoints(i+1) + sSubgridPoints(2,nSubgridTri) = xPoints(j+1) + + rSubgridPoints(3,nSubgridTri) = xPoints(i) + sSubgridPoints(3,nSubgridTri) = xPoints(j+1) + + endif + + enddo + enddo + + print*, "exit ocn_init_define_subgrid_points" + + + !-------------------------------------------------------------------- + + end subroutine ocn_init_define_subgrid_points!}}} + +!*********************************************************************** +! +! routine ocn_init_evaluate_subgrid_data +! +!> \brief Evaluate subgrid infromation +!> \author Steven Brus +!> \date November 2022 +!> \details Evaluate data on subgrid triangles for a given triangluar grid cell region +!> +! +!----------------------------------------------------------------------- + + subroutine ocn_init_evaluate_subgrid_data(xTri, yTri, nSubgridTri, nSubgridCV, rSubgridPoints, sSubgridPoints, & + problem_bathymetry, problem_velocity, problem_ssh, & + subgridBathymetryValues, subgridAreas, & + subgridSshValues, subgridUValues, subgridVValues, & + xSubgridCell, ySubgridCell) !{{{ + + + implicit none + real (kind=RKIND), intent(inout) :: xTri(3), yTri(3) + integer, intent(in) :: nSubgridTri + integer, intent(inout) :: nSubgridCV + + procedure(bathymetry_function) :: problem_bathymetry + procedure(velocity_function) :: problem_velocity + procedure(ssh_function) :: problem_ssh + + real (kind=RKIND), dimension(:,:), intent(in) :: rSubgridPoints, sSubgridPoints + real (kind=RKIND), dimension(:), intent(inout) :: subgridBathymetryValues + real (kind=RKIND), dimension(:), intent(inout) :: subgridAreas + real (kind=RKIND), dimension(:), intent(inout),optional :: subgridSshValues + real (kind=RKIND), dimension(:), intent(inout),optional :: subgridUValues, subgridVValues + + real (kind=RKIND), dimension(:), intent(inout), optional :: xSubgridCell, ySubgridCell + + real (kind=RKIND) :: rCenter, sCenter + real (kind=RKIND) :: xCenter, yCenter + real (kind=RKIND) :: xSubgridPoints(3), ySubgridPoints(3) + real (kind=RKIND) :: x, y + real (kind=RKIND) :: area + + + integer :: iPt,i + + !-------------------------------------------------------------------- + + ! Coordinates of physical triangle + ! (ensure counter-clockwise numering) + call ocn_init_tri_area(xTri(:), yTri(:), area) + if (area < 0.0_RKIND) then + x = xTri(1) + y = yTri(1) + xTri(1) = xTri(2) + yTri(1) = yTri(2) + xTri(2) = x ; + yTri(2) = y ; + area = abs(area) ; + endif + + do iPt = 1,nSubgridTri + + ! Counter over all subcells within cell/edge/vertex control volume + nSubgridCV = nSubgridCV + 1 + + ! Center sub-triangle (on reference triangle) + rCenter = sum(rSubgridPoints(:,iPt))/3.0_RKIND + sCenter = sum(sSubgridPoints(:,iPt))/3.0_RKIND + + ! Transformation of sub-triangle center to physical coordinates + call ocn_init_tri_coordinate_transform(rCenter, sCenter, xTri, yTri, xCenter, yCenter) + + ! Evaluate bathymetry + call problem_bathymetry(xCenter, yCenter, subgridBathymetryValues(nSubgridCV)) + + ! Transformation of sub-triangle vertices to physical coordinates + do i = 1,3 + call ocn_init_tri_coordinate_transform(rSubgridPoints(i,iPt), sSubgridPoints(i,iPt), xTri, yTri, xSubgridPoints(i), ySubgridPoints(i)) + enddo + + ! Calculate area of sub-triangle + call ocn_init_tri_area(xSubgridPoints(:), ySubgridPoints(:), subgridAreas(nSubgridCV)) + + ! Optionally evalulate ssh ! + if (present(subgridsshValues)) then + call problem_ssh(xCenter, yCenter, subgridBathymetryValues(nSubgridCV), subgridSshValues(nSubgridCV)) + endif + + if (present(subgridUValues).and.present(subgridVValues)) then + call problem_velocity(xCenter, yCenter, subgridUValues(nSubgridCV), subgridVValues(nSubgridCV)) + endif + + if ( present(xSubgridCell) ) then + xSubgridCell(nSubgridCV) = xCenter ; + endif + + if ( present(ySubgridCell) ) then + ySubgridCell(nSubgridCV) = yCenter ; + end if + enddo + + !-------------------------------------------------------------------- + + end subroutine ocn_init_evaluate_subgrid_data!}}} + +!*********************************************************************** +! +! routine ocn_init_tri_area +! +!> \brief Compute triangle area +!> \author Steven Brus +!> \date November 2022 +!> \details Compute area of triangle given vertex coordinates +!> +! +!----------------------------------------------------------------------- + + subroutine ocn_init_tri_area(x, y, area)!{{{ + + implicit none + + real (kind=RKIND), intent(in) :: x(3), y(3) + real (kind=RKIND), intent(out) :: area + + !-------------------------------------------------------------------- + + area = 0.5_RKIND*(x(2)*y(3) - y(2)*x(3) - x(1)*y(3) + y(1)*x(3) + x(1)*y(2) - y(1)*x(2)) + + !-------------------------------------------------------------------- + + end subroutine ocn_init_tri_area!}}} + +!*********************************************************************** +! +! routine ocn_init_tri_coordinate_transform +! +!> \brief Transform reference triangle coordinates to mesh coordinates +!> \author Steven Brus +!> \date November 2022 +!> \details Evaluate r,s reference coordinates in x,y mesh +!> +! +!----------------------------------------------------------------------- + + subroutine ocn_init_tri_coordinate_transform(r, s, xTri, yTri, x, y)!{{{ + + implicit none + + real (kind=RKIND), intent(in) :: r, s + real (kind=RKIND), intent(in) :: xTri(3), yTri(3) + real (kind=RKIND), intent(out) :: x, y + + !-------------------------------------------------------------------- + + ! Transformation of sub-triangle center to physical coordinates + x = 0.5_RKIND*(-(r+s)*xTri(1) + (1.0_RKIND+r)*xTri(2) + (1.0_RKIND+s)*xTri(3)) + y = 0.5_RKIND*(-(r+s)*yTri(1) + (1.0_RKIND+r)*yTri(2) + (1.0_RKIND+s)*yTri(3)) + + !-------------------------------------------------------------------- + + end subroutine ocn_init_tri_coordinate_transform!}}} + +!*********************************************************************** +! +! routine ocn_init_vertical_integration +! +!> \brief Compute the wet volume per unit area lookup table +!> \author Steven Brus +!> \date November 2022 +!> \details Integrate the wet fraction over discrete ssh values +!> +! +!----------------------------------------------------------------------- + + subroutine ocn_init_vertical_integration(iCV, subgridSshTableRange, nSubgridCV, subgridBathymetryValues, subgridAreas, & + subgridWetVolumeTable, subgridWetFractionTable)!{{{ + + implicit none + + integer, intent(in) :: iCV + real (kind=RKIND), dimension(:,:), intent(in) :: subgridSshTableRange + integer, intent(in) :: nSubgridCV + real (kind=RKIND), dimension(:), intent(in) :: subgridBathymetryValues, subgridAreas + real (kind=RKIND), dimension(:,:), intent(inout) :: subgridWetVolumeTable + real (kind=RKIND), dimension(:,:), intent(inout) :: subgridWetFractionTable + + real (kind=RKIND) :: deltaZ, ssh, pVal + integer :: lev, tri + + + !-------------------------------------------------------------------- + + deltaZ = (subgridSshTableRange(2,iCV)-subgridSshTableRange(1,iCV))/real(nSubgridTableLevels-1,RKIND); + + subgridWetVolumeTable(1,iCV) = subgrid_thin_layer + ssh = subgridSshTableRange(1,iCV) + deltaZ + subgridWetVolumeTable(2,iCV) = subgridWetVolumeTable(1,iCV)*sum(subgridAreas(1:nSubgridCV)) + subgridWetFractionTable(1,iCV) = 0.0_RKIND + do lev = 2,nSubgridTableLevels + do tri = 1,nSubgridCV + + pVal = 0.0_RKIND + if (subgridBathymetryValues(tri) + ssh >= 0.0_RKIND) then + pVal = 1.0_RKIND + endif + + subgridWetVolumeTable(lev,iCV) = subgridWetVolumeTable(lev,iCV) + pVal*deltaZ*subgridAreas(tri) + subgridWetFractionTable(lev,iCV) = subgridWetFractionTable(lev,iCV) + pVal*subgridAreas(tri) + + enddo + + if (lev < nSubgridTableLevels) then + subgridWetVolumeTable(lev+1,iCV) = subgridWetVolumeTable(lev,iCV) + endif + + subgridWetVolumeTable(lev,iCV) = subgridWetVolumeTable(lev,iCV)/sum(subgridAreas(1:nSubgridCV)) + subgridWetFractionTable(lev,iCV) = subgridWetFractionTable(lev,iCV)/sum(subgridAreas(1:nSubgridCV)) + + ssh = ssh + deltaZ + enddo + + !-------------------------------------------------------------------- + + end subroutine ocn_init_vertical_integration!}}} + + +!*********************************************************************** +! +! routine ocn_init_grid_average +! +!> \brief Compute thickness and velocity averages over wet area +!> \author Steven Brus +!> \date November 2022 +!> \details Compute the thickness and velocity averages over an area +!> based on the subgrid wet area +! +!----------------------------------------------------------------------- + + subroutine ocn_init_grid_average(nSubgridCV, subgridBathymetryValues, subgridSshValues, subgridAreas, subgridThicknessAverage, & + subgridUValues, subgridVValues, subgridUAverage, subgridVAverage)!{{{ + + implicit none + + integer, intent(in) :: nSubgridCV + real (kind=RKIND), dimension(:), intent(in) :: subgridBathymetryValues, subgridAreas + real (kind=RKIND), dimension(:), intent(in) :: subgridSshValues + real (kind=RKIND), intent(inout) :: subgridThicknessAverage + real (kind=RKIND), dimension(:), intent(in), optional :: subgridUValues, subgridVValues + real (kind=RKIND), intent(inout), optional :: subgridUAverage, subgridVAverage + + real (kind=RKIND) :: deltaZ, ssh, pVal + real (kind=RKIND) :: averageDepth, cellArea, layerThicknessValue + real (kind=RKIND) :: H_int + real (kind=RKIND) :: HU_int, HV_int + logical :: computeVelAverage + integer :: lev, tri + + + !-------------------------------------------------------------------- + + averageDepth = sum(subgridBathymetryValues(1:nSubgridCV))/real(nSubgridCV,RKIND) + + computeVelAverage = .false. + if (present(subgridUValues) .and. present(subgridVValues) .and. & + present(subgridUAverage) .and. present(subgridVAverage)) then + computeVelAverage = .true. + endif + + H_int = 0.0_RKIND + cellArea = 0.0_RKIND + HU_int = 0.0_RKIND + HV_int = 0.0_RKIND + do tri = 1,nSubgridCV + + layerThicknessValue = subgridBathymetryValues(tri) + subgridSshValues(tri) + if (layerThicknessValue <= config_drying_min_cell_height + eps) then + layerThicknessValue = config_drying_min_cell_height + eps + endif + H_int = H_int + layerThicknessValue*subgridAreas(tri) + cellArea = cellArea + subgridAreas(tri) + + if (computeVelAverage) then + HU_int = HU_int + layerThicknessValue*subgridUValues(tri)*subgridAreas(tri) + HV_int = HV_int + layerThicknessValue*subgridVValues(tri)*subgridAreas(tri) + endif + enddo + + if (computeVelAverage) then + subgridUAverage = HU_int/H_int + subgridVAverage = HV_int/H_int + endif + subgridThicknessAverage = H_int/cellArea + + + !-------------------------------------------------------------------- + + end subroutine ocn_init_grid_average!}}} + +!*********************************************************************** +! +! routine ocn_init_wet_average_ssh +! +!> \brief +!> \author Steven Brus +!> \date November 2022 +!> \details +!> +! +!----------------------------------------------------------------------- + subroutine ocn_init_wet_average_ssh( nSubgridCV, subgridBathymetryValues, subgridSshValues, subgridAreas, sshWetAverage)!{{{ + implicit none + + integer, intent(in) :: nSubgridCV + real (kind=RKIND), dimension(:), intent(in) :: subgridBathymetryValues, subgridAreas + real (kind=RKIND), dimension(:), intent(in) :: subgridSshValues + real (kind=RKIND), intent(inout) :: sshWetAverage + + real (kind=RKIND) :: deltaZ, ssh, pVal + real (kind=RKIND) :: averageDepth, wetArea, layerThicknessValue + integer :: lev, tri + + + sshWetAverage = 0.0_RKIND + wetArea = 0.0_RKIND + + do tri = 1, nsubgridCV + layerThicknessValue = subgridBathymetryValues(tri) + subgridSshValues(tri) + + if ( layerThicknessValue > subgrid_thin_layer) then + sshWetAverage = sshWetAverage + subgridSshValues(tri)*subgridAreas(tri) + wetArea = wetArea + subgridAreas(tri) + endif + end do + + if ( WetArea > 0.0_RKIND ) then + sshWetAverage = sshWetAverage/WetArea ; + else + sshWetAverage = -maxval(subgridBathymetryValues(1:nsubgridCV)) + subgrid_thin_layer + endif + + return ; + end subroutine ocn_init_wet_average_ssh + + +end module ocn_init_subgrid + +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! vim: foldmethod=marker diff --git a/components/mpas-ocean/src/ocean.cmake b/components/mpas-ocean/src/ocean.cmake index 3c0614ee38d5..8866a8cea3d6 100644 --- a/components/mpas-ocean/src/ocean.cmake +++ b/components/mpas-ocean/src/ocean.cmake @@ -32,6 +32,7 @@ list(APPEND RAW_SOURCES core_ocean/mode_forward/mpas_ocn_time_integration_split.F core_ocean/mode_forward/mpas_ocn_time_integration_si.F core_ocean/mode_forward/mpas_ocn_time_integration_lts.F + core_ocean/mode_forward/mpas_ocn_time_integration_fblts.F core_ocean/mode_forward/mpas_ocn_time_integration_split_ab2.F core_ocean/mode_analysis/mpas_ocn_analysis_mode.F @@ -112,6 +113,7 @@ list(APPEND RAW_SOURCES core_ocean/shared/mpas_ocn_vel_tidal_potential.F core_ocean/shared/mpas_ocn_stokes_drift.F core_ocean/shared/mpas_ocn_manufactured_solution.F + core_ocean/shared/mpas_ocn_subgrid.F ) set(OCEAN_DRIVER diff --git a/components/mpas-ocean/src/shared/Makefile b/components/mpas-ocean/src/shared/Makefile index 5c5bc21cff12..35017551c754 100644 --- a/components/mpas-ocean/src/shared/Makefile +++ b/components/mpas-ocean/src/shared/Makefile @@ -69,6 +69,7 @@ OBJS = mpas_ocn_init_routines.o \ mpas_ocn_time_average_coupled.o \ mpas_ocn_framework_forcing.o \ mpas_ocn_time_varying_forcing.o \ + mpas_ocn_subgrid.o \ mpas_ocn_wetting_drying.o \ mpas_ocn_vel_tidal_potential.o \ mpas_ocn_vel_forcing_topographic_wave_drag.o \ @@ -84,7 +85,7 @@ mpas_ocn_init_routines.o: mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_mesh.o mpas_ocn_tendency.o: mpas_ocn_high_freq_thickness_hmix_del2.o mpas_ocn_tracer_surface_restoring.o mpas_ocn_thick_surface_flux.o mpas_ocn_tracer_short_wave_absorption.o mpas_ocn_tracer_advection.o mpas_ocn_tracer_hmix.o mpas_ocn_tracer_nonlocalflux.o mpas_ocn_surface_bulk_forcing.o mpas_ocn_surface_land_ice_fluxes.o mpas_ocn_tracer_surface_flux_to_tend.o mpas_ocn_tracer_interior_restoring.o mpas_ocn_tracer_exponential_decay.o mpas_ocn_tracer_ideal_age.o mpas_ocn_tracer_TTD.o mpas_ocn_vmix.o mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_frazil_forcing.o mpas_ocn_tidal_forcing.o mpas_ocn_tracer_ecosys.o mpas_ocn_tracer_DMS.o mpas_ocn_tracer_MacroMolecules.o mpas_ocn_tracer_CFC.o mpas_ocn_diagnostics.o mpas_ocn_wetting_drying.o mpas_ocn_vel_self_attraction_loading.o mpas_ocn_vel_tidal_potential.o mpas_ocn_mesh.o mpas_ocn_diagnostics_variables.o mpas_ocn_thick_hadv.o mpas_ocn_thick_vadv.o mpas_ocn_vel_hadv_coriolis.o mpas_ocn_vel_pressure_grad.o mpas_ocn_vel_vadv.o mpas_ocn_vel_hmix.o mpas_ocn_vel_forcing.o mpas_ocn_manufactured_solution.o -mpas_ocn_diagnostics.o: mpas_ocn_thick_ale.o mpas_ocn_equation_of_state.o mpas_ocn_gm.o mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_mesh.o mpas_ocn_diagnostics_variables.o mpas_ocn_surface_land_ice_fluxes.o mpas_ocn_vertical_advection.o mpas_ocn_submesoscale_eddies.o +mpas_ocn_diagnostics.o: mpas_ocn_thick_ale.o mpas_ocn_equation_of_state.o mpas_ocn_gm.o mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_mesh.o mpas_ocn_diagnostics_variables.o mpas_ocn_surface_land_ice_fluxes.o mpas_ocn_vertical_advection.o mpas_ocn_submesoscale_eddies.o mpas_ocn_subgrid.o mpas_ocn_diagnostics_variables.o: mpas_ocn_config.o @@ -184,7 +185,7 @@ mpas_ocn_surface_land_ice_fluxes.o: mpas_ocn_constants.o mpas_ocn_config.o mpas_ mpas_ocn_frazil_forcing.o: mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_mesh.o mpas_ocn_diagnostics_variables.o mpas_ocn_equation_of_state.o -mpas_ocn_tidal_forcing.o: mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_equation_of_state.o mpas_ocn_diagnostics_variables.o mpas_ocn_mesh.o +mpas_ocn_tidal_forcing.o: mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_equation_of_state.o mpas_ocn_diagnostics_variables.o mpas_ocn_mesh.o mpas_ocn_subgrid.o mpas_ocn_transport_tests.o: mpas_ocn_config.o @@ -238,6 +239,8 @@ mpas_ocn_stokes_drift.o: mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_diagnos mpas_ocn_manufactured_solution.o: mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_mesh.o +mpas_ocn_subgrid.o: mpas_ocn_diagnostics_variables.o mpas_ocn_config.o mpas_ocn_constants.o + clean: $(RM) *.o *.i *.mod *.f90 diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F index 7b8e5154495b..cdb398f43ff7 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F @@ -38,6 +38,7 @@ module ocn_diagnostics use ocn_mesh use ocn_surface_land_ice_fluxes use ocn_vertical_advection + use ocn_subgrid implicit none private @@ -181,7 +182,7 @@ subroutine ocn_diagnostic_solve(dt, statePool, forcingPool, meshPool, verticalMe call mpas_pool_get_array(forcingPool, 'landIceDraft', landIceDraft) call mpas_pool_get_array(forcingPool, 'landIceFloatingMask', landIceFloatingMask) call mpas_pool_get_array(forcingPool, 'landIceFraction', landIceFraction) - if (landIceFloatingMask(1) == -1) then + if (nCellsAll.gt.0 .AND. landIceFloatingMask(1) == -1) then call mpas_log_write('landIceFloatingMask contains the default value which likely indicates that this field is missing in the initial condition file (e.g. because it is meant for an older E3SM version).', & MPAS_LOG_CRIT) endif @@ -239,10 +240,19 @@ subroutine ocn_diagnostic_solve(dt, statePool, forcingPool, meshPool, verticalMe ! endif #endif + ! + ! Z-coordinates + ! This section must be placed in the code before computing the density. + ! + ! inputs : layerThickness + ! outputs : zMid, zTop, ssh) + call ocn_diagnostic_solve_z_coordinates(layerThickness, zMid, zTop, ssh) + ! inputs: layerThickness, normalVelocity ! output: layerThickEdgeMean, layerThickEdgeDrag, layerThickEdgeFlux call ocn_diagnostic_solve_layerThicknessEdge(normalVelocity, & - layerThickness, restingThickness) + layerThickness, restingThickness, & + ssh) ! inputs: normalVelocity ! outputs: relativeVorticity, circulation @@ -337,13 +347,6 @@ subroutine ocn_diagnostic_solve(dt, statePool, forcingPool, meshPool, verticalMe call ocn_diagnostic_solve_surface_pressure(forcingPool, atmosphericPressure, & seaIcePressure, surfacePressure) - ! - ! Z-coordinates - ! This section must be placed in the code before computing the density. - ! - ! inputs : layerThickness - ! outputs : zMid, zTop, ssh) - call ocn_diagnostic_solve_z_coordinates(layerThickness, zMid, zTop, ssh) ! ! equation of state @@ -807,7 +810,8 @@ end subroutine ocn_diagnostic_solve_wctEdge!}}} !----------------------------------------------------------------------- subroutine ocn_diagnostic_solve_layerThicknessEdge(normalVelocity, & - layerThickness, restingThickness)!{{{ + layerThickness, restingThickness, & + ssh)!{{{ !----------------------------------------------------------------- ! input variables @@ -817,6 +821,8 @@ subroutine ocn_diagnostic_solve_layerThicknessEdge(normalVelocity, & normalVelocity, &!< [in] transport layerThickness, &!< [in] layer thickness at cell center restingThickness !< [in] initial layer thickness at cell center + real (kind=RKIND), dimension(:), intent(in) :: & + ssh !< [in] sea surface height at cell center !----------------------------------------------------------------- ! output variables @@ -879,64 +885,79 @@ subroutine ocn_diagnostic_solve_layerThicknessEdge(normalVelocity, & case (thickEdgeFluxCenter) ! Use centered (mean) thickness as flux value + if ( config_use_subgrid_wetting_drying ) then + + call ocn_subgrid_layerThickEdgeFlux_center(ssh, layerThickEdgeFlux) + + else + #ifdef MPAS_OPENACC - !$acc parallel loop collapse(2) & - !$acc present(layerThickEdgeFlux, layerThickEdgeMean) + !$acc parallel loop collapse(2) & + !$acc present(layerThickEdgeFlux, layerThickEdgeMean) #else - !$omp parallel - !$omp do schedule(runtime) private(k) + !$omp parallel + !$omp do schedule(runtime) private(k) #endif - do iEdge = 1, nEdgesAll - do k = 1,nVertLevels - layerThickEdgeFlux(k,iEdge) = & - layerThickEdgeMean(k,iEdge) - end do - end do + do iEdge = 1, nEdgesAll + do k = 1,nVertLevels + layerThickEdgeFlux(k,iEdge) = & + layerThickEdgeMean(k,iEdge) + end do + end do #ifndef MPAS_OPENACC - !$omp end do - !$omp end parallel + !$omp end do + !$omp end parallel #endif + end if + case (thickEdgeFluxUpwind) ! Use upwind thickness as the edge flux value + if (config_use_subgrid_wetting_drying) then + + call ocn_subgrid_layerThickEdgeFlux_upwind(ssh, normalVelocity, layerThickness, layerThickEdgeFlux) + + else #ifdef MPAS_OPENACC - !$acc parallel loop & - !$acc present(normalVelocity, layerThickness, & - !$acc minLevelEdgeBot, maxLevelEdgeTop, & - !$acc layerThickEdgeFlux, cellsOnEdge) & - !$acc private(k, kmin, kmax, cell1, cell2) + !$acc parallel loop & + !$acc present(normalVelocity, layerThickness, & + !$acc minLevelEdgeBot, maxLevelEdgeTop, & + !$acc layerThickEdgeFlux, cellsOnEdge) & + !$acc private(k, kmin, kmax, cell1, cell2) #else - !$omp parallel - !$omp do schedule(runtime) private(k, kmin, kmax, cell1, cell2) + !$omp parallel + !$omp do schedule(runtime) private(k, kmin, kmax, cell1, cell2) #endif - do iEdge = 1, nEdgesAll - kmin = minLevelEdgeBot(iEdge) - kmax = maxLevelEdgeTop(iEdge) - cell1 = cellsOnEdge(1,iEdge) - cell2 = cellsOnEdge(2,iEdge) - do k=1,nVertLevels - ! initialize layerThicknessEdgeFlux to avoid divide by - ! zero and NaN problems. - layerThickEdgeFlux(k,iEdge) = -1.0e34_RKIND - end do - do k = kmin,kmax - if (normalVelocity(k,iEdge) > 0.0_RKIND) then - layerThickEdgeFlux(k,iEdge) = layerThickness(k,cell1) - elseif (normalVelocity(k,iEdge) < 0.0_RKIND) then - layerThickEdgeFlux(k,iEdge) = layerThickness(k,cell2) - else - layerThickEdgeFlux(k,iEdge) = & - max(layerThickness(k,cell1), & - layerThickness(k,cell2)) - end if + do iEdge = 1, nEdgesAll + kmin = minLevelEdgeBot(iEdge) + kmax = maxLevelEdgeTop(iEdge) + cell1 = cellsOnEdge(1,iEdge) + cell2 = cellsOnEdge(2,iEdge) + do k=1,nVertLevels + ! initialize layerThicknessEdgeFlux to avoid divide by + ! zero and NaN problems. + layerThickEdgeFlux(k,iEdge) = -1.0e34_RKIND + end do + do k = kmin,kmax + if (normalVelocity(k,iEdge) > 0.0_RKIND) then + layerThickEdgeFlux(k,iEdge) = layerThickness(k,cell1) + elseif (normalVelocity(k,iEdge) < 0.0_RKIND) then + layerThickEdgeFlux(k,iEdge) = layerThickness(k,cell2) + else + layerThickEdgeFlux(k,iEdge) = & + max(layerThickness(k,cell1), & + layerThickness(k,cell2)) + end if + end do end do - end do #ifndef MPAS_OPENACC - !$omp end do - !$omp end parallel + !$omp end do + !$omp end parallel #endif + end if + case (thickEdgeFluxConstant) ! Use linearized version H*u where H is constant in time @@ -2600,8 +2621,21 @@ subroutine ocn_diagnostic_solve_z_coordinates(layerThickness, zMid, zTop, ssh)!{ + layerThickness(k ,iCell) end do - ! copy zTop(1,iCell) into sea-surface height array - ssh(iCell) = zTop(minLevelCell(iCell),iCell) + if (config_use_subgrid_wetting_drying) then + call ocn_subgrid_ssh_lookup(layerThickness(1,iCell), & + subgridWetVolumeCellTable(:,iCell), & + subgridSshCellTableRange(:,iCell), & + bottomDepth(iCell), & + subgridCellBathymetryMin(iCell), & + ssh(iCell)) + + + zTop(1,iCell) = ssh(iCell) + zMid(1,iCell) = -bottomDepth(iCell) + 0.5_RKIND*layerThickness(1,iCell) + else + ! copy zTop(1,iCell) into sea-surface height array + ssh(iCell) = zTop(minLevelCell(iCell),iCell) + end if end do #ifndef MPAS_OPENACC @@ -4485,8 +4519,10 @@ subroutine ocn_diagnostics_init(domain, err)!{{{ ke_cell_flag = 1 endif - if ((trim(config_time_integrator) == 'RK4') .or.(trim(config_time_integrator) == 'LTS') ) then - ! For RK4 or LTS, PV includes f: PV = (eta+f)/h. + if ( (trim(config_time_integrator) == 'RK4') & + .or. (trim(config_time_integrator) == 'LTS') & + .or. (trim(config_time_integrator) == 'FB_LTS') ) then + ! For RK4, LTS, or FB_LTS, PV includes f: PV = (eta+f)/h. fCoef = 1 elseif (trim(config_time_integrator) == 'split_explicit' & .or.trim(config_time_integrator) == 'unsplit_explicit' & diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics_variables.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics_variables.F index 567c3efc183c..045112919536 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics_variables.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics_variables.F @@ -140,7 +140,11 @@ module ocn_diagnostics_variables real (kind=RKIND), dimension(:,:), pointer :: wettingVelocityFactor type (field2DReal), pointer :: wettingVelocityField + real (kind=RKIND), dimension(:,:), pointer :: layerThicknessCellWetDry + real (kind=RKIND), dimension(:), pointer :: sshCellWetDry + real (kind=RKIND), dimension(:,:), pointer :: vertDiffTopOfCell + real (kind=RKIND), dimension(:,:), pointer :: vertDiffPassiveTopOfCell real (kind=RKIND), dimension(:,:,:), pointer :: vertNonLocalFlux integer, dimension(:,:), pointer :: rediLimiterCount real (kind=RKIND), dimension(:,:,:), pointer :: activeTracerSurfaceFluxTendency @@ -262,6 +266,10 @@ subroutine ocn_diagnostics_variables_init(domain, jenkinsOn, hollandJenkinsOn, e wettingVelocityFactor) call mpas_pool_get_field(diagnosticsPool, 'wettingVelocityFactor', & wettingVelocityField) + call mpas_pool_get_array(diagnosticsPool, 'layerThicknessCellWetDry', & + layerThicknessCellWetDry) + call mpas_pool_get_array(diagnosticsPool, 'sshCellWetDry', & + sshCellWetDry) end if if (config_use_GM.or.config_use_Redi) then @@ -494,6 +502,8 @@ subroutine ocn_diagnostics_variables_init(domain, jenkinsOn, hollandJenkinsOn, e vertViscTopOfCell) call mpas_pool_get_array(diagnosticsPool, 'vertDiffTopOfCell', & vertDiffTopOfCell) + call mpas_pool_get_array(diagnosticsPool, 'vertDiffPassiveTopOfCell', & + vertDiffPassiveTopOfCell) call mpas_pool_get_array(diagnosticsPool, 'vertAleTransportTop', & vertAleTransportTop) call mpas_pool_get_array(diagnosticsPool, 'vertTransportVelocityTop', & @@ -752,6 +762,7 @@ subroutine ocn_diagnostics_variables_init(domain, jenkinsOn, hollandJenkinsOn, e !$acc barotropicForcing, & !$acc vertViscTopOfCell, & !$acc vertDiffTopOfCell, & + !$acc vertDiffPassiveTopOfCell, & !$acc GMStreamFuncZonal, & !$acc relativeVorticity, & !$acc kineticEnergyCell, & @@ -999,6 +1010,7 @@ subroutine ocn_diagnostics_variables_destroy(err) !{{{ !$acc barotropicForcing, & !$acc vertViscTopOfCell, & !$acc vertDiffTopOfCell, & + !$acc vertDiffPassiveTopOfCell, & !$acc GMStreamFuncZonal, & !$acc relativeVorticity, & !$acc kineticEnergyCell, & @@ -1204,6 +1216,7 @@ subroutine ocn_diagnostics_variables_destroy(err) !{{{ barotropicForcing, & vertViscTopOfCell, & vertDiffTopOfCell, & + vertDiffPassiveTopOfCell, & GMStreamFuncZonal, & relativeVorticity, & kineticEnergyCell, & @@ -1246,6 +1259,8 @@ subroutine ocn_diagnostics_variables_destroy(err) !{{{ end if if ( trim(config_ocean_run_mode) == 'forward' ) then nullify(wettingVelocityFactor) + nullify(layerThicknessCellWetDry) + nullify(sshCellWetDry) end if if (config_use_GM.or.config_use_Redi) then nullify(RediKappa, & diff --git a/components/mpas-ocean/src/shared/mpas_ocn_frazil_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_frazil_forcing.F index 16c223986840..0f06a8d39c16 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_frazil_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_frazil_forcing.F @@ -380,6 +380,7 @@ subroutine ocn_frazil_forcing_build_arrays(domain, meshPool, forcingPool, stateP real (kind=RKIND), pointer, dimension(:) :: accumulatedFrazilIceSalinityOld real (kind=RKIND), pointer, dimension(:) :: accumulatedLandIceFrazilMassNew real (kind=RKIND), pointer, dimension(:) :: accumulatedLandIceFrazilMassOld + real (kind=RKIND), pointer, dimension(:) :: frazilIceFreshwaterFlux real (kind=RKIND), pointer, dimension(:) :: ssh real (kind=RKIND), pointer, dimension(:,:) :: layerThickness real (kind=RKIND), pointer, dimension(:,:,:) :: activeTracers @@ -436,6 +437,7 @@ subroutine ocn_frazil_forcing_build_arrays(domain, meshPool, forcingPool, stateP call mpas_pool_get_array(forcingPool, 'frazilSurfacePressure', frazilSurfacePressure) call mpas_pool_get_array(forcingPool, 'landIceMask', landIceMask) call mpas_pool_get_array(forcingPool, 'landIceFloatingMask', landIceFloatingMask) + call mpas_pool_get_array(forcingPool, 'frazilIceFreshwaterFlux', frazilIceFreshwaterFlux) ! get time step in units of seconds timeStep = mpas_get_clock_timestep(domain % clock, ierr=err) @@ -607,6 +609,8 @@ subroutine ocn_frazil_forcing_build_arrays(domain, meshPool, forcingPool, stateP enddo ! do k=kBottom,minLevelCell,-1 + ! frazilIceFreshwaterFlux forms part of the landIceFreshwaterFluxTotal computed in surface land ice fluxes + frazilIceFreshwaterFlux(iCell) = - (sumNewFrazilIceThickness * config_frazil_ice_density) / dt ! accumulate frazil mass to column total ! note: the accumulatedFrazilIceMass (at both time levels) is reset to zero after being sent to the coupler accumulatedFrazilIceMassNew(iCell) = accumulatedFrazilIceMassOld(iCell) + sumNewFrazilIceThickness & diff --git a/components/mpas-ocean/src/shared/mpas_ocn_subgrid.F b/components/mpas-ocean/src/shared/mpas_ocn_subgrid.F new file mode 100644 index 000000000000..82208a971687 --- /dev/null +++ b/components/mpas-ocean/src/shared/mpas_ocn_subgrid.F @@ -0,0 +1,589 @@ +! Copyright (c) 2013, Los Alamos National Security, LLC (LANS) +! and the University Corporation for Atmospheric Research (UCAR). +! +! Unless noted otherwise source code is licensed under the BSD license. +! Additional copyright and license information can be found in the LICENSE file +! distributed with this code, or at http://mpas-dev.github.com/license.html +! +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! ocn_subgrid +! +!> \brief MPAS ocean subgrid wetting and drying +!> \authors Steven Brus, Damrongsak Wirasaet +!> \date September 2022 +!> \details +!> This module contains routines for subgrid wetting and drying +!> These routines primarily determine the relationship between the +!> wet volume per unit area (which can be thought of a as a layer thickness +!> for a completely wet cell) of a control volume and a the ssh +!> defined over the wet fraction of that control volume. When a +!> control volume is wet, this relationship is linear, i.e. H = \eta + b. +!> However, for partially wet cells, the nonlinear relationship between +!> these values can be pre-computed based on subgrid scale high-resolution data. +!> The routines in this module are responsible for providing interpolated +!> values for the forward and inverse relationship to be used in +!> computing the tendency terms from pre-computed look-up tables. +!> +!> See other details in Kennedy et al. (2019): +!> https://doi.org/10.1016/j.ocemod.2019.101491 +! +!----------------------------------------------------------------------- + +module ocn_subgrid + + use mpas_kind_types + use mpas_constants + use mpas_derived_types + use mpas_pool_routines + use mpas_timekeeping + use mpas_timer + use ocn_constants + use ocn_config + use ocn_mesh + use ocn_diagnostics_variables + + implicit none + private + save + + !-------------------------------------------------------------------- + ! + ! Public parameters + ! + !-------------------------------------------------------------------- + + real(kind=RKIND), public, dimension(:,:), pointer :: & + subgridWetVolumeCellTable, & + subgridWetVolumeEdgeTable, & + subgridWetVolumeVertexTable, & + subgridWetFractionCellTable, & + subgridWetFractionEdgeTable, & + subgridWetFractionVertexTable, & + subgridSshCellTableRange, & + subgridSshEdgeTableRange, & + subgridSshVertexTableRange + real(kind=RKIND), public, dimension(:), pointer :: & + subgridEdgeBathymetryMean, & + subgridVertexBathymetryMean, & + subgridCellBathymetryMin, & + subgridEdgeBathymetryMin, & + subgridVertexBathymetryMin, & + subgridLayerThicknessDebug + + integer, public, pointer :: nSubgridTableLevels + + !-------------------------------------------------------------------- + ! + ! Public member functions + ! + !-------------------------------------------------------------------- + + public :: ocn_subgrid_layerThickEdgeFlux_center, & + ocn_subgrid_layerThickEdgeFlux_upwind, & + ocn_subgrid_vorticity + + public :: ocn_subgrid_layer_thickness_lookup, & + ocn_subgrid_wet_fraction_lookup, & + ocn_subgrid_ssh_lookup, & + ocn_subgrid_init + + !-------------------------------------------------------------------- + ! + ! Private module variables + ! + !-------------------------------------------------------------------- + + + +contains + +!*********************************************************************** +! +! routine ocn_subgrid_layerThickEdgeFlux_center +! +!> \brief Copmutes centered layer thickness edge flux using subgrid info +!> \author Steven Brus, Damrongsak Wirasaet +!> \date September 2022 +!> \details +!> +!> +! +!----------------------------------------------------------------------- + + subroutine ocn_subgrid_layerThickEdgeFlux_center(ssh, layerThickEdgeFlux)!{{{ + + !----------------------------------------------------------------- + ! input variables + !----------------------------------------------------------------- + + real (kind=RKIND), dimension(:), intent(in) :: ssh + + !----------------------------------------------------------------- + ! output variables + !----------------------------------------------------------------- + + real (kind=RKIND), dimension(:,:), intent(out) :: layerThickEdgeFlux + + !----------------------------------------------------------------- + ! local variables + !----------------------------------------------------------------- + + integer :: iEdge + integer :: cell1, cell2 + real (kind=RKIND) :: sshMean + real(kind=RKIND):: eps = 1.0e-10_RKIND + + + do iEdge = 1, nEdgesAll + cell1 = cellsOnEdge(1,iEdge) + cell2 = cellsOnEdge(2,iEdge) + sshMean = 0.5_RKIND * (ssh(cell1) + ssh(cell2)) + call ocn_subgrid_layer_thickness_lookup(sshMean, & + subgridWetVolumeEdgeTable(:,iEdge), & + subgridSshEdgeTableRange(:,iEdge), & + subgridEdgeBathymetryMean(iEdge), & + layerThickEdgeFlux(1,iEdge) ) + + if ( layerThickEdgeFlux(1,iEdge) < eps ) then + layerThickEdgeFlux(1,iEdge) = layerThickEdgeMean(1,iEdge) ; + end if + end do + + end subroutine ocn_subgrid_layerThickEdgeFlux_center!}}} + +!*********************************************************************** +! +! routine ocn_subgrid_layerThickEdgeFlux_upwind +! +!> \brief Copmutes upwind layer thickness edge flux using subgrid info +!> \author Steven Brus, Damrongsak Wirasaet +!> \date September 2022 +!> \details +!> +!> +! +!----------------------------------------------------------------------- + + subroutine ocn_subgrid_layerThickEdgeFlux_upwind(ssh, normalVelocity, layerThickness, layerThickEdgeFlux)!{{{ + + !----------------------------------------------------------------- + ! input variables + !----------------------------------------------------------------- + + real (kind=RKIND), dimension(:), intent(in) :: ssh + real (kind=RKIND), dimension(:,:), intent(in) :: normalVelocity + real (kind=RKIND), dimension(:,:), intent(in) :: layerThickness + + !----------------------------------------------------------------- + ! output variables + !----------------------------------------------------------------- + + real (kind=RKIND), dimension(:,:), intent(out) :: layerThickEdgeFlux + + !----------------------------------------------------------------- + ! local variables + !----------------------------------------------------------------- + + integer :: iEdge + integer :: cell1, cell2 + real (kind=RKIND) :: sshMean + real(kind=RKIND):: eps = 1.0e-10_RKIND + + do iEdge = 1, nEdgesAll + cell1 = cellsOnEdge(1,iEdge) + cell2 = cellsOnEdge(2,iEdge) + if (normalVelocity(1,iEdge) > 0.0_RKIND) then + call ocn_subgrid_layer_thickness_lookup(ssh(cell1), & + subgridWetVolumeEdgeTable(:,iEdge), & + subgridSshEdgeTableRange(:,iEdge), & + subgridEdgeBathymetryMean(iEdge), & + layerThickEdgeFlux(1,iEdge)) + elseif (normalVelocity(1,iEdge) < 0.0_RKIND) then + call ocn_subgrid_layer_thickness_lookup(ssh(cell2), & + subgridWetVolumeEdgeTable(:,iEdge), & + subgridSshEdgeTableRange(:,iEdge), & + subgridEdgeBathymetryMean(iEdge), & + layerThickEdgeFlux(1,iEdge)) + else + sshMean = 0.5_RKIND*(ssh(cell1) + ssh(cell2)) + call ocn_subgrid_layer_thickness_lookup(sshMean, & + subgridWetVolumeEdgeTable(:,iEdge), & + subgridSshEdgeTableRange(:,iEdge), & + subgridEdgeBathymetryMean(iEdge), & + layerThickEdgeFlux(1,iEdge)) + end if + + if ( layerThickEdgeFlux(1,iEdge) < eps ) then + cell1 = cellsOnEdge(1,iEdge) + cell2 = cellsOnEdge(2,iEdge) + + layerThickEdgeFlux(1,iEdge) = 0.5_RKIND * & + ( layerThickness(1,cell1) + & + layerThickness(1,cell2) ) + end if + end do + + end subroutine ocn_subgrid_layerThickEdgeFlux_upwind!}}} + +!*********************************************************************** +! +! routine ocn_subgrid_vorticity +! +!> \brief Copmutes vorticity using subgrid info +!> \author Steven Brus, Damrongsak Wirasaet +!> \date September 2022 +!> \details +!> +!> +! +!----------------------------------------------------------------------- + + subroutine ocn_subgrid_vorticity(ssh, normalizedRelativeVorticityVertex, normalizedPlanetaryVorticityVertex)!{{{ + + !----------------------------------------------------------------- + ! input variables + !----------------------------------------------------------------- + + real (kind=RKIND), dimension(:), intent(in) :: ssh + + !----------------------------------------------------------------- + ! output variables + !----------------------------------------------------------------- + + real (kind=RKIND), dimension(:,:), intent(out) :: normalizedRelativeVorticityVertex + real (kind=RKIND), dimension(:,:), intent(out) :: normalizedPlanetaryVorticityVertex + + !----------------------------------------------------------------- + ! local variables + !----------------------------------------------------------------- + + integer :: iVertex, vertex + integer :: i, k + real (kind=RKIND) :: sshVertex + real (kind=RKIND) :: invAreaTri1 + real (kind=RKIND) :: layerThicknessVertex + + do iVertex = 1, nVerticesHalo(2) + invAreaTri1 = 1.0_RKIND / areaTriangle(iVertex) + do k = 1, maxLevelVertexBot(iVertex) + sshVertex = 0.0_RKIND + do i = 1, vertexDegree + sshVertex = sshVertex + ssh(cellsOnVertex(i,iVertex)) & + * kiteAreasOnVertex(i,iVertex) + end do + sshVertex = sshVertex * invAreaTri1 + + call ocn_subgrid_layer_thickness_lookup(sshVertex, & + subgridWetVolumeVertexTable(:,iVertex), & + subgridSshVertexTableRange(:,iVertex), & + subgridVertexBathymetryMean(iVertex), & + layerThicknessVertex) + if (layerThicknessVertex == 0) cycle + + normalizedRelativeVorticityVertex(k,iVertex) = relativeVorticity(k,iVertex) / layerThicknessVertex + normalizedPlanetaryVorticityVertex(k,iVertex) = fVertex(iVertex) / layerThicknessVertex + end do + end do + + end subroutine ocn_subgrid_vorticity!}}} + +!*********************************************************************** +! +! routine ocn_subgrid_layer_thickness_lookup +! +!> \brief Forward subgrid lookup (ssh -> layerThicknes) +!> \author Steven Brus, Damrongsak Wirasaet +!> \date September 2022 +!> \details Returns the wet volume per unit area (layerThick) for a +!> given ssh (defined over the wet-fraction based on subgrid +!> lookup table information +! +!----------------------------------------------------------------------- + + subroutine ocn_subgrid_layer_thickness_lookup(zeta, & + subgridTable, & + subgridTableRange, & + bathymetry, & + layerThick)!{{{ + + !----------------------------------------------------------------- + ! input variables + !----------------------------------------------------------------- + + real (kind=RKIND), intent(in) :: zeta + real (kind=RKIND), dimension(:), intent(in) :: subgridTable + real (kind=RKIND), dimension(:), intent(in) :: subgridTableRange + real (kind=RKIND), intent(in) :: bathymetry + + !----------------------------------------------------------------- + ! output variables + !----------------------------------------------------------------- + + real (kind=RKIND), intent(out) :: layerThick + + !----------------------------------------------------------------- + ! local variables + !----------------------------------------------------------------- + + integer :: lev + + real (kind=RKIND) :: tableMin + real (kind=RKIND) :: tableMax + real (kind=RKIND) :: deltaZ + real (kind=RKIND) :: zeta0, zeta1 + + tableMin = subgridTableRange(1) + tableMax = subgridTableRange(2) + deltaZ = (tableMax - tableMin)/real(nSubgridTableLevels-1,RKIND) + + if (zeta >= tableMax) then + layerThick = zeta + bathymetry + else if (zeta <= tableMin) then + layerThick = 0.0_RKIND + else + do lev = 1, nSubgridTableLevels-1 + zeta0 = (real(lev,RKIND)-1.0_RKIND)*deltaZ + tableMin + zeta1 = zeta0 + deltaZ + + if ((zeta <= zeta1) .and. (zeta >= zeta0)) then + layerThick = ((zeta-zeta0)*subgridTable(lev+1) - (zeta-zeta1)*subgridTable(lev))/deltaZ + return + end if + + end do + end if + + end subroutine ocn_subgrid_layer_thickness_lookup!}}} + +!*********************************************************************** +! +! routine ocn_subgrid_wet_fraction_lookup +! +!> \brief Wet fraction lookup +!> \author Steven Brus, Damrongsak Wirasaet +!> \date September 2022 +!> \details Returns the wet fraction for a given ssh (defined over the +!> wet fraction) value based on subgrid lookup table information +! +!----------------------------------------------------------------------- + + subroutine ocn_subgrid_wet_fraction_lookup(zeta, & + subgridTable, & + subgridTableRange, & + wetFraction)!{{{ + + !----------------------------------------------------------------- + ! input variables + !----------------------------------------------------------------- + + real (kind=RKIND), intent(in) :: zeta + real (kind=RKIND), dimension(:), intent(in) :: subgridTable + real (kind=RKIND), dimension(:), intent(in) :: subgridTableRange + + !----------------------------------------------------------------- + ! output variables + !----------------------------------------------------------------- + + real (kind=RKIND), intent(out) :: wetFraction + + !----------------------------------------------------------------- + ! local variables + !----------------------------------------------------------------- + + integer :: lev + + real (kind=RKIND) :: tableMin + real (kind=RKIND) :: tableMax + real (kind=RKIND) :: deltaZ + real (kind=RKIND) :: zeta0, zeta1 + + tableMin = subgridTableRange(1) + tableMax = subgridTableRange(2) + deltaZ = (tableMax - tableMin)/real(nSubgridTableLevels-1,RKIND) + + if (zeta >= tableMax) then + wetFraction = 1.0_RKIND + else if (zeta <= tableMin) then + wetFraction = 0.0_RKIND + else + do lev = 1, nSubgridTableLevels-1 + zeta0 = (real(lev,RKIND)-1.0_RKIND)*deltaZ + tableMin + zeta1 = zeta0 + deltaZ + + if ((zeta <= zeta1) .and. (zeta >= zeta0)) then + wetFraction = ((zeta-zeta0)*subgridTable(lev+1) - (zeta-zeta1)*subgridTable(lev))/deltaZ + return + end if + + end do + end if + + end subroutine ocn_subgrid_wet_fraction_lookup!}}} + +!*********************************************************************** +! +! routine ocn_subgrid_ssh_lookup +!> \brief Inverse subgrid lookup (layerThickness -> ssh) +!> \author Steven Brus, Damrongsak Wirasaet +!> \date September 2022 +!> \details Returns the ssh (defined over the wet fraction) for a given +!> wet volume per unit area value (layerThick) based on subgrid +!> lookup table information +! +!----------------------------------------------------------------------- + + subroutine ocn_subgrid_ssh_lookup(layerThick, & + subgridTable, & + subgridTableRange, & + bathymetryMean, & + bathymetryMin, & + zeta)!{{{ + + !----------------------------------------------------------------- + ! input variables + !----------------------------------------------------------------- + + real (kind=RKIND), intent(in) :: layerThick + real (kind=RKIND), dimension(:), intent(in) :: subgridTable + real (kind=RKIND), dimension(:), intent(in) :: subgridTableRange + real (kind=RKIND), intent(in) :: bathymetryMean + real (kind=RKIND), intent(in) :: bathymetryMin + + !----------------------------------------------------------------- + ! output variables + !----------------------------------------------------------------- + + real (kind=RKIND), intent(inout) :: zeta + + !----------------------------------------------------------------- + ! local variables + !----------------------------------------------------------------- + + integer :: lev + + real (kind=RKIND) :: deltaZ + real (kind=RKIND) :: tableMin, tableMax + real (kind=RKIND) :: layerThickMin, layerThickMax + real (kind=RKIND) :: layerThick0, layerThick1 + real (kind=RKIND) :: zeta0, zeta1 + real (kind=RKIND) :: phi0, phi1 + + tableMin = subgridTableRange(1) + tableMax = subgridTableRange(2) + deltaZ = (tableMax - tableMin)/real(nSubgridTableLevels-1,RKIND) + + layerThickMin = subgridTable(1) + layerThickMax = subgridTable(nSubgridTableLevels) + + if (layerThick >= layerThickMax) then + zeta = layerThick - bathymetryMean + + else if (layerThick <= layerThickMin) then + zeta = - bathymetryMin ! prevent_drying likely fails to ensure positive water columbn + else + + do lev = 1, nSubgridTableLevels-1 + zeta0 = (real(lev,RKIND)-1.0_RKIND)*deltaZ + tableMin + zeta1 = zeta0 + deltaZ + + layerThick0 = subgridTable(lev) + layerThick1 = subgridTable(lev+1) + + if ((layerThick <= layerThick1) .and. (layerThick >= layerThick0)) then + phi0 = (layerThick-layerThick1)/(layerThick0-layerThick1) + phi1 = (layerThick-layerThick0)/(layerThick1-layerThick0) + zeta = phi0*zeta0 + phi1*zeta1 + return + end if + + end do + end if + + end subroutine ocn_subgrid_ssh_lookup!}}} + +!*********************************************************************** +! +! routine ocn_subgrid_init +! +!> \brief Initializes subgrid wetting and drying module. +!> \author Steven Brus, Damrongsak Wirasaet +!> \date September 2022 +!> \details +!> This routine initializes the subgrid wetting and drying module +! +!----------------------------------------------------------------------- + + subroutine ocn_subgrid_init(domain,err)!{{{ + + !----------------------------------------------------------------- + ! input/output variables + !----------------------------------------------------------------- + + type (domain_type), intent(inout) :: domain + + !----------------------------------------------------------------- + ! output variables + !----------------------------------------------------------------- + + integer, intent(out) :: err + + !----------------------------------------------------------------- + ! local variables + !----------------------------------------------------------------- + + type(block_type), pointer :: block + type(mpas_pool_type), pointer :: meshPool + + err = 0 + if (.not. config_use_subgrid_wetting_drying) then + return + end if + + if ((config_ocean_run_mode == 'forward') .and. (nVertLevels .ne. 1)) then + call mpas_log_write('config_config_use_subgrid_wetting_drying = .true. requires single layer' , MPAS_LOG_CRIT) + end if + + block => domain%blocklist + call mpas_pool_get_subpool(block%structs, 'mesh', meshPool) + + call mpas_pool_get_array(meshPool, 'subgridWetVolumeCellTable', & + subgridWetVolumeCellTable) + call mpas_pool_get_array(meshPool, 'subgridWetVolumeEdgeTable', & + subgridWetVolumeEdgeTable) + call mpas_pool_get_array(meshPool, 'subgridWetVolumeVertexTable', & + subgridWetVolumeVertexTable) + call mpas_pool_get_array(meshPool, 'subgridWetFractionCellTable', & + subgridWetFractionCellTable) + call mpas_pool_get_array(meshPool, 'subgridWetFractionEdgeTable', & + subgridWetFractionEdgeTable) + call mpas_pool_get_array(meshPool, 'subgridWetFractionVertexTable', & + subgridWetFractionVertexTable) + call mpas_pool_get_array(meshPool, 'subgridSshCellTableRange', & + subgridSshCellTableRange) + call mpas_pool_get_array(meshPool, 'subgridSshEdgeTableRange', & + subgridSshEdgeTableRange) + call mpas_pool_get_array(meshPool, 'subgridSshVertexTableRange', & + subgridSshVertexTableRange) + call mpas_pool_get_array(meshPool, 'subgridEdgeBathymetryMean', & + subgridEdgeBathymetryMean) + call mpas_pool_get_array(meshPool, 'subgridVertexBathymetryMean', & + subgridVertexBathymetryMean) + call mpas_pool_get_array(meshPool, 'subgridCellBathymetryMin', & + subgridCellBathymetryMin) + call mpas_pool_get_array(meshPool, 'subgridEdgeBathymetryMin', & + subgridEdgeBathymetryMin) + call mpas_pool_get_array(meshPool, 'subgridVertexBathymetryMin', & + subgridVertexBathymetryMin) + call mpas_pool_get_array(meshPool, 'subgridLayerThicknessDebug', & + subgridLayerThicknessDebug) + + call mpas_pool_get_dimension(meshPool, 'nSubgridTableLevels', & + nSubgridTableLevels) + + end subroutine ocn_subgrid_init!}}} + +end module ocn_subgrid + +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! vim: foldmethod=marker diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F index 69c60c79ca51..c249f202757d 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F @@ -468,6 +468,8 @@ subroutine ocn_surface_land_ice_fluxes_build_arrays(meshPool, & landIceFreshwaterFlux, & landIceHeatFlux, heatFluxToLandIce + real (kind=RKIND), dimension(:), pointer :: frazilIceFreshwaterFlux, landIceFreshwaterFluxTotal + integer, dimension(:), pointer :: landIceFloatingMask real (kind=RKIND), dimension(:,:), pointer :: & @@ -505,6 +507,9 @@ subroutine ocn_surface_land_ice_fluxes_build_arrays(meshPool, & call mpas_pool_get_array(forcingPool, 'landIceHeatFlux', landIceHeatFlux) call mpas_pool_get_array(forcingPool, 'heatFluxToLandIce', heatFluxToLandIce) + call mpas_pool_get_array(forcingPool, 'frazilIceFreshwaterFlux', frazilIceFreshwaterFlux) + call mpas_pool_get_array(forcingPool, 'landIceFreshwaterFluxTotal', landIceFreshwaterFluxTotal) + call mpas_pool_get_array(forcingPool, 'landIceInterfaceTracers', landIceInterfaceTracers) call mpas_pool_get_dimension(forcingPool, & 'index_landIceInterfaceTemperature', & @@ -650,6 +655,17 @@ subroutine ocn_surface_land_ice_fluxes_build_arrays(meshPool, & endif ! jenkinsOn or hollandJenkinsOn + ! Add frazil and interface melt/freeze to get total fresh water flux + if ( associated(frazilIceFreshwaterFlux) ) then + do iCell = 1, nCells + landIceFreshwaterFluxTotal(iCell) = landIceFreshwaterFlux(iCell) + landIceFloatingMask(iCell)*frazilIceFreshwaterFlux(iCell) + end do + else + do iCell = 1, nCells + landIceFreshwaterFluxTotal(iCell) = landIceFreshwaterFlux(iCell) + end do + end if + if(useHollandJenkinsAdvDiff) then deallocate(freezeInterfaceSalinity, & freezeInterfaceTemperature, & diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F index 2719e1526930..91f182db9993 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F @@ -437,7 +437,9 @@ subroutine ocn_tend_vel(domain, tendPool, statePool, forcingPool, & ! Add tidal potential (if needed) call ocn_compute_tidal_potential_forcing(err) - if ((config_time_integrator == 'RK4') .or. (config_time_integrator =='LTS')) then + if ( (config_time_integrator == 'RK4') & + .or. (config_time_integrator =='LTS') & + .or. (config_time_integrator == 'FB_LTS') ) then ! for split explicit, tidal forcing is added in barotropic subcycles call ocn_vel_tidal_potential_tend(ssh,surfacePressure, tendVel, err) endif diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F index e2c1705c39b8..2a9b400dd90b 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F @@ -29,6 +29,7 @@ module ocn_tidal_forcing use ocn_config use ocn_mesh use ocn_diagnostics_variables + use ocn_subgrid implicit none private @@ -260,7 +261,11 @@ subroutine ocn_tidal_forcing_build_array(domain, meshPool, forcingPool, statePoo if (config_use_wetting_drying .and. tidalInputMask(iCell) == 1.0_RKIND) then ! ensure that tidal height can't force below total minimum thickness ! condition wrong to ensure that there isn't any drying according to criteria - tidalHeight = max(-bottomDepth(iCell) + (float(maxLevelCell(iCell))+1.0_RKIND)*config_drying_min_cell_height, tidalHeight) + if ( config_use_subgrid_wetting_drying ) then + tidalHeight = max(tidalHeight, subgridSShCellTableRange(3,iCell) ) ; + else + tidalHeight = max(-bottomDepth(iCell) + (float(maxLevelCell(iCell))+1.0_RKIND)*config_drying_min_cell_height, tidalHeight) + end if end if ! compute total depth for relative thickness contribution diff --git a/components/mpas-ocean/src/shared/mpas_ocn_time_average_coupled.F b/components/mpas-ocean/src/shared/mpas_ocn_time_average_coupled.F index b796c56fa1a6..751c547d95bd 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_time_average_coupled.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_time_average_coupled.F @@ -53,7 +53,10 @@ subroutine ocn_time_average_coupled_init(forcingPool)!{{{ real (kind=RKIND), dimension(:,:), pointer :: avgTracersSurfaceValue, avgSurfaceVelocity, avgSSHGradient, & avgLandIceBoundaryLayerTracers, avgLandIceTracerTransferVelocities - real (kind=RKIND), dimension(:), pointer :: avgEffectiveDensityInLandIce, avgTotalFreshWaterTemperatureFlux + real (kind=RKIND), dimension(:), pointer :: avgEffectiveDensityInLandIce, avgTotalFreshWaterTemperatureFlux, & + avgLandIceFreshwaterFlux, & + avgRemovedRiverRunoffFlux, avgRemovedIceRunoffFlux, & + avgLandIceHeatFlux, avgRemovedIceRunoffHeatFlux integer :: iCell integer, pointer :: nAccumulatedCoupled, nCells @@ -116,6 +119,37 @@ subroutine ocn_time_average_coupled_init(forcingPool)!{{{ !$omp end parallel end if + ! Set up polar fields if necessary + if(trim(config_land_ice_flux_mode)=='standalone' .or. trim(config_land_ice_flux_mode) == 'data') then + call mpas_pool_get_array(forcingPool, 'avgLandIceFreshwaterFlux', avgLandIceFreshwaterFlux) + call mpas_pool_get_array(forcingPool, 'avgLandIceHeatFlux', avgLandIceHeatFlux) + + !$omp parallel + !$omp do schedule(runtime) + do iCell = 1, nCells + avgLandIceFreshwaterFlux(iCell) = 0.0_RKIND + avgLandIceHeatFlux(iCell) = 0.0_RKIND + end do + !$omp end do + !$omp end parallel + end if + + if(config_remove_AIS_coupler_runoff) then + call mpas_pool_get_array(forcingPool, 'avgRemovedRiverRunoffFlux', avgRemovedRiverRunoffFlux) + call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoffFlux', avgRemovedIceRunoffFlux) + call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoffHeatFlux', avgRemovedIceRunoffHeatFlux) + + !$omp parallel + !$omp do schedule(runtime) + do iCell = 1, nCells + avgRemovedRiverRunoffFlux(iCell) = 0.0_RKIND + avgRemovedIceRunoffFlux(iCell) = 0.0_RKIND + avgRemovedIceRunoffHeatFlux(iCell) = 0.0_RKIND + end do + !$omp end do + !$omp end parallel + end if + ! set up BGC coupling fields if necessary if (config_use_ecosysTracers) then @@ -201,6 +235,9 @@ end subroutine ocn_time_average_coupled_init!}}} ! !----------------------------------------------------------------------- subroutine ocn_time_average_coupled_accumulate(statePool, forcingPool, timeLevel)!{{{ + use ocn_constants, only: & + latent_heat_fusion_mks + type (mpas_pool_type), intent(in) :: statePool type (mpas_pool_type), intent(inout) :: forcingPool integer, intent(in) :: timeLevel @@ -215,7 +252,12 @@ subroutine ocn_time_average_coupled_accumulate(statePool, forcingPool, timeLevel real (kind=RKIND), dimension(:,:), pointer :: & avgLandIceBoundaryLayerTracers, avgLandIceTracerTransferVelocities real (kind=RKIND), dimension(:), pointer :: effectiveDensityInLandIce, avgEffectiveDensityInLandIce, & - totalFreshWaterTemperatureFlux, avgTotalFreshWaterTemperatureFlux + totalFreshWaterTemperatureFlux, avgTotalFreshWaterTemperatureFlux, & + landIceFreshwaterFlux, avgLandIceFreshwaterFlux, & + landIceHeatFlux, avgLandIceHeatFlux, & + removedRiverRunoffFlux, avgRemovedRiverRunoffFlux, & + removedIceRunoffFlux, avgRemovedIceRunoffFlux, & + avgRemovedIceRunoffHeatFlux type (mpas_pool_type), pointer :: tracersPool @@ -311,6 +353,48 @@ subroutine ocn_time_average_coupled_accumulate(statePool, forcingPool, timeLevel !$omp end parallel end if + ! Accumulate polar fields if necessary + if(trim(config_land_ice_flux_mode) == 'standalone' .or. trim(config_land_ice_flux_mode) == 'data') then + call mpas_pool_get_array(forcingPool, 'avgLandIceFreshwaterFlux', avgLandIceFreshwaterFlux) + call mpas_pool_get_array(forcingPool, 'landIceFreshwaterFlux', landIceFreshwaterFlux) + call mpas_pool_get_array(forcingPool, 'avgLandIceHeatFlux', avgLandIceHeatFlux) + call mpas_pool_get_array(forcingPool, 'landIceHeatFlux', landIceHeatFlux) + + !$omp parallel + !$omp do schedule(runtime) + do iCell = 1, nCells + avgLandIceFreshwaterFlux(iCell) = ( avgLandIceFreshwaterFlux(iCell) * nAccumulatedCoupled & + + landIceFreshwaterFlux(iCell) ) / ( nAccumulatedCoupled + 1) + avgLandIceHeatFlux(iCell) = ( avgLandIceHeatFlux(iCell) * nAccumulatedCoupled & + + landIceHeatFlux(iCell) ) / ( nAccumulatedCoupled + 1) + end do + !$omp end do + !$omp end parallel + + end if + + if (config_remove_AIS_coupler_runoff) then + call mpas_pool_get_array(forcingPool, 'avgRemovedRiverRunoffFlux', avgRemovedRiverRunoffFlux) + call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoffFlux', avgRemovedIceRunoffFlux) + call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoffHeatFlux', avgRemovedIceRunoffHeatFlux) + call mpas_pool_get_array(forcingPool, 'removedRiverRunoffFlux', removedRiverRunoffFlux) + call mpas_pool_get_array(forcingPool, 'removedIceRunoffFlux', removedIceRunoffFlux) + + !$omp parallel + !$omp do schedule(runtime) + do iCell = 1, nCells + avgRemovedRiverRunoffFlux(iCell) = ( avgRemovedRiverRunoffFlux(iCell) * nAccumulatedCoupled & + + removedRiverRunoffFlux(iCell) ) / ( nAccumulatedCoupled + 1) + avgRemovedIceRunoffFlux(iCell) = ( avgRemovedIceRunoffFlux(iCell) * nAccumulatedCoupled & + + removedIceRunoffFlux(iCell) ) / ( nAccumulatedCoupled + 1) + avgRemovedIceRunoffHeatFlux(iCell) = ( avgRemovedIceRunoffHeatFlux(iCell) * nAccumulatedCoupled & + + removedIceRunoffFlux(iCell)*latent_heat_fusion_mks ) / ( nAccumulatedCoupled + 1) + end do + !$omp end do + !$omp end parallel + + end if + ! accumulate BGC coupling fields if necessary if (config_use_ecosysTracers) then diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tracer_ecosys.F b/components/mpas-ocean/src/shared/mpas_ocn_tracer_ecosys.F index d0bad1133b16..0148c32947cd 100755 --- a/components/mpas-ocean/src/shared/mpas_ocn_tracer_ecosys.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tracer_ecosys.F @@ -1975,6 +1975,8 @@ subroutine ocn_tracer_ecosys_surface_flux_compute(activeTracers, ecosysTracers, ecosysSurfaceFlux(fe_ind_MPAS,iCell) + iceFluxFeDissolved(iCell)/parm_Fe_bioavail ecosysSurfaceFlux(dic_ind_MPAS,iCell) = & ecosysSurfaceFlux(dic_ind_MPAS,iCell) + iceFluxDIC(iCell) + ecosysSurfaceFlux(alk_ind_MPAS,iCell) = & + ecosysSurfaceFlux(alk_ind_MPAS,iCell) + iceFluxDIC(iCell) ecosysSurfaceFlux(dic_alt_co2_ind_MPAS,iCell) = & ecosysSurfaceFlux(dic_alt_co2_ind_MPAS,iCell) + iceFluxDIC(iCell) ecosysSurfaceFlux(doc_ind_MPAS,iCell) = & diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vel_hadv_coriolis.F b/components/mpas-ocean/src/shared/mpas_ocn_vel_hadv_coriolis.F index 3bc9e3e9284c..8ba260596b31 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vel_hadv_coriolis.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vel_hadv_coriolis.F @@ -359,6 +359,10 @@ subroutine ocn_vel_hadv_coriolis_init(err)!{{{ case ('LTS','lts') ! For LTS, coriolis tendency term includes f: (eta+f)/h. usePlanetVorticity = .true. + + case ('FB_LTS','fb_lts') + ! For FB_LTS, coriolis tendency term includes f: (eta+f)/h. + usePlanetVorticity = .true. case ('split_explicit') ! For split explicit, Coriolis tendency uses eta/h because diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vel_pressure_grad.F b/components/mpas-ocean/src/shared/mpas_ocn_vel_pressure_grad.F index 0615856f6673..b8f83cd01dce 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vel_pressure_grad.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vel_pressure_grad.F @@ -39,6 +39,19 @@ module ocn_vel_pressure_grad ! !-------------------------------------------------------------------- + integer, public :: & + pGradType ! id for pressure gradient method selected + + integer, parameter, public :: &! ids for supported methods + pGradTypeNone = 0, &! none selected + pGradTypeSSHgrad = 1, &! ssh gradient + pGradTypePZmid = 2, &! pressure and zMid + pGradTypeMontPot = 3, &! Montgomery potential + pGradTypeMontPotDens = 4, &! Montgomery potential and density + pGradTypeJacobDens = 5, &! Jacobian from density + pGradTypeJacobTS = 6, &! Jacobian from T,S + pGradTypeConstForced = 7 ! constant forced + !-------------------------------------------------------------------- ! ! Public member functions @@ -58,19 +71,6 @@ module ocn_vel_pressure_grad pGradOff, &! flag for turning pressure gradient on/off timeIntegratorLTS ! flag for Local Time Stepping - integer :: & - pGradType ! id for pressure gradient method selected - - integer, parameter :: &! ids for supported methods - pGradTypeNone = 0, &! none selected - pGradTypeSSHgrad = 1, &! ssh gradient - pGradTypePZmid = 2, &! pressure and zMid - pGradTypeMontPot = 3, &! Montgomery potential - pGradTypeMontPotDens = 4, &! Montgomery potential and density - pGradTypeJacobDens = 5, &! Jacobian from density - pGradTypeJacobTS = 6, &! Jacobian from T,S - pGradTypeConstForced = 7 ! constant forced - real (kind=RKIND) :: &! precomputed constants for efficiency density0Inv, &! 1/density0 gdensity0Inv, &! g/density0 @@ -709,7 +709,8 @@ subroutine ocn_vel_pressure_grad_init(err)!{{{ end select - if (config_time_integrator == 'LTS') then + if ( (config_time_integrator == 'LTS') & + .or. (config_time_integrator == 'FB_LTS') ) then timeIntegratorLTS = .true. else timeIntegratorLTS = .false. diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vel_tidal_potential.F b/components/mpas-ocean/src/shared/mpas_ocn_vel_tidal_potential.F index a13105200ed5..092c52c54d09 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vel_tidal_potential.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vel_tidal_potential.F @@ -503,7 +503,8 @@ subroutine ocn_vel_tidal_potential_init(domain,err)!{{{ call mpas_log_write(' ') end do - if (config_time_integrator == 'LTS') then + if ( (config_time_integrator == 'LTS') & + .or. (config_time_integrator == 'FB_LTS') ) then timeIntegratorLTS = .true. else timeIntegratorLTS = .false. diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vmix.F b/components/mpas-ocean/src/shared/mpas_ocn_vmix.F index 87f23d65aa1f..5eefa945a68c 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vmix.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vmix.F @@ -1583,9 +1583,44 @@ subroutine ocn_vmix_implicit(dt, meshPool, statePool, forcingPool, scratchPool, !$acc enter data copyin(tracerGroupSurfaceFlux) #endif - call ocn_tracer_vmix_tend_implicit(dt, vertDiffTopOfCell, layerThickness, tracersGroup, & + if (trim(groupItr % memberName) /= 'activeTracers' .and. & + config_cvmix_background_diffusion_passive_enable) then + + +#ifdef MPAS_OPENACC + !$acc parallel loop gang vector collapse(3) present(vertDiffPassiveTopOfCell, vertDiffTopOfCell) +#else + !$omp parallel + !$omp do schedule(runtime) private(k, iTracer) +#endif + do iCell = 1, nCellsOwned + do k = 2, maxLevelCell(iCell) + vertDiffPassiveTopOfCell(k,iCell) = vertDiffTopOfCell(k,iCell) - & + config_cvmix_background_diffusion + config_cvmix_background_diffusion_passive + end do + end do +#ifndef MPAS_OPENACC + !$omp end do + !$omp end parallel +#endif + + + + + + +#ifdef MPAS_OPENACC + !$acc update host (vertDiffPassiveTopOfCell) +#endif + + call ocn_tracer_vmix_tend_implicit(dt, vertDiffPassiveTopOfCell, layerThickness, tracersGroup, & + vertNonLocalFlux, tracerGroupSurfaceFlux, & + config_cvmix_kpp_nonlocal_with_implicit_mix, err) + else + call ocn_tracer_vmix_tend_implicit(dt, vertDiffTopOfCell, layerThickness, tracersGroup, & vertNonLocalFlux, tracerGroupSurfaceFlux, & config_cvmix_kpp_nonlocal_with_implicit_mix, err) + end if #ifdef MPAS_OPENACC !$acc exit data delete(tracerGroupSurfaceFlux) #endif diff --git a/components/mpas-ocean/src/shared/mpas_ocn_wetting_drying.F b/components/mpas-ocean/src/shared/mpas_ocn_wetting_drying.F index 92cb65b2bbaf..16d052df0288 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_wetting_drying.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_wetting_drying.F @@ -33,6 +33,8 @@ module ocn_wetting_drying use ocn_diagnostics_variables use ocn_gm use ocn_mesh + use ocn_subgrid + use ocn_vel_pressure_grad implicit none private @@ -200,7 +202,7 @@ end subroutine ocn_wetting_drying_verify !}}} ! !----------------------------------------------------------------------- - subroutine ocn_prevent_drying_rk4(block, dt, rkSubstepWeight, config_zero_drying_velocity, err) !{{{ + subroutine ocn_prevent_drying_rk4(domain, block, dt, rkSubstepWeight, config_zero_drying_velocity, err) !{{{ !----------------------------------------------------------------- ! @@ -208,6 +210,7 @@ subroutine ocn_prevent_drying_rk4(block, dt, rkSubstepWeight, config_zero_drying ! !----------------------------------------------------------------- + type (domain_type), intent(inout) :: domain type (block_type), intent(in) :: block real (kind=RKIND), intent(in) :: dt real (kind=RKIND), intent(in) :: rkSubstepWeight @@ -263,10 +266,10 @@ subroutine ocn_prevent_drying_rk4(block, dt, rkSubstepWeight, config_zero_drying !$omp end do !$omp end parallel - ! ensure cells stay wet by selectively damping cells with a damping tendency to make + ! ensure cells stay wet by selectively damping cells with a damping tendency to make ! sure tendency doesn't dry cells - call ocn_wetting_drying_wettingVelocity(layerThickEdgeFlux, layerThicknessCur, layerThicknessProvis, & + call ocn_wetting_drying_wettingVelocity(domain, layerThickEdgeFlux, layerThicknessCur, layerThicknessProvis, & normalTransportVelocity, rkSubstepWeight, wettingVelocityFactor, err) ! prevent drying from happening with selective wettingVelocityFactor @@ -304,9 +307,9 @@ end subroutine ocn_prevent_drying_rk4 !}}} !> to prevent cells from drying. ! !----------------------------------------------------------------------- - subroutine ocn_wetting_drying_wettingVelocity(layerThickEdgeFlux, layerThicknessCur, layerThicknessProvis, & - normalVelocity, dt, wettingVelocityFactor, err)!{{{ + subroutine ocn_wetting_drying_wettingVelocity(domain, layerThickEdgeFlux, layerThicknessCur, layerThicknessProvis, & + normalVelocity, dt, wettingVelocityFactor, err)!{{{ !----------------------------------------------------------------- ! @@ -335,6 +338,10 @@ subroutine ocn_wetting_drying_wettingVelocity(layerThickEdgeFlux, layerThickness ! !----------------------------------------------------------------- + type (domain_type), optional, intent(inout) :: & + domain !< Input/Output: domain information, needed for halo exchange + + real (kind=RKIND), dimension(:,:), intent(inout) :: & wettingVelocityFactor !< Input/Output: velocity wettingVelocityFactor @@ -355,21 +362,39 @@ subroutine ocn_wetting_drying_wettingVelocity(layerThickEdgeFlux, layerThickness integer :: cell1, cell2, iEdge, iCell, k, i real (kind=RKIND) :: divOutFlux - real (kind=RKIND) :: layerThickness + real (kind=RKIND) :: columnThickness + real (kind=RKIND) :: hCrit, hRampMin, hRampMax, hEdgeTotal character (len=100) :: log_string + integer:: cellDummy(2), cellCur, CellNei + real (kind=RKIND) :: sshCur, sshNei + real (kind=RKIND), dimension(:), pointer :: sshCell + real (kind=RKIND), dimension(:, :), pointer :: layerThicknessCell + + err = 0 if (.not. config_zero_drying_velocity) return + if (config_use_ssh_gradient_wetting_drying .and. & + pGradType /= pGradTypeSSHgrad) then + + call mpas_log_write("config_use_ssh_gradient_wetting_drying requires " // & + "config_pressure_gradient_type = 'ssh_gradient'", MPAS_LOG_CRIT) + endif + + hRampMin = config_zero_drying_velocity_ramp_hmin + hRampMax = config_zero_drying_velocity_ramp_hmax + hCrit = config_drying_min_cell_height + config_drying_safety_height + + layerThicknessCell => layerThicknessCellWetDry ! need predicted transport velocity to limit drying flux !$omp parallel - !$omp do schedule(runtime) private(i, iEdge, k, divOutFlux, layerThickness) + !$omp do schedule(runtime) private(i, iEdge, k, divOutFlux) do iCell = 1, nCellsAll do k = minLevelCell(iCell), maxLevelCell(iCell) divOutFlux = 0.0_RKIND - layerThickness = min(layerThicknessProvis(k, iCell), layerThicknessCur(k, iCell)) do i = 1, nEdgesOnCell(iCell) iEdge = edgesOnCell(i, iCell) if (k <= maxLevelEdgeTop(iEdge) .and. k >= minLevelEdgeBot(iEdge)) then @@ -382,15 +407,148 @@ subroutine ocn_wetting_drying_wettingVelocity(layerThickEdgeFlux, layerThickness end if end if end do - layerThickness = layerThickness + dt * divOutFlux - - call ocn_wetting_velocity_factor_on_cell_edges(wettingVelocityFactor, layerThickness, normalVelocity, iCell, k) + layerThicknessCell(k, iCell) = min(layerThicknessProvis(k, iCell), layerThicknessCur(k, iCell)) + dt * divOutFlux + if ( .not. config_use_ssh_gradient_wetting_drying ) then + call ocn_wetting_velocity_factor_on_cell_edges( & + wettingVelocityFactor, layerThicknessCell(k, iCell), normalVelocity, iCell, k) + end if end do end do !$omp end do !$omp end parallel + call mpas_dmpar_field_halo_exch(domain, 'layerThicknessCellWetDry') + + + ! This wetting and drying implementation is based on the one above. + ! However, it has special considerations for edges with normal velocities + ! that have been previously zeroed to limit drying (or are zero initially), + ! but have ssh gradients that would cause flow into the current cell, + ! i.e., the neighboring cell's ssh is higher. + ! There are two cases that require special treatment: + ! 1) In the case that the neighbor cell is predicted to be wet, flow should + ! be allowed into the current cell. + ! 2) In the case that the neighor cell is predicted to be dry, flow should + ! be prevented completely (and not ramped). + ! Currently, this only works for single layer configurations with the 'ssh_gradient' + ! pressure gradient option + if (config_use_ssh_gradient_wetting_drying ) then + ! Compute predicted ssh + sshCell => sshCellWetDry + sshCell = 0.0_RKIND + + wettingVelocityFactor = 0.0_RKIND + + if ( config_use_subgrid_wetting_drying ) then + do iCell = 1, nCellsAll + k = 1 + call ocn_subgrid_ssh_lookup(layerThicknessCell(k, iCell), & + subgridWetVolumeCellTable(:,iCell), & + subgridSshCellTableRange(:,iCell), & + bottomDepth(iCell), & + subgridCellBathymetryMin(iCell), & + sshCell(iCell)) + enddo + else + !$omp parallel + !$omp do schedule(runtime) private(k, columnThickness) + do iCell = 1, nCellsAll + columnThickness = 0.0_RKIND + do k = minLevelCell(iCell), maxLevelCell(iCell) + columnThickness = columnThickness + layerThicknessCell(k, iCell) + enddo + sshCell(iCell) = columnThickness - bottomDepth(iCell) + enddo + !$omp end do + !$omp end parallel + end if + + !$omp parallel + !$omp do schedule(runtime) private(k, i, iEdge, cellDummy, cellNei, cellCur, sshCur, sshNei) + do iCell = 1, nCellsAll + do i = 1, nEdgesOnCell(iCell) + iEdge = edgesOnCell(i, iCell) + do k = minLevelCell(iCell), maxLevelCell(iCell) + if (k <= maxLevelEdgeTop(iEdge) .and. k >= minLevelEdgeBot(iEdge) .and. & + layerThicknessCell(k, iCell) <= hCrit ) then + ! If edge velocity has been zeroed, check neighbor and currnt cell ssh. + ! Allow flow into current cell if neighbor cell is wet and the ssh gradient + ! would cause flow from neighbor cell into current cell. + if ( normalVelocity(k,iEdge)*edgeSignOnCell(i, iCell) < 0.0_RKIND ) then + + wettingVelocityFactor(k,iEdge) = 1.0_RKIND + + elseif ( normalVelocity(k,iEdge)*edgeSignOnCell(i, iCell) == 0.0_RKIND ) then + + cellDummy(1:2) = cellsOnEdge(1:2,iEdge) + cellNei = merge( cellDummy(2), cellDummy(1), iCell == cellDummy(1) ) + cellCur = iCell + if ( cellNei > nCellsAll ) cellNei = cellCur + sshCur = sshCell(cellCur) + sshNei = sshCell(cellNei) + + wettingVelocityFactor(k,iEdge) = 1.0_RKIND + + ! if the neigbor cell is anticipated wet + if ( layerThicknessCell(k, cellNei) > hCrit ) then + if ( sshCur < sshNei ) then + wettingVelocityFactor(k,iEdge) = 0.0_RKIND + end if + end if + + end if ! velocity check + endif ! k check and thickness check + enddo ! k + enddo ! i + end do ! iCell + !$omp end do + !$omp end parallel + + !$omp parallel + !$omp do schedule(runtime) private(k, i, iEdge, cellDummy, cellNei, cellCur, sshCur, sshNei) + do iCell = 1, nCellsAll + do i = 1, nEdgesOnCell(iCell) + iEdge = edgesOnCell(i, iCell) + do k = minLevelCell(iCell), maxLevelCell(iCell) + if (k <= maxLevelEdgeTop(iEdge) .and. k >= minLevelEdgeBot(iEdge) .and. & + config_zero_drying_velocity_ramp .and. & + (layerThicknessCell(k, iCell) > hCrit) .and. & + (layerThicknessCell(k, iCell) <= hRampMax) ) then + + if ( normalVelocity(k, iEdge) * edgeSignOnCell(i, iCell) <= 0.0_RKIND ) then + wettingVelocityFactor(k, iEdge) = 1.0_RKIND - & + tanh(50.0_RKIND * (layerThicknessCell(k, iCell) - hRampMin)/hRampMin) + endif + + ! If edge velocity has been zeroed, check neighbor and current cell ssh. + ! Prevent flow out of cell if neighbor cell is dry and the ssh gradient + ! would cause flow from neighbor cell into current cell. + if ( normalVelocity(k, iEdge) * edgeSignOnCell(i,iCell) == 0.0_RKIND ) then + + cellDummy(1:2) = cellsOnEdge(1:2,iEdge) + cellNei = merge( cellDummy(2), cellDummy(1), iCell == cellDummy(1) ) + cellCur = iCell + if ( cellNei > nCellsAll ) cellNei = cellCur + sshCur = sshCell(cellCur) + sshNei = sshCell(cellNei) + + ! if the neigbor cell is anticipated dry + if ( layerThicknessCell(k, cellNei) <= hCrit ) then + if ( sshCur < sshNei ) then + wettingVelocityFactor(k,iEdge) = 1.0_RKIND + end if + end if + end if ! velocity check + end if ! thickness check + enddo ! k + enddo ! iEdge + end do ! iCell + !$omp end do + !$omp end parallel + + end if + end subroutine ocn_wetting_drying_wettingVelocity !}}} !*********************************************************************** diff --git a/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml b/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml index 1414ffc777de..23f258f2a308 100644 --- a/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml +++ b/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml @@ -25,6 +25,11 @@ 1800.0 1800.0 1800.0 +1800.0 +480.0 +240.0 +120.0 +60.0 'noleap' '2000-01-01_00:00:00' 'none' @@ -77,6 +82,11 @@ 75.0 70.0 +70.0 +85.0 +85.0 +85.0 +85.0 75.0 85.0 85.0 @@ -87,6 +97,11 @@ -75.0 -60.0 +-60.0 +-85.0 +-85.0 +-85.0 +-85.0 -85.0 -85.0 -85.0 @@ -148,6 +163,11 @@ 1 1 1 +1 +1 +1 +1 +1 true true 120 diff --git a/components/mpas-seaice/cime_config/buildnml b/components/mpas-seaice/cime_config/buildnml index 92e5eaf0d004..2e602311033d 100755 --- a/components/mpas-seaice/cime_config/buildnml +++ b/components/mpas-seaice/cime_config/buildnml @@ -129,6 +129,7 @@ def buildnml(case, caseroot, compname): decomp_prefix = 'partitions/mpas-seaice.graph.info.' grid_date = '160929' grid_prefix = 'cice.QU240wLI' + data_iceberg_file = 'Iceberg_Climatology_Merino.oQU240wLI.20240404.nc' if ice_ic_mode == 'spunup': logger.warning("WARNING: The specified compset is requesting seaice ICs spunup from a G-case") logger.warning(" But no file available for this grid.") @@ -243,6 +244,46 @@ def buildnml(case, caseroot, compname): grid_date = '210203' grid_prefix = 'mpassi.SOwISC12to60E2r4.rstFromG-anvil' + elif ice_grid == 'FRISwISC08to60E3r1': + decomp_date = '20230913' # changed to date of partiotions in ../files_for_e3sm/assembled_files/inputdata/ocn/mpas-seaice/FRISwISC08to60E3r1/partitions + decomp_prefix = 'partitions/mpas-seaice.graph.info.' + grid_date = '20230913' # changed to same as decomp_date + grid_prefix = 'mpassi.FRISwISC08to60E3r1' # name from ../files_for_e3sm/assembled_files/inputdata/ice/mpas-seaice/FRISwISC08to60E3r1/mpassi.FRISwISC08to60E3r1.20230913.nc + data_iceberg_file += 'Iceberg_Climatology_Merino.FRISwISC08to60E3r1.20230913.nc' + if ice_ic_mode == 'spunup': + grid_date = '20230913' # changed to same as decomp_date, but the spun up file does not yet exist + grid_prefix = 'mpassi.FRISwISC08to60E3r1.rstFromG-anvil' #the spun up file does not yet exist + + elif ice_grid == 'FRISwISC04to60E3r1': + decomp_date = '20230913' # changed to date of partiotions in ../files_for_e3sm/assembled_files/inputdata/ocn/mpas-seaice/FRISwISC04to60E3r1/partitions + decomp_prefix = 'partitions/mpas-seaice.graph.info.' + grid_date = '20230913' # changed to same as decomp_date + grid_prefix = 'mpassi.FRISwISC04to60E3r1' # name from ../files_for_e3sm/assembled_files/inputdata/ice/mpas-seaice/FRISwISC04to60E3r1/mpassi.FRISwISC04to60E3r1.20230913.nc + data_iceberg_file += 'Iceberg_Climatology_Merino.FRISwISC04to60E3r1.20230913.nc' + if ice_ic_mode == 'spunup': + grid_date = '20230913' # changed to same as decomp_date, but the spun up file does not yet exist + grid_prefix = 'mpassi.FRISwISC04to60E3r1.rstFromG-anvil' #the spun up file does not yet exist + + elif ice_grid == 'FRISwISC02to60E3r1': + decomp_date = '20230914' # changed to date of partiotions in ../files_for_e3sm/assembled_files/inputdata/ocn/mpas-seaice/FRISwISC02to60E3r1/partitions + decomp_prefix = 'partitions/mpas-seaice.graph.info.' + grid_date = '20230914' # changed to same as decomp_date + grid_prefix = 'mpassi.FRISwISC02to60E3r1' # name from ../files_for_e3sm/assembled_files/inputdata/ice/mpas-seaice/FRISwISC02to60E3r1/mpassi.FRISwISC02to60E3r1.20230914.nc + data_iceberg_file += 'Iceberg_Climatology_Merino.FRISwISC02to60E3r1.20230914.nc' + if ice_ic_mode == 'spunup': + grid_date = '20230914' # changed to same as decomp_date, but the spun up file does not yet exist + grid_prefix = 'mpassi.FRISwISC02to60E3r1.rstFromG-anvil' #the spun up file does not yet exist + + elif ice_grid == 'FRISwISC01to60E3r1': + decomp_date = '20230915' # changed to date of partiotions in ../files_for_e3sm/assembled_files/inputdata/ocn/mpas-seaice/FRISwISC01to60E3r1/partitions + decomp_prefix = 'partitions/mpas-seaice.graph.info.' + grid_date = '20230915' # changed to same as decomp_date + grid_prefix = 'mpassi.FRISwISC01to60E3r1' # name from ../files_for_e3sm/assembled_files/inputdata/ice/mpas-seaice/FRISwISC01to60E3r1/mpassi.FRISwISC01to60E3r1.20230915.nc + data_iceberg_file += 'Iceberg_Climatology_Merino.FRISwISC01to60E3r1.20230915.nc' + if ice_ic_mode == 'spunup': + grid_date = '20230915' # changed to same as decomp_date, but the spun up file does not yet exist + grid_prefix = 'mpassi.FRISwISC01to60E3r1.rstFromG-anvil' #the spun up file does not yet exist + elif ice_grid == 'ECwISC30to60E2r1': grid_date = '210413' grid_prefix = 'seaice.ECwISC30to60E2r1' @@ -260,8 +301,22 @@ def buildnml(case, caseroot, compname): decomp_prefix = 'partitions/mpas-seaice.graph.info.' data_iceberg_file = 'Iceberg_Climatology_Merino.IcoswISC30E3r5.20231120.nc' if ice_ic_mode == 'spunup': - grid_date = '20231121' - grid_prefix = 'mpassi.IcoswISC30E3r5.rstFromG-chrysalis' + if iceberg_mode == 'data': + grid_date = '20240207' + grid_prefix = 'mpassi.IcoswISC30E3r5.rstFromG-DIB-chrysalis' + else: + grid_date = '20231121' + grid_prefix = 'mpassi.IcoswISC30E3r5.rstFromG-chrysalis' + + elif ice_grid == 'IcosXISC30E3r7': + grid_date = '20240314' + grid_prefix = 'mpassi.IcosXISC30E3r7' + decomp_date = '20240314' + decomp_prefix = 'partitions/mpas-seaice.graph.info.' + data_iceberg_file = 'Iceberg_Climatology_Merino.IcosXISC30E3r7.20240314.nc' + if ice_ic_mode == 'spunup': + grid_date = '20240314' + grid_prefix = 'mpassi.IcosXISC30E3r7.rstFromPiControlSpinup-chrysalis' elif ice_grid == 'ICOS10': grid_date = '211015' @@ -396,7 +451,7 @@ def buildnml(case, caseroot, compname): lines.append('') lines.append('') +``` + +to + +```text +lines.append(' output_interval="00-00-01_00:00:00">') +``` + +for ``stream name=“output”`` and add + +```text +lines.append(' ') +``` + +a few lines below that: + +```text + lines.append('') ++ lines.append(' output_interval="00-00-01_00:00:00">') + lines.append('') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') ++ lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append('') + lines.append('') +``` + +Build and run baseline case for 5 years (60 months): + +```text +./E3SM-Polar-Developer.sh -s qcbaseline -k qcbase.nlk -e -d60 -nb +./E3SM-Polar-Developer.sh -s qcbaseline -k qcbase.nlk -e -d60 -q +``` + +Copy the thickness analysis member changes into your development directory: + +```text +cd ~/E3SM-Polar/code/newdev01/components/mpas-seaice/cime_config/ +cp ~/E3SM-Polar/code/qcbaseline/components/mpas-seaice/cime_config/buildnml . +``` + +If your development case adds namelist parameters, add the thickness analysis member to your .nlk file as above. This example uses the default configuration. + +Build and run the development case: + +```text +cd ~/SimulationScripts/archive/PolarGroup/ +./E3SM-Polar-Developer.sh -s newdev01 -k qcbase.nlk -e -d60 -nb +./E3SM-Polar-Developer.sh -s newdev01 -k qcbase.nlk -e -d60 -q +``` + +### Run QC comparison + +```text +cd ~/E3SM-Polar/code/newdev01/components/mpas-seaice/testing/cice-qc +``` + +See README.md. This example is for anvil. + +Edit ``job_script.cice-qc.anvil`` to export (insert your username) + +```text +BASE = /lcrc/group/e3sm/[username]/E3SM-Polar/D12.qcbase.emc.qcbaseline.master.E3SM-Project.anvil/run.k000/ +TEST = /lcrc/group/e3sm/[username]/E3SM-Polar/D12.qcbase.emc.newdev01.branch.E3SM-Project.anvil/run.k000 +``` + +Submit QC test. Test results will be in the file ``qc_log.txt``. + +```text +sbatch job_script.qc-testing-mpassi.anvil +less qc_log.txt +``` + +Example of desired result: + +```text +Running QC test on the following directories: + /lcrc/group/e3sm/ac.eclare/E3SM-Polar/D12.qcbase.emc.qcbaseline.master.E3SM-Project.anvil/run.k000/ + /lcrc/group/e3sm/ac.eclare/E3SM-Polar/D12.qcbase.emc.newdev01.branch.E3SM-Project.anvil/run.k000 +Number of files: 61 +2 Stage Test Passed +Quadratic Skill Test Passed for Northern Hemisphere +Quadratic Skill Test Passed for Southern Hemisphere +``` + +### Generate statistics from the CICE-QC runs + +This only works if the .nlk filename is the same for both cases. If comparing only namelist changes within MPAS-seaice, use the ``./E3SM-Polar-Developer.sh`` script with a single .nlk file that includes each option. + +```text +cd ~/SimulationScripts/archive/PolarGroup/ +$ ./E3SM-Polar-Developer.sh -s qcbaseline -k qcbase.nlk -e -d60 -a D12.qcbase.emc.newdev01.branch.E3SM-Project.anvil -v +``` + +### Create comparison plots + +To generate MPAS-Analysis plots from the CICE-QC runs and compare: + +Copy the scripts in the file [MPAS-Analysis_scripts.zip](./MPAS-Analysis_scripts.zip) to anvil or chrysalis. + +Edit each script for your run names, directories, etc (search for 'echmod' to find settings used for a QC comparison) + +Edit and submit (on chrysalis) the job script 3 times, once for icepack, once for column, and finally for the comparison. + +Browse the html output by navigating to the location indicated by ``htmlSubdirectory`` in the comparison script, e.g. +``https://web.lcrc.anl.gov/public/e3sm/diagnostic_output/ac.eclare/icepack-testing/D12.qcPR19.emc.qcPR19.snicar_active.eclare108213.anvil/mpas_analysis_output/`` diff --git a/components/mpas-seaice/docs/figures/mesh.png b/components/mpas-seaice/docs/figures/mesh.png new file mode 100644 index 000000000000..f4ddc3612388 Binary files /dev/null and b/components/mpas-seaice/docs/figures/mesh.png differ diff --git a/components/mpas-seaice/docs/index.md b/components/mpas-seaice/docs/index.md new file mode 100644 index 000000000000..e4239d697fd7 --- /dev/null +++ b/components/mpas-seaice/docs/index.md @@ -0,0 +1,44 @@ +The E3SM Sea Ice Model (MPAS-seaice) +==================================== + +MPAS-seaice is an unstructured-mesh sea-ice model that uses the Modeling for Prediction Across Scales (MPAS) framework, allowing enhanced horizontal resolution in regions of interest. MPAS-seaice incorporates many of the methods used in the Los Alamos CICE sea-ice model, but adapted to the Spherical Centroidal Vornoi Tesselation (SCVT) meshes used by the MPAS framework. + +* The [MPAS-seaice User's Guide](user-guide/index.md) outlines the MPAS Framework, on which MPAS-seaice is built, and Icepack, the column physics submodule in MPAS-seaice, and it provides guidance for controlling MPAS-seaice within E3SM. +* The [MPAS-seaice Technical Guide](tech-guide/index.md) describes the mesh and major physics components underlying MPAS-seaice code and its coupling to E3SM. +* The [MPAS-seaice Developer's Guide](dev-guide/index.md) provides additional information relevant for model development, including the Icepack interface and development/testing scripts. + +**Icepack** +----------- + +MPAS-seaice incorporates the Icepack software package for sea ice column physics, developed by the [CICE Consortium](https://github.com/cice-consortium), as a submodule. [Icepack documentation](https://e3sm-icepack.readthedocs.io/en/latest/). provides a complete description of the column physics and instructions for using Icepack as a standalone model. The source code for this documentation is maintained in [E3SM's Icepack fork](https://github.com/E3SM-Project/Icepack/) (navigate to the desired branch, then to doc/source/, etc). This is the documentation associated with the latest Icepack version that has been merged into E3SM, plus any documentation changes made within E3SM itself. This documentation is fully rendered in [E3SM's Icepack readthedocs](https://e3sm-icepack.readthedocs.io/en/latest/). + + + +[Guidance for developing Icepack documentation](https://github.com/CICE-Consortium/About-Us/wiki/Documentation-Workflow-Guide) includes instructions for building the readthedocs documentation yourself. + +**MPAS-seaice code structure** +------------------------------ + +Some MPAS-seaice functionality is sourced from the MPAS Framework: +``E3SM/components/mpas-framework``. In particular, see ``E3SM/components/mpas-framework/core_seaice``. + +Code structure within the ``mpas-seaice/``component-level directory: + +| Directories | Function | +| ----------- | -------- | +| ``bld`` | namelist configuration files | +| ``cime_config`` | build and configuration scripts | +| ``docs`` | this documentation | +| ``driver`` | coupling modules | +| ``src`` | source code for the model physics and output | +| ``src/analysis_members`` | source code for model output | +| ``src/column`` | source code for the (original) ``column_package`` | +| ``src/icepack`` | link to the icepack submodule | +| ``src/model_forward`` | top-level mpas-seaice modules | +| ``src/shared`` | dynamics and general-purpose modules (e.g. mesh, constants) | +| ``testing`` | testing scripts | diff --git a/components/mpas-seaice/docs/references.md b/components/mpas-seaice/docs/references.md new file mode 100644 index 000000000000..95dbe8f36eea --- /dev/null +++ b/components/mpas-seaice/docs/references.md @@ -0,0 +1,47 @@ +References +========== + +Bitz, C. M., and W. H. Lipscomb (1999). An energy-conserving thermodynamic model of sea ice, Journal of Geophysical Research: Oceans, 104(C7), 15,669–15,677, doi: 10.1029/1999JC900100. + +Briegleb, B. P., and B. Light (2007). A Delta-Eddington multiple scattering parameterization for solar radiation in the sea ice component of the Community Climate Sys- tem Model, Tech. Rep. NCAR/TN-472+STR, National Center for Atmospheric Research, Boulder, Colorado USA. + +Dang, C., C. S. Zender, and M. G. Flanner (2019). Intercomparison and improvement of two-stream shortwave radiative transfer schemes in earth system models for a unified treatment of cryospheric surfaces. The Cryosphere, 13:2325–2343. doi:10.5194/tc-13-2325-2019. + +Dasgupta, G. (2003). Interpolants within convex polygons: Wachpress' shape functions. Journal of Aerospace Engineering, 16, 1–8. + +Dukowicz, J. K., & Baumgardner, J. R. (2000). Incremental remapping as a transport/advection algorithm. Journal of Computational Physics, 160(1), 318–335. + +Dunavant, D. A. (1985). High degree efficient symmetrical Gaussian quadrature rules for the triangle. International Journal for Numerical Methods in Engineering, 21(6), 1129–1148. + +Flocco, D., D. L. Feltham, and A. K. Turner (2010). Incorporation of a physically based melt pond scheme into the sea ice component of a climate model, Journal of Geophysi- cal Research: Oceans, 115(C8), doi:10.1029/2009JC005568, C08012. + +Golaz, J.-C., Caldwell, P. M.,
Van Roekel, L. P., Petersen, M. R., Tang, Q., Wolfe, J. D., et al. (2019). The DOE E3SM coupled model version 1: Overview and evaluation at
standard resolution. Journal of Advances in Modeling Earth
Systems, 11, 2089–2129. + +Golaz, J.-C., Van Roekel, L. P., Zheng, X., Roberts, A. F., Wolfe, J. D., Lin, W., et al. (2022). The DOE E3SM Model version 2: Overview of the physical model and initial model evaluation. Journal of Advances in Modeling Earth Systems, 14, e2022MS003156. + +Hibler, W. D. III (1979). A dynamic thermodynamic sea ice model. Journal of Physical Oceanography, 9(4), 815–846. + +Holland, M. M., D. A. Bailey, B. P. Briegleb, B. Light, and E. Hunke (2012). Improved sea ice shortwave radiation physics in CCSM4: The impact of melt ponds and aerosols on arctic sea ice, Journal of Climate, 25(5), 1413–1430, doi:10.1175/JCLI-D-11-00078.1. + + +Hunke, E., et al. (2018). CICE-Consortium/Icepack. Zenodo. + +Hunke, E. C., & Dukowicz, J. K. (1997). An elastic-viscous-plastic model for sea ice dynamics. Journal of Physical Oceanography, 27(9), 1849–1867. + +Hunke, E. C., & Dukowicz, J. K. (2002). The elastic-viscous-plastic sea ice dynamics model in general orthogonal curvilinear coordinates on a sphere—Incorporation of metric terms. Monthly Weather Review, 130(7), 1848–1865. + +Hunke, E. C., Hebert, D. A., & Lecomte, O. (2013). Level-ice melt ponds in the Los Alamos sea ice model, CICE. Ocean Modelling, 71, 26–42. + +Lipscomb, W. H. (2001). Remapping the thickness distribution in sea ice models, Journal of Geophysical Research: Oceans, 106(C7), 13,989–14,000, doi:10.1029/2000JC000518. + +Lipscomb, W. H., & Hunke, E. C. (2004). Modeling sea ice transport using incremental remapping. Monthly Weather Review, 132(6), 1341–1354. + +Lipscomb, W. H., Hunke, E. C., Maslowski, W., & Jakacki, J. (2007). Ridging, strength, and stability in high-resolution sea ice models. Journal of Geophysical Research, 112. C03S91. + +Lipscomb, W. H., & Ringler, T. D. (2005). An incremental remapping transport scheme on a spherical geodesic grid. Monthly Weather Review, 133(8), 2335–2350. + +Turner, A. K., and E. C. Hunke (2015). Impacts of a mushy-layer thermodynamic ap- proach in global sea-ice simulations using the CICE sea-ice model, Journal of Geophys- ical Research: Oceans, 120(2), 1253–1275, doi:10.1002/2014JC010358. + +Turner, A. K., E. C. Hunke, and C. M. Bitz (2013). Two modes of sea-ice gravity drainage: A parameterization for large-scale modeling, Journal of Geophysical Research: Oceans, 118(5), 2279–2294, doi:10.1002/jgrc.20171. + +Turner, A. K., Lipscomb, W. H., Hunke, E. C., Jacobsen, D. W., Jeffery, N., Engwirda, D., Ringer, T. D., Wolfe, J. D. (2021). MPAS-seaice (v1.0.0): Sea-ice dynamics on unstructured Voronoi meshes. Geoscientific Model Development Discussions, 1–46. diff --git a/components/mpas-seaice/docs/tech-guide/index.md b/components/mpas-seaice/docs/tech-guide/index.md new file mode 100644 index 000000000000..3d6d4a1786dd --- /dev/null +++ b/components/mpas-seaice/docs/tech-guide/index.md @@ -0,0 +1,97 @@ +# Technical Guide + +## Primary documentation for MPAS-seaice + +See complete citations in [References](../references.md). + +**E3SM v1 Overview**: Golaz et al., JAMES 2019 + +**MPAS-seaice v1**: Turner et al., GMD Discussions, 2021. + +**E3SM v2 Overview**: Golaz et al., JAMES 2022 + +**Icepack**: Full documentation for E3SM's version of Icepack can be found in [E3SM's Icepack readthedocs](https://e3sm-icepack.readthedocs.io/en/latest). The most up-to-date documentation from the CICE Consortium's main Icepack repository is [here](https://cice-consortium-icepack.readthedocs.io/en/main). + +A comprehensive paper describing MPAS-seaice is in preparation. + +## Meshes + +MPAS-Seaice is the sea ice component of E3SMv1. MPAS-Seaice and MPAS-Ocean share identical meshes, but MPAS-Seaice uses B-grid discretizations (Arakawa & Lamb, 1977) with sea ice concentration, volume, and tracers defined at cell centers and velocity defined at cell vertices. + +The MPAS mesh system requires the definition of seven elements. These seven elements are composed of two types of _cells_, two types of _lines_, and three types of _points_. These elements can be defined on either the plane or the surface of the sphere. The two types of cells form two meshes, a primal mesh composed of Voronoi regions and a dual mesh composed of Delaunay triangles. Each corner of a primal mesh cell is uniquely associated with the "center" of a dual mesh cell and vice versa. The boundary of a given primal mesh cell is composed of the set of lines that connect the centers of the dual mesh cells. Similarly, the boundary of a given dual mesh cell is composed of the set of lines that connect the center points of the associated primal mesh cells. A line segment that connects two primal mesh cell centers is uniquely associated with a line seqment that connects two dual mesh cell centers. We assume that these two line seqments cross and are orthogonal. Since the two line seqments crossing are othogonal, they form a convenient local coordinate system for each edge. +![mesh](../figures/mesh.png) +Figure: Sample from an MPAS mesh showing the primal mesh (solid lines), the dual mesh (dashed), and velocity components aligned with a locally Cartesian coordinate system (east/north). + +## Velocity and Stresses + +Velocity components at cell vertices are not aligned with the mesh, as in sea ice models with structured meshes and quadrilateral cells. Instead, the velocity components are aligned with a spherical coordinate system that is locally Cartesian, eastwards (u) and northwards (v), irrespective of the orientation of edges joining that vertex. Such a definition, however, would result in a convergence of v components at the geographic North Pole and strong metric terms in the velocity solution. Consequently, in addition, these definitions of u and v are rotated so that their pole lies on the geographical equator at 0 deg longitude. + +Velocities are determined by solving the sea ice momentum equation (Hibler, 1979; Hunke & Dukowicz, 1997). During coupled simulations the ocean model provides the ocean surface tilt term; the only other term that depends on the properties of the horizontal grid is the divergence of internal stress. Therefore only this stress term must be adapted for use on MPAS meshes. Otherwise the velocity solver is identical to that in CICE’s standard EVP approach. Determination of the divergence of the internal stress can be broken down into three stages: + +1. The strain rate tensor is determined from the velocity field. + +2. The stress tensor at a point is determined, through a constitutive relation, from the strain rate tensor at that point. + +3. The divergence of this stress tensor is calculated. + +Two schemes to calculate the strain rate tensor and the divergence of internal stress on MPAS meshes are implemented in MPAS-Seaice, a variational scheme based on that used in CICE (Hunke and Dukowicz, 2002), and a weak scheme that uses the line integral forms of the symmetric gradient and divergence operators. The variational scheme is based on the fact that over the entire domain, Ω, and ignoring boundary effects, the total work done by the internal stress is equal to the dissipation of mechanical energy. Instead of the bilinear basis functions used by CICE, MPAS-Seaice uses Wachspress basis functions (Dasgupta, 2003), which are integrated with the quadrature rules of Dunavant (1985). + +## Horizontal Transport of Ice Area Fraction and Tracers + +Horizontal transport of ice concentration, volume, and tracers is achieved with an incremental remapping (IR) scheme similar to that described in Dukowicz and Baumgardner (2000), Lipscomb and Hunke (2004), and Lipscomb and Ringler (2005). For MPAS-Seaice the IR scheme was generalized to work on either the standard MPAS mesh (hexagons and other n-gons of varying sizes, with a vertex degree of 3, or a quadrilateral mesh with a vertex degree of 4 as in CICE. Since MPAS meshes are unstructured, the IR scheme had to be rewritten from scratch. Most of the code is mesh-agnostic, but a small amount of code is specific to quad meshes. +The transport equations describe conservation of quantities such as volume and energy. Fractional ice area (also known as sea ice concentration) is a mass-like quantity whose transport equation forms the basis for all other transported quantities in the model. In particular, ice volume is the product of ice area and thickness; therefore thickness is treated as a tracer on ice area, transported with the continuity equation for conservation of volume. Likewise, snow depth is carried as a tracer on ice area via a conservation of snow volume equation. Ice thickness and snow depth are referred to as “type 1” tracers (carried directly on ice area). Ice and snow enthalpy in each vertical layer are type 2 tracers, carried on ice and snow volume. When run with advanced options (e.g., active melt ponds and biogeochemistry), MPAS-Seaice advects tracers up to type 3. Thus, the mass-like field (area) is the “parent field” for type 1 tracers; type 1 tracers are parents of type 2; and type 2 tracers are parents of type 3. Sources and sinks of mass and tracers (e.g., ice growth and melting) are treated separately from transport. + +The transport time step is limited by the requirement that trajectories projected backward from vertices are confined to the cells sharing the vertex (i.e., 3 cells for the standard MPAS mesh and 4 for the quad mesh). This is what is meant by incremental as opposed to general remapping. For highly divergent velocity fields, the maximum time step may have to be reduced by a factor of 2 to ensure that trajectories do not cross. The incremental remapping algorithm consists of the following steps: + +1. Given mean values of the ice area and tracer fields in each grid cell and thickness category, construct linear approximations of these fields. Limit the field gradients to preserve mono- tonicity. +2. Given ice velocities at grid cell vertices, identify departure regions for the transport across each cell edge. Divide these departure regions into triangles and compute the coordinates of the triangle vertices. +3. Integrate the area and tracer fields over the departure triangles to obtain the area, volume, and other conserved quantities transported across each cell edge. +4. Given these transports, update the area and tracers. + +Since all fields are transported by the same velocity field, the second step is done only once per time step. The other steps are repeated for each field. + +With advanced physics and biogeochemistry (BGC) options, MPAS-Seaice can be configured to include numerous tracer fields, each of which is advected in every thickness category, and many of which are defined in each vertical ice or snow layer. In order to accommodate different tracer combinations and make it easy to add new tracers, the tracer fields are organized in a linked list that depends on which physics and BGC packages are active. The list is arranged with fractional ice area first, followed by the type 1 tracers, type 2 tracers, and finally type 3 tracers. In this way, values computed for parent tracers are always available when needed for computations involving child tracers. + +## Column Physics + +The Icepack software has replaced the original ``colpkg`` column physics code in MPAS-seaice. The ``config_column_physics_type = 'column_package'`` option is still available but is no longer being supported in MPAS-seaice. + +Because of the strong thermal gradients between the (cold) atmosphere and (relatively warm) oceans in polar regions, a large portion of the physics in sea ice models can be described in a vertical column, without reference to neighboring grid cells. MPAS-Seaice shares the same column physics code as CICE through the Icepack library (Hunke et al., 2018), which is maintained by the CICE Consortium. This code includes several options for simulating sea ice thermodynamics, mechanical redistribution (ridging) and associated area and thickness changes. In addition, the model supports a number of tracers, including thickness, enthalpy, ice age, first-year ice area, deformed ice area and volume, melt ponds, snow properties and biogeochemistry. + +Icepack is implemented in MPAS-seaice as a git submodule. Icepack consists of three independent parts, the column physics code, the Icepack driver that supports stand-alone testing of the column physics code, and the Icepack scripts that build and test the Icepack model. E3SM uses only the column physics code, which is called for each ocean grid cell. Icepack’s own driver and testing scripts are used when preparing new developments to be merged back to the CICE Consortium’s Icepack repository. + +Icepack includes sophisticated vertical physics and biogeochemical schemes, which include vertical thermodynamics schemes (Bitz and Lipscomb, 1999; Turner et al., 2013; Turner and Hunke, 2015), melt-pond parameterizations (Flocco et al., 2010; Hunke et al., 2013), a delta-Eddington radiation scheme (Briegleb and Light, 2007; Holland et al., 2012a), schemes for transport in thickness space (Lipscomb, 2001), and representations of mechanical redistribution (Lipscomb et al., 2007). + +Full documentation for E3SM's version of Icepack can be found in [E3SM's Icepack readthedocs](https://e3sm-icepack.readthedocs.io/en/latest). The most up-to-date documentation from the CICE Consortium's main Icepack repository is [here](https://cice-consortium-icepack.readthedocs.io/en/main). + +### Thermodynamics + +In its default configuration, MPAS-Seaice uses the “mushy layer” vertical thermodynamics scheme of Turner et al. (2013) and Turner and Hunke (2015). The mushy layer formulation describes the sea ice as a two-phase system of crystalline, fresh ice and liquid brine. Enthalpy depends on temperature and salinity, all of which are prognostic variables. The mushy layer equations are derived from conservation of energy, conservation of salt, an ice-brine liquidus relation that determines the temperature- and salinity-dependent phase, and Darcy flow through a porous medium to describe the vertical movement of brine within the ice. When or where the ice is cold, brine pockets are isolated from each other, but warmer temperatures cause the brine pockets to expand and connect into vertical channels in which meltwater, seawater, biology and nutrients may move through the ice. + +### Melt Ponds + +MPAS-seaice uses the level-ice melt pond scheme of Hunke et al. (2013). The ponds are carried as tracers on the level (undeformed) ice area of each thickness category, thus limiting their spatial extent based on the simulated sea ice topography. This limiting is meant to approximate the horizontal drainage of melt water into depressions in ice floes. The ponds evolve according to physically based process descriptions, assuming a thickness-area ratio for changes in pond volume. Melt pond processes include addition of liquid water from rain, melting snow and melting surface ice, drainage of pond water when its weight pushes the ice surface below sea level or when the ice interior becomes permeable, and refreezing of the pond water. If snow falls after a layer of ice has formed on the ponds, the snow may block sunlight from reaching the ponds below. When melt water forms with snow still on the ice, the water is assumed to infiltrate the snow. If there is enough water to fill the air spaces within the snowpack, then the pond becomes visible above the snow, thus decreasing the albedo and ultimately causing the snow to melt faster. The albedo also decreases as snow depth decreases, and thus a thin layer of snow remaining above a pond-saturated layer of snow will have a lower albedo than if the melt water were not present. Level-ice melt ponds are “virtual” in the sense that rain and meltwater is sent to the ocean immediately, and the tracers are only used thereafter to adjust the radiative calculations as if the ponds were present. The delta-Eddington radiative transfer scheme must be active for this purpose. + +### Radiation + +The Delta-Eddington radiation scheme of Briegleb & Light (2007) has been updated to the Dang et al. (2019) SNICAR-AD model, to ensure radiative consistency across all snow surfaces in E3SM, including on land, ice sheets and sea ice. The SNICAR-AD radiative transfer code includes five-band snow single-scattering properties, two-stream Delta-Eddington approximation with the adding–doubling technique, and parameterization for correcting the near-infrared (NIR) snow albedo biases when solar zenith angle exceeds 75 degrees (Dang et al., 2019). + +### Snow + +A new snow-on-sea-ice morphology has been added to E3SMv2 that includes the effects of wind redistribution: losses to leads and meltponds, and the piling of snow against ridges. Snow grain radius, now a prognosed tracer field on sea ice, evolves according to temperature gradient and wet snow metamorphism and feeds back to the SNICAR-AD radiative model up to a dry maximum of 2800 μm. Fresh snow falls at a grain radius of 54.5 μm, and five vertical snow layers replace the previous single snow layer atop each of the five sea ice thickness categories retained from E3SMv1. + +A paper describing the advanced snow physics is in preparation. + +### Biogeochemistry + +This section is under construction, pending the full merge of BGC codes in Icepack and the older column physics package. + +## Coupling of MPAS-seaice within E3SM + +This description is taken from the v1 overview paper (Golaz et al. 2019). Refer to that paper for further information. Coupling of the sea ice component to the ocean takes advantage of z star ocean coordinates and is a departure from the coupling of CICE and POP (Parallel Ocean Program) in CESM1. The weight of sea ice contributes to the ocean's barotropic mode, notably affecting the free surface over continental shelves. In shallow water depths at or less than the floating ice draft, the weight passed to the ocean model is limited to prevent evacuation of the underlying liquid column. When frazil ice forms in the ocean model, the volume of newly formed crystals is passed to the sea ice model with a fixed salinity of 4 PSU, rather than exchanging a freezing potential as in other models. Future versions of E3SM will permit progressive brine drainage to the ocean from the mushy layer physics used in MPAS-Seaice. For E3SMv1, brine drainage occurs internally in MPAS-Seaice for thermodynamic calculations, but for the sake of freshwater coupling, the ocean model only receives mass fluxes back from melted sea ice at the fixed salinity that it originally passed to its cryospheric counterpart (4 PSU). The ocean temperature immediately under the ice is the same as the liquid phase in the lowest layer of the sea ice model and is not fixed at −1.8 ◦C as is typical of previous generation coupled models. For the current version, we have addressed these long-standing ocean-ice coupling issues identified by the modeling community: explicit sea ice mass and salt exchange, a pressure force of the ice on the ocean, a basal sea ice temperature consistent with the ocean model's equation of state, and resolved inertial oscillations. + +This paragraph, taken from the v2 overview paper (Golaz et al. 2022), describes changes since v1. The most significant improvement to the sea ice climate since E3SMv1 was achieved with coupling changes associated with mushy-layer thermodynamics. Whereas the basal temperature of the ice was held fixed at -1.8◦C in E3SMv1, the new version of the model assumes the mushy liquidus basal temperature from the sea ice as described by Turner & Hunke (2015). Conversion of frazil ice from MPAS-Ocean with a fixed reference salinity of 4 PSU to the mushy layer now conserves to computational accuracy over a 500-year control integration. This was achieved by exchanging additional mass between the upper ocean and sea ice model to accommodate an assumed 25% mushy liquid content, assumed from heat and mass transferred adiabatically from the MPAS-Ocean frazil scheme active from a depth of 100 m. In addition to achieving perfect heat and mass conservation between sea ice and ocean models, this improvement greatly reduces a negative sea ice thickness bias in the summer Arctic reported by Golaz et al. (2019) for E3SMv1; it only minimally impacts Southern Ocean sea ice mass that was better simulated as compared to northern hemisphere sea ice in E3SMv1. Note that E3SM does not use virtual ice-ocean fluxes, but instead full mass and heat flux exchange consistent with a Boussinesq ocean model. Radiative coupling with the atmosphere still integrates across just two bands (visible and NIR) separated at 700nm, which does not fully exploit the five-band capability available in the delta-Eddington scheme. + +### Prescribed Ice Mode + +E3SM also includes a prescribed-extent ice mode for MPAS-SeaIce based the CESM implementation. This mode is needed for Atmospheric Model Intercomparison Project (AMIP) style simulations where a full prognostic sea ice model is not desired but sea ice surface fluxes, albedos, snow depth, and surface temperature are needed by the atmosphere model. These fields are calculated by the vertical thermodynamics module of the sea ice component. The prescribed-ice mode is intended for atmosphere sensitivity experiments and does not conserve energy or mass. In this mode, sea ice thermodynamics is active but sea ice dynamics are disabled, and at each time step ice area and thickness are reset to specified values. Ice area is interpolated in time and space from an input data set, while ice thickness in grid cells containing sea ice is set to 2 m in the Northern hemisphere and 1 m in the Southern hemisphere. During each area and thickness adjustment, snow volume preserves the snow thickness prognosed in the previous time step. Snow temperatures are reset to the surface temperature, as prognosed in the previous time step, while ice temperatures are set so that the ice temperature gradient is linear, with the ice temperature at the top equal to the prognosed surface temperature, and equal to the sea freezing temperature at the base of the ice. The vertical ice salinity profile is reset to the profile from Bitz & Lipscomb (1999). The prescribed-ice mode implemented in MPAS-SeaIce can now replace that in CICE in such configurations, but CICE continues to be used for those requiring exceptional computational efficiency. diff --git a/components/mpas-seaice/docs/user-guide/index.md b/components/mpas-seaice/docs/user-guide/index.md new file mode 100644 index 000000000000..21fbca2c32df --- /dev/null +++ b/components/mpas-seaice/docs/user-guide/index.md @@ -0,0 +1,65 @@ +# User's Guide + +Guidance for using E3SM is available from [E3SM's public web site](). + +## Configuring MPAS-seaice + +MPAS-seaice is controlled using namelist options. + +- Default namelist values are found in +``E3SM/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml``. +- Namelist options are defined in +``E3SM/components/mpas-seaice/bld/namelist_files/namelist_definitions_mpassi.xml``, +including type, category (``seaice_model``), group, valid values and a brief description. Each namelist variable is defined in an ``entry`` element. The content of the element is the documentation of how the variable is used. Other aspects of the variable's definition are expressed as attributes of the ``entry`` element. +- Users can change namelist options from defaults by entering ``[namelist option] = [changed value]`` as separate lines in the ``user_nl_mpassi`` file in the case directory. +- Some namelist values or combinations are not allowed and will generate warnings and often abort the code. The consistency checks for using MPAS-seaice within E3SM are in ``mpas_seaice_initialize`` (subroutines ``seaice_check_configs_coupled``, ``seaice_check_constants_coupled``), and those specific to Icepack can be found in subroutine ``check_column_package_configs`` in ``mpas_seaice_icepack.F``. + +Related namelist variables are grouped according to their application. + +| Namelist Groups | Relevant application | +| ------------------- | -------------------- | +| ``seaice_model`` | general options | +| ``io`` | input/output | +| ``decomposition`` | mesh parallelization | +| ``restart`` | restarting the code | +| ``dimensions`` | column physics dimensions (layers, categories) | +| ``initialize`` | initialization | +| ``use_sections`` | turn entire parameterizations on and off | +| ``forcing`` | forcing for standalone configurations | +| ``velocity_solver`` | algorithms for solving the dynamics (velocity and stress) equations | +| ``advection`` | advection | +| ``column_package`` | general column package software configurations | +| ``biogeochemistry`` | biogeochemistry | +| ``shortwave`` | radiation | +| ``snow`` | advanced snow physics | +| ``meltponds`` | melt pond parameterization flags and parameters | +| ``thermodynamics`` | basic thermodynamics | +| ``itd`` | ice thickness distribution | +| ``floesize`` | floe size distribution | +| ``ridging`` | mechanical redistribution | +| ``atmosphere`` | atmospheric boundary layer and coupling | +| ``ocean`` | oceanic boundary layer and coupling | +| ``diagnostics`` | diagnostic output | +| ``prescribed_ice`` | for testing atmosphere simulations | + +## Icepack + +The Icepack software has replaced the original ``colpkg`` column physics code in MPAS-seaice. The ``column_package`` option is still available but is no longer being supported in MPAS-seaice. + +Full documentation for E3SM's version of Icepack can be found in [E3SM's Icepack readthedocs](). The most up-to-date documentation from the CICE Consortium's main Icepack repository is [here](). + +The MPAS-seaice driver for Icepack is + +``E3SM/components/mpas-seaice/src/shared/mpas_seaice_icepack.f`` + +and the mapping between the names of Icepack's namelist options and those in MPAS-seaice can be found in subroutine ``init_icepack_package_configs`` (see the argument list for calls to subroutine ``icepack_init_parameters`` and comments at the end of ``init_icepack_package_configs``. + +## Configuring Model Input and Output + +The reading and writing of model fields in MPAS is handled by user-configurable streams. A stream represents a fixed set of model fields, together with dimensions and attributes, that are all written or read together to or from the same file or set of files. Besides these default streams, users may define new streams to, e.g., write certain diagnostic fields at a higher temporal frequency than the usual model history fields. + +Streams are defined in XML configuration files that are created at build time for each model core. The name of this XML file is simply ‘streams.’ suffixed with the name of the core. For example, the streams for the sea-ice core are defined in a file named ‘streams.seaice’. An XML stream file may further reference other text files that contain lists of the model fields that are read or written in each of the streams defined in the XML stream file. + +The stream file can be modified by copying the ``streams.seaice`` file (generated during ``./case.setup``) from the run directory to the case directory under ``SourceMods/src.mpassi/``, and making changes to the stream file there. Changes to the stream file will take effect on the next case submission. Alternatively, changes to the streams file can be made directly in the code in ``components/mpas-seaice/cime_config/buildnml``. + +Two classes of streams exist in MPAS: immutable streams and mutable streams. Immutable streams are those for which the set of fields that belong to the stream may not be modified at model run-time; however, it is possible to modify the interval at which the stream is read or written, the filename template describing the files containing the stream on disk, and several other parameters of the stream. In contrast, all aspects of mutable streams, including the set of fields that belong to the stream, may be modified at run-time. The motivation for the creation of two stream classes is the idea that an MPAS core may not function correctly if certain fields are not read in upon model start-up or written to restart files, and it is therefore not reasonable for users to modify this set of required fields at run-time. An MPAS core developer may choose to implement such streams as immutable streams. Since fields may not be added to an immutable stream at run-time, new immutable streams may not be defined at run-time, and the only type of new stream that may be defined at run-time is the mutable stream type. diff --git a/components/mpas-seaice/mkdocs.yml b/components/mpas-seaice/mkdocs.yml new file mode 100644 index 000000000000..c89222d5e58b --- /dev/null +++ b/components/mpas-seaice/mkdocs.yml @@ -0,0 +1,9 @@ +site_name: MPAS-seaice + +nav: + - Introduction: 'index.md' + - User's Guide: 'user-guide/index.md' + - Technical Guide: 'tech-guide/index.md' + - Developer's Guide: 'dev-guide/index.md' + - References: 'references.md' + diff --git a/docs/.markdownlint.json b/docs/.markdownlint.json new file mode 100644 index 000000000000..23e8b392689f --- /dev/null +++ b/docs/.markdownlint.json @@ -0,0 +1,4 @@ +{ + "line-length": false, + "no-reversed-links": false +} diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 000000000000..9b1e171a9724 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,13 @@ +# Development Guide + +Development of E3SM is carefully planned and tied to goals in the proposals that fund +work on E3SM. + +Most development occurs within one component and developers should consult the +relevant component's development guide. + +More information + ++ [Development Big Picture](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/7997024/Development+Big+Picture) ++ [Getting Started](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/1868455/Development+Getting+Started+Guide) ++ [Development Reference](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/2523163/Development+Reference) diff --git a/docs/index.md b/docs/index.md index 36584c99bf7c..4bc48af94a5c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,10 +1,31 @@ # The Energy Exascale Earth System Model (E3SM) -The documentation for the components of E3SM is found here. +E3SM is a state-of-the-art fully coupled model of the Earth's climate including important biogeochemical +and cryospheric processes. It is intended to address the most challenging and demanding climate-change +research problems and Department of Energy mission needs while efficiently using DOE Leadership Computing Facilities. + +!!! note + This is the future home for all documentation specific to E3SM and its components. Until complete, + some documentation will be found at the project's + [Confluence wiki](https://acme-climate.atlassian.net/wiki/spaces/DOC/overview) + +## E3SM Basics + +- [Installation](installation.md) +- [User Guide](user-guide/index.md) +- [Development](development.md) + +## Component Models -## Components - [EAM](./EAM/index.md) -- [EAMxx](./EAMxx/index.md) +- EAMxx — not yet supported. - [ELM](./ELM/index.md) - [MOSART](./MOSART/index.md) +- [MPAS-Ocean](./MPAS-Ocean/index.md) +- [MPAS-seaice](./MPAS-seaice/index.md) + +## Tools + +- [generate_domain_files](./generate_domain_files/index.md) +Please see [Developing Docs](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/3924787306/Developing+Documentation) to learn about how to contribute to the documentation. diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 000000000000..38b725e9f95f --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,23 @@ +# Installation Guide + +E3SM is not available as a pre-compiled binary. You install +E3SM by cloning the source code using [git](https://git-scm.com/) +and building the executable on your local platform after making some +choices on model configuration (addressed in the [User Guide](user-guide/index.md)). + +It is recommended that you install E3SM on a supported platform. +(See [Hardware/Software Configuration](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/4116447351/Hardware+Software+Configuration)) + +The E3SM Project can not assist with installation on an un-supported platform but can point you in the right direction. +If you wish to port E3SM to your machine, first check that the target machine has +the software prerequisites detailed in +[Hardware/Software Configuration](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/4116447351/Hardware+Software+Configuration#Software-prerequisites). + +E3SM uses the CIME Case Control System (CCS) and a machine must be described to the CCS +in order to build and run E3SM. See the +[CIME Porting Guide](https://esmci.github.io/cime/versions/master/html/users_guide/porting-cime.html). + +Once you are on a supported machine, clone the source code by following the first steps (2-4 and 6) +in the [Development Getting Started Guide](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/1868455/Development+Getting+Started+Guide). + +To start configuration cases and building the model, see the User Guide diff --git a/docs/refs/eam.bib b/docs/refs/eam.bib new file mode 100644 index 000000000000..ff4415a519b8 --- /dev/null +++ b/docs/refs/eam.bib @@ -0,0 +1,1037 @@ + +@article{moncrieff_simulation_2017, + title = {Simulation, {Modeling}, and {Dynamically} {Based} {Parameterization} of {Organized} {Tropical} {Convection} for {Global} {Climate} {Models}}, + volume = {74}, + issn = {0022-4928, 1520-0469}, + url = {https://journals.ametsoc.org/view/journals/atsc/74/5/jas-d-16-0166.1.xml}, + doi = {10.1175/JAS-D-16-0166.1}, + language = {EN}, + number = {5}, + urldate = {2024-03-29}, + journal = {Journal of the Atmospheric Sciences}, + author = {Moncrieff, Mitchell W. and Liu, Changhai and Bogenschutz, Peter}, + month = may, + year = {2017}, + pages = {1363--1380}, +} + +@article{chen_effects_2021, + title = {Effects of {Organized} {Convection} {Parameterization} on the {MJO} and {Precipitation} in {E3SMv1}. {Part} {I}: {Mesoscale} {Heating}}, + volume = {13}, + issn = {1942-2466, 1942-2466}, + shorttitle = {Effects of {Organized} {Convection} {Parameterization} on the {MJO} and {Precipitation} in {E3SMv1}. {Part} {I}}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/10.1029/2020MS002401}, + doi = {10.1029/2020MS002401}, + language = {en}, + number = {6}, + urldate = {2024-03-29}, + journal = {Journal of Advances in Modeling Earth Systems}, + author = {Chen, C.‐C. and Richter, J. H. and Liu, C. and Moncrieff, M. W. and Tang, Q. and Lin, W. and Xie, S. and Rasch, P. J.}, + month = jun, + year = {2021}, + pages = {e2020MS002401}, +} + +@article{morrison_parameterization_2015, + title = {Parameterization of {Cloud} {Microphysics} {Based} on the {Prediction} of {Bulk} {Ice} {Particle} {Properties}. {Part} {I}: {Scheme} {Description} and {Idealized} {Tests}}, + volume = {72}, + issn = {0022-4928, 1520-0469}, + shorttitle = {Parameterization of {Cloud} {Microphysics} {Based} on the {Prediction} of {Bulk} {Ice} {Particle} {Properties}. {Part} {I}}, + url = {https://journals.ametsoc.org/view/journals/atsc/72/1/jas-d-14-0065.1.xml}, + doi = {10.1175/JAS-D-14-0065.1}, + language = {EN}, + number = {1}, + urldate = {2024-03-29}, + journal = {Journal of the Atmospheric Sciences}, + author = {Morrison, Hugh and Milbrandt, Jason A.}, + month = jan, + year = {2015}, + pages = {287--311}, +} + +@article{milbrandt_parameterization_2016, + title = {Parameterization of {Cloud} {Microphysics} {Based} on the {Prediction} of {Bulk} {Ice} {Particle} {Properties}. {Part} {III}: {Introduction} of {Multiple} {Free} {Categories}}, + volume = {73}, + issn = {0022-4928, 1520-0469}, + shorttitle = {Parameterization of {Cloud} {Microphysics} {Based} on the {Prediction} of {Bulk} {Ice} {Particle} {Properties}. {Part} {III}}, + url = {https://journals.ametsoc.org/view/journals/atsc/73/3/jas-d-15-0204.1.xml}, + doi = {10.1175/JAS-D-15-0204.1}, + language = {EN}, + number = {3}, + urldate = {2024-03-29}, + journal = {Journal of the Atmospheric Sciences}, + author = {Milbrandt, J. A. and Morrison, H.}, + month = mar, + year = {2016}, + pages = {975--995}, +} + +@article{liu_ice_2005, + title = {Ice nucleation parameterization for global models}, + issn = {,}, + url = {https://www.schweizerbart.de/papers/metz/detail/14/54281/Ice_nucleation_parameterization_for_global_models}, + doi = {10.1127/0941-2948/2005/0059}, + language = {pt}, + urldate = {2024-03-29}, + journal = {Meteorologische Zeitschrift}, + author = {Liu, Xiaohong and Penner, Joyce E.}, + month = sep, + year = {2005}, + pages = {499--514}, +} + +@article{wang_impact_2021, + title = {Impact of a {New} {Cloud} {Microphysics} {Parameterization} on the {Simulations} of {Mesoscale} {Convective} {Systems} in {E3SM}}, + volume = {13}, + issn = {1942-2466, 1942-2466}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/10.1029/2021MS002628}, + doi = {10.1029/2021MS002628}, + language = {en}, + number = {11}, + urldate = {2024-03-29}, + journal = {Journal of Advances in Modeling Earth Systems}, + author = {Wang, Jingyu and Fan, Jiwen and Feng, Zhe and Zhang, Kai and Roesler, Erika and Hillman, Benjamin and Shpund, Jacob and Lin, Wuyin and Xie, Shaocheng}, + month = nov, + year = {2021}, + pages = {e2021MS002628}, +} + +@misc{larson_clubb-silhs_2022, + title = {{CLUBB}-{SILHS}: {A} parameterization of subgrid variability in the atmosphere}, + shorttitle = {{CLUBB}-{SILHS}}, + url = {http://arxiv.org/abs/1711.03675}, + doi = {10.48550/arXiv.1711.03675}, + urldate = {2024-03-29}, + publisher = {arXiv}, + author = {Larson, Vincent E.}, + month = mar, + year = {2022}, + note = {arXiv:1711.03675 [physics]}, + keywords = {Physics - Atmospheric and Oceanic Physics}, +} + +@article{bogenschutz_path_2018, + title = {The path to {CAM6}: coupled simulations with {CAM5}.4 and {CAM5}.5}, + volume = {11}, + issn = {1991-959X}, + shorttitle = {The path to {CAM6}}, + url = {https://gmd.copernicus.org/articles/11/235/2018/}, + doi = {10.5194/gmd-11-235-2018}, + language = {English}, + number = {1}, + urldate = {2024-03-29}, + journal = {Geoscientific Model Development}, + author = {Bogenschutz, Peter A. and Gettelman, Andrew and Hannay, Cecile and Larson, Vincent E. and Neale, Richard B. and Craig, Cheryl and Chen, Chih-Chieh}, + month = jan, + year = {2018}, + pages = {235--255}, +} + +@article{golaz_pdf-based_2002, + title = {A {PDF}-{Based} {Model} for {Boundary} {Layer} {Clouds}. {Part} {I}: {Method} and {Model} {Description}}, + volume = {59}, + issn = {0022-4928, 1520-0469}, + shorttitle = {A {PDF}-{Based} {Model} for {Boundary} {Layer} {Clouds}. {Part} {I}}, + url = {https://journals.ametsoc.org/view/journals/atsc/59/24/1520-0469_2002_059_3540_apbmfb_2.0.co_2.xml}, + doi = {10.1175/1520-0469(2002)059<3540:APBMFB>2.0.CO;2}, + language = {EN}, + number = {24}, + urldate = {2024-03-29}, + journal = {Journal of the Atmospheric Sciences}, + author = {Golaz, Jean-Christophe and Larson, Vincent E. and Cotton, William R.}, + month = dec, + year = {2002}, + pages = {3540--3551}, +} + +@article{larson_using_2005, + title = {Using {Probability} {Density} {Functions} to {Derive} {Consistent} {Closure} {Relationships} among {Higher}-{Order} {Moments}}, + volume = {133}, + issn = {1520-0493, 0027-0644}, + url = {https://journals.ametsoc.org/view/journals/mwre/133/4/mwr2902.1.xml}, + doi = {10.1175/MWR2902.1}, + language = {EN}, + number = {4}, + urldate = {2024-03-29}, + journal = {Monthly Weather Review}, + author = {Larson, Vincent E. and Golaz, Jean-Christophe}, + month = apr, + year = {2005}, + pages = {1023--1042}, +} + +@article{neale_impact_2008, + title = {The {Impact} of {Convection} on {ENSO}: {From} a {Delayed} {Oscillator} to a {Series} of {Events}}, + volume = {21}, + issn = {0894-8755, 1520-0442}, + shorttitle = {The {Impact} of {Convection} on {ENSO}}, + url = {https://journals.ametsoc.org/view/journals/clim/21/22/2008jcli2244.1.xml}, + doi = {10.1175/2008JCLI2244.1}, + language = {EN}, + number = {22}, + urldate = {2024-03-29}, + journal = {Journal of Climate}, + author = {Neale, Richard B. and Richter, Jadwiga H. and Jochum, Markus}, + month = nov, + year = {2008}, + pages = {5904--5924}, +} + +@article{zhang_sensitivity_1995, + title = {Sensitivity of climate simulations to the parameterization of cumulus convection in the {Canadian} climate centre general circulation model}, + volume = {33}, + issn = {0705-5900, 1480-9214}, + url = {http://www.tandfonline.com/doi/abs/10.1080/07055900.1995.9649539}, + doi = {10.1080/07055900.1995.9649539}, + language = {en}, + number = {3}, + urldate = {2024-03-29}, + journal = {Atmosphere-Ocean}, + author = {Zhang, G.J. and McFarlane, Norman A.}, + month = sep, + year = {1995}, + pages = {407--446}, +} + +@article{xie_improved_2019, + title = {Improved {Diurnal} {Cycle} of {Precipitation} in {E3SM} {With} a {Revised} {Convective} {Triggering} {Function}}, + volume = {11}, + issn = {1942-2466, 1942-2466}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/10.1029/2019MS001702}, + doi = {10.1029/2019MS001702}, + language = {en}, + number = {7}, + urldate = {2024-03-29}, + journal = {Journal of Advances in Modeling Earth Systems}, + author = {Xie, Shaocheng and Wang, Yi‐Chi and Lin, Wuyin and Ma, Hsi‐Yen and Tang, Qi and Tang, Shuaiqi and Zheng, Xue and Golaz, Jean‐Christophe and Zhang, Guang J. and Zhang, Minghua}, + month = jul, + year = {2019}, + pages = {2290--2310}, +} + +@article{xie_impact_2000, + title = {Impact of the convection triggering function on single‐column model simulations}, + volume = {105}, + issn = {0148-0227}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/10.1029/2000JD900170}, + doi = {10.1029/2000JD900170}, + language = {en}, + number = {D11}, + urldate = {2024-03-29}, + journal = {Journal of Geophysical Research: Atmospheres}, + author = {Xie, Shaocheng and Zhang, Minghua}, + month = jun, + year = {2000}, + pages = {14983--14996}, +} + +@article{wang_impacts_2015, + title = {Impacts of the triggering function of cumulus parameterization on warm-season diurnal rainfall cycles at the {Atmospheric} {Radiation} {Measurement} {Southern} {Great} {Plains} site: {CONVECTIVE} {TRIGGER} {ON} {SGP} {NOCTURNAL} {RAIN}}, + volume = {120}, + issn = {2169897X}, + shorttitle = {Impacts of the triggering function of cumulus parameterization on warm-season diurnal rainfall cycles at the {Atmospheric} {Radiation} {Measurement} {Southern} {Great} {Plains} site}, + url = {http://doi.wiley.com/10.1002/2015JD023337}, + doi = {10.1002/2015JD023337}, + language = {en}, + number = {20}, + urldate = {2024-03-29}, + journal = {Journal of Geophysical Research: Atmospheres}, + author = {Wang, Yi-Chi and Pan, Hua-Lu and Hsu, Huang-Hsiung}, + month = oct, + year = {2015}, + pages = {10,681--10,702}, +} + +@article{song_microphysics_2011, + title = {Microphysics parameterization for convective clouds in a global climate model: {Description} and single-column model tests}, + volume = {116}, + issn = {0148-0227}, + shorttitle = {Microphysics parameterization for convective clouds in a global climate model}, + url = {http://doi.wiley.com/10.1029/2010JD014833}, + doi = {10.1029/2010JD014833}, + language = {en}, + number = {D2}, + urldate = {2024-03-29}, + journal = {Journal of Geophysical Research}, + author = {Song, Xiaoliang and Zhang, Guang J.}, + month = jan, + year = {2011}, + pages = {D02201}, +} + +@article{song_evaluation_2012, + title = {Evaluation of {Microphysics} {Parameterization} for {Convective} {Clouds} in the {NCAR} {Community} {Atmosphere} {Model} {CAM5}}, + volume = {25}, + issn = {0894-8755, 1520-0442}, + url = {http://journals.ametsoc.org/doi/10.1175/JCLI-D-11-00563.1}, + doi = {10.1175/JCLI-D-11-00563.1}, + language = {en}, + number = {24}, + urldate = {2024-03-29}, + journal = {Journal of Climate}, + author = {Song, Xiaoliang and Zhang, Guang J. and Li, J.-L. F.}, + month = dec, + year = {2012}, + pages = {8568--8590}, +} + +@article{storer_effects_2015, + title = {Effects of {Convective} {Microphysics} {Parameterization} on {Large}-{Scale} {Cloud} {Hydrological} {Cycle} and {Radiative} {Budget} in {Tropical} and {Midlatitude} {Convective} {Regions}}, + volume = {28}, + issn = {0894-8755, 1520-0442}, + url = {https://journals.ametsoc.org/view/journals/clim/28/23/jcli-d-15-0064.1.xml}, + doi = {10.1175/JCLI-D-15-0064.1}, + language = {EN}, + number = {23}, + urldate = {2024-03-29}, + journal = {Journal of Climate}, + author = {Storer, Rachel L. and Zhang, Guang J. and Song, Xiaoliang}, + month = dec, + year = {2015}, + pages = {9277--9297}, +} + +@article{song_incorporating_2023, + title = {Incorporating the {Effect} of {Large}‐{Scale} {Vertical} {Motion} on {Convection} {Through} {Convective} {Mass} {Flux} {Adjustment} in {E3SMv2}}, + volume = {15}, + issn = {1942-2466, 1942-2466}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/10.1029/2022MS003553}, + doi = {10.1029/2022MS003553}, + language = {en}, + number = {10}, + urldate = {2024-03-29}, + journal = {Journal of Advances in Modeling Earth Systems}, + author = {Song, Xiaoliang and Zhang, Guang and Wan, Hui and Xie, Shaocheng}, + month = oct, + year = {2023}, + pages = {e2022MS003553}, +} + +@article{mlawer_radiative_1997, + title = {Radiative transfer for inhomogeneous atmospheres: {RRTM}, a validated correlated‐k model for the longwave}, + volume = {102}, + issn = {0148-0227}, + shorttitle = {Radiative transfer for inhomogeneous atmospheres}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/10.1029/97JD00237}, + doi = {10.1029/97JD00237}, + language = {en}, + number = {D14}, + urldate = {2024-03-29}, + journal = {Journal of Geophysical Research: Atmospheres}, + author = {Mlawer, Eli J. and Taubman, Steven J. and Brown, Patrick D. and Iacono, Michael J. and Clough, Shepard A.}, + month = jul, + year = {1997}, + pages = {16663--16682}, +} + +@article{iacono_radiative_2008, + title = {Radiative forcing by long‐lived greenhouse gases: {Calculations} with the {AER} radiative transfer models}, + volume = {113}, + issn = {0148-0227}, + shorttitle = {Radiative forcing by long‐lived greenhouse gases}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/10.1029/2008JD009944}, + doi = {10.1029/2008JD009944}, + language = {en}, + number = {D13}, + urldate = {2024-03-29}, + journal = {Journal of Geophysical Research: Atmospheres}, + author = {Iacono, Michael J. and Delamere, Jennifer S. and Mlawer, Eli J. and Shephard, Mark W. and Clough, Shepard A. and Collins, William D.}, + month = jul, + year = {2008}, + pages = {2008JD009944}, +} + +@article{pincus_fast_2003, + title = {A fast, flexible, approximate technique for computing radiative transfer in inhomogeneous cloud fields}, + volume = {108}, + issn = {0148-0227}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/10.1029/2002JD003322}, + doi = {10.1029/2002JD003322}, + language = {en}, + number = {D13}, + urldate = {2024-03-29}, + journal = {Journal of Geophysical Research: Atmospheres}, + author = {Pincus, Robert and Barker, Howard W. and Morcrette, Jean‐Jacques}, + month = jul, + year = {2003}, + pages = {2002JD003322}, +} + +@article{liu_description_2016, + title = {Description and evaluation of a new four-mode version of the {Modal} {Aerosol} {Module} ({MAM4}) within version 5.3 of the {Community} {Atmosphere} {Model}}, + volume = {9}, + issn = {1991-9603}, + url = {https://gmd.copernicus.org/articles/9/505/2016/}, + doi = {10.5194/gmd-9-505-2016}, + language = {en}, + number = {2}, + urldate = {2024-03-29}, + journal = {Geoscientific Model Development}, + author = {Liu, X. and Ma, P.-L. and Wang, H. and Tilmes, S. and Singh, B. and Easter, R. C. and Ghan, S. J. and Rasch, P. J.}, + month = feb, + year = {2016}, + pages = {505--522}, +} + +@article{wu_development_2022, + title = {Development and {Evaluation} of {E3SM}‐{MOSAIC}: {Spatial} {Distributions} and {Radiative} {Effects} of {Nitrate} {Aerosol}}, + volume = {14}, + issn = {1942-2466, 1942-2466}, + shorttitle = {Development and {Evaluation} of {E3SM}‐{MOSAIC}}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/10.1029/2022MS003157}, + doi = {10.1029/2022MS003157}, + language = {en}, + number = {11}, + urldate = {2024-03-29}, + journal = {Journal of Advances in Modeling Earth Systems}, + author = {Wu, Mingxuan and Wang, Hailong and Easter, Richard C. and Lu, Zheng and Liu, Xiaohong and Singh, Balwinder and Ma, Po‐Lun and Tang, Qi and Zaveri, Rahul A. and Ke, Ziming and Zhang, Rudong and Emmons, Louisa K. and Tilmes, Simone and Dibb, Jack E. and Zheng, Xue and Xie, Shaocheng and Leung, L. Ruby}, + month = nov, + year = {2022}, + pages = {e2022MS003157}, +} + +@article{lou_new_2020, + title = {New {SOA} {Treatments} {Within} the {Energy} {Exascale} {Earth} {System} {Model} ({E3SM}): {Strong} {Production} and {Sinks} {Govern} {Atmospheric} {SOA} {Distributions} and {Radiative} {Forcing}}, + volume = {12}, + issn = {1942-2466, 1942-2466}, + shorttitle = {New {SOA} {Treatments} {Within} the {Energy} {Exascale} {Earth} {System} {Model} ({E3SM})}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/10.1029/2020MS002266}, + doi = {10.1029/2020MS002266}, + language = {en}, + number = {12}, + urldate = {2024-03-29}, + journal = {Journal of Advances in Modeling Earth Systems}, + author = {Lou, Sijia and Shrivastava, Manish and Easter, Richard C. and Yang, Yang and Ma, Po‐Lun and Wang, Hailong and Cubison, Michael J. and Campuzano‐Jost, Pedro and Jimenez, Jose L. and Zhang, Qi and Rasch, Philip J. and Shilling, John E. and Zelenyuk, Alla and Dubey, Manvendra and Cameron‐Smith, Philip and Martin, Scot T. and Schneider, Johannes and Schulz, Christiane}, + month = dec, + year = {2020}, + pages = {e2020MS002266}, +} + +@article{shrivastava_global_2015, + title = {Global transformation and fate of {SOA}: {Implications} of low‐volatility {SOA} and gas‐phase fragmentation reactions}, + volume = {120}, + issn = {2169-897X, 2169-8996}, + shorttitle = {Global transformation and fate of {SOA}}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/10.1002/2014JD022563}, + doi = {10.1002/2014JD022563}, + language = {en}, + number = {9}, + urldate = {2024-03-29}, + journal = {Journal of Geophysical Research: Atmospheres}, + author = {Shrivastava, Manish and Easter, Richard C. and Liu, Xiaohong and Zelenyuk, Alla and Singh, Balwinder and Zhang, Kai and Ma, Po‐Lun and Chand, Duli and Ghan, Steven and Jimenez, Jose L. and Zhang, Qi and Fast, Jerome and Rasch, Philip J. and Tiitta, Petri}, + month = may, + year = {2015}, + pages = {4169--4195}, +} + + +@article{hodzic_rethinking_2016, + title = {Rethinking the global secondary organic aerosol ({SOA}) budget: stronger production, faster removal, shorter lifetime}, + volume = {16}, + issn = {1680-7316}, + shorttitle = {Rethinking the global secondary organic aerosol ({SOA}) budget}, + url = {https://acp.copernicus.org/articles/16/7917/2016/}, + doi = {10.5194/acp-16-7917-2016}, + language = {English}, + number = {12}, + urldate = {2024-03-29}, + journal = {Atmospheric Chemistry and Physics}, + author = {Hodzic, Alma and Kasibhatla, Prasad S. and Jo, Duseong S. and Cappa, Christopher D. and Jimenez, Jose L. and Madronich, Sasha and Park, Rokjin J.}, + month = jun, + year = {2016}, + pages = {7917--7941}, +} + +@article{kok_improved_2014, + title = {An improved dust emission model – {Part} 1: {Model} description and comparison against measurements}, + volume = {14}, + issn = {1680-7316}, + shorttitle = {An improved dust emission model – {Part} 1}, + url = {https://acp.copernicus.org/articles/14/13023/2014/}, + doi = {10.5194/acp-14-13023-2014}, + language = {English}, + number = {23}, + urldate = {2024-03-29}, + journal = {Atmospheric Chemistry and Physics}, + author = {Kok, J. F. and Mahowald, N. M. and Fratini, G. and Gillies, J. A. and Ishizuka, M. and Leys, J. F. and Mikami, M. and Park, M.-S. and Park, S.-U. and Van Pelt, R. S. and Zobeck, T. M.}, + month = dec, + year = {2014}, + pages = {13023--13041}, +} + +@article{zender_mineral_2003, + title = {Mineral {Dust} {Entrainment} and {Deposition} ({DEAD}) model: {Description} and 1990s dust climatology}, + volume = {108}, + issn = {0148-0227}, + shorttitle = {Mineral {Dust} {Entrainment} and {Deposition} ({DEAD}) model}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/10.1029/2002JD002775}, + doi = {10.1029/2002JD002775}, + language = {en}, + number = {D14}, + urldate = {2024-03-29}, + journal = {Journal of Geophysical Research: Atmospheres}, + author = {Zender, Charles S. and Bian, Huisheng and Newman, David}, + month = jul, + year = {2003}, + pages = {2002JD002775}, +} + +@article{martensson_laboratory_2003, + title = {Laboratory simulations and parameterization of the primary marine aerosol production}, + volume = {108}, + issn = {0148-0227}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/10.1029/2002JD002263}, + doi = {10.1029/2002JD002263}, + language = {en}, + number = {D9}, + urldate = {2024-03-29}, + journal = {Journal of Geophysical Research: Atmospheres}, + author = {Mårtensson, E. M. and Nilsson, E. D. and De Leeuw, G. and Cohen, L. H. and Hansson, H.‐C.}, + month = may, + year = {2003}, + pages = {2002JD002263}, +} + +@incollection{monahan_model_1986, + address = {Dordrecht}, + title = {A {Model} of {Marine} {Aerosol} {Generation} {Via} {Whitecaps} and {Wave} {Disruption}}, + isbn = {9789400946682}, + url = {https://doi.org/10.1007/978-94-009-4668-2_16}, + language = {en}, + urldate = {2024-03-29}, + booktitle = {Oceanic {Whitecaps}: {And} {Their} {Role} in {Air}-{Sea} {Exchange} {Processes}}, + publisher = {Springer Netherlands}, + author = {Monahan, E. C. and Spiel, D. E. and Davidson, K. L.}, + editor = {Monahan, Edward C. and Niocaill, Gearóid Mac}, + year = {1986}, + doi = {10.1007/978-94-009-4668-2_16}, + keywords = {Wind Speed, Droplet Radius, Marine Aerosol, Aerosol Generation, Aerosol Droplet}, + pages = {167--174}, +} + +@article{lee_e3sm_2024, + title = {{E3SM} {Chemistry} {Diagnostics} {Package} ({ChemDyg}) {Version} 0.1.4}, + url = {https://gmd.copernicus.org/preprints/gmd-2023-203/}, + doi = {10.5194/gmd-2023-203}, + language = {English}, + urldate = {2024-03-29}, + journal = {Geoscientific Model Development Discussions}, + author = {Lee, Hsiang-He and Tang, Qi and Prather, Michael}, + month = jan, + year = {2024}, + pages = {1--46}, +} + +@article{hsu_global_2010, + title = {Global long‐lived chemical modes excited in a 3‐{D} chemistry transport model: {Stratospheric} {N} $_{\textrm{2}}$ {O}, {NO} $_{\textrm{ \textit{y} }}$ , {O} $_{\textrm{3}}$ and {CH} $_{\textrm{4}}$ chemistry}, + volume = {37}, + issn = {0094-8276, 1944-8007}, + shorttitle = {Global long‐lived chemical modes excited in a 3‐{D} chemistry transport model}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/10.1029/2009GL042243}, + doi = {10.1029/2009GL042243}, + language = {en}, + number = {7}, + urldate = {2024-03-29}, + journal = {Geophysical Research Letters}, + author = {Hsu, Juno and Prather, Michael J.}, + month = apr, + year = {2010}, + pages = {2009GL042243}, +} + +@article{tang_evaluation_2021, + title = {Evaluation of the interactive stratospheric ozone ({O3v2}) module in the {E3SM} version 1 {Earth} system model}, + volume = {14}, + issn = {1991-9603}, + url = {https://gmd.copernicus.org/articles/14/1219/2021/}, + doi = {10.5194/gmd-14-1219-2021}, + language = {en}, + number = {3}, + urldate = {2024-03-29}, + journal = {Geoscientific Model Development}, + author = {Tang, Qi and Prather, Michael J. and Hsu, Juno and Ruiz, Daniel J. and Cameron-Smith, Philip J. and Xie, Shaocheng and Golaz, Jean-Christophe}, + month = mar, + year = {2021}, + pages = {1219--1236}, +} + +@article{zhang_understanding_2024, + title = {Understanding changes in cloud simulations from {E3SM} version 1 to version 2}, + volume = {17}, + issn = {1991-9603}, + url = {https://gmd.copernicus.org/articles/17/169/2024/}, + doi = {10.5194/gmd-17-169-2024}, + language = {en}, + number = {1}, + urldate = {2024-03-29}, + journal = {Geoscientific Model Development}, + author = {Zhang, Yuying and Xie, Shaocheng and Qin, Yi and Lin, Wuyin and Golaz, Jean-Christophe and Zheng, Xue and Ma, Po-Lun and Qian, Yun and Tang, Qi and Terai, Christopher R. and Zhang, Meng}, + month = jan, + year = {2024}, + pages = {169--189}, +} + +@article{zhang_evaluation_2019, + title = {Evaluation of {Clouds} in {Version} 1 of the {E3SM} {Atmosphere} {Model} {With} {Satellite} {Simulators}}, + volume = {11}, + issn = {1942-2466, 1942-2466}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/10.1029/2018MS001562}, + doi = {10.1029/2018MS001562}, + language = {en}, + number = {5}, + urldate = {2024-03-29}, + journal = {Journal of Advances in Modeling Earth Systems}, + author = {Zhang, Yuying and Xie, Shaocheng and Lin, Wuyin and Klein, Stephen A. and Zelinka, Mark and Ma, Po‐Lun and Rasch, Philip J. and Qian, Yun and Tang, Qi and Ma, Hsi‐Yen}, + month = may, + year = {2019}, + pages = {1253--1268}, +} + +@article{swales_cloud_2018, + title = {The {Cloud} {Feedback} {Model} {Intercomparison} {Project} {Observational} {Simulator} {Package}: {Version} 2}, + volume = {11}, + issn = {1991-9603}, + shorttitle = {The {Cloud} {Feedback} {Model} {Intercomparison} {Project} {Observational} {Simulator} {Package}}, + url = {https://gmd.copernicus.org/articles/11/77/2018/}, + doi = {10.5194/gmd-11-77-2018}, + language = {en}, + number = {1}, + urldate = {2024-03-29}, + journal = {Geoscientific Model Development}, + author = {Swales, Dustin J. and Pincus, Robert and Bodas-Salcedo, Alejandro}, + month = jan, + year = {2018}, + pages = {77--81}, +} + +@article{bodas-salcedo_cosp_2011, + title = {{COSP}: {Satellite} simulation software for model assessment}, + volume = {92}, + issn = {0003-0007, 1520-0477}, + shorttitle = {{COSP}}, + url = {https://journals.ametsoc.org/doi/10.1175/2011BAMS2856.1}, + doi = {10.1175/2011BAMS2856.1}, + language = {en}, + number = {8}, + urldate = {2024-03-29}, + journal = {Bulletin of the American Meteorological Society}, + author = {Bodas-Salcedo, A. and Webb, M. J. and Bony, S. and Chepfer, H. and Dufresne, J.-L. and Klein, S. A. and Zhang, Y. and Marchand, R. and Haynes, J. M. and Pincus, R. and John, V. O.}, + month = aug, + year = {2011}, + pages = {1023--1043}, +} + +@article{zhang_arm_2020, + title = {The {ARM} {Data}-{Oriented} {Metrics} and {Diagnostics} {Package} for {Climate} {Models}: {A} {New} {Tool} for {Evaluating} {Climate} {Models} with {Field} {Data}}, + volume = {101}, + issn = {0003-0007, 1520-0477}, + shorttitle = {The {ARM} {Data}-{Oriented} {Metrics} and {Diagnostics} {Package} for {Climate} {Models}}, + url = {https://journals.ametsoc.org/view/journals/bams/101/10/bamsD190282.xml}, + doi = {10.1175/BAMS-D-19-0282.1}, + language = {EN}, + number = {10}, + urldate = {2024-03-29}, + journal = {Bulletin of the American Meteorological Society}, + author = {Zhang, C. and Xie, S. and Tao, C. and Tang, S. and Emmenegger, T. and Neelin, J. D. and Schiro, K. A. and Lin, W. and Shaheen, Z.}, + month = oct, + year = {2020}, + pages = {E1619--E1627}, +} + +@article{zheng_assessment_2023, + title = {Assessment of {CMIP5} and {CMIP6} {AMIP} {Simulated} {Clouds} and {Surface} {Shortwave} {Radiation} {Using} {ARM} {Observations} over {Different} {Climate} {Regions}}, + volume = {36}, + issn = {0894-8755, 1520-0442}, + url = {https://journals.ametsoc.org/view/journals/clim/36/24/JCLI-D-23-0247.1.xml}, + doi = {10.1175/JCLI-D-23-0247.1}, + language = {EN}, + number = {24}, + urldate = {2024-03-29}, + journal = {Journal of Climate}, + author = {Zheng, Xiaojian and Tao, Cheng and Zhang, Chengzhu and Xie, Shaocheng and Zhang, Yuying and Xi, Baike and Dong, Xiquan}, + month = nov, + year = {2023}, + pages = {8475--8495}, +} + +@article{zhang_causes_2018, + title = {{CAUSES}: {Diagnosis} of the {Summertime} {Warm} {Bias} in {CMIP5} {Climate} {Models} at the {ARM} {Southern} {Great} {Plains} {Site}}, + volume = {123}, + issn = {2169-897X, 2169-8996}, + shorttitle = {{CAUSES}}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/10.1002/2017JD027200}, + doi = {10.1002/2017JD027200}, + language = {en}, + number = {6}, + urldate = {2024-03-29}, + journal = {Journal of Geophysical Research: Atmospheres}, + author = {Zhang, Chengzhu and Xie, Shaocheng and Klein, Stephen A. and Ma, Hsi‐yen and Tang, Shuaiqi and Van Weverberg, Kwinten and Morcrette, Cyril J. and Petch, Jon}, + month = mar, + year = {2018}, + pages = {2968--2992}, +} + +@article{emmenegger_evaluating_2022, + title = {Evaluating {Tropical} {Precipitation} {Relations} in {CMIP6} {Models} with {ARM} {Data}}, + volume = {35}, + issn = {0894-8755, 1520-0442}, + url = {https://journals.ametsoc.org/view/journals/clim/35/19/JCLI-D-21-0386.1.xml}, + doi = {10.1175/JCLI-D-21-0386.1}, + number = {19}, + urldate = {2024-03-29}, + journal = {Journal of Climate}, + author = {Emmenegger, Todd and Kuo, Yi-Hung and Xie, Shaocheng and Zhang, Chengzhu and Tao, Cheng and Neelin, J. David}, + month = oct, + year = {2022}, + pages = {6343--6360}, +} + +@article{zhang_e3sm_2022, + title = {The {E3SM} {Diagnostics} {Package} ({E3SM} {Diags} v2.7): a {Python}-based diagnostics package for {Earth} system model evaluation}, + volume = {15}, + issn = {1991-9603}, + shorttitle = {The {E3SM} {Diagnostics} {Package} ({E3SM} {Diags} v2.7)}, + url = {https://gmd.copernicus.org/articles/15/9031/2022/}, + doi = {10.5194/gmd-15-9031-2022}, + language = {en}, + number = {24}, + urldate = {2024-03-29}, + journal = {Geoscientific Model Development}, + author = {Zhang, Chengzhu and Golaz, Jean-Christophe and Forsyth, Ryan and Vo, Tom and Xie, Shaocheng and Shaheen, Zeshawn and Potter, Gerald L. and Asay-Davis, Xylar S. and Zender, Charles S. and Lin, Wuyin and Chen, Chih-Chieh and Terai, Chris R. and Mahajan, Salil and Zhou, Tian and Balaguru, Karthik and Tang, Qi and Tao, Cheng and Zhang, Yuying and Emmenegger, Todd and Burrows, Susannah and Ullrich, Paul A.}, + month = dec, + year = {2022}, + pages = {9031--9056}, +} + + +@article{taylor_energy_2020, + title = {An {Energy} {Consistent} {Discretization} of the {Nonhydrostatic} {Equations} in {Primitive} {Variables}}, + volume = {12}, + issn = {1942-2466, 1942-2466}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/10.1029/2019MS001783}, + doi = {10.1029/2019MS001783}, + language = {en}, + number = {1}, + urldate = {2024-03-29}, + journal = {Journal of Advances in Modeling Earth Systems}, + author = {Taylor, Mark A. and Guba, Oksana and Steyer, Andrew and Ullrich, Paul A. and Hall, David M. and Eldred, Christopher}, + month = jan, + year = {2020}, + pages = {e2019MS001783}, +} + +@article{bradley_islet_2022, + title = {Islet: interpolation semi-{Lagrangian} element-based transport}, + volume = {15}, + issn = {1991-9603}, + shorttitle = {Islet}, + url = {https://gmd.copernicus.org/articles/15/6285/2022/}, + doi = {10.5194/gmd-15-6285-2022}, + language = {en}, + number = {16}, + urldate = {2024-03-29}, + journal = {Geoscientific Model Development}, + author = {Bradley, Andrew M. and Bosler, Peter A. and Guba, Oksana}, + month = aug, + year = {2022}, + pages = {6285--6310}, +} + +@techreport{guba_spectral_2014, + type = {preprint}, + title = {The spectral element method on variable resolution grids: evaluating grid sensitivity and resolution-aware numerical viscosity}, + shorttitle = {The spectral element method on variable resolution grids}, + url = {https://gmd.copernicus.org/preprints/7/4081/2014/gmdd-7-4081-2014.pdf}, + urldate = {2024-03-29}, + institution = {Numerical Methods}, + author = {Guba, O. and Taylor, M. A. and Ullrich, P. A. and Overfelt, J. R. and Levy, M. N.}, + month = jun, + year = {2014}, + doi = {10.5194/gmdd-7-4081-2014}, +} + +@article{taylor_compatible_2010, + title = {A compatible and conservative spectral element method on unstructured grids}, + volume = {229}, + issn = {00219991}, + url = {https://linkinghub.elsevier.com/retrieve/pii/S0021999110001841}, + doi = {10.1016/j.jcp.2010.04.008}, + language = {en}, + number = {17}, + urldate = {2024-03-29}, + journal = {Journal of Computational Physics}, + author = {Taylor, Mark A. and Fournier, Aimé}, + month = aug, + year = {2010}, + pages = {5879--5895}, +} + +@misc{elliott_macromolecule_2015, + title = {Macromolecule distributions input file for the {OCEANFILMS} parameterization}, + copyright = {Creative Commons Attribution 4.0 International, Open Access}, + url = {https://zenodo.org/record/6320812}, + doi = {10.5281/ZENODO.6320812}, + urldate = {2024-03-29}, + publisher = {[object Object]}, + author = {Elliott, Scott M. and Maltrud, Mathew and Burrows, Susannah M.}, + month = nov, + year = {2015}, + keywords = {Sea spray organic matter, Emissions parameterization}, +} + +@article{wang_influence_2015, + title = {Influence of explicit \textit{{Phaeocystis}} parameterizations on the global distribution of marine dimethyl sulfide}, + volume = {120}, + issn = {2169-8953, 2169-8961}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/10.1002/2015JG003017}, + doi = {10.1002/2015JG003017}, + language = {en}, + number = {11}, + urldate = {2024-03-29}, + journal = {Journal of Geophysical Research: Biogeosciences}, + author = {Wang, Shanlin and Elliott, Scott and Maltrud, Mathew and Cameron‐Smith, Philip}, + month = nov, + year = {2015}, + pages = {2158--2177}, +} + +@article{burrows_physically_2014, + title = {A physically based framework for modeling the organic fractionation of sea spray aerosol from bubble film {Langmuir} equilibria}, + volume = {14}, + issn = {1680-7316}, + url = {https://acp.copernicus.org/articles/14/13601/2014/}, + doi = {10.5194/acp-14-13601-2014}, + language = {English}, + number = {24}, + urldate = {2024-03-29}, + journal = {Atmospheric Chemistry and Physics}, + author = {Burrows, S. M. and Ogunro, O. and Frossard, A. A. and Russell, L. M. and Rasch, P. J. and Elliott, S. M.}, + month = dec, + year = {2014}, + pages = {13601--13629}, +} + +@article{burrows_oceanfilms_2022, + title = {{OCEANFILMS} ({Organic} {Compounds} from {Ecosystems} to {Aerosols}: {Natural} {Films} and {Interfaces} via {Langmuir} {Molecular} {Surfactants}) sea spray organic aerosol emissions – implementation in a global climate model and impacts on clouds}, + volume = {22}, + issn = {1680-7316}, + shorttitle = {{OCEANFILMS} ({Organic} {Compounds} from {Ecosystems} to {Aerosols}}, + url = {https://acp.copernicus.org/articles/22/5223/2022/}, + doi = {10.5194/acp-22-5223-2022}, + language = {English}, + number = {8}, + urldate = {2024-03-29}, + journal = {Atmospheric Chemistry and Physics}, + author = {Burrows, Susannah M. and Easter, Richard C. and Liu, Xiaohong and Ma, Po-Lun and Wang, Hailong and Elliott, Scott M. and Singh, Balwinder and Zhang, Kai and Rasch, Philip J.}, + month = apr, + year = {2022}, + pages = {5223--5251}, +} + +@article{maltrud_global_1998, + title = {Global eddy‐resolving ocean simulations driven by 1985–1995 atmospheric winds}, + volume = {103}, + issn = {0148-0227}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/10.1029/1998JC900013}, + doi = {10.1029/1998JC900013}, + language = {en}, + number = {C13}, + urldate = {2024-03-29}, + journal = {Journal of Geophysical Research: Oceans}, + author = {Maltrud, Mathew E. and Smith, Richard D. and Semtner, Albert J. and Malone, Robert C.}, + month = dec, + year = {1998}, + pages = {30825--30853}, +} + +@article{moore_upper_2004, + title = {Upper ocean ecosystem dynamics and iron cycling in a global three‐dimensional model}, + volume = {18}, + issn = {0886-6236, 1944-9224}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/10.1029/2004GB002220}, + doi = {10.1029/2004GB002220}, + language = {en}, + number = {4}, + urldate = {2024-03-29}, + journal = {Global Biogeochemical Cycles}, + author = {Moore, J. Keith and Doney, Scott C. and Lindsay, Keith}, + month = dec, + year = {2004}, + pages = {2004GB002220}, +} + +@article{liu_toward_2012, + title = {Toward a minimal representation of aerosols in climate models: description and evaluation in the {Community} {Atmosphere} {Model} {CAM5}}, + volume = {5}, + issn = {1991-9603}, + shorttitle = {Toward a minimal representation of aerosols in climate models}, + url = {https://gmd.copernicus.org/articles/5/709/2012/}, + doi = {10.5194/gmd-5-709-2012}, + language = {en}, + number = {3}, + urldate = {2024-03-29}, + journal = {Geoscientific Model Development}, + author = {Liu, X. and Easter, R. C. and Ghan, S. J. and Zaveri, R. and Rasch, P. and Shi, X. and Lamarque, J.-F. and Gettelman, A. and Morrison, H. and Vitt, F. and Conley, A. and Park, S. and Neale, R. and Hannay, C. and Ekman, A. M. L. and Hess, P. and Mahowald, N. and Collins, W. and Iacono, M. J. and Bretherton, C. S. and Flanner, M. G. and Mitchell, D.}, + month = may, + year = {2012}, + pages = {709--739}, +} + +@article{wang_aerosols_2020, + title = {Aerosols in the {E3SM} {Version} 1: {New} {Developments} and {Their} {Impacts} on {Radiative} {Forcing}}, + volume = {12}, + issn = {1942-2466, 1942-2466}, + shorttitle = {Aerosols in the {E3SM} {Version} 1}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/10.1029/2019MS001851}, + doi = {10.1029/2019MS001851}, + language = {en}, + number = {1}, + urldate = {2024-03-29}, + journal = {Journal of Advances in Modeling Earth Systems}, + author = {Wang, Hailong and Easter, Richard C. and Zhang, Rudong and Ma, Po‐Lun and Singh, Balwinder and Zhang, Kai and Ganguly, Dilip and Rasch, Philip J. and Burrows, Susannah M. and Ghan, Steven J. and Lou, Sijia and Qian, Yun and Yang, Yang and Feng, Yan and Flanner, Mark and Leung, L. Ruby and Liu, Xiaohong and Shrivastava, Manish and Sun, Jian and Tang, Qi and Xie, Shaocheng and Yoon, Jin‐Ho}, + month = jan, + year = {2020}, + pages = {e2019MS001851}, +} + +@article{feng_global_2022, + title = {Global {Dust} {Cycle} and {Direct} {Radiative} {Effect} in {E3SM} {Version} 1: {Impact} of {Increasing} {Model} {Resolution}}, + volume = {14}, + issn = {1942-2466, 1942-2466}, + shorttitle = {Global {Dust} {Cycle} and {Direct} {Radiative} {Effect} in {E3SM} {Version} 1}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/10.1029/2021MS002909}, + doi = {10.1029/2021MS002909}, + language = {en}, + number = {7}, + urldate = {2024-03-29}, + journal = {Journal of Advances in Modeling Earth Systems}, + author = {Feng, Y. and Wang, H. and Rasch, P. J. and Zhang, K. and Lin, W. and Tang, Q. and Xie, S. and Hamilton, D. S. and Mahowald, N. and Yu, H.}, + month = jul, + year = {2022}, + pages = {e2021MS002909}, +} + +@article{dentener_emissions_2006, + title = {Emissions of primary aerosol and precursor gases in the years 2000 and 1750 prescribed data-sets for {AeroCom}}, + volume = {6}, + issn = {1680-7316}, + url = {https://acp.copernicus.org/articles/6/4321/2006/acp-6-4321-2006.html}, + doi = {10.5194/acp-6-4321-2006}, + language = {English}, + number = {12}, + urldate = {2024-04-24}, + journal = {Atmospheric Chemistry and Physics}, + author = {Dentener, F. and Kinne, S. and Bond, T. and Boucher, O. and Cofala, J. and Generoso, S. and Ginoux, P. and Gong, S. and Hoelzemann, J. J. and Ito, A. and Marelli, L. and Penner, J. E. and Putaud, J.-P. and Textor, C. and Schulz, M. and van der Werf, G. R. and Wilson, J.}, + year = {2006}, + pages = {4321--4344}, +} + +@article{visioni_limitations_2022, + title = {Limitations of assuming internal mixing between different aerosol species: a case study with sulfate geoengineering simulations}, + volume = {22}, + issn = {1680-7316}, + shorttitle = {Limitations of assuming internal mixing between different aerosol species}, + url = {https://acp.copernicus.org/articles/22/1739/2022/}, + doi = {10.5194/acp-22-1739-2022}, + language = {English}, + number = {3}, + urldate = {2024-04-25}, + journal = {Atmospheric Chemistry and Physics}, + author = {Visioni, Daniele and Tilmes, Simone and Bardeen, Charles and Mills, Michael and MacMartin, Douglas G. and Kravitz, Ben and Richter, Jadwiga H.}, + month = feb, + year = {2022}, + pages = {1739--1756}, +} + +@misc{neely_iii_volcaneesm_2016, + title = {{VolcanEESM}: {Global} volcanic sulphur dioxide ({SO2}) emissions database from 1850 to present - {Version} 1.0}, + shorttitle = {{VolcanEESM}}, + url = {https://catalogue.ceda.ac.uk/uuid/a8a7e52b299a46c9b09d8e56b283d385}, + doi = {10.5285/76EBDC0B-0EED-4F70-B89E-55E606BCD568}, + language = {en}, + urldate = {2024-04-25}, + publisher = {[object Object]}, + author = {Neely III, R. R. and Schmidt, A.}, + year = {2016}, + keywords = {Atmospheric Sciences, FOS: Earth and related environmental sciences}, +} + +@article{mills_global_2016, + title = {Global volcanic aerosol properties derived from emissions, 1990–2014, using {CESM1}({WACCM})}, + volume = {121}, + copyright = {http://onlinelibrary.wiley.com/termsAndConditions\#am}, + issn = {2169-897X, 2169-8996}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/10.1002/2015JD024290}, + doi = {10.1002/2015JD024290}, + language = {en}, + number = {5}, + urldate = {2024-04-25}, + journal = {Journal of Geophysical Research: Atmospheres}, + author = {Mills, Michael J. and Schmidt, Anja and Easter, Richard and Solomon, Susan and Kinnison, Douglas E. and Ghan, Steven J. and Neely, Ryan R. and Marsh, Daniel R. and Conley, Andrew and Bardeen, Charles G. and Gettelman, Andrew}, + month = mar, + year = {2016}, + pages = {2332--2348}, +} + + +@article{rinaldi_is_2013, + title = {Is chlorophyll‐ \textit{a} the best surrogate for organic matter enrichment in submicron primary marine aerosol?}, + volume = {118}, + copyright = {http://onlinelibrary.wiley.com/termsAndConditions\#vor}, + issn = {2169-897X, 2169-8996}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/10.1002/jgrd.50417}, + doi = {10.1002/jgrd.50417}, + language = {en}, + number = {10}, + urldate = {2024-04-25}, + journal = {Journal of Geophysical Research: Atmospheres}, + author = {Rinaldi, Matteo and Fuzzi, Sandro and Decesari, Stefano and Marullo, Salvatore and Santoleri, Rosalia and Provenzale, Antonello and Von Hardenberg, Jost and Ceburnis, Darius and Vaishya, Aditya and O'Dowd, Colin D. and Facchini, Maria Cristina}, + month = may, + year = {2013}, + pages = {4964--4973}, +} + +@article{gantt_wind_2011, + title = {Wind speed dependent size-resolved parameterization for the organic mass fraction of sea spray aerosol}, + volume = {11}, + issn = {1680-7316}, + url = {https://acp.copernicus.org/articles/11/8777/2011/acp-11-8777-2011.html}, + doi = {10.5194/acp-11-8777-2011}, + language = {English}, + number = {16}, + urldate = {2024-04-25}, + journal = {Atmospheric Chemistry and Physics}, + author = {Gantt, B. and Meskhidze, N. and Facchini, M. C. and Rinaldi, M. and Ceburnis, D. and O'Dowd, C. D.}, + month = aug, + year = {2011}, + pages = {8777--8790}, +} + +@article{quinn_contribution_2014, + title = {Contribution of sea surface carbon pool to organic matter enrichment in sea spray aerosol}, + volume = {7}, + copyright = {2014 Springer Nature Limited}, + issn = {1752-0908}, + url = {https://www.nature.com/articles/ngeo2092}, + doi = {10.1038/ngeo2092}, + language = {en}, + number = {3}, + urldate = {2024-04-25}, + journal = {Nature Geoscience}, + author = {Quinn, Patricia K. and Bates, Timothy S. and Schulz, Kristen S. and Coffman, D. J. and Frossard, A. A. and Russell, L. M. and Keene, W. C. and Kieber, D. J.}, + month = mar, + year = {2014}, + keywords = {Atmospheric chemistry, Marine biology, Marine chemistry}, + pages = {228--232}, +} + +@article{neale_description_2012, + title = {Description of the {NCAR} {Community} {Atmosphere} {Model} ({CAM} 5.0)}, + url = {https://opensky.ucar.edu/islandora/object/technotes%3A594/}, + doi = {10.5065/wgtk-4g06}, + language = {en}, + urldate = {2024-04-25}, + author = {Neale, Richard B. and Gettelman, Andrew and Park, Sungsu and Chen, Chih-Chieh and Lauritzen, Peter H. and Williamson, David L. and Conley, Andrew J. and Kinnison, Doug and Marsh, Dan and Smith, Anne K. and Vitt, Francis M. and Garcia, Rolando and Lamarque, Jean-Francois and Mills, Michael J. and Tilmes, Simone and Morrison, Hugh and Cameron-Smith, Philip and Collins, William D. and Iacono, Michael J. and Easter, Richard C. and Liu, Xiaohong and Ghan, Steven J. and Rasch, Philip J. and Taylor, Mark A.}, + journal = {UNKNOWN}, + year = {2012}, +} diff --git a/components/elm/docs/refs.bib b/docs/refs/elm.bib similarity index 100% rename from components/elm/docs/refs.bib rename to docs/refs/elm.bib diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md new file mode 100644 index 000000000000..c2daf519a42d --- /dev/null +++ b/docs/user-guide/index.md @@ -0,0 +1,69 @@ +# User Guide + +E3SM is not just one climate model but a modeling system that allows +many different configurations of atmosphere, ocean, land and other +components with both full model and data model options. Also, the configurations of model +components can run at different resolutions. Some configurations +can run easily on a laptop. Other's require the most powerful +supercomputers. + +Using the model requires first deciding what configuration to use and creating +a case with that configuration. +The configuration options are managed by the Case Control System (CCS) +within the +[Community Infrastructure for Modeling the Earth](https://esmci.github.io/cime/versions/master/html/what_cime/index.html) (CIME). + +Before reading the rest of this guide, you should become familiar with +cases, compsets and grids by reading the +[Case Control System Basic Usage](https://esmci.github.io/cime/versions/master/html/users_guide/index.html#case-control-system-part-1-basic-usage) + +A [step-by-step guide](https://docs.e3sm.org/running-e3sm-guide/) for running E3SM with a run script +is available. + +## Supported Coupled Compsets + +A *fully coupled* compset is one which has active components for at least the atmosphere, ocean, land surface, ocean and +sea-ice all interacting. Each compset is associated with a specific forcing condition. +Coupled compsets in E3SM are developed for three science-driven simulation campaigns: `water cycle change and impacts`, `human-earth system feedbacks`, and `polar processes, sea-level rise and coastal impacts`. The standard coupled configurations -- which consist of prognostic atmosphere, land, river, ocean and sea-ice components -- form the base physical coupled system and are mainly designed for `water cycle change and impacts` simulation campaign. +Below list the standard configuration compsets supported in the current version of E3SM: + +|Compset alias | Description | +|:----------- |:----------- | +|`WCYCL1850` | Standard configuration with pre-industrial climatological forcings | +|`WCYCL1850-4xCO2` | Same as `WCYCL1850` except with abrupt (then persistent) 4xCO2 forcing. | +|`WCYCL1850-1pctCO2` | Same as `WCYCL1850` except with 1 percent per year increase of CO2 concentration | +|`WCYCL1950` | Standard configuration with perpetual 1950 forcings | +|`WCYCL20TR` | Standard configuration with prescribed transient forcings over the historical period (1850-2014) | +|`WCYCLSSP245` | Standard configuration with prescribed SSP-245 forcings | +|`WCYCLSSP370` | Standard configuration with prescribed SSP-370 forcings | +|`WCYCLSSP585` | Standard configuration with prescribed SSP-585 forcings | + +The compsets for the other two science simulation campaigns are being finalized, with additional components and/or features. +The compset naming follows the same convention, e.g., `CRYO1850` and `CRYO1850-4xCO2` are with prognostic ice-shelf melt fluxes for the `polar processes` simulation campaign. + +Compsets are also available for standalone component model configurations, See the User Guides for the components for more information. + +## Supported resolution + +Currently two grid sets are supported for the above compsets, including a nominal low-resosluton confiuguration and one regionally refined mesh. Additional regionally refined meshes and a high-resolution grid will become available in the near future. + +| Grid Alias | Description | +| ----------- | ------------ | +|ne30pg2_r05_IcoswISC30E3r5 | For this grid set, the atmosphere is on the ne30pg2 cubed-sphere mesh with approximately 100km resolution, the land and river are on a 0.5deg x 0.5deg structured grid, and the ocean and sea ice are on a hexagonal mesh dervied from the dual of a 30km resolution icosahedral mesh with ice shelf cavities (wISC) around Antarctica.| +|northamericax4v1pg2_r025_IcoswISC30E3r5 | The atmosphere for this grid set uses North America regionally refined mesh from about 110 km down to 25 km over the refined region. The land and river are on 0.25deg x 0.25deg structured grid. The ocean and sea ice are on the same icosahedral mesh as for `ne30pg2_r05_IcoswISC30E3r5`.| + +## Input data + +Inputdata for coupled compsets at component model levels are the same as for the standalone component configurations +for a given forcing scenario (e.g., `1850` for the pre-industrial period, `20TR` for the historical period, `2010` +for present-day condition, and `SSPs` for Shared Socioeconomic Pathways of climate change scenarios). +Between the coupled compsets, the differences are in the prescribed solar forcing, volcanic emissions, +atmospheric forcing data, and land use and land cover. The required inputdata for the pre-industrial and the historical periods +as well as the present-day condition are described in [the EAM User's Guide](https://e3sm-project.github.io/E3SM/EAM) and +[the ELM's User's Guide](https://e3sm-project.github.io/E3SM/ELM). Below are the prescribed forcing data for the SSP scenarios. + +### [SSP245 forcing data](ssp245-forcings.md) + +### [SSP370 forcing data](ssp370-forcings.md) + +### [SSP585 forcing data](ssp585-forcings.md) diff --git a/docs/user-guide/ssp245-forcings.md b/docs/user-guide/ssp245-forcings.md new file mode 100644 index 000000000000..d4cbded160e1 --- /dev/null +++ b/docs/user-guide/ssp245-forcings.md @@ -0,0 +1,68 @@ +# SSP245 Forcing data + +These are the prescribed inputdata specifically for the SSP245 scenario, in place of the files for the historical period. + +## Solar constant + +`\$DIN_LOC_ROOT/atm/cam/solar/Solar_1850-2299_input4MIPS_c20181106.nc` + +## Greenhouse gas concentrations + +`\$DIN_LOC_ROOT/atm/cam/ggas/GHG_CMIP_SSP245-1-2-1_Annual_Global_2015-2500_c20200807.nc` + +## Elevated external forcings + +```fortran +NO2 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_NO2_aircraft_vertical_2015-2100_1.9x2.5_c20240219.nc +SO2 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_so2_volc_elev_2015-2100_c240331.nc +SOAG0 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_SOAG0_elev_2015-2100_1.9x2.5_c20240219.nc +bc_a4 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_bc_a4_elev_2015-2100_c200716.nc +num_a1 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_num_a1_elev_2015-2100_c200716.nc +num_a2 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_num_a2_elev_2015-2100_c200716.nc +num_a4 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_num_a4_elev_2015-2100_c200716.nc +pom_a4 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_pom_a4_elev_2015-2100_c200716.nc +so4_a1 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_so4_a1_elev_2015-2100_c200716.nc +so4_a2 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_so4_a2_elev_2015-2100_c200716.nc +``` + +## Surface emissions + +```fortran +C2H4 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_C2H4_surface_2015-2100_1.9x2.5_c20240219.nc +C2H6 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_C2H6_surface_2015-2100_1.9x2.5_c20240219.nc +C3H8 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_C3H8_surface_2015-2100_1.9x2.5_c20240219.nc +CH2O \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_CH2O_surface_2015-2100_1.9x2.5_c20240219.nc +CH3CHO \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_CH3CHO_surface_2015-2100_1.9x2.5_c20240219.nc +CH3COCH3 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_CH3COCH3_surface_2015-2100_1.9x2.5_c20240219.nc +CO \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_CO_surface_2015-2100_1.9x2.5_c20240219.nc +ISOP \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_ISOP_surface_2015-2100_1.9x2.5_c20240219.nc +ISOP_VBS \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_ISOP_surface_2015-2100_1.9x2.5_c20240219.nc +C10H16 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_MTERP_surface_2015-2100_1.9x2.5_c20240219.nc +NOX \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_NO_surface_2015-2100_1.9x2.5_c20240219.nc +DMS \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DMSflux.1850-2100.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20160727.nc +SO2 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_so2_surf_2015-2100_c200716.nc +SOAG0 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_SOAG0_surf_2015-2100_1.9x2.5_c20240219.nc +bc_a4 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_bc_a4_surf_2015-2100_c200716.nc +num_a1 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_num_a1_surf_2015-2100_c200716.nc +num_a2 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_num_a2_surf_2015-2100_c200716.nc +num_a4 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_num_a4_surf_2015-2100_c200716.nc +pom_a4 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_pom_a4_surf_2015-2100_c200716.nc +so4_a1 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_so4_a1_surf_2015-2100_c200716.nc +so4_a2 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/cmip6_ssp245_mam4_so4_a2_surf_2015-2100_c200716.nc +E90 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart/ub/emissions_E90_surface_1750-2101_1.9x2.5_c20231222.nc +``` + +## Prescribed oxidant for aerosol chemistry + +`$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/oxid/oxid_SSP245_1.9x2.5_L70_1849-2101_c20240228.nc` + +## Stratospheric ozone (linoz) and chlorine loading data + +```fortran +\$DIN_LOC_ROOT/atm/cam/chem/trop_mozart/ub/Linoz_Chlorine_Loading_CMIP6_Hist_SSP245_0003-2503_c20200808.nc +\$DIN_LOC_ROOT/atm/cam/chem/trop_mozart/ub/linv3_1849-2101_CMIP6_Hist_SSP245_10deg_58km_c20230705.nc +``` + +## Land use and land cover + +`\$DIN_LOC_ROOT/lnd/clm2/surfdata_map/landuse.timeseries_0.5x0.5_ssp2_rcp45_simyr2015-2100_c240408.nc` diff --git a/docs/user-guide/ssp370-forcings.md b/docs/user-guide/ssp370-forcings.md new file mode 100644 index 000000000000..29477a5d1796 --- /dev/null +++ b/docs/user-guide/ssp370-forcings.md @@ -0,0 +1,68 @@ +# SSP370 Forcing data + +These are the prescribed inputdata specifically for the SSP370 scenario, in place of the files for the historical period. + +## Solar constant + +`\$DIN_LOC_ROOT/atm/cam/solar/Solar_1850-2299_input4MIPS_c20181106.nc` + +## Greenhouse gas concentrations + +`\$DIN_LOC_ROOT/atm/cam/ggas/GHG_CMIP_SSP370-1-2-1_Annual_Global_2015-2500_c20210509.nc` + +## Elevated external forcings + +```fortran +NO2 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_NO2_aircraft_vertical_2015-2100_1.9x2.5_c20240208.nc +SO2 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_so2_volc_elev_2015-2100_c240331.nc +SOAG0 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_SOAG0_elev_2015-2100_1.9x2.5_c20240208.nc +bc_a4 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_bc_a4_elev_2015-2100_c210216.nc +num_a1 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_num_a1_elev_2015-2100_c210216.nc +num_a2 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_num_a2_elev_2015-2100_c210216.nc +num_a4 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_num_a4_elev_2015-2100_c210216.nc +pom_a4 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_pom_a4_elev_2015-2100_c210216.nc +so4_a1 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_so4_a1_elev_2015-2100_c210216.nc +so4_a2 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_so4_a2_elev_2015-2100_c210216.nc +``` + +## Surface emissions + +```fortran +C2H4 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_C2H4_surface_2015-2100_1.9x2.5_c20240208.nc +C2H6 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_C2H6_surface_2015-2100_1.9x2.5_c20240208.nc +C3H8 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_C3H8_surface_2015-2100_1.9x2.5_c20240208.nc +CH2O \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_CH2O_surface_2015-2100_1.9x2.5_c20240208.nc +CH3CHO \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_CH3CHO_surface_2015-2100_1.9x2.5_c20240208.nc +CH3COCH3 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_CH3COCH3_surface_2015-2100_1.9x2.5_c20240208.nc +CO \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_CO_surface_2015-2100_1.9x2.5_c20240208.nc +ISOP \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_ISOP_surface_2015-2100_1.9x2.5_c20240208.nc +ISOP_VBS \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_ISOP_surface_2015-2100_1.9x2.5_c20240208.nc +C10H16 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_MTERP_surface_2015-2100_1.9x2.5_c20240208.nc +NOX \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_NO_surface_2015-2100_1.9x2.5_c20240208.nc +DMS \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DMSflux.1850-2100.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20160727.nc +SO2 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_so2_surf_2015-2100_c210216.nc +SOAG0 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_SOAG0_surf_2015-2100_1.9x2.5_c20240208.nc +bc_a4 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_bc_a4_surf_2015-2100_c210216.nc +num_a1 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_num_a1_surf_2015-2100_c210216.nc +num_a2 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_num_a2_surf_2015-2100_c210216.nc +num_a4 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_num_a4_surf_2015-2100_c210216.nc +pom_a4 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_pom_a4_surf_2015-2100_c210216.nc +so4_a1 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_so4_a1_surf_2015-2100_c210216.nc +so4_a2 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/cmip6_ssp370_mam4_so4_a2_surf_2015-2100_c210216.nc +E90 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart/ub/emissions_E90_surface_1750-2101_1.9x2.5_c20231222.nc +``` + +## Prescribed oxidant for aerosol chemistry + +`\$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/oxid/oxid_SSP370_1.9x2.5_L70_1849-2101_c20240228.nc` + +## Stratospheric ozone (linoz) and chlorine loading data + +```fortran +\$DIN_LOC_ROOT/atm/cam/chem/trop_mozart/ub/Linoz_Chlorine_Loading_CMIP6_Hist_SSP370_0003-2503_c20210202.nc +\$DIN_LOC_ROOT/atm/cam/chem/trop_mozart/ub/linv3_1849-2101_CMIP6_Hist_SSP370_10deg_58km_c20230705.nc +``` + +## Land use and land cover + +`\$DIN_LOC_ROOT/lnd/clm2/surfdata_map/landuse.timeseries_0.5x0.5_ssp3_rcp70_simyr2015-2100_c240308.nc` diff --git a/docs/user-guide/ssp585-forcings.md b/docs/user-guide/ssp585-forcings.md new file mode 100644 index 000000000000..2c5b6ec26492 --- /dev/null +++ b/docs/user-guide/ssp585-forcings.md @@ -0,0 +1,68 @@ +# SSP585 Forcing data + +These are the prescribed inputdata specifically for the SSP585 scenario, in place of the files for the historical period. + +## Solar constant + +`\$DIN_LOC_ROOT/atm/cam/solar/Solar_1850-2299_input4MIPS_c20181106.nc` + +## Greenhouse gas concentrations + +`\$DIN_LOC_ROOT/atm/cam/ggas/GHG_CMIP_SSP585-1-2-1_Annual_Global_2015-2500_c20190310.nc` + +## Elevated external forcings + +```fortran +NO2 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_NO2_aircraft_vertical_2015-2100_1.9x2.5_c20240304.nc +SO2 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_so2_volc_elev_2015-2100_c240331.nc +SOAG0 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_SOAG0_elev_2015-2100_1.9x2.5_c20240304.nc +bc_a4 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_bc_a4_elev_2015-2100_c190828.nc +num_a1 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_num_a1_elev_2015-2100_c190828.nc +num_a2 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_num_a2_elev_2015-2100_c190828.nc +num_a4 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_num_a4_elev_2015-2100_c190828.nc +pom_a4 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_pom_a4_elev_2015-2100_c190828.nc +so4_a1 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_so4_a1_elev_2015-2100_c190828.nc +so4_a2 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_so4_a2_elev_2015-2100_c190828.nc +``` + +## Surface emissions + +```fortran +C2H4 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_C2H4_surface_2015-2100_1.9x2.5_c20240304.nc +C2H6 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_C2H6_surface_2015-2100_1.9x2.5_c20240304.nc +C3H8 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_C3H8_surface_2015-2100_1.9x2.5_c20240304.nc +CH2O \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_CH2O_surface_2015-2100_1.9x2.5_c20240304.nc +CH3CHO \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_CH3CHO_surface_2015-2100_1.9x2.5_c20240304.nc +CH3COCH3 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_CH3COCH3_surface_2015-2100_1.9x2.5_c20240304.nc +CO \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_CO_surface_2015-2100_1.9x2.5_c20240304.nc +ISOP \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_ISOP_surface_2015-2100_1.9x2.5_c20240304.nc +ISOP_VBS \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_ISOP_surface_2015-2100_1.9x2.5_c20240304.nc +C10H16 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_MTERP_surface_2015-2100_1.9x2.5_c20240304.nc +NOX \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_NO_surface_2015-2100_1.9x2.5_c20240304.nc +DMS \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DMSflux.1850-2100.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20160727.nc +SO2 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_so2_surf_2015-2100_c190828.nc +SOAG0 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/emissions-cmip6_ssp585_e3sm_SOAG0_surf_2015-2100_1.9x2.5_c20240304.nc +bc_a4 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_bc_a4_surf_2015-2100_c190828.nc +num_a1 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_num_a1_surf_2015-2100_c190828.nc +num_a2 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_num_a2_surf_2015-2100_c190828.nc +num_a3 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_num_a4_surf_2015-2100_c190828.nc +pom_a4 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_pom_a4_surf_2015-2100_c190828.nc +so4_a1 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_so4_a1_surf_2015-2100_c190828.nc +so4_a2 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP585_ne30/cmip6_ssp585_mam4_so4_a2_surf_2015-2100_c190828.nc +E90 \$DIN_LOC_ROOT/atm/cam/chem/trop_mozart/ub/emissions_E90_surface_1750-2101_1.9x2.5_c20231222.nc +``` + +## Prescribed oxidant for aerosol chemistry + +`\$DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/oxid/oxid_SSP585_1.9x2.5_L70_2014-2101_c20240228.nc` + +## Stratospheric ozone (linoz) and chlorine loading data + +```fortran +\$DIN_LOC_ROOT/atm/cam/chem/trop_mozart/ub/Linoz_Chlorine_Loading_CMIP6_Hist_SSP585_0003-2503_c20190414.nc +\$DIN_LOC_ROOT/atm/cam/chem/trop_mozart/ub/linv3_1849-2101_CMIP6_Hist_SSP585_10deg_58km_c20230705.nc +``` + +## Land use and land cover + +`\$DIN_LOC_ROOT/lnd/clm2/surfdata_map/landuse.timeseries_0.5x0.5_ssp5_rcp85_simyr2015-2100_c240408.nc` diff --git a/driver-mct/cime_config/buildnml b/driver-mct/cime_config/buildnml index 426995b8ffb6..4938e9da0b18 100755 --- a/driver-mct/cime_config/buildnml +++ b/driver-mct/cime_config/buildnml @@ -40,6 +40,7 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files): config['CPL_ALBAV'] = case.get_value('CPL_ALBAV') config['CPL_EPBAL'] = case.get_value('CPL_EPBAL') config['FLDS_WISO'] = case.get_value('FLDS_WISO') + config['FLDS_POLAR'] = case.get_value('FLDS_POLAR') config['BUDGETS'] = case.get_value('BUDGETS') config['MACH'] = case.get_value('MACH') config['MPILIB'] = case.get_value('MPILIB') diff --git a/driver-mct/cime_config/config_component.xml b/driver-mct/cime_config/config_component.xml index 9ebfb78234b4..5b5126af8930 100644 --- a/driver-mct/cime_config/config_component.xml +++ b/driver-mct/cime_config/config_component.xml @@ -1085,8 +1085,8 @@ char - gland20,gland10,gland5,gland5UM,gland4,mpas.aisgis20km,mpas.gis20km,mpas.ais20km,mpas.gis1to10km,null - gland5UM + mpas.aisgis20km,mpas.gis20km,mpas.ais20km,mpas.gis1to10km,mpas.gis1to10kmR2,null + mpas.gis20km build_grid env_build.xml glacier (glc) grid - DO NOT EDIT (for experts only) @@ -1375,6 +1375,14 @@ atm2ocn flux mapping file + + char + idmap_ignore + run_domain + env_run.xml + atm2ice flux mapping file + + char idmap @@ -2684,10 +2692,10 @@ integer - 1,2 + 1,2,3 run_pio env_run.xml - pio rearranger choice box=1, subset=2 + pio rearranger choice box=1, subset=2, any=3 $PIO_VERSION $PIO_VERSION diff --git a/driver-mct/cime_config/config_component_e3sm.xml b/driver-mct/cime_config/config_component_e3sm.xml index 5d582b41b438..2895650d2426 100755 --- a/driver-mct/cime_config/config_component_e3sm.xml +++ b/driver-mct/cime_config/config_component_e3sm.xml @@ -172,6 +172,19 @@ Turn on the passing of water isotope fields through the coupler + + logical + TRUE,FALSE + FALSE + + TRUE + TRUE + + run_flags + env_run.xml + Turn on the passing of polar fields through the coupler + + char minus1p8,linear_salt,mushy @@ -265,6 +278,7 @@ CO2A CO2A CO2A + CO2A CO2A_OI CO2A_OI CO2C @@ -369,6 +383,10 @@ 24 48 48 + 180 + 360 + 720 + 1440 48 48 96 diff --git a/driver-mct/cime_config/namelist_definition_drv.xml b/driver-mct/cime_config/namelist_definition_drv.xml index 6f0bdbcd9794..7fbf83688c8a 100644 --- a/driver-mct/cime_config/namelist_definition_drv.xml +++ b/driver-mct/cime_config/namelist_definition_drv.xml @@ -137,6 +137,18 @@ + + logical + seq_flds + seq_cplflds_inparm + + If set to .true. Polar fields will be passed from the ocean to the coupler. + + + $FLDS_POLAR + + + logical seq_flds @@ -228,7 +240,6 @@ .false. - .true. diff --git a/driver-mct/cime_config/namelist_definition_modelio.xml b/driver-mct/cime_config/namelist_definition_modelio.xml index ce0275e59bf3..2860ffd7106f 100644 --- a/driver-mct/cime_config/namelist_definition_modelio.xml +++ b/driver-mct/cime_config/namelist_definition_modelio.xml @@ -90,9 +90,9 @@ integer pio pio_inparm - -99,1,2 + -99,1,2,3 - Rearranger method for pio 1=box, 2=subset. + Rearranger method for pio 1=box, 2=subset, 3=any. $CPL_PIO_REARRANGER diff --git a/driver-mct/main/cime_comp_mod.F90 b/driver-mct/main/cime_comp_mod.F90 index 1dd8526cee63..2131d0c86844 100644 --- a/driver-mct/main/cime_comp_mod.F90 +++ b/driver-mct/main/cime_comp_mod.F90 @@ -652,6 +652,37 @@ module cime_comp_mod integer, parameter :: comp_num_esp = 8 integer, parameter :: comp_num_iac = 9 + !---------------------------------------------------------------------------- + ! Data structures and parameters for rpointer-consistency management. + !---------------------------------------------------------------------------- + ! The number of components, including the driver. This should be 1 more than + ! the maximum comp_num_x integer. + integer, parameter :: rpointer_ncomp = 10 + ! Suffixes x for rpointer.x files. + character(3), parameter :: rpointer_suffixes(rpointer_ncomp) = & + ['atm', 'lnd', 'ice', 'ocn', 'glc', 'rof', 'wav', 'esp', 'iac', 'drv'] + ! Wrapper to a clock pointer. + type, private :: EClockPointer_t + type (ESMF_Clock), pointer :: ptr + end type EClockPointer_t + ! Manager data structure. + type, private :: RpointerMgr_t + ! Number of components active. + integer :: npresent + ! Program state variable. + logical :: remove_prev_in_next_call + ! Component i is present. + logical :: cpresent(rpointer_ncomp) + ! Component i's restart alarm rang. + logical :: rang(rpointer_ncomp) + ! Component i's clock. + type (EClockPointer_t) :: clock(rpointer_ncomp) + ! Verbosity flag + logical :: verbose + end type RpointerMgr_t + ! Manager object. + type (RpointerMgr_t) :: rpointer_mgr + !---------------------------------------------------------------------------- ! misc !---------------------------------------------------------------------------- @@ -1069,6 +1100,13 @@ subroutine cime_pre_init2() !mt call shr_mem_init(prt=.true.) call shr_mem_init(prt=iamroot_CPLID) + !---------------------------------------------------------- + !| Prepare consistent rpointer.x files + !---------------------------------------------------------- + ! This call must be made before the seq_infodata_init call below to make + ! rpointer.drv consistent. + call rpointer_prepare_restart() + !---------------------------------------------------------- !| Initialize infodata !---------------------------------------------------------- @@ -1383,7 +1421,6 @@ subroutine cime_pre_init2() call seq_infodata_GetData(infodata, bfbflag=bfbflag) write(logunit,'(2A,L4)') subname,'BFBFLAG is:',bfbflag endif - call t_stopf('CPL:cime_pre_init2') @@ -1649,6 +1686,11 @@ subroutine cime_init() iac_nx=iac_nx, iac_ny=iac_ny, & atm_aero=atm_aero ) + ! Initialize the rpointer manager. This is called after the restart routine + ! because we need to be further along in initialization to fully initialize + ! the manager. The restart routine doesn't need the initialized manager. + call rpointer_init_manager() + ! derive samegrid flags samegrid_ao = .true. @@ -2717,6 +2759,12 @@ subroutine cime_run() ! Does the driver need to pause? drv_pause = pause_alarm .and. seq_timemgr_pause_component_active(drv_index) + ! Monitor each component's restart alarm. Determine when to prepare + ! backup copies of the rpointer files and when it's OK to remove + ! these. .false. on input means never to force removal of the backup + ! files until the monitor's state machine says they can be removed. + call rpointer_manage(.false.) + if (glc_prognostic .or. do_hist_l2x1yrg) then ! Is it time to average fields to pass to glc? ! @@ -3544,6 +3592,12 @@ subroutine cime_final() call component_final(EClock_l, lnd, lnd_final) call component_final(EClock_a, atm, atm_final) + ! Finalize the rpointer manager. The manager can't always tell the earliest + ! time at which it can safely remove the backup rpointer files; pass + ! .true. here to force their removal if they still exist, because we now are + ! certain they can be removed. + call rpointer_manage(.true.) + !------------------------------------------------------------------------ ! End the run cleanly !------------------------------------------------------------------------ @@ -5125,4 +5179,413 @@ subroutine cime_write_performance_checkpoint(output_ckpt, ckpt_filename, & end subroutine cime_write_performance_checkpoint +!---------------------------------------------------------------------------------- +! +! The following subroutines improve robustness of restart writing and reading. +! +! It's possible for a crash that occurs during restart writing to lead to +! inconsistent or incomplete rpointer files. While we can't salvage the restart +! files in general, we can at least provide a consistent set of rpointer files +! -- namely, the previous ones -- for the next restart. +! +! These routines provide this capability by copying all rpointer.X files to +! rpointer.X.prev before components write restart files, then removing +! rpointer.X.prev files when all components are done. +! +! If a crash occurs midway through, on restart, the consistent rpointer.X.prev +! files will be used. +! +!---------------------------------------------------------------------------------- + + subroutine rpointer_prepare_restart() + ! Prepare to restart. If .prev file are present, something went wrong in the + ! previous run's final restart write. Use the .prev files instead of the + ! invalid regular ones. If there are no prev files, then this subroutine + ! doesn't do anything. + ! + ! This routine is called independently of the ones after it; in particular, + ! it does not require the manager to be initialized. + + integer :: i, n, idxlist(rpointer_ncomp), sleep_len, rcode, unit + logical :: file_exists, ok, same, complete + + ! Each rank checks if .prev files exist. + n = 0 + do i = 1, rpointer_ncomp + inquire(file='rpointer.'//rpointer_suffixes(i)//'.prev', & + exist=file_exists) + if (file_exists) then + n = n + 1 + idxlist(n) = i + end if + end do + + if (n == 0) return + + ! .prev files exist. + + ! Check that there is not an rpointer.x file with no corresponding + ! rpointer.x.prev file. If there is, then we assume the .prev files are + ! incomplete and error out. Note the presence of at least one + ! rpointer.x.prev file means something went wrong in the previous run, so + ! it's best to let the user sort things out. + if (iamroot_CPLID) then + complete = .true. + do i = 1, rpointer_ncomp + inquire(file='rpointer.'//rpointer_suffixes(i), & + exist=file_exists) + if (file_exists) then + inquire(file='rpointer.'//rpointer_suffixes(i)//'.prev', & + exist=file_exists) + if (.not. file_exists) then + complete = .false. + write(logunit,'(3a)') 'rpointer> ERROR: ', rpointer_suffixes(i), & + ' has an rpointer.x file with no corresponding rpointer.x.prev file' + end if + end if + end do + if (.not. complete) then + call shr_sys_abort('rpointer_prepare_restart: & + &rpointer.x.prev files exist but rpointer.y & + &has no corresponding rpointer.y.prev file.') + end if + end if + + ! The root rank copies the .prev files to regular files. + if (iamroot_CPLID) then + do i = 1, n + rcode = copy_and_trim_rpointer_file( & + 'rpointer.'//rpointer_suffixes(idxlist(i))//'.prev', & + 'rpointer.'//rpointer_suffixes(idxlist(i))) + if (rcode /= 0) write(logunit,*) 'rpointer> copy x.prev->x', rcode + end do + end if + + ! Read-after-write consistency generally does not hold, so each rank + ! waits until it does, as follows: Check if rpointer.x is the same as + ! rpointer.x.prev. If not, then sleep and loop to try again. The sleep + ! period doubles each try until 15 seconds have elapsed, at which point, + ! if consistency still doesn't hold, give up. + sleep_len = 1 + do while (.true.) + ok = .true. + do i = 1, n + same = are_files_same( & + 'rpointer.'//rpointer_suffixes(idxlist(i))//'.prev', & + 'rpointer.'//rpointer_suffixes(idxlist(i))) + if (.not. same) then + ok = .false. + exit + end if + end do + if (ok) exit + call sleep(sleep_len) + sleep_len = 2*sleep_len + ! Wait for up to 8 + 4 + 2 + 1 = 15 seconds. + if (sleep_len > 8) exit + end do + if (.not. ok) then + call shr_sys_abort('rpointer_prepare_restart: & + &Could not copy rpointer.x.prev to rpointer.x') + end if + ! This rank is consistent. Wait for everyone else. + call mpi_barrier(mpicom_GLOID, rcode) + + ! After the barrier exits, the root rank can delete the .prev files. + if (iamroot_CPLID) then + unit = shr_file_getUnit() + do i = 1, n + open(file='rpointer.'//rpointer_suffixes(i)//'.prev', & + unit=unit, iostat=rcode) + if (rcode == 0) close(unit=unit, status='delete', iostat=rcode) + end do + call shr_file_freeUnit(unit) + end if + + contains + + function are_files_same(afname, bfname) result(same) + ! Do files afname and bfname contain the same contents? + + character(*), intent(in) :: afname, bfname + + integer :: aunit, bunit, astat, bstat, i, line + character(1024) :: abuf, bbuf + logical :: same + + same = .true. + + aunit = shr_file_getUnit() + bunit = shr_file_getUnit() + + open(aunit, file=trim(afname), action='READ', iostat=astat) + open(bunit, file=trim(bfname), action='READ', iostat=bstat) + + if ((astat == 0) .neqv. (bstat == 0)) same = .false. + + if (same .and. astat /= 0) same = .false. + + if (same) then + astat = 0 + bstat = 0 + abuf(:) = ' ' + bbuf(:) = ' ' + line = 1 + do while (same .and. astat == 0 .and. bstat == 0) + read(aunit, '(a1024)', iostat=astat) abuf + read(bunit, '(a1024)', iostat=bstat) bbuf + if ((astat == 0) .neqv. (bstat == 0)) then + same = .false. + exit + end if + if (astat /= 0) exit + do i = 1, 1024 + if (abuf(i:i) /= bbuf(i:i)) then + same = .false. + exit + end if + end do + line = line + 1 + end do + end if + + close(aunit) + close(bunit) + call shr_file_freeUnit(aunit) + call shr_file_freeUnit(bunit) + + end function are_files_same + + end subroutine rpointer_prepare_restart + + subroutine rpointer_init_manager() + ! Initialize a manager that is accessed through calls to rpointer_manage. + + integer :: i, n + + rpointer_mgr%verbose = .false. + rpointer_mgr%rang(:) = .false. + + do i = 1, rpointer_ncomp + rpointer_mgr%clock(i)%ptr => null() + end do + + rpointer_mgr%cpresent(:) = .false. + n = 0 + + if (atm_present) then + n = n + 1 + rpointer_mgr%cpresent(comp_num_atm) = .true. + rpointer_mgr%clock(comp_num_atm)%ptr => EClock_a + end if + if (lnd_present) then + n = n + 1 + rpointer_mgr%cpresent(comp_num_lnd) = .true. + rpointer_mgr%clock(comp_num_lnd)%ptr => EClock_l + end if + if (ice_present) then + n = n + 1 + rpointer_mgr%cpresent(comp_num_ice) = .true. + rpointer_mgr%clock(comp_num_ice)%ptr => EClock_i + end if + if (ocn_present) then + n = n + 1 + rpointer_mgr%cpresent(comp_num_ocn) = .true. + rpointer_mgr%clock(comp_num_ocn)%ptr => EClock_o + end if + if (glc_present) then + n = n + 1 + rpointer_mgr%cpresent(comp_num_glc) = .true. + rpointer_mgr%clock(comp_num_glc)%ptr => EClock_g + end if + if (rof_present) then + n = n + 1 + rpointer_mgr%cpresent(comp_num_rof) = .true. + rpointer_mgr%clock(comp_num_rof)%ptr => EClock_r + end if + if (wav_present) then + n = n + 1 + rpointer_mgr%cpresent(comp_num_wav) = .true. + rpointer_mgr%clock(comp_num_wav)%ptr => EClock_w + end if + if (esp_present) then + n = n + 1 + rpointer_mgr%cpresent(comp_num_esp) = .true. + rpointer_mgr%clock(comp_num_esp)%ptr => EClock_e + end if + if (iac_present) then + n = n + 1 + rpointer_mgr%cpresent(comp_num_iac) = .true. + rpointer_mgr%clock(comp_num_iac)%ptr => EClock_z + end if + n = n + 1 + rpointer_mgr%cpresent(rpointer_ncomp) = .true. + rpointer_mgr%clock(rpointer_ncomp)%ptr => EClock_d + + rpointer_mgr%npresent = n + rpointer_mgr%remove_prev_in_next_call = .false. + + end subroutine rpointer_init_manager + + subroutine rpointer_manage(force_remove) + ! Call this routine at certain places in the driver loop. This subroutine + ! monitors the restart alarms of all the active components and carries out + ! the steps required for rpointer file consistency based on state. + + logical, intent(in) :: force_remove ! force removal of .prev files in this call + + integer :: i, n, rcode, unit + logical :: previous_rings, file_exists + character(32) :: buf + + if (.not. iamroot_CPLID) return + + if (rpointer_mgr%remove_prev_in_next_call .or. force_remove) then + ! Now we've been told the restart writes really are all valid. Remove the + ! .prev files. + unit = shr_file_getUnit() + do i = 1, size(rpointer_suffixes,1) + if (rpointer_mgr%cpresent(i)) then + buf = 'rpointer.'//rpointer_suffixes(i)//'.prev' + inquire(file=trim(buf), exist=file_exists) + if (file_exists) then + open(file=trim(buf), unit=unit, iostat=rcode) + if (rcode == 0) close(unit, status='delete', iostat=rcode) + end if + end if + end do + call shr_file_freeUnit(unit) + rpointer_mgr%rang(:) = .false. + rpointer_mgr%remove_prev_in_next_call = .false. + if (rpointer_mgr%verbose) write(logunit,*) 'rpointer> rm .prev files' + return + end if + + previous_rings = .false. + do i = 1, rpointer_ncomp + if (rpointer_mgr%rang(i)) previous_rings = .true. + end do + + n = 0 + do i = 1, rpointer_ncomp + if (.not. rpointer_mgr%cpresent(i)) cycle + if (.not. rpointer_mgr%rang(i)) then + if (seq_timemgr_alarmIsOn(rpointer_mgr%clock(i)%ptr, & + seq_timemgr_alarm_restart)) then + rpointer_mgr%rang(i) = .true. + end if + end if + if (rpointer_mgr%rang(i)) n = n + 1 + end do + + if (n > 0 .and. rpointer_mgr%verbose) then + write (logunit,*) 'rpointer> state' + do i = 1, rpointer_ncomp + if (rpointer_mgr%cpresent(i)) then + write (logunit, '(a,i2,i2,i2,a4,l2)') & + 'rpointer>',i,n,rpointer_mgr%npresent,rpointer_suffixes(i), & + rpointer_mgr%rang(i) + end if + end do + end if + + if (previous_rings .and. n == rpointer_mgr%npresent) then + ! All restart timers have rung. Get ready to remove the .prev files. We + ! don't want to do it in this rpointer_manage call, however. Because of + ! things like partial steps in the atmosphere model (initiated from + ! cime_final rather than cime_run), we can't yet be sure all + ! restart-related writes at the level of history tapes are complete. Set + ! our state to tell us that in the next call, we can remove the files. + rpointer_mgr%remove_prev_in_next_call = .true. + if (rpointer_mgr%verbose) & + write(logunit,*) 'rpointer> set remove_prev_in_next_call=true' + return + else if (.not. previous_rings) then + if (n == 0) then + ! Nothing happened. + return + else + ! A new round of restart writes is starting. Copy previous, valid + ! rpointer files to .prev in case one or more of the restart writes that + ! are about to occur fail. + do i = 1, size(rpointer_suffixes,1) + if (rpointer_mgr%cpresent(i)) then + buf = 'rpointer.'//rpointer_suffixes(i) + inquire(file=trim(buf), exist=file_exists) + if (file_exists) then + rcode = copy_and_trim_rpointer_file(trim(buf), & + 'rpointer.'//rpointer_suffixes(i)//'.prev') + if (rcode /= 0 .and. rpointer_mgr%verbose) & + write(logunit,*) 'rpointer> copy x->x.prev',rcode + if (rpointer_mgr%verbose) then + if (rcode == 0) then + write(logunit,*) 'rpointer> copied: ', rpointer_suffixes(i) + else + write(logunit,*) 'rpointer> failed to copy: ', rpointer_suffixes(i) + end if + end if + end if + end if + end do + return + end if + end if + + ! If we reach this point, there were previous rings and n < npresent, + ! meaning > 2 iterations of the driver run loop will occur before all + ! restart alarms have rung, now that at least one has rung. Don't do + ! anything more yet. + + end subroutine rpointer_manage + + function copy_and_trim_rpointer_file(src, dst) result(out) + ! Copy rpointer file src to dst, with the caveat that the lines are + ! trimmed. We found that shr_file_put would result in mysterious errors + ! preventing copying, whereas this manual approach has yet to exhibit this + ! problem. + + character(*), intent(in) :: src, dst + + character(1024) :: buf + character(16) :: status + integer :: soi, doi, stat, out + logical :: file_exists + + soi = shr_file_getUnit() + out = 0 + open(soi, file=trim(src), status='old', action='read', iostat=stat) + if (stat /= 0) then + out = -3 + call shr_file_freeUnit(soi) + return + end if + inquire(file=trim(dst), exist=file_exists) + ! Probably not needed; always using 'replace' I think should work. + if (file_exists) then + status = 'replace' + else + status = 'new' + end if + doi = shr_file_getUnit() + open(doi, file=trim(dst), status=trim(status), action='write', iostat=stat) + if (stat /= 0) then + close(soi) + out = -2 + call shr_file_freeUnit(soi) + call shr_file_freeUnit(doi) + return + end if + do while (.true.) + read(soi, '(a1024)', iostat=stat) buf + if (stat /= 0) exit + write(doi, '(a)', iostat=stat) trim(buf) + if (stat /= 0) out = -1 + end do + close(soi) + close(doi) + call shr_file_freeUnit(soi) + call shr_file_freeUnit(doi) + + end function copy_and_trim_rpointer_file + end module cime_comp_mod diff --git a/driver-mct/main/seq_diag_mct.F90 b/driver-mct/main/seq_diag_mct.F90 index 2a7d999ccc5e..8534008f9be7 100644 --- a/driver-mct/main/seq_diag_mct.F90 +++ b/driver-mct/main/seq_diag_mct.F90 @@ -138,13 +138,13 @@ module seq_diag_mct integer(in),parameter :: f_hlatf = 8 ! heat : latent, fusion, snow integer(in),parameter :: f_hioff = 9 ! heat : latent, fusion, frozen runoff integer(in),parameter :: f_hsen =10 ! heat : sensible - integer(in),parameter :: f_hberg =11 ! heat : data icebergs + integer(in),parameter :: f_hpolar =11 ! heat : AIS imbalance integer(in),parameter :: f_hh2ot =12 ! heat : water temperature integer(in),parameter :: f_wfrz =13 ! water: freezing integer(in),parameter :: f_wmelt =14 ! water: melting integer(in),parameter :: f_wrain =15 ! water: precip, liquid integer(in),parameter :: f_wsnow =16 ! water: precip, frozen - integer(in),parameter :: f_wberg =17 ! water: data icebergs + integer(in),parameter :: f_wpolar =17 ! water: AIS imbalance integer(in),parameter :: f_wevap =18 ! water: evaporation integer(in),parameter :: f_wroff =19 ! water: runoff/flood integer(in),parameter :: f_wioff =20 ! water: frozen runoff @@ -189,8 +189,8 @@ module seq_diag_mct (/' area',' hfreeze',' hmelt',' hnetsw',' hlwdn', & ' hlwup',' hlatvap',' hlatfus',' hiroff',' hsen', & - ' hberg',' hh2otemp',' wfreeze',' wmelt',' wrain', & - ' wsnow',' wberg',' wevap',' wrunoff',' wfrzrof', & + ' hpolar',' hh2otemp',' wfreeze',' wmelt',' wrain', & + ' wsnow',' wpolar',' wevap',' wrunoff',' wfrzrof', & ' wirrig', & ' wfreeze_16O',' wmelt_16O',' wrain_16O',' wsnow_16O', & ' wevap_16O',' wrunoff_16O',' wfrzrof_16O', & @@ -286,7 +286,15 @@ module seq_diag_mct integer :: index_o2x_Faoo_h2otemp integer :: index_o2x_Fioo_frazil + integer :: index_o2x_Foxo_frazil_li integer :: index_o2x_Fioo_q + integer :: index_o2x_Foxo_q_li + + integer :: index_o2x_Foxo_ismw + integer :: index_o2x_Foxo_rrofl + integer :: index_o2x_Foxo_rrofi + integer :: index_o2x_Foxo_ismh + integer :: index_o2x_Foxo_rrofih integer :: index_xao_Faox_lwup integer :: index_xao_Faox_lat @@ -311,8 +319,6 @@ module seq_diag_mct integer :: index_i2x_Fioi_melth integer :: index_i2x_Fioi_meltw - integer :: index_i2x_Fioi_bergh - integer :: index_i2x_Fioi_bergw integer :: index_i2x_Fioi_salt integer :: index_i2x_Faii_swnet integer :: index_i2x_Fioi_swpen @@ -1337,9 +1343,10 @@ subroutine seq_diag_ocn_mct( ocn, xao_o, frac_o, infodata, do_o2x, do_x2o, do_xa integer(in) :: kArea ! index of area field in aVect integer(in) :: ko,ki ! fraction indices integer(in) :: lSize ! size of aVect - real(r8) :: ca_i,ca_o ! area of a grid cell + real(r8) :: ca_i,ca_o,ca_c ! area of a grid cell logical,save :: first_time = .true. logical,save :: flds_wiso_ocn = .false. + logical,save :: flds_polar = .false. !----- formats ----- character(*),parameter :: subName = '(seq_diag_ocn_mct) ' @@ -1371,8 +1378,18 @@ subroutine seq_diag_ocn_mct( ocn, xao_o, frac_o, infodata, do_o2x, do_x2o, do_xa if (present(do_o2x)) then if (first_time) then index_o2x_Fioo_frazil = mct_aVect_indexRA(o2x_o,'Fioo_frazil') + index_o2x_Foxo_frazil_li= mct_aVect_indexRA(o2x_o,'Foxo_frazil_li',perrWith='quiet') index_o2x_Fioo_q = mct_aVect_indexRA(o2x_o,'Fioo_q') + index_o2x_Foxo_q_li = mct_aVect_indexRA(o2x_o,'Foxo_q_li',perrWith='quiet') index_o2x_Faoo_h2otemp = mct_aVect_indexRA(o2x_o,'Faoo_h2otemp') + index_o2x_Foxo_ismw = mct_aVect_indexRA(o2x_o,'Foxo_ismw',perrWith='quiet') + if ( index_o2x_Foxo_ismw /= 0 ) flds_polar = .true. + if ( flds_polar ) then + index_o2x_Foxo_rrofl = mct_aVect_indexRA(o2x_o,'Foxo_rrofl',perrWith='quiet') + index_o2x_Foxo_rrofi = mct_aVect_indexRA(o2x_o,'Foxo_rrofi',perrWith='quiet') + index_o2x_Foxo_ismh = mct_aVect_indexRA(o2x_o,'Foxo_ismh',perrWith='quiet') + index_o2x_Foxo_rrofih = mct_aVect_indexRA(o2x_o,'Foxo_rrofih',perrWith='quiet') + end if end if lSize = mct_avect_lSize(o2x_o) @@ -1380,10 +1397,20 @@ subroutine seq_diag_ocn_mct( ocn, xao_o, frac_o, infodata, do_o2x, do_x2o, do_xa do n=1,lSize ca_o = dom_o%data%rAttr(kArea,n) * frac_o%rAttr(ko,n) ca_i = dom_o%data%rAttr(kArea,n) * frac_o%rAttr(ki,n) + ca_c = dom_o%data%rAttr(kArea,n) nf = f_area; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + ca_o nf = f_wfrz; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - (ca_o+ca_i)*max(0.0_r8,o2x_o%rAttr(index_o2x_Fioo_frazil,n)) nf = f_hfrz; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*max(0.0_r8,o2x_o%rAttr(index_o2x_Fioo_q,n)) nf = f_hh2ot; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*o2x_o%rAttr(index_o2x_Faoo_h2otemp,n) + if (flds_polar) then + nf = f_wpolar;budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - ca_c*o2x_o%rAttr(index_o2x_Foxo_frazil_li,n) + nf = f_wpolar;budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + ca_c*o2x_o%rAttr(index_o2x_Foxo_ismw,n) + nf = f_wpolar;budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - (ca_o+ca_i)*o2x_o%rAttr(index_o2x_Foxo_rrofl,n) + nf = f_wpolar;budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - (ca_o+ca_i)*o2x_o%rAttr(index_o2x_Foxo_rrofi,n) + nf = f_hpolar;budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + ca_c*o2x_o%rAttr(index_o2x_Foxo_q_li,n) + nf = f_hpolar;budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + ca_c*o2x_o%rAttr(index_o2x_Foxo_ismh,n) + nf = f_hpolar;budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + ca_c*o2x_o%rAttr(index_o2x_Foxo_rrofih,n) + end if end do end if @@ -1490,11 +1517,11 @@ subroutine seq_diag_ocn_mct( ocn, xao_o, frac_o, infodata, do_o2x, do_x2o, do_xa nf = f_hmelt ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*x2o_o%rAttr(index_x2o_Fioi_melth,n) nf = f_hswnet; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*x2o_o%rAttr(index_x2o_Foxx_swnet,n) nf = f_hlwdn ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*x2o_o%rAttr(index_x2o_Faxa_lwdn,n) - nf = f_hberg ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*x2o_o%rAttr(index_x2o_Fioi_bergh,n) + nf = f_hpolar; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*x2o_o%rAttr(index_x2o_Fioi_bergh,n) nf = f_wmelt ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*x2o_o%rAttr(index_x2o_Fioi_meltw,n) nf = f_wrain ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*x2o_o%rAttr(index_x2o_Faxa_rain,n) nf = f_wsnow ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*x2o_o%rAttr(index_x2o_Faxa_snow,n) - nf = f_wberg ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*x2o_o%rAttr(index_x2o_Fioi_bergw,n) + nf = f_wpolar; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*x2o_o%rAttr(index_x2o_Fioi_bergw,n) nf = f_wroff ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*x2o_o%rAttr(index_x2o_Foxx_rofl,n) nf = f_wioff ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*x2o_o%rAttr(index_x2o_Foxx_rofi,n) @@ -1613,8 +1640,6 @@ subroutine seq_diag_ice_mct( ice, frac_i, infodata, do_i2x, do_x2i) if (present(do_i2x)) then index_i2x_Fioi_melth = mct_aVect_indexRA(i2x_i,'Fioi_melth') index_i2x_Fioi_meltw = mct_aVect_indexRA(i2x_i,'Fioi_meltw') - index_i2x_Fioi_bergh = mct_aVect_indexRA(i2x_i,'PFioi_bergh') - index_i2x_Fioi_bergw = mct_aVect_indexRA(i2x_i,'PFioi_bergw') index_i2x_Fioi_swpen = mct_aVect_indexRA(i2x_i,'Fioi_swpen') index_i2x_Faii_swnet = mct_aVect_indexRA(i2x_i,'Faii_swnet') index_i2x_Faii_lwup = mct_aVect_indexRA(i2x_i,'Faii_lwup') @@ -1650,9 +1675,7 @@ subroutine seq_diag_ice_mct( ice, frac_i, infodata, do_i2x, do_x2i) nf = f_hlwup ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + ca_i*i2x_i%rAttr(index_i2x_Faii_lwup,n) nf = f_hlatv ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + ca_i*i2x_i%rAttr(index_i2x_Faii_lat,n) nf = f_hsen ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + ca_i*i2x_i%rAttr(index_i2x_Faii_sen,n) - nf = f_hberg ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - (ca_o+ca_i)*i2x_i%rAttr(index_i2x_Fioi_bergh,n) nf = f_wmelt ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - ca_i*i2x_i%rAttr(index_i2x_Fioi_meltw,n) - nf = f_wberg ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - (ca_o+ca_i)*i2x_i%rAttr(index_i2x_Fioi_bergw,n) nf = f_wevap ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + ca_i*i2x_i%rAttr(index_i2x_Faii_evap,n) if ( flds_wiso_ice )then diff --git a/driver-mct/main/seq_hist_mod.F90 b/driver-mct/main/seq_hist_mod.F90 index cff6a8f07aa3..8ade839130b7 100644 --- a/driver-mct/main/seq_hist_mod.F90 +++ b/driver-mct/main/seq_hist_mod.F90 @@ -161,6 +161,8 @@ subroutine seq_hist_write(infodata, EClock_d, & integer(IN) :: start_tod ! Starting time-of-day (s) real(r8) :: curr_time ! Time interval since reference time integer(IN) :: fk ! index + integer(IN) :: latlonid(2) ! ids of lat lon dimensions + integer(IN) :: alatlonid(2) ! ids of atmosphere lat lon dimensions character(CL) :: time_units ! units of time variable character(CL) :: calendar ! calendar type character(CL) :: case_name ! case name @@ -258,16 +260,16 @@ subroutine seq_hist_write(infodata, EClock_d, & gsmap => component_get_gsmap_cx(atm(1)) dom => component_get_dom_cx(atm(1)) call seq_io_write(hist_file, gsmap, dom%data, 'dom_ax', & - nx=atm_nx, ny=atm_ny, nt=1, whead=whead, wdata=wdata, pre='doma', & + nx=atm_nx, ny=atm_ny, nt=1, whead=whead, wdata=wdata,dims2do=alatlonid, pre='doma', & scolumn=single_column) call seq_io_write(hist_file, gsmap, fractions_ax, 'fractions_ax', & - nx=atm_nx, ny=atm_ny, nt=1, whead=whead, wdata=wdata, pre='fraca', & + nx=atm_nx, ny=atm_ny, nt=1, whead=whead, wdata=wdata, dims2din=alatlonid, pre='fraca', & scolumn=single_column) call seq_io_write(hist_file, atm, 'x2c', 'x2a_ax', & - nx=atm_nx, ny=atm_ny, nt=1, whead=whead, wdata=wdata, pre='x2a', & + nx=atm_nx, ny=atm_ny, nt=1, whead=whead, wdata=wdata, dims2din=alatlonid, pre='x2a', & scolumn=single_column) call seq_io_write(hist_file, atm, 'c2x', 'a2x_ax', & - nx=atm_nx, ny=atm_ny, nt=1, whead=whead, wdata=wdata, pre='a2x', & + nx=atm_nx, ny=atm_ny, nt=1, whead=whead, wdata=wdata, dims2din=alatlonid, pre='a2x', & scolumn=single_column) !call seq_io_write(hist_file, gsmap, l2x_ax, 'l2x_ax', & ! nx=atm_nx, ny=atm_ny, nt=1, whead=whead, wdata=wdata, pre='l2x_ax') @@ -281,51 +283,45 @@ subroutine seq_hist_write(infodata, EClock_d, & gsmap => component_get_gsmap_cx(lnd(1)) dom => component_get_dom_cx(lnd(1)) call seq_io_write(hist_file, gsmap, dom%data, 'dom_lx', & - nx=lnd_nx, ny=lnd_ny, nt=1, whead=whead, wdata=wdata, pre='doml') + nx=lnd_nx, ny=lnd_ny, nt=1, whead=whead, wdata=wdata, dims2do=latlonid, pre='doml') call seq_io_write(hist_file, gsmap, fractions_lx, 'fractions_lx', & - nx=lnd_nx, ny=lnd_ny, nt=1, whead=whead, wdata=wdata, pre='fracl') + nx=lnd_nx, ny=lnd_ny, nt=1, whead=whead, wdata=wdata, dims2din=latlonid, pre='fracl') call seq_io_write(hist_file, lnd, 'c2x', 'l2x_lx', & - nx=lnd_nx, ny=lnd_ny, nt=1, whead=whead, wdata=wdata, pre='l2x') + nx=lnd_nx, ny=lnd_ny, nt=1, whead=whead, wdata=wdata, dims2din=latlonid, pre='l2x') call seq_io_write(hist_file, lnd, 'x2c', 'x2l_lx',& - nx=lnd_nx, ny=lnd_ny, nt=1, whead=whead, wdata=wdata, pre='x2l') + nx=lnd_nx, ny=lnd_ny, nt=1, whead=whead, wdata=wdata, dims2din=latlonid, pre='x2l') endif if (rof_present) then gsmap => component_get_gsmap_cx(rof(1)) dom => component_get_dom_cx(rof(1)) call seq_io_write(hist_file, gsmap, dom%data, 'dom_rx', & - nx=rof_nx, ny=rof_ny, nt=1, whead=whead, wdata=wdata, pre='domr') + nx=rof_nx, ny=rof_ny, nt=1, whead=whead, wdata=wdata, dims2do=latlonid, pre='domr') call seq_io_write(hist_file, gsmap, fractions_rx, 'fractions_rx', & - nx=rof_nx, ny=rof_ny, nt=1, whead=whead, wdata=wdata, pre='fracr') + nx=rof_nx, ny=rof_ny, nt=1, whead=whead, wdata=wdata, dims2din=latlonid, pre='fracr') call seq_io_write(hist_file, rof, 'c2x', 'r2x_rx', & - nx=rof_nx, ny=rof_ny, nt=1, whead=whead, wdata=wdata, pre='r2x') + nx=rof_nx, ny=rof_ny, nt=1, whead=whead, wdata=wdata, dims2din=latlonid, pre='r2x') call seq_io_write(hist_file, rof, 'x2c', 'x2r_rx', & - nx=rof_nx, ny=rof_ny, nt=1, whead=whead, wdata=wdata, pre='x2r') - endif - - if (rof_present .and. ocnrof_prognostic) then - gsmap => component_get_gsmap_cx(ocn(1)) - r2x_ox => prep_ocn_get_r2x_ox() - call seq_io_write(hist_file, gsmap, r2x_ox, 'r2x_ox', & - nx=ocn_nx, ny=ocn_ny, nt=1, whead=whead, wdata=wdata, pre='r2xo') + nx=rof_nx, ny=rof_ny, nt=1, whead=whead, wdata=wdata, dims2din=latlonid, pre='x2r') endif if (ocn_present) then gsmap => component_get_gsmap_cx(ocn(1)) dom => component_get_dom_cx(ocn(1)) call seq_io_write(hist_file, gsmap, dom%data, 'dom_ox', & - nx=ocn_nx, ny=ocn_ny, nt=1, whead=whead, wdata=wdata, pre='domo') + nx=ocn_nx, ny=ocn_ny, nt=1, whead=whead, wdata=wdata, dims2do=latlonid, pre='domo') call seq_io_write(hist_file, gsmap, fractions_ox, 'fractions_ox', & - nx=ocn_nx, ny=ocn_ny, nt=1, whead=whead, wdata=wdata, pre='fraco') + nx=ocn_nx, ny=ocn_ny, nt=1, whead=whead, wdata=wdata, dims2din=latlonid, pre='fraco') call seq_io_write(hist_file, ocn, 'c2x', 'o2x_ox', & - nx=ocn_nx, ny=ocn_ny, nt=1, whead=whead, wdata=wdata, pre='o2x') + nx=ocn_nx, ny=ocn_ny, nt=1, whead=whead, wdata=wdata, dims2din=latlonid, pre='o2x') + !call seq_io_write(hist_file, ocn, 'x2c', 'x2o_ox', & ! nx=ocn_nx, ny=ocn_ny, nt=1, whead=whead, wdata=wdata, pre='x2o') gsmap => component_get_gsmap_cx(ocn(1)) x2oacc_ox => prep_ocn_get_x2oacc_ox() call seq_io_write(hist_file, gsmap, x2oacc_ox, 'x2oacc_ox', & - nx=ocn_nx, ny=ocn_ny, nt=1, whead=whead, wdata=wdata, pre='x2oacc') + nx=ocn_nx, ny=ocn_ny, nt=1, whead=whead, wdata=wdata, dims2din=latlonid, pre='x2oacc') gsmap => component_get_gsmap_cx(ocn(1)) x2oacc_ox_cnt => prep_ocn_get_x2oacc_ox_cnt() @@ -334,17 +330,24 @@ subroutine seq_hist_write(infodata, EClock_d, & gsmap => component_get_gsmap_cx(ocn(1)) xao_ox => prep_aoflux_get_xao_ox() call seq_io_write(hist_file, gsmap, xao_ox, 'xao_ox', & - nx=ocn_nx, ny=ocn_ny, nt=1, whead=whead, wdata=wdata, pre='xaoo') + nx=ocn_nx, ny=ocn_ny, nt=1, whead=whead, wdata=wdata, dims2din=latlonid, pre='xaoo') gsmap => component_get_gsmap_cx(atm(1)) o2x_ax => prep_atm_get_o2x_ax() call seq_io_write(hist_file, gsmap, o2x_ax, 'o2x_ax', & - nx=atm_nx, ny=atm_ny, nt=1, whead=whead, wdata=wdata, pre='o2xa') + nx=atm_nx, ny=atm_ny, nt=1, whead=whead, wdata=wdata, dims2din=alatlonid, pre='o2xa') gsmap => component_get_gsmap_cx(atm(1)) xao_ax => prep_aoflux_get_xao_ax() call seq_io_write(hist_file, gsmap, xao_ax, 'xao_ax', & - nx=atm_nx, ny=atm_ny, nt=1, whead=whead, wdata=wdata, pre='xaoa') + nx=atm_nx, ny=atm_ny, nt=1, whead=whead, wdata=wdata, dims2din=alatlonid, pre='xaoa') + endif + + if (rof_present .and. ocnrof_prognostic) then + gsmap => component_get_gsmap_cx(ocn(1)) + r2x_ox => prep_ocn_get_r2x_ox() + call seq_io_write(hist_file, gsmap, r2x_ox, 'r2x_ox', & + nx=ocn_nx, ny=ocn_ny, nt=1, whead=whead, wdata=wdata, dims2din=latlonid, pre='r2xo') endif if (ice_present) then @@ -352,52 +355,52 @@ subroutine seq_hist_write(infodata, EClock_d, & dom => component_get_dom_cx(ice(1)) nmask = mct_aVect_indexRA(dom%data,'mask') call seq_io_write(hist_file, gsmap, dom%data, 'dom_ix', & - nx=ice_nx, ny=ice_ny, nt=1, whead=whead, wdata=wdata, pre='domi') + nx=ice_nx, ny=ice_ny, nt=1, whead=whead, wdata=wdata, dims2do=latlonid, pre='domi') call seq_io_write(hist_file, gsmap, fractions_ix, 'fractions_ix', & - nx=ice_nx, ny=ice_ny, nt=1, whead=whead, wdata=wdata, pre='fraci') + nx=ice_nx, ny=ice_ny, nt=1, whead=whead, wdata=wdata, dims2din=latlonid, pre='fraci') call seq_io_write(hist_file, ice, 'c2x', 'i2x_ix', & - nx=ice_nx, ny=ice_ny, nt=1, whead=whead, wdata=wdata, pre='i2x', mask=dom%data%rattr(nmask,:)) + nx=ice_nx, ny=ice_ny, nt=1, whead=whead, wdata=wdata, dims2din=latlonid, pre='i2x', mask=dom%data%rattr(nmask,:)) call seq_io_write(hist_file, ice, 'x2c', 'x2i_ix', & - nx=ice_nx, ny=ice_ny, nt=1, whead=whead, wdata=wdata, pre='x2i', mask=dom%data%rattr(nmask,:)) + nx=ice_nx, ny=ice_ny, nt=1, whead=whead, wdata=wdata, dims2din=latlonid, pre='x2i', mask=dom%data%rattr(nmask,:)) endif if (glc_present) then gsmap => component_get_gsmap_cx(glc(1)) dom => component_get_dom_cx(glc(1)) call seq_io_write(hist_file, gsmap, dom%data, 'dom_gx', & - nx=glc_nx, ny=glc_ny, nt=1, whead=whead, wdata=wdata, pre='domg') + nx=glc_nx, ny=glc_ny, nt=1, whead=whead, wdata=wdata, dims2do=latlonid, pre='domg') call seq_io_write(hist_file, gsmap, fractions_gx, 'fractions_gx', & - nx=glc_nx, ny=glc_ny, nt=1, whead=whead, wdata=wdata, pre='fracg') + nx=glc_nx, ny=glc_ny, nt=1, whead=whead, wdata=wdata, dims2din=latlonid, pre='fracg') call seq_io_write(hist_file, glc, 'c2x', 'g2x_gx', & - nx=glc_nx, ny=glc_ny, nt=1, whead=whead, wdata=wdata, pre='g2x') + nx=glc_nx, ny=glc_ny, nt=1, whead=whead, wdata=wdata, dims2din=latlonid, pre='g2x') call seq_io_write(hist_file, glc, 'x2c', 'x2g_gx', & - nx=glc_nx, ny=glc_ny, nt=1, whead=whead, wdata=wdata, pre='x2g') + nx=glc_nx, ny=glc_ny, nt=1, whead=whead, wdata=wdata, dims2din=latlonid, pre='x2g') endif if (wav_present) then gsmap => component_get_gsmap_cx(wav(1)) dom => component_get_dom_cx(wav(1)) call seq_io_write(hist_file, gsmap, dom%data, 'dom_wx', & - nx=wav_nx, ny=wav_ny, nt=1, whead=whead, wdata=wdata, pre='domw') + nx=wav_nx, ny=wav_ny, nt=1, whead=whead, wdata=wdata, dims2do=latlonid, pre='domw') call seq_io_write(hist_file, gsmap, fractions_wx, 'fractions_wx', & - nx=wav_nx, ny=wav_ny, nt=1, whead=whead, wdata=wdata, pre='fracw') + nx=wav_nx, ny=wav_ny, nt=1, whead=whead, wdata=wdata, dims2din=latlonid, pre='fracw') call seq_io_write(hist_file, wav, 'c2x', 'w2x_wx', & - nx=wav_nx, ny=wav_ny, nt=1, whead=whead, wdata=wdata, pre='w2x') + nx=wav_nx, ny=wav_ny, nt=1, whead=whead, wdata=wdata, dims2din=latlonid, pre='w2x') call seq_io_write(hist_file, wav, 'x2c', 'x2w_wx', & - nx=wav_nx, ny=wav_ny, nt=1, whead=whead, wdata=wdata, pre='x2w') + nx=wav_nx, ny=wav_ny, nt=1, whead=whead, wdata=wdata, dims2din=latlonid, pre='x2w') endif if (iac_present) then gsmap => component_get_gsmap_cx(iac(1)) dom => component_get_dom_cx(iac(1)) call seq_io_write(hist_file, gsmap, dom%data, 'dom_zx', & - nx=iac_nx, ny=iac_ny, nt=1, whead=whead, wdata=wdata, pre='domz') + nx=iac_nx, ny=iac_ny, nt=1, whead=whead, wdata=wdata, dims2do=latlonid, pre='domz') call seq_io_write(hist_file, gsmap, fractions_zx, 'fractions_zx', & - nx=iac_nx, ny=iac_ny, nt=1, whead=whead, wdata=wdata, pre='fracz') + nx=iac_nx, ny=iac_ny, nt=1, whead=whead, wdata=wdata, dims2din=latlonid, pre='fracz') call seq_io_write(hist_file, iac, 'c2x', 'z2x_zx', & - nx=iac_nx, ny=iac_ny, nt=1, whead=whead, wdata=wdata, pre='w2x') + nx=iac_nx, ny=iac_ny, nt=1, whead=whead, wdata=wdata, dims2din=latlonid, pre='w2x') call seq_io_write(hist_file, iac, 'x2c', 'x2z_zx', & - nx=iac_nx, ny=iac_ny, nt=1, whead=whead, wdata=wdata, pre='x2w') + nx=iac_nx, ny=iac_ny, nt=1, whead=whead, wdata=wdata, dims2din=latlonid, pre='x2w') endif enddo call seq_io_close(hist_file) diff --git a/driver-mct/main/seq_io_mod.F90 b/driver-mct/main/seq_io_mod.F90 index 9739fbf256a6..063b60409b51 100644 --- a/driver-mct/main/seq_io_mod.F90 +++ b/driver-mct/main/seq_io_mod.F90 @@ -392,8 +392,8 @@ end function seq_io_sec2hms ! ! !INTERFACE: ------------------------------------------------------------------ - subroutine seq_io_write_av(filename,gsmap,AV,dname,whead,wdata,nx,ny,nt,fillval,pre,tavg,& - use_float, file_ind, mask, scolumn) + subroutine seq_io_write_av(filename,gsmap,AV,dname,whead,wdata,nx,ny,nt,fillval,dims2din,& + dims2do,pre,tavg,use_float, file_ind, mask, scolumn) ! !INPUT/OUTPUT PARAMETERS: implicit none @@ -407,6 +407,8 @@ subroutine seq_io_write_av(filename,gsmap,AV,dname,whead,wdata,nx,ny,nt,fillval, integer(in),optional,intent(in) :: ny ! 2d grid size if available integer(in),optional,intent(in) :: nt ! time sample real(r8),optional,intent(in) :: fillval ! fill value + integer(in),optional,intent(out) :: dims2do(2) ! dim ids to output + integer(in),optional,intent(in) :: dims2din(2) ! dim ids to output character(len=*),optional,intent(in) :: pre ! prefix to variable name logical,optional,intent(in) :: tavg ! is this a tavg logical,optional,intent(in) :: use_float ! write output as float rather than double @@ -503,8 +505,17 @@ subroutine seq_io_write_av(filename,gsmap,AV,dname,whead,wdata,nx,ny,nt,fillval, endif if (lwhead) then - rcode = pio_def_dim(cpl_io_file(lfile_ind),trim(lpre)//'_nx',lnx,dimid2(1)) - rcode = pio_def_dim(cpl_io_file(lfile_ind),trim(lpre)//'_ny',lny,dimid2(2)) + if (present(dims2din)) then + dimid2(1)=dims2din(1) + dimid2(2)=dims2din(2) + else + rcode = pio_def_dim(cpl_io_file(lfile_ind),trim(lpre)//'_nx',lnx,dimid2(1)) + rcode = pio_def_dim(cpl_io_file(lfile_ind),trim(lpre)//'_ny',lny,dimid2(2)) + endif + if (present(dims2do)) then + dims2do(1)=dimid2(1) + dims2do(2)=dimid2(2) + endif if (present(nt)) then dimid3(1:2) = dimid2 @@ -611,8 +622,8 @@ end subroutine seq_io_write_av ! ! !INTERFACE: ------------------------------------------------------------------ - subroutine seq_io_write_avs(filename,gsmap,AVS,dname,whead,wdata,nx,ny,nt,fillval,pre,tavg,& - use_float,file_ind,scolumn) + subroutine seq_io_write_avs(filename,gsmap,AVS,dname,whead,wdata,nx,ny,nt,fillval,dims2din,& + dims2do,pre,tavg,use_float,file_ind,scolumn) ! !INPUT/OUTPUT PARAMETERS: implicit none @@ -625,6 +636,8 @@ subroutine seq_io_write_avs(filename,gsmap,AVS,dname,whead,wdata,nx,ny,nt,fillva integer(in),optional,intent(in) :: nx ! 2d grid size if available integer(in),optional,intent(in) :: ny ! 2d grid size if available integer(in),optional,intent(in) :: nt ! time sample + integer(in),optional,intent(in) :: dims2din(2) ! dim ids to output + integer(in),optional,intent(out) :: dims2do(2) ! dim ids for output real(r8),optional,intent(in) :: fillval ! fill value character(len=*),optional,intent(in) :: pre ! prefix to variable name logical,optional,intent(in) :: tavg ! is this a tavg @@ -726,8 +739,17 @@ subroutine seq_io_write_avs(filename,gsmap,AVS,dname,whead,wdata,nx,ny,nt,fillva endif if (lwhead) then - rcode = pio_def_dim(cpl_io_file(lfile_ind),trim(lpre)//'_nx',lnx,dimid2(1)) - rcode = pio_def_dim(cpl_io_file(lfile_ind),trim(lpre)//'_ny',lny,dimid2(2)) + if (present(dims2din)) then + dimid2(1)=dims2din(1) + dimid2(2)=dims2din(2) + else + rcode = pio_def_dim(cpl_io_file(lfile_ind),trim(lpre)//'_nx',lnx,dimid2(1)) + rcode = pio_def_dim(cpl_io_file(lfile_ind),trim(lpre)//'_ny',lny,dimid2(2)) + endif + if (present(dims2do)) then + dims2do(1)=dimid2(1) + dims2do(2)=dimid2(2) + endif if (ni > 1) then rcode = pio_def_dim(cpl_io_file(lfile_ind),trim(lpre)//'_ni',ni,dimid3(3)) @@ -835,7 +857,7 @@ end subroutine seq_io_write_avs ! !INTERFACE: ------------------------------------------------------------------ subroutine seq_io_write_avscomp(filename, comp, flow, dname, & - whead, wdata, nx, ny, nt, fillval, pre, tavg, use_float, file_ind, scolumn, mask) + whead, wdata, nx, ny, nt, fillval, dims2din, pre, tavg, use_float, file_ind, scolumn, mask) ! !INPUT/OUTPUT PARAMETERS: implicit none @@ -849,6 +871,7 @@ subroutine seq_io_write_avscomp(filename, comp, flow, dname, & integer(in) ,optional,intent(in) :: ny ! 2d grid size if available integer(in) ,optional,intent(in) :: nt ! time sample real(r8) ,optional,intent(in) :: fillval ! fill value + integer(in) ,optional,intent(in) :: dims2din(2) ! use previously made 2d dims character(len=*) ,optional,intent(in) :: pre ! prefix to variable name logical ,optional,intent(in) :: tavg ! is this a tavg logical ,optional,intent(in) :: use_float ! write output as float rather than double @@ -957,8 +980,13 @@ subroutine seq_io_write_avscomp(filename, comp, flow, dname, & endif if (lwhead) then - rcode = pio_def_dim(cpl_io_file(lfile_ind),trim(lpre)//'_nx',lnx,dimid2(1)) - rcode = pio_def_dim(cpl_io_file(lfile_ind),trim(lpre)//'_ny',lny,dimid2(2)) + if (present(dims2din)) then + dimid2(1)=dims2din(1) + dimid2(2)=dims2din(2) + else + rcode = pio_def_dim(cpl_io_file(lfile_ind),trim(lpre)//'_nx',lnx,dimid2(1)) + rcode = pio_def_dim(cpl_io_file(lfile_ind),trim(lpre)//'_ny',lny,dimid2(2)) + endif if (ni > 1) then rcode = pio_def_dim(cpl_io_file(lfile_ind),trim(lpre)//'_ni',ni,dimid3(3)) diff --git a/driver-mct/main/seq_map_mod.F90 b/driver-mct/main/seq_map_mod.F90 index 13623d0b920b..0eaa45296d99 100644 --- a/driver-mct/main/seq_map_mod.F90 +++ b/driver-mct/main/seq_map_mod.F90 @@ -910,9 +910,11 @@ subroutine seq_map_avNormArr(mapper, av_i, av_o, norm_i, rList, norm, omit_nonli call mct_aVect_init(avp_i, rList=trim( rList)//trim(appnd), lsize=lsize_i) call mct_aVect_init(avp_o, rList=trim( rList)//trim(appnd), lsize=lsize_o) else - lrList = mct_aVect_exportRList2c(av_i) + lrList = '' + if(mct_aVect_nRAttr(av_i) /= 0) lrList = mct_aVect_exportRList2c(av_i) call mct_aVect_init(avp_i, rList=trim(lrList)//trim(appnd), lsize=lsize_i) - lrList = mct_aVect_exportRList2c(av_o) + lrList = '' + if(mct_aVect_nRAttr(av_o) /= 0) lrList = mct_aVect_exportRList2c(av_o) call mct_aVect_init(avp_o, rList=trim(lrList)//trim(appnd), lsize=lsize_o) endif diff --git a/driver-mct/shr/seq_flds_mod.F90 b/driver-mct/shr/seq_flds_mod.F90 index 4b37e40b7b3e..dbfba0889d0c 100644 --- a/driver-mct/shr/seq_flds_mod.F90 +++ b/driver-mct/shr/seq_flds_mod.F90 @@ -379,10 +379,11 @@ subroutine seq_flds_set(nmlfile, ID, infodata) logical :: flds_co2_dmsa logical :: flds_bgc_oi logical :: flds_wiso + logical :: flds_polar integer :: glc_nec namelist /seq_cplflds_inparm/ & - flds_co2a, flds_co2b, flds_co2c, flds_co2_dmsa, flds_wiso, glc_nec, & + flds_co2a, flds_co2b, flds_co2c, flds_co2_dmsa, flds_wiso, flds_polar, glc_nec, & ice_ncat, seq_flds_i2o_per_cat, flds_bgc_oi, & nan_check_component_fields, rof_heat, atm_flux_method, atm_gustiness, & rof2ocn_nutrients, lnd_rof_two_way, ocn_rof_two_way, rof_sed @@ -416,6 +417,7 @@ subroutine seq_flds_set(nmlfile, ID, infodata) flds_co2_dmsa = .false. flds_bgc_oi = .false. flds_wiso = .false. + flds_polar = .false. glc_nec = 0 ice_ncat = 1 seq_flds_i2o_per_cat = .false. @@ -449,6 +451,7 @@ subroutine seq_flds_set(nmlfile, ID, infodata) call shr_mpi_bcast(flds_co2_dmsa, mpicom) call shr_mpi_bcast(flds_bgc_oi , mpicom) call shr_mpi_bcast(flds_wiso , mpicom) + call shr_mpi_bcast(flds_polar , mpicom) call shr_mpi_bcast(glc_nec , mpicom) call shr_mpi_bcast(ice_ncat , mpicom) call shr_mpi_bcast(seq_flds_i2o_per_cat, mpicom) @@ -1583,6 +1586,70 @@ subroutine seq_flds_set(nmlfile, ID, infodata) attname = 'PFioi_bergw' call metadata_set(attname, longname, stdname, units) + !-------------------------------- + ! ocn<->cpl only exchange - Polar + !-------------------------------- + + if (flds_polar) then + + ! Ocean Land ice freeze potential + call seq_flds_add(o2x_fluxes,"Foxo_q_li") + longname = 'Ocean land ice freeze potential' + stdname = 'ice_shelf_cavity_ice_heat_flux' + units = 'W m-2' + attname = 'Foxo_q_li' + call metadata_set(attname, longname, stdname, units) + + ! Ocean land ice frazil production + call seq_flds_add(o2x_fluxes,"Foxo_frazil_li") + longname = 'Ocean land ice frazil production' + stdname = 'ocean_land_ice_frazil_ice_production' + units = 'kg m-2 s-1' + attname = 'Foxo_frazil_li' + call metadata_set(attname, longname, stdname, units) + + ! Water flux from ice shelf melt + call seq_flds_add(o2x_fluxes,"Foxo_ismw") + longname = 'Water flux due to basal melting of ice shelves' + stdname = 'basal_iceshelf_melt_flux' + units = 'kg m-2 s-1' + attname = 'Foxo_ismw' + call metadata_set(attname, longname, stdname, units) + + ! Heat flux from ice shelf melt + call seq_flds_add(o2x_fluxes,"Foxo_ismh") + longname = 'Heat flux due to basal melting of ice shelves' + stdname = 'basal_iceshelf_heat_flux' + units = 'W m-2' + attname = 'Foxo_ismh' + call metadata_set(attname, longname, stdname, units) + + ! Water flux from removed liquid runoff + call seq_flds_add(o2x_fluxes,"Foxo_rrofl") + longname = 'Water flux due to removed liqiud runoff' + stdname = 'removed_liquid_runoff_flux' + units = 'kg m-2 s-1' + attname = 'Foxo_rrofl' + call metadata_set(attname, longname, stdname, units) + + ! Water flux from removed solid runoff + call seq_flds_add(o2x_fluxes,"Foxo_rrofi") + longname = 'Water flux due to removed solid runoff' + stdname = 'removed_solid_runoff_flux' + units = 'kg m-2 s-1' + attname = 'Foxo_rrofi' + call metadata_set(attname, longname, stdname, units) + + ! Heat flux from removed solid runoff + call seq_flds_add(o2x_fluxes,"Foxo_rrofih") + longname = 'Heat flux due to removed solid runoff' + stdname = 'removed_solid_runoff_heat_flux' + units = 'W m-2' + attname = 'Foxo_rrofih' + call metadata_set(attname, longname, stdname, units) + + end if + ! Salt flux call seq_flds_add(i2x_fluxes,"Fioi_salt") call seq_flds_add(x2o_fluxes,"Fioi_salt") diff --git a/driver-moab/cime_config/buildnml b/driver-moab/cime_config/buildnml index 0d282d402159..cfaa3c63f8bc 100755 --- a/driver-moab/cime_config/buildnml +++ b/driver-moab/cime_config/buildnml @@ -40,6 +40,7 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files): config['CPL_ALBAV'] = case.get_value('CPL_ALBAV') config['CPL_EPBAL'] = case.get_value('CPL_EPBAL') config['FLDS_WISO'] = case.get_value('FLDS_WISO') + config['FLDS_POLAR'] = case.get_value('FLDS_POLAR') config['BUDGETS'] = case.get_value('BUDGETS') config['MACH'] = case.get_value('MACH') config['MPILIB'] = case.get_value('MPILIB') diff --git a/driver-moab/cime_config/config_component.xml b/driver-moab/cime_config/config_component.xml index 8e4e686aa1c5..be0ee0fbd776 100644 --- a/driver-moab/cime_config/config_component.xml +++ b/driver-moab/cime_config/config_component.xml @@ -1085,8 +1085,8 @@ char - gland20,gland10,gland5,gland5UM,gland4,mpas.aisgis20km,mpas.gis20km,mpas.ais20km,mpas.gis1to10km,null - gland5UM + mpas.aisgis20km,mpas.gis20km,mpas.ais20km,mpas.gis1to10km,gis1to10kmR2,null + mpas.gis20km build_grid env_build.xml glacier (glc) grid - DO NOT EDIT (for experts only) @@ -2684,10 +2684,10 @@ integer - 1,2 + 1,2,3 run_pio env_run.xml - pio rearranger choice box=1, subset=2 + pio rearranger choice box=1, subset=2, any=3 $PIO_VERSION $PIO_VERSION diff --git a/driver-moab/cime_config/config_component_e3sm.xml b/driver-moab/cime_config/config_component_e3sm.xml index dd7a5ddb6c15..044cc1e9f9d6 100644 --- a/driver-moab/cime_config/config_component_e3sm.xml +++ b/driver-moab/cime_config/config_component_e3sm.xml @@ -172,6 +172,19 @@ Turn on the passing of water isotope fields through the coupler + + logical + TRUE,FALSE + FALSE + + TRUE + TRUE + + run_flags + env_run.xml + Turn on the passing of polar fields through the coupler + + char minus1p8,linear_salt,mushy @@ -265,6 +278,7 @@ CO2A CO2A CO2A + CO2A CO2A_OI CO2A_OI CO2C diff --git a/driver-moab/cime_config/namelist_definition_drv.xml b/driver-moab/cime_config/namelist_definition_drv.xml index 8a10291f69fc..5510783582f9 100644 --- a/driver-moab/cime_config/namelist_definition_drv.xml +++ b/driver-moab/cime_config/namelist_definition_drv.xml @@ -149,6 +149,18 @@ + + logical + seq_flds + seq_cplflds_inparm + + If set to .true. Polar fields will be passed from the ocean to the coupler. + + + $FLDS_POLAR + + + char seq_flds @@ -228,7 +240,6 @@ .false. - .true. diff --git a/driver-moab/cime_config/namelist_definition_modelio.xml b/driver-moab/cime_config/namelist_definition_modelio.xml index ce0275e59bf3..2860ffd7106f 100644 --- a/driver-moab/cime_config/namelist_definition_modelio.xml +++ b/driver-moab/cime_config/namelist_definition_modelio.xml @@ -90,9 +90,9 @@ integer pio pio_inparm - -99,1,2 + -99,1,2,3 - Rearranger method for pio 1=box, 2=subset. + Rearranger method for pio 1=box, 2=subset, 3=any. $CPL_PIO_REARRANGER diff --git a/driver-moab/main/cime_comp_mod.F90 b/driver-moab/main/cime_comp_mod.F90 index 0f9eb4d01bdb..265f8e11338e 100644 --- a/driver-moab/main/cime_comp_mod.F90 +++ b/driver-moab/main/cime_comp_mod.F90 @@ -129,6 +129,8 @@ module cime_comp_mod use seq_rest_mod, only : seq_rest_read, seq_rest_mb_read, seq_rest_write, seq_rest_mb_write #ifdef MOABDEBUG use seq_rest_mod, only : write_moab_state + use iMOAB, only: iMOAB_GetDoubleTagStorage, iMOAB_GetMeshInfo + use component_type_mod, only: component_get_name, component_get_c2x_cc #endif ! flux calc routines @@ -178,7 +180,7 @@ module cime_comp_mod use component_mod, only: component_init_areacor_moab #endif use component_mod, only: component_exch, component_diag - use component_mod, only: component_exch_moab + use cplcomp_exchange_mod, only: component_exch_moab ! used to send from components to coupler instances ! use component_mod, only: ocn_cpl_moab @@ -209,9 +211,7 @@ module cime_comp_mod use component_type_mod , only: expose_mct_grid_moab #endif -#ifdef MOABCOMP - use iso_c_binding -#endif + use iso_c_binding implicit none @@ -681,7 +681,7 @@ module cime_comp_mod !---------------------------------------------------------------------------- ! formats !---------------------------------------------------------------------------- - character(*), parameter :: subname = '(seq_mct_drv)' + character(*), parameter :: subname = '(moab_driver)' character(*), parameter :: F00 = "('"//subname//" : ', 4A )" character(*), parameter :: F0L = "('"//subname//" : ', A, L6 )" character(*), parameter :: F01 = "('"//subname//" : ', A, 2i8, 3x, A )" @@ -690,6 +690,14 @@ module cime_comp_mod character(*), parameter :: FormatD = '(A,": =============== ", A20,I10.8,I8,6x, " ===============")' character(*), parameter :: FormatR = '(A,": =============== ", A31,F12.3,1x, " ===============")' character(*), parameter :: FormatQ = '(A,": =============== ", A20,2F10.2,4x," ===============")' + +#ifdef MOABDEBUG +! allocate to get data frpm moab + real(r8) , private, pointer :: moab_tag_vals(:,:) ! various tags for debug purposes + integer nvert(3), nvise(3), nbl(3), nsurf(3), nvisBC(3), arrsize, ent_type + character(100) :: tagname + type(mct_aVect) , pointer :: a2x_aa => null() +#endif !=============================================================================== contains !=============================================================================== @@ -720,15 +728,15 @@ subroutine cime_pre_init1(esmf_log_option) integer(i8) :: beg_count ! start time integer(i8) :: end_count ! end time integer(i8) :: irtc_rate ! factor to convert time to seconds - + beg_count = shr_sys_irtc(irtc_rate) - + call mpi_init(ierr) call shr_mpi_chkerr(ierr,subname//' mpi_init') end_count = shr_sys_irtc(irtc_rate) mpi_init_time = real( (end_count-beg_count), r8)/real(irtc_rate, r8) - + call mpi_comm_dup(MPI_COMM_WORLD, global_comm, ierr) call shr_mpi_chkerr(ierr,subname//' mpi_comm_dup') @@ -1398,10 +1406,10 @@ subroutine cime_pre_init2() !Print BFBFLAG value in the log file if (iamroot_CPLID) then call seq_infodata_GetData(infodata, bfbflag=bfbflag) - write(logunit,'(2A,L4)') subname,'BFBFLAG is:',bfbflag + write(logunit,'(2A,L4)') subname,'BFBFLAG is:',bfbflag endif - + call t_stopf('CPL:cime_pre_init2') ! CPL:cime_pre_init2 timer elapsed time will be double counted @@ -1541,6 +1549,7 @@ subroutine cime_init() call t_adj_detailf(-2) call t_stopf('CPL:comp_init_cc_iac') + !--------------------------------------------------------------------------------------- ! Initialize coupler-component data ! if processor has cpl or model @@ -2185,7 +2194,7 @@ subroutine cime_init() call mpi_barrier(mpicom_GLOID,ierr) if (ice_present) call component_init_areacor(ice, areafact_samegrid, seq_flds_i2x_fluxes) if (ice_present) call component_init_areacor_moab(ice, mpsiid, mbixid, seq_flds_i2x_fluxes, seq_flds_i2x_fields) - !component_exch_moab(ice(1), mpsiid, mbixid, 0, seq_flds_i2x_fields) + !component_exch_moab(ice(1), mpsiid, mbixid, 0, seq_flds_i2x_fields) call mpi_barrier(mpicom_GLOID,ierr) if (glc_present) call component_init_areacor(glc, areafact_samegrid, seq_flds_g2x_fluxes) @@ -2431,7 +2440,7 @@ subroutine cime_init() ! will call prep_atm_merge for each instance. call prep_atm_mrg(infodata, & fractions_ax=fractions_ax, xao_ax=xao_ax, timer_mrg='CPL:init_atminit') - ! MOAB + ! MOAB call prep_atm_mrg_moab(infodata, xao_ax) endif endif @@ -2481,7 +2490,7 @@ subroutine cime_init() ! Send atm output data from atm pes to cpl pes call component_exch(atm, flow='c2x', infodata=infodata, & infodata_string='atm2cpl_init') - ! + ! call component_exch_moab(atm(1), mphaid, mbaxid, 0, seq_flds_a2x_fields) if (iamin_CPLID) then @@ -2516,7 +2525,7 @@ subroutine cime_init() write(logunit,103) subname,' Reading restart file ',trim(rest_file) call shr_sys_flush(logunit) end if - + call t_startf('CPL:seq_rest_read-init') call seq_rest_read(rest_file, infodata, & atm, lnd, ice, ocn, rof, glc, wav, esp, iac, & @@ -2528,7 +2537,7 @@ subroutine cime_init() call shr_sys_flush(logunit) end if call t_startf('CPL:seq_rest_read-moab') - call seq_rest_mb_read(rest_file, infodata, samegrid_al) + call seq_rest_mb_read(rest_file, infodata, samegrid_al) call t_stopf('CPL:seq_rest_read-moab') #ifdef MOABDEBUG call write_moab_state(.false.) @@ -2673,7 +2682,7 @@ subroutine cime_run() integer :: i, nodeId character(len=15) :: c_ymdtod character(len=18) :: c_mprof_file - integer :: cur_step_no ! step number + integer :: cur_step_no ! step number 101 format( A, i10.8, i8, 12A, A, F8.2, A, F8.2 ) 102 format( A, i10.8, i8, A, 8L3 ) @@ -3358,7 +3367,7 @@ subroutine cime_run() call cime_run_atm_recv_post endif - + !---------------------------------------------------------- !| Budget with new fractions !---------------------------------------------------------- @@ -3516,7 +3525,7 @@ subroutine cime_run() call shr_sys_flush(logunit) end if call t_startf('CPL:seq_rest_read-moab') - call seq_rest_mb_read(drv_resume_file, infodata, samegrid_al) + call seq_rest_mb_read(drv_resume_file, infodata, samegrid_al) call t_stopf('CPL:seq_rest_read-moab') end if ! Clear the resume file so we don't try to read it again @@ -3975,7 +3984,7 @@ subroutine cime_run_atmocn_fluxes(hashint) o2x_ax => prep_atm_get_o2x_ax() ! array over all instances xao_ax => prep_aoflux_get_xao_ax() ! array over all instances call seq_flux_atmocn_mct(infodata, tod, dtime, a2x_ax, o2x_ax(eoi), xao_ax(exi)) - !call seq_flux_atmocn_moab( atm(eai), xao_ax(exi) ) ! should be only one ensemble probably + !call seq_flux_atmocn_moab( atm(eai), xao_ax(exi) ) ! should be only one ensemble probably enddo call t_drvstopf ('CPL:atmocna_fluxa',hashint=hashint(6)) @@ -4057,7 +4066,7 @@ subroutine cime_run_atm_setup_send() endif if (associated(xao_ax)) then call prep_atm_mrg(infodata, fractions_ax, xao_ax=xao_ax, timer_mrg='CPL:atmprep_mrgx2a') - ! call moab atm merge too + ! call moab atm merge too call prep_atm_mrg_moab(infodata, xao_ax) endif @@ -4134,7 +4143,7 @@ subroutine cime_run_atm_recv_post() ! ! migrate that tag from coupler pes to ocean pes ! call prep_lnd_migrate_moab(infodata) ! endif - + end subroutine cime_run_atm_recv_post @@ -4472,7 +4481,7 @@ subroutine cime_run_lnd_setup_send() if (lnd_prognostic) then call prep_lnd_mrg(infodata, timer_mrg='CPL:lndprep_mrgx2l') - call prep_lnd_mrg_moab(infodata) + call prep_lnd_mrg_moab(infodata) call component_diag(infodata, lnd, flow='x2c', comment= 'send lnd', & info_debug=info_debug, timer_diag='CPL:lndprep_diagav') @@ -4510,7 +4519,7 @@ subroutine cime_run_lnd_recv_post() mpicom_barrier=mpicom_CPLALLLNDID, run_barriers=run_barriers, & timer_barrier='CPL:L2C_BARRIER', timer_comp_exch='CPL:L2C', & timer_map_exch='CPL:l2c_lndl2lndx', timer_infodata_exch='lnd2cpl_run') - ! send from land to coupler, + ! send from land to coupler, call component_exch_moab(lnd(1), mlnid, mblxid, 0, seq_flds_l2x_fields) endif @@ -4725,7 +4734,7 @@ subroutine cime_run_rof_recv_post() mpicom_barrier=mpicom_CPLALLROFID, run_barriers=run_barriers, & timer_barrier='CPL:R2C_BARRIER', timer_comp_exch='CPL:R2C', & timer_map_exch='CPL:r2c_rofr2rofx', timer_infodata_exch='CPL:r2c_infoexch') - ! this is for one hop + ! this is for one hop call component_exch_moab(rof(1), mrofid, mbrxid, 0, seq_flds_r2x_fields) !call prep_rof_migrate_moab(infodata) @@ -4759,7 +4768,7 @@ end subroutine cime_run_rof_recv_post !---------------------------------------------------------------------------------- subroutine cime_run_ice_setup_send() - + use seq_flds_mod, only : seq_flds_x2i_fields use seq_comm_mct, only : mpsiid, mbixid ! Note that for atm->ice mapping below will leverage the assumption that the @@ -4834,7 +4843,7 @@ subroutine cime_run_ice_recv_post() ! this needs to happen between ice comp and ocn coupler directly ! it needs to be called on the joint comm between ice and coupler ! if we do a proper component_exch, then would need another hop, just on coupler pes - ! TODO when do we need to send from ice to ocn? Usually after ice run ? + ! TODO when do we need to send from ice to ocn? Usually after ice run ? call component_exch_moab(ice(1), mpsiid, mbixid, 0, seq_flds_i2x_fields) ! this migrates all fields from ice to coupler endif @@ -4973,7 +4982,7 @@ subroutine cime_run_calc_budgets1(in_cplrun) if (present(in_cplrun)) then lcplrun = .not. in_cplrun endif - + if (iamin_CPLID) then call cime_comp_barriers(mpicom=mpicom_CPLID, timer='CPL:BUDGET1_BARRIER') call t_drvstartf ('CPL:BUDGET1',cplrun=lcplrun,budget=.true.,barrier=mpicom_CPLID) @@ -5014,7 +5023,7 @@ subroutine cime_run_calc_budgets2(in_cplrun) if (present(in_cplrun)) then lcplrun = .not. in_cplrun endif - + if (iamin_CPLID) then call cime_comp_barriers(mpicom=mpicom_CPLID, timer='CPL:BUDGET2_BARRIER') @@ -5083,7 +5092,7 @@ subroutine cime_run_calc_budgets3(in_cplrun) if (present(in_cplrun)) then lcplrun = .not. in_cplrun endif - + if (iamin_CPLID) then call cime_comp_barriers(mpicom=mpicom_CPLID, timer='CPL:BUDGET0_BARRIER') call t_drvstartf ('CPL:BUDGET0',cplrun=lcplrun,budget=.true.,barrier=mpicom_CPLID) diff --git a/driver-moab/main/component_mod.F90 b/driver-moab/main/component_mod.F90 index a08f229ca152..88d152a95685 100644 --- a/driver-moab/main/component_mod.F90 +++ b/driver-moab/main/component_mod.F90 @@ -32,6 +32,7 @@ module component_mod use perf_mod use ESMF use seq_flds_mod, only: nan_check_component_fields + implicit none #include @@ -55,7 +56,6 @@ module component_mod public :: component_final ! mct and esmf versions public :: component_exch public :: component_diag - public :: component_exch_moab ! public :: ocn_cpl_moab @@ -463,6 +463,15 @@ subroutine component_init_aream(infodata, rof_c2_ocn, samegrid_ao, samegrid_al, use prep_ice_mod, only : prep_ice_get_mapper_SFo2i use prep_glc_mod, only : prep_glc_get_mapper_Sl2g use component_type_mod, only : atm, lnd, ice, ocn, rof, glc +#ifdef HAVE_MOAB + use iMOAB, only : iMOAB_DefineTagStorage, iMOAB_GetDoubleTagStorage, & + iMOAB_SetDoubleTagStorageWithGid, iMOAB_WriteMesh + + use iso_c_binding + ! character(1024) :: domain_file ! file containing domain info (set my input) + use seq_comm_mct, only: mboxid ! iMOAB id for MPAS ocean migrated mesh to coupler pes + use seq_comm_mct, only: mbaxid ! iMOAB id for atm migrated mesh to coupler pes +#endif ! ! Arguments type (seq_infodata_type) , intent(inout) :: infodata @@ -486,6 +495,12 @@ subroutine component_init_aream(infodata, rof_c2_ocn, samegrid_ao, samegrid_al, logical :: glc_present ! glc present flag integer :: ka,km character(*), parameter :: subname = '(component_init_aream)' +#ifdef HAVE_MOAB + integer :: tagtype, nloc, ent_type, tagindex, ierr + character*100 tagname + real(R8), allocatable, target :: data1(:) + integer , allocatable :: gids(:) ! used for setting values associated with ids +#endif !--------------------------------------------------------------- ! Note that the following is assumed to hold - all gsmaps_cx for a given @@ -511,7 +526,38 @@ subroutine component_init_aream(infodata, rof_c2_ocn, samegrid_ao, samegrid_al, km = mct_aVect_indexRa(dom_s%data, "aream" ) dom_s%data%rAttr(km,:) = dom_s%data%rAttr(ka,:) - call seq_map_map(mapper_Fa2o, av_s=dom_s%data, av_d=dom_d%data, fldlist='aream') +#ifdef HAVE_MOAB + tagtype = 1 ! dense, double + tagname='aream'//C_NULL_CHAR + nloc = mct_avect_lsize(dom_s%data) + allocate(data1(nloc)) + data1 = dom_s%data%rAttr(ka,:) + ent_type = 1 ! element dense double tags + allocate(gids(nloc)) + gids = dom_s%data%iAttr(mct_aVect_indexIA(dom_s%data,"GlobGridNum"),:) + ! ! now set data on the coupler side too + ierr = iMOAB_SetDoubleTagStorageWithGid ( mbaxid, tagname, nloc, ent_type, & + data1, gids) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in setting the aream tag on atm ' + call shr_sys_abort(subname//' ERROR in setting aream tag on atm ') + endif + ! project now aream on ocean (from atm) +#endif + call seq_map_map(mapper_Fa2o, av_s=dom_s%data, av_d=dom_d%data, fldlist='aream') + +#ifdef HAVE_MOAB +#ifdef MOABDEBUG + ierr = iMOAB_WriteMesh(mboxid, trim('recMeshOcnWithArea.h5m'//C_NULL_CHAR), & + trim(';PARALLEL=WRITE_PART'//C_NULL_CHAR)) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in writing ocean mesh coupler ' + call shr_sys_abort(subname//' ERROR in writing ocean mesh coupler ') + endif +#endif +#endif + + else gsmap_s => component_get_gsmap_cx(ocn(1)) ! gsmap_ox gsmap_d => component_get_gsmap_cx(atm(1)) ! gsmap_ax @@ -526,7 +572,7 @@ subroutine component_init_aream(infodata, rof_c2_ocn, samegrid_ao, samegrid_al, call t_stopf('CPL:seq_map_readdata-ocn2atm') endif - end if + endif if (ice_present .and. ocn_present) then dom_s => component_get_dom_cx(ocn(1)) !dom_ox @@ -664,6 +710,7 @@ subroutine component_init_areacor_moab (comp, mbccid, mbcxid, seq_flds_c2x_fluxe ! ! Uses use seq_domain_mct, only : seq_domain_areafactinit + use cplcomp_exchange_mod, only: component_exch_moab use ISO_C_BINDING, only : C_NULL_CHAR use shr_kind_mod , only : CXX => shr_kind_CXX use iMOAB, only: iMOAB_DefineTagStorage, iMOAB_GetDoubleTagStorage, & @@ -682,7 +729,7 @@ subroutine component_init_areacor_moab (comp, mbccid, mbcxid, seq_flds_c2x_fluxe character(*), parameter :: subname = '(component_init_areacor_moab)' character(CXX) :: tagname integer :: tagtype, numco, tagindex, lsize, i, j, arrsize, ierr, nfields - real (kind=r8) , allocatable :: areas (:,:), factors(:,:), vals(:,:) ! 2 tags values, area, aream, + real (kind=r8) , allocatable :: areas (:,:), factors(:,:), vals(:,:) ! 2 tags values, area, aream, real (kind=r8) :: rarea, raream, rmask, fact integer nvert(3), nvise(3), nbl(3), nsurf(3), nvisBC(3) type(mct_list) :: temp_list ! used to count number of fields @@ -690,14 +737,14 @@ subroutine component_init_areacor_moab (comp, mbccid, mbcxid, seq_flds_c2x_fluxe if (comp(1)%iamin_cplcompid) then tagname='aream'//C_NULL_CHAR - ! bring on the comp side the aream from maps + ! bring on the comp side the aream from maps ! (it is either computed by mapping routine or read from mapping files) call component_exch_moab(comp(1), mbcxid, mbccid, 1, tagname) ! For only component pes if (comp(1)%iamin_compid) then ! Allocate and initialize area correction factors on component processes - ! get areas, first allocate memory + ! get areas, first allocate memory lsize = comp(1)%mblsize allocate(areas (lsize, 3)) ! lsize is along grid; read mask too allocate(factors (lsize, 2)) @@ -717,7 +764,7 @@ subroutine component_init_areacor_moab (comp, mbccid, mbcxid, seq_flds_c2x_fluxe if ( abs(rmask) >= 1.0e-06) then if (rarea * raream /= 0.0_R8) then factors(i,1) = rarea/raream - factors(i,2)= 1.0_R8/factors(i,1) + factors(i,2)= 1.0_R8/factors(i,1) else write(logunit,*) trim(subname),' ERROR area,aream= ', & rarea,raream,' in ',i,lsize @@ -748,14 +795,14 @@ subroutine component_init_areacor_moab (comp, mbccid, mbcxid, seq_flds_c2x_fluxe call mct_list_init(temp_list, seq_flds_c2x_fluxes) nfields=mct_list_nitem (temp_list) call mct_list_clean(temp_list) - + allocate(vals(lsize, nfields)) tagname = trim(seq_flds_c2x_fluxes)//C_NULL_CHAR arrsize = lsize * nfields - ierr = iMOAB_GetDoubleTagStorage( mbccid, tagname, arrsize , comp(1)%mbGridType, vals) + ierr = iMOAB_GetDoubleTagStorage( mbccid, tagname, arrsize, comp(1)%mbGridType, vals ) if (ierr .ne. 0) then - call shr_sys_abort(subname//' cannot get flux values ') + call shr_sys_abort(subname//' cannot get flux values: '//tagname) endif ! multiply them with the factors(i,1) do i=1,lsize @@ -767,11 +814,11 @@ subroutine component_init_areacor_moab (comp, mbccid, mbcxid, seq_flds_c2x_fluxe enddo endif enddo - ierr = iMOAB_SetDoubleTagStorage( mbccid, tagname, arrsize , comp(1)%mbGridType, vals) + ierr = iMOAB_SetDoubleTagStorage( mbccid, tagname, arrsize, comp(1)%mbGridType, vals) if (ierr .ne. 0) then call shr_sys_abort(subname//' cannot set new flux values ') endif - + ! call mct_avect_vecmult(comp(eci)%c2x_cc, comp(eci)%mdl2drv, seq_flds_c2x_fluxes, mask_spval=.true.) ! send to coupler corrected values @@ -781,7 +828,7 @@ subroutine component_init_areacor_moab (comp, mbccid, mbcxid, seq_flds_c2x_fluxe deallocate(vals) endif - ! send data to coupler exchange ? everything, not only fluxes ? + ! send data to coupler exchange ? everything, not only fluxes ? call component_exch_moab(comp(1), mbccid, mbcxid, 0, seq_flds_c2x_fields) endif @@ -1134,95 +1181,6 @@ subroutine component_diag(infodata, comp, flow, comment, info_debug, timer_diag end subroutine component_diag - ! can exchange data between mesh in component and mesh on coupler. Either way. - ! used in first hop of 2-hop - subroutine component_exch_moab(comp, mbAPPid1, mbAppid2, direction, fields ) - - use iMOAB , only: iMOAB_SendElementTag, iMOAB_ReceiveElementTag, iMOAB_WriteMesh, iMOAB_FreeSenderBuffers - use seq_comm_mct, only : num_moab_exports ! for debugging - use ISO_C_BINDING, only : C_NULL_CHAR - use shr_kind_mod , only : CXX => shr_kind_CXX - !--------------------------------------------------------------- - ! Description - ! send tags (fields) from component to coupler or from coupler to component - - type(component_type) , intent(in) :: comp - ! direction 0 is from component to coupler; 1 is from coupler to component - integer, intent(in) :: mbAPPid1, mbAppid2, direction - character(CXX) , intent(in) :: fields - - character(*), parameter :: subname = '(component_exch_moab)' - integer :: id_join, source_id, target_id, ierr - integer :: mpicom_join - character(CXX) :: tagname - character*100 outfile, wopts, lnum, dir - - ! how to get mpicomm for joint comp + coupler - id_join = comp%cplcompid - call seq_comm_getinfo(ID_join,mpicom=mpicom_join) -! - tagName = trim(fields)//C_NULL_CHAR - - if (direction .eq. 0) then - source_id = comp%compid - target_id = comp%cplcompid - else ! direction eq 1 - source_id = comp%cplcompid - target_id = comp%compid - endif - ! for atm, add 200 to component side, because we will involve always the point cloud - ! we are not supporting anymore the spectral case, at least for the time being - ! we need to fix fv-cgll projection first - if (comp%oneletterid == 'a' .and. direction .eq. 0 ) then - source_id = source_id + 200 - endif - if (comp%oneletterid == 'a' .and. direction .eq. 1 ) then - target_id = target_id + 200 - endif - if (mbAPPid1 .ge. 0) then ! send - - ! basically, use the initial partitioning - ierr = iMOAB_SendElementTag(mbAPPid1, tagName, mpicom_join, target_id) - if (ierr .ne. 0) then - call shr_sys_abort(subname//' cannot send element tag') - endif - - endif - if ( mbAPPid2 .ge. 0 ) then ! we are on receiving end - ierr = iMOAB_ReceiveElementTag(mbAPPid2, tagName, mpicom_join, source_id) - if (ierr .ne. 0) then - call shr_sys_abort(subname//' cannot receive element tag') - endif - endif - -! ! we can now free the sender buffers - if (mbAPPid1 .ge. 0) then - ierr = iMOAB_FreeSenderBuffers(mbAPPid1, target_id) - if (ierr .ne. 0) then - call shr_sys_abort(subname//' cannot free sender buffers') - endif - endif - -#ifdef MOABDEBUG - if (mbAPPid2 .ge. 0 ) then ! we are on receiving pes, for sure - ! number_proj = number_proj+1 ! count the number of projections - write(lnum,"(I0.2)") num_moab_exports - if (direction .eq. 0 ) then - dir = 'c2x' - else - dir = 'x2c' - endif - outfile = comp%ntype//'_'//trim(dir)//'_'//trim(lnum)//'.h5m'//C_NULL_CHAR - wopts = ';PARALLEL=WRITE_PART'//C_NULL_CHAR ! - ierr = iMOAB_WriteMesh(mbAPPid2, trim(outfile), trim(wopts)) - if (ierr .ne. 0) then - call shr_sys_abort(subname//' cannot write file '// outfile) - endif - endif -#endif - - end subroutine component_exch_moab - subroutine factor_moab_comp(comp, type, seq_flds_fluxes) use ISO_C_BINDING, only : C_NULL_CHAR use shr_kind_mod , only : CXX => shr_kind_CXX @@ -1232,11 +1190,11 @@ subroutine factor_moab_comp(comp, type, seq_flds_fluxes) character(len=*) , intent(in) :: type character(len=*) , intent(in) :: seq_flds_fluxes - character(CXX) :: tagname + character(CXX) :: tagname type(mct_list) :: temp_list ! used to count number of fields integer :: nfields, arrsize, ierr, i, j real (kind=r8) , allocatable :: vals(:,:) ! tags values to be multiplied - real (kind=r8) , allocatable :: factors(:) + real (kind=r8) , allocatable :: factors(:) character(*), parameter :: subname = '(factor_moab_comp)' diff --git a/driver-moab/main/component_type_mod.F90 b/driver-moab/main/component_type_mod.F90 index 8407ac2b05d8..85a300356c9a 100644 --- a/driver-moab/main/component_type_mod.F90 +++ b/driver-moab/main/component_type_mod.F90 @@ -482,7 +482,7 @@ subroutine compare_mct_av_moab_tag(comp, attrVect, mct_field, appId, tagname, en deallocate(values) ! now start comparing tags after set - ierr = iMOAB_GetMeshInfo ( appId, nvert, nvise, nbl, nsurf, nvisBC ); + ierr = iMOAB_GetMeshInfo ( appId, nvert, nvise, nbl, nsurf, nvisBC ) if (ierr > 0 ) & call shr_sys_abort(subname//'Error: fail to get mesh info') if (ent_type .eq. 0) then @@ -503,6 +503,7 @@ subroutine compare_mct_av_moab_tag(comp, attrVect, mct_field, appId, tagname, en values = mct_values - values difference = dot_product(values, values) + differenceg = 0. ! intel complained; why ? call shr_mpi_sum(difference,differenceg,mpicom,subname) difference = sqrt(differenceg) iamroot = seq_comm_iamroot(CPLID) diff --git a/driver-moab/main/cplcomp_exchange_mod.F90 b/driver-moab/main/cplcomp_exchange_mod.F90 index 8f06ff714891..2fd8bb1f5f60 100644 --- a/driver-moab/main/cplcomp_exchange_mod.F90 +++ b/driver-moab/main/cplcomp_exchange_mod.F90 @@ -29,7 +29,7 @@ module cplcomp_exchange_mod use seq_comm_mct, only : MPSIID, mbixid ! sea-ice on comp pes and on coupler pes use seq_comm_mct, only : mrofid, mbrxid ! iMOAB id of moab rof app on comp pes and on coupler too use shr_mpi_mod, only: shr_mpi_max - use dimensions_mod, only : np ! for atmosphere + ! use dimensions_mod, only : np ! for atmosphere use iso_c_binding implicit none @@ -50,6 +50,7 @@ module cplcomp_exchange_mod public :: seq_mctext_avExtend public :: cplcomp_moab_Init ! called to migrate MOAB mesh from ! component pes to coupler pes + public :: component_exch_moab !-------------------------------------------------------------------------- ! Private interfaces !-------------------------------------------------------------------------- @@ -993,8 +994,7 @@ subroutine cplcomp_moab_Init(infodata,comp) ! use iMOAB, only: iMOAB_RegisterApplication, iMOAB_ReceiveMesh, iMOAB_SendMesh, & iMOAB_WriteMesh, iMOAB_DefineTagStorage, iMOAB_GetMeshInfo, & - iMOAB_SetIntTagStorage, iMOAB_FreeSenderBuffers, iMOAB_ComputeCommGraph, iMOAB_LoadMesh - ! use component_mod, only: component_exch_moab + iMOAB_FreeSenderBuffers, iMOAB_ComputeCommGraph, iMOAB_LoadMesh ! use seq_infodata_mod ! @@ -1019,13 +1019,18 @@ subroutine cplcomp_moab_Init(infodata,comp) character*200 :: appname, outfile, wopts, ropts character(CL) :: rtm_mesh character(CL) :: lnd_domain + character(CL) :: ocn_domain + character(CL) :: atm_mesh integer :: maxMH, maxMPO, maxMLID, maxMSID, maxMRID ! max pids for moab apps atm, ocn, lnd, sea-ice, rof integer :: tagtype, numco, tagindex, partMethod, nghlay integer :: rank, ent_type integer :: typeA, typeB, ATM_PHYS_CID ! used to compute par graph between atm phys ! and atm spectral on coupler - character(CXX) :: tagname - integer nvert(3), nvise(3), nbl(3), nsurf(3), nvisBC(3) + character(CXX) :: tagname + integer nvert(3), nvise(3), nbl(3), nsurf(3), nvisBC(3) + ! type(mct_list) :: temp_list + ! integer :: nfields, arrsize + ! real(R8), allocatable, target :: values(:) !----------------------------------------------------- @@ -1051,7 +1056,7 @@ subroutine cplcomp_moab_Init(infodata,comp) call seq_comm_getinfo(ID_new ,mpicom=mpicom_new) call seq_comm_getinfo(ID_join,mpicom=mpicom_join) - call shr_mpi_max(mhid, maxMH, mpicom_join, all=.true.) ! if on atm / cpl joint, maxMH /= -1 + call shr_mpi_max(mphaid, maxMH, mpicom_join, all=.true.) ! if on atm / cpl joint, maxMH /= -1 call shr_mpi_max(mpoid, maxMPO, mpicom_join, all=.true.) call shr_mpi_max(mlnid, maxMLID, mpicom_join, all=.true.) call shr_mpi_max(MPSIID, maxMSID, mpicom_join, all=.true.) @@ -1063,18 +1068,31 @@ subroutine cplcomp_moab_Init(infodata,comp) if ( comp%oneletterid == 'a' .and. maxMH /= -1) then call seq_comm_getinfo(cplid ,mpigrp=mpigrp_cplid) ! receiver group call seq_comm_getinfo(id_old,mpigrp=mpigrp_old) ! component group pes + + ! find atm mesh/domain file if it exists; it would be for data atm model (atm_prognostic false) + call seq_infodata_GetData(infodata,atm_mesh = atm_mesh) ! now, if on coupler pes, receive mesh; if on comp pes, send mesh + + if (mphaid >= 0) then + ierr = iMOAB_GetMeshInfo ( mphaid, nvert, nvise, nbl, nsurf, nvisBC ) + comp%mbApCCid = mphaid ! phys atm + comp%mbGridType = 0 ! point cloud + comp%mblsize = nvert(1) ! point cloud + endif + if (MPI_COMM_NULL /= mpicom_old ) then ! it means we are on the component pes (atmosphere) ! send mesh to coupler - if (atm_pg_active) then ! change : send the pg2 mesh, not coarse mesh, when atm pg active - ierr = iMOAB_SendMesh(mhpgid, mpicom_join, mpigrp_cplid, id_join, partMethod) - else - ! still use the mhid, original coarse mesh - ierr = iMOAB_SendMesh(mhid, mpicom_join, mpigrp_cplid, id_join, partMethod) - endif - if (ierr .ne. 0) then - write(logunit,*) subname,' error in sending mesh from atm comp ' - call shr_sys_abort(subname//' ERROR in sending mesh from atm comp') + if ( trim(atm_mesh) == 'none' ) then + if (atm_pg_active) then ! change : send the pg2 mesh, not coarse mesh, when atm pg active + ierr = iMOAB_SendMesh(mhpgid, mpicom_join, mpigrp_cplid, id_join, partMethod) + else + ! still use the mhid, original coarse mesh + ierr = iMOAB_SendMesh(mhid, mpicom_join, mpigrp_cplid, id_join, partMethod) + endif + if (ierr .ne. 0) then + write(logunit,*) subname,' error in sending mesh from atm comp ' + call shr_sys_abort(subname//' ERROR in sending mesh from atm comp') + endif endif endif if (MPI_COMM_NULL /= mpicom_new ) then ! we are on the coupler pes @@ -1085,25 +1103,50 @@ subroutine cplcomp_moab_Init(infodata,comp) write(logunit,*) subname,' error in registering ', appname call shr_sys_abort(subname//' ERROR registering '// appname) endif - ierr = iMOAB_ReceiveMesh(mbaxid, mpicom_join, mpigrp_old, id_old) - if (ierr .ne. 0) then - write(logunit,*) subname,' error in receiving mesh on atm coupler ' - call shr_sys_abort(subname//' ERROR in receiving mesh on atm coupler ') + if ( trim(atm_mesh) == 'none' ) then + ierr = iMOAB_ReceiveMesh(mbaxid, mpicom_join, mpigrp_old, id_old) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in receiving mesh on atm coupler ' + call shr_sys_abort(subname//' ERROR in receiving mesh on atm coupler ') + endif + else + ! we need to read the atm mesh on coupler, from domain file + ierr = iMOAB_LoadMesh(mbaxid, trim(atm_mesh)//C_NULL_CHAR, & + "PARALLEL=READ_PART;PARTITION_METHOD=SQIJ;NO_CULLING", 0) + if ( ierr /= 0 ) then + write(logunit,*) 'Failed to load atm domain mesh on coupler' + call shr_sys_abort(subname//' ERROR Failed to load atm domain mesh on coupler ') + endif + ! right now, turn atm_pg_active to true + atm_pg_active = .true. ! FIXME TODO + ! need to add global id tag to the app, it will be used in restart + tagtype = 0 ! dense, integer + numco = 1 + tagname='GLOBAL_ID'//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mbaxid, tagname, tagtype, numco, tagindex ) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in adding global id tag to atmx' + call shr_sys_abort(subname//' ERROR in adding global id tag to atmx ') + endif endif + + endif ! iMOAB_FreeSenderBuffers needs to be called after receiving the mesh if (mhid .ge. 0) then ! we are on component atm pes - context_id = id_join - if (atm_pg_active) then! we send mesh from mhpgid app - ierr = iMOAB_FreeSenderBuffers(mhpgid, context_id) - else - ierr = iMOAB_FreeSenderBuffers(mhid, context_id) - endif - if (ierr .ne. 0) then - write(logunit,*) subname,' error in freeing send buffers ' - call shr_sys_abort(subname//' ERROR in freeing send buffers') + if ( trim(atm_mesh) == 'none' ) then + context_id = id_join + if (atm_pg_active) then! we send mesh from mhpgid app + ierr = iMOAB_FreeSenderBuffers(mhpgid, context_id) + else + ierr = iMOAB_FreeSenderBuffers(mhid, context_id) + endif + if (ierr .ne. 0) then + write(logunit,*) subname,' error in freeing send buffers ' + call shr_sys_abort(subname//' ERROR in freeing send buffers') + endif endif endif @@ -1115,13 +1158,8 @@ subroutine cplcomp_moab_Init(infodata,comp) typeB = 3 ! in this case, we will have cells associated with DOFs as GLOBAL_ID tag endif ATM_PHYS_CID = 200 + id_old ! 200 + 5 for atm, see line 969 ATM_PHYS = 200 + ATMID ! in - ! components/cam/src/cpl/atm_comp_mct.F90 - if (mphaid >= 0) then - ierr = iMOAB_GetMeshInfo ( mphaid, nvert, nvise, nbl, nsurf, nvisBC ) - comp%mbApCCid = mphaid ! phys atm - comp%mbGridType = 0 ! point cloud - comp%mblsize = nvert(1) ! point cloud - endif + ! components/cam/src/cpl/atm_comp_mct.F90 + ! components/data_comps/datm/src/atm_comp_mct.F90 ! line 177 !! ierr = iMOAB_ComputeCommGraph( mphaid, mbaxid, mpicom_join, mpigrp_old, mpigrp_cplid, & typeA, typeB, ATM_PHYS_CID, id_join) ! ID_JOIN is now 6 @@ -1135,7 +1173,7 @@ subroutine cplcomp_moab_Init(infodata,comp) numco = 1 ! usually 1 value per cell else ! this is not supported now, but leave it here tagname = trim(seq_flds_a2x_ext_fields)//C_NULL_CHAR ! MOAB versions of a2x for spectral - numco = np*np ! usually 16 values per cell, GLL points; should be 4 x 4 = 16 + numco = 16 ! np*np ! usually 16 values per cell, GLL points; should be 4 x 4 = 16 endif ierr = iMOAB_DefineTagStorage(mbaxid, tagname, tagtype, numco, tagindex ) if (ierr .ne. 0) then @@ -1148,6 +1186,13 @@ subroutine cplcomp_moab_Init(infodata,comp) write(logunit,*) subname,' error in defining tags seq_flds_x2a_fields on atm on coupler ' call shr_sys_abort(subname//' ERROR in defining tags ') endif + ! zero out the fields for seq_flds_x2a_fields and seq_flds_a2x_fields, on mbaxid + ! first determine the size of array + ierr = iMOAB_GetMeshInfo ( mbaxid, nvert, nvise, nbl, nsurf, nvisBC ) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in getting mesh info on atm on coupler ' + call shr_sys_abort(subname//' ERROR in getting mesh info on atm on coupler ') + endif !add the normalization tag tagname = trim(seq_flds_dom_fields)//":norm8wt"//C_NULL_CHAR @@ -1156,6 +1201,12 @@ subroutine cplcomp_moab_Init(infodata,comp) write(logunit,*) subname,' error in defining tags seq_flds_dom_fields on atm on coupler ' call shr_sys_abort(subname//' ERROR in defining tags ') endif + if ( trim(atm_mesh) /= 'none' ) then + ! also, frac, area, aream, masks has to come from atm mphaid, not from domain file reader + ! this is hard to digest :( + tagname = 'area:aream:frac:mask'//C_NULL_CHAR + call component_exch_moab(comp, mphaid, mbaxid, 0, tagname) + endif endif @@ -1185,6 +1236,9 @@ subroutine cplcomp_moab_Init(infodata,comp) call seq_comm_getinfo(cplid ,mpigrp=mpigrp_cplid) ! receiver group call seq_comm_getinfo(id_old,mpigrp=mpigrp_old) ! component group pes + ! find ocean domain file if it exists; it would be for data ocean model (ocn_prognostic false) + call seq_infodata_GetData(infodata,ocn_domain=ocn_domain) + if (MPI_COMM_NULL /= mpicom_old ) then ! it means we are on the component pes (ocean) #ifdef MOABDEBUG ! write out the mesh file to disk, in parallel @@ -1199,26 +1253,58 @@ subroutine cplcomp_moab_Init(infodata,comp) endif #endif - ! send mesh to coupler - ierr = iMOAB_SendMesh(mpoid, mpicom_join, mpigrp_cplid, id_join, partMethod) + ierr = iMOAB_GetMeshInfo ( mpoid, nvert, nvise, nbl, nsurf, nvisBC ) if (ierr .ne. 0) then - write(logunit,*) subname,' error in sending ocean mesh to coupler ' - call shr_sys_abort(subname//' ERROR in sending ocean mesh to coupler ') + write(logunit,*) subname,' error in getting mesh info on ocn ' + call shr_sys_abort(subname//' ERROR in getting mesh info on ocn ') endif - if (mpoid >= 0) then - ierr = iMOAB_GetMeshInfo ( mpoid, nvert, nvise, nbl, nsurf, nvisBC ) - comp%mbApCCid = mpoid ! phys atm + comp%mbApCCid = mpoid ! ocn comp app in moab + if ( trim(ocn_domain) == 'none' ) then + ! send mesh to coupler + ierr = iMOAB_SendMesh(mpoid, mpicom_join, mpigrp_cplid, id_join, partMethod) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in sending ocean mesh to coupler ' + call shr_sys_abort(subname//' ERROR in sending ocean mesh to coupler ') + endif comp%mbGridType = 1 ! cells comp%mblsize = nvise(1) ! cells + else + comp%mbGridType = 0 ! vertices + comp%mblsize = nvert(1) ! vertices endif - endif if (MPI_COMM_NULL /= mpicom_new ) then ! we are on the coupler pes appname = "COUPLE_MPASO"//C_NULL_CHAR ! migrated mesh gets another app id, moab ocean to coupler (mbox) ierr = iMOAB_RegisterApplication(trim(appname), mpicom_new, id_join, mboxid) - ierr = iMOAB_ReceiveMesh(mboxid, mpicom_join, mpigrp_old, id_old) - + if (ierr .ne. 0) then + write(logunit,*) subname,' cannot register ocn app on coupler ' + call shr_sys_abort(subname//' ERROR cannot register ocn app on coupler ') + endif + if ( trim(ocn_domain) == 'none' ) then + ierr = iMOAB_ReceiveMesh(mboxid, mpicom_join, mpigrp_old, id_old) + if (ierr .ne. 0) then + write(logunit,*) subname,' cannot receive ocn mesh on coupler ' + call shr_sys_abort(subname//' ERROR cannot receive ocn mesh on coupler ') + endif + else + ! we need to read the ocean mesh on coupler, from domain file + ierr = iMOAB_LoadMesh(mboxid, trim(ocn_domain)//C_NULL_CHAR, & + "PARALLEL=READ_PART;PARTITION_METHOD=SQIJ;NO_CULLING", 0) + if ( ierr /= 0 ) then + write(logunit,*) 'Failed to load ocean domain mesh on coupler' + call shr_sys_abort(subname//' ERROR Failed to load ocean domain mesh on coupler ') + endif + ! need to add global id tag to the app, it will be used in restart + tagtype = 0 ! dense, integer + numco = 1 + tagname='GLOBAL_ID'//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mboxid, tagname, tagtype, numco, tagindex ) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in adding global id tag to ocnx' + call shr_sys_abort(subname//' ERROR in adding global id tag to ocnx ') + endif + endif tagname = trim(seq_flds_o2x_fields)//C_NULL_CHAR tagtype = 1 ! dense, double numco = 1 ! one value per cell @@ -1256,39 +1342,89 @@ subroutine cplcomp_moab_Init(infodata,comp) #endif endif if (mpoid .ge. 0) then ! we are on component ocn pes - context_id = id_join - ierr = iMOAB_FreeSenderBuffers(mpoid, context_id) - if (ierr .ne. 0) then - write(logunit,*) subname,' error in freeing buffers ' - call shr_sys_abort(subname//' ERROR in freeing buffers ') + if ( trim(ocn_domain) == 'none' ) then + context_id = id_join + ierr = iMOAB_FreeSenderBuffers(mpoid, context_id) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in freeing buffers ' + call shr_sys_abort(subname//' ERROR in freeing buffers ') + endif endif endif + ! in case of domain read, we need to compute the comm graph + if ( trim(ocn_domain) /= 'none' ) then + ! we are now on joint pes, compute comm graph between data ocn and coupler model ocn + typeA = 2 ! point cloud on component PEs + typeB = 3 ! full mesh on coupler pes, we just read it + ierr = iMOAB_ComputeCommGraph( mpoid, mboxid, mpicom_join, mpigrp_old, mpigrp_cplid, & + typeA, typeB, id_old, id_join) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in computing comm graph for data ocn model ' + call shr_sys_abort(subname//' ERROR in computing comm graph for data ocn model ') + endif + ! also, frac, area, aream, masks has to come from ocean mpoid, not from domain file reader + ! this is hard to digest :( + tagname = 'area:aream:frac:mask'//C_NULL_CHAR + call component_exch_moab(comp, mpoid, mboxid, 0, tagname) + endif + ! start copy ! do another ocean copy of the mesh on the coupler, just because So_fswpen field ! would appear twice on original mboxid, once from xao states, once from o2x states id_join = id_join + 1000! kind of random if (MPI_COMM_NULL /= mpicom_old ) then ! it means we are on the component pes (ocean) - ! send mesh to coupler, the second time! a copy would be cheaper - ierr = iMOAB_SendMesh(mpoid, mpicom_join, mpigrp_cplid, id_join, partMethod) - if (ierr .ne. 0) then - write(logunit,*) subname,' error in sending ocean mesh to coupler the second time' - call shr_sys_abort(subname//' ERROR in sending ocean mesh to coupler the second time ') + if ( trim(ocn_domain) == 'none' ) then + ! send mesh to coupler, the second time! a copy would be cheaper + ierr = iMOAB_SendMesh(mpoid, mpicom_join, mpigrp_cplid, id_join, partMethod) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in sending ocean mesh to coupler the second time' + call shr_sys_abort(subname//' ERROR in sending ocean mesh to coupler the second time ') + endif endif endif if (MPI_COMM_NULL /= mpicom_new ) then ! we are on the coupler pes appname = "COUPLE_MPASOF"//C_NULL_CHAR ! migrated mesh gets another app id, moab ocean to coupler (mbox) ierr = iMOAB_RegisterApplication(trim(appname), mpicom_new, id_join, mbofxid) - ierr = iMOAB_ReceiveMesh(mbofxid, mpicom_join, mpigrp_old, id_old) - - endif - if (mpoid .ge. 0) then ! we are on component ocn pes again, release buffers - context_id = id_join - ierr = iMOAB_FreeSenderBuffers(mpoid, context_id) if (ierr .ne. 0) then - write(logunit,*) subname,' error in freeing buffers ' - call shr_sys_abort(subname//' ERROR in freeing buffers ') + write(logunit,*) subname,' cant register second ocean mesh on coupler' + call shr_sys_abort(subname//' ERROR cant register second ocean mesh on coupler') endif + if ( trim(ocn_domain) == 'none' ) then + ierr = iMOAB_ReceiveMesh(mbofxid, mpicom_join, mpigrp_old, id_old) + if (ierr .ne. 0) then + write(logunit,*) subname,' cant receive second ocean mesh on coupler' + call shr_sys_abort(subname//' ERROR cant receive second ocean mesh on coupler') + endif + else + ! we need to read the ocean mesh on coupler, from domain file + ierr = iMOAB_LoadMesh(mbofxid, trim(ocn_domain)//C_NULL_CHAR, & + "PARALLEL=READ_PART;PARTITION_METHOD=SQIJ;NO_CULLING", 0) + if ( ierr /= 0 ) then + write(logunit,*) 'Failed to load second ocean domain mesh on coupler' + call shr_sys_abort(subname//' ERROR Failed to load second ocean domain mesh on coupler ') + endif + ! need to add global id tag to the app, it will be used in restart + tagtype = 0 ! dense, integer + numco = 1 + tagname='GLOBAL_ID'//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mbofxid, tagname, tagtype, numco, tagindex ) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in adding global id tag to ocnx' + call shr_sys_abort(subname//' ERROR in adding global id tag to ocnx ') + endif + endif + + endif + if (mpoid .ge. 0) then ! we are on component ocn pes again, release buffers + if ( trim(ocn_domain) == 'none' ) then + context_id = id_join + ierr = iMOAB_FreeSenderBuffers(mpoid, context_id) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in freeing buffers ' + call shr_sys_abort(subname//' ERROR in freeing buffers ') + endif + endif endif ! end copy endif @@ -1577,4 +1713,98 @@ subroutine cplcomp_moab_Init(infodata,comp) end subroutine cplcomp_moab_Init + + ! can exchange data between mesh in component and mesh on coupler. Either way. + ! used in first hop of 2-hop + subroutine component_exch_moab(comp, mbAPPid1, mbAppid2, direction, fields ) + + use iMOAB , only: iMOAB_SendElementTag, iMOAB_ReceiveElementTag, iMOAB_WriteMesh, iMOAB_FreeSenderBuffers + use seq_comm_mct, only : num_moab_exports ! for debugging + use ISO_C_BINDING, only : C_NULL_CHAR + use shr_kind_mod , only : CXX => shr_kind_CXX + !--------------------------------------------------------------- + ! Description + ! send tags (fields) from component to coupler or from coupler to component + + type(component_type) , intent(in) :: comp + ! direction 0 is from component to coupler; 1 is from coupler to component + integer, intent(in) :: mbAPPid1, mbAppid2, direction + character(CXX) , intent(in) :: fields + + character(*), parameter :: subname = '(component_exch_moab)' + integer :: id_join, source_id, target_id, ierr + integer :: mpicom_join + character(CXX) :: tagname + character*100 outfile, wopts, lnum, dir + + ! how to get mpicomm for joint comp + coupler + id_join = comp%cplcompid + call seq_comm_getinfo(ID_join,mpicom=mpicom_join) +! + tagName = trim(fields)//C_NULL_CHAR + + if (direction .eq. 0) then + source_id = comp%compid + target_id = comp%cplcompid + else ! direction eq 1 + source_id = comp%cplcompid + target_id = comp%compid + endif + ! for atm, add 200 to component side, because we will involve always the point cloud + ! we are not supporting anymore the spectral case, at least for the time being + ! we need to fix fv-cgll projection first + if (comp%oneletterid == 'a' .and. direction .eq. 0 ) then + source_id = source_id + 200 + endif + if (comp%oneletterid == 'a' .and. direction .eq. 1 ) then + target_id = target_id + 200 + endif + if (mbAPPid1 .ge. 0) then ! send + + ! basically, use the initial partitioning + ierr = iMOAB_SendElementTag(mbAPPid1, tagName, mpicom_join, target_id) + if (ierr .ne. 0) then + call shr_sys_abort(subname//' cannot send element tag') + endif + + endif + if ( mbAPPid2 .ge. 0 ) then ! we are on receiving end + ierr = iMOAB_ReceiveElementTag(mbAPPid2, tagName, mpicom_join, source_id) + if (ierr .ne. 0) then + call shr_sys_abort(subname//' cannot receive element tag') + endif + endif + +! ! we can now free the sender buffers + if (mbAPPid1 .ge. 0) then + ierr = iMOAB_FreeSenderBuffers(mbAPPid1, target_id) + if (ierr .ne. 0) then + call shr_sys_abort(subname//' cannot free sender buffers') + endif + endif + +#ifdef MOABDEBUG + if (direction .eq. 0 ) then + dir = 'c2x' + else + dir = 'x2c' + endif + if (seq_comm_iamroot(CPLID) ) then + write(logunit,'(A)') subname//' '//comp%ntype//' called in direction '//trim(dir)//' for fields '//trim(tagname) + endif + if (mbAPPid2 .ge. 0 ) then ! we are on receiving pes, for sure + ! number_proj = number_proj+1 ! count the number of projections + write(lnum,"(I0.2)") num_moab_exports + + outfile = comp%ntype//'_'//trim(dir)//'_'//trim(lnum)//'.h5m'//C_NULL_CHAR + wopts = ';PARALLEL=WRITE_PART'//C_NULL_CHAR ! + ierr = iMOAB_WriteMesh(mbAPPid2, trim(outfile), trim(wopts)) + if (ierr .ne. 0) then + call shr_sys_abort(subname//' cannot write file '// outfile) + endif + endif +#endif + + end subroutine component_exch_moab + end module cplcomp_exchange_mod diff --git a/driver-moab/main/prep_atm_mod.F90 b/driver-moab/main/prep_atm_mod.F90 index 040c617fe7c2..c8c2de763622 100644 --- a/driver-moab/main/prep_atm_mod.F90 +++ b/driver-moab/main/prep_atm_mod.F90 @@ -35,7 +35,7 @@ module prep_atm_mod use seq_comm_mct, only : seq_comm_getinfo => seq_comm_setptrs use seq_comm_mct, only : num_moab_exports - use dimensions_mod, only : np ! for atmosphere + !use dimensions_mod, only : np ! for atmosphere #ifdef MOABCOMP use component_type_mod, only: compare_mct_av_moab_tag #endif @@ -250,50 +250,19 @@ subroutine prep_atm_init(infodata, ocn_c2_atm, ice_c2_atm, lnd_c2_atm, iac_c2_at write(logunit,*) subname,' error in registering atm ocn intx' call shr_sys_abort(subname//' ERROR in registering atm ocn intx') endif - ierr = iMOAB_ComputeMeshIntersectionOnSphere (mboxid, mbaxid, mbintxoa) - if (ierr .ne. 0) then - write(logunit,*) subname,' error in computing ocn atm intx' - call shr_sys_abort(subname//' ERROR in computing ocn atm intx') - endif - if (iamroot_CPLID) then - write(logunit,*) 'iMOAB intersection between ocean and atm with id:', idintx - endif - - ! we also need to compute the comm graph for the second hop, from the ocn on coupler to the - ! ocean for the intx ocean-atm context (coverage) - ! call seq_comm_getinfo(CPLID ,mpigrp=mpigrp_CPLID) - type1 = 3; ! fv for ocean and atm; fv-cgll does not work anyway - type2 = 3; - ! ierr = iMOAB_ComputeCommGraph( mboxid, mbintxoa, &mpicom_CPLID, &mpigrp_CPLID, &mpigrp_CPLID, &type1, &type2, - ! &ocn_id, &idintx) - ierr = iMOAB_ComputeCommGraph( mboxid, mbintxoa, mpicom_CPLID, mpigrp_CPLID, mpigrp_CPLID, type1, type2, & - ocn(1)%cplcompid, idintx) - if (ierr .ne. 0) then - write(logunit,*) subname,' error in computing comm graph for second hop, ocn-atm' - call shr_sys_abort(subname//' ERROR in computing comm graph for second hop, ocn-atm') - endif - ! now take care of the mapper - if ( mapper_So2a%src_mbid .gt. -1 ) then - if (iamroot_CPLID) then - write(logunit,F00) 'overwriting '//trim(mapper_So2a%mbname) & - //' mapper_So2a' - endif - endif mapper_So2a%src_mbid = mboxid mapper_So2a%tgt_mbid = mbaxid ! mapper_So2a%intx_mbid = mbintxoa mapper_So2a%src_context = ocn(1)%cplcompid - mapper_So2a%intx_context = idintx wgtIdef = 'scalar'//C_NULL_CHAR mapper_So2a%weight_identifier = wgtIdef mapper_So2a%mbname = 'mapper_So2a' - ! because we will project fields from ocean to atm phys grid, we need to define - ! ocean o2x fields to atm phys grid (or atm spectral ext ) on coupler side - + ! Since we are projecting fields from OCN to ATM-PHY grid, we need to define + ! OCN o2x fields to ATM-PHY grid (or ATM-DYN (spectral) ) on coupler side if (atm_pg_active) then tagname = trim(seq_flds_o2x_fields)//C_NULL_CHAR tagtype = 1 ! dense @@ -304,75 +273,126 @@ subroutine prep_atm_init(infodata, ocn_c2_atm, ice_c2_atm, lnd_c2_atm, iac_c2_at call shr_sys_abort(subname//' ERROR in coin defining tags for seq_flds_o2x_fields') endif else ! spectral case, fix later TODO - numco = np*np ! + !numco = np*np ! + numco = 16 endif ! - volumetric = 0 ! can be 1 only for FV->DGLL or FV->CGLL; - if (atm_pg_active) then - dm2 = "fv"//C_NULL_CHAR - dofnameT="GLOBAL_ID"//C_NULL_CHAR - orderT = 1 ! fv-fv - else - dm2 = "cgll"//C_NULL_CHAR - dofnameT="GLOBAL_DOFS"//C_NULL_CHAR - orderT = np ! it should be 4 - endif - dm1 = "fv"//C_NULL_CHAR - dofnameS="GLOBAL_ID"//C_NULL_CHAR - orderS = 1 ! not much arguing - fNoBubble = 1 - monotonicity = 0 ! - noConserve = 0 - validate = 0 ! less verbose - fInverseDistanceMap = 0 - if (iamroot_CPLID) then - write(logunit,*) subname, 'launch iMOAB weights with args ', 'mbintxoa=', mbintxoa, ' wgtIdef=', wgtIdef, & - 'dm1=', trim(dm1), ' orderS=', orderS, 'dm2=', trim(dm2), ' orderT=', orderT, & - fNoBubble, monotonicity, volumetric, fInverseDistanceMap, & - noConserve, validate, & - trim(dofnameS), trim(dofnameT) - endif - ierr = iMOAB_ComputeScalarProjectionWeights ( mbintxoa, wgtIdef, & - trim(dm1), orderS, trim(dm2), orderT, ''//C_NULL_CHAR, & - fNoBubble, monotonicity, volumetric, fInverseDistanceMap, & - noConserve, validate, & - trim(dofnameS), trim(dofnameT) ) - if (ierr .ne. 0) then - write(logunit,*) subname,' error in iMOAB_ComputeScalarProjectionWeights ocn atm ' - call shr_sys_abort(subname//' ERROR in iMOAB_ComputeScalarProjectionWeights ocn atm ') - endif + if (.not. samegrid_ao) then ! data-OCN case + + ierr = iMOAB_ComputeMeshIntersectionOnSphere (mboxid, mbaxid, mbintxoa) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in computing ocn atm intx' + call shr_sys_abort(subname//' ERROR in computing ocn atm intx') + endif + if (iamroot_CPLID) then + write(logunit,*) 'iMOAB intersection between ocean and atm with id:', idintx + endif + + ! we also need to compute the comm graph for the second hop, from the ocn on coupler to the + ! ocean for the intx ocean-atm context (coverage) + ! + type1 = 3; ! fv for ocean and atm; fv-cgll does not work anyway + type2 = 3; + ! ierr = iMOAB_ComputeCommGraph( mboxid, mbintxoa, &mpicom_CPLID, &mpigrp_CPLID, &mpigrp_CPLID, &type1, &type2, + ! &ocn_id, &idintx) + ierr = iMOAB_ComputeCommGraph( mboxid, mbintxoa, mpicom_CPLID, mpigrp_CPLID, mpigrp_CPLID, type1, type2, & + ocn(1)%cplcompid, idintx) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in computing comm graph for second hop, ocn-atm' + call shr_sys_abort(subname//' ERROR in computing comm graph for second hop, ocn-atm') + endif + ! now take care of the mapper + if ( mapper_So2a%src_mbid .gt. -1 ) then + if (iamroot_CPLID) then + write(logunit,F00) 'overwriting '//trim(mapper_So2a%mbname) & + //' mapper_So2a' + endif + endif + + mapper_So2a%intx_context = idintx + + if (atm_pg_active) then + dm2 = "fv"//C_NULL_CHAR + dofnameT="GLOBAL_ID"//C_NULL_CHAR + orderT = 1 ! fv-fv + else + dm2 = "cgll"//C_NULL_CHAR + dofnameT="GLOBAL_DOFS"//C_NULL_CHAR + orderT = 4 ! np ! it should be 4 + endif + dm1 = "fv"//C_NULL_CHAR + dofnameS="GLOBAL_ID"//C_NULL_CHAR + orderS = 1 ! not much arguing + fNoBubble = 1 + monotonicity = 0 ! + noConserve = 0 + validate = 0 ! less verbose + fInverseDistanceMap = 0 + volumetric = 0 ! can be 1 only for FV->DGLL or FV->CGLL; + if (iamroot_CPLID) then + write(logunit,*) subname, 'launch iMOAB weights with args ', 'mbintxoa=', mbintxoa, ' wgtIdef=', wgtIdef, & + 'dm1=', trim(dm1), ' orderS=', orderS, 'dm2=', trim(dm2), ' orderT=', orderT, & + fNoBubble, monotonicity, volumetric, fInverseDistanceMap, & + noConserve, validate, & + trim(dofnameS), trim(dofnameT) + endif + ierr = iMOAB_ComputeScalarProjectionWeights ( mbintxoa, wgtIdef, & + trim(dm1), orderS, trim(dm2), orderT, ''//C_NULL_CHAR, & + fNoBubble, monotonicity, volumetric, fInverseDistanceMap, & + noConserve, validate, & + trim(dofnameS), trim(dofnameT) ) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in iMOAB_ComputeScalarProjectionWeights ocn atm ' + call shr_sys_abort(subname//' ERROR in iMOAB_ComputeScalarProjectionWeights ocn atm ') + endif #ifdef MOABDEBUG - wopts = C_NULL_CHAR - call shr_mpi_commrank( mpicom_CPLID, rank ) - if (rank .lt. 3) then - write(lnum,"(I0.2)")rank ! - outfile = 'intx_oa_'//trim(lnum)// '.h5m' // C_NULL_CHAR - ierr = iMOAB_WriteMesh(mbintxoa, outfile, wopts) ! write local intx file - if (ierr .ne. 0) then - write(logunit,*) subname,' error in writing intx file ' - call shr_sys_abort(subname//' ERROR in writing intx file ') - endif - endif + wopts = C_NULL_CHAR + call shr_mpi_commrank( mpicom_CPLID, rank ) + if (rank .lt. 3) then + write(lnum,"(I0.2)")rank ! + outfile = 'intx_oa_'//trim(lnum)// '.h5m' // C_NULL_CHAR + ierr = iMOAB_WriteMesh(mbintxoa, outfile, wopts) ! write local intx file + if (ierr .ne. 0) then + write(logunit,*) subname,' error in writing intx file ' + call shr_sys_abort(subname//' ERROR in writing intx file ') + endif + endif ! endif for MOABDEBUG #endif + else ! samegrid_ao = TRUE + + ! ATM and OCN components use the same mesh and DoF numbering (OCN is a subset of ATM); + ! We do not need to compute intersection since the "map" from ATM to OCN is essentially a + ! permutation operator and can be achieved by simply sending/receiving data from ATM to OCN + ! and viceversa, based on element GLOBAL_ID matching. In order to seamless produce the + ! permutation operator, we will compute a communication graph between ATM and OCN DoFs on the + ! coupler. + type1 = 3; ! FV mesh on coupler OCN + if (atm_pg_active) then + type2 = 3; ! FV for ATM; CGLL does not work correctly in parallel at the moment + else + type2 = 1 ! This projection works (CGLL to FV), but reverse does not (FV - CGLL) + endif + ierr = iMOAB_ComputeCommGraph( mboxid, mbaxid, mpicom_CPLID, mpigrp_CPLID, mpigrp_CPLID, type1, type2, & + ocn(1)%cplcompid, atm(1)%cplcompid ) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in computing communication graph for second hop, ATM-OCN' + call shr_sys_abort(subname//' ERROR in computing communication graph for second hop, ATM-OCN') + endif + mapper_So2a%intx_context = atm(1)%cplcompid + + endif ! if (.not. samegrid_ao) endif ! if ((mbaxid .ge. 0) .and. (mboxid .ge. 0)) then ! FLUX make the app and mapper for the a2o flux mappings if ((mbaxid .ge. 0) .and. (mbofxid .ge. 0)) then - ! we also need to compute the comm graph for the second hop, from the ocn on coupler to the - ! ocean for the intx ocean-atm context (coverage) - ! + ! We also need to compute the comm graph for the second hop, from the OCN on the coupler to the + ! OCN for the intersection of OCN-ATM context (coverage) call seq_comm_getinfo(CPLID ,mpigrp=mpigrp_CPLID) - type1 = 3; ! fv for ocean and atm; fv-cgll does not work anyway - type2 = 3; - ! we ideintified the app mbofxid with !id_join = id_join + 1000! kind of random - ! line 1267 in cplcomp_exchange_mod.F90 - context_id = ocn(1)%cplcompid + 1000 - ierr = iMOAB_ComputeCommGraph( mbofxid, mbintxoa, mpicom_CPLID, mpigrp_CPLID, mpigrp_CPLID, type1, type2, & - context_id, idintx) + if (ierr .ne. 0) then write(logunit,*) subname,' error in computing comm graph for second hop, ocnf -atm' call shr_sys_abort(subname//' ERROR in computing comm graph for second hop, ocnf-atm') @@ -384,14 +404,33 @@ subroutine prep_atm_init(infodata, ocn_c2_atm, ice_c2_atm, lnd_c2_atm, iac_c2_at //' mapper_Sof2a' endif endif + ! we identified the app mbofxid with !id_join = id_join + 1000! kind of random + ! line 1267 in cplcomp_exchange_mod.F90 + context_id = ocn(1)%cplcompid + 1000 mapper_Sof2a%src_mbid = mbofxid mapper_Sof2a%tgt_mbid = mbaxid mapper_Sof2a%intx_mbid = mbintxoa mapper_Sof2a%src_context = context_id - mapper_Sof2a%intx_context = idintx + mapper_Sof2a%intx_context = mapper_So2a%intx_context ! basically will use the same intx as ocean on coupler wgtIdef = 'scalar'//C_NULL_CHAR mapper_Sof2a%weight_identifier = wgtIdef mapper_Sof2a%mbname = 'mapper_Sof2a' + + type1 = 3; ! fv for ocean and atm; fv-cgll does not work anyway + type2 = 3; + if (.not. samegrid_ao) then ! data-OCN case + ! we use the same intx, because the mesh will be the same, between mbofxid and mboxid + ierr = iMOAB_ComputeCommGraph( mbofxid, mbintxoa, mpicom_CPLID, mpigrp_CPLID, mpigrp_CPLID, type1, type2, & + context_id, idintx) + else + ! this is a case appearing in the data ocean case --res ne4pg2_ne4pg2 --compset FAQP + ierr = iMOAB_ComputeCommGraph( mbofxid, mbaxid, mpicom_CPLID, mpigrp_CPLID, mpigrp_CPLID, type1, type2, & + context_id, atm(1)%cplcompid ) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in computing communication graph for second hop, ATM-OCN' + call shr_sys_abort(subname//' ERROR in computing communication graph for second hop, ATM-OCN') + endif + endif endif ! endif for HAVE_MOAB @@ -431,8 +470,8 @@ subroutine prep_atm_init(infodata, ocn_c2_atm, ice_c2_atm, lnd_c2_atm, iac_c2_at mapper_Fo2a%src_mbid = mboxid mapper_Fo2a%tgt_mbid = mbaxid mapper_Fo2a%intx_mbid = mbintxoa - mapper_Fo2a%src_context = ocn(1)%cplcompid - mapper_Fo2a%intx_context = idintx + mapper_Fo2a%src_context = mapper_So2a%src_context ! ocn(1)%cplcompid + mapper_Fo2a%intx_context = mapper_So2a%intx_context ! it could be different, based on samegrid_ao wgtIdef = 'scalar'//C_NULL_CHAR mapper_Fo2a%weight_identifier = wgtIdef mapper_Fo2a%mbname = 'mapper_Fo2a' @@ -447,8 +486,8 @@ subroutine prep_atm_init(infodata, ocn_c2_atm, ice_c2_atm, lnd_c2_atm, iac_c2_at mapper_Fof2a%src_mbid = mbofxid mapper_Fof2a%tgt_mbid = mbaxid mapper_Fof2a%intx_mbid = mbintxoa - mapper_Fof2a%src_context = ocn(1)%cplcompid - mapper_Fof2a%intx_context = idintx + mapper_Fof2a%src_context = mapper_Sof2a%src_context ! we use the same source 1000 + ? + mapper_Fof2a%intx_context = mapper_Sof2a%intx_context ! depends on samegrid_ao wgtIdef = 'scalar'//C_NULL_CHAR mapper_Fof2a%weight_identifier = wgtIdef mapper_Fof2a%mbname = 'mapper_Fof2a' @@ -459,6 +498,21 @@ subroutine prep_atm_init(infodata, ocn_c2_atm, ice_c2_atm, lnd_c2_atm, iac_c2_at endif ! endif (ocn_present) then call shr_sys_flush(logunit) +! because we will project fields from ocean to atm phys grid, we need to define + ! ice i2x fields to atm phys grid (or atm spectral ext ) on coupler side + if (atm_pg_active) then + tagname = trim(seq_flds_i2x_fields)//C_NULL_CHAR + tagtype = 1 ! dense + numco = 1 ! + ierr = iMOAB_DefineTagStorage(mbaxid, tagname, tagtype, numco, tagindex ) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in defining tags for seq_flds_i2x_fields' + call shr_sys_abort(subname//' ERROR in coin defining tags for seq_flds_i2x_fields') + endif + else ! spectral case, TODO + tagtype = 1 ! dense + endif + if (ice_c2_atm) then if (iamroot_CPLID) then write(logunit,*) ' ' @@ -472,7 +526,7 @@ subroutine prep_atm_init(infodata, ocn_c2_atm, ice_c2_atm, lnd_c2_atm, iac_c2_at ! similar to ocn-atm mapping, do ice 2 atm mapping / set up #ifdef HAVE_MOAB - ! Call moab intx only if atm and ice are init in moab coupler + ! Call moab intx only if ATM and ICE are init in moab coupler if ((mbaxid .ge. 0) .and. (mbixid .ge. 0)) then appname = "ICE_ATM_COU"//C_NULL_CHAR ! idintx is a unique number of MOAB app that takes care of intx between ice and atm mesh @@ -521,20 +575,7 @@ subroutine prep_atm_init(infodata, ocn_c2_atm, ice_c2_atm, lnd_c2_atm, iac_c2_at wgtIdef = 'scalar'//C_NULL_CHAR mapper_Si2a%weight_identifier = wgtIdef mapper_Si2a%mbname = 'mapper_Si2a' - ! because we will project fields from ocean to atm phys grid, we need to define - ! ice i2x fields to atm phys grid (or atm spectral ext ) on coupler side - if (atm_pg_active) then - tagname = trim(seq_flds_i2x_fields)//C_NULL_CHAR - tagtype = 1 ! dense - numco = 1 ! - ierr = iMOAB_DefineTagStorage(mbaxid, tagname, tagtype, numco, tagindex ) - if (ierr .ne. 0) then - write(logunit,*) subname,' error in defining tags for seq_flds_i2x_fields' - call shr_sys_abort(subname//' ERROR in coin defining tags for seq_flds_i2x_fields') - endif - else ! spectral case, TODO - tagtype = 1 ! dense - endif + volumetric = 0 ! can be 1 only for FV->DGLL or FV->CGLL; @@ -545,7 +586,7 @@ subroutine prep_atm_init(infodata, ocn_c2_atm, ice_c2_atm, lnd_c2_atm, iac_c2_at else dm2 = "cgll"//C_NULL_CHAR dofnameT="GLOBAL_DOFS"//C_NULL_CHAR - orderT = np ! it should be 4 + orderT = 4 ! np ! it should be 4 endif dm1 = "fv"//C_NULL_CHAR dofnameS="GLOBAL_ID"//C_NULL_CHAR @@ -605,14 +646,15 @@ subroutine prep_atm_init(infodata, ocn_c2_atm, ice_c2_atm, lnd_c2_atm, iac_c2_at 'mapper_Fi2a initialization',esmf_map_flag, no_match) #ifdef HAVE_MOAB - ! now take care of the mapper for MOAB + ! now take care of the mapper for MOAB, only if ice is coupled to atm ! + if (ice_c2_atm) then if ( mapper_Fi2a%src_mbid .gt. -1 ) then if (iamroot_CPLID) then write(logunit,F00) 'overwriting '//trim(mapper_Fi2a%mbname) & //' mapper_Fi2a' endif endif - + mapper_Fi2a%src_mbid = mbixid mapper_Fi2a%tgt_mbid = mbaxid mapper_Fi2a%intx_mbid = mbintxia @@ -621,10 +663,26 @@ subroutine prep_atm_init(infodata, ocn_c2_atm, ice_c2_atm, lnd_c2_atm, iac_c2_at wgtIdef = 'scalar'//C_NULL_CHAR mapper_Fi2a%weight_identifier = wgtIdef mapper_Fi2a%mbname = 'mapper_Fi2a' + endif #endif endif ! if (ice_present) then call shr_sys_flush(logunit) + if (mbaxid > 0) then + ! we still need to define seq_flds_l2x_fields on atm cpl mesh + if (atm_pg_active) then + tagname = trim(seq_flds_l2x_fields)//C_NULL_CHAR + tagtype = 1 ! dense + numco = 1 ! + ierr = iMOAB_DefineTagStorage(mbaxid, tagname, tagtype, numco, tagindex ) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in defining tags for seq_flds_l2x_fields' + call shr_sys_abort(subname//' ERROR in coin defining tags for seq_flds_l2x_fields') + endif + else ! spectral case, TODO + tagtype = 1 ! dense + endif + endif ! needed for domain checking if (lnd_present) then if (iamroot_CPLID) then @@ -710,7 +768,7 @@ subroutine prep_atm_init(infodata, ocn_c2_atm, ice_c2_atm, lnd_c2_atm, iac_c2_at else dm2 = "cgll"//C_NULL_CHAR dofnameT="GLOBAL_DOFS"//C_NULL_CHAR - orderT = np ! it should be 4 + orderT = 4 ! np ! it should be 4 endif dm1 = "fv"//C_NULL_CHAR dofnameS="GLOBAL_ID"//C_NULL_CHAR @@ -741,6 +799,7 @@ subroutine prep_atm_init(infodata, ocn_c2_atm, ice_c2_atm, lnd_c2_atm, iac_c2_at ! we do not compute intersection, so we will have to just send data from atm to land and viceversa, by GLOBAL_ID matching ! so we compute just a comm graph, between lnd and atm dofs, on the coupler; target is atm ! land is point cloud in this case, type1 = 2 + call seq_comm_getinfo(CPLID ,mpigrp=mpigrp_CPLID) ! make sure we have the right MPI group type1 = 3; ! full mesh for land now type2 = 3; ! fv for target atm ierr = iMOAB_ComputeCommGraph( mblxid, mbaxid, mpicom_CPLID, mpigrp_CPLID, mpigrp_CPLID, type1, type2, & @@ -754,19 +813,6 @@ subroutine prep_atm_init(infodata, ocn_c2_atm, ice_c2_atm, lnd_c2_atm, iac_c2_at mapper_Fl2a%intx_context = atm(1)%cplcompid endif ! if tri-grid - ! we still need to define seq_flds_l2x_fields on atm cpl mesh - if (atm_pg_active) then - tagname = trim(seq_flds_l2x_fields)//C_NULL_CHAR - tagtype = 1 ! dense - numco = 1 ! - ierr = iMOAB_DefineTagStorage(mbaxid, tagname, tagtype, numco, tagindex ) - if (ierr .ne. 0) then - write(logunit,*) subname,' error in defining tags for seq_flds_l2x_fields' - call shr_sys_abort(subname//' ERROR in coin defining tags for seq_flds_l2x_fields') - endif - else ! spectral case, TODO - tagtype = 1 ! dense - endif endif ! if ((mbaxid .ge. 0) .and. (mblxid .ge. 0) ) then #endif endif ! if lnd_present @@ -1128,6 +1174,18 @@ subroutine prep_atm_mrg_moab(infodata, xao_ax) endif end do +#ifdef HAVE_MOAB + ! just to mark it; we know that l2x_am should be 0, and we should init everything to 0 just because it will affect + l2x_am = 0! + ! allocate(l2x_am (lsize, nlflds)) + tagname = trim(seq_flds_l2x_fields)//C_NULL_CHAR + arrsize = lsize * nlflds + ent_type = 1 ! always atm on coupler is cells + ierr = iMOAB_SetDoubleTagStorage ( mbaxid, tagname, arrsize , ent_type, l2x_am) + if (ierr .ne. 0) then + call shr_sys_abort(subname//' error in setting l2x_am array in init ') + endif +#endif endif ! Zero attribute vector @@ -1345,7 +1403,7 @@ subroutine prep_atm_mrg_moab(infodata, xao_ax) arrsize = naflds * lsize ierr = iMOAB_SetDoubleTagStorage ( mbaxid, tagname, arrsize , ent_type, x2a_am) if (ierr .ne. 0) then - call shr_sys_abort(subname//' error in setting x2o_om array ') + call shr_sys_abort(subname//' error in setting x2a_am array in atm merging ') endif #ifdef MOABCOMP !compare_mct_av_moab_tag(comp, attrVect, field, imoabApp, tag_name, ent_type, difference) diff --git a/driver-moab/main/prep_lnd_mod.F90 b/driver-moab/main/prep_lnd_mod.F90 index f1d2f169ee2d..44ae5490e2c6 100644 --- a/driver-moab/main/prep_lnd_mod.F90 +++ b/driver-moab/main/prep_lnd_mod.F90 @@ -18,10 +18,10 @@ module prep_lnd_mod use seq_comm_mct, only: mbrxid ! iMOAB id of moab rof on coupler pes (FV now) use seq_comm_mct, only: mbintxal ! iMOAB id for intx mesh between atm and lnd use seq_comm_mct, only: mbintxrl ! iMOAB id for intx mesh between river and land - + use seq_comm_mct, only: mbaxid ! iMOAB id for atm migrated mesh to coupler pes use seq_comm_mct, only: atm_pg_active ! whether the atm uses FV mesh or not ; made true if fv_nphys > 0 - use dimensions_mod, only: np ! for atmosphere + ! use dimensions_mod, only: np ! for atmosphere use seq_comm_mct, only: seq_comm_getinfo => seq_comm_setptrs use seq_map_type_mod use seq_map_mod @@ -35,7 +35,7 @@ module prep_lnd_mod use iso_c_binding #ifdef HAVE_MOAB use iMOAB , only: iMOAB_ComputeCommGraph, iMOAB_ComputeMeshIntersectionOnSphere, & - iMOAB_ComputeScalarProjectionWeights, iMOAB_DefineTagStorage, iMOAB_RegisterApplication, & + iMOAB_ComputeScalarProjectionWeights, iMOAB_DefineTagStorage, iMOAB_RegisterApplication, & iMOAB_WriteMesh, iMOAB_GetMeshInfo, iMOAB_SetDoubleTagStorage use seq_comm_mct, only : num_moab_exports #endif @@ -143,8 +143,8 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln character(CL) :: rof_gnam ! rof grid character(CL) :: glc_gnam ! glc grid type(mct_avect), pointer :: l2x_lx -#ifdef HAVE_MOAB - ! MOAB stuff +#ifdef HAVE_MOAB + ! MOAB stuff integer :: ierr, idintx, rank character*32 :: appname, outfile, wopts, lnum character*32 :: dm1, dm2, dofnameS, dofnameT, wgtIdef @@ -236,9 +236,9 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln call shr_sys_abort(subname//' ERROR in registering rof lnd intx') endif if (samegrid_lr)then -! the same mesh , lnd and rof use the same dofs, but restricted +! the same mesh , lnd and rof use the same dofs, but restricted ! we do not compute intersection, so we will have to just send data from lnd to rof and viceversa, by GLOBAL_ID matching - ! so we compute just a comm graph, between lnd and rof dofs, on the coupler; target is rof + ! so we compute just a comm graph, between lnd and rof dofs, on the coupler; target is rof ! land is full mesh call seq_comm_getData(CPLID ,mpigrp=mpigrp_CPLID) type1 = 3; ! full mesh for lrofarnd now @@ -270,10 +270,10 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln if (iamroot_CPLID) then write(logunit,*) 'iMOAB intersection between rof and lnd with id:', idintx end if - ! we also need to compute the comm graph for the second hop, from the rof on coupler to the + ! we also need to compute the comm graph for the second hop, from the rof on coupler to the ! rof for the intx rof-lnd context (coverage) - ! - call seq_comm_getData(CPLID ,mpigrp=mpigrp_CPLID) + ! + call seq_comm_getData(CPLID ,mpigrp=mpigrp_CPLID) type1 = 3 ! land is FV now on coupler side type2 = 3; @@ -283,7 +283,7 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln write(logunit,*) subname,' error in computing comm graph for second hop, lnd-rof' call shr_sys_abort(subname//' ERROR in computing comm graph for second hop, lnd-rof') endif - ! now take care of the mapper + ! now take care of the mapper if ( mapper_Fr2l%src_mbid .gt. -1 ) then if (iamroot_CPLID) then write(logunit,F00) 'overwriting '//trim(mapper_Fr2l%mbname) & @@ -292,22 +292,22 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln endif mapper_Fr2l%src_mbid = mbrxid mapper_Fr2l%tgt_mbid = mblxid - mapper_Fr2l%intx_mbid = mbintxrl + mapper_Fr2l%intx_mbid = mbintxrl mapper_Fr2l%src_context = rof(1)%cplcompid mapper_Fr2l%intx_context = idintx wgtIdef = 'scalar'//C_NULL_CHAR mapper_Fr2l%weight_identifier = wgtIdef mapper_Fr2l%mbname = 'mapper_Fr2l' - ! because we will project fields from rof to lnd grid, we need to define + ! because we will project fields from rof to lnd grid, we need to define ! the r2x fields to lnd grid on coupler side - volumetric = 0 ! can be 1 only for FV->DGLL or FV->CGLL; - + volumetric = 0 ! can be 1 only for FV->DGLL or FV->CGLL; + dm1 = "fv"//C_NULL_CHAR dofnameS="GLOBAL_ID"//C_NULL_CHAR orderS = 1 ! fv-fv - + dm2 = "fv"//C_NULL_CHAR dofnameT="GLOBAL_ID"//C_NULL_CHAR orderT = 1 ! not much arguing @@ -349,13 +349,13 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln endif tagname = trim(seq_flds_r2x_fields)//C_NULL_CHAR tagtype = 1 ! dense - numco = 1 ! + numco = 1 ! ierr = iMOAB_DefineTagStorage(mblxid, tagname, tagtype, numco, tagindex ) if (ierr .ne. 0) then write(logunit,*) subname,' error in defining tags for seq_flds_r2x_fields on lnd cpl' call shr_sys_abort(subname//' ERROR in defining tags for seq_flds_r2x_fields on lnd cpl') endif - + ! find out the number of local elements in moab mesh land instance on coupler ierr = iMOAB_GetMeshInfo ( mblxid, nvert, nvise, nbl, nsurf, nvisBC ) if (ierr .ne. 0) then @@ -364,12 +364,12 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln endif ! land is now cell mesh on coupler side mlsize = nvise(1) - ent_type = 1 ! cell + ent_type = 1 ! cell ! set to 0 all fields that are projected from river nrflds = mct_aVect_nRattr(r2x_lx(1)) ! these are the numbers of fields in seq_flds_r2x_fields arrsize = nrflds*mlsize allocate (tmparray(arrsize)) ! mlsize is the size of local land - ! do we need to zero out others or just river ? + ! do we need to zero out others or just river ? tmparray = 0._r8 ierr = iMOAB_SetDoubleTagStorage(mblxid, tagname, arrsize , ent_type, tmparray) if (ierr .ne. 0) then @@ -379,7 +379,7 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln deallocate (tmparray) end if ! if ((mbrxid .ge. 0) .and. (mblxid .ge. 0)) -! endif HAVE_MOAB +! endif HAVE_MOAB #endif call shr_sys_flush(logunit) @@ -398,11 +398,11 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln call seq_map_init_rcfile(mapper_Fa2l, atm(1), lnd(1), & 'seq_maps.rc','atm2lnd_fmapname:','atm2lnd_fmaptype:',samegrid_al, & 'mapper_Fa2l initialization',esmf_map_flag) -! similar to prep_atm_init, lnd and atm reversed +! similar to prep_atm_init, lnd and atm reversed #ifdef HAVE_MOAB - ! important change: do not compute intx at all between atm and land when we have samegrid_al + ! important change: do not compute intx at all between atm and land when we have samegrid_al ! we will use just a comm graph to send data from atm to land on coupler - ! this is just a rearrange in a way + ! this is just a rearrange in a way if ((mbaxid .ge. 0) .and. (mblxid .ge. 0) ) then appname = "ATM_LND_COU"//C_NULL_CHAR ! idintx is a unique number of MOAB app that takes care of intx between lnd and atm mesh @@ -418,16 +418,18 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln //' mapper_Sa2l' endif endif + + ! set up the scalar mapper context mapper_Sa2l%src_mbid = mbaxid mapper_Sa2l%tgt_mbid = mblxid mapper_Sa2l%intx_mbid = mbintxal mapper_Sa2l%src_context = atm(1)%cplcompid mapper_Sa2l%intx_context = idintx wgtIdef = 'scalar'//C_NULL_CHAR - mapper_Sa2l%weight_identifier = wgtIdef + mapper_Sa2l%weight_identifier = wgtIdef mapper_Sa2l%mbname = 'mapper_Sa2l' - - call seq_comm_getinfo(CPLID ,mpigrp=mpigrp_CPLID) + + call seq_comm_getinfo(CPLID ,mpigrp=mpigrp_CPLID) if (.not. samegrid_al) then ! tri grid case if (iamroot_CPLID) then write(logunit,*) 'iMOAB intersection between atm and land with id:', idintx @@ -451,14 +453,14 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln endif endif #endif - ! we also need to compute the comm graph for the second hop, from the atm on coupler to the + ! we also need to compute the comm graph for the second hop, from the atm on coupler to the ! lnd for the intx atm-lnd context (coverage) - ! + ! if (atm_pg_active) then type1 = 3; ! fv for atm; cgll does not work anyway else type1 = 1 ! this projection works (cgll to fv), but reverse does not ( fv - cgll) - endif + endif type2 = 3; ! land is fv in this case (separate grid) ierr = iMOAB_ComputeCommGraph( mbaxid, mbintxal, mpicom_CPLID, mpigrp_CPLID, mpigrp_CPLID, type1, type2, & @@ -467,9 +469,9 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln write(logunit,*) subname,' error in computing comm graph for second hop, atm-lnd' call shr_sys_abort(subname//' ERROR in computing comm graph for second hop, atm-lnd') endif - - volumetric = 0 ! can be 1 only for FV->DGLL or FV->CGLL; - + + volumetric = 0 ! can be 1 only for FV->DGLL or FV->CGLL; + if (atm_pg_active) then dm1 = "fv"//C_NULL_CHAR dofnameS="GLOBAL_ID"//C_NULL_CHAR @@ -477,7 +479,7 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln else dm1 = "cgll"//C_NULL_CHAR dofnameS="GLOBAL_DOFS"//C_NULL_CHAR - orderS = np ! it should be 4 + orderS = 4 ! np ! it should be 4 endif dm2 = "fv"//C_NULL_CHAR dofnameT="GLOBAL_ID"//C_NULL_CHAR @@ -504,16 +506,16 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln call shr_sys_abort(subname//' ERROR in computing weights for atm-lnd ') endif - else ! the same mesh , atm and lnd use the same dofs, but lnd is a subset of atm + else ! the same mesh , atm and lnd use the same dofs, but lnd is a subset of atm ! we do not compute intersection, so we will have to just send data from atm to land and viceversa, by GLOBAL_ID matching - ! so we compute just a comm graph, between atm and lnd dofs, on the coupler; target is lnd + ! so we compute just a comm graph, between atm and lnd dofs, on the coupler; target is lnd ! land is point cloud in this case, type1 = 2 - + if (atm_pg_active) then type1 = 3; ! fv for atm; cgll does not work anyway else type1 = 1 ! this projection works (cgll to fv), but reverse does not ( fv - cgll) - endif + endif type2 = 3; ! FV mesh on coupler land ierr = iMOAB_ComputeCommGraph( mbaxid, mblxid, mpicom_CPLID, mpigrp_CPLID, mpigrp_CPLID, type1, type2, & atm(1)%cplcompid, lnd(1)%cplcompid) @@ -521,7 +523,6 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln write(logunit,*) subname,' error in computing comm graph for second hop, atm-lnd' call shr_sys_abort(subname//' ERROR in computing comm graph for second hop, atm-lnd') endif - mapper_Sa2l%tgt_mbid = mblxid mapper_Sa2l%intx_context = lnd(1)%cplcompid endif ! if tri-grid @@ -534,15 +535,15 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln endif endif mapper_Fa2l%src_mbid = mbaxid - mapper_Fa2l%tgt_mbid = mapper_Sa2l%tgt_mbid ! mblxid + mapper_Fa2l%tgt_mbid = mblxid mapper_Fa2l%intx_mbid = mbintxal mapper_Fa2l%src_context = atm(1)%cplcompid mapper_Fa2l%intx_context = mapper_Sa2l%intx_context wgtIdef = 'scalar'//C_NULL_CHAR - mapper_Fa2l%weight_identifier = wgtIdef + mapper_Fa2l%weight_identifier = wgtIdef mapper_Fa2l%mbname = 'mapper_Fa2l' - + ! in any case, we need to define the tags on landx from the phys atm seq_flds_a2x_fields tagtype = 1 ! dense, double numco = 1 ! one value per vertex / entity @@ -551,11 +552,11 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln if ( ierr > 0) then call shr_sys_abort(subname//' fail to define seq_flds_a2x_fields for lnd x moab mesh ') endif - - endif ! if ((mbaxid .ge. 0) .and. (mblxid .ge. 0) ) then - - -#endif + + endif ! if ((mbaxid .ge. 0) .and. (mblxid .ge. 0) ) then + + +#endif endif call shr_sys_flush(logunit) @@ -658,12 +659,12 @@ subroutine prep_lnd_mrg(infodata, timer_mrg) end subroutine prep_lnd_mrg -! this does almost nothing now, except documenting +! this does almost nothing now, except documenting subroutine prep_lnd_mrg_moab (infodata) type(seq_infodata_type) , intent(in) :: infodata - type(mct_avect) , pointer :: a2x_l ! used just for indexing + type(mct_avect) , pointer :: a2x_l ! used just for indexing type(mct_avect) , pointer :: r2x_l type(mct_avect) , pointer :: g2x_l type(mct_avect) , pointer :: x2l_l @@ -783,7 +784,7 @@ subroutine prep_lnd_mrg_moab (infodata) #endif - end subroutine prep_lnd_mrg_moab + end subroutine prep_lnd_mrg_moab !================================================================================================ subroutine prep_lnd_merge( a2x_l, r2x_l, g2x_l, x2l_l ) diff --git a/driver-moab/main/prep_ocn_mod.F90 b/driver-moab/main/prep_ocn_mod.F90 index de4a89e235f5..7514cc40db01 100644 --- a/driver-moab/main/prep_ocn_mod.F90 +++ b/driver-moab/main/prep_ocn_mod.F90 @@ -12,7 +12,6 @@ module prep_ocn_mod use seq_comm_mct, only: CPLID, OCNID, logunit use seq_comm_mct, only: seq_comm_getData=>seq_comm_setptrs - use seq_comm_mct, only: mpoid ! iMOAB pid for ocean mesh on component pes use seq_comm_mct, only: mboxid ! iMOAB id for mpas ocean migrated mesh to coupler pes use seq_comm_mct, only: mbrmapro ! iMOAB id for map read from rof2ocn map file use seq_comm_mct, only: mbrxoid ! iMOAB id for rof on coupler in ocean context; @@ -23,7 +22,7 @@ module prep_ocn_mod use seq_comm_mct, only : mbintxoa ! iMOAB id for intx mesh between ocean and atmosphere use seq_comm_mct, only : mhid ! iMOAB id for atm instance use seq_comm_mct, only : mhpgid ! iMOAB id for atm pgx grid, on atm pes; created with se and gll grids - use dimensions_mod, only : np ! for atmosphere degree + ! use dimensions_mod, only : np ! for atmosphere degree use seq_comm_mct, only : mbixid ! iMOAB for sea-ice migrated to coupler use seq_comm_mct, only : CPLALLICEID use seq_comm_mct, only : seq_comm_iamin @@ -80,7 +79,7 @@ module prep_ocn_mod public :: prep_ocn_get_x2oacc_ox public :: prep_ocn_get_x2oacc_ox_cnt - public :: prep_ocn_get_x2oacc_om ! will return a pointer to the local private matrix + public :: prep_ocn_get_x2oacc_om ! will return a pointer to the local private matrix public :: prep_ocn_get_x2oacc_om_cnt #ifdef SUMMITDEV_PGI ! Sarat: Dummy variable added to workaround PGI compiler bug (PGI 17.9) as of Oct 23, 2017 @@ -264,7 +263,7 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc integer :: rank_on_cpl ! just for debugging ! these are just to zero out r2x fields on ocean integer nvert(3), nvise(3), nbl(3), nsurf(3), nvisBC(3) ! for moab info - integer mlsize ! moab local ocean size + integer mlsize ! moab local ocean size integer nrflds ! number of rof fields projected on land integer arrsize ! for setting the r2x fields on land to 0 integer ent_type ! for setting tags @@ -366,8 +365,8 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc ! during "first_time" entering merge routine; this was wrong ! allocate accumulation variable , parallel to x2o_om noflds = mct_aVect_nRattr(x2o_ox) ! these are saved after first time - ! size of the x2oacc_om depends on the size of the ocean mesh locally - + ! size of the x2oacc_om depends on the size of the ocean mesh locally + ! find out the number of local elements in moab mesh ocean instance on coupler ierr = iMOAB_GetMeshInfo ( mboxid, nvert, nvise, nbl, nsurf, nvisBC ) if (ierr .ne. 0) then @@ -376,11 +375,11 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc endif ! ocn is cell mesh on coupler side mlsize = nvise(1) - allocate(x2oacc_om(mlsize, noflds)) + allocate(x2oacc_om(mlsize, noflds)) x2oacc_om_cnt = 0 x2oacc_om(:,:)=0. - ! moab accumulation variable + ! moab accumulation variable samegrid_ao = .true. samegrid_ro = .true. @@ -411,145 +410,165 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc write(logunit,*) subname,' error in registering atm ocn intx' call shr_sys_abort(subname//' ERROR in registering atm ocn intx') endif - ierr = iMOAB_ComputeMeshIntersectionOnSphere (mbaxid, mboxid, mbintxao) - if (ierr .ne. 0) then - write(logunit,*) subname,' error in computing atm ocn intx' - call shr_sys_abort(subname//' ERROR in computing atm ocn intx') - endif - if (iamroot_CPLID) then - write(logunit,*) 'iMOAB intersection between atm and ocean with id:', idintx - end if - - ! we also need to compute the comm graph for the second hop, from the atm on coupler to the - ! atm for the intx atm-ocn context (coverage) - ! - call seq_comm_getinfo(CPLID ,mpigrp=mpigrp_CPLID) - if (atm_pg_active) then - type1 = 3; ! fv for both ocean and atm; fv-cgll does not work anyway - else - type1 = 1 ! this works in this direction, but it will not be used - endif - type2 = 3; - ! ierr = iMOAB_ComputeCommGraph( mboxid, mbintxoa, &mpicom_CPLID, &mpigrp_CPLID, &mpigrp_CPLID, &type1, &type2, - ! &ocn_id, &idintx) - ierr = iMOAB_ComputeCommGraph( mbaxid, mbintxao, mpicom_CPLID, mpigrp_CPLID, mpigrp_CPLID, type1, type2, & - atm(1)%cplcompid, idintx) - if (ierr .ne. 0) then - write(logunit,*) subname,' error in computing comm graph for second hop, atm-ocn' - call shr_sys_abort(subname//' ERROR in computing comm graph for second hop, atm-ocn') - endif - ! now take care of the mapper - if ( mapper_Fa2o%src_mbid .gt. -1 ) then - if (iamroot_CPLID) then - write(logunit,F00) 'overwriting '//trim(mapper_Fa2o%mbname) & - //' mapper_Fa2o' - endif - endif mapper_Fa2o%src_mbid = mbaxid mapper_Fa2o%tgt_mbid = mboxid mapper_Fa2o%intx_mbid = mbintxao mapper_Fa2o%src_context = atm(1)%cplcompid - mapper_Fa2o%intx_context = idintx wgtIdef = 'scalar'//C_NULL_CHAR mapper_Fa2o%weight_identifier = wgtIdef mapper_Fa2o%mbname = 'mapper_Fa2o' - ! because we will project fields from atm to ocn grid, we need to define - ! atm a2x fields to ocn grid on coupler side - tagname = trim(seq_flds_a2x_fields)//C_NULL_CHAR - tagtype = 1 ! dense - numco = 1 ! - ierr = iMOAB_DefineTagStorage(mboxid, tagname, tagtype, numco, tagindex ) - if (ierr .ne. 0) then - write(logunit,*) subname,' error in defining tags for seq_flds_a2x_fields on ocn cpl' - call shr_sys_abort(subname//' ERROR in coin defining tags for seq_flds_a2x_fields on ocn cpl') - endif - volumetric = 0 ! can be 1 only for FV->DGLL or FV->CGLL; - - if (atm_pg_active) then - dm1 = "fv"//C_NULL_CHAR - dofnameS="GLOBAL_ID"//C_NULL_CHAR - orderS = 1 ! fv-fv - else - dm1 = "cgll"//C_NULL_CHAR - dofnameS="GLOBAL_DOFS"//C_NULL_CHAR - orderS = np ! it should be 4 - endif - dm2 = "fv"//C_NULL_CHAR - dofnameT="GLOBAL_ID"//C_NULL_CHAR - orderT = 1 ! not much arguing - fNoBubble = 1 - monotonicity = 0 ! - noConserve = 0 - validate = 0 ! less verbose - fInverseDistanceMap = 0 - - ! First compute the non-conservative bilinear map for projection of scalar fields - if (iamroot_CPLID) then - call print_weight_map_details(subname, mbintxao, "FV-FV", "bilinear", & - trim(dm1), orderS, trim(dofnameS), trim(dm2), orderT, trim(dofnameT), "bilinear", & - fNoBubble, monotonicity, volumetric, fInverseDistanceMap, noConserve, validate) - ! write(logunit,*) subname, 'launch iMOAB weights with args ', 'mbintxao=', mbintxao, ' wgtIdef=bilinear ', & - ! 'dm1=', trim(dm1), ' orderS=', orderS, 'dm2=', trim(dm2), ' orderT=', orderT, 'fvMethod=bilinear', & - ! fNoBubble, monotonicity, volumetric, fInverseDistanceMap, & - ! noConserve, validate, & - ! trim(dofnameS), trim(dofnameT) - endif - ierr = iMOAB_ComputeScalarProjectionWeights ( mbintxao, 'bilinear'//C_NULL_CHAR, & - trim(dm1), orderS, trim(dm2), orderT, 'bilin'//C_NULL_CHAR, & - fNoBubble, monotonicity, volumetric, fInverseDistanceMap, & - noConserve, validate, & - trim(dofnameS), trim(dofnameT) ) - if (ierr .ne. 0) then - write(logunit,*) subname,' error in computing ao weights ' - call shr_sys_abort(subname//' ERROR in computing ao weights ') - endif + ! we also need to compute the comm graph for the second hop, from the atm on coupler to the + ! atm for the intx atm-ocn context (coverage) + call seq_comm_getinfo(CPLID, mpigrp=mpigrp_CPLID) - ierr = iMOAB_WriteMappingWeightsToFile(mbintxao, 'bilinear'//C_NULL_CHAR, 'bilinear_a2o.nc'//C_NULL_CHAR) + ! next, let us compute the ATM and OCN data transfer + if (.not. samegrid_ao) then ! not a data OCN model - ! Next compute the conservative map for projection of flux fields - if (iamroot_CPLID) then - call print_weight_map_details(subname, mbintxao, "FV-FV", wgtIdef, & - trim(dm1), orderS, trim(dofnameS), trim(dm2), orderT, trim(dofnameT), "bilinear", & - fNoBubble, monotonicity, volumetric, fInverseDistanceMap, noConserve, validate) + ! first compute the overlap mesh between mbaxid (ATM) and mboxid (OCN) on coupler PEs + ierr = iMOAB_ComputeMeshIntersectionOnSphere (mbaxid, mboxid, mbintxao) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in computing ATM-OCN intersection' + call shr_sys_abort(subname//' ERROR in computing ATM-OCN intersection') + endif + if (iamroot_CPLID) then + write(logunit,*) 'iMOAB intersection completed between atm and ocean with id:', idintx + end if + + if (atm_pg_active) then + type1 = 3; ! FV for ATM; CGLL does not work correctly in parallel at the moment + else + type1 = 1 ! This projection works (CGLL to FV), but reverse does not (FV - CGLL) + endif + type2 = 3; ! FV mesh on coupler OCN + ! ierr = iMOAB_ComputeCommGraph( mboxid, mbintxoa, &mpicom_CPLID, &mpigrp_CPLID, &mpigrp_CPLID, &type1, &type2, + ! &ocn_id, &idintx) + ierr = iMOAB_ComputeCommGraph( mbaxid, mbintxao, mpicom_CPLID, mpigrp_CPLID, mpigrp_CPLID, type1, type2, & + atm(1)%cplcompid, idintx) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in computing comm graph for second hop, atm-ocn' + call shr_sys_abort(subname//' ERROR in computing comm graph for second hop, atm-ocn') + endif + ! now take care of the mapper + if ( mapper_Fa2o%src_mbid .gt. -1 ) then + if (iamroot_CPLID) then + write(logunit,F00) 'overwriting '//trim(mapper_Fa2o%mbname) & + //' mapper_Fa2o' + endif + endif - ! write(logunit,*) subname, 'launch iMOAB weights with args ', 'mbintxao=', mbintxao, ' wgtIdef=', wgtIdef, & - ! 'dm1=', trim(dm1), ' orderS=', orderS, 'dm2=', trim(dm2), ' orderT=', orderT, & - ! fNoBubble, monotonicity, volumetric, fInverseDistanceMap, & - ! noConserve, validate, & - ! trim(dofnameS), trim(dofnameT) - endif - ierr = iMOAB_ComputeScalarProjectionWeights ( mbintxao, wgtIdef, & - trim(dm1), orderS, trim(dm2), orderT, ''//C_NULL_CHAR, & - fNoBubble, monotonicity, volumetric, fInverseDistanceMap, & - noConserve, validate, & - trim(dofnameS), trim(dofnameT) ) - if (ierr .ne. 0) then - write(logunit,*) subname,' error in computing ao weights ' - call shr_sys_abort(subname//' ERROR in computing ao weights ') - endif + ! To project fields from ATM to OCN grid, we need to define + ! ATM a2x fields to OCN grid on coupler side + tagname = trim(seq_flds_a2x_fields)//C_NULL_CHAR + tagtype = 1 ! dense + numco = 1 ! + ierr = iMOAB_DefineTagStorage(mboxid, tagname, tagtype, numco, tagindex ) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in defining tags for seq_flds_a2x_fields on ocn cpl' + call shr_sys_abort(subname//' ERROR in coin defining tags for seq_flds_a2x_fields on ocn cpl') + endif + volumetric = 0 ! can be 1 only for FV->DGLL or FV->CGLL; + + if (atm_pg_active) then + dm1 = "fv"//C_NULL_CHAR + dofnameS="GLOBAL_ID"//C_NULL_CHAR + orderS = 1 ! fv-fv + else + dm1 = "cgll"//C_NULL_CHAR + dofnameS="GLOBAL_DOFS"//C_NULL_CHAR + orderS = 4 ! np ! it should be 4 + endif + dm2 = "fv"//C_NULL_CHAR + dofnameT="GLOBAL_ID"//C_NULL_CHAR + orderT = 1 ! not much arguing + fNoBubble = 1 + monotonicity = 0 ! + noConserve = 0 + validate = 0 ! less verbose + fInverseDistanceMap = 0 + + ! First compute the non-conservative bilinear map for projection of scalar fields + if (iamroot_CPLID) then + call print_weight_map_details(subname, mbintxao, "FV-FV", "bilinear", & + trim(dm1), orderS, trim(dofnameS), trim(dm2), orderT, trim(dofnameT), "bilinear", & + fNoBubble, monotonicity, volumetric, fInverseDistanceMap, noConserve, validate) + endif + ierr = iMOAB_ComputeScalarProjectionWeights ( mbintxao, 'bilinear'//C_NULL_CHAR, & + trim(dm1), orderS, trim(dm2), orderT, 'bilin'//C_NULL_CHAR, & + fNoBubble, monotonicity, volumetric, fInverseDistanceMap, & + noConserve, validate, & + trim(dofnameS), trim(dofnameT) ) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in computing ao weights ' + call shr_sys_abort(subname//' ERROR in computing ao weights ') + endif + + ! ierr = iMOAB_WriteMappingWeightsToFile(mbintxao, 'bilinear'//C_NULL_CHAR, 'bilinear_a2o.nc'//C_NULL_CHAR) + + ! Next compute the conservative map for projection of flux fields + if (iamroot_CPLID) then + call print_weight_map_details(subname, mbintxao, "FV-FV", wgtIdef, & + trim(dm1), orderS, trim(dofnameS), trim(dm2), orderT, trim(dofnameT), "", & + fNoBubble, monotonicity, volumetric, fInverseDistanceMap, noConserve, validate) + endif + ierr = iMOAB_ComputeScalarProjectionWeights ( mbintxao, wgtIdef, & + trim(dm1), orderS, trim(dm2), orderT, ''//C_NULL_CHAR, & + fNoBubble, monotonicity, volumetric, fInverseDistanceMap, & + noConserve, validate, & + trim(dofnameS), trim(dofnameT) ) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in computing ao weights ' + call shr_sys_abort(subname//' ERROR in computing ao weights ') + endif + + mapper_Fa2o%intx_context = idintx + !! All done for mapper_Fa2o -- #ifdef MOABDEBUG - wopts = C_NULL_CHAR - call shr_mpi_commrank( mpicom_CPLID, rank ) - if (rank .lt. 5) then - write(lnum,"(I0.2)")rank ! - outfile = 'intx_ao_'//trim(lnum)// '.h5m' // C_NULL_CHAR - ierr = iMOAB_WriteMesh(mbintxao, outfile, wopts) ! write local intx file + wopts = C_NULL_CHAR + call shr_mpi_commrank( mpicom_CPLID, rank ) + if (rank .lt. 5) then + write(lnum,"(I0.2)")rank ! + outfile = 'intx_ao_'//trim(lnum)// '.h5m' // C_NULL_CHAR + ierr = iMOAB_WriteMesh(mbintxao, outfile, wopts) ! write local intx file + if (ierr .ne. 0) then + write(logunit,*) subname,' error in writing intx file ' + call shr_sys_abort(subname//' ERROR in writing intx file ') + endif + endif +#endif + else ! if (samegrid_ao) + + ! ATM and OCN components use the same mesh and DoF numbering (OCN is a subset of ATM); + ! We do not need to compute intersection since the "map" from ATM to OCN is essentially a + ! permutation operator and can be achieved by simply sending/receiving data from ATM to OCN + ! and viceversa, based on element GLOBAL_ID matching. In order to seamless produce the + ! permutation operator, we will compute a communication graph between ATM and OCN DoFs on the + ! coupler. + if (atm_pg_active) then + type1 = 3; ! FV for ATM; CGLL does not work correctly in parallel at the moment + else + type1 = 1 ! This projection works (CGLL to FV), but reverse does not (FV - CGLL) + endif + type2 = 3; ! FV mesh on coupler OCN + ierr = iMOAB_ComputeCommGraph( mbaxid, mboxid, mpicom_CPLID, mpigrp_CPLID, mpigrp_CPLID, type1, type2, & + atm(1)%cplcompid, ocn(1)%cplcompid ) if (ierr .ne. 0) then - write(logunit,*) subname,' error in writing intx file ' - call shr_sys_abort(subname//' ERROR in writing intx file ') + write(logunit,*) subname,' error in computing communication graph for second hop, ATM-OCN' + call shr_sys_abort(subname//' ERROR in computing communication graph for second hop, ATM-OCN') endif - endif -#endif - end if ! if ((mbaxid .ge. 0) .and. (mboxid .ge. 0)) + mapper_Fa2o%intx_context = ocn(1)%cplcompid + + endif ! if (.not. samegrid_ao) + + endif ! if ((mbaxid .ge. 0) .and. (mboxid .ge. 0)) ! endif HAVE_MOAB #endif end if ! if (atm_present) - ! atm_c2_ice flag is here because ice and ocn are constrained to be on the same - ! grid so the atm->ice mapping is set to the atm->ocn mapping to improve performance + ! atm_c2_ice flag is here because ICE and OCN are constrained to be on the same + ! grid so the ATM->ICE mapping is set to the atm->ocn mapping to improve performance if (atm_c2_ocn .or. atm_c2_ice) then if (iamroot_CPLID) then write(logunit,*) ' ' @@ -575,6 +594,7 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc ! will use the same map for mapper_Sa2o and Va2o, using the bilinear map option if ((mbaxid .ge. 0) .and. (mboxid .ge. 0)) then + ! now take care of the 2 new mappers if ( mapper_Sa2o%src_mbid .gt. -1 ) then if (iamroot_CPLID) then @@ -586,7 +606,7 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc mapper_Sa2o%tgt_mbid = mboxid mapper_Sa2o%intx_mbid = mbintxao mapper_Sa2o%src_context = atm(1)%cplcompid - mapper_Sa2o%intx_context = idintx + mapper_Sa2o%intx_context = mapper_Fa2o%intx_context mapper_Sa2o%weight_identifier = 'bilinear'//C_NULL_CHAR mapper_Sa2o%mbname = 'mapper_Sa2o' @@ -600,9 +620,10 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc mapper_Va2o%tgt_mbid = mboxid mapper_Va2o%intx_mbid = mbintxao mapper_Va2o%src_context = atm(1)%cplcompid - mapper_Va2o%intx_context = idintx + mapper_Va2o%intx_context = mapper_Fa2o%intx_context mapper_Va2o%weight_identifier = 'bilinear'//C_NULL_CHAR mapper_Va2o%mbname = 'mapper_Va2o' + endif ! if ((mbaxid .ge. 0) .and. (mboxid .ge. 0)) endif ! if (atm_c2_ocn .or. atm_c2_ice) call shr_sys_flush(logunit) @@ -621,9 +642,9 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc call seq_comm_getinfo(CPLID ,mpigrp=mpigrp_CPLID) ! second group, the coupler group CPLID is global variable type1 = 3 - type2 = 3 ! fv-fv graph + type2 = 3 ! FV-FV graph - ! iMOAB: compute the communication graph for ice-ocn, based on the same global id + ! iMOAB: compute the communication graph for ICE-OCN, based on the same global id ! it will be a simple permutation from ice mesh directly to ocean, using the comm graph computed here ierr = iMOAB_ComputeCommGraph( mbixid, mboxid, mpicom_CPLID, mpigrp_CPLID, mpigrp_CPLID, & type1, type2, ice(1)%cplcompid, ocn(1)%cplcompid) @@ -632,8 +653,7 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc call shr_sys_abort(subname//' ERROR in computing graph ice - ocn x ') endif - - ! define tags according to the seq_flds_i2x_fields + ! define tags according to the seq_flds_i2x_fields tagtype = 1 ! dense, double numco = 1 ! one value per cell / entity tagname = trim(seq_flds_i2x_fields)//C_NULL_CHAR @@ -751,7 +771,7 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc if (iamroot_CPLID) then write(logunit,*) subname,' created moab tags for seq_flds_r2x_fields ' endif - + ! find out the number of local elements in moab mesh ocean instance on coupler ierr = iMOAB_GetMeshInfo ( mboxid, nvert, nvise, nbl, nsurf, nvisBC ) if (ierr .ne. 0) then @@ -961,7 +981,7 @@ subroutine prep_ocn_accum_moab() character(*) , parameter :: subname = '(prep_ocn_accum_moab)' !--------------------------------------------------------------- - ! this method is called after merge, so it is not really necessary, because + ! this method is called after merge, so it is not really necessary, because ! x2o_om should be saved between these calls tagname = trim(seq_flds_x2o_fields)//C_NULL_CHAR ent_type = 1 ! cell type @@ -969,7 +989,7 @@ subroutine prep_ocn_accum_moab() if (ierr .ne. 0) then call shr_sys_abort(subname//' error in getting x2o_om array ') endif - + if (x2oacc_om_cnt == 0) then x2oacc_om = x2o_om @@ -1050,7 +1070,7 @@ subroutine prep_ocn_accum_avg_moab() if (mboxid .ge. 0 ) then ! we are on coupler pes, for sure write(lnum,"(I0.2)")num_moab_exports outfile = 'OcnCplAftAvg'//trim(lnum)//'.h5m'//C_NULL_CHAR - wopts = ';PARALLEL=WRITE_PART'//C_NULL_CHAR + wopts = ';PARALLEL=WRITE_PART'//C_NULL_CHAR ierr = iMOAB_WriteMesh(mboxid, trim(outfile), trim(wopts)) endif #endif @@ -1699,10 +1719,10 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) ierr = iMOAB_GetDoubleTagStorage ( mboxid, tagname, arrsize , ent_type, x2o_om) if (ierr .ne. 0) then call shr_sys_abort(subname//' error in getting x2o_om array ') - endif + endif ! zero out the output first (see line 1358) !x2o_om(:,:)=0. - ! no, we should zero out only some indices, that accumulate + ! no, we should zero out only some indices, that accumulate do ko = 1, noflds if ( (aindx(ko) .gt. 0 ) .and. amerge(ko) ) then x2o_om(:, ko) = 0. @@ -1981,25 +2001,25 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) enddo write(logunit, *) ' Atm fields projected on coupler' do ka = 1,naflds - write(logunit, *) trim(field_atm(ka)) + write(logunit, *) trim(field_atm(ka)) enddo write(logunit, *) ' Ice fields projected on coupler' do ki = 1,niflds - write(logunit, *) trim(field_ice(ki)) + write(logunit, *) trim(field_ice(ki)) enddo write(logunit, *) ' Runoff fields projected on coupler' do kr = 1,nrflds - write(logunit, *) trim(field_rof(kr)) + write(logunit, *) trim(field_rof(kr)) enddo write(logunit, *) ' xao flux fields ' do kx = 1,nxflds - write(logunit, *) trim(field_xao(kx)) + write(logunit, *) trim(field_xao(kx)) enddo #endif endif deallocate(mrgstr) - + deallocate(field_atm,itemc_atm) deallocate(field_ocn,itemc_ocn) deallocate(field_ice,itemc_ice) diff --git a/driver-moab/main/prep_rof_mod.F90 b/driver-moab/main/prep_rof_mod.F90 index a8cce9455a44..59cc61c58890 100644 --- a/driver-moab/main/prep_rof_mod.F90 +++ b/driver-moab/main/prep_rof_mod.F90 @@ -15,7 +15,7 @@ module prep_rof_mod use seq_comm_mct, only: mboxid use seq_comm_mct, only: mbintxlr ! iMOAB id for intx mesh between land and river use seq_comm_mct, only : atm_pg_active ! whether the atm uses FV mesh or not ; made true if fv_nphys > 0 - use dimensions_mod, only : np ! for atmosphere degree + !use dimensions_mod, only : np ! for atmosphere degree use seq_comm_mct, only: seq_comm_getData=>seq_comm_setptrs use seq_infodata_mod, only: seq_infodata_type, seq_infodata_getdata use shr_log_mod , only: errMsg => shr_log_errMsg @@ -264,8 +264,9 @@ subroutine prep_rof_init(infodata, lnd_c2_rof, atm_c2_rof, ocn_c2_rof) l2racc_lx_cnt = 0 #ifdef HAVE_MOAB ! this l2racc_lm will be over land size ? - sharedFieldsLndRof=trim( mct_aVect_exportRList2c(l2racc_lx(1)) ) + sharedFieldsLndRof='' nfields_sh_lr = mct_aVect_nRAttr(l2racc_lx(1)) + if( nfields_sh_lr /= 0 ) sharedFieldsLndRof=trim( mct_aVect_exportRList2c(l2racc_lx(1)) ) tagname = trim(sharedFieldsLndRof)//C_NULL_CHAR ! find the size of land mesh locally ! find out the number of local elements in moab mesh lnd instance on coupler @@ -340,12 +341,12 @@ subroutine prep_rof_init(infodata, lnd_c2_rof, atm_c2_rof, ocn_c2_rof) call shr_sys_abort(subname//' ERROR in computing comm graph , lnd-rof') endif ! context for rearrange is target in this case - if ( mapper_Fl2r%src_mbid .gt. -1 ) then - if (iamroot_CPLID) then - write(logunit,F00) 'overwriting '//trim(mapper_Fl2r%mbname) & - //' mapper_Fl2r' - endif - endif + if ( mapper_Fl2r%src_mbid .gt. -1 ) then + if (iamroot_CPLID) then + write(logunit,F00) 'overwriting '//trim(mapper_Fl2r%mbname) & + //' mapper_Fl2r' + endif + endif mapper_Fl2r%src_mbid = mblxid mapper_Fl2r%tgt_mbid = mbrxid mapper_Fl2r%src_context = lnd(1)%cplcompid @@ -373,12 +374,12 @@ subroutine prep_rof_init(infodata, lnd_c2_rof, atm_c2_rof, ocn_c2_rof) call shr_sys_abort(subname//' ERROR in computing comm graph for second hop, lnd-rof') endif ! now take care of the mapper - if ( mapper_Fl2r%src_mbid .gt. -1 ) then - if (iamroot_CPLID) then - write(logunit,F00) 'overwriting '//trim(mapper_Fl2r%mbname) & - //' mapper_Fl2r' - endif - endif + if ( mapper_Fl2r%src_mbid .gt. -1 ) then + if (iamroot_CPLID) then + write(logunit,F00) 'overwriting '//trim(mapper_Fl2r%mbname) & + //' mapper_Fl2r' + endif + endif mapper_Fl2r%src_mbid = mblxid mapper_Fl2r%tgt_mbid = mbrxid mapper_Fl2r%intx_mbid = mbintxlr @@ -435,7 +436,6 @@ subroutine prep_rof_init(infodata, lnd_c2_rof, atm_c2_rof, ocn_c2_rof) endif #endif end if ! if ((mblxid .ge. 0) .and. (mbrxid .ge. 0)) - ! endif HAVE_MOAB endif ! samegrid_lr #endif ! We'll map irrigation specially, so exclude this from the list of l2r fields @@ -469,9 +469,10 @@ subroutine prep_rof_init(infodata, lnd_c2_rof, atm_c2_rof, ocn_c2_rof) a2racc_ax_cnt = 0 #ifdef HAVE_MOAB ! this a2racc_am will be over atm size - sharedFieldsAtmRof=trim( mct_aVect_exportRList2c(a2racc_ax(1)) ) - tagname = trim(sharedFieldsAtmRof)//C_NULL_CHAR + sharedFieldsAtmRof='' nfields_sh_ar = mct_aVect_nRAttr(a2racc_ax(1)) + if (nfields_sh_ar /= 0 ) sharedFieldsAtmRof = trim( mct_aVect_exportRList2c(a2racc_ax(1)) ) + tagname = trim(sharedFieldsAtmRof)//C_NULL_CHAR ! find the size of atm mesh locally ! find out the number of local elements in moab mesh atm instance on coupler ierr = iMOAB_GetMeshInfo ( mbaxid, nvert, nvise, nbl, nsurf, nvisBC ) @@ -582,7 +583,7 @@ subroutine prep_rof_init(infodata, lnd_c2_rof, atm_c2_rof, ocn_c2_rof) else ! this part does not work, anyway dm1 = "cgll"//C_NULL_CHAR dofnameS="GLOBAL_DOFS"//C_NULL_CHAR - orderS = np ! it should be 4 + orderS = 4 ! np ! it should be 4 endif dm2 = "fv"//C_NULL_CHAR dofnameT="GLOBAL_ID"//C_NULL_CHAR @@ -675,9 +676,10 @@ subroutine prep_rof_init(infodata, lnd_c2_rof, atm_c2_rof, ocn_c2_rof) #ifdef HAVE_MOAB ! this o2racc_om will be over ocn size - sharedFieldsOcnRof=trim( mct_aVect_exportRList2c(o2racc_ox(1)) ) - tagname = trim(sharedFieldsOcnRof)//C_NULL_CHAR + sharedFieldsOcnRof='' nfields_sh_or = mct_aVect_nRAttr(o2racc_ox(1)) + if ( nfields_sh_or /= 0 ) sharedFieldsOcnRof = trim( mct_aVect_exportRList2c(o2racc_ox(1)) ) + tagname = trim(sharedFieldsOcnRof)//C_NULL_CHAR ! find the size of ocn mesh locally ! find out the number of local elements in moab mesh ocn instance on coupler @@ -1033,10 +1035,12 @@ subroutine prep_rof_accum_avg_moab() tagname = trim(sharedFieldsLndRof)//C_NULL_CHAR arrsize = nfields_sh_lr * lsize_lm ent_type = 1 ! cell type - ierr = iMOAB_SetDoubleTagStorage ( mblxid, tagname, arrsize , ent_type, l2racc_lm) - if (ierr .ne. 0) then - call shr_sys_abort(subname//' error in setting accumulated shared fields on rof on land instance ') - endif + if (arrsize > 0) then + ierr = iMOAB_SetDoubleTagStorage ( mblxid, tagname, arrsize , ent_type, l2racc_lm) + if (ierr .ne. 0) then + call shr_sys_abort(subname//' error in setting accumulated shared fields on rof on land instance ') + endif + endif #ifdef MOABDEBUG if (mblxid .ge. 0 ) then ! we are on coupler pes, for sure @@ -1058,10 +1062,12 @@ subroutine prep_rof_accum_avg_moab() tagname = trim(sharedFieldsAtmRof)//C_NULL_CHAR arrsize = nfields_sh_ar * lsize_am ent_type = 1 ! cell type - ierr = iMOAB_SetDoubleTagStorage ( mbaxid, tagname, arrsize , ent_type, a2racc_am) - if (ierr .ne. 0) then - call shr_sys_abort(subname//' error in setting accumulated shared fields on rof on atm instance ') - endif + if (arrsize > 0) then + ierr = iMOAB_SetDoubleTagStorage ( mbaxid, tagname, arrsize , ent_type, a2racc_am) + if (ierr .ne. 0) then + call shr_sys_abort(subname//' error in setting accumulated shared fields on rof on atm instance ') + endif + endif #ifdef MOABDEBUG if (mbaxid .ge. 0 ) then ! we are on coupler pes, for sure write(lnum,"(I0.2)")num_moab_exports @@ -1081,10 +1087,12 @@ subroutine prep_rof_accum_avg_moab() tagname = trim(sharedFieldsOcnRof)//C_NULL_CHAR arrsize = nfields_sh_or * lsize_om ent_type = 1 ! cell type - ierr = iMOAB_SetDoubleTagStorage ( mboxid, tagname, arrsize , ent_type, o2racc_om) - if (ierr .ne. 0) then - call shr_sys_abort(subname//' error in setting accumulated shared fields on rof on ocn instance ') - endif + if (arrsize > 0 ) then + ierr = iMOAB_SetDoubleTagStorage ( mboxid, tagname, arrsize , ent_type, o2racc_om) + if (ierr .ne. 0) then + call shr_sys_abort(subname//' error in setting accumulated shared fields on rof on ocn instance ') + endif + endif #ifdef MOABDEBUG if (mboxid .ge. 0 ) then ! we are on coupler pes, for sure write(lnum,"(I0.2)")num_moab_exports diff --git a/driver-moab/main/seq_diag_mct.F90 b/driver-moab/main/seq_diag_mct.F90 index 2a7d999ccc5e..8534008f9be7 100644 --- a/driver-moab/main/seq_diag_mct.F90 +++ b/driver-moab/main/seq_diag_mct.F90 @@ -138,13 +138,13 @@ module seq_diag_mct integer(in),parameter :: f_hlatf = 8 ! heat : latent, fusion, snow integer(in),parameter :: f_hioff = 9 ! heat : latent, fusion, frozen runoff integer(in),parameter :: f_hsen =10 ! heat : sensible - integer(in),parameter :: f_hberg =11 ! heat : data icebergs + integer(in),parameter :: f_hpolar =11 ! heat : AIS imbalance integer(in),parameter :: f_hh2ot =12 ! heat : water temperature integer(in),parameter :: f_wfrz =13 ! water: freezing integer(in),parameter :: f_wmelt =14 ! water: melting integer(in),parameter :: f_wrain =15 ! water: precip, liquid integer(in),parameter :: f_wsnow =16 ! water: precip, frozen - integer(in),parameter :: f_wberg =17 ! water: data icebergs + integer(in),parameter :: f_wpolar =17 ! water: AIS imbalance integer(in),parameter :: f_wevap =18 ! water: evaporation integer(in),parameter :: f_wroff =19 ! water: runoff/flood integer(in),parameter :: f_wioff =20 ! water: frozen runoff @@ -189,8 +189,8 @@ module seq_diag_mct (/' area',' hfreeze',' hmelt',' hnetsw',' hlwdn', & ' hlwup',' hlatvap',' hlatfus',' hiroff',' hsen', & - ' hberg',' hh2otemp',' wfreeze',' wmelt',' wrain', & - ' wsnow',' wberg',' wevap',' wrunoff',' wfrzrof', & + ' hpolar',' hh2otemp',' wfreeze',' wmelt',' wrain', & + ' wsnow',' wpolar',' wevap',' wrunoff',' wfrzrof', & ' wirrig', & ' wfreeze_16O',' wmelt_16O',' wrain_16O',' wsnow_16O', & ' wevap_16O',' wrunoff_16O',' wfrzrof_16O', & @@ -286,7 +286,15 @@ module seq_diag_mct integer :: index_o2x_Faoo_h2otemp integer :: index_o2x_Fioo_frazil + integer :: index_o2x_Foxo_frazil_li integer :: index_o2x_Fioo_q + integer :: index_o2x_Foxo_q_li + + integer :: index_o2x_Foxo_ismw + integer :: index_o2x_Foxo_rrofl + integer :: index_o2x_Foxo_rrofi + integer :: index_o2x_Foxo_ismh + integer :: index_o2x_Foxo_rrofih integer :: index_xao_Faox_lwup integer :: index_xao_Faox_lat @@ -311,8 +319,6 @@ module seq_diag_mct integer :: index_i2x_Fioi_melth integer :: index_i2x_Fioi_meltw - integer :: index_i2x_Fioi_bergh - integer :: index_i2x_Fioi_bergw integer :: index_i2x_Fioi_salt integer :: index_i2x_Faii_swnet integer :: index_i2x_Fioi_swpen @@ -1337,9 +1343,10 @@ subroutine seq_diag_ocn_mct( ocn, xao_o, frac_o, infodata, do_o2x, do_x2o, do_xa integer(in) :: kArea ! index of area field in aVect integer(in) :: ko,ki ! fraction indices integer(in) :: lSize ! size of aVect - real(r8) :: ca_i,ca_o ! area of a grid cell + real(r8) :: ca_i,ca_o,ca_c ! area of a grid cell logical,save :: first_time = .true. logical,save :: flds_wiso_ocn = .false. + logical,save :: flds_polar = .false. !----- formats ----- character(*),parameter :: subName = '(seq_diag_ocn_mct) ' @@ -1371,8 +1378,18 @@ subroutine seq_diag_ocn_mct( ocn, xao_o, frac_o, infodata, do_o2x, do_x2o, do_xa if (present(do_o2x)) then if (first_time) then index_o2x_Fioo_frazil = mct_aVect_indexRA(o2x_o,'Fioo_frazil') + index_o2x_Foxo_frazil_li= mct_aVect_indexRA(o2x_o,'Foxo_frazil_li',perrWith='quiet') index_o2x_Fioo_q = mct_aVect_indexRA(o2x_o,'Fioo_q') + index_o2x_Foxo_q_li = mct_aVect_indexRA(o2x_o,'Foxo_q_li',perrWith='quiet') index_o2x_Faoo_h2otemp = mct_aVect_indexRA(o2x_o,'Faoo_h2otemp') + index_o2x_Foxo_ismw = mct_aVect_indexRA(o2x_o,'Foxo_ismw',perrWith='quiet') + if ( index_o2x_Foxo_ismw /= 0 ) flds_polar = .true. + if ( flds_polar ) then + index_o2x_Foxo_rrofl = mct_aVect_indexRA(o2x_o,'Foxo_rrofl',perrWith='quiet') + index_o2x_Foxo_rrofi = mct_aVect_indexRA(o2x_o,'Foxo_rrofi',perrWith='quiet') + index_o2x_Foxo_ismh = mct_aVect_indexRA(o2x_o,'Foxo_ismh',perrWith='quiet') + index_o2x_Foxo_rrofih = mct_aVect_indexRA(o2x_o,'Foxo_rrofih',perrWith='quiet') + end if end if lSize = mct_avect_lSize(o2x_o) @@ -1380,10 +1397,20 @@ subroutine seq_diag_ocn_mct( ocn, xao_o, frac_o, infodata, do_o2x, do_x2o, do_xa do n=1,lSize ca_o = dom_o%data%rAttr(kArea,n) * frac_o%rAttr(ko,n) ca_i = dom_o%data%rAttr(kArea,n) * frac_o%rAttr(ki,n) + ca_c = dom_o%data%rAttr(kArea,n) nf = f_area; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + ca_o nf = f_wfrz; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - (ca_o+ca_i)*max(0.0_r8,o2x_o%rAttr(index_o2x_Fioo_frazil,n)) nf = f_hfrz; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*max(0.0_r8,o2x_o%rAttr(index_o2x_Fioo_q,n)) nf = f_hh2ot; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*o2x_o%rAttr(index_o2x_Faoo_h2otemp,n) + if (flds_polar) then + nf = f_wpolar;budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - ca_c*o2x_o%rAttr(index_o2x_Foxo_frazil_li,n) + nf = f_wpolar;budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + ca_c*o2x_o%rAttr(index_o2x_Foxo_ismw,n) + nf = f_wpolar;budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - (ca_o+ca_i)*o2x_o%rAttr(index_o2x_Foxo_rrofl,n) + nf = f_wpolar;budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - (ca_o+ca_i)*o2x_o%rAttr(index_o2x_Foxo_rrofi,n) + nf = f_hpolar;budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + ca_c*o2x_o%rAttr(index_o2x_Foxo_q_li,n) + nf = f_hpolar;budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + ca_c*o2x_o%rAttr(index_o2x_Foxo_ismh,n) + nf = f_hpolar;budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + ca_c*o2x_o%rAttr(index_o2x_Foxo_rrofih,n) + end if end do end if @@ -1490,11 +1517,11 @@ subroutine seq_diag_ocn_mct( ocn, xao_o, frac_o, infodata, do_o2x, do_x2o, do_xa nf = f_hmelt ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*x2o_o%rAttr(index_x2o_Fioi_melth,n) nf = f_hswnet; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*x2o_o%rAttr(index_x2o_Foxx_swnet,n) nf = f_hlwdn ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*x2o_o%rAttr(index_x2o_Faxa_lwdn,n) - nf = f_hberg ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*x2o_o%rAttr(index_x2o_Fioi_bergh,n) + nf = f_hpolar; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*x2o_o%rAttr(index_x2o_Fioi_bergh,n) nf = f_wmelt ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*x2o_o%rAttr(index_x2o_Fioi_meltw,n) nf = f_wrain ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*x2o_o%rAttr(index_x2o_Faxa_rain,n) nf = f_wsnow ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*x2o_o%rAttr(index_x2o_Faxa_snow,n) - nf = f_wberg ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*x2o_o%rAttr(index_x2o_Fioi_bergw,n) + nf = f_wpolar; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*x2o_o%rAttr(index_x2o_Fioi_bergw,n) nf = f_wroff ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*x2o_o%rAttr(index_x2o_Foxx_rofl,n) nf = f_wioff ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + (ca_o+ca_i)*x2o_o%rAttr(index_x2o_Foxx_rofi,n) @@ -1613,8 +1640,6 @@ subroutine seq_diag_ice_mct( ice, frac_i, infodata, do_i2x, do_x2i) if (present(do_i2x)) then index_i2x_Fioi_melth = mct_aVect_indexRA(i2x_i,'Fioi_melth') index_i2x_Fioi_meltw = mct_aVect_indexRA(i2x_i,'Fioi_meltw') - index_i2x_Fioi_bergh = mct_aVect_indexRA(i2x_i,'PFioi_bergh') - index_i2x_Fioi_bergw = mct_aVect_indexRA(i2x_i,'PFioi_bergw') index_i2x_Fioi_swpen = mct_aVect_indexRA(i2x_i,'Fioi_swpen') index_i2x_Faii_swnet = mct_aVect_indexRA(i2x_i,'Faii_swnet') index_i2x_Faii_lwup = mct_aVect_indexRA(i2x_i,'Faii_lwup') @@ -1650,9 +1675,7 @@ subroutine seq_diag_ice_mct( ice, frac_i, infodata, do_i2x, do_x2i) nf = f_hlwup ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + ca_i*i2x_i%rAttr(index_i2x_Faii_lwup,n) nf = f_hlatv ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + ca_i*i2x_i%rAttr(index_i2x_Faii_lat,n) nf = f_hsen ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + ca_i*i2x_i%rAttr(index_i2x_Faii_sen,n) - nf = f_hberg ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - (ca_o+ca_i)*i2x_i%rAttr(index_i2x_Fioi_bergh,n) nf = f_wmelt ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - ca_i*i2x_i%rAttr(index_i2x_Fioi_meltw,n) - nf = f_wberg ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - (ca_o+ca_i)*i2x_i%rAttr(index_i2x_Fioi_bergw,n) nf = f_wevap ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + ca_i*i2x_i%rAttr(index_i2x_Faii_evap,n) if ( flds_wiso_ice )then diff --git a/driver-moab/main/seq_flux_mct.F90 b/driver-moab/main/seq_flux_mct.F90 index 1bb08e10ed94..db876bb3b0d8 100644 --- a/driver-moab/main/seq_flux_mct.F90 +++ b/driver-moab/main/seq_flux_mct.F90 @@ -134,7 +134,7 @@ module seq_flux_mct real(r8), allocatable :: tagValues(:) ! used for copying tag values from frac to frad real(r8), allocatable :: tagValues2(:) ! used for copying tag values for albedos integer , allocatable :: GlobalIds(:) ! used for setting values associated with ids - + ! Coupler field indices integer :: index_a2x_Sa_z @@ -802,6 +802,7 @@ subroutine seq_flux_ocnalb_mct( infodata, ocn, a2x_o, fractions_o, xao_o ) ! Local variables ! type(mct_gGrid), pointer :: dom_o + type(mct_gsMap) , pointer :: gsMap ! model global seg map logical :: flux_albav ! flux avg option integer(in) :: n ! indices real(r8) :: rlat ! gridcell latitude in radians @@ -826,12 +827,16 @@ subroutine seq_flux_ocnalb_mct( infodata, ocn, a2x_o, fractions_o, xao_o ) integer(in) :: klat,klon ! field indices logical :: update_alb ! was albedo updated + integer(IN), pointer :: idata(:) ! temporary for getting global ids from gsmap + integer nvert(3), nvise(3), nbl(3), nsurf(3), nvisBC(3) ! for moab info character(CXX) ::tagname integer :: ent_type, ierr, kgg integer , save :: arrSize ! local size for moab tag arrays (number of cells locally) + integer mpicom ! just to get the global ids from gsmap + integer my_task ! again, just for global ids logical,save :: first_call = .true. integer, save :: lSize @@ -890,8 +895,19 @@ subroutine seq_flux_ocnalb_mct( infodata, ocn, a2x_o, fractions_o, xao_o ) lSize = mct_aVect_lSize(xao_o) allocate(tagValues2(lSize) ) allocate(GlobalIds(lSize) ) - kgg = mct_aVect_indexIA(dom_o%data ,"GlobGridNum" ,perrWith=subName) - GlobalIds = dom_o%data%iAttr(kgg,:) + ! use gsmap instead of domain; for data models, it seems to be not initialized + ! same problem during data ocean init + gsmap => component_get_gsmap_cx( ocn ) + ! get list of global IDs for Dofs + call seq_comm_setptrs(CPLID, mpicom=mpicom) + ! Determine communicator task + call mpi_comm_rank(mpicom, my_task, ierr) + call mct_gsMap_orderedPoints(gsMap, my_task, idata) + do n = 1, lSize + GlobalIds (n) = idata (n) + enddo + !kgg = mct_aVect_indexIA(dom_o%data ,"GlobGridNum" ,perrWith=subName) + !GlobalIds = dom_o%data%iAttr(kgg,:) endif first_call = .false. @@ -1802,20 +1818,15 @@ subroutine seq_flux_atmocn_moab(comp, xao) real(r8) , pointer :: local_xao_mct(:,:) ! atm-ocn fluxes, transpose, mct local sizes integer appId ! moab app id integer i,j - integer nloc, listSize, kgg - - type(mct_ggrid), pointer :: dom + integer nloc, listSize ! moab integer :: tagtype, numco, tagindex, ent_type, ierr, arrSize character(CXX) :: tagname - integer , allocatable :: GlobalIdsLocal(:) ! used for setting values associated with ids character*100 outfile, wopts, lnum - character(*),parameter :: subName = '(seq_flux_atmocn_moab) ' - if (comp%oneletterid == 'a' ) then appId = mbaxid ! atm on coupler local_xao_mct => prep_aoflux_get_xao_amct() @@ -1828,12 +1839,6 @@ subroutine seq_flux_atmocn_moab(comp, xao) ! transpose into moab double array, then set with global id nloc = mct_avect_lsize(xao) listSize = mct_aVect_nRAttr(xao) - dom => component_get_dom_cx(comp) - kgg = mct_aVect_indexIA(dom%data ,"GlobGridNum" ,perrWith=subName) - - allocate(GlobalIdsLocal(nloc)) - GlobalIdsLocal = dom%data%iAttr(kgg,:) - do j = 1, listSize local_xao_mct(:, j) = xao%rAttr(j, :) @@ -1842,12 +1847,12 @@ subroutine seq_flux_atmocn_moab(comp, xao) tagname = trim(seq_flds_xao_fields)//C_NULL_CHAR arrSize = nloc * listSize ent_type = 1 ! cells - ierr = iMOAB_SetDoubleTagStorageWithGid ( appId, tagname, arrSize , ent_type, local_xao_mct, GlobalIdsLocal ) + ! global ids are retrieved by albedo first call; it is a local module variable + ierr = iMOAB_SetDoubleTagStorageWithGid ( appId, tagname, arrSize , ent_type, local_xao_mct, GlobalIds ) if (ierr .ne. 0) then write(logunit,*) subname,' error in setting atm-ocn fluxes ' call shr_sys_abort(subname//' ERROR in setting atm-ocn fluxes') endif - deallocate(GlobalIdsLocal) #ifdef MOABDEBUG ! debug out file diff --git a/driver-moab/main/seq_frac_mct.F90 b/driver-moab/main/seq_frac_mct.F90 index 88622ff45e45..a89fc0b9aa4f 100644 --- a/driver-moab/main/seq_frac_mct.F90 +++ b/driver-moab/main/seq_frac_mct.F90 @@ -662,6 +662,16 @@ subroutine seq_frac_init( infodata, & ko = mct_aVect_indexRa(fractions_o,"ofrac",perrWith=subName) kf = mct_aVect_indexRA(dom_o%data ,"frac" ,perrWith=subName) fractions_o%rAttr(ko,:) = dom_o%data%rAttr(kf,:) + ! so the frac var from dom is copied into ofrac field + ! frac is on ocean mesh, ofrac is on ocean mesh too; + tagname = 'frac'//C_NULL_CHAR + ent_type = 1! cells + allocate(tagValues(local_size_mb_ocn) ) + ierr = iMOAB_GetDoubleTagStorage ( mboxid, tagname, local_size_mb_ocn , ent_type, tagValues) + tagname = 'ofrac'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mboxid, tagname, local_size_mb_ocn , ent_type, tagValues) + ierr = iMOAB_SetDoubleTagStorage ( mbofxid, tagname, local_size_mb_ocn , ent_type, tagValues) + deallocate(tagValues) mapper_o2a => prep_atm_get_mapper_Fo2a() call seq_map_map(mapper_o2a, fractions_o, fractions_a, fldlist='ofrac',norm=.false.) endif diff --git a/driver-moab/main/seq_map_mod.F90 b/driver-moab/main/seq_map_mod.F90 index b893bf53ac73..be47473f444c 100644 --- a/driver-moab/main/seq_map_mod.F90 +++ b/driver-moab/main/seq_map_mod.F90 @@ -327,7 +327,7 @@ subroutine seq_map_map( mapper, av_s, av_d, fldlist, norm, avwts_s, avwtsfld_s, use iso_c_binding use iMOAB, only: iMOAB_GetMeshInfo, iMOAB_GetDoubleTagStorage, iMOAB_SetDoubleTagStorage, & - iMOAB_GetIntTagStorage, iMOAB_SetDoubleTagStorageWithGid, iMOAB_ApplyScalarProjectionWeights, & + iMOAB_GetIntTagStorage, iMOAB_ApplyScalarProjectionWeights, & iMOAB_SendElementTag, iMOAB_ReceiveElementTag, iMOAB_FreeSenderBuffers implicit none @@ -354,7 +354,7 @@ subroutine seq_map_map( mapper, av_s, av_d, fldlist, norm, avwts_s, avwtsfld_s, integer, dimension(:), allocatable :: globalIds real(r8), dimension(:), allocatable :: wghts real(kind=r8) , allocatable :: targtags(:,:), targtags_ini(:,:) - real(kind=r8) :: factor + real(kind=r8) :: factor #endif ! ! Local Variables @@ -423,7 +423,8 @@ subroutine seq_map_map( mapper, av_s, av_d, fldlist, norm, avwts_s, avwtsfld_s, else ! Extract character strings from attribute vector nfields = mct_aVect_nRAttr(av_s) - fldlist_moab = trim(mct_aVect_exportRList2c(av_s)) + fldlist_moab = '' + if ( nfields /= 0 ) fldlist_moab = trim(mct_aVect_exportRList2c(av_s)) endif if (mbnorm) then @@ -495,7 +496,7 @@ subroutine seq_map_map( mapper, av_s, av_d, fldlist, norm, avwts_s, avwtsfld_s, #ifdef MOABDEBUG if (seq_comm_iamroot(CPLID)) then write(logunit, *) subname,' iMOAB mapper rearrange or copy ', mapper%mbname, ' send/recv tags ', trim(fldlist_moab), & - ' mbpresent=', mbpresent, ' mbnorm=', mbnorm + ' mbpresent=', mbpresent, ' mbnorm=', mbnorm call shr_sys_flush(logunit) endif #endif @@ -519,7 +520,7 @@ subroutine seq_map_map( mapper, av_s, av_d, fldlist, norm, avwts_s, avwtsfld_s, call shr_sys_abort(subname//' ERROR in freeing buffers') ! serious enough endif endif ! if (valid_moab_context) - + #endif else @@ -609,11 +610,23 @@ subroutine seq_map_map( mapper, av_s, av_d, fldlist, norm, avwts_s, avwtsfld_s, if ( valid_moab_context ) then ! receive in the intx app, because it is redistributed according to coverage (trick) ! for true intx cases, tgt_mbid is set to be the same as intx_mbid - ! just read map is special + ! just read map is special if (mapper%read_map) then ! receive indeed in target app +#ifdef MOABDEBUG + if (seq_comm_iamroot(CPLID)) then + write(logunit, *) subname,' iMOAB mapper receiving tags with read_map and tgt_mbid: ', & + mapper%mbname, trim(fldlist_moab) + endif +#endif ierr = iMOAB_ReceiveElementTag( mapper%tgt_mbid, fldlist_moab, mapper%mpicom, mapper%src_context ) - else ! receive in the intx app, trick - ierr = iMOAB_ReceiveElementTag( mapper%intx_mbid, fldlist_moab, mapper%mpicom, mapper%src_context ) + else ! receive in the intx app, trick +#ifdef MOABDEBUG + if (seq_comm_iamroot(CPLID)) then + write(logunit, *) subname,' iMOAB mapper receiving tags with intx and intx_mbid: ', & + mapper%mbname, trim(fldlist_moab) + endif +#endif + ierr = iMOAB_ReceiveElementTag( mapper%intx_mbid, fldlist_moab, mapper%mpicom, mapper%src_context ) endif if (ierr .ne. 0) then write(logunit,*) subname,' error in receiving tags ', mapper%mbname, 'recv:', mapper%intx_mbid, trim(fldlist_moab) @@ -684,7 +697,7 @@ subroutine seq_map_map( mapper, av_s, av_d, fldlist, norm, avwts_s, avwtsfld_s, write(logunit,*) subname,' error getting destination tag values ', mapper%mbname call shr_sys_abort(subname//' ERROR getting source tag values') ! serious enough endif - + deallocate(wghts, targtags) if (mbpresent) then #ifdef MOABDEBUG @@ -1232,9 +1245,11 @@ subroutine seq_map_avNormArr(mapper, av_i, av_o, norm_i, rList, norm) call mct_aVect_init(avp_i, rList=trim( rList)//trim(appnd), lsize=lsize_i) call mct_aVect_init(avp_o, rList=trim( rList)//trim(appnd), lsize=lsize_o) else - lrList = mct_aVect_exportRList2c(av_i) + lrList = '' + if(mct_aVect_nRAttr(av_i) /= 0) lrList = mct_aVect_exportRList2c(av_i) call mct_aVect_init(avp_i, rList=trim(lrList)//trim(appnd), lsize=lsize_i) - lrList = mct_aVect_exportRList2c(av_o) + lrList = '' + if(mct_aVect_nRAttr(av_o) /= 0) lrList = mct_aVect_exportRList2c(av_o) call mct_aVect_init(avp_o, rList=trim(lrList)//trim(appnd), lsize=lsize_o) endif diff --git a/driver-moab/shr/seq_comm_mct.F90 b/driver-moab/shr/seq_comm_mct.F90 index c167824cf3b0..10a23b9c5094 100644 --- a/driver-moab/shr/seq_comm_mct.F90 +++ b/driver-moab/shr/seq_comm_mct.F90 @@ -1599,6 +1599,7 @@ subroutine seq_comm_compare_mb_mct( modelstr, mpicom, attrVect, mct_field, appId values = mct_values - values difference = dot_product(values, values) + differenceg = 0. ! initialize to 0 the total sum call shr_mpi_sum(difference,differenceg,mpicom,subname) difference = sqrt(differenceg) call shr_mpi_commrank( mpicom, rank2 ) diff --git a/driver-moab/shr/seq_flds_mod.F90 b/driver-moab/shr/seq_flds_mod.F90 index e03771a7a9c6..84e200b0d35e 100644 --- a/driver-moab/shr/seq_flds_mod.F90 +++ b/driver-moab/shr/seq_flds_mod.F90 @@ -293,6 +293,8 @@ module seq_flds_mod ! namelist variables logical :: nan_check_component_fields + public moab_set_tag_from_av ! will be caled usually from data models, to set moab tags from data fields in AVs + !---------------------------------------------------------------------------- contains !---------------------------------------------------------------------------- @@ -395,10 +397,11 @@ subroutine seq_flds_set(nmlfile, ID, infodata) logical :: flds_co2_dmsa logical :: flds_bgc_oi logical :: flds_wiso + logical :: flds_polar integer :: glc_nec namelist /seq_cplflds_inparm/ & - flds_co2a, flds_co2b, flds_co2c, flds_co2_dmsa, flds_wiso, glc_nec, & + flds_co2a, flds_co2b, flds_co2c, flds_co2_dmsa, flds_wiso, flds_polar, glc_nec, & ice_ncat, seq_flds_i2o_per_cat, flds_bgc_oi, & nan_check_component_fields, rof_heat, atm_flux_method, atm_gustiness, & rof2ocn_nutrients, lnd_rof_two_way, ocn_rof_two_way, rof_sed @@ -435,6 +438,7 @@ subroutine seq_flds_set(nmlfile, ID, infodata) flds_co2_dmsa = .false. flds_bgc_oi = .false. flds_wiso = .false. + flds_polar = .false. glc_nec = 0 ice_ncat = 1 seq_flds_i2o_per_cat = .false. @@ -468,6 +472,7 @@ subroutine seq_flds_set(nmlfile, ID, infodata) call shr_mpi_bcast(flds_co2_dmsa, mpicom) call shr_mpi_bcast(flds_bgc_oi , mpicom) call shr_mpi_bcast(flds_wiso , mpicom) + call shr_mpi_bcast(flds_polar , mpicom) call shr_mpi_bcast(glc_nec , mpicom) call shr_mpi_bcast(ice_ncat , mpicom) call shr_mpi_bcast(seq_flds_i2o_per_cat, mpicom) @@ -1602,6 +1607,69 @@ subroutine seq_flds_set(nmlfile, ID, infodata) attname = 'PFioi_bergw' call metadata_set(attname, longname, stdname, units) + !-------------------------------- + ! ocn<->cpl only exchange - Polar + !-------------------------------- + if (flds_polar) then + + ! Ocean Land ice freeze potential + call seq_flds_add(o2x_fluxes,"Foxo_q_li") + longname = 'Ocean land ice freeze potential' + stdname = 'ice_shelf_cavity_ice_heat_flux' + units = 'W m-2' + attname = 'Foxo_q_li' + call metadata_set(attname, longname, stdname, units) + + ! Ocean land ice frazil production + call seq_flds_add(o2x_fluxes,"Foxo_frazil_li") + longname = 'Ocean land ice frazil production' + stdname = 'ocean_land_ice_frazil_ice_production' + units = 'kg m-2 s-1' + attname = 'Foxo_frazil_li' + call metadata_set(attname, longname, stdname, units) + + ! Water flux from ice shelf melt + call seq_flds_add(o2x_fluxes,"Foxo_ismw") + longname = 'Water flux due to basal melting of ice shelves' + stdname = 'basal_iceshelf_melt_flux' + units = 'kg m-2 s-1' + attname = 'Foxo_ismw' + call metadata_set(attname, longname, stdname, units) + + ! Heat flux from ice shelf melt + call seq_flds_add(o2x_fluxes,"Foxo_ismh") + longname = 'Heat flux due to basal melting of ice shelves' + stdname = 'basal_iceshelf_heat_flux' + units = 'W m-2' + attname = 'Foxo_ismh' + call metadata_set(attname, longname, stdname, units) + + ! Water flux from removed liquid runoff + call seq_flds_add(o2x_fluxes,"Foxo_rrofl") + longname = 'Water flux due to removed liqiud runoff' + stdname = 'removed_liquid_runoff_flux' + units = 'kg m-2 s-1' + attname = 'Foxo_rrofl' + call metadata_set(attname, longname, stdname, units) + + ! Water flux from removed solid runoff + call seq_flds_add(o2x_fluxes,"Foxo_rrofi") + longname = 'Water flux due to removed solid runoff' + stdname = 'removed_solid_runoff_flux' + units = 'kg m-2 s-1' + attname = 'Foxo_rrofi' + call metadata_set(attname, longname, stdname, units) + + ! Heat flux from removed solid runoff + call seq_flds_add(o2x_fluxes,"Foxo_rrofih") + longname = 'Heat flux due to removed solid runoff' + stdname = 'removed_solid_runoff_heat_flux' + units = 'W m-2' + attname = 'Foxo_rrofih' + call metadata_set(attname, longname, stdname, units) + + end if + ! Salt flux call seq_flds_add(i2x_fluxes,"Fioi_salt") call seq_flds_add(x2o_fluxes,"Fioi_salt") @@ -4369,4 +4437,32 @@ subroutine seq_flds_esmf_metadata_get(shortname, longname, stdname, units) end subroutine seq_flds_esmf_metadata_get +#ifdef HAVE_MOAB + !=============================================================================== + + subroutine moab_set_tag_from_av(tagname, avx, index, mbapid, dataarr, lsize) + + ! !DESCRIPTION: set field method for data atm model + use iMOAB, only: iMOAB_SetDoubleTagStorage + use shr_kind_mod , only: r8 => SHR_KIND_R8 + implicit none + + integer :: ierr, lsize + character(len=*), intent(in) :: tagname + type(mct_aVect), intent(in) :: avx + integer, intent(in) :: index + integer, intent(in) :: mbapid ! moab app id + real(R8), intent(inout) :: dataarr(:) + + !write(*,* ) "Setting data for tag: ", tagname, " with size = ", lsize + dataarr(:) = avx%rAttr(index, :) + ierr = iMOAB_SetDoubleTagStorage ( mbapid, tagname, lsize, & + 0, & ! data on vertices + dataarr ) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to set tag values for '//tagname) + + end subroutine moab_set_tag_from_av + +#endif end module seq_flds_mod diff --git a/driver-moab/shr/seq_infodata_mod.F90 b/driver-moab/shr/seq_infodata_mod.F90 index 29ba2c89d983..77236e027cc3 100644 --- a/driver-moab/shr/seq_infodata_mod.F90 +++ b/driver-moab/shr/seq_infodata_mod.F90 @@ -232,6 +232,8 @@ MODULE seq_infodata_mod integer(SHR_KIND_IN) :: iac_ny ! nx, ny of "2d" grid character(SHR_KIND_CL) :: lnd_domain ! path to land domain file character(SHR_KIND_CL) :: rof_mesh ! path to river mesh file + character(SHR_KIND_CL) :: ocn_domain ! path to ocean domain file, used by data ocean models only + character(SHR_KIND_CL) :: atm_mesh ! path to atmosphere domain/mesh file, used by data atm models only !--- set via components and may be time varying --- real(SHR_KIND_R8) :: nextsw_cday ! calendar of next atm shortwave @@ -790,6 +792,9 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid, cpl_tag) infodata%iac_ny = 0 infodata%lnd_domain = 'none' infodata%rof_mesh = 'none' + infodata%ocn_domain = 'none' ! will be used for ocean data models only; will be used as a signal + infodata%atm_mesh = 'none' ! will be used for atmosphere data models only; will be used as a signal + ! not sure if it exists always actually infodata%nextsw_cday = -1.0_SHR_KIND_R8 infodata%precip_fact = 1.0_SHR_KIND_R8 @@ -1032,7 +1037,8 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ glc_phase, rof_phase, atm_phase, lnd_phase, ocn_phase, ice_phase, & wav_phase, iac_phase, esp_phase, wav_nx, wav_ny, atm_nx, atm_ny, & lnd_nx, lnd_ny, rof_nx, rof_ny, ice_nx, ice_ny, ocn_nx, ocn_ny, & - iac_nx, iac_ny, glc_nx, glc_ny, lnd_domain, rof_mesh, eps_frac, & + iac_nx, iac_ny, glc_nx, glc_ny, lnd_domain, rof_mesh, ocn_domain, & + atm_mesh, eps_frac, & eps_amask, eps_agrid, eps_aarea, eps_omask, eps_ogrid, eps_oarea, & reprosum_use_ddpdd, reprosum_allow_infnan, & reprosum_diffmax, reprosum_recompute, & @@ -1206,6 +1212,8 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ integer(SHR_KIND_IN), optional, intent(OUT) :: iac_ny character(SHR_KIND_CL), optional, intent(OUT) :: lnd_domain character(SHR_KIND_CL), optional, intent(OUT) :: rof_mesh + character(SHR_KIND_CL), optional, intent(OUT) :: ocn_domain + character(SHR_KIND_CL), optional, intent(OUT) :: atm_mesh real(SHR_KIND_R8), optional, intent(OUT) :: nextsw_cday ! calendar of next atm shortwave real(SHR_KIND_R8), optional, intent(OUT) :: precip_fact ! precip factor @@ -1393,6 +1401,8 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ if ( present(iac_ny) ) iac_ny = infodata%iac_ny if ( present(lnd_domain) ) lnd_domain = infodata%lnd_domain if ( present(rof_mesh) ) rof_mesh = infodata%rof_mesh + if ( present(ocn_domain) ) ocn_domain = infodata%ocn_domain + if ( present(atm_mesh) ) atm_mesh = infodata%atm_mesh if ( present(nextsw_cday) ) nextsw_cday = infodata%nextsw_cday if ( present(precip_fact) ) precip_fact = infodata%precip_fact @@ -1588,7 +1598,8 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ wav_phase, iac_phase, esp_phase, wav_nx, wav_ny, atm_nx, atm_ny, & lnd_nx, lnd_ny, rof_nx, rof_ny, ice_nx, ice_ny, ocn_nx, ocn_ny, & iac_nx, iac_ny, glc_nx, glc_ny, eps_frac, eps_amask, lnd_domain, & - rof_mesh, eps_agrid, eps_aarea, eps_omask, eps_ogrid, eps_oarea, & + rof_mesh, ocn_domain, atm_mesh, eps_agrid, eps_aarea, eps_omask, & + eps_ogrid, eps_oarea, & reprosum_use_ddpdd, reprosum_allow_infnan, & reprosum_diffmax, reprosum_recompute, & mct_usealltoall, mct_usevector, glc_valid_input, nlmaps_verbosity) @@ -1760,6 +1771,8 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ integer(SHR_KIND_IN), optional, intent(IN) :: iac_ny character(SHR_KIND_CL), optional, intent(IN) :: lnd_domain character(SHR_KIND_CL), optional, intent(IN) :: rof_mesh + character(SHR_KIND_CL), optional, intent(IN) :: ocn_domain + character(SHR_KIND_CL), optional, intent(IN) :: atm_mesh real(SHR_KIND_R8), optional, intent(IN) :: nextsw_cday ! calendar of next atm shortwave real(SHR_KIND_R8), optional, intent(IN) :: precip_fact ! precip factor @@ -1946,6 +1959,8 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ if ( present(iac_ny) ) infodata%iac_ny = iac_ny if ( present(lnd_domain) ) infodata%lnd_domain = lnd_domain if ( present(rof_mesh) ) infodata%rof_mesh = rof_mesh + if ( present(ocn_domain) ) infodata%ocn_domain = ocn_domain + if ( present(atm_mesh) ) infodata%atm_mesh = atm_mesh if ( present(nextsw_cday) ) infodata%nextsw_cday = nextsw_cday if ( present(precip_fact) ) infodata%precip_fact = precip_fact @@ -2256,7 +2271,8 @@ subroutine seq_infodata_bcast(infodata,mpicom) call shr_mpi_bcast(infodata%iac_ny, mpicom) call shr_mpi_bcast(infodata%lnd_domain, mpicom) call shr_mpi_bcast(infodata%rof_mesh, mpicom) - + call shr_mpi_bcast(infodata%ocn_domain, mpicom) + call shr_mpi_bcast(infodata%atm_mesh, mpicom) call shr_mpi_bcast(infodata%nextsw_cday, mpicom) call shr_mpi_bcast(infodata%precip_fact, mpicom) call shr_mpi_bcast(infodata%atm_phase, mpicom) @@ -2474,6 +2490,7 @@ subroutine seq_infodata_Exchange(infodata,ID,type) call shr_mpi_bcast(infodata%atm_nx, mpicom, pebcast=cmppe) call shr_mpi_bcast(infodata%atm_ny, mpicom, pebcast=cmppe) call shr_mpi_bcast(infodata%atm_aero, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%atm_mesh, mpicom, pebcast=cmppe) ! dead_comps is true if it's ever set to true deads = infodata%dead_comps call shr_mpi_bcast(deads, mpicom, pebcast=cmppe) @@ -2514,6 +2531,7 @@ subroutine seq_infodata_Exchange(infodata,ID,type) call shr_mpi_bcast(infodata%ocn_c2_glcshelf, mpicom, pebcast=cmppe) call shr_mpi_bcast(infodata%ocn_nx, mpicom, pebcast=cmppe) call shr_mpi_bcast(infodata%ocn_ny, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%ocn_domain, mpicom, pebcast=cmppe) ! dead_comps is true if it's ever set to true deads = infodata%dead_comps call shr_mpi_bcast(deads, mpicom, pebcast=cmppe) @@ -2972,6 +2990,8 @@ SUBROUTINE seq_infodata_print( infodata ) write(logunit,F0I) subname,'iac_ny = ', infodata%iac_ny write(logunit,F0I) subname,'lnd_domain = ', infodata%lnd_domain write(logunit,F0I) subname,'rof_mesh = ', infodata%rof_mesh + write(logunit,F0I) subname,'ocn_domain = ', infodata%ocn_domain + write(logunit,F0I) subname,'atm_mesh = ', infodata%atm_mesh write(logunit,F0R) subname,'nextsw_cday = ', infodata%nextsw_cday write(logunit,F0R) subname,'precip_fact = ', infodata%precip_fact diff --git a/externals/ekat b/externals/ekat index 2ff5853316e1..44977d9eab51 160000 --- a/externals/ekat +++ b/externals/ekat @@ -1 +1 @@ -Subproject commit 2ff5853316e15d4e8004c21890329fd257fa7459 +Subproject commit 44977d9eab51b812952b6bac7dfcb30aafdf7cb5 diff --git a/externals/haero b/externals/haero index 574787f9b31c..77c4a45584f2 160000 --- a/externals/haero +++ b/externals/haero @@ -1 +1 @@ -Subproject commit 574787f9b31cc77b7902f07e73e10dcd18cdf29d +Subproject commit 77c4a45584f2bd3bf38a18ea5bc24ef9ce3776a2 diff --git a/externals/mam4xx b/externals/mam4xx index 8f6af492c862..a19bc75891e2 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 8f6af492c8627ebe5ec7904c3185ab8294e49a12 +Subproject commit a19bc75891e235071426840742a2c530bbea9e52 diff --git a/externals/scorpio b/externals/scorpio index de0b1ca2200f..cdd541e0cd70 160000 --- a/externals/scorpio +++ b/externals/scorpio @@ -1 +1 @@ -Subproject commit de0b1ca2200f62c6eb5e3fd40147965409e97123 +Subproject commit cdd541e0cd708bece13b1f3ee42f66dfd6440aa7 diff --git a/mkdocs.yaml b/mkdocs.yaml index 27fafbeba4d0..3c7a2f93bdb3 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -1,8 +1,20 @@ site_name: E3SM nav: - - Introduction: 'index.md' + - Home: 'index.md' + - E3SM Basics: + - 'index.md' + - Installation: 'installation.md' + - User Guide: 'user-guide/index.md' + - Development: 'development.md' - Components: '*include ./components/*/mkdocs.yml' + - Tools: '*include ./tools/*/mkdocs.yml' + - More Information: + - 'E3SM-Project' : 'http://docs.e3sm.org' + - 'e3sm.org' : 'http://e3sm.org' + +repo_name: E3SM-Project/E3SM +repo_url: https://github.com/E3SM-Project/E3SM theme: name: material @@ -17,15 +29,30 @@ theme: toggle: icon: material/weather-night name: Switch to light mode + icon: + admonition: + note: octicons/tag-16 features: - - navigation.indices + - navigation.indexes - navigation.instant + - navigation.instant.prefetch - navigation.sections + - navigation.path + - navigation.tracking - navigation.top -# - navigation.tabs + - search.suggest + - search.highlight + - search.share + - content.code.select + - content.code.copy + - content.action.view + - content.tooltips + markdown_extensions: + - admonition - footnotes + - pymdownx.details - pymdownx.highlight - pymdownx.superfences - pymdownx.tabbed: @@ -33,11 +60,13 @@ markdown_extensions: - pymdownx.arithmatex: generic: true - md_in_html + - tables plugins: - monorepo + - search - bibtex: - bib_file: components/elm/docs/refs.bib + bib_dir: docs/refs extra_javascript: - javascript/mathjax.js diff --git a/run_e3sm.template.sh b/run_e3sm.template.sh index e471faa42ed8..623867c01904 100755 --- a/run_e3sm.template.sh +++ b/run_e3sm.template.sh @@ -1,8 +1,6 @@ #!/bin/bash -fe -# E3SM Water Cycle v2 run_e3sm script template. -# -# Inspired by v1 run_e3sm script as well as SCREAM group simplified run script. +# E3SM Coupled Model Group run_e3sm script template. # # Bash coding style inspired by: # http://kfirlavi.herokuapp.com/blog/2012/11/14/defensive-bash-programming @@ -18,36 +16,38 @@ main() { # --- Configuration flags ---- # Machine and project -readonly MACHINE=cori-knl +readonly MACHINE=pm-cpu +# BEFORE RUNNING: CHANGE this to your project readonly PROJECT="e3sm" # Simulation readonly COMPSET="WCYCL1850" -readonly RESOLUTION="ne30pg2_EC30to60E2r2" +readonly RESOLUTION="ne30pg2_r05_IcoswISC30E3r5" # BEFORE RUNNING : CHANGE the following CASE_NAME to desired value readonly CASE_NAME="your_casename" # If this is part of a simulation campaign, ask your group lead about using a case_group label # readonly CASE_GROUP="" # Code and compilation -readonly CHECKOUT="20210806" +# BEFORE RUNNING: CHANGE CHECKOUT to date string like 20240301 +readonly CHECKOUT="latest" readonly BRANCH="master" readonly CHERRY=( ) readonly DEBUG_COMPILE=false # Run options -readonly MODEL_START_TYPE="hybrid" # 'initial', 'continue', 'branch', 'hybrid' +readonly MODEL_START_TYPE="initial" # 'initial', 'continue', 'branch', 'hybrid' readonly START_DATE="0001-01-01" # Additional options for 'branch' and 'hybrid' readonly GET_REFCASE=TRUE -readonly RUN_REFDIR="/global/cscratch1/sd/forsyth/E3SMv2/v2.LR.piControl/init" -readonly RUN_REFCASE="20210625.v2rc3c-GWD.piControl.ne30pg2_EC30to60E2r2.chrysalis" -readonly RUN_REFDATE="1001-01-01" # same as MODEL_START_DATE for 'branch', can be different for 'hybrid' +#readonly RUN_REFDIR="" +#readonly RUN_REFCASE="" +#readonly RUN_REFDATE="" # same as MODEL_START_DATE for 'branch', can be different for 'hybrid' # Set paths -readonly CODE_ROOT="${HOME}/E3SMv2/code/${CHECKOUT}" -readonly CASE_ROOT="/global/cscratch1/sd/${USER}/E3SMv2/${CASE_NAME}" +readonly CODE_ROOT="${HOME}/E3SMv3/code/${CHECKOUT}" +readonly CASE_ROOT="/pscratch/sd/r/${USER}/e3sm-scratch/${CASE_NAME}" # Sub-directories readonly CASE_BUILD_DIR=${CASE_ROOT}/build @@ -59,7 +59,7 @@ readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/archive # or 'production' for full simulation readonly run='XS_2x5_ndays' if [ "${run}" != "production" ]; then - + echo "setting up Short test simulations: ${run}" # Short test simulations tmp=($(echo $run | tr "_" " ")) layout=${tmp[0]} @@ -145,36 +145,111 @@ echo $'\n----- All done -----\n' user_nl() { cat << EOF >> user_nl_eam - nhtfrq = 0,-24,-6,-6,-3,-24,0 - mfilt = 1,30,120,120,240,30,1 - avgflag_pertape = 'A','A','I','A','A','A','I' - fexcl1 = 'CFAD_SR532_CAL', 'LINOZ_DO3', 'LINOZ_DO3_PSC', 'LINOZ_O3CLIM', 'LINOZ_O3COL', 'LINOZ_SSO3', 'hstobie_linoz' - fincl1 = 'extinct_sw_inp','extinct_lw_bnd7','extinct_lw_inp','CLD_CAL', 'TREFMNAV', 'TREFMXAV' - fincl2 = 'FLUT','PRECT','U200','V200','U850','V850','Z500','OMEGA500','UBOT','VBOT','TREFHT','TREFHTMN:M','TREFHTMX:X','QREFHT','TS','PS','TMQ','TUQ','TVQ','TOZ', 'FLDS', 'FLNS', 'FSDS', 'FSNS', 'SHFLX', 'LHFLX', 'TGCLDCWP', 'TGCLDIWP', 'TGCLDLWP', 'CLDTOT', 'T250', 'T200', 'T150', 'T100', 'T050', 'T025', 'T010', 'T005', 'T002', 'T001', 'TTOP', 'U250', 'U150', 'U100', 'U050', 'U025', 'U010', 'U005', 'U002', 'U001', 'UTOP', 'FSNT', 'FLNT' - fincl3 = 'PSL','T200','T500','U850','V850','UBOT','VBOT','TREFHT', 'Z700', 'TBOT:M' - fincl4 = 'FLUT','U200','U850','PRECT','OMEGA500' - fincl5 = 'PRECT','PRECC','TUQ','TVQ','QFLX','SHFLX','U90M','V90M' - fincl6 = 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANTAU_ISCCP','MEANPTOP_ISCCP','MEANTB_ISCCP','CLDTOT_CAL','CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN','CLDHGH_CAL','CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN','CLDMED_CAL','CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN','CLDLOW_CAL','CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN' - fincl7 = 'O3', 'PS', 'TROP_P' - -! Additional retuning - clubb_tk1 = 268.15D0 - gw_convect_hcf = 10.0 + cosp_lite = .true. + + empty_htapes = .true. + + avgflag_pertape = 'A','A','A','A','I','I' + nhtfrq = 0,-24,-6,-3,-1,0 + mfilt = 1,30,120,240,720,1 + + fincl1 = 'AODALL','AODBC','AODDUST','AODPOM','AODSO4','AODSOA','AODSS','AODVIS', + 'CLDLOW','CLDMED','CLDHGH','CLDTOT', + 'CLDHGH_CAL','CLDLOW_CAL','CLDMED_CAL','CLD_MISR','CLDTOT_CAL', + 'CLMODIS','FISCCP1_COSP','FLDS','FLNS','FLNSC','FLNT','FLUT', + 'FLUTC','FSDS','FSDSC','FSNS','FSNSC','FSNT','FSNTOA','FSNTOAC','FSNTC', + 'ICEFRAC','LANDFRAC','LWCF','OCNFRAC','OMEGA','PRECC','PRECL','PRECSC','PRECSL','PS','PSL','Q', + 'QFLX','QREFHT','RELHUM','SCO','SHFLX','SOLIN','SWCF','T','TAUX','TAUY','TCO', + 'TGCLDLWP','TMQ','TREFHT','TREFMNAV','TREFMXAV','TS','U','U10','V','Z3', + 'O3','LHFLX', + 'O3_2DTDA_trop','O3_2DTDB_trop','O3_2DTDD_trop','O3_2DTDE_trop','O3_2DTDI_trop','O3_2DTDL_trop', + 'O3_2DTDN_trop','O3_2DTDO_trop','O3_2DTDS_trop','O3_2DTDU_trop','O3_2DTRE_trop','O3_2DTRI_trop', + 'O3_SRF','NO_2DTDS','NO_TDLgt','NO2_2DTDD','NO2_2DTDS','NO2_TDAcf','CO_SRF','TROPE3D_P','TROP_P', + 'CDNUMC','SFDMS','so4_a1_sfgaex1','so4_a2_sfgaex1','so4_a3_sfgaex1','so4_a5_sfgaex1','soa_a1_sfgaex1', + 'soa_a2_sfgaex1','soa_a3_sfgaex1','GS_soa_a1','GS_soa_a2','GS_soa_a3','AQSO4_H2O2','AQSO4_O3', + 'SFSO2','SO2_CLXF','SO2','DF_SO2','AQ_SO2','GS_SO2','WD_SO2','ABURDENSO4_STR','ABURDENSO4_TRO', + 'ABURDENSO4','ABURDENBC','ABURDENDUST','ABURDENMOM','ABURDENPOM','ABURDENSEASALT', + 'ABURDENSOA','AODSO4_STR','AODSO4_TRO', + 'EXTINCT','AODABS','AODABSBC','CLDICE','CLDLIQ','CLD_CAL_TMPLIQ','CLD_CAL_TMPICE','Mass_bc_srf', + 'Mass_dst_srf','Mass_mom_srf','Mass_ncl_srf','Mass_pom_srf','Mass_so4_srf','Mass_soa_srf','Mass_bc_850', + 'Mass_dst_850','Mass_mom_850','Mass_ncl_850','Mass_pom_850','Mass_so4_850','Mass_soa_850','Mass_bc_500', + 'Mass_dst_500','Mass_mom_500','Mass_ncl_500','Mass_pom_500','Mass_so4_500','Mass_soa_500','Mass_bc_330', + 'Mass_dst_330','Mass_mom_330','Mass_ncl_330','Mass_pom_330','Mass_so4_330','Mass_soa_330','Mass_bc_200', + 'Mass_dst_200','Mass_mom_200','Mass_ncl_200','Mass_pom_200','Mass_so4_200','Mass_soa_200', + 'O3_2DTDD','O3_2DCIP','O3_2DCIL','CO_2DTDS','CO_2DTDD','CO_2DCEP','CO_2DCEL','NO_2DTDD', + 'FLNTC','SAODVIS', + 'H2OLNZ', + 'dst_a1SF','dst_a3SF', + 'PHIS','CLOUD','TGCLDIWP','TGCLDCWP','AREL', + 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANPTOP_ISCCP','CLD_CAL', + 'CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN', + 'CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN', + 'CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN', + 'CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN', + 'CLWMODIS','CLIMODIS' + + fincl2 = 'PS', 'FLUT','PRECT','U200','V200','U850','V850', + 'TCO','SCO','TREFHTMN','TREFHTMX','TREFHT','QREFHT' + fincl3 = 'PS', 'PSL','PRECT','TUQ','TVQ','UBOT','VBOT','TREFHT','FLUT','OMEGA500','TBOT','U850','V850','U200','V200','T200','T500','Z700' + fincl4 = 'PRECT' + fincl5 = 'O3_SRF' + fincl6 = 'CO_2DMSD','NO2_2DMSD','NO_2DMSD','O3_2DMSD','O3_2DMSD_trop' + + ! -- chemUCI settings ------------------ + history_chemdyg_summary = .true. + history_gaschmbudget_2D = .false. + history_gaschmbudget_2D_levels = .false. + history_gaschmbudget_num = 6 !! no impact if history_gaschmbudget_2D = .false. + + ! -- MAM5 settings ------------------ + is_output_interactive_volc = .true. EOF cat << EOF >> user_nl_elm - hist_dov2xy = .true.,.true. +finidat = '' +hist_dov2xy = .true.,.true. +hist_fexcl1 = 'AGWDNPP','ALTMAX_LASTYEAR','AVAIL_RETRANSP','AVAILC','BAF_CROP', + 'BAF_PEATF','BIOCHEM_PMIN_TO_PLANT','CH4_SURF_AERE_SAT','CH4_SURF_AERE_UNSAT','CH4_SURF_DIFF_SAT', + 'CH4_SURF_DIFF_UNSAT','CH4_SURF_EBUL_SAT','CH4_SURF_EBUL_UNSAT','CMASS_BALANCE_ERROR','cn_scalar', + 'COL_PTRUNC','CONC_CH4_SAT','CONC_CH4_UNSAT','CONC_O2_SAT','CONC_O2_UNSAT', + 'cp_scalar','CWDC_HR','CWDC_LOSS','CWDC_TO_LITR2C','CWDC_TO_LITR3C', + 'CWDC_vr','CWDN_TO_LITR2N','CWDN_TO_LITR3N','CWDN_vr','CWDP_TO_LITR2P', + 'CWDP_TO_LITR3P','CWDP_vr','DWT_CONV_CFLUX_DRIBBLED','F_CO2_SOIL','F_CO2_SOIL_vr', + 'F_DENIT_vr','F_N2O_DENIT','F_N2O_NIT','F_NIT_vr','FCH4_DFSAT', + 'FINUNDATED_LAG','FPI_P_vr','FPI_vr','FROOTC_LOSS','HR_vr', + 'LABILEP_TO_SECONDP','LABILEP_vr','LAND_UPTAKE','LEAF_MR','leaf_npimbalance', + 'LEAFC_LOSS','LEAFC_TO_LITTER','LFC2','LITR1_HR','LITR1C_TO_SOIL1C', + 'LITR1C_vr','LITR1N_TNDNCY_VERT_TRANS','LITR1N_TO_SOIL1N','LITR1N_vr','LITR1P_TNDNCY_VERT_TRANS', + 'LITR1P_TO_SOIL1P','LITR1P_vr','LITR2_HR','LITR2C_TO_SOIL2C','LITR2C_vr', + 'LITR2N_TNDNCY_VERT_TRANS','LITR2N_TO_SOIL2N','LITR2N_vr','LITR2P_TNDNCY_VERT_TRANS','LITR2P_TO_SOIL2P', + 'LITR2P_vr','LITR3_HR','LITR3C_TO_SOIL3C','LITR3C_vr','LITR3N_TNDNCY_VERT_TRANS', + 'LITR3N_TO_SOIL3N','LITR3N_vr','LITR3P_TNDNCY_VERT_TRANS','LITR3P_TO_SOIL3P','LITR3P_vr', + 'M_LITR1C_TO_LEACHING','M_LITR2C_TO_LEACHING','M_LITR3C_TO_LEACHING','M_SOIL1C_TO_LEACHING','M_SOIL2C_TO_LEACHING', + 'M_SOIL3C_TO_LEACHING','M_SOIL4C_TO_LEACHING','NDEPLOY','NEM','nlim_m', + 'o2_decomp_depth_unsat','OCCLP_vr','PDEPLOY','PLANT_CALLOC','PLANT_NDEMAND', + 'PLANT_NDEMAND_COL','PLANT_PALLOC','PLANT_PDEMAND','PLANT_PDEMAND_COL','plim_m', + 'POT_F_DENIT','POT_F_NIT','POTENTIAL_IMMOB','POTENTIAL_IMMOB_P','PRIMP_TO_LABILEP', + 'PRIMP_vr','PROD1P_LOSS','QOVER_LAG','RETRANSN_TO_NPOOL','RETRANSP_TO_PPOOL', + 'SCALARAVG_vr','SECONDP_TO_LABILEP','SECONDP_TO_OCCLP','SECONDP_vr','SMIN_NH4_vr', + 'SMIN_NO3_vr','SMINN_TO_SOIL1N_L1','SMINN_TO_SOIL2N_L2','SMINN_TO_SOIL2N_S1','SMINN_TO_SOIL3N_L3', + 'SMINN_TO_SOIL3N_S2','SMINN_TO_SOIL4N_S3','SMINP_TO_SOIL1P_L1','SMINP_TO_SOIL2P_L2','SMINP_TO_SOIL2P_S1', + 'SMINP_TO_SOIL3P_L3','SMINP_TO_SOIL3P_S2','SMINP_TO_SOIL4P_S3','SMINP_vr','SOIL1_HR','SOIL1C_TO_SOIL2C','SOIL1C_vr','SOIL1N_TNDNCY_VERT_TRANS','SOIL1N_TO_SOIL2N','SOIL1N_vr', + 'SOIL1P_TNDNCY_VERT_TRANS','SOIL1P_TO_SOIL2P','SOIL1P_vr','SOIL2_HR','SOIL2C_TO_SOIL3C', + 'SOIL2C_vr','SOIL2N_TNDNCY_VERT_TRANS','SOIL2N_TO_SOIL3N','SOIL2N_vr','SOIL2P_TNDNCY_VERT_TRANS', + 'SOIL2P_TO_SOIL3P','SOIL2P_vr','SOIL3_HR','SOIL3C_TO_SOIL4C','SOIL3C_vr', + 'SOIL3N_TNDNCY_VERT_TRANS','SOIL3N_TO_SOIL4N','SOIL3N_vr','SOIL3P_TNDNCY_VERT_TRANS','SOIL3P_TO_SOIL4P', + 'SOIL3P_vr','SOIL4_HR','SOIL4C_vr','SOIL4N_TNDNCY_VERT_TRANS','SOIL4N_TO_SMINN', + 'SOIL4N_vr','SOIL4P_TNDNCY_VERT_TRANS','SOIL4P_TO_SMINP','SOIL4P_vr','SOLUTIONP_vr', + 'TCS_MONTH_BEGIN','TCS_MONTH_END','TOTCOLCH4','water_scalar','WF', + 'wlim_m','WOODC_LOSS','WTGQ' + hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC' hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' hist_mfilt = 1,365 hist_nhtfrq = 0,-24 hist_avgflag_pertape = 'A','A' -EOF - -cat << EOF >> user_nl_mosart - rtmhist_fincl2 = 'RIVER_DISCHARGE_OVER_LAND_LIQ' - rtmhist_mfilt = 1,365 - rtmhist_ndens = 2 - rtmhist_nhtfrq = 0,-24 + check_finidat_year_consistency = .false. + check_dynpft_consistency = .false. + create_crop_landunit = .false. EOF } @@ -369,6 +444,7 @@ case_build() { # Some user_nl settings won't be updated to *_in files under the run directory # Call preview_namelists to make sure *_in and user_nl files are consistent. + echo $'\n----- Preview namelists -----\n' ./preview_namelists fi diff --git a/share/build/buildlib.kokkos b/share/build/buildlib.kokkos index 1b08afa3dd63..035e86b0df2a 100755 --- a/share/build/buildlib.kokkos +++ b/share/build/buildlib.kokkos @@ -99,7 +99,7 @@ def buildlib(bldroot, installpath, case): gmake_cmd = case.get_value("GMAKE") gmake_j = case.get_value("GMAKE_J") - gen_makefile_cmd = f"cmake {kokkos_options} {cxx} -DCMAKE_INSTALL_PREFIX={installpath} {kokkos_dir}" + gen_makefile_cmd = f"cmake {kokkos_options} {cxx} -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX={installpath} {kokkos_dir}" # When later we use find_package to get kokkos in CMake, the folder # install_sharedpath/kokkos (which is bldroot here) gets picked over diff --git a/share/build/buildlib.spio b/share/build/buildlib.spio index d65d55479361..49b898202f72 100755 --- a/share/build/buildlib.spio +++ b/share/build/buildlib.spio @@ -106,6 +106,9 @@ def buildlib(bldroot, installpath, case): if "ADIOS2_ROOT" in os.environ: cmake_opts += "-DWITH_ADIOS2:BOOL=ON " + if "FROM_CREATE_TEST" in os.environ and os.environ["FROM_CREATE_TEST"] == "True": + cmake_opts += "-DADIOS_BP2NC_TEST:BOOL=ON " + if debug: cmake_opts += "-DPIO_ENABLE_LOGGING=ON " # Case changes for NetCDF/NETCDF forces us to do this. For other packages diff --git a/share/util/shr_pio_mod.F90 b/share/util/shr_pio_mod.F90 index ed7dfdaa2ed8..0935bce351a5 100644 --- a/share/util/shr_pio_mod.F90 +++ b/share/util/shr_pio_mod.F90 @@ -730,7 +730,8 @@ subroutine shr_pio_namelist_set(npes,mycomm, pio_stride, pio_root, pio_numiotask if(pio_stride == 1 .and. .not. pio_async_interface) then pio_root = 0 endif - if(pio_rearranger .ne. PIO_REARR_SUBSET .and. pio_rearranger .ne. PIO_REARR_BOX) then + if(pio_rearranger .ne. PIO_REARR_SUBSET .and. pio_rearranger .ne. PIO_REARR_BOX .and.& + pio_rearranger .ne. PIO_REARR_ANY) then write(shr_log_unit,*) 'pio_rearranger value, ',pio_rearranger,& ', not supported - using PIO_REARR_BOX' pio_rearranger = PIO_REARR_BOX diff --git a/tools/generate_domain_files/docs/index.md b/tools/generate_domain_files/docs/index.md new file mode 100644 index 000000000000..578582051403 --- /dev/null +++ b/tools/generate_domain_files/docs/index.md @@ -0,0 +1,47 @@ +# Generating Domain Files + +Domain files are needed at runtime by the coupler, data models, and land model. The land model uses the mask to determine where to run and the coupler use the land fraction to merge fluxes from multiple surface types to the atmosphere above them. + +Domain files are created from a conservative, monotone mapping file from the ocean grid (where the mask is defined) to the atmosphere grid. + +## Environment + +The new domain generation tool requires a few special packages, such as xarray, numba, and itertools. These are all included in the E3SM unified environment: + + +Alternatively, a simple conda environment can be created with the following command: + +```shell +conda create --name example_env --channel conda-forge xarray numpy numba scikit-learn netcdf4 +``` + +## Map File Generation + +The map file used to generate the domain files can be created a few different ways. For a typical E3SM configuration we recommend using a conservative, monotone map. Here is an example command that can be used to generate one (as of NCO version 5.2.2) + +```shell +ncremap -5 -a traave --src_grd=${OCN_GRID} --dst_grd=${ATM_GRID} --map_file=${MAP_FILE} +``` + +Note that existing ocean grid files can be found in the inputdata repository: `inputdata/ocn/mpas-o//` + +The atmosphere grid file should be on the "pg2" grid. These grid files are easily generated with three TempestRemap commands as follows: + +```shell +NE=30 +GenerateCSMesh --alt --res ${NE} --file ${GRID_FILE_PATH}/ne${NE}.g +GenerateVolumetricMesh --in ${GRID_FILE_PATH}/ne${NE}.g --out ${GRID_FILE_PATH}/ne${NE}pg2.g --np 2 --uniform +ConvertMeshToSCRIP --in ${GRID_FILE_PATH}/ne${NE}pg2.g --out ${GRID_FILE_PATH}/ne${NE}pg2_scrip.nc +``` + +For RRM grids the last two commands would be used on the exodus file produced by [SQuadGen](https://github.com/ClimateGlobalChange/squadgen) (See the [Adding Support for New Grids](https://docs.e3sm.org/user-guides/adding-grid-support-overview.md) tutorial for more information.). + +## Running the Domain Generation Tool + +Below is a typical example of how to invoke the domain generation tool from the command line: + +```shell +NE=30 +MAP_FILE=${MAP_FILE_ROOT}/map_oEC60to30v3_to_ne${NE}pg2_traave.20240313.nc +python generate_domain_files_E3SM.py -m ${MAP_FILE} -o oEC60to30v3 -l ne${NE}pg2 --date-stamp=9999 --output-root=${OUTPUT_ROOT} +``` diff --git a/tools/generate_domain_files/generate_domain_files_E3SM.py b/tools/generate_domain_files/generate_domain_files_E3SM.py new file mode 100644 index 000000000000..efaa7c60c159 --- /dev/null +++ b/tools/generate_domain_files/generate_domain_files_E3SM.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 +#--------------------------------------------------------------------------------------------------- +''' +This is a replacement for the legacy gen_domain tool created for CESM. +Most legacy functionality is reproduced, with the notable exception of +the pole point latitude adjustment needed for the CESM FV grid. + +Created April, 2024 by Walter Hannah (LLNL) +''' +#--------------------------------------------------------------------------------------------------- +''' +The map file used to generate the domain files can be created a few different ways. +For a typical E3SM configuration we recommend using a conservative, monotone map. +Here is an example command that can be used to generate one as of NCO version 5.2.2 + + ncremap -5 -a traave --src_grd=${OCN_GRID} --dst_grd=${ATM_GRID} --map_file=${MAP_FILE} + +''' +#--------------------------------------------------------------------------------------------------- +import datetime, os, numpy as np, xarray as xr, numba, itertools +user, host = os.getenv('USER'), os.getenv('HOST') +source_code_meta = 'generate_domain_E3SM.py' +#--------------------------------------------------------------------------------------------------- +class clr:END,RED,GREEN,MAGENTA,CYAN = '\033[0m','\033[31m','\033[32m','\033[35m','\033[36m' +#--------------------------------------------------------------------------------------------------- +usage = ''' +python generate_domain_files_E3SM.py -m + -o + -l + [--output-root ] + [--date-stamp ] + [--fminval ] + [--fmaxval ] + [--set-omask] + +Purpose: + For "bi-grid" configurations of E3SM (land grid is same as atmos): + Given a mapping file from the ocean grid (where the mask is defined) + to the atmosphere grid, this tool creates land and ocean domain files + needed by data model components (ex. datm, dlnd, docn) + + For "tri-grid" configurations of E3SM (land grid is different from atmos/ocn): + In addition to running this tool with the ocn->atm map as above, + a second iteration is needed with a similar ocn->lnd map. + +Environment + + This tool requires a few special packages, such as xarray, numba, and itertools. + These are all included in the E3SM unified environment: + https://e3sm.org/resources/tools/other-tools/e3sm-unified-environment/ + + Otherwise a simple conda environment can be created: + conda create --name example_env --channel conda-forge xarray numpy numba scikit-learn netcdf4 + +The following output domain files are created: + + domain.lnd._..nc + land domain file on the land/atmos grid with a land fraction + corresponding to (1-ocnfrac) mask mapped to the land grid + + domain.ocn._..nc + ocean domain on the land/atmos grid with an ocean fraction based + on the ocean grid mask mapped to the land/atmos grid for when + atm,lnd,ice,ocn are all on the same grid + (not compatible with MPAS sea-ice) + + domain.ocn...nc + ocean domain on the ocean grid +''' +from optparse import OptionParser +parser = OptionParser(usage=usage) +parser.add_option('-m', + dest='map_file', + default=None, + help='Input mapping file from ocean to atmosphere grid - '+\ + 'This is the primary source from which the ocean and'+\ + 'land domains will be determined') +parser.add_option('-l', + dest='lnd_grid', + default=None, + help='Output land grid name') +parser.add_option('-o', + dest='ocn_grid', + default=None, + help='Output ocean grid name') +parser.add_option('--output-root', + dest='output_root', + default='./', + help='Output path for domain files') +parser.add_option('--date-stamp', + dest='date_stamp', + default=None, + help='Creation date stamp for domain files') +parser.add_option('--fminval', + dest='fminval', + default=1e-3, + help='Minimum allowable land fraction (reset to 0 below fminval)') +parser.add_option('--fmaxval', + dest='fmaxval', + default=1, + help='Maximum allowable land fraction (reset to 1 above fmaxval)') +parser.add_option('--set-omask', + dest='set_omask', + default=False, + action='store_true', + help='If True then an ocean mask is not required and will '+\ + 'simply be set to a vector of 1\'s if mask_a is not '+\ + 'present in the input mapping file. If --set-omask is '+\ + 'omitted, then mask_a is required and an error will be '+\ + 'raised if it does not exist in the input mapping file.') +(opts, args) = parser.parse_args() +#--------------------------------------------------------------------------------------------------- +def main(): + global domain_a_grid_file, domain_b_grid_file, ocn_grid_file, atm_grid_file + #------------------------------------------------------------------------------- + # check for valid input arguments + + if opts.map_file is None: + raise ValueError(f'{clr.RED}input map file was not specified{clr.END}') + if opts.lnd_grid is None: + raise ValueError(f'{clr.RED}land grid name was not specified{clr.END}') + if opts.ocn_grid is None: + raise ValueError(f'{clr.RED}ocean grid name was not specified{clr.END}') + if not os.path.exists(opts.output_root) : + raise ValueError(f'{clr.RED}Output root path does not exist{clr.END}') + + #------------------------------------------------------------------------------- + # Set date stamp for file name + + if opts.date_stamp is None: + cdate = datetime.datetime.now().strftime('%Y%m%d') + else: + cdate = opts.date_stamp + + #------------------------------------------------------------------------------- + # specify output file names + + domain_file_ocn_on_ocn = f'{opts.output_root}/domain.ocn.{opts.ocn_grid}.{cdate}.nc' + domain_file_lnd_on_atm = f'{opts.output_root}/domain.lnd.{opts.lnd_grid}_{opts.ocn_grid}.{cdate}.nc' + domain_file_ocn_on_atm = f'{opts.output_root}/domain.ocn.{opts.lnd_grid}_{opts.ocn_grid}.{cdate}.nc' + + # cosmetic clean up of file names + domain_file_ocn_on_ocn = domain_file_ocn_on_ocn.replace('//','/') + domain_file_lnd_on_atm = domain_file_lnd_on_atm.replace('//','/') + domain_file_ocn_on_atm = domain_file_ocn_on_atm.replace('//','/') + + #----------------------------------------------------------------------------- + # print some informative stuff + + print(f''' + Input files and parameter values: + {clr.GREEN}map_file {clr.END}: {opts.map_file} + {clr.GREEN}lnd_grid {clr.END}: {opts.lnd_grid} + {clr.GREEN}ocn_grid {clr.END}: {opts.ocn_grid} + {clr.GREEN}fminval {clr.END}: {opts.fminval} + {clr.GREEN}fmaxval {clr.END}: {opts.fmaxval} + {clr.GREEN}set_omask {clr.END}: {opts.set_omask} + ''') + + #----------------------------------------------------------------------------- + # open map file as dataset + + ds = xr.open_dataset(opts.map_file) + + #----------------------------------------------------------------------------- + # read grid meta-data from map file + + if 'domain_a' in ds.attrs.keys(): domain_a_grid_file = ds.attrs['domain_a'] + if 'domain_b' in ds.attrs.keys(): domain_b_grid_file = ds.attrs['domain_b'] + + if 'grid_file_ocn' in ds.attrs.keys(): + ocn_grid_file = ds.attrs['grid_file_ocn'] + else: + ocn_grid_file = ds.attrs['grid_file_src'] + + if 'grid_file_ocn' in ds.attrs.keys(): + atm_grid_file = ds.attrs['grid_file_atm'] + else: + atm_grid_file = ds.attrs['grid_file_dst'] + + #----------------------------------------------------------------------------- + # print some useful information from the map file + + print(f''' + Grid information from map file: + {clr.CYAN}domain_a file {clr.END}: {domain_a_grid_file} + {clr.CYAN}domain_b file {clr.END}: {domain_b_grid_file} + {clr.CYAN}ocn_grid_file {clr.END}: {ocn_grid_file} + {clr.CYAN}atm_grid_file {clr.END}: {atm_grid_file} + {clr.CYAN}ocn grid size (n_a){clr.END}: {len(ds.n_a)} + {clr.CYAN}atm grid size (n_b){clr.END}: {len(ds.n_b)} + {clr.CYAN}sparse mat size (n_s){clr.END}: {len(ds.n_s)} + ''') + + #----------------------------------------------------------------------------- + # Create ocean domain on ocean grid (domain_file_ocn_on_ocn) + + # Get ocn mask on ocn grid + omask = get_mask(ds,opts,suffix='_a') + ofrac = xr.zeros_like(ds['area_a']) + + ds_out = xr.Dataset() + ds_out['xc'] = ds['xc_a'] .expand_dims(dim='nj').rename({'n_a':'ni'}) + ds_out['yc'] = ds['yc_a'] .expand_dims(dim='nj').rename({'n_a':'ni'}) + ds_out['xv'] = ds['xv_a'] .expand_dims(dim='nj').rename({'n_a':'ni','nv_a':'nv'}) + ds_out['yv'] = ds['yv_a'] .expand_dims(dim='nj').rename({'n_a':'ni','nv_a':'nv'}) + ds_out['area'] = ds['area_a'].expand_dims(dim='nj').rename({'n_a':'ni'}) + ds_out['frac'] = ofrac .expand_dims(dim='nj').rename({'n_a':'ni'}) + ds_out['mask'] = omask .expand_dims(dim='nj').rename({'n_a':'ni'}) + + add_metadata(ds_out) + + ds_out.to_netcdf(path=domain_file_ocn_on_ocn,mode='w') + + print(f'successfully created domain file: {clr.MAGENTA}{domain_file_ocn_on_ocn}{clr.END}') + + #----------------------------------------------------------------------------- + # Create land and ocean domains on atmosphere grid + + xc = ds['xc_b'] + yc = ds['yc_b'] + xv = ds['xv_b'] + yv = ds['yv_b'] + area = ds['area_b'] + + mask_a = get_mask(ds,opts,suffix='_a') + frac_a = xr.where( mask_a!=0, xr.ones_like(ds['area_a']), xr.zeros_like(ds['area_a']) ) + + # compute ocn fraction on atm grid + ofrac = compute_ofrac_on_atm( len(ds['n_s']), np.zeros(ds['area_b'].shape), + frac_a.values, ds['S'].values, + ds['row'].values-1, ds['col'].values-1 ) + ofrac = xr.DataArray(ofrac,dims=['n_b']) + + # lfrac is area frac of mask "_a" on grid "_b" or float(mask) + lfrac = xr.zeros_like(ds['area_b']) + + # convert to land fraction + lfrac_min = opts.fmaxval + lfrac_max = opts.fminval + omask = xr.ones_like(ds['area_b'],dtype=np.int32) + lmask = xr.zeros_like(ds['area_b'],dtype=np.int32) + lfrac = 1 - ofrac + lfrac_min = lfrac.min().values + lfrac_max = lfrac.max().values + lfrac = xr.where( lfrac>opts.fmaxval, 1, lfrac ) + lfrac = xr.where( lfrac