From b3bb4a7e144aab1e92a3abdffdf3fc772be9f38a Mon Sep 17 00:00:00 2001 From: Lukas Date: Wed, 6 Nov 2024 11:34:15 +0100 Subject: [PATCH 01/17] change authors name (#3806) --- esmvaltool/config-references.yml | 10 +++++----- esmvaltool/diag_scripts/monitor/multi_datasets.py | 6 +++++- .../recipes/monitor/recipe_monitor_with_refs.yml | 2 +- esmvaltool/recipes/recipe_shapeselect.yml | 2 +- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/esmvaltool/config-references.yml b/esmvaltool/config-references.yml index 199dc671e0..79a85c9866 100644 --- a/esmvaltool/config-references.yml +++ b/esmvaltool/config-references.yml @@ -336,6 +336,11 @@ authors: name: Lillis, Jon institute: MetOffice, UK orcid: + lindenlaub_lukas: + name: Lindenlaub, Lukas + institute: University of Bremen, Germany + orcid: https://orcid.org/0000-0001-6349-9118 + github: lukruh little_bill: name: Little, Bill institute: MetOffice, UK @@ -466,11 +471,6 @@ authors: rol_evert: name: Rol, Evert orcid: https://orcid.org/0000-0001-8357-4453 - ruhe_lukas: - name: Ruhe, Lukas - institute: University of Bremen, Germany - orcid: https://orcid.org/0000-0001-6349-9118 - github: lukruh russell_joellen: name: Russell, Joellen institute: Univ. of Arizona, USA diff --git a/esmvaltool/diag_scripts/monitor/multi_datasets.py b/esmvaltool/diag_scripts/monitor/multi_datasets.py index 70faee96c2..41f238a64e 100644 --- a/esmvaltool/diag_scripts/monitor/multi_datasets.py +++ b/esmvaltool/diag_scripts/monitor/multi_datasets.py @@ -2576,7 +2576,11 @@ def create_hovmoeller_time_vs_lat_or_lon_plot(self, datasets): # Provenance tracking provenance_record = { 'ancestors': ancestors, - 'authors': ['schlund_manuel', 'kraft_jeremy', 'ruhe_lukas'], + 'authors': [ + 'schlund_manuel', + 'kraft_jeremy', + 'lindenlaub_lukas' + ], 'caption': caption, 'plot_types': ['zonal'], 'long_names': [dataset['long_name']], diff --git a/esmvaltool/recipes/monitor/recipe_monitor_with_refs.yml b/esmvaltool/recipes/monitor/recipe_monitor_with_refs.yml index 48c5153287..4277313428 100644 --- a/esmvaltool/recipes/monitor/recipe_monitor_with_refs.yml +++ b/esmvaltool/recipes/monitor/recipe_monitor_with_refs.yml @@ -10,7 +10,7 @@ documentation: - heuer_helge - kraft_jeremy - kuehbacher_birgit - - ruhe_lukas + - lindenlaub_lukas - sarauer_ellen - winterstein_franziska maintainer: diff --git a/esmvaltool/recipes/recipe_shapeselect.yml b/esmvaltool/recipes/recipe_shapeselect.yml index ee56810f03..b463f09df8 100644 --- a/esmvaltool/recipes/recipe_shapeselect.yml +++ b/esmvaltool/recipes/recipe_shapeselect.yml @@ -11,7 +11,7 @@ documentation: - berg_peter maintainer: - - ruhe_lukas + - lindenlaub_lukas projects: - c3s-magic From 7d8d72c43b2c3cd80fd68d53eee7231ce589f210 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 11 Nov 2024 13:22:45 +0000 Subject: [PATCH 02/17] [Condalock] Update Linux condalock file (#3809) Co-authored-by: valeriupredoi --- conda-linux-64.lock | 145 ++++++++++++++++++++++---------------------- 1 file changed, 72 insertions(+), 73 deletions(-) diff --git a/conda-linux-64.lock b/conda-linux-64.lock index 7521c7f30c..a3ad9b680c 100644 --- a/conda-linux-64.lock +++ b/conda-linux-64.lock @@ -27,9 +27,9 @@ https://conda.anaconda.org/conda-forge/linux-64/binutils_impl_linux-64-2.43-h4bf https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2#fee5683a3f04bd15cbd8318b096a27ab https://conda.anaconda.org/conda-forge/linux-64/libgcc-14.2.0-h77fa898_1.conda#3cb76c3f10d3bc7f1105b2fc9db984df https://conda.anaconda.org/conda-forge/linux-64/aws-c-common-0.9.28-hb9d3cd8_0.conda#1b53af320b24547ce0fb8196d2604542 -https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.34.2-heb4867d_0.conda#2b780c0338fc0ffa678ac82c54af51fd +https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.34.3-heb4867d_0.conda#09a6c610d002e54e18353c06ef61a253 https://conda.anaconda.org/conda-forge/linux-64/libbrotlicommon-1.1.0-hb9d3cd8_2.conda#41b599ed2b02abcfdd84302bff174b23 -https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.6.3-h5888daf_0.conda#59f4c43bb1b5ef1c71946ff2cbf59524 +https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.6.4-h5888daf_0.conda#db833e03127376d461e1e13e76f09b6c https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-14.2.0-h69a702a_1.conda#e39480b9ca41323497b05492a63bc35b https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-14.2.0-hd5240d6_1.conda#9822b874ea29af082e5d36098d25427d https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-14.2.0-hc0a3c3a_1.conda#234a5554c53625688d51062645337328 @@ -49,7 +49,7 @@ https://conda.anaconda.org/conda-forge/linux-64/aws-c-sdkutils-0.1.19-h756ea98_3 https://conda.anaconda.org/conda-forge/linux-64/aws-checksums-0.1.18-h756ea98_11.conda#eadcc12bedac44f13223a2909c0e5bcc https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h4bc722e_7.conda#62ee74e96c5ebb0af99386de58cf9553 https://conda.anaconda.org/conda-forge/linux-64/dav1d-1.2.1-hd590300_0.conda#418c6ca5929a611cbd69204907a83995 -https://conda.anaconda.org/conda-forge/linux-64/expat-2.6.3-h5888daf_0.conda#6595440079bed734b113de44ffd3cd0a +https://conda.anaconda.org/conda-forge/linux-64/expat-2.6.4-h5888daf_0.conda#1d6afef758879ef5ee78127eb4cd2c4a https://conda.anaconda.org/conda-forge/linux-64/fribidi-1.0.10-h36c2ea0_0.tar.bz2#ac7bc6a654f8f41b352b38f4051135f8 https://conda.anaconda.org/conda-forge/linux-64/gettext-tools-0.22.5-he02047a_3.conda#fcd2016d1d299f654f81021e27496818 https://conda.anaconda.org/conda-forge/linux-64/gflags-2.2.2-h5888daf_1005.conda#d411fc29e338efb48c5fd4576d71d881 @@ -128,9 +128,11 @@ https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-14.2.0-h69a702a_1 https://conda.anaconda.org/conda-forge/linux-64/libhwy-1.1.0-h00ab1b0_0.conda#88928158ccfe797eac29ef5e03f7d23d https://conda.anaconda.org/conda-forge/linux-64/libllvm14-14.0.6-hcd5def8_4.conda#73301c133ded2bf71906aa2104edae8b https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.64.0-h161d5f1_0.conda#19e57602824042dfd0446292ef90488b +https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.28-pthreads_h94d23a6_1.conda#62857b389e42b36b686331bec0922050 +https://conda.anaconda.org/conda-forge/linux-64/libopenblas-ilp64-0.3.28-pthreads_h3e26593_1.conda#9d5c316d93ee4c5effd9afda8e8af823 https://conda.anaconda.org/conda-forge/linux-64/libthrift-0.20.0-h0e7cc3e_1.conda#d0ed81c4591775b70384f4cc78e05cd1 https://conda.anaconda.org/conda-forge/linux-64/libunwind-1.6.2-h9c3ff4c_0.tar.bz2#a730b2badd586580c5752cc73842e068 -https://conda.anaconda.org/conda-forge/linux-64/libzip-1.11.1-hf83b1b0_0.conda#e8536ec89df2aec5f65fefcf4ccd58ba +https://conda.anaconda.org/conda-forge/linux-64/libzip-1.11.2-h6991a6a_0.conda#a7b27c075c9b7f459f1c022090697cba https://conda.anaconda.org/conda-forge/linux-64/libzopfli-1.0.3-h9c3ff4c_0.tar.bz2#c66fe2d123249af7651ebde8984c51c2 https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.9.4-hcb278e6_0.conda#318b08df404f9c9be5712aaa5a6f0bb0 https://conda.anaconda.org/conda-forge/linux-64/mbedtls-3.5.1-h59595ed_0.conda#a7b444a6e008b804b35521895e3440e2 @@ -165,12 +167,11 @@ https://conda.anaconda.org/conda-forge/linux-64/hdfeos2-2.20-h3e53b52_1004.conda https://conda.anaconda.org/conda-forge/linux-64/krb5-1.21.3-h659f571_0.conda#3f43953b7d3fb3aaa1d0d0723d91e368 https://conda.anaconda.org/conda-forge/linux-64/libasprintf-devel-0.22.5-he8f35ee_3.conda#1091193789bb830127ed067a9e01ac57 https://conda.anaconda.org/conda-forge/linux-64/libavif16-1.1.1-h1909e37_2.conda#21e468ed3786ebcb2124b123aa2484b7 +https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-25_linux64_openblas.conda#8ea26d42ca88ec5258802715fe1ee10b https://conda.anaconda.org/conda-forge/linux-64/libgit2-1.8.4-hd24f944_0.conda#94887b4deb460378a34e1533beaacfd5 https://conda.anaconda.org/conda-forge/linux-64/libglib-2.82.2-h2ff4ddf_0.conda#13e8e54035ddd2b91875ba399f0f7c04 https://conda.anaconda.org/conda-forge/linux-64/libjxl-0.11.0-hdb8da77_2.conda#9c4554fafc94db681543804037e65de2 https://conda.anaconda.org/conda-forge/linux-64/libkml-1.3.0-hf539b9f_1021.conda#e8c7620cc49de0c6a2349b6dd6e39beb -https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.28-pthreads_h94d23a6_0.conda#9ebc9aedafaa2515ab247ff6bb509458 -https://conda.anaconda.org/conda-forge/linux-64/libopenblas-ilp64-0.3.28-pthreads_h3e26593_0.conda#2bd7dc48907a3b6bf766ed87867f3459 https://conda.anaconda.org/conda-forge/linux-64/libprotobuf-4.25.3-hd5b35b9_1.conda#06def97690ef90781a91b786cb48a0a9 https://conda.anaconda.org/conda-forge/linux-64/libre2-11-2023.09.01-h5a48ba9_2.conda#41c69fba59d495e8cf5ffda48a607e35 https://conda.anaconda.org/conda-forge/linux-64/librttopo-1.1.0-hc670b87_16.conda#3d9f3a2e5d7213c34997e4464d2f938c @@ -179,6 +180,7 @@ https://conda.anaconda.org/conda-forge/linux-64/libxgboost-2.1.2-cuda118_h09a87b https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.13.4-hb346dea_2.conda#69b90b70c434b916abf5a1d5ee5d55fb https://conda.anaconda.org/conda-forge/linux-64/minizip-4.0.7-h401b404_0.conda#4474532a312b2245c5c77f1176989b46 https://conda.anaconda.org/conda-forge/linux-64/mpfr-4.2.1-h90cbb55_3.conda#2eeb50cab6652538eee8fc0bc3340c81 +https://conda.anaconda.org/conda-forge/linux-64/openblas-ilp64-0.3.28-pthreads_h3d04fff_1.conda#fdaa89df7b34f5c904f8f1348e5a62a5 https://conda.anaconda.org/conda-forge/linux-64/python-3.12.7-hc5c86c4_0_cpython.conda#0515111a9cdf69f83278f7c197db9807 https://conda.anaconda.org/conda-forge/linux-64/s2geometry-0.10.0-h8413349_4.conda#d19f88cf8812836e6a4a2a7902ed0e77 https://conda.anaconda.org/conda-forge/linux-64/spdlog-1.14.1-hed91bc2_1.conda#909188c8979846bac8e586908cf1ca6a @@ -191,7 +193,6 @@ https://conda.anaconda.org/conda-forge/linux-64/xorg-libxt-1.3.0-hb9d3cd8_2.cond https://conda.anaconda.org/conda-forge/noarch/affine-2.4.0-pyhd8ed1ab_0.conda#ae5f4ad87126c55ba3f690ef07f81d64 https://conda.anaconda.org/conda-forge/noarch/aiohappyeyeballs-2.4.3-pyhd8ed1ab_0.conda#ec763b0a58960558ca0ad7255a51a237 https://conda.anaconda.org/conda-forge/noarch/alabaster-1.0.0-pyhd8ed1ab_0.conda#7d78a232029458d0077ede6cda30ed0c -https://conda.anaconda.org/conda-forge/noarch/antlr-python-runtime-4.11.1-pyhd8ed1ab_0.tar.bz2#15109c4977d39ad7aa3423f57243e286 https://conda.anaconda.org/conda-forge/noarch/asciitree-0.3.3-py_2.tar.bz2#c0481c9de49f040272556e2cedf42816 https://conda.anaconda.org/conda-forge/linux-64/astroid-3.3.5-py312h7900ff3_0.conda#e1ed4d572a4a16b97368ab00fd646487 https://conda.anaconda.org/conda-forge/linux-64/atk-1.0-2.38.0-h04ea711_2.conda#f730d54ba9cd543666d7220c9f7ed563 @@ -235,7 +236,7 @@ https://conda.anaconda.org/conda-forge/noarch/geographiclib-2.0-pyhd8ed1ab_0.tar https://conda.anaconda.org/conda-forge/linux-64/gettext-0.22.5-he02047a_3.conda#c7f243bbaea97cd6ea1edd693270100e https://conda.anaconda.org/conda-forge/linux-64/gts-0.7.6-h977cf35_4.conda#4d8df0b0db060d33c9a702ada998a8fe https://conda.anaconda.org/conda-forge/noarch/hpack-4.0.0-pyh9f0ad1d_0.tar.bz2#914d6646c4dbb1fd3ff539830a12fd71 -https://conda.anaconda.org/conda-forge/noarch/humanfriendly-10.0-pyhd8ed1ab_6.conda#2ed1fe4b9079da97c44cfe9c2e5078fd +https://conda.anaconda.org/conda-forge/noarch/humanfriendly-10.0-pyhd81877a_7.conda#74fbff91ca7c1b9a36b15903f2242f86 https://conda.anaconda.org/conda-forge/noarch/hyperframe-6.0.1-pyhd8ed1ab_0.tar.bz2#9f765cbfab6870c8435b9eefecd7a1f4 https://conda.anaconda.org/conda-forge/noarch/idna-3.10-pyhd8ed1ab_0.conda#7ba2ede0e7c795ff95088daf0dc59753 https://conda.anaconda.org/conda-forge/noarch/imagesize-1.4.1-pyhd8ed1ab_0.tar.bz2#7de5386c8fea29e76b303f37dde4c352 @@ -246,12 +247,13 @@ https://conda.anaconda.org/conda-forge/linux-64/kiwisolver-1.4.7-py312h68727a3_0 https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.16-hb7c19ff_0.conda#51bb7010fc86f70eee639b4bb7a894f5 https://conda.anaconda.org/conda-forge/noarch/legacy-cgi-2.6.1-pyh5b84bb0_3.conda#f258b7f54b5d9ddd02441f10c4dca2ac https://conda.anaconda.org/conda-forge/linux-64/libarchive-3.7.4-hfca40fe_0.conda#32ddb97f897740641d8d46a829ce1704 -https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-25_linux64_openblas.conda#8ea26d42ca88ec5258802715fe1ee10b +https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-25_linux64_openblas.conda#5dbd1b0fc0d01ec5e0e1fbe667281a11 https://conda.anaconda.org/conda-forge/linux-64/libcurl-8.9.1-hdb1bdb2_0.conda#7da1d242ca3591e174a3c7d82230d3c0 https://conda.anaconda.org/conda-forge/linux-64/libgd-2.3.3-hd3e95f3_10.conda#30ee3a29c84cf7b842a8c5828c4b7c13 https://conda.anaconda.org/conda-forge/linux-64/libglu-9.0.0-ha6d2627_1004.conda#df069bea331c8486ac21814969301c1f https://conda.anaconda.org/conda-forge/linux-64/libheif-1.18.2-gpl_hffcb242_100.conda#76ac2c07b62d45c192940f010eea11fa -https://conda.anaconda.org/conda-forge/linux-64/libhwloc-2.11.1-default_hecaa2ac_1000.conda#f54aeebefb5c5ff84eca4fb05ca8aa3a +https://conda.anaconda.org/conda-forge/linux-64/libhwloc-2.11.2-default_he43201b_1000.conda#36247217c4e1018085bd9db41eb3526a +https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-25_linux64_openblas.conda#4dc03a53fc69371a6158d0ed37214cd3 https://conda.anaconda.org/conda-forge/linux-64/libwebp-1.4.0-h2c329e2_0.conda#80030debaa84cfc31755d53742df3ca6 https://conda.anaconda.org/conda-forge/linux-64/libxslt-1.1.39-h76b75d6_0.conda#e71f31f8cfb0a91439f2086fc8aa0461 https://conda.anaconda.org/conda-forge/linux-64/llvmlite-0.43.0-py312h374181b_1.conda#ed6ead7e9ab9469629c6cfb363b5c6e2 @@ -266,7 +268,6 @@ https://conda.anaconda.org/conda-forge/noarch/munkres-1.1.4-pyh9f0ad1d_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/mypy_extensions-1.0.0-pyha770c72_0.conda#4eccaeba205f0aed9ac3a9ea58568ca3 https://conda.anaconda.org/conda-forge/noarch/natsort-8.4.0-pyhd8ed1ab_0.conda#70959cd1db3cf77b2a27a0836cfd08a7 https://conda.anaconda.org/conda-forge/noarch/networkx-3.4.2-pyhd8ed1ab_1.conda#1d4c088869f206413c59acdd309908b7 -https://conda.anaconda.org/conda-forge/linux-64/openblas-ilp64-0.3.28-pthreads_h3d04fff_0.conda#eb2736b14329cf5650917caa43a549c6 https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.2-h488ebb8_0.conda#7f2e286780f072ed750df46dc2631138 https://conda.anaconda.org/conda-forge/linux-64/orc-2.0.2-h669347b_0.conda#1e6c10f7d749a490612404efeb179eb8 https://conda.anaconda.org/conda-forge/noarch/packaging-24.1-pyhd8ed1ab_0.conda#cbe1bb1f21567018ce595d9c2be0f0db @@ -290,7 +291,7 @@ https://conda.anaconda.org/conda-forge/linux-64/python-xxhash-3.5.0-py312h66e93f https://conda.anaconda.org/conda-forge/noarch/pytz-2024.2-pyhd8ed1ab_0.conda#260009d03c9d5c0f111904d851f053dc https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.2-py312h66e93f0_1.conda#549e5930e768548a89c23f595dac5a95 https://conda.anaconda.org/conda-forge/linux-64/re2-2023.09.01-h7f4b329_2.conda#8f70e36268dea8eb666ef14c29bd3cda -https://conda.anaconda.org/conda-forge/linux-64/rpds-py-0.20.0-py312h12e396e_1.conda#9ae193ac9c1ead5024d5a4ee0024e9a6 +https://conda.anaconda.org/conda-forge/linux-64/rpds-py-0.21.0-py312h12e396e_0.conda#37f4ad7cb4214c799f32e5f411c6c69f https://conda.anaconda.org/conda-forge/linux-64/ruamel.yaml.clib-0.2.8-py312h66e93f0_1.conda#532c3e5d0280be4fea52396ec1fa7d5d https://conda.anaconda.org/conda-forge/noarch/semver-3.0.2-pyhd8ed1ab_0.conda#5efb3fccda53974aed800b6d575f72ed https://conda.anaconda.org/conda-forge/noarch/setoptconf-tmp-0.3.1-pyhd8ed1ab_0.tar.bz2#af3e36d4effb85b9b9f93cd1db0963df @@ -318,7 +319,7 @@ https://conda.anaconda.org/conda-forge/linux-64/ujson-5.10.0-py312h2ec8cdc_1.con https://conda.anaconda.org/conda-forge/linux-64/unicodedata2-15.1.0-py312h66e93f0_1.conda#588486a61153f94c7c13816f7069e440 https://conda.anaconda.org/conda-forge/noarch/untokenize-0.1.1-pyhd8ed1ab_1.conda#6042b782b893029aa40335782584a092 https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-pyhd8ed1ab_2.conda#daf5160ff9cde3a468556965329085b9 -https://conda.anaconda.org/conda-forge/noarch/wheel-0.44.0-pyhd8ed1ab_0.conda#d44e3b085abcaef02983c6305b84b584 +https://conda.anaconda.org/conda-forge/noarch/wheel-0.45.0-pyhd8ed1ab_0.conda#f9751d7c71df27b2d29f5cab3378982e https://conda.anaconda.org/conda-forge/noarch/xlsxwriter-3.2.0-pyhd8ed1ab_0.conda#a1f7264726115a2f8eac9773b1f27eba https://conda.anaconda.org/conda-forge/linux-64/xorg-libxi-1.8.2-hb9d3cd8_0.conda#17dcc85db3c7886650b8908b183d6876 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxmu-1.2.1-hb9d3cd8_1.conda#f35a9a2da717ade815ffa70c0e8bdfbd @@ -326,12 +327,13 @@ https://conda.anaconda.org/conda-forge/noarch/xyzservices-2024.9.0-pyhd8ed1ab_0. https://conda.anaconda.org/conda-forge/noarch/yapf-0.32.0-pyhd8ed1ab_0.tar.bz2#177cba0b4bdfacad5c5fbb0ed31504c4 https://conda.anaconda.org/conda-forge/linux-64/zeromq-4.3.5-h3b0a872_6.conda#113506c8d2d558e733f5c38f6bf08c50 https://conda.anaconda.org/conda-forge/noarch/zict-3.0.0-pyhd8ed1ab_0.conda#cf30c2c15b82aacb07f9c09e28ff2275 -https://conda.anaconda.org/conda-forge/noarch/zipp-3.20.2-pyhd8ed1ab_0.conda#4daaed111c05672ae669f7036ee5bba3 +https://conda.anaconda.org/conda-forge/noarch/zipp-3.21.0-pyhd8ed1ab_0.conda#fee389bf8a4843bd7a2248ce11b7f188 https://conda.anaconda.org/conda-forge/noarch/accessible-pygments-0.0.5-pyhd8ed1ab_0.conda#1bb1ef9806a9a20872434f58b3e7fc1a https://conda.anaconda.org/conda-forge/noarch/aiosignal-1.3.1-pyhd8ed1ab_0.tar.bz2#d1e1eb7e21a9e2c74279d87dafb68156 +https://conda.anaconda.org/conda-forge/linux-64/arpack-3.9.1-nompi_h77f6705_101.conda#ff39030debb47f6b53b45bada38e0903 https://conda.anaconda.org/conda-forge/linux-64/aws-c-s3-0.6.5-hbaf354b_4.conda#2cefeb144de7712995d1b52cc6a3864c https://conda.anaconda.org/conda-forge/linux-64/azure-core-cpp-1.13.0-h935415a_0.conda#debd1677c2fea41eb2233a260f48a298 -https://conda.anaconda.org/conda-forge/noarch/babel-2.14.0-pyhd8ed1ab_0.conda#9669586875baeced8fc30c0826c3270e +https://conda.anaconda.org/conda-forge/noarch/babel-2.16.0-pyhd8ed1ab_0.conda#6d4e9ecca8d88977147e109fc7053184 https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.12.3-pyha770c72_0.conda#332493000404d8411859539a5a630865 https://conda.anaconda.org/conda-forge/noarch/bleach-6.2.0-pyhd8ed1ab_0.conda#461bcfab8e65c166e297222ae919a2d4 https://conda.anaconda.org/conda-forge/linux-64/cffi-1.17.1-py312h06ac9bb_0.conda#a861504bbea4161a9170b85d4d2be840 @@ -350,6 +352,7 @@ https://conda.anaconda.org/conda-forge/linux-64/freeglut-3.2.2-ha6d2627_3.conda# https://conda.anaconda.org/conda-forge/noarch/geopy-2.4.1-pyhd8ed1ab_1.conda#358c17429c97883b2cb9ab5f64bc161b https://conda.anaconda.org/conda-forge/linux-64/git-2.46.0-pl5321hb5640b7_0.conda#825d146359bc8b85083d92259d0a0e1b https://conda.anaconda.org/conda-forge/noarch/gitdb-4.0.11-pyhd8ed1ab_0.conda#623b19f616f2ca0c261441067e18ae40 +https://conda.anaconda.org/conda-forge/linux-64/gsl-2.7-he838d99_0.tar.bz2#fec079ba39c9cca093bf4c00001825de https://conda.anaconda.org/conda-forge/noarch/h2-4.1.0-pyhd8ed1ab_0.tar.bz2#b748fbf7060927a6e82df7cb5ee8f097 https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-9.0.0-hda332d3_1.conda#76b32dcf243444aea9c6b804bcfa40b8 https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.14.3-nompi_hdf9ad27_105.conda#7e1729554e209627636a0f6fabcdd115 @@ -361,13 +364,12 @@ https://conda.anaconda.org/conda-forge/noarch/joblib-1.4.2-pyhd8ed1ab_0.conda#25 https://conda.anaconda.org/conda-forge/noarch/jupyter_core-5.7.2-pyh31011fe_1.conda#0a2980dada0dd7fd0998f0342308b1b1 https://conda.anaconda.org/conda-forge/noarch/jupyterlab_pygments-0.3.0-pyhd8ed1ab_1.conda#afcd1b53bcac8844540358e33f33d28f https://conda.anaconda.org/conda-forge/noarch/latexcodec-2.0.1-pyh9f0ad1d_0.tar.bz2#8d67904973263afd2985ba56aa2d6bb4 -https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-25_linux64_openblas.conda#5dbd1b0fc0d01ec5e0e1fbe667281a11 https://conda.anaconda.org/conda-forge/linux-64/libgrpc-1.62.2-h15f2491_0.conda#8dabe607748cb3d7002ad73cd06f1325 -https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-25_linux64_openblas.conda#4dc03a53fc69371a6158d0ed37214cd3 https://conda.anaconda.org/conda-forge/noarch/logilab-common-1.7.3-py_0.tar.bz2#6eafcdf39a7eb90b6d951cfff59e8d3b https://conda.anaconda.org/conda-forge/linux-64/lxml-5.3.0-py312he28fd5a_2.conda#3acf38086326f49afed094df4ba7c9d9 https://conda.anaconda.org/conda-forge/noarch/nested-lookup-0.2.25-pyhd8ed1ab_1.tar.bz2#2f59daeb14581d41b1e2dda0895933b2 https://conda.anaconda.org/conda-forge/noarch/nodeenv-1.9.1-pyhd8ed1ab_0.conda#dfe0528d0f1c16c1f7c528ea5536ab30 +https://conda.anaconda.org/conda-forge/linux-64/numpy-1.26.4-py312heda63a1_0.conda#d8285bea2a350f63fab23bf460221f3f https://conda.anaconda.org/conda-forge/linux-64/openldap-2.6.8-hedd0468_0.conda#dcd0ed5147d8876b0848a552b416ce76 https://conda.anaconda.org/conda-forge/linux-64/openpyxl-3.1.5-py312h710cb58_1.conda#69a8838436435f59d72ddcb8dfd24a28 https://conda.anaconda.org/conda-forge/noarch/partd-1.4.2-pyhd8ed1ab_0.conda#0badf9c54e24cecfb0ad2f99d680c163 @@ -380,16 +382,16 @@ https://conda.anaconda.org/conda-forge/noarch/pydocstyle-6.3.0-pyhd8ed1ab_0.cond https://conda.anaconda.org/conda-forge/noarch/pyproject_hooks-1.2.0-pyh7850678_0.conda#5003da197661e40a2509e9c4651f1eea https://conda.anaconda.org/conda-forge/noarch/pytest-8.3.3-pyhd8ed1ab_0.conda#c03d61f31f38fdb9facf70c29958bf7a https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0-pyhd8ed1ab_0.conda#2cf4264fffb9e6eff6031c5b6884d61c -https://conda.anaconda.org/conda-forge/noarch/python-utils-3.8.2-pyhd8ed1ab_0.conda#89703b4f38bd1c0353881f085bc8fdaa +https://conda.anaconda.org/conda-forge/noarch/python-utils-3.9.0-pyhff2d567_0.conda#ae8d4e318695c0d3e3464ed95cc8b385 https://conda.anaconda.org/conda-forge/linux-64/pyzmq-26.2.0-py312hbf22597_3.conda#746ce19f0829ec3e19c93007b1a224d3 https://conda.anaconda.org/conda-forge/noarch/rdflib-7.1.1-pyh0610db2_0.conda#325219de79481bcf5b6446d327e3d492 https://conda.anaconda.org/conda-forge/noarch/referencing-0.35.1-pyhd8ed1ab_0.conda#0fc8b52192a8898627c3efae1003e9f6 https://conda.anaconda.org/conda-forge/noarch/requirements-detector-1.3.1-pyhd8ed1ab_0.conda#f921ea6a1138cc7edee77de8ed12b226 https://conda.anaconda.org/conda-forge/noarch/retrying-1.3.3-pyhd8ed1ab_3.conda#1f7482562f2082f1b2abf8a3e2a41b63 https://conda.anaconda.org/conda-forge/linux-64/ruamel.yaml-0.18.6-py312h66e93f0_1.conda#28ed869ade5601ee374934a31c9d628e -https://conda.anaconda.org/conda-forge/linux-64/tbb-2021.13.0-h84d6215_0.conda#ee6f7fd1e76061ef1fa307d41fa86a96 +https://conda.anaconda.org/conda-forge/linux-64/tbb-2022.0.0-hceb3a55_0.conda#79f0161f3ca73804315ca980f65d9c60 https://conda.anaconda.org/conda-forge/noarch/tinycss2-1.4.0-pyhd8ed1ab_0.conda#f1acf5fdefa8300de697982bcb1761c9 -https://conda.anaconda.org/conda-forge/noarch/tqdm-4.66.6-pyhd8ed1ab_0.conda#92718e1f892e1e4623dcc59b9f9c4e55 +https://conda.anaconda.org/conda-forge/noarch/tqdm-4.67.0-pyhd8ed1ab_0.conda#196a9e6ab4e036ceafa516ea036619b0 https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.12.2-hd8ed1ab_0.conda#52d648bd608f5737b123f510bb5514b5 https://conda.anaconda.org/conda-forge/noarch/url-normalize-1.4.3-pyhd8ed1ab_0.tar.bz2#7c4076e494f0efe76705154ac9302ba6 https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.27.1-pyhd8ed1ab_0.conda#dae21509d62aa7bf676279ced3edcb3f @@ -399,18 +401,22 @@ https://conda.anaconda.org/conda-forge/noarch/yamale-5.2.1-pyhca7485f_0.conda#c0 https://conda.anaconda.org/conda-forge/noarch/yamllint-1.35.1-pyhd8ed1ab_0.conda#a1240b99a7ccd953879dc63111823986 https://conda.anaconda.org/conda-forge/linux-64/yarl-1.16.0-py312h66e93f0_0.conda#c3f4a6b56026c22319bf31514662b283 https://conda.anaconda.org/conda-forge/linux-64/aiohttp-3.10.10-py312h178313f_0.conda#d2f9e490ab2eae3e661b281346618a82 -https://conda.anaconda.org/conda-forge/linux-64/arpack-3.9.1-nompi_h77f6705_101.conda#ff39030debb47f6b53b45bada38e0903 https://conda.anaconda.org/conda-forge/linux-64/aws-crt-cpp-0.28.2-h6c0439f_6.conda#4e472c316d08af60faeb71f86d7563e1 https://conda.anaconda.org/conda-forge/linux-64/azure-identity-cpp-1.8.0-hd126650_2.conda#36df3cf05459de5d0a41c77c4329634b https://conda.anaconda.org/conda-forge/linux-64/azure-storage-common-cpp-12.7.0-h10ac4d7_1.conda#ab6d507ad16dbe2157920451d662e4a1 https://conda.anaconda.org/conda-forge/noarch/cattrs-24.1.2-pyhd8ed1ab_0.conda#ac582de2324988b79870b50c89c91c75 +https://conda.anaconda.org/conda-forge/linux-64/cftime-1.6.4-py312hc0a28a1_1.conda#990033147b0a998e756eaaed6b28f48d +https://conda.anaconda.org/conda-forge/noarch/colorspacious-1.1.2-pyh24bf2e0_0.tar.bz2#b73afa0d009a51cabd3ec99c4d2ef4f3 +https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.3.0-py312h68727a3_2.conda#ff28f374b31937c048107521c814791e https://conda.anaconda.org/conda-forge/linux-64/cryptography-43.0.3-py312hda17c39_0.conda#2abada8c216dd6e32514535a3fa245d4 +https://conda.anaconda.org/conda-forge/noarch/eofs-1.4.1-pyhd8ed1ab_1.conda#5fc43108dee4106f23050acc7a101233 https://conda.anaconda.org/conda-forge/noarch/flake8-polyfill-1.0.2-py_0.tar.bz2#a53db35e3d07f0af2eccd59c2a00bffe https://conda.anaconda.org/conda-forge/noarch/funcargparse-0.2.5-pyhd8ed1ab_0.tar.bz2#e557b70d736251fa0bbb7c4497852a92 https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.3-hf7fa9e8_2.conda#1d6bdc6b2c62c8cc90c67b50142d7b7f https://conda.anaconda.org/conda-forge/noarch/gitpython-3.1.43-pyhd8ed1ab_0.conda#0b2154c1818111e17381b1df5b4b0176 -https://conda.anaconda.org/conda-forge/linux-64/gsl-2.7-he838d99_0.tar.bz2#fec079ba39c9cca093bf4c00001825de https://conda.anaconda.org/conda-forge/linux-64/hdfeos5-5.1.16-hf1a501a_15.conda#d2e16a32f41d67c7d280da11b2846328 +https://conda.anaconda.org/conda-forge/linux-64/imagecodecs-2024.6.1-py312h6d9a048_4.conda#a810fadedc4edc06b4282d1222467837 +https://conda.anaconda.org/conda-forge/noarch/imageio-2.36.0-pyh12aca89_1.conda#36349844ff73fcd0140ee7f30745f0bf https://conda.anaconda.org/conda-forge/noarch/importlib_metadata-7.2.1-hd8ed1ab_0.conda#d6c936d009aa63e5f82d216c95cdcaee https://conda.anaconda.org/conda-forge/linux-64/jasper-4.2.4-h536e39c_0.conda#9518ab7016cf4564778aef08b6bd8792 https://conda.anaconda.org/conda-forge/noarch/jsonschema-specifications-2024.10.1-pyhd8ed1ab_0.conda#720745920222587ef942acfbc578b584 @@ -421,67 +427,72 @@ https://conda.anaconda.org/conda-forge/linux-64/libgoogle-cloud-2.28.0-h26d7fe4_ https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.9.2-nompi_h135f659_114.conda#a908e463c710bd6b10a9eaa89fdf003c https://conda.anaconda.org/conda-forge/linux-64/libpq-17.0-h04577a9_4.conda#392cae2a58fbcb9db8c2147c6d6d1620 https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.1.0-h15fa968_9.conda#4957a903bd6a68cc2e53e47476f9c6f4 -https://conda.anaconda.org/conda-forge/linux-64/numpy-1.26.4-py312heda63a1_0.conda#d8285bea2a350f63fab23bf460221f3f +https://conda.anaconda.org/conda-forge/noarch/magics-python-1.5.8-pyhd8ed1ab_1.conda#3fd7e3db129f12362642108f23fde521 +https://conda.anaconda.org/conda-forge/linux-64/numba-0.60.0-py312h83e6fd3_0.conda#e064ca33edf91ac117236c4b5dee207a +https://conda.anaconda.org/conda-forge/linux-64/numcodecs-0.13.1-py312hf9745cd_0.conda#33c27209bfd7af6766211facd24839ce +https://conda.anaconda.org/conda-forge/linux-64/pandas-2.1.4-py312hfb8ada1_0.conda#d0745ae74c2b26571b692ddde112eebb https://conda.anaconda.org/conda-forge/linux-64/pango-1.54.0-h4c5309f_1.conda#7df02e445367703cd87a574046e3a6f0 +https://conda.anaconda.org/conda-forge/noarch/patsy-0.5.6-pyhd8ed1ab_0.conda#a5b55d1cb110cdcedc748b5c3e16e687 https://conda.anaconda.org/conda-forge/noarch/progressbar2-4.5.0-pyhd8ed1ab_0.conda#6f9eb38d0a87898cf5a7c91adaccd691 https://conda.anaconda.org/conda-forge/noarch/pybtex-0.24.0-pyhd8ed1ab_2.tar.bz2#2099b86a7399c44c0c61cdb6de6915ba https://conda.anaconda.org/conda-forge/noarch/pylint-3.3.1-pyhd8ed1ab_0.conda#2a3426f75e2172c932131f4e3d51bcf4 https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.6.1-py312h9211aeb_9.conda#173afeb0d112c854fd1a9fcac4b5cce3 +https://conda.anaconda.org/conda-forge/linux-64/pys2index-0.1.5-py312hfb10629_0.conda#325cc5f0e0dc36562f3de2a4dbded572 https://conda.anaconda.org/conda-forge/noarch/pytest-cov-6.0.0-pyhd8ed1ab_0.conda#cb8a11b6d209e3d85e5094bdbd9ebd9c https://conda.anaconda.org/conda-forge/noarch/pytest-env-1.1.5-pyhd8ed1ab_0.conda#ecd5e850bcd3eca02143e7df030ee50f https://conda.anaconda.org/conda-forge/noarch/pytest-metadata-3.1.1-pyhd8ed1ab_0.conda#52b91ecba854d55b28ad916a8b10da24 https://conda.anaconda.org/conda-forge/noarch/pytest-mock-3.14.0-pyhd8ed1ab_0.conda#4b9b5e086812283c052a9105ab1e254e https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.6.1-pyhd8ed1ab_0.conda#b39568655c127a9c4a44d178ac99b6d0 https://conda.anaconda.org/conda-forge/noarch/python-build-1.2.2.post1-pyhff2d567_0.conda#bd5ae3c630d5eed353badb091fd3e603 +https://conda.anaconda.org/conda-forge/linux-64/pywavelets-1.7.0-py312hc0a28a1_2.conda#8300d634adec4a6aed35a87e90e9cb07 +https://conda.anaconda.org/conda-forge/linux-64/scipy-1.14.1-py312h62794b6_1.conda#b43233a9e2f62fb94affe5607ea79473 +https://conda.anaconda.org/conda-forge/linux-64/shapely-2.0.6-py312h6cab151_1.conda#5be02e05e1adaa42826cc6800ce399bc +https://conda.anaconda.org/conda-forge/noarch/snuggs-1.4.7-pyhd8ed1ab_1.conda#5abeaa41ec50d4d1421a8bc8fbc93054 https://conda.anaconda.org/conda-forge/linux-64/suitesparse-7.8.3-hb42a789_0.conda#216922e19843f5662a2b260f905640cb https://conda.anaconda.org/conda-forge/linux-64/ukkonen-1.0.1-py312h68727a3_5.conda#f9664ee31aed96c85b7319ab0a693341 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxaw-1.0.16-hb9d3cd8_0.conda#7c0a9bf62d573409d12ad14b362a96e5 https://conda.anaconda.org/conda-forge/linux-64/zstandard-0.23.0-py312hef9b889_1.conda#8b7069e9792ee4e5b4919a7a306d2e67 https://conda.anaconda.org/conda-forge/linux-64/aws-sdk-cpp-1.11.379-h5a9005d_9.conda#5dc18b385893b7991a3bbeb135ad7c3e https://conda.anaconda.org/conda-forge/linux-64/azure-storage-blobs-cpp-12.12.0-hd2e3451_0.conda#61f1c193452f0daa582f39634627ea33 -https://conda.anaconda.org/conda-forge/linux-64/cftime-1.6.4-py312hc0a28a1_1.conda#990033147b0a998e756eaaed6b28f48d -https://conda.anaconda.org/conda-forge/noarch/colorspacious-1.1.2-pyh24bf2e0_0.tar.bz2#b73afa0d009a51cabd3ec99c4d2ef4f3 -https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.3.0-py312h68727a3_2.conda#ff28f374b31937c048107521c814791e -https://conda.anaconda.org/conda-forge/noarch/dask-core-2024.10.0-pyhd8ed1ab_0.conda#7823092a3cf14e98a52d2a2875c47c80 +https://conda.anaconda.org/conda-forge/noarch/bokeh-3.6.1-pyhd8ed1ab_0.conda#e88d74bb7b9b89d4c9764286ceb94cc9 +https://conda.anaconda.org/conda-forge/linux-64/cf-units-3.3.0-py312hc0a28a1_0.conda#8b5b812d4c18cb37bda7a7c8d3a6acb3 +https://conda.anaconda.org/conda-forge/noarch/dask-core-2024.11.0-pyhd8ed1ab_0.conda#75c96f0655908f596a57be60251b78d4 https://conda.anaconda.org/conda-forge/linux-64/eccodes-2.38.3-h8bb6dbc_1.conda#73265d4acc551063cc5c5beab37f33c5 -https://conda.anaconda.org/conda-forge/noarch/eofs-1.4.1-pyhd8ed1ab_1.conda#5fc43108dee4106f23050acc7a101233 https://conda.anaconda.org/conda-forge/linux-64/gtk2-2.24.33-h6470451_5.conda#1483ba046164be27df7f6eddbcec3a12 -https://conda.anaconda.org/conda-forge/noarch/identify-2.6.1-pyhd8ed1ab_0.conda#43f629202f9eec21be5f71171fb5daf8 -https://conda.anaconda.org/conda-forge/linux-64/imagecodecs-2024.6.1-py312h6d9a048_4.conda#a810fadedc4edc06b4282d1222467837 -https://conda.anaconda.org/conda-forge/noarch/imageio-2.36.0-pyh12aca89_1.conda#36349844ff73fcd0140ee7f30745f0bf +https://conda.anaconda.org/conda-forge/noarch/identify-2.6.2-pyhd8ed1ab_0.conda#636950f839e065401e2031624a414f0b +https://conda.anaconda.org/conda-forge/noarch/imagehash-4.3.1-pyhd8ed1ab_0.tar.bz2#132ad832787a2156be1f1b309835001a https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.23.0-pyhd8ed1ab_0.conda#da304c192ad59975202859b367d0f6a2 https://conda.anaconda.org/conda-forge/linux-64/julia-1.10.4-hf18f99d_1.conda#cc0ef9c191bab16211970a29b6787d69 https://conda.anaconda.org/conda-forge/noarch/lazy_loader-0.4-pyhd8ed1ab_1.conda#ec6f70b8a5242936567d4f886726a372 https://conda.anaconda.org/conda-forge/linux-64/libgdal-core-3.9.2-h353785f_1.conda#c363d0b330b4b21b4c1b10e0981d3a99 https://conda.anaconda.org/conda-forge/linux-64/libgoogle-cloud-storage-2.28.0-ha262f82_0.conda#9e7960f0b9ab3895ef73d92477c47dae https://conda.anaconda.org/conda-forge/linux-64/librsvg-2.58.4-hc0ffecb_0.conda#83f045969988f5c7a65f3950b95a8b35 -https://conda.anaconda.org/conda-forge/noarch/magics-python-1.5.8-pyhd8ed1ab_1.conda#3fd7e3db129f12362642108f23fde521 +https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.9.2-py312hd3ec401_2.conda#2380c9ba933ffaac9ad16d8eac8e3318 https://conda.anaconda.org/conda-forge/linux-64/netcdf-fortran-4.6.1-nompi_h22f9119_106.conda#5b911bfe75855326bae6857451268e59 -https://conda.anaconda.org/conda-forge/linux-64/numba-0.60.0-py312h83e6fd3_0.conda#e064ca33edf91ac117236c4b5dee207a -https://conda.anaconda.org/conda-forge/linux-64/numcodecs-0.13.1-py312hf9745cd_0.conda#33c27209bfd7af6766211facd24839ce -https://conda.anaconda.org/conda-forge/linux-64/pandas-2.1.4-py312hfb8ada1_0.conda#d0745ae74c2b26571b692ddde112eebb -https://conda.anaconda.org/conda-forge/noarch/patsy-0.5.6-pyhd8ed1ab_0.conda#a5b55d1cb110cdcedc748b5c3e16e687 +https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.7.1-nompi_py312h21d6d8e_102.conda#9049ba34261ce7106220711d313fcf61 https://conda.anaconda.org/conda-forge/noarch/pep8-naming-0.10.0-pyh9f0ad1d_0.tar.bz2#b3c5536e4f9f58a4b16adb6f1e11732d https://conda.anaconda.org/conda-forge/linux-64/postgresql-17.0-h1122569_4.conda#028ea131f116f13bb2a4a382b5863a04 https://conda.anaconda.org/conda-forge/noarch/pylint-plugin-utils-0.8.2-pyhd8ed1ab_0.conda#84377261c09c02182d76fbe79e69c9bf https://conda.anaconda.org/conda-forge/noarch/pyopenssl-24.2.1-pyhd8ed1ab_2.conda#85fa2fdd26d5a38792eb57bc72463f07 -https://conda.anaconda.org/conda-forge/linux-64/pys2index-0.1.5-py312hfb10629_0.conda#325cc5f0e0dc36562f3de2a4dbded572 https://conda.anaconda.org/conda-forge/noarch/pytest-html-4.1.1-pyhd8ed1ab_0.conda#4d2040212307d18392a2687772b3a96d -https://conda.anaconda.org/conda-forge/linux-64/pywavelets-1.7.0-py312hc0a28a1_2.conda#8300d634adec4a6aed35a87e90e9cb07 https://conda.anaconda.org/conda-forge/linux-64/r-base-4.2.3-h32f4cee_16.conda#feee98a221344be7a447b80b410df867 -https://conda.anaconda.org/conda-forge/linux-64/scipy-1.14.1-py312h62794b6_1.conda#b43233a9e2f62fb94affe5607ea79473 -https://conda.anaconda.org/conda-forge/linux-64/shapely-2.0.6-py312h6cab151_1.conda#5be02e05e1adaa42826cc6800ce399bc -https://conda.anaconda.org/conda-forge/noarch/snuggs-1.4.7-pyhd8ed1ab_1.conda#5abeaa41ec50d4d1421a8bc8fbc93054 +https://conda.anaconda.org/conda-forge/linux-64/scikit-learn-1.5.2-py312h7a48858_1.conda#6b5f4c68483bd0c22bca9094dafc606b +https://conda.anaconda.org/conda-forge/noarch/seawater-3.3.5-pyhd8ed1ab_0.conda#8e1b01f05e8f97b0fcc284f957175903 +https://conda.anaconda.org/conda-forge/noarch/sparse-0.15.4-pyh267e887_1.conda#40d80cd9fa4cc759c6dba19ea96642db +https://conda.anaconda.org/conda-forge/linux-64/statsmodels-0.14.4-py312hc0a28a1_0.conda#97dc960f3d9911964d73c2cf240baea5 https://conda.anaconda.org/conda-forge/linux-64/tempest-remap-2.2.0-h13910d2_3.conda#7f10762cd62c8ad03323c4dc3ee544b1 +https://conda.anaconda.org/conda-forge/noarch/tifffile-2024.9.20-pyhd8ed1ab_0.conda#6de55c7859ed314159eaf2b7b4f19cc7 https://conda.anaconda.org/conda-forge/noarch/urllib3-2.2.3-pyhd8ed1ab_0.conda#6b55867f385dd762ed99ea687af32a69 +https://conda.anaconda.org/conda-forge/noarch/xarray-2024.10.0-pyhd8ed1ab_0.conda#53e365732dfa053c4d19fc6b927392c4 +https://conda.anaconda.org/conda-forge/noarch/zarr-2.18.3-pyhd8ed1ab_0.conda#41abde21508578e02e3fd492e82a05cd https://conda.anaconda.org/conda-forge/linux-64/azure-storage-files-datalake-cpp-12.11.0-h325d260_1.conda#11d926d1f4a75a1b03d1c053ca20424b -https://conda.anaconda.org/conda-forge/noarch/bokeh-3.6.0-pyhd8ed1ab_0.conda#6728ca650187933a007b89f00ece4279 -https://conda.anaconda.org/conda-forge/linux-64/cf-units-3.2.0-py312hc0a28a1_6.conda#fa4853d25b6fbfef5eb7b3e1b5616dd5 -https://conda.anaconda.org/conda-forge/noarch/distributed-2024.10.0-pyhd8ed1ab_0.conda#b3b498f7bcc9a2543ad72a3501f3d87b +https://conda.anaconda.org/conda-forge/linux-64/cartopy-0.23.0-py312hf9745cd_2.conda#cc3ecff140731b46b970a7c4787b1823 +https://conda.anaconda.org/conda-forge/noarch/cf_xarray-0.10.0-pyhd8ed1ab_0.conda#9437cfe346eab83b011b4def99f0e879 +https://conda.anaconda.org/conda-forge/noarch/cmocean-4.0.3-pyhd8ed1ab_0.conda#53df00540de0348ed1b2a62684dd912b +https://conda.anaconda.org/conda-forge/noarch/distributed-2024.11.0-pyhd8ed1ab_0.conda#497f3535cbb69cd2f02158e2e18ee0bb https://conda.anaconda.org/conda-forge/linux-64/esmf-8.4.2-nompi_h9e768e6_3.conda#c330e87e698bae8e7381c0315cf25dd0 https://conda.anaconda.org/conda-forge/linux-64/gdal-3.9.2-py312h1299960_7.conda#9cf27e3f9d97ea13f250db9253a25dc8 https://conda.anaconda.org/conda-forge/linux-64/graphviz-12.0.0-hba01fac_0.conda#953e31ea00d46beb7e64a79fc291ec44 -https://conda.anaconda.org/conda-forge/noarch/imagehash-4.3.1-pyhd8ed1ab_0.tar.bz2#132ad832787a2156be1f1b309835001a https://conda.anaconda.org/conda-forge/linux-64/libgdal-fits-3.9.2-h2db6552_7.conda#524e64f1aa0ebc87230109e684f392f4 https://conda.anaconda.org/conda-forge/linux-64/libgdal-grib-3.9.2-hc3b29a1_7.conda#56a7436a66a1a4636001ce4b621a3a33 https://conda.anaconda.org/conda-forge/linux-64/libgdal-hdf4-3.9.2-hd5ecb85_7.conda#9c8431dc0b83d5fe9c12a2c0b6861a72 @@ -492,11 +503,12 @@ https://conda.anaconda.org/conda-forge/linux-64/libgdal-pg-3.9.2-h5e77dd0_7.cond https://conda.anaconda.org/conda-forge/linux-64/libgdal-postgisraster-3.9.2-h5e77dd0_7.conda#3392965ffc4e8b7c66a532750ce0e91f https://conda.anaconda.org/conda-forge/linux-64/libgdal-xls-3.9.2-h03c987c_7.conda#165f12373452e8d17889e9c877431acf https://conda.anaconda.org/conda-forge/linux-64/magics-4.15.4-h24e9adf_1.conda#9731bb0d2a3917cab718fd7c90dea857 -https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.9.2-py312hd3ec401_1.conda#2f4f3854f23be30de29e9e4d39758349 https://conda.anaconda.org/conda-forge/noarch/myproxyclient-2.1.1-pyhd8ed1ab_0.conda#bcdbeb2b693eba886583a907840c6421 https://conda.anaconda.org/conda-forge/noarch/nbformat-5.10.4-pyhd8ed1ab_0.conda#0b57b5368ab7fc7cdc9e3511fa867214 -https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.7.1-nompi_py312h21d6d8e_102.conda#9049ba34261ce7106220711d313fcf61 +https://conda.anaconda.org/conda-forge/noarch/nc-time-axis-1.4.1-pyhd8ed1ab_0.tar.bz2#281b58948bf60a2582de9e548bcc5369 https://conda.anaconda.org/conda-forge/noarch/pre-commit-4.0.1-pyha770c72_0.conda#5971cc64048943605f352f7f8612de6c +https://conda.anaconda.org/conda-forge/linux-64/psyplot-1.5.1-py312h7900ff3_1.conda#f110e71421e5c86e50232cc027c6d85c +https://conda.anaconda.org/conda-forge/noarch/py-xgboost-2.1.2-cuda118_pyh40095f8_0.conda#aa5881b02bd9555a7b06c709aa33bd20 https://conda.anaconda.org/conda-forge/noarch/pylint-celery-0.3-py_1.tar.bz2#e29456a611a62d3f26105a2f9c68f759 https://conda.anaconda.org/conda-forge/noarch/pylint-django-2.6.1-pyhd8ed1ab_0.conda#d1023ccf92d8235cd4808ef53e274a5e https://conda.anaconda.org/conda-forge/noarch/pylint-flask-0.6-py_0.tar.bz2#5a9afd3d0a61b08d59eed70fab859c1b @@ -559,36 +571,28 @@ https://conda.anaconda.org/conda-forge/linux-64/r-xfun-0.45-r42ha18555a_0.conda# https://conda.anaconda.org/conda-forge/noarch/r-xmlparsedata-1.0.5-r42hc72bb7e_2.conda#2f3614450b54f222c1eff786ec2a45ec https://conda.anaconda.org/conda-forge/linux-64/r-yaml-2.3.8-r42h57805ef_0.conda#97f60a93ca12f4fdd5f44049dcee4345 https://conda.anaconda.org/conda-forge/noarch/requests-2.32.3-pyhd8ed1ab_0.conda#5ede4753180c7a550a443c430dc8ab52 -https://conda.anaconda.org/conda-forge/linux-64/scikit-learn-1.5.2-py312h7a48858_1.conda#6b5f4c68483bd0c22bca9094dafc606b -https://conda.anaconda.org/conda-forge/noarch/seawater-3.3.5-pyhd8ed1ab_0.conda#8e1b01f05e8f97b0fcc284f957175903 -https://conda.anaconda.org/conda-forge/noarch/sparse-0.15.4-pyh267e887_1.conda#40d80cd9fa4cc759c6dba19ea96642db -https://conda.anaconda.org/conda-forge/linux-64/statsmodels-0.14.4-py312hc0a28a1_0.conda#97dc960f3d9911964d73c2cf240baea5 -https://conda.anaconda.org/conda-forge/noarch/tifffile-2024.9.20-pyhd8ed1ab_0.conda#6de55c7859ed314159eaf2b7b4f19cc7 +https://conda.anaconda.org/conda-forge/linux-64/scikit-image-0.24.0-py312hf9745cd_3.conda#3612f99c589d51c363c8b90c0bcf3a18 +https://conda.anaconda.org/conda-forge/noarch/seaborn-base-0.13.2-pyhd8ed1ab_2.conda#b713b116feaf98acdba93ad4d7f90ca1 https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.26.0-h86fa3b2_0.conda#061175d9d4c046a1cf8bffe95a359fab -https://conda.anaconda.org/conda-forge/noarch/xarray-2024.10.0-pyhd8ed1ab_0.conda#53e365732dfa053c4d19fc6b927392c4 -https://conda.anaconda.org/conda-forge/noarch/zarr-2.18.3-pyhd8ed1ab_0.conda#41abde21508578e02e3fd492e82a05cd -https://conda.anaconda.org/conda-forge/linux-64/cartopy-0.23.0-py312hf9745cd_2.conda#cc3ecff140731b46b970a7c4787b1823 https://conda.anaconda.org/conda-forge/linux-64/cdo-2.4.1-h9fe33b1_1.conda#a326dab3d2a1a8e32c2a6f792fac3161 -https://conda.anaconda.org/conda-forge/noarch/cf_xarray-0.10.0-pyhd8ed1ab_0.conda#9437cfe346eab83b011b4def99f0e879 https://conda.anaconda.org/conda-forge/noarch/cfgrib-0.9.14.1-pyhd8ed1ab_0.conda#1870fe8c9bd8967429e227be28ab94d2 https://conda.anaconda.org/conda-forge/noarch/chart-studio-1.1.0-pyh9f0ad1d_0.tar.bz2#acd9a12a35e5a0221bdf39eb6e4811dc -https://conda.anaconda.org/conda-forge/noarch/cmocean-4.0.3-pyhd8ed1ab_0.conda#53df00540de0348ed1b2a62684dd912b https://conda.anaconda.org/conda-forge/noarch/dask-jobqueue-0.9.0-pyhd8ed1ab_0.conda#a201de7d36907f2355426e019168d337 https://conda.anaconda.org/conda-forge/noarch/esmpy-8.4.2-pyhc1e730c_4.conda#ddcf387719b2e44df0cc4dd467643951 https://conda.anaconda.org/conda-forge/linux-64/imagemagick-7.1.1_39-imagemagick_hcfc5581_1.conda#1144fe07cf76921ec664b868453027d3 +https://conda.anaconda.org/conda-forge/noarch/iris-3.10.0-pyha770c72_2.conda#5d8984ceb5fdf85110ca7108114ecc18 https://conda.anaconda.org/conda-forge/linux-64/libarrow-17.0.0-h8d2e343_13_cpu.conda#dc379f362829d5df5ce6722565110029 https://conda.anaconda.org/conda-forge/linux-64/libgdal-kea-3.9.2-h1df15e4_7.conda#c693e703649051ee9db0fabd4fcd0483 https://conda.anaconda.org/conda-forge/linux-64/libgdal-netcdf-3.9.2-hf2d2f32_7.conda#4015ef020928219acc0b5c9edbce8d30 https://conda.anaconda.org/conda-forge/linux-64/libgdal-tiledb-3.9.2-h4a3bace_2.conda#c3fac34ecba2fcf9d5d31a03b975d5a1 +https://conda.anaconda.org/conda-forge/noarch/lime-0.2.0.1-pyhd8ed1ab_1.tar.bz2#789ce01416721a5533fb74aa4361fd13 https://conda.anaconda.org/conda-forge/noarch/multiurl-0.3.2-pyhd8ed1ab_0.conda#9b6cf42ef472b332970282ec87d2e5d4 https://conda.anaconda.org/conda-forge/noarch/nbclient-0.10.0-pyhd8ed1ab_0.conda#15b51397e0fe8ea7d7da60d83eb76ebc -https://conda.anaconda.org/conda-forge/noarch/nc-time-axis-1.4.1-pyhd8ed1ab_0.tar.bz2#281b58948bf60a2582de9e548bcc5369 https://conda.anaconda.org/conda-forge/linux-64/nco-5.2.8-hf7c1f58_0.conda#6cd18a9c6b8269b0cd101ba9cc3d02ab https://conda.anaconda.org/conda-forge/noarch/pooch-1.8.2-pyhd8ed1ab_0.conda#8dab97d8a9616e07d779782995710aed https://conda.anaconda.org/conda-forge/noarch/prospector-1.12.1-pyhd8ed1ab_0.conda#8621ba9cf057da26d371b87cd2264259 -https://conda.anaconda.org/conda-forge/linux-64/psyplot-1.5.1-py312h7900ff3_1.conda#f110e71421e5c86e50232cc027c6d85c -https://conda.anaconda.org/conda-forge/noarch/py-xgboost-2.1.2-cuda118_pyh40095f8_0.conda#aa5881b02bd9555a7b06c709aa33bd20 -https://conda.anaconda.org/conda-forge/linux-64/pydot-3.0.1-py312h7900ff3_1.conda#c3d006b1d90fa9f5ae436ff9d6c40249 +https://conda.anaconda.org/conda-forge/linux-64/psy-simple-1.5.1-py312h7900ff3_0.conda#683ec8787a523de54b02c885e2c2aefa +https://conda.anaconda.org/conda-forge/linux-64/pydot-3.0.2-py312h7900ff3_0.conda#a972ba77217a2cac592c41dd3cc56dfd https://conda.anaconda.org/conda-forge/noarch/pyroma-4.2-pyhd8ed1ab_0.conda#fe2aca9a5d4cb08105aefc451ef96950 https://conda.anaconda.org/conda-forge/linux-64/r-bigmemory-4.6.4-r42ha503ecb_0.conda#12b6fa8fe80a6494a948c6ea2f34340d https://conda.anaconda.org/conda-forge/linux-64/r-checkmate-2.3.1-r42h57805ef_0.conda#9febce7369c72d991e2399d7d28f3390 @@ -615,19 +619,18 @@ https://conda.anaconda.org/conda-forge/linux-64/r-timechange-0.3.0-r42ha503ecb_0 https://conda.anaconda.org/conda-forge/linux-64/r-xml2-1.3.6-r42hbfba7a4_1.conda#5c3d7a89a2d5e1c0885f92d1aa6fde30 https://conda.anaconda.org/conda-forge/linux-64/r-zoo-1.8_12-r42h57805ef_1.conda#5367d265c0c9c151dea85f1ccb515ec1 https://conda.anaconda.org/conda-forge/noarch/requests-cache-1.2.1-pyhd8ed1ab_0.conda#c6089540fed51a9a829aa19590fa925b -https://conda.anaconda.org/conda-forge/linux-64/scikit-image-0.24.0-py312hf9745cd_3.conda#3612f99c589d51c363c8b90c0bcf3a18 -https://conda.anaconda.org/conda-forge/noarch/seaborn-base-0.13.2-pyhd8ed1ab_2.conda#b713b116feaf98acdba93ad4d7f90ca1 -https://conda.anaconda.org/conda-forge/noarch/cads-api-client-1.5.0-pyhd8ed1ab_0.conda#0ca8f6f735f6171aa178364cdbbebe4d +https://conda.anaconda.org/conda-forge/noarch/seaborn-0.13.2-hd8ed1ab_2.conda#a79d8797f62715255308d92d3a91ef2e +https://conda.anaconda.org/conda-forge/noarch/xgboost-2.1.2-cuda118_pyh256f914_0.conda#2dcf3e60ef65fd4cb95048f2491f6a89 +https://conda.anaconda.org/conda-forge/noarch/cads-api-client-1.5.2-pyhd8ed1ab_0.conda#e7005effa79f1493a51404873d6eb5a0 https://conda.anaconda.org/conda-forge/noarch/esgf-pyclient-0.3.1-pyhd8ed1ab_4.conda#f481c17430f801e68ee3b57cc30ecd2e -https://conda.anaconda.org/conda-forge/noarch/iris-3.10.0-pyha770c72_2.conda#5d8984ceb5fdf85110ca7108114ecc18 https://conda.anaconda.org/conda-forge/linux-64/libarrow-acero-17.0.0-h5888daf_13_cpu.conda#b654d072b8d5da807495e49b28a0b884 https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.9.2-ha770c72_7.conda#63779711c7afd4fcf9cea67538baa67a https://conda.anaconda.org/conda-forge/linux-64/libparquet-17.0.0-h39682fd_13_cpu.conda#49c60a8dc089d8127b9368e9eb6c1a77 -https://conda.anaconda.org/conda-forge/noarch/lime-0.2.0.1-pyhd8ed1ab_1.tar.bz2#789ce01416721a5533fb74aa4361fd13 https://conda.anaconda.org/conda-forge/noarch/mapgenerator-1.0.7-pyhd8ed1ab_0.conda#d18db96ef2a920b0ecefe30282b0aecf https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.16.4-pyhd8ed1ab_1.conda#e2d2abb421c13456a9a9f80272fdf543 https://conda.anaconda.org/conda-forge/noarch/prov-2.0.0-pyhd3deb0d_0.tar.bz2#aa9b3ad140f6c0668c646f32e20ccf82 -https://conda.anaconda.org/conda-forge/linux-64/psy-simple-1.5.1-py312h7900ff3_0.conda#683ec8787a523de54b02c885e2c2aefa +https://conda.anaconda.org/conda-forge/linux-64/psy-maps-1.5.0-py312h7900ff3_1.conda#080bc8f34a9cb0ab81ae0369fd43b7ab +https://conda.anaconda.org/conda-forge/linux-64/psy-reg-1.5.0-py312h7900ff3_1.conda#ea719cfcc2e5b815b137b7082ece8aeb https://conda.anaconda.org/conda-forge/noarch/py-cordex-0.8.0-pyhd8ed1ab_0.conda#fba377622e74ee0bbeb8ccae9fa593d3 https://conda.anaconda.org/conda-forge/linux-64/pyarrow-core-17.0.0-py312h01725c0_2_cpu.conda#add603bfa43d9bf3f06783f780e1a817 https://conda.anaconda.org/conda-forge/noarch/python-cdo-1.6.0-pyhd8ed1ab_0.conda#3fd1a0b063c1fbbe4b7bd5a5a7601e84 @@ -646,16 +649,12 @@ https://conda.anaconda.org/conda-forge/noarch/r-scales-1.3.0-r42hc72bb7e_0.conda https://conda.anaconda.org/conda-forge/linux-64/r-specsverification-0.5_3-r42h7525677_2.tar.bz2#1521b8a303852af0496245e368d3c61c https://conda.anaconda.org/conda-forge/linux-64/r-splancs-2.01_45-r42hbcb9c34_0.conda#bcd96dc088f54514a54d57e6b8ed51b6 https://conda.anaconda.org/conda-forge/linux-64/r-vctrs-0.6.5-r42ha503ecb_0.conda#5689030c60302fb5bb7a48b54c11dbe8 -https://conda.anaconda.org/conda-forge/noarch/seaborn-0.13.2-hd8ed1ab_2.conda#a79d8797f62715255308d92d3a91ef2e https://conda.anaconda.org/conda-forge/noarch/xesmf-0.8.7-pyhd8ed1ab_0.conda#42301f78a4c6d2500f891b9723160d5c -https://conda.anaconda.org/conda-forge/noarch/xgboost-2.1.2-cuda118_pyh256f914_0.conda#2dcf3e60ef65fd4cb95048f2491f6a89 https://conda.anaconda.org/conda-forge/noarch/cdsapi-0.7.4-pyhd8ed1ab_0.conda#67a29b663023b8c0e3d8a73013ea3e23 https://conda.anaconda.org/conda-forge/linux-64/fiona-1.10.1-py312h5aa26c2_1.conda#4a30f4277a1894928a7057d0e14c1c95 https://conda.anaconda.org/conda-forge/linux-64/libarrow-dataset-17.0.0-h5888daf_13_cpu.conda#cd2c36e8865b158b82f61c6aac28b7e1 https://conda.anaconda.org/conda-forge/noarch/nbconvert-pandoc-7.16.4-hd8ed1ab_1.conda#37cec2cf68f4c09563d8bc833791096b https://conda.anaconda.org/conda-forge/linux-64/ncl-6.6.2-h7cb714c_54.conda#7363202c15302898deb49e82ca3e5f58 -https://conda.anaconda.org/conda-forge/linux-64/psy-maps-1.5.0-py312h7900ff3_1.conda#080bc8f34a9cb0ab81ae0369fd43b7ab -https://conda.anaconda.org/conda-forge/linux-64/psy-reg-1.5.0-py312h7900ff3_1.conda#ea719cfcc2e5b815b137b7082ece8aeb https://conda.anaconda.org/conda-forge/noarch/r-cyclocomp-1.1.1-r42hc72bb7e_0.conda#6bd41a85dc43541400311eca03d4e2d4 https://conda.anaconda.org/conda-forge/noarch/r-gridextra-2.3-r42hc72bb7e_1005.conda#da116b29105a8d48571975a185e9bb94 https://conda.anaconda.org/conda-forge/noarch/r-lmomco-2.5.1-r42hc72bb7e_0.conda#6efbdfe5d41b3ef5652be1ea2e0a6e3c @@ -672,11 +671,11 @@ https://conda.anaconda.org/conda-forge/linux-64/r-tibble-3.2.1-r42h57805ef_2.con https://conda.anaconda.org/conda-forge/linux-64/pyarrow-17.0.0-py312h9cebb41_2.conda#5f7d505626cb057e1320bbd46dd02ef2 https://conda.anaconda.org/conda-forge/noarch/r-ggplot2-3.5.1-r42hc72bb7e_0.conda#77cc0254e0dc92e5e7791ce20a170f74 https://conda.anaconda.org/conda-forge/noarch/r-rematch2-2.1.2-r42hc72bb7e_3.conda#5ccfee6f3b94e6b247c7e1929b24f1cc -https://conda.anaconda.org/conda-forge/noarch/dask-expr-1.1.16-pyhd8ed1ab_0.conda#81de1c44ab7f6cadab4a59b6d76dfa87 +https://conda.anaconda.org/conda-forge/noarch/dask-expr-1.1.17-pyhd8ed1ab_0.conda#4f75a3a76e9f693fc33be59485f46fcf https://conda.anaconda.org/conda-forge/noarch/r-styler-1.10.3-r42hc72bb7e_0.conda#1b2b8fa85a9d0556773abac4763d8ef9 https://conda.anaconda.org/conda-forge/linux-64/r-tlmoments-0.7.5.3-r42ha503ecb_1.conda#6aa1414e06dfffc39d3b5ca78b60b377 https://conda.anaconda.org/conda-forge/noarch/r-viridis-0.6.5-r42hc72bb7e_0.conda#959f69b6dfd4b620a15489975fa27670 -https://conda.anaconda.org/conda-forge/noarch/dask-2024.10.0-pyhd8ed1ab_0.conda#719832923b1d98803d07b2ca38eb3baa +https://conda.anaconda.org/conda-forge/noarch/dask-2024.11.0-pyhd8ed1ab_0.conda#9a25bf7e2a910e85209218896f2adeb9 https://conda.anaconda.org/conda-forge/linux-64/r-fields-15.2-r42h61816a4_0.conda#d84fe2f9e893e92089370b195e2263a0 https://conda.anaconda.org/conda-forge/noarch/r-spei-1.8.1-r42hc72bb7e_1.conda#7fe060235dac0fc0b3d387f98e79d128 https://conda.anaconda.org/conda-forge/noarch/iris-esmf-regrid-0.11.0-pyhd8ed1ab_1.conda#86286b197e33e3b034416c18ba0f574c From eb627592325e91fc5021bb38a86f7905de88e2d5 Mon Sep 17 00:00:00 2001 From: Manuel Schlund <32543114+schlunma@users.noreply.github.com> Date: Wed, 13 Nov 2024 13:16:39 +0100 Subject: [PATCH 03/17] Remove recipe filler utility (#3777) --- doc/sphinx/source/utils.rst | 57 -- esmvaltool/utils/recipe_filler.py | 914 ------------------------ setup.py | 2 - tests/integration/test_recipe_filler.py | 211 ------ 4 files changed, 1184 deletions(-) delete mode 100755 esmvaltool/utils/recipe_filler.py delete mode 100644 tests/integration/test_recipe_filler.py diff --git a/doc/sphinx/source/utils.rst b/doc/sphinx/source/utils.rst index 536b78ebee..d0783ff2a4 100644 --- a/doc/sphinx/source/utils.rst +++ b/doc/sphinx/source/utils.rst @@ -383,63 +383,6 @@ klaus.zimmermann@smhi.se .. _pygithub: https://pygithub.readthedocs.io/en/latest/introduction.html -Recipe filler -============= - -If you need to fill in a blank recipe with additional datasets, you can do that with -the command `recipe_filler`. This runs a tool to obtain a set of additional datasets when -given a blank recipe, and you can give an arbitrary number of data parameters. The blank recipe -should contain, to the very least, a list of diagnostics, each with their variable(s). -Example of running the tool: - -.. code-block:: bash - - recipe_filler recipe.yml - -where `recipe.yml` is the recipe that needs to be filled with additional datasets; a minimal -example of this recipe could be: - -.. code-block:: yaml - - diagnostics: - diagnostic: - variables: - ta: - mip: Amon # required - start_year: 1850 # required - end_year: 1900 # required - - -Key features ------------- - -- you can add as many variable parameters as are needed; if not added, the - tool will use the ``"*"`` wildcard and find all available combinations; -- you can restrict the number of datasets to be looked for with the ``dataset:`` - key for each variable, pass a list of datasets as value, e.g. - ``dataset: [MPI-ESM1-2-LR, MPI-ESM-LR]``; -- you can specify a pair of experiments, e.g. ``exp: [historical, rcp85]`` - for each variable; this will look for each available dataset per experiment - and assemble an aggregated data stretch from each experiment to complete - for the total data length specified by ``start_year`` and ``end_year``; equivalent to - ESMValTool's syntax on multiple experiments; this option needs an ensemble - to be declared explicitly; it will return no entry if there are gaps in data; -- ``start_year`` and ``end_year`` are required and are used to filter out the - datasets that don't have data in the interval; as noted above, the tool will not - return datasets with partial coverage from ``start_year`` to ``end_year``; - if you want all possible years hence no filtering on years just use ``"*"`` - for start and end years; -- ``config-user: rootpath: CMIPX`` may be a list, rootpath lists are supported; -- all major DRS paths (including ``default``, ``BADC``, ``ETHZ`` etc) are supported; -- speedup is achieved through CMIP mip tables lookup, so ``mip`` is required in recipe; - -Caveats -------- - -- the tool doesn't yet work with derived variables; it will not return any available datasets; -- operation restricted to CMIP data only, OBS lookup is not available yet. - - Extracting a list of input files from the provenance ==================================================== diff --git a/esmvaltool/utils/recipe_filler.py b/esmvaltool/utils/recipe_filler.py deleted file mode 100755 index 40f637c6d5..0000000000 --- a/esmvaltool/utils/recipe_filler.py +++ /dev/null @@ -1,914 +0,0 @@ -""" -Fill in a blank recipe with additional datasets. - -Tool to obtain a set of additional datasets when given a blank recipe. -The blank recipe should contain, to the very least, a list of diagnostics -each with their variable(s). Example of minimum settings: - -diagnostics: - diagnostic: - variables: - ta: - mip: Amon - start_year: 1850 - end_year: 1900 - -Note that the tool will exit if any of these minimum settings are missing! - -Key features: - -- you can add as many variable parameters as are needed; if not added, the - tool will use the "*" wildcard and find all available combinations; -- you can restrict the number of datasets to be looked for with the `dataset:` - key for each variable, pass a list of datasets as value, e.g. - `dataset: [MPI-ESM1-2-LR, MPI-ESM-LR]`; -- you can specify a pair of experiments eg `exp: [rcp26, rcp85]` - for each variable; this will look for each available dataset per experiment - and assemble an aggregated data stretch from each experiment; equivalent to - esmvaltool's syntax of multiple experiments; this option needs an ensemble - to be declared explicitly; it will return no entry if there are gaps in data -- `start_year` and `end_year` are mandatory and are used to filter out the - datasets that don't have data in the interval; if you want all possible years - hence no filtering on years just use "*" for start and end years; -- `config-user: rootpath: CMIPX` may be a list, rootpath lists are supported; - -Caveats: - -- the tool doesn't yet work for derived variables; -- operation restricted to CMIP data. - -Have fun! -""" -import argparse -import datetime -import itertools -import logging -import logging.config -import os -import shutil -import time -from glob import glob -from pathlib import Path - -import esmvalcore -import yaml - -from esmvalcore import __version__ as core_ver -from esmvalcore.cmor.table import CMOR_TABLES, read_cmor_tables -from packaging import version as pkg_version -from ruamel.yaml import YAML - -logger = logging.getLogger(__name__) - -CFG = {} - - -def _purge_file_handlers(cfg: dict) -> None: - """Remove handlers with filename set. - - This is used to remove file handlers which require an output - directory to be set. - """ - cfg['handlers'] = { - name: handler - for name, handler in cfg['handlers'].items() - if 'filename' not in handler - } - prev_root = cfg['root']['handlers'] - cfg['root']['handlers'] = [ - name for name in prev_root if name in cfg['handlers'] - ] - - -def _update_stream_level(cfg: dict, level=None): - """Update the log level for the stream handlers.""" - handlers = cfg['handlers'] - - for handler in handlers.values(): - if level is not None and 'stream' in handler: - if handler['stream'] in ('ext://sys.stdout', 'ext://sys.stderr'): - handler['level'] = level.upper() - - -def _get_log_files(cfg: dict, output_dir: str = None) -> list: - """Initialize log files for the file handlers.""" - log_files = [] - - handlers = cfg['handlers'] - - for handler in handlers.values(): - filename = handler.get('filename', None) - - if filename: - if not os.path.isabs(filename): - handler['filename'] = os.path.join(output_dir, filename) - log_files.append(handler['filename']) - - return log_files - - -def configure_logging(cfg_file: str = None, - output_dir: str = None, - console_log_level: str = None) -> list: - """Configure logging. - - Parameters - ---------- - cfg_file : str, optional - Logging config file. If `None`, defaults to `configure-logging.yml` - output_dir : str, optional - Output directory for the log files. If `None`, log only to the console. - console_log_level : str, optional - If `None`, use the default (INFO). - - Returns - ------- - log_files : list - Filenames that will be logged to. - """ - if cfg_file is None: - cfg_loc = Path(esmvalcore.__file__ + "esmvalcore") - if pkg_version.parse(core_ver) < pkg_version.parse('2.8.0'): - cfg_file = cfg_loc.parents[0] / '_config' / 'config-logging.yml' - else: - cfg_file = cfg_loc.parents[0] / 'config' / 'config-logging.yml' - - cfg_file = Path(cfg_file).absolute() - - with open(cfg_file) as file_handler: - cfg = yaml.safe_load(file_handler) - - if output_dir is None: - _purge_file_handlers(cfg) - - log_files = _get_log_files(cfg, output_dir=output_dir) - _update_stream_level(cfg, level=console_log_level) - - logging.config.dictConfig(cfg) - logging.Formatter.converter = time.gmtime - logging.captureWarnings(True) - - return log_files - - -def read_config_developer_file(cfg_file=None): - """Read the developer's configuration file.""" - if cfg_file is None: - cfg_loc = Path(esmvalcore.__file__ + "esmvalcore") - cfg_file = cfg_loc.parents[0] / 'config-developer.yml' - - with open(cfg_file, 'r') as file: - cfg = yaml.safe_load(file) - - return cfg - - -def _normalize_path(path): - """Normalize paths. - - Expand ~ character and environment variables and convert path to absolute. - - Parameters - ---------- - path: str - Original path - - Returns - ------- - str: - Normalized path - """ - if path is None: - return None - return os.path.abspath(os.path.expanduser(os.path.expandvars(path))) - - -def read_config_user_file(config_file, folder_name, options=None): - """Read config user file and store settings in a dictionary.""" - if not config_file: - config_file = '~/.esmvaltool/config-user.yml' - config_file = os.path.abspath( - os.path.expandvars(os.path.expanduser(config_file))) - # Read user config file - if not os.path.exists(config_file): - print(f"ERROR: Config file {config_file} does not exist") - - with open(config_file, 'r') as file: - cfg = yaml.safe_load(file) - - if options is None: - options = dict() - for key, value in options.items(): - cfg[key] = value - - # set defaults - defaults = { - 'compress_netcdf': False, - 'exit_on_warning': False, - 'output_file_type': 'png', - 'output_dir': 'esmvaltool_output', - 'auxiliary_data_dir': 'auxiliary_data', - 'save_intermediary_cubes': False, - 'remove_preproc_dir': True, - 'max_parallel_tasks': None, - 'run_diagnostic': True, - 'profile_diagnostic': False, - 'config_developer_file': None, - 'drs': {}, - } - - for key in defaults: - if key not in cfg: - logger.info( - "No %s specification in config file, " - "defaulting to %s", key, defaults[key]) - cfg[key] = defaults[key] - - cfg['output_dir'] = _normalize_path(cfg['output_dir']) - cfg['auxiliary_data_dir'] = _normalize_path(cfg['auxiliary_data_dir']) - - cfg['config_developer_file'] = _normalize_path( - cfg['config_developer_file']) - - for key in cfg['rootpath']: - root = cfg['rootpath'][key] - if isinstance(root, str): - cfg['rootpath'][key] = [_normalize_path(root)] - else: - cfg['rootpath'][key] = [_normalize_path(path) for path in root] - - # insert a directory date_time_recipe_usertag in the output paths - now = datetime.datetime.utcnow().strftime("%Y%m%d_%H%M%S") - new_subdir = '_'.join((folder_name, now)) - cfg['output_dir'] = os.path.join(cfg['output_dir'], new_subdir) - - # create subdirectories - cfg['preproc_dir'] = os.path.join(cfg['output_dir'], 'preproc') - cfg['work_dir'] = os.path.join(cfg['output_dir'], 'work') - cfg['plot_dir'] = os.path.join(cfg['output_dir'], 'plots') - cfg['run_dir'] = os.path.join(cfg['output_dir'], 'run') - - # Read developer configuration file - read_cmor_tables(cfg['config_developer_file']) - - return cfg - - -HEADER = r""" -______________________________________________________________________ - _____ ____ __ ____ __ _ _____ _ - | ____/ ___|| \/ \ \ / /_ _| |_ _|__ ___ | | - | _| \___ \| |\/| |\ \ / / _` | | | |/ _ \ / _ \| | - | |___ ___) | | | | \ V / (_| | | | | (_) | (_) | | - |_____|____/|_| |_| \_/ \__,_|_| |_|\___/ \___/|_| -______________________________________________________________________ - -""" + __doc__ - -dataset_order = [ - 'dataset', 'project', 'exp', 'mip', 'ensemble', 'grid', 'start_year', - 'end_year' -] - -# cmip eras -cmip_eras = ["CMIP5", "CMIP6"] - -# The base dictionairy (all wildcards): -base_dict = { - 'institute': '*', - 'dataset': '*', - 'project': '*', - 'exp': '*', - 'frequency': '*', - 'ensemble': '*', - 'mip': '*', - 'modeling_realm': '*', - 'short_name': '*', - 'grid': '*', - 'start_year': '*', - 'end_year': '*', - 'activity': '*', -} - - -def _get_download_dir(yamlconf, cmip_era): - """Get the Download Directory from user config file.""" - if 'download_dir' in yamlconf: - return os.path.join(yamlconf['download_dir'], cmip_era) - return False - - -def _get_site_rootpath(cmip_era): - """Get site (drs) from config-user.yml.""" - config_yml = get_args().config_file - with open(config_yml, 'r') as yamf: - yamlconf = yaml.safe_load(yamf) - drs = yamlconf['drs'][cmip_era] - - download_dir = _get_download_dir(yamlconf, cmip_era) - rootdir = [yamlconf['rootpath'][cmip_era], ] - - if download_dir: - rootdir.append(download_dir) - logger.debug("%s root directory %s", cmip_era, rootdir) - if drs == 'default' and 'default' in yamlconf['rootpath']: - rootdir = [yamlconf['rootpath']['default'], ] - if download_dir: - rootdir.append(download_dir) - - logger.debug("Using drs default and " - "default: %s data directory", rootdir) - - return drs, rootdir - - -def _get_input_dir(cmip_era): - """Get input_dir from config-developer.yml.""" - site = _get_site_rootpath(cmip_era)[0] - yamlconf = read_config_developer_file() - - return yamlconf[cmip_era]['input_dir'][site] - - -def _get_input_file(cmip_era): - """Get input_file from config-developer.yml.""" - yamlconf = read_config_developer_file() - return yamlconf[cmip_era]['input_file'] - - -def _determine_basepath(cmip_era): - """Determine a basepath.""" - if isinstance(_get_site_rootpath(cmip_era)[1], list): - rootpaths = _get_site_rootpath(cmip_era)[1] - else: - rootpaths = [_get_site_rootpath(cmip_era)[1]] - - basepaths = [] - for rootpath in rootpaths: - if _get_input_dir(cmip_era) != os.path.sep: - basepath = os.path.join(rootpath, _get_input_dir(cmip_era), - _get_input_file(cmip_era)) - else: - basepath = os.path.join(rootpath, _get_input_file(cmip_era)) - basepath = basepath.replace('//', '/') - basepaths.append(basepath) - logger.debug("We will look for files of patterns %s", basepaths) - - return basepaths - - -def _overlapping_datasets(files, all_years, start_year, end_year): - """Process overlapping datasets and check for avail data in time range.""" - valid_files = [] - ay_sorted = sorted(all_years) - if ay_sorted[0] <= start_year and ay_sorted[-1] >= end_year: - yr_pairs = sorted( - [all_years[i:i + 2] for i in range(0, len(all_years), 2)]) - yr_pairs = list(k for k, _ in itertools.groupby(yr_pairs)) - d_y = [ - yr_pairs[j][1] - yr_pairs[j + 1][0] - for j in range(len(yr_pairs) - 1) - ] - gaps = [c for c in d_y if c < -1] - if not gaps: - valid_files = files - logger.info("Contiguous data from multiple experiments.") - else: - logger.warning("Data from multiple exps has >1 year gaps! ") - logger.debug("Start %s/end %s requested - " - "files covering %s found.", - start_year, end_year, yr_pairs) - - return valid_files - - -def filter_years(files, start_year, end_year, overlap=False): - """ - Filter out files that are outside requested time range. - - Nifty function that takes a list of files and two years - as arguments; it will build a series of filter dictionaries - and check if data is available for the entire interval; - it will return a single file per dataset, the first file - in the list of files that cover the specified interval; - optional argument `overlap` used if multiple experiments are - used and overlap between datasets is present. - - Parameters - ---------- - files: list - A list of files that need filtering by requested time range. - - start_year: int - Integer start year of requested range. - - end_year: int - Integer end year of requested range. - - overlap: bool - Flag if datasets overlap; defaults to False. - - Returns - ------- - list - List of files which have been identified as falling in - the requested time range; if multiple files within time range - per dataset, the first file will be returned. - - """ - valid_files = [] - available_years = {} - - if start_year == "*" and end_year == "*": - return files - - if not files: - return valid_files - - all_files_roots = [("").join(fil.split("_")[0:-1]) for fil in files] - for fil in files: - available_years[("").join(fil.split("_")[0:-1])] = [] - for fil in files: - available_years[("").join(fil.split("_")[0:-1])].append( - fil.split("_")[-1].strip(".nc").split("-")) - - all_years = [] - for root, yr_list in available_years.items(): - actual_years = [] - yr_list = list(itertools.chain.from_iterable(yr_list)) - for year in yr_list: - if len(year) == 4: - actual_years.append(int(year)) - else: - actual_years.append(int(year[0:4])) - actual_years = sorted(actual_years) - all_years.extend(actual_years) - if not overlap: - actual_years = sorted(list(set(actual_years))) - if actual_years[0] <= start_year and actual_years[-1] >= end_year: - idx = all_files_roots.index(root) - valid_files.append(files[idx]) - - # multiple experiments to complete each other - if overlap: - valid_files = _overlapping_datasets(files, all_years, start_year, - end_year) - - if not valid_files: - logger.warning("No data found to fully cover start " - "%s / end %s as requested!", start_year, end_year) - - return valid_files - - -def _resolve_latestversion(dirname_template): - """Resolve the 'latestversion' tag.""" - for version_separator in ['{latestversion}', '{version}']: - if version_separator in dirname_template: - break - else: - return dirname_template - - # Find latest version - part1, part2 = dirname_template.split(version_separator) - part2 = part2.lstrip(os.sep) - part1_contents = glob(part1) - if part1_contents: - versions = os.listdir(part1_contents[0]) - versions.sort(reverse=True) - for version in ['latest'] + versions: - dirname = os.path.join(part1, version, part2) - if glob(dirname): - return dirname - - return dirname_template - - -def list_all_files(file_dict, cmip_era): - """ - List all files that match the dataset dictionary. - - Function that returns all files that are determined by a - file_dict dictionary; file_dict is keyed on usual parameters - like `dataset`, `project`, `mip` etc; glob.glob is used - to find files; speedup is achieved by replacing wildcards - with values from CMOR tables. - - Parameters - ---------- - file_dict: dict - Dictionary to hold dataset specifications. - - cmip_era: str - Either CMIP5 or CMIP6. - - Returns - ------- - list: - List of found files. - - """ - mip = file_dict['mip'] - short_name = file_dict['short_name'] - try: - frequency = CMOR_TABLES[cmip_era].get_variable(mip, - short_name).frequency - realms = CMOR_TABLES[cmip_era].get_variable(mip, - short_name).modeling_realm - except AttributeError: - logger.warning("Could not find %s CMOR table " - "for variable %s with mip %s", - cmip_era, short_name, mip) - return [] - file_dict['frequency'] = frequency - - basepaths = _determine_basepath(cmip_era) - all_files = [] - - for basepath in basepaths: - new_path = basepath[:] - - # could have multiple realms - for realm in realms: - file_dict['modeling_realm'] = realm - - # load all the files in the custom dict - for key, value in file_dict.items(): - new_path = new_path.replace('{' + key + '}', str(value)) - new_path = _resolve_latestversion(new_path) - if new_path.startswith("~"): - new_path = os.path.expanduser(new_path) - if not new_path.startswith(os.sep): - raise ValueError( - "Could not expand ~ to user home dir " - "please expand it in the config user file!") - logger.info("Expanding path to %s", new_path) - - # Globs all the wildcards into a list of files. - files = glob(new_path) - all_files.extend(files) - if not all_files: - logger.warning("Could not find any file for data specifications.") - - return all_files - - -def _file_to_recipe_dataset(fn_path, cmip_era, file_dict): - """Convert a filename to an recipe ready dataset.""" - # Add the obvious ones - ie the one you requested! - output_dataset = {} - output_dataset['project'] = cmip_era - for key, value in file_dict.items(): - if value == '*': - continue - if key in dataset_order: - output_dataset[key] = value - - # Split file name and base path into directory structure and filenames. - basefiles = _determine_basepath(cmip_era) - _, fnfile = os.path.split(fn_path) - - for basefile in basefiles: - _, basefile = os.path.split(basefile) - # Some of the key words include the splitting character '_' ! - basefile = basefile.replace('short_name', 'shortname') - basefile = basefile.replace('start_year', 'startyear') - basefile = basefile.replace('end_year', 'endyear') - - # Assume filename is separated by '_' - basefile_split = [key.replace("{", "") for key in basefile.split('_')] - basefile_split = [key.replace("}", "") for key in basefile_split] - fnfile_split = fnfile.split('_') - - # iterate through directory structure looking for useful bits. - for base_key, fn_key in zip(basefile_split, fnfile_split): - if base_key == '*.nc': - fn_key = fn_key.replace('.nc', '') - start_year, end_year = fn_key.split('-') - output_dataset['start_year'] = start_year - output_dataset['end_year'] = end_year - elif base_key == "ensemble*.nc": - output_dataset['ensemble'] = fn_key - elif base_key == "grid*.nc": - output_dataset['grid'] = fn_key - elif base_key == "shortname": - pass - else: - output_dataset[base_key] = fn_key - if "exp" in file_dict: - if isinstance(file_dict["exp"], list): - output_dataset["exp"] = file_dict["exp"] - - return output_dataset - - -def _remove_duplicates(add_datasets): - """ - Remove accidental duplicates. - - Close to 0% chances this will ever be used. - May be used when there are actual duplicates in data - storage, we've seen these before, but seldom. - """ - datasets = [] - seen = set() - - for dataset in add_datasets: - orig_exp = dataset["exp"] - dataset["exp"] = str(dataset["exp"]) - tup_dat = tuple(dataset.items()) - if tup_dat not in seen: - seen.add(tup_dat) - dataset["exp"] = orig_exp - datasets.append(dataset) - - return datasets - - -def _check_recipe(recipe_dict): - """Perform a quick recipe check for mandatory fields.""" - do_exit = False - if "diagnostics" not in recipe_dict: - logger.error("Recipe missing diagnostics section.") - do_exit = True - for diag_name, diag in recipe_dict["diagnostics"].items(): - if "variables" not in diag: - logger.error("Diagnostic %s missing variables.", diag_name) - do_exit = True - for var_name, var_pars in diag["variables"].items(): - if "mip" not in var_pars: - logger.error("Variable %s missing mip.", var_name) - do_exit = True - if "start_year" not in var_pars: - logger.error("Variable %s missing start_year.", var_name) - do_exit = True - if "end_year" not in var_pars: - logger.error("Variable %s missing end_year.", var_name) - do_exit = True - if "exp" in var_pars: - if isinstance(var_pars["exp"], - list) and "ensemble" not in var_pars: - logger.error("Asking for experiments list for ") - logger.error("variable %s - you need to ", var_name) - logger.error("define an ensemble for this case.") - do_exit = True - if do_exit: - raise ValueError("Please fix the issues in recipe and rerun") - - -def _check_config_file(user_config_file): - """Perform a quick recipe check for mandatory fields.""" - do_exit = False - if "rootpath" not in user_config_file: - logger.error("Config file missing rootpath section.") - do_exit = True - if "drs" not in user_config_file: - logger.error("Config file missing drs section.") - do_exit = True - for proj in cmip_eras: - if proj not in user_config_file["rootpath"].keys(): - logger.error("Config file missing rootpath for %s", proj) - do_exit = True - if proj not in user_config_file["drs"].keys(): - logger.error("Config file missing drs for %s", proj) - do_exit = True - if do_exit: - raise ValueError("Please fix issues in config file and rerun") - - -def _parse_recipe_to_dicts(yamlrecipe): - """Parse a recipe's variables into a dictionary of dictionairies.""" - output_dicts = {} - for diag in yamlrecipe['diagnostics']: - for variable, var_dict in yamlrecipe['diagnostics'][diag][ - 'variables'].items(): - new_dict = base_dict.copy() - for var_key, var_value in var_dict.items(): - if var_key in new_dict: - new_dict[var_key] = var_value - output_dicts[(diag, variable)] = new_dict - - return output_dicts - - -def _add_datasets_into_recipe(additional_datasets, output_recipe): - """Add the datasets into a new recipe.""" - yaml = YAML() - yaml.default_flow_style = False - with open(output_recipe, 'r') as yamlfile: - cur_yaml = yaml.load(yamlfile) - for diag_var, add_dat in additional_datasets.items(): - if add_dat: - if 'additional_datasets' in cur_yaml['diagnostics']: - cur_yaml['diagnostics'][diag_var[0]]['variables'][ - diag_var[1]]['additional_datasets'].extend(add_dat) - else: - cur_yaml['diagnostics'][diag_var[0]]['variables'][ - diag_var[1]]['additional_datasets'] = add_dat - if cur_yaml: - with open(output_recipe, 'w') as yamlfile: - yaml.dump(cur_yaml, yamlfile) - - -def _find_all_datasets(recipe_dict, cmip_eras): - """Find all datasets explicitly.""" - datasets = [] - for cmip_era in cmip_eras: - if cmip_era == "CMIP6": - activity = "CMIP" - else: - activity = "" - drs, site_path = _get_site_rootpath(cmip_era) - if drs in ["default", "SMHI"]: - logger.info("DRS is %s; filter on dataset disabled.", drs) - datasets = ["*"] - else: - if not isinstance(site_path, list): - site_path = [site_path] - for site_pth in site_path: - if drs in ["BADC", "DKRZ", "CP4CDS"]: - institutes_path = os.path.join(site_pth, activity) - elif drs in ["ETHZ", "RCAST"]: - exp = recipe_dict["exp"][0] - if exp == "*": - exp = "piControl" # all institutes have piControl - mip = recipe_dict["mip"] - var = recipe_dict["short_name"] - institutes_path = os.path.join(site_pth, exp, mip, var) - - if not os.path.isdir(institutes_path): - logger.warning("Path to data %s " - "does not exist; will look everywhere.", - institutes_path) - datasets = ["*"] - return datasets - - institutes = os.listdir(institutes_path) - if drs in ["BADC", "DKRZ", "CP4CDS"]: - for institute in institutes: - datasets.extend( - os.listdir(os.path.join(institutes_path, - institute))) - else: - datasets.extend(institutes) - - return datasets - - -def _get_exp(recipe_dict): - """Get the correct exp as list of single or multiple exps.""" - if isinstance(recipe_dict["exp"], list): - exps_list = recipe_dict["exp"] - logger.info("Multiple %s experiments requested", exps_list) - else: - exps_list = [recipe_dict["exp"]] - logger.info("Single %s experiment requested", exps_list) - - return exps_list - - -def _get_datasets(recipe_dict, cmip_eras): - """Get the correct datasets as list if needed.""" - if recipe_dict["dataset"] == "*": - datasets = _find_all_datasets(recipe_dict, cmip_eras) - return datasets - if isinstance(recipe_dict['dataset'], list): - datasets = recipe_dict['dataset'] - logger.info("Multiple %s datasets requested", datasets) - else: - datasets = [recipe_dict['dataset']] - logger.info("Single %s dataset requested", datasets) - - return datasets - - -def get_args(): - """Parse command line arguments.""" - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('recipe', help='Path/name of yaml pilot recipe file') - parser.add_argument('-c', - '--config-file', - default=os.path.join(os.environ["HOME"], '.esmvaltool', - 'config-user.yml'), - help='User configuration file') - - parser.add_argument('-o', - '--output', - default=os.path.join(os.getcwd(), - 'recipe_autofilled.yml'), - help='Output recipe, default recipe_autofilled.yml') - - args = parser.parse_args() - return args - - -def _get_timefiltered_files(recipe_dict, exps_list, cmip_era): - """Obtain all files that correspond to requested time range.""" - # multiple experiments allowed, complement data from each exp - if len(exps_list) > 1: - files = [] - for exp in exps_list: - recipe_dict["exp"] = exp - files.extend(list_all_files(recipe_dict, cmip_era)) - files = filter_years(files, - recipe_dict["start_year"], - recipe_dict["end_year"], - overlap=True) - recipe_dict["exp"] = exps_list - - else: - files = list_all_files(recipe_dict, cmip_era) - files = filter_years(files, recipe_dict["start_year"], - recipe_dict["end_year"]) - - return files - - -def run(): - """Run the `recipe_filler` tool. Help in __doc__ and via --help.""" - # Get arguments - args = get_args() - input_recipe = args.recipe - output_recipe = args.output - cmip_eras = ["CMIP5", "CMIP6"] - - # read the config file - config_user = read_config_user_file(args.config_file, - 'recipe_filler', - options={}) - - # configure logger - run_dir = os.path.join(config_user['output_dir'], 'recipe_filler') - if not os.path.isdir(run_dir): - os.makedirs(run_dir) - log_files = configure_logging(output_dir=run_dir, - console_log_level=config_user['log_level']) - logger.info(HEADER) - logger.info("Using user configuration file: %s", args.config_file) - logger.info("Using pilot recipe file: %s", input_recipe) - logger.info("Writing filled out recipe to: %s", output_recipe) - log_files = "\n".join(log_files) - logger.info("Writing program log files to:\n%s", log_files) - - # check config user file - _check_config_file(config_user) - - # parse recipe - with open(input_recipe, 'r') as yamlfile: - yamlrecipe = yaml.safe_load(yamlfile) - _check_recipe(yamlrecipe) - recipe_dicts = _parse_recipe_to_dicts(yamlrecipe) - - # Create a list of additional_datasets for each diagnostic/variable. - additional_datasets = {} - for (diag, variable), recipe_dict in recipe_dicts.items(): - logger.info("Looking for data for " - "variable %s in diagnostic %s", variable, diag) - new_datasets = [] - if "short_name" not in recipe_dict: - recipe_dict['short_name'] = variable - elif recipe_dict['short_name'] == "*": - recipe_dict['short_name'] = variable - - # adjust cmip era if needed - if recipe_dict['project'] != "*": - cmip_eras = [recipe_dict['project']] - - # get datasets depending on user request; always a list - datasets = _get_datasets(recipe_dict, cmip_eras) - - # get experiments depending on user request; always a list - exps_list = _get_exp(recipe_dict) - - # loop through datasets - for dataset in datasets: - recipe_dict['dataset'] = dataset - logger.info("Seeking data for dataset: %s", dataset) - for cmip_era in cmip_eras: - files = _get_timefiltered_files(recipe_dict, exps_list, - cmip_era) - - # assemble in new recipe - add_datasets = [] - for fn in sorted(files): - fn_dir = os.path.dirname(fn) - logger.info("Data directory: %s", fn_dir) - out = _file_to_recipe_dataset(fn, cmip_era, recipe_dict) - logger.info("New recipe entry: %s", out) - if out is None: - continue - add_datasets.append(out) - new_datasets.extend(add_datasets) - additional_datasets[(diag, variable, cmip_era)] = \ - _remove_duplicates(new_datasets) - - # add datasets to recipe as additional_datasets - shutil.copyfile(input_recipe, output_recipe, follow_symlinks=True) - _add_datasets_into_recipe(additional_datasets, output_recipe) - logger.info("Finished recipe filler. Go get some science done now!") - - -if __name__ == "__main__": - run() diff --git a/setup.py b/setup.py index 6b4636d1f7..86aab79854 100755 --- a/setup.py +++ b/setup.py @@ -250,8 +250,6 @@ def read_description(filename): 'nclcodestyle = esmvaltool.utils.nclcodestyle.nclcodestyle:_main', 'test_recipe = ' 'esmvaltool.utils.testing.recipe_settings.install_expand_run:main', - 'recipe_filler = ' - 'esmvaltool.utils.recipe_filler:run' ], 'esmvaltool_commands': [ 'colortables = ' diff --git a/tests/integration/test_recipe_filler.py b/tests/integration/test_recipe_filler.py deleted file mode 100644 index b78ac8c5f8..0000000000 --- a/tests/integration/test_recipe_filler.py +++ /dev/null @@ -1,211 +0,0 @@ -"""Tests for _data_finder.py.""" -import contextlib -import os -import shutil -import sys -import tempfile - -import pytest -import yaml - -from esmvaltool.utils.recipe_filler import run - - -# Load test configuration -with open(os.path.join(os.path.dirname(__file__), - 'recipe_filler.yml')) as file: - CONFIG = yaml.safe_load(file) - - -@contextlib.contextmanager -def arguments(*args): - backup = sys.argv - sys.argv = list(args) - yield - sys.argv = backup - - -def print_path(path): - """Print path.""" - txt = path - if os.path.isdir(path): - txt += '/' - if os.path.islink(path): - txt += ' -> ' + os.readlink(path) - print(txt) - - -def tree(path): - """Print path, similar to the the `tree` command.""" - print_path(path) - for dirpath, dirnames, filenames in os.walk(path): - for dirname in dirnames: - print_path(os.path.join(dirpath, dirname)) - for filename in filenames: - print_path(os.path.join(dirpath, filename)) - - -def create_file(filename): - """Create an empty file.""" - dirname = os.path.dirname(filename) - if not os.path.exists(dirname): - os.makedirs(dirname) - - with open(filename, 'a'): - pass - - -def create_tree(path, filenames=None, symlinks=None): - """Create directory structure and files.""" - for filename in filenames or []: - create_file(os.path.join(path, filename)) - - for symlink in symlinks or []: - link_name = os.path.join(path, symlink['link_name']) - os.symlink(symlink['target'], link_name) - - -def write_config_user_file(dirname, file_path, drs): - config_file = dirname / 'config-user.yml' - cfg = { - 'log_level': 'info', - 'output_dir': str(dirname / 'recipe_filler_output'), - 'rootpath': { - 'CMIP5': str(dirname / file_path), - 'CMIP6': str(dirname / file_path), - }, - 'drs': { - 'CMIP5': drs, - 'CMIP6': drs, - }, - } - config_file.write_text(yaml.safe_dump(cfg, encoding=None)) - return str(config_file) - - -def write_recipe(dirname, recipe_dict): - recipe_file = dirname / 'recipe.yml' - diags = {'diagnostics': recipe_dict} - recipe_file.write_text(yaml.safe_dump(diags, encoding=None)) - return str(recipe_file) - - -@pytest.fixture -def root(): - """Root function for tests.""" - dirname = tempfile.mkdtemp() - yield os.path.join(dirname, 'output1') - print("Directory structure was:") - tree(dirname) - shutil.rmtree(dirname) - - -def setup_files(tmp_path, root, cfg): - """Create config, recipe ,output recipe etc.""" - user_config_file = write_config_user_file(tmp_path, root, cfg['drs']) - diagnostics = {} - diagnostics["test_diagnostic"] = {} - diagnostics["test_diagnostic"]["variables"] = {} - diagnostics["test_diagnostic"]["variables"]["test_var"] = cfg["variable"] - recipe = write_recipe(tmp_path, diagnostics) - output_recipe = str(tmp_path / "recipe_auto.yml") - - return user_config_file, recipe, output_recipe - - -@pytest.mark.parametrize('cfg', CONFIG['has_additional_datasets']) -def test_adding_datasets(tmp_path, root, cfg): - """Test retrieving additional datasets.""" - create_tree(root, cfg.get('available_files'), - cfg.get('available_symlinks')) - - user_config_file, recipe, output_recipe = setup_files(tmp_path, root, cfg) - - with arguments( - 'recipe_filler', - recipe, - '-c', - user_config_file, - '-o', - output_recipe, - ): - run() - - with open(output_recipe, 'r') as file: - autofilled_recipe = yaml.safe_load(file) - diag = autofilled_recipe["diagnostics"]["test_diagnostic"] - var = diag["variables"]["test_var"] - assert "additional_datasets" in var - - -@pytest.mark.parametrize('cfg', CONFIG['no_additional_datasets']) -def test_not_adding_datasets(tmp_path, root, cfg): - """Test retrieving no additional datasets.""" - create_tree(root, cfg.get('available_files'), - cfg.get('available_symlinks')) - - user_config_file, recipe, output_recipe = setup_files(tmp_path, root, cfg) - - with arguments( - 'recipe_filler', - recipe, - '-c', - user_config_file, - '-o', - output_recipe, - ): - run() - - with open(output_recipe, 'r') as file: - autofilled_recipe = yaml.safe_load(file) - diag = autofilled_recipe["diagnostics"]["test_diagnostic"] - var = diag["variables"]["test_var"] - assert "additional_datasets" not in var - - -def test_bad_var(tmp_path, root): - """Test a bad variable in the works.""" - cfg = CONFIG['bad_variable'][0] - user_config_file, recipe, output_recipe = setup_files(tmp_path, root, cfg) - - # this doesn't fail and it shouldn't since it can go on - # and look for data for other valid variables - with arguments( - 'recipe_filler', - recipe, - '-c', - user_config_file, - '-o', - output_recipe, - ): - run() - - with open(output_recipe, 'r') as file: - autofilled_recipe = yaml.safe_load(file) - diag = autofilled_recipe["diagnostics"]["test_diagnostic"] - var = diag["variables"]["test_var"] - assert "additional_datasets" not in var - - -def test_no_short_name(tmp_path, root): - """Test a bad variable in the works.""" - cfg = CONFIG['no_short_name'][0] - user_config_file, recipe, output_recipe = setup_files(tmp_path, root, cfg) - - # this doesn't fail and it shouldn't since it can go on - # and look for data for other valid variables - with arguments( - 'recipe_filler', - recipe, - '-c', - user_config_file, - '-o', - output_recipe, - ): - run() - - with open(output_recipe, 'r') as file: - autofilled_recipe = yaml.safe_load(file) - diag = autofilled_recipe["diagnostics"]["test_diagnostic"] - var = diag["variables"]["test_var"] - assert "additional_datasets" not in var From c4f757638ea3e78c635cc130ed965d47c32c1d9e Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 13 Nov 2024 12:23:41 +0000 Subject: [PATCH 04/17] Fix issue related to removal/change of private function imported in `diag_scripts/shared/_supermeans.py` (deprecation in iris=3.11) (#3810) --- esmvaltool/diag_scripts/shared/_supermeans.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/esmvaltool/diag_scripts/shared/_supermeans.py b/esmvaltool/diag_scripts/shared/_supermeans.py index 7099ba4725..8543ca99cf 100644 --- a/esmvaltool/diag_scripts/shared/_supermeans.py +++ b/esmvaltool/diag_scripts/shared/_supermeans.py @@ -13,7 +13,6 @@ import cf_units import iris import iris.coord_categorisation -from iris.coord_categorisation import _pt_date import numpy as np @@ -206,6 +205,28 @@ def add_start_hour(cube, coord, name='diurnal_sampling_hour'): _add_categorised_coord(cube, name, coord, start_hour_from_bounds) +# lifted from iris==3.10 last iris to have it in iris.coord_categorisation +# Private "helper" function +def _pt_date(coord, time): + """Return the datetime of a time-coordinate point. + + Parameters + ---------- + coord : Coord + Coordinate (must be Time-type). + time : float + Value of a coordinate point. + + Returns + ------- + cftime.datetime + + """ + # NOTE: All of the currently defined categorisation functions are + # calendar operations on Time coordinates. + return coord.units.num2date(time, only_use_cftime_datetimes=True) + + def start_hour_from_bounds(coord, _, bounds): """Add hour from bounds.""" return np.array([_pt_date(coord, _bounds[0]).hour for _bounds in bounds]) From de43833ff1238d1c0b5e70bf4b12d67583d8057e Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Thu, 14 Nov 2024 16:35:32 +0000 Subject: [PATCH 05/17] Update environment: pin `iris>=3.11`, unpin `cartopy` and allow for `numpy >=2` (#3811) Co-authored-by: Manuel Schlund <32543114+schlunma@users.noreply.github.com> --- environment.yml | 10 +++++----- environment_osx.yml | 8 ++++---- setup.py | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/environment.yml b/environment.yml index 270f0f6ecd..72ccf127f6 100644 --- a/environment.yml +++ b/environment.yml @@ -10,27 +10,27 @@ channels: dependencies: - aiohttp - - cartopy <0.24 # https://github.com/ESMValGroup/ESMValTool/issues/3767 + - cartopy - cdo >=2.3.0 - cdsapi - cf-units - cfgrib - cftime - cmocean - - curl <8.10 + - curl <8.10 # https://github.com/ESMValGroup/ESMValTool/issues/3758 - cython - dask !=2024.8.0 # https://github.com/dask/dask/issues/11296 - distributed - ecmwf-api-client - eofs - - esmpy # <8.6 safe https://github.com/SciTools/iris-esmf-regrid/issues/415 + - esmpy - esmvalcore 2.11.* - fiona - fire - fsspec - gdal >=3.9.0 - importlib_metadata <8 # https://github.com/ESMValGroup/ESMValTool/issues/3699 only for Python 3.10/11 and esmpy<8.6 - - iris >=3.6.1 + - iris >=3.11 - iris-esmf-regrid >=0.10.0 # github.com/SciTools-incubator/iris-esmf-regrid/pull/342 - jinja2 - joblib @@ -41,7 +41,7 @@ dependencies: - nc-time-axis - netCDF4 - numba - - numpy !=1.24.3,<2.0 # severe masking bug + - numpy !=1.24.3 # severe masking bug - openpyxl - packaging - pandas==2.1.4 # unpin when ESMValCore released with https://github.com/ESMValGroup/ESMValCore/pull/2529 diff --git a/environment_osx.yml b/environment_osx.yml index 8285b43ecd..242f0a4f56 100644 --- a/environment_osx.yml +++ b/environment_osx.yml @@ -10,7 +10,7 @@ channels: dependencies: - aiohttp - - cartopy <0.24 # https://github.com/ESMValGroup/ESMValTool/issues/3767 + - cartopy - cdo >=2.3.0 - cdsapi - cf-units @@ -22,14 +22,14 @@ dependencies: - distributed - ecmwf-api-client - eofs - - esmpy # <8.6 safe https://github.com/SciTools/iris-esmf-regrid/issues/415 + - esmpy - esmvalcore 2.11.* - fiona - fire - fsspec - gdal >=3.9.0 - importlib_metadata <8 # https://github.com/ESMValGroup/ESMValTool/issues/3699 only for Python 3.10/11 and esmpy<8.6 - - iris >=3.6.1 + - iris >=3.11 - iris-esmf-regrid >=0.10.0 # github.com/SciTools-incubator/iris-esmf-regrid/pull/342 - jinja2 - joblib @@ -40,7 +40,7 @@ dependencies: - nc-time-axis - netCDF4 - numba - - numpy !=1.24.3,<2.0 # severe masking bug + - numpy !=1.24.3 # severe masking bug - openpyxl - packaging - pandas==2.1.4 # unpin when ESMValCore released with https://github.com/ESMValGroup/ESMValCore/pull/2529 diff --git a/setup.py b/setup.py index 86aab79854..cdadaca2d2 100755 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ # Use with pip install . to install from source 'install': [ 'aiohttp', - 'cartopy<0.24', # github.com/ESMValGroup/ESMValTool/issues/3767 + 'cartopy', 'cdo', 'cdsapi', 'cf-units', @@ -67,7 +67,7 @@ 'scikit-image', 'scikit-learn>=1.4.0', # github.com/ESMValGroup/ESMValTool/issues/3504 'scipy', - 'scitools-iris>=3.6.1', + 'scitools-iris>=3.11', 'seaborn', 'seawater', 'shapely>=2', From e070fd5a86bc3832c82e801832cc5cfbdabf7ffb Mon Sep 17 00:00:00 2001 From: Axel Lauer Date: Thu, 21 Nov 2024 12:09:12 +0100 Subject: [PATCH 06/17] Add info on obs tiers to docu (#3624) Co-authored-by: Bouwe Andela Co-authored-by: Romain Beucher --- doc/sphinx/source/input.rst | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/doc/sphinx/source/input.rst b/doc/sphinx/source/input.rst index fbc16b45ec..f9bcfafc3e 100644 --- a/doc/sphinx/source/input.rst +++ b/doc/sphinx/source/input.rst @@ -112,6 +112,21 @@ ESMValTool currently supports two ways to perform this reformatting (aka checks and fixes'). Details on this second method are given at the :ref:`end of this chapter `. +Tiers +----- + +All observational datasets are grouped into in three tiers: + +* **Tier 1**: obs4mips and ana4mips datasets. These datasets are publicly and freely available without any license restrictions. These datasets do not need any reformatting and can be used as is with ESMValTool. +* **Tier 2** other freely available datasets that are not obs4mips. There are no license restrictions. These datasets need to be reformatted to be used with ESMValTool ('CMORization', see above). +* **Tier 3** restricted datasets. Datasets which require registration to be downloaded or that can only be obtained upon request from the respective authors. License restrictions do not allow us to redistribute Tier 3 datasets. The data have to be obtained and reformatted by the user ('CMORization', see above). + +[!NOTE] +.. _tier3_note: +For some of the Tier 3 datasets, we obtained permission from the dataset providers to share the data among ESMValTool users on HPC systems. These Tier 3 datasets are marked with an asterisk in the table in section :ref:`supported datasets below`. + +An overview of the Tier 2 and Tier 3 datasets for which a CMORizing script is available in ESMValTool v2.0 is given in section :ref:`supported datasets below`. + A collection of readily CMORized OBS and OBS6 datasets can be accessed directly on CEDA/JASMIN and DKRZ. At CEDA/JASMIN OBS and OBS6 data is stored in the `esmeval` Group Workspace (GWS), and to be granted read (and execute) permissions to the GWS, one must apply at https://accounts.jasmin.ac.uk/services/group_workspaces/esmeval/ ; after permission has been granted, the user @@ -246,7 +261,7 @@ A list of the datasets for which a CMORizers is available is provided in the fol +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ | CALIPSO-GOCCP | clcalipso (cfMon) | 2 | NCL | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ -| CALIPSO-ICECLOUD | cli (AMon) | 3 | NCL | +| CALIPSO-ICECLOUD* [#t3]_ | cli (AMon) | 3 | NCL | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ | CDS-SATELLITE-ALBEDO | bdalb (Lmon), bhalb (Lmon) | 3 | Python | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ @@ -330,7 +345,7 @@ A list of the datasets for which a CMORizers is available is provided in the fol +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ | ESRL | co2s (Amon) | 2 | NCL | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ -| FLUXCOM | gpp (Lmon) | 3 | Python | +| FLUXCOM* [#t3]_ | gpp (Lmon) | 3 | Python | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ | GCP2018 | fgco2 (Omon [#note3]_), nbp (Lmon [#note3]_) | 2 | Python | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ @@ -380,17 +395,17 @@ A list of the datasets for which a CMORizers is available is provided in the fol +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ | Landschuetzer2020 | spco2 (Omon) | 2 | Python | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ -| MAC-LWP | lwp, lwpStderr (Amon) | 3 | NCL | +| MAC-LWP* [#t3]_ | lwp, lwpStderr (Amon) | 3 | NCL | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ | MERRA | cli, clivi, clt, clw, clwvi, hur, hus, lwp, pr, prw, ps, psl, rlut, rlutcs, rsdt, rsut, rsutcs, ta, | 3 | NCL | | | tas, ts, ua, va, wap, zg (Amon) | | | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ -| MERRA2 | sm (Lmon) | 3 | Python | +| MERRA2* [#t3]_ | sm (Lmon) | 3 | Python | | | clt, pr, evspsbl, hfss, hfls, huss, prc, prsn, prw, ps, psl, rlds, rldscs, rlus, rlut, rlutcs, rsds, | | | | | rsdscs, rsdt, tas, tasmin, tasmax, tauu, tauv, ts, uas, vas, rsus, rsuscs, rsut, rsutcs, ta, ua, va, | | | | | tro3, zg, hus, wap, hur, cl, clw, cli, clwvi, clivi (Amon) | | | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ -| MLS-AURA | hur, hurStderr (day) | 3 | Python | +| MLS-AURA* [#t3]_ | hur, hurStderr (day) | 3 | Python | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ | MOBO-DIC_MPIM | dissic (Omon) | 2 | Python | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ @@ -400,7 +415,7 @@ A list of the datasets for which a CMORizers is available is provided in the fol +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ | MSWEP [#note1]_ | pr | 3 | n/a | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ -| MTE | gpp, gppStderr (Lmon) | 3 | Python | +| MTE* [#t3]_ | gpp, gppStderr (Lmon) | 3 | Python | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ | NCEP-NCAR-R1 | clt, hur, hurs, hus, pr, prw, psl, rlut, rlutcs, rsut, rsutcs, sfcWind, ta, tas, | 2 | Python | | | tasmax, tasmin, ts, ua, va, wap, zg (Amon) | | | @@ -410,7 +425,7 @@ A list of the datasets for which a CMORizers is available is provided in the fol +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ | NDP | cVeg (Lmon) | 3 | Python | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ -| NIWA-BS | toz, tozStderr (Amon) | 3 | NCL | +| NIWA-BS* [#t3]_ | toz, tozStderr (Amon) | 3 | NCL | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ | NOAA-CIRES-20CR-V2 | clt, clwvi, hus, prw, rlut, rsut, pr, tauu, tauv (Amon) | 2 | Python | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ @@ -448,7 +463,7 @@ A list of the datasets for which a CMORizers is available is provided in the fol +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ | TCOM-N2O | n2o (Amon [#note3]_) | 2 | Python | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ -| UWisc | clwvi, lwpStderr (Amon) | 3 | NCL | +| UWisc* [#t3]_ | clwvi, lwpStderr (Amon) | 3 | NCL | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ | WFDE5 | tas, pr (Amon, day) | 2 | Python | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ @@ -456,6 +471,9 @@ A list of the datasets for which a CMORizers is available is provided in the fol | | no3, o2, po4, si (Oyr) | | | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ +.. [#t3] We obtained permission from the dataset provider to share this dataset + among ESMValTool users on HPC systems. + .. [#note1] CMORization is built into ESMValTool through the native6 project, so there is no separate CMORizer script. From dc23cdf484d4194aad6fbc8d673452b1021e2c5f Mon Sep 17 00:00:00 2001 From: Emma Hogan Date: Thu, 21 Nov 2024 16:15:47 +0000 Subject: [PATCH 07/17] Recipe Test Workflow (RTW) prototype (#3210) Co-authored-by: mo-tgeddes <108924122+mo-tgeddes@users.noreply.github.com> Co-authored-by: Katherine Tomkins Co-authored-by: Jon Lillis Co-authored-by: Jon Lillis <68286976+Jon-Lillis@users.noreply.github.com> Co-authored-by: Andrew Clark Co-authored-by: Alistair Sellar Co-authored-by: Alistair Sellar Co-authored-by: Alistair Sellar <16133375+alistairsellar@users.noreply.github.com> Co-authored-by: Ed <146008263+mo-gill@users.noreply.github.com> Co-authored-by: chrisbillowsMO <152496175+chrisbillowsMO@users.noreply.github.com> Co-authored-by: Valeriu Predoi Co-authored-by: sloosvel <45196700+sloosvel@users.noreply.github.com> --- .codacy.yml | 3 +- .github/CODEOWNERS | 1 + .github/workflows/check-rtw.yml | 83 ++++++++ .zenodo.json | 32 ++- CITATION.cff | 34 +++- doc/sphinx/source/gensidebar.py | 2 +- doc/sphinx/source/utils/RTW/about.rst | 14 ++ doc/sphinx/source/utils/RTW/add_a_recipe.rst | 118 +++++++++++ doc/sphinx/source/utils/RTW/common.txt | 33 ++++ doc/sphinx/source/utils/RTW/glossary.rst | 39 ++++ doc/sphinx/source/utils/RTW/index.rst | 11 ++ .../source/utils/RTW/tested_recipes.rst | 19 ++ .../source/utils/RTW/user_guide/index.rst | 9 + .../utils/RTW/user_guide/quick_start.rst | 42 ++++ .../source/utils/RTW/user_guide/workflow.rst | 105 ++++++++++ doc/sphinx/source/{ => utils}/utils.rst | 14 ++ .../app/compare/rose-app.conf | 4 + .../app/configure/bin/__init__.py | 0 .../app/configure/bin/configure.py | 145 ++++++++++++++ .../app/configure/bin/test_configure.py | 76 +++++++ .../app/configure/rose-app.conf | 2 + .../app/get_esmval/bin/clone_latest_esmval.sh | 19 ++ .../app/get_esmval/opt/rose-app-jasmin.conf | 10 + .../get_esmval/opt/rose-app-metoffice.conf | 7 + .../app/get_esmval/rose-app.conf | 0 .../app/install_env_file/rose-app.conf | 11 ++ .../app/process/rose-app.conf | 5 + .../utils/recipe_test_workflow/flow.cylc | 120 ++++++++++++ .../recipe_test_workflow/meta/rose-meta.conf | 185 ++++++++++++++++++ .../opt/rose-suite-jasmin.conf | 10 + .../opt/rose-suite-metoffice.conf | 10 + .../recipe_test_workflow/rose-suite.conf | 24 +++ .../recipe_test_workflow/rose-suite.info | 6 + .../recipe_test_workflow/site/jasmin-env | 59 ++++++ .../recipe_test_workflow/site/jasmin.cylc | 44 +++++ .../recipe_test_workflow/site/metoffice-env | 55 ++++++ .../recipe_test_workflow/site/metoffice.cylc | 60 ++++++ setup.cfg | 3 +- 38 files changed, 1398 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/check-rtw.yml create mode 100644 doc/sphinx/source/utils/RTW/about.rst create mode 100644 doc/sphinx/source/utils/RTW/add_a_recipe.rst create mode 100644 doc/sphinx/source/utils/RTW/common.txt create mode 100644 doc/sphinx/source/utils/RTW/glossary.rst create mode 100644 doc/sphinx/source/utils/RTW/index.rst create mode 100644 doc/sphinx/source/utils/RTW/tested_recipes.rst create mode 100644 doc/sphinx/source/utils/RTW/user_guide/index.rst create mode 100644 doc/sphinx/source/utils/RTW/user_guide/quick_start.rst create mode 100644 doc/sphinx/source/utils/RTW/user_guide/workflow.rst rename doc/sphinx/source/{ => utils}/utils.rst (98%) create mode 100644 esmvaltool/utils/recipe_test_workflow/app/compare/rose-app.conf create mode 100644 esmvaltool/utils/recipe_test_workflow/app/configure/bin/__init__.py create mode 100755 esmvaltool/utils/recipe_test_workflow/app/configure/bin/configure.py create mode 100644 esmvaltool/utils/recipe_test_workflow/app/configure/bin/test_configure.py create mode 100644 esmvaltool/utils/recipe_test_workflow/app/configure/rose-app.conf create mode 100755 esmvaltool/utils/recipe_test_workflow/app/get_esmval/bin/clone_latest_esmval.sh create mode 100644 esmvaltool/utils/recipe_test_workflow/app/get_esmval/opt/rose-app-jasmin.conf create mode 100644 esmvaltool/utils/recipe_test_workflow/app/get_esmval/opt/rose-app-metoffice.conf create mode 100644 esmvaltool/utils/recipe_test_workflow/app/get_esmval/rose-app.conf create mode 100644 esmvaltool/utils/recipe_test_workflow/app/install_env_file/rose-app.conf create mode 100644 esmvaltool/utils/recipe_test_workflow/app/process/rose-app.conf create mode 100644 esmvaltool/utils/recipe_test_workflow/flow.cylc create mode 100644 esmvaltool/utils/recipe_test_workflow/meta/rose-meta.conf create mode 100644 esmvaltool/utils/recipe_test_workflow/opt/rose-suite-jasmin.conf create mode 100644 esmvaltool/utils/recipe_test_workflow/opt/rose-suite-metoffice.conf create mode 100644 esmvaltool/utils/recipe_test_workflow/rose-suite.conf create mode 100644 esmvaltool/utils/recipe_test_workflow/rose-suite.info create mode 100755 esmvaltool/utils/recipe_test_workflow/site/jasmin-env create mode 100644 esmvaltool/utils/recipe_test_workflow/site/jasmin.cylc create mode 100755 esmvaltool/utils/recipe_test_workflow/site/metoffice-env create mode 100644 esmvaltool/utils/recipe_test_workflow/site/metoffice.cylc diff --git a/.codacy.yml b/.codacy.yml index 06a0ea342f..afe979f5c7 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -21,5 +21,6 @@ engines: exclude_paths: [ 'doc/sphinx/**', 'esmvaltool/cmor/tables/**', - 'tests/**' + 'tests/**', + 'esmvaltool/utils/recipe_test_workflow/app/configure/bin/test_configure.py' ] diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2086d60173..3478d469b4 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,3 @@ esmvaltool/cmorizers @ESMValGroup/obs-maintainers .github/workflows @valeriupredoi +esmvaltool/utils/recipe_test_workflow/ @alistairsellar @ehogan diff --git a/.github/workflows/check-rtw.yml b/.github/workflows/check-rtw.yml new file mode 100644 index 0000000000..611601dfd7 --- /dev/null +++ b/.github/workflows/check-rtw.yml @@ -0,0 +1,83 @@ +# This workflow performs various validation steps for Cylc and Rose. +name: Check Recipe Test Workflow (RTW) + +# Controls when the action will run +on: + # Triggers the workflow on push events + push: + paths: +# - esmvaltool/utils/recipe_test_workflow/** + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Common variables are defined here +env: + RTW_ROOT_DIR: esmvaltool/utils/recipe_test_workflow + +# Required shell entrypoint to have properly configured bash shell +defaults: + run: + shell: bash -l {0} + +# A workflow run is made up of one or more jobs that can run +# sequentially or in parallel +jobs: + # This workflow contains a single job called "check-rtw" + check-rtw: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part + # of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job + # can access it + - uses: actions/checkout@v4 + - uses: conda-incubator/setup-miniconda@v3 + with: + miniforge-version: "latest" + miniforge-variant: Miniforge3 + use-mamba: true + conda-remove-defaults: "true" + + - name: Install Cylc and Rose + run: conda install cylc-flow>=8.2 cylc-rose metomi-rose + + - name: Check current environment + run: conda list + + - name: Validate Cylc workflow + run: | + cd ${RTW_ROOT_DIR} + cylc validate . -O metoffice + + - name: Run Cylc configuration linter + run: | + cd ${RTW_ROOT_DIR} + cylc lint + + - name: Validate format of Rose configuration files + run: | + cd ${RTW_ROOT_DIR} + output="$(rose config-dump)" + msg="Run 'rose config-dump' to re-dump the Rose configuration files" + msg="${msg} in the common format, then commit the changes." + # The '-z' option returns true if 'output' is empty. + if [[ -z "${output}" ]]; then true; else echo "${msg}" && exit 1; fi + + - name: Validate Rose configuration metadata + run: | + cd ${RTW_ROOT_DIR} + rose metadata-check -C meta/ + + - name: Run Rose configuration validation macros + run: | + cd ${RTW_ROOT_DIR} + rose macro -V + + - name: Lint shell scripts + run: | + cd ${RTW_ROOT_DIR} + output=$(find . -name "*.sh" -exec shellcheck {} \;) + if [ "$output" ]; then echo "${output}" && exit 1; fi diff --git a/.zenodo.json b/.zenodo.json index c087c4ae21..be799a9dc1 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -81,13 +81,17 @@ "name": "Berg, Peter", "orcid": "0000-0002-1469-2568" }, + { + "affiliation": "Met Office, UK", + "name": "Billows, Chris" + }, { "affiliation": "DLR, Germany", "name": "Bock, Lisa", "orcid": "0000-0001-7058-5938" }, { - "affiliation": "MetOffice, UK", + "affiliation": "Met Office, UK", "name": "Bodas-Salcedo, Alejandro", "orcid": "0000-0002-7890-2536" }, @@ -142,7 +146,7 @@ "name": "Docquier, David" }, { - "affiliation": "MetOffice, UK", + "affiliation": "Met Office, UK", "name": "Dreyer, Laura" }, { @@ -150,13 +154,21 @@ "name": "Ehbrecht, Carsten" }, { - "affiliation": "MetOffice, UK", + "affiliation": "Met Office, UK", "name": "Earnshaw, Paul" }, + { + "affiliation": "Met Office, UK", + "name": "Geddes, Theo" + }, { "affiliation": "University of Bremen, Germany", "name": "Gier, Bettina" }, + { + "affiliation": "Met Office, UK", + "name": "Gillett, Ed" + }, { "affiliation": "BSC, Spain", "name": "Gonzalez-Reviriego, Nube", @@ -191,6 +203,10 @@ "name": "Heuer, Helge", "orcid": "0000-0003-2411-7150" }, + { + "affiliation": "Met Office, UK", + "name": "Hogan, Emma" + }, { "affiliation": "BSC, Spain", "name": "Hunter, Alasdair", @@ -227,7 +243,7 @@ "orcid": "0000-0001-6085-5914" }, { - "affiliation": "MetOffice, UK", + "affiliation": "Met Office, UK", "name": "Little, Bill" }, { @@ -279,7 +295,7 @@ "name": "Sandstad, Marit" }, { - "affiliation": "MetOffice, UK", + "affiliation": "Met Office, UK", "name": "Sellar, Alistair" }, { @@ -305,6 +321,10 @@ "name": "Swaminathan, Ranjini", "orcid": "0000-0001-5853-2673" }, + { + "affiliation": "Met Office, UK", + "name": "Tomkins, Katherine" + }, { "affiliation": "BSC, Spain", "name": "Torralba, Verónica" @@ -387,7 +407,7 @@ "orcid": "0000-0003-3780-0784" }, { - "affiliation": "MetOffice, UK", + "affiliation": "Met Office, UK", "name": "Munday, Gregory", "orcid": "0000-0003-4750-9923" } diff --git a/CITATION.cff b/CITATION.cff index 1934c36ef1..ab158d2436 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -85,13 +85,17 @@ authors: family-names: Berg given-names: Peter orcid: "https://orcid.org/0000-0002-1469-2568" + - + affiliation: "Met Office, UK" + family-names: Billows + given-names: Chris - affiliation: "DLR, Germany" family-names: Bock given-names: Lisa orcid: "https://orcid.org/0000-0001-7058-5938" - - affiliation: "MetOffice, UK" + affiliation: "Met Office, UK" family-names: Bodas-Salcedo given-names: Alejandro orcid: "https://orcid.org/0000-0002-7890-2536" @@ -146,7 +150,7 @@ authors: family-names: Docquier given-names: David - - affiliation: "MetOffice, UK" + affiliation: "Met Office, UK" family-names: Dreyer given-names: Laura - @@ -154,13 +158,21 @@ authors: family-names: Ehbrecht given-names: Carsten - - affiliation: "MetOffice, UK" + affiliation: "Met Office, UK" family-names: Earnshaw given-names: Paul + - + affiliation: "Met Office, UK" + family-names: Geddes + given-names: Theo - affiliation: "University of Bremen, Germany" family-names: Gier given-names: Bettina + - + affiliation: "Met Office, UK" + family-names: Gillett + given-names: Ed - affiliation: "BSC, Spain" family-names: Gonzalez-Reviriego @@ -196,6 +208,10 @@ authors: family-names: Heuer given-names: Helge orcid: "https://orcid.org/0000-0003-2411-7150" + - + affiliation: "Met Office, UK" + family-names: Hogan + given-names: Emma - affiliation: "BSC, Spain" family-names: Hunter @@ -232,7 +248,7 @@ authors: given-names: Valerio orcid: "https://orcid.org/0000-0001-6085-5914" - - affiliation: "MetOffice, UK" + affiliation: "Met Office, UK" family-names: Little given-names: Bill - @@ -289,7 +305,7 @@ authors: family-names: Sandstad given-names: Marit - - affiliation: "MetOffice, UK" + affiliation: "Met Office, UK" family-names: Sellar given-names: Alistair - @@ -315,6 +331,10 @@ authors: family-names: Swaminathan given-names: Ranjini orcid: "https://orcid.org/0000-0001-5853-2673" + - + affiliation: "Met Office, UK" + family-names: Tomkins + given-names: Katherine - affiliation: "BSC, Spain" family-names: Torralba @@ -396,8 +416,8 @@ authors: family-names: Bonnet given-names: Pauline orcid: "https://orcid.org/0000-0003-3780-0784" - - - affiliation: "MetOffice, UK" + - + affiliation: "Met Office, UK" family-names: Munday given-names: Gregory orcid: "https://orcid.org/0000-0003-4750-9923" diff --git a/doc/sphinx/source/gensidebar.py b/doc/sphinx/source/gensidebar.py index 970722ff0a..f8b766ab7d 100644 --- a/doc/sphinx/source/gensidebar.py +++ b/doc/sphinx/source/gensidebar.py @@ -65,7 +65,7 @@ def _header(project, text): _write("esmvaltool", "Obtaining input data", "input") _write("esmvaltool", "Making a recipe or diagnostic", "develop/index") _write("esmvaltool", "Contributing to the community", "community/index") - _write("esmvaltool", "Utilities", "utils") + _write("esmvaltool", "Utilities", "utils/utils") _write("esmvaltool", "Diagnostics API Reference", "api/esmvaltool") _write("esmvaltool", "Frequently Asked Questions", "faq") _write("esmvaltool", "Changelog", "changelog") diff --git a/doc/sphinx/source/utils/RTW/about.rst b/doc/sphinx/source/utils/RTW/about.rst new file mode 100644 index 0000000000..62883fe2e1 --- /dev/null +++ b/doc/sphinx/source/utils/RTW/about.rst @@ -0,0 +1,14 @@ +***** +About +***** + +.. include:: common.txt + +The Recipe Test Workflow (|RTW|) is a workflow that is used to regularly run +recipes so issues can be discovered during the development process sooner +rather than later. + +|Cylc| v8 and |Rose| v2 are used as the workflow engine and application +configuration system for the |RTW|, respectively. |Cylc| and |Rose| are not +included in the ESMValTool environment as they are typically already centrally +installed at sites e.g. JASMIN and the Met Office. diff --git a/doc/sphinx/source/utils/RTW/add_a_recipe.rst b/doc/sphinx/source/utils/RTW/add_a_recipe.rst new file mode 100644 index 0000000000..6e495e1f1c --- /dev/null +++ b/doc/sphinx/source/utils/RTW/add_a_recipe.rst @@ -0,0 +1,118 @@ +How to add a recipe to the |RTW| +================================ + +.. include:: common.txt + +.. note:: + Before you follow these steps to add your recipe, you must be able to + successfully run the recipe with the latest version of ESMValTool on the + compute server you use at your site, as detailed by the ``platform`` option + in the ``[[COMPUTE]]`` section in the site-specific ``.cylc`` file in the + ``esmvaltool/utils/recipe_test_workflow/site/`` directory. + +#. Open a `new ESMValTool issue`_ on GitHub, assign yourself to the issue, and + add the ``Recipe Test Workflow (RTW)`` label to the issue, see + `ESMValTool issue #3663`_ for an example. + +#. Create a branch. + +#. Obtain the duration and memory usage of the recipe from the messages printed + to screen, or at the end of the ``run/main_log.txt`` file in the recipe + output directory after running your recipe on the compute cluster you use at + your site; these messages will look something like:: + + YYYY-MM-DD HH:MM:SS:sss UTC [12345] INFO Time for running the recipe was: 0:02:13.334742 + YYYY-MM-DD HH:MM:SS:sss UTC [12345] INFO Maximum memory used (estimate): 2.4 GB + [...] + YYYY-MM-DD HH:MM:SS:sss UTC [12345] INFO Run was successful + +#. Add the recipe to the ``[task parameters]`` section in the + ``esmvaltool/utils/recipe_test_workflow/flow.cylc`` file. + + .. hint:: + If the recipe takes less than 10 minutes to run then it should be added + to the ``fast`` option. Recipes that take longer than ten minutes should + be added to the ``medium`` option. + + .. hint:: + The line added should follow the format of ``recipe_new_recipe, \``, + unless the line is the last one in the list, in which case the line added + should follow the format of ``recipe_new_recipe``. + +#. If the duration of the recipe is larger than the value specified by the + ``execution time limit`` option in the ``[[COMPUTE]]`` section in the + aforementioned site-specific ``.cylc`` file, and / or the memory usage of + the recipe is larger than the value specified by the ``--mem`` option in the + ``[[[directives]]]`` section in the ``[[COMPUTE]]`` section, add a section + (in alphabetical order) to this file as shown below (round the duration to + the nearest second):: + + [[process]] + # Actual: 0m31s, 2.5 GB on 2024-04-08. + execution time limit = PT2M + [[[directives]]] + --mem = 3G + + .. hint:: + The ``fast`` key in the example task definition above + (``[[process]]``) should match name of the + option the recipe was added to in the ``[task parameters]`` section in + the ``esmvaltool/utils/recipe_test_workflow/flow.cylc`` file + + .. hint:: + Set the ``execution time limit`` to 10-20% more than the actual duration. + For actual durations of up to ``1m45s``, set the ``execution time limit`` + to ``PT2M`` (2 minutes). + + .. hint:: + Try not to regularly waste more than 500 MiB in memory usage. Typically, + rounding the actual memory usage up to the nearest integer is acceptable. + +#. Stop any running ``recipe_test_workflow`` workflows:: + + cylc stop recipe_test_workflow/* + +#. Run the |RTW|, as detailed in the :ref:`quick_start_guide`; it is expected + that the ``compare`` task will fail. + +#. Update the Known Good Outputs (|KGOs|): + + * Recursively copy the recipe output directory (i.e. + ``recipe___