Python 3.9: remove pynio as dependency and replace with rasterio and …

…pin Matplotlib>3.3.1 and pin cartopy>=0.18 (#1997)


Co-authored-by: Tomas Lovato <[email protected]>

.circleci/config.yml

@@ -71,7 +71,8 @@ jobs:
             tar xfz julia-*-linux-x86_64.tar.gz
             ln -s $(pwd)/julia-*/bin/julia /usr/bin/julia
             # conda update -y conda > /logs/conda.txt 2>&1
-            conda env update >> /logs/conda.txt 2>&1
+            # conda env update >> /logs/conda.txt 2>&1
+            conda env create -n esmvaltool -f environment.yml >> /logs/conda.txt 2>&1
             set +x; conda activate esmvaltool; set -x
             pip install .[test] > /logs/install.txt 2>&1
             esmvaltool install R
@@ -122,7 +123,8 @@ jobs:
             tar xfz julia-*-linux-x86_64.tar.gz
             ln -s $(pwd)/julia-*/bin/julia /usr/bin/julia
             # conda update -y conda > /logs/conda.txt 2>&1
-            conda env update >> /logs/conda.txt 2>&1
+            # conda env update >> /logs/conda.txt 2>&1
+            conda env create -n esmvaltool -f environment.yml >> /logs/conda.txt 2>&1
             set +x; conda activate esmvaltool; set -x
             pip install -e .[develop] > /logs/install.txt 2>&1
             esmvaltool install R
@@ -183,16 +185,16 @@ jobs:
             set -x
             # Install prerequisites
             mkdir /logs
+            apt update && apt install time
             wget https://julialang-s3.julialang.org/bin/linux/x64/1.0/julia-1.0.3-linux-x86_64.tar.gz
             tar xfz julia-*-linux-x86_64.tar.gz
             ln -s $(pwd)/julia-*/bin/julia /usr/bin/julia
-            # conda update -y conda > /logs/conda_base.txt 2>&1
-            conda install -y conda-build conda-verify >> /logs/conda_base.txt 2>&1
+            conda create -y -n build conda-build conda-verify
+            conda activate build
             # Log versions
             dpkg -l > /logs/versions.txt
-            conda env export -n base > /logs/build_environment.yml
+            conda env export > /logs/build_environment.yml
             # Build conda package
-            apt update && apt install time
             \time -v conda build package -c conda-forge -c esmvalgroup > /logs/build_log.txt
           no_output_timeout: 30m
       - store_artifacts:

doc/sphinx/source/conf.py

@@ -79,7 +79,7 @@
     'GDAL',
     'iris',
     'psutil',
-    'pynio',
+    'rasterio',
     'scipy',
     'sklearn',
     'xesmf',

environment.yml

@@ -6,18 +6,18 @@ channels:
 
 dependencies:
   # Python packages that cannot be installed from PyPI:
+  - cartopy>=0.18
   - compilers
   - gdal
   - esmpy
   - esmvalcore>=2.1.0,<2.2
   - iris>=2.2.1,<3
-  - matplotlib>=3,<3.3
+  - matplotlib>3.3.1  # bug in 3.3.1 but 3.3.3 gets installed automatically
   # Non-Python dependencies
   - cdo>=1.9.7
   - eccodes!=2.19.0  # cdo dependency; something messed up with libeccodes.so
   - imagemagick
   - nco
-  - pynio
   - scikit-learn  # may hit hw-specific issue if from pypi https://github.com/scikit-learn/scikit-learn/issues/14485
 
   # Multi language support:

esmvaltool/cmorizers/obs/cmorize_obs_eppley_vgpm_modis.py

@@ -21,9 +21,9 @@
 import logging
 import os
 import glob
-from datetime import datetime as dt
 import xarray as xr
 import numpy as np
+import pandas as pd
 
 import iris
 
@@ -70,24 +70,23 @@ def merge_data(in_dir, out_dir, raw_info):
     var = raw_info['name']
     filelist = sorted(glob.glob(in_dir + '/' + raw_info['file'] + '*.hdf'))
     for filename in filelist:
-        ds = xr.open_dataset(filename, engine='pynio').rename({
-            'fakeDim0': 'lat',
-            'fakeDim1': 'lon'
-        })
+        ds = xr.open_rasterio(filename).rename({
+            'y': 'lat',
+            'x': 'lon'
+        }).squeeze().drop('band')
         # create coordinates
         ds = ds.assign_coords(
-            time=dt.strptime(ds.attrs['Start_Time_String'],
-                             '%m/%d/%Y %H:%M:%S'))
+            time=pd.to_datetime(filename[-11:-4], format='%Y%j'))
         ds = ds.expand_dims(dim='time', axis=0)
-        dx = 90. / ds.dims['lat']
-        ds = ds.assign_coords(
-            lat=np.linspace(-90. + dx, 90. - dx, ds.dims['lat']))
+        dx = 90. / ds.lat.size
+        ds = ds.assign_coords(lat=np.linspace(-90. + dx, 90. -
+                                              dx, ds.lat.size))
         ds.lat.attrs = {'long_name': 'Latitude', 'units': 'degrees_north'}
-        ds = ds.assign_coords(
-            lon=np.linspace(-180. + dx, 180. - dx, ds.dims['lon']))
+        ds = ds.assign_coords(lon=np.linspace(-180. + dx, 180. -
+                                              dx, ds.lon.size))
         ds.lon.attrs = {'long_name': 'Longitude', 'units': 'degrees_east'}
         # get current file data
-        da.append(ds[var])
+        da.append(ds)
     damerge = xr.concat(da, dim='time')
 
     # need data flip to match coordinates
@@ -106,7 +105,7 @@ def merge_data(in_dir, out_dir, raw_info):
             'calendar': 'gregorian'
         },
         var: {
-            '_FillValue': ds[var].attrs['Hole_Value']
+            '_FillValue': -9999.0
         }
     }
     filename = os.path.join(out_dir, raw_info['file'] + '_merged.nc')
@@ -128,7 +127,7 @@ def cmorization(in_dir, out_dir, cfg, _):
         glob_attrs['mip'] = vals['mip']
         raw_info = {'name': vals['raw'], 'file': vals['file']}
 
-        # merge yearly data and apply binning
+        # merge data
         inpfile = merge_data(in_dir, out_dir, raw_info)
 
         logger.info("CMORizing var %s from file %s", var, inpfile)

package/meta.yaml

@@ -66,7 +66,7 @@ outputs:
         - python>=3.6
         - setuptools_scm
       run:
-        - cartopy
+        - cartopy>=0.18
         - cdo>=1.9.7
         - eccodes!=2.19.0  # cdo dependency; something messed up with libeccodes.so
         - cdsapi
@@ -84,17 +84,17 @@ outputs:
         - jinja2
         - joblib
         - lime
-        - matplotlib>=3,<3.3
+        - matplotlib>3.3.1  # bug in 3.3.1
         - natsort
         - nc-time-axis
         - netCDF4
         - numpy
         - pandas
-        - pynio
         - pyproj>=2.1
         - python>=3.6
         - python-cdo
         - pyyaml
+        - rasterio  # replaces pynio
         - scikit-image
         - scikit-learn
         - seaborn

setup.py

@@ -23,7 +23,7 @@
     # Installation dependencies
     # Use with pip install . to install from source
     'install': [
-        'cartopy',
+        'cartopy>=0.18',
         'cdo',
         'cdsapi',
         'cf-units',
@@ -39,15 +39,15 @@
         'jinja2',
         'joblib',
         'lime',
-        'matplotlib>=3,<3.3',
+        'matplotlib>3.3.1',  # bug in 3.3.1, 3.3.2 and 3 fine
         'natsort',
         'nc-time-axis',  # needed by iris.plot
         'netCDF4',
         'numpy',
         'pandas',
-        'pynio',
         'pyproj>=2.1'
         'pyyaml',
+        'rasterio',  # replaces pynio
         'scikit-image',
         'scikit-learn',
         'scipy',