Skip to content

Commit 29e359a

Browse files
marqhmarqh
committed
adopt eccodes, use gribapi.grib_is_missing
1 parent 077b6ee commit 29e359a

19 files changed

+69
-82
lines changed

‎.travis.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ install:
5959
conda install --quiet --file minimal-conda-requirements.txt;
6060
else
6161
if [[ "$TRAVIS_PYTHON_VERSION" == 3* ]]; then
62-
sed -e '/ecmwf_grib/d' -e '/esmpy/d' -e 's/#.\+$//' conda-requirements.txt | xargs conda install --quiet;
62+
sed -e '/esmpy/d' -e 's/#.\+$//' conda-requirements.txt | xargs conda install --quiet;
6363
else
6464
conda install --quiet --file conda-requirements.txt;
6565
fi

‎conda-requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ imagehash
2525
requests
2626

2727
# Optional iris dependencies
28-
python-ecmwf_grib
28+
python-eccodes
2929
esmpy>=7.0
3030
gdal
3131
libmo_unpack

‎lib/iris/fileformats/grib/_load_convert.py‎

Lines changed: 5 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,9 @@ def _unscale(v, f):
158158

159159
if isinstance(value, Iterable) or isinstance(factor, Iterable):
160160
def _masker(item):
161-
result = ma.masked_equal(item, _MDI)
161+
numerical_mdi = 2 ** 32 - 1
162+
item = [numerical_mdi if i is None else i for i in item]
163+
result = ma.masked_equal(item, numerical_mdi)
162164
if ma.count_masked(result):
163165
# Circumvent downstream NumPy "RuntimeWarning"
164166
# of "overflow encountered in power" in _unscale
@@ -177,30 +179,8 @@ def _masker(item):
177179
return result
178180

179181

180-
# Regulations 92.1.4 and 92.1.5.
181-
_MDI = 2 ** 32 - 1
182-
# Note:
183-
# 1. Integer "on-disk" values (aka. coded keys) in GRIB messages:
184-
# - Are 8-, 16-, or 32-bit.
185-
# - Are either signed or unsigned, with signed values stored as
186-
# sign-and-magnitude (*not* twos-complement).
187-
# - Use all bits set to indicate a missing value (MDI).
188-
# 2. Irrespective of the on-disk form, the ECMWF GRIB API *always*:
189-
# - Returns values as 64-bit signed integers, either as native
190-
# Python 'int' or numpy 'int64'.
191-
# - Returns missing values as 2**32 - 1, but not all keys are
192-
# defined as supporting missing values.
193-
# NB. For keys which support missing values, the MDI value is reliably
194-
# distinct from the valid range of either signed or unsigned 8-, 16-,
195-
# or 32-bit values. For example:
196-
# unsigned 32-bit:
197-
# min = 0b000...000 = 0
198-
# max = 0b111...110 = 2**32 - 2
199-
# MDI = 0b111...111 = 2**32 - 1
200-
# signed 32-bit:
201-
# MDI = 0b111...111 = 2**32 - 1
202-
# min = 0b111...110 = -(2**31 - 2)
203-
# max = 0b011...111 = 2**31 - 1
182+
# Use ECCodes gribapi to recognise missing value
183+
_MDI = None
204184

205185

206186
# Non-standardised usage for negative forecast times.

‎lib/iris/fileformats/grib/_save_rules.py‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -257,8 +257,21 @@ def latlon_first_last(x_coord, y_coord, grib):
257257
def dx_dy(x_coord, y_coord, grib):
258258
x_step = regular_step(x_coord)
259259
y_step = regular_step(y_coord)
260-
gribapi.grib_set(grib, "DxInDegrees", float(abs(x_step)))
261-
gribapi.grib_set(grib, "DyInDegrees", float(abs(y_step)))
260+
# Set x and y step. For degrees, this is encoded as an integer:
261+
# 1 * 10^6 * floating point value.
262+
# WMO Manual on Codes regulation 92.1.6
263+
if x_coord.units == 'degrees':
264+
gribapi.grib_set(grib, "iDirectionIncrement",
265+
round(1e6 * float(abs(x_step))))
266+
else:
267+
raise ValueError('X coordinate must be in degrees, not {}'
268+
'.'.format(x_coord.units))
269+
if y_coord.units == 'degrees':
270+
gribapi.grib_set(grib, "jDirectionIncrement",
271+
round(1e6 * float(abs(y_step))))
272+
else:
273+
raise ValueError('Y coordinate must be in degrees, not {}'
274+
'.'.format(y_coord.units))
262275

263276

264277
def scanning_mode_flags(x_coord, y_coord, grib):

‎lib/iris/fileformats/grib/message.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -459,8 +459,12 @@ def _get_key_value(self, key):
459459
# By default these values are returned as unhelpful strings but
460460
# we can use int representation to compare against instead.
461461
res = gribapi.grib_get(self._message_id, key, int)
462+
if gribapi.grib_is_missing(self._message_id, key) == 1:
463+
res = None
462464
else:
463465
res = gribapi.grib_get(self._message_id, key)
466+
if gribapi.grib_is_missing(self._message_id, key) == 1:
467+
res = None
464468
return res
465469

466470
def get_computed_key(self, key):
@@ -482,6 +486,8 @@ def get_computed_key(self, key):
482486
res = gribapi.grib_get_array(self._message_id, key)
483487
else:
484488
res = gribapi.grib_get(self._message_id, key)
489+
if gribapi.grib_is_missing(self._message_id, key) == 1:
490+
res = None
485491
return res
486492

487493
def keys(self):

‎lib/iris/tests/integration/format_interop/test_name_grib.py‎

Lines changed: 13 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# (C) British Crown Copyright 2013 - 2016, Met Office
1+
# (C) British Crown Copyright 2013 - 2017, Met Office
22
#
33
# This file is part of Iris.
44
#
@@ -72,21 +72,19 @@ def check_common(self, name_cube, grib_cube):
7272
def test_name2_field(self):
7373
filepath = tests.get_data_path(('NAME', 'NAMEII_field.txt'))
7474
name_cubes = iris.load(filepath)
75-
# Check gribapi version, because we currently have a known load/save
76-
# problem with gribapi 1v14 (at least).
77-
gribapi_ver = gribapi.grib_get_api_version()
78-
gribapi_fully_supported_version = \
79-
(StrictVersion(gribapi.grib_get_api_version()) <
80-
StrictVersion('1.13'))
75+
76+
# There is a known load/save problem with numerous
77+
# gribapi/eccodes versions and
78+
# zero only data, where min == max.
79+
# This may be a problem with data scaling.
8180
for i, name_cube in enumerate(name_cubes):
82-
if not gribapi_fully_supported_version:
83-
data = name_cube.data
84-
if np.min(data) == np.max(data):
85-
msg = ('NAMEII cube #{}, "{}" has empty data : '
86-
'SKIPPING test for this cube, as save/load will '
87-
'not currently work with gribabi > 1v12.')
88-
warnings.warn(msg.format(i, name_cube.name()))
89-
continue
81+
data = name_cube.data
82+
if np.min(data) == np.max(data):
83+
msg = ('NAMEII cube #{}, "{}" has empty data : '
84+
'SKIPPING test for this cube, as save/load will '
85+
'not currently work.')
86+
warnings.warn(msg.format(i, name_cube.name()))
87+
continue
9088

9189
with self.temp_filename('.grib2') as temp_filename:
9290
iris.save(name_cube, temp_filename)

‎lib/iris/tests/test_grib_save.py‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# (C) British Crown Copyright 2010 - 2016, Met Office
1+
# (C) British Crown Copyright 2010 - 2017, Met Office
22
#
33
# This file is part of Iris.
44
#
@@ -34,6 +34,7 @@
3434

3535
if tests.GRIB_AVAILABLE:
3636
import gribapi
37+
from iris.fileformats.grib._load_convert import _MDI as MDI
3738

3839

3940
@tests.skip_data
@@ -50,10 +51,10 @@ def test_latlon_forecast_plev(self):
5051
iris.save(cubes, temp_file_path)
5152
expect_diffs = {'totalLength': (4837, 4832),
5253
'productionStatusOfProcessedData': (0, 255),
53-
'scaleFactorOfRadiusOfSphericalEarth': (4294967295,
54+
'scaleFactorOfRadiusOfSphericalEarth': (MDI,
5455
0),
5556
'shapeOfTheEarth': (0, 1),
56-
'scaledValueOfRadiusOfSphericalEarth': (4294967295,
57+
'scaledValueOfRadiusOfSphericalEarth': (MDI,
5758
6367470),
5859
'typeOfGeneratingProcess': (0, 255),
5960
'generatingProcessIdentifier': (128, 255),
@@ -70,10 +71,10 @@ def test_rotated_latlon(self):
7071
iris.save(cubes, temp_file_path)
7172
expect_diffs = {'totalLength': (648196, 648191),
7273
'productionStatusOfProcessedData': (0, 255),
73-
'scaleFactorOfRadiusOfSphericalEarth': (4294967295,
74+
'scaleFactorOfRadiusOfSphericalEarth': (MDI,
7475
0),
7576
'shapeOfTheEarth': (0, 1),
76-
'scaledValueOfRadiusOfSphericalEarth': (4294967295,
77+
'scaledValueOfRadiusOfSphericalEarth': (MDI,
7778
6367470),
7879
'longitudeOfLastGridPoint': (392109982, 32106370),
7980
'latitudeOfLastGridPoint': (19419996, 19419285),
@@ -91,10 +92,10 @@ def test_time_mean(self):
9192
cubes = iris.load(source_grib)
9293
expect_diffs = {'totalLength': (21232, 21227),
9394
'productionStatusOfProcessedData': (0, 255),
94-
'scaleFactorOfRadiusOfSphericalEarth': (4294967295,
95+
'scaleFactorOfRadiusOfSphericalEarth': (MDI,
9596
0),
9697
'shapeOfTheEarth': (0, 1),
97-
'scaledValueOfRadiusOfSphericalEarth': (4294967295,
98+
'scaledValueOfRadiusOfSphericalEarth': (MDI,
9899
6367470),
99100
'longitudeOfLastGridPoint': (356249908, 356249809),
100101
'latitudeOfLastGridPoint': (-89999938, -89999944),

‎lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_12.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,10 @@
3333
import iris.coords
3434
import iris.exceptions
3535
from iris.fileformats.grib._load_convert import grid_definition_template_12
36+
from iris.fileformats.grib._load_convert import _MDI as MDI
3637
from iris.tests.unit.fileformats.grib.load_convert import empty_metadata
3738

3839

39-
MDI = 2 ** 32 - 1
40-
41-
4240
class Test(tests.IrisTest):
4341
def section_3(self):
4442
section = {

‎lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_20.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,10 @@
3232
import iris.coord_systems
3333
import iris.coords
3434
from iris.fileformats.grib._load_convert import grid_definition_template_20
35+
from iris.fileformats.grib._load_convert import _MDI as MDI
3536
from iris.tests.unit.fileformats.grib.load_convert import empty_metadata
3637

3738

38-
MDI = 2 ** 32 - 1
39-
40-
4139
class Test(tests.IrisTest):
4240

4341
def section_3(self):

‎lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_30.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,10 @@
3232
import iris.coord_systems
3333
import iris.coords
3434
from iris.fileformats.grib._load_convert import grid_definition_template_30
35+
from iris.fileformats.grib._load_convert import _MDI as MDI
3536
from iris.tests.unit.fileformats.grib.load_convert import empty_metadata
3637

3738

38-
MDI = 2 ** 32 - 1
39-
40-
4139
class Test(tests.IrisTest):
4240

4341
def section_3(self):

0 commit comments

Comments
 (0)