Import aliases from CLDR data and use them better

We had some manual fallbacks for missing field/unit lengths,
and ignored many `<alias>`es in CLDR data entirely.

This PR imports those aliases and uses them to resolve missing lengths,
improving output in corner cases and fixes some rare KeyError crashes too.

There's a quirk related to person-variant unit aliases,
which are only expressed in CLDR for the short length.
If one follows the spec to the letter,
a request for a long/narrow person-variant unit will route
through the short length and lose the requested length.
We use the same "workaround" as ICU to preserve the length,
by importing these as whole-unit aliases. Not totally spec-
compliant, but the end result is simply better.

Fixes #1076

Author: akx

babel/dates.py

@@ -992,20 +992,13 @@ def _iter_patterns(a_unit):
         if add_direction:
             # Try to find the length variant version first ("year-narrow")
             # before falling back to the default.
-            unit_rel_patterns = date_fields.get(f"{a_unit}-{format}") or date_fields[a_unit]
+            unit_rel_patterns = date_fields.get(f"{a_unit}-{format}") or date_fields.get(a_unit) or {}
             if seconds >= 0:
                 yield unit_rel_patterns['future']
             else:
                 yield unit_rel_patterns['past']
         a_unit = f"duration-{a_unit}"
-        unit_pats = unit_patterns.get(a_unit, {})
-        yield unit_pats.get(format)
-        # We do not support `<alias>` tags at all while ingesting CLDR data,
-        # so these aliases specified in `root.xml` are hard-coded here:
-        # <unitLength type="long"><alias source="locale" path="../unitLength[@type='short']"/></unitLength>
-        # <unitLength type="narrow"><alias source="locale" path="../unitLength[@type='short']"/></unitLength>
-        if format in ("long", "narrow"):
-            yield unit_pats.get("short")
+        yield unit_patterns.get(a_unit, {}).get(format)  # resolves aliases
 
     for unit, secs_per_unit in TIMEDELTA_UNITS:
         value = abs(seconds) / secs_per_unit

babel/units.py

@@ -174,21 +174,12 @@ def format_unit(
         )
         plural_form = locale.plural_form(value)
 
-    unit_patterns = locale._data["unit_patterns"][q_unit]
-
-    # We do not support `<alias>` tags at all while ingesting CLDR data,
-    # so these aliases specified in `root.xml` are hard-coded here:
-    # <unitLength type="long"><alias source="locale" path="../unitLength[@type='short']"/></unitLength>
-    # <unitLength type="narrow"><alias source="locale" path="../unitLength[@type='short']"/></unitLength>
-    lengths_to_check = [length, "short"] if length in ("long", "narrow") else [length]
-
-    for real_length in lengths_to_check:
-        length_patterns = unit_patterns.get(real_length, {})
-        # Fall back from the correct plural form to "other"
-        # (this is specified in LDML "Lateral Inheritance")
-        pat = length_patterns.get(plural_form) or length_patterns.get("other")
-        if pat:
-            return pat.format(formatted_value)
+    length_patterns = locale._data["unit_patterns"][q_unit].get(length, {})  # resolves aliases
+    # Fall back from the correct plural form to "other"
+    # (this is specified in LDML "Lateral Inheritance")
+    pat = length_patterns.get(plural_form) or length_patterns.get("other")
+    if pat:
+        return pat.format(formatted_value)
 
     # Fall back to a somewhat bad representation.
     # nb: This is marked as no-cover, as the current CLDR seemingly has no way for this to happen.

scripts/import_cldr.py

@@ -78,6 +78,19 @@ def _translate_alias(ctxt, path):
     return keys
 
 
+def _sibling_alias_type(alias_elem) -> str:
+    """
+    Extract target type from an alias path pointing at a sibling.
+
+    E.g. ``../unit[@type='duration-year']`` -> ``duration-year``.
+    """
+    path_attr = alias_elem.attrib['path']
+    parent, _, leaf = path_attr.rpartition('/')
+    if parent != "..":
+        raise ValueError(f"not a sibling alias: {path_attr}")
+    return TYPE_ATTR_RE.match(leaf).group(1)
+
+
 def _parse_currency_date(s):
     if not s:
         return None
@@ -805,8 +818,20 @@ def parse_decimal_formats(data, tree):
             length_type = elem.attrib.get('type')
             if _should_skip_elem(elem, length_type, decimal_formats):
                 continue
-            if elem.findall('./alias'):
-                # TODO map the alias to its target
+            alias_elem = elem.find('./alias')
+            if alias_elem is not None:
+                if alias_elem.attrib['path'] != "../decimalFormatLength[@type='short']":
+                    raise ValueError("Unexpected decimal format alias in new CLDR data; this may require hand-holding.")
+                    # In particular: if you're reading this, the tribal knowledge below
+                    # (that this is compact_decimal_format) needs to be double-checked.
+
+                # `root.xml` aliases `long` to `short`.
+                # The typed lengths only carry compact patterns, so the alias lands in `compact_decimal_formats`.
+                # Only root has this.
+                compact_decimal_formats = data.setdefault('compact_decimal_formats', {})
+                compact_decimal_formats[length_type] = Alias(
+                    ('compact_decimal_formats', _sibling_alias_type(alias_elem)),
+                )
                 continue
             for pattern_el in elem.findall('./decimalFormat/pattern'):
                 pattern_type = pattern_el.attrib.get('type')
@@ -879,11 +904,58 @@ def parse_unit_patterns(data, tree):
     unit_patterns = data.setdefault('unit_patterns', {})
     compound_patterns = data.setdefault('compound_unit_patterns', {})
     unit_display_names = data.setdefault('unit_display_names', {})
+    # `root.xml` aliases the `long` and `narrow` unit lengths to `short`,
+    # and some units to others (e.g. `duration-day-person` to `duration-day`).
+    # This is keyed unit-first instead of length-first (XML),
+    # so length aliases are expanded into per-unit aliases below.
+    # Only root has aliases.
+    length_aliases = {}
 
     for elem in tree.findall('.//units/unitLength'):
         unit_length_type = elem.attrib['type']
+        alias_elem = elem.find('./alias')
+        if alias_elem is not None:
+            length_aliases[unit_length_type] = _sibling_alias_type(alias_elem)
+            continue
         for unit in elem.findall('unit'):
             unit_type = unit.attrib['type']
+            alias_elem = unit.find('./alias')
+            if alias_elem is not None:
+                target_unit = _sibling_alias_type(alias_elem)
+                if unit_type.endswith('-person'):
+                    # HACK (and this is a mouthful):
+                    # The `duration-*-person` units (used for formatting ages) are
+                    # aliased to their base units. The alias element only exists
+                    # inside `unitLength[short]`, so literal resolution would route
+                    # a long/narrow request through the short chain and lose the
+                    # requested length ("3 y" where "3 years" was wanted).
+                    # ICU considers this a deficiency of the alias data structure
+                    # (https://unicode-org.atlassian.net/browse/ICU-20400) and
+                    # compensates by stripping the `-person` suffix before lookup
+                    # This seems to make sense, so we do that too:
+                    # alias the whole unit, preserving the requested length.
+                    #
+                    # NB: this deliberately bends literal TR35 alias resolution to
+                    # match ICU's output. Should CLDR ever gain a length-preserving
+                    # alias representation, spec compliance is restored by deleting
+                    # this special-case `-person` branch and translating those
+                    # aliases as written, as the branch below does..
+                    unit_patterns[unit_type] = Alias(('unit_patterns', target_unit))
+                    unit_display_names[unit_type] = Alias(('unit_display_names', target_unit))
+                else:
+                    # The other aliased units (`graphics-dot*` -> `graphics-pixel*`,
+                    # `energy-foodcalorie` -> `energy-kilocalorie`) get no such
+                    # compensation in ICU and follow the literal chain: the alias
+                    # applies within this length only, and a locale's own data for
+                    # the unit at other lengths (plus the length aliases below)
+                    # takes precedence over the target unit's.
+                    unit_patterns.setdefault(unit_type, {})[unit_length_type] = Alias(
+                        ('unit_patterns', target_unit, unit_length_type),
+                    )
+                    unit_display_names.setdefault(unit_type, {})[unit_length_type] = Alias(
+                        ('unit_display_names', target_unit, unit_length_type),
+                    )
+                continue
             unit_and_length_patterns = unit_patterns.setdefault(unit_type, {}).setdefault(unit_length_type, {})
             for pattern in unit.findall('unitPattern'):
                 if pattern.attrib.get('case', 'nominative') != 'nominative':
@@ -922,11 +994,32 @@ def parse_unit_patterns(data, tree):
                     compound_unit_info['compound_variations'] = compound_variations
             compound_patterns.setdefault(unit_type, {})[unit_length_type] = compound_unit_info
 
+    for aliased_length, target_length in length_aliases.items():
+        for dict_name, dst in (
+            ('unit_patterns', unit_patterns),
+            ('compound_unit_patterns', compound_patterns),
+            ('unit_display_names', unit_display_names),
+        ):
+            for unit_type, lengths in dst.items():
+                if isinstance(lengths, Alias):
+                    # A length-preserving whole-unit alias installed above.
+                    continue
+                lengths.setdefault(aliased_length, Alias((dict_name, unit_type, target_length)))
+
 
 def parse_date_fields(data, tree):
     date_fields = data.setdefault('date_fields', {})
     for elem in tree.findall('.//dates/fields/field'):
         field_type = elem.attrib['type']
+        # `root` aliases `x-narrow` to `x-short` and `x-short` to `x`.
+        # Locales defining only some length variants inherit these.
+        # Only root has these.
+        alias_elem = elem.find('alias')
+        if alias_elem is not None:
+            date_fields[field_type] = Alias(
+                _translate_alias(['date_fields', field_type], alias_elem.attrib['path']),
+            )
+            continue
         date_fields.setdefault(field_type, {})
         for rel_time in elem.findall('relativeTime'):
             rel_time_type = rel_time.attrib['type']

tests/test_dates.py

@@ -1190,6 +1190,22 @@ def test_issue_1162(locale, format, negative, expected):
     assert dates.format_timedelta(delta, add_direction=True, format=format, locale=locale) == expected
 
 
+@pytest.mark.parametrize(('locale', 'format', 'expected'), [
+    # `am` defines `year-short` with <relative> names but no <relativeTime> patterns; this used to raise KeyError.
+    ('am', 'short', 'በ2 ዓመታት ውስጥ'),
+    ('am', 'narrow', 'በ2 ዓመታት ውስጥ'),
+    # `af` has no `year-narrow`; root aliases narrow to short,
+    # so the narrow form must use `year-short` ('oor 2 j.')
+    # rather than skipping straight to `year`.
+    ('af', 'narrow', 'oor 2 j.'),
+    ('af', 'short', 'oor 2 j.'),
+    ('af', 'long', 'oor 2 jaar'),
+])
+def test_issue_1076_date_field_length_aliases(locale, format, expected):
+    delta = timedelta(days=800)
+    assert dates.format_timedelta(delta, add_direction=True, format=format, locale=locale) == expected
+
+
 def test_issue_1192():
     # The actual returned value here is not actually strictly specified ("get_timezone_name"
     # is not an operation specified as such). Issue #1192 concerned this invocation returning

tests/test_numbers.py

@@ -673,3 +673,11 @@ def test_format_decimal_with_none_locale(monkeypatch):
     monkeypatch.setattr(numbers, "LC_NUMERIC", None)  # Pretend we couldn't find any locale when importing the module
     with pytest.raises(TypeError, match="Empty"):
         numbers.format_decimal(0, locale=None)
+
+
+def test_issue_1076_fixes():
+    # Japanese only defines short compact decimal formats; root aliases long to short. This used to raise a KeyError.
+    assert numbers.format_compact_decimal(12345, format_type="long", locale="ja") == "1万"
+    assert numbers.format_compact_decimal(2345678, format_type="long", locale="ja") == "235万"
+    # Swahili does not define an accounting currency format; root aliases it to the standard format.
+    assert numbers.format_currency(-2, "USD", format_type="accounting", locale="sw") == "-US$\xa02.00"

tests/test_units.py

@@ -1,6 +1,6 @@
 import pytest
 
-from babel.units import format_unit
+from babel.units import format_unit, get_unit_name
 
 
 # New units in CLDR 46
@@ -35,3 +35,40 @@ def test_deprecated_unit_ids():
     for id in ("concentr-permillion", "concentr-portion", "concentr-portion-per-1e9"):
         with pytest.warns(DeprecationWarning, match=id):
             format_unit(1, id, locale='en')
+
+
+@pytest.mark.parametrize('count, unit, locale, length, expected', [
+    # Root aliases `duration-*-person` to `duration-*`;
+    # no locale defines the person variants at all.
+    # These resolve length-preservingly (person long -> base long),
+    # matching ICU's `-person`-stripping behavior
+    # (see `getMeasureData` in https://github.com/unicode-org/icu/blob/main/icu4c/source/i18n/number_longnames.cpp
+    # and https://unicode-org.atlassian.net/browse/ICU-20400).
+    # This deliberately bends literal TR35 alias resolution.
+    # See `parse_unit_patterns` in `scripts/import_cldr.py` for the full story.
+    (2, 'duration-day-person', 'af', 'long', '2 dae'),
+    (3, 'duration-day-person', 'fi', 'long', '3 päivää'),
+    (3, 'duration-day-person', 'fi', 'short', '3 pv'),
+    (3, 'duration-day-person', 'fi', 'narrow', '3pv'),
+    (3, 'duration-year-person', 'fi', 'long', '3 vuotta'),
+    # `fi` defines `energy-foodcalorie` at long and narrow but not short;
+    # the root alias fills short in from `energy-kilocalorie`.
+    (3, 'energy-foodcalorie', 'fi', 'short', '3 kcal'),
+    # `fi` defines `graphics-dot` at short and narrow but not long;
+    # the root alias fills long in from short.
+    (3, 'graphics-dot', 'fi', 'long', '3 pistettä'),
+    # `cs` defines `graphics-dot` itself but not its short forms;
+    # the root aliases short to `graphics-pixel` short rather than the display name.
+    (3, 'graphics-dot', 'cs', 'short', '3 px'),
+])
+def test_issue_1076_unit_aliases(count, unit, locale, length, expected):
+    assert format_unit(count, unit, length, locale=locale) == expected
+
+
+def test_issue_1076_unit_name_length_aliases():
+    # Finnish defines no narrow display name for days; root aliases narrow to
+    # short. This used to return None.
+    assert get_unit_name('duration-day', length='narrow', locale='fi') == 'pv'
+    # The person variant resolves length-preservingly to `duration-day`.
+    assert get_unit_name('duration-day-person', length='long', locale='fi') == 'päivät'
+    assert get_unit_name('duration-day-person', length='short', locale='fi') == 'pv'