Add compressible_bytes data

As a side effect, always load CollationSpecialPrimaries, since we no longer
know at collator instantiation time if some of the data in the struct is
going to be used.

Preparation for unicode-org#6537

Author: hsivonen

components/collator/src/comparison.rs

@@ -36,7 +36,7 @@ use crate::provider::CollationReordering;
 use crate::provider::CollationReorderingV1;
 use crate::provider::CollationRootV1;
 use crate::provider::CollationSpecialPrimaries;
-use crate::provider::CollationSpecialPrimariesV1;
+use crate::provider::CollationSpecialPrimariesV2;
 use crate::provider::CollationTailoringV1;
 use core::cmp::Ordering;
 use core::convert::TryFrom;
@@ -379,7 +379,7 @@ impl LocaleSpecificDataHolder {
 /// Compares strings according to culturally-relevant ordering.
 #[derive(Debug)]
 pub struct Collator {
-    special_primaries: Option<DataPayload<CollationSpecialPrimariesV1>>,
+    special_primaries: DataPayload<CollationSpecialPrimariesV2>,
     root: DataPayload<CollationRootV1>,
     tailoring: Option<DataPayload<CollationTailoringV1>>,
     jamo: DataPayload<CollationJamoV1>,
@@ -395,7 +395,7 @@ impl Collator {
     /// Constructs a borrowed version of this type for more efficient querying.
     pub fn as_borrowed(&self) -> CollatorBorrowed {
         CollatorBorrowed {
-            special_primaries: self.special_primaries.as_ref().map(|s| s.get()),
+            special_primaries: self.special_primaries.get(),
             root: self.root.get(),
             tailoring: self.tailoring.as_ref().map(|s| s.get()),
             jamo: self.jamo.get(),
@@ -434,7 +434,7 @@ impl Collator {
         options: CollatorOptions,
     ) -> Result<Self, DataError>
     where
-        D: DataProvider<CollationSpecialPrimariesV1>
+        D: DataProvider<CollationSpecialPrimariesV2>
             + DataProvider<CollationRootV1>
             + DataProvider<CollationTailoringV1>
             + DataProvider<CollationDiacriticsV1>
@@ -451,7 +451,7 @@ impl Collator {
             provider.load(Default::default())?.payload,
             provider.load(Default::default())?.payload,
             provider.load(Default::default())?.payload,
-            || provider.load(Default::default()).map(|r| r.payload),
+            provider.load(Default::default())?.payload,
             prefs,
             options,
         )
@@ -464,7 +464,7 @@ impl Collator {
         decompositions: DataPayload<NormalizerNfdDataV1>,
         tables: DataPayload<NormalizerNfdTablesV1>,
         jamo: DataPayload<CollationJamoV1>,
-        special_primaries: impl FnOnce() -> Result<DataPayload<CollationSpecialPrimariesV1>, DataError>,
+        special_primaries: DataPayload<CollationSpecialPrimariesV2>,
         prefs: CollatorPreferences,
         options: CollatorOptions,
     ) -> Result<Self, DataError>
@@ -484,22 +484,14 @@ impl Collator {
             return Err(DataError::custom("invalid").with_marker(CollationJamoV1::INFO));
         }
 
-        let special_primaries = if locale_dependent.merged_options.alternate_handling()
-            == AlternateHandling::Shifted
-            || locale_dependent.merged_options.numeric()
-        {
-            let special_primaries = special_primaries()?;
-            // `variant_count` isn't stable yet:
-            // https://github.com/rust-lang/rust/issues/73662
-            if special_primaries.get().last_primaries.len() <= (MaxVariable::Currency as usize) {
-                return Err(
-                    DataError::custom("invalid").with_marker(CollationSpecialPrimariesV1::INFO)
-                );
-            }
-            Some(special_primaries)
-        } else {
-            None
-        };
+        // `variant_count` isn't stable yet:
+        // https://github.com/rust-lang/rust/issues/73662
+        if special_primaries.get().last_primaries.len() <= (MaxVariable::Currency as usize) {
+            return Err(DataError::custom("invalid").with_marker(CollationSpecialPrimariesV2::INFO));
+        }
+        if special_primaries.get().compressible_bytes.len() != 32 {
+            return Err(DataError::custom("invalid").with_marker(CollationSpecialPrimariesV2::INFO));
+        }
 
         Ok(Collator {
             special_primaries,
@@ -543,7 +535,7 @@ macro_rules! compare {
 /// borrowed version.
 #[derive(Debug)]
 pub struct CollatorBorrowed<'a> {
-    special_primaries: Option<&'a CollationSpecialPrimaries<'a>>,
+    special_primaries: &'a CollationSpecialPrimaries<'a>,
     root: &'a CollationData<'a>,
     tailoring: Option<&'a CollationData<'a>>,
     jamo: &'a CollationJamo<'a>,
@@ -579,23 +571,15 @@ impl CollatorBorrowed<'static> {
             return Err(DataError::custom("invalid").with_marker(CollationJamoV1::INFO));
         }
 
-        let special_primaries = if locale_dependent.merged_options.alternate_handling()
-            == AlternateHandling::Shifted
-            || locale_dependent.merged_options.numeric()
-        {
-            let special_primaries =
-                crate::provider::Baked::SINGLETON_COLLATION_SPECIAL_PRIMARIES_V1;
-            // `variant_count` isn't stable yet:
-            // https://github.com/rust-lang/rust/issues/73662
-            if special_primaries.last_primaries.len() <= (MaxVariable::Currency as usize) {
-                return Err(
-                    DataError::custom("invalid").with_marker(CollationSpecialPrimariesV1::INFO)
-                );
-            }
-            Some(special_primaries)
-        } else {
-            None
-        };
+        let special_primaries = crate::provider::Baked::SINGLETON_COLLATION_SPECIAL_PRIMARIES_V2;
+        // `variant_count` isn't stable yet:
+        // https://github.com/rust-lang/rust/issues/73662
+        if special_primaries.last_primaries.len() <= (MaxVariable::Currency as usize) {
+            return Err(DataError::custom("invalid").with_marker(CollationSpecialPrimariesV2::INFO));
+        }
+        if special_primaries.compressible_bytes.len() != 32 {
+            return Err(DataError::custom("invalid").with_marker(CollationSpecialPrimariesV2::INFO));
+        }
 
         // Attribute belongs closer to `unwrap`, but
         // https://github.com/rust-lang/rust/issues/15701
@@ -623,12 +607,7 @@ impl CollatorBorrowed<'static> {
     /// compile-time optimizations that are possible with [`CollatorBorrowed`].
     pub const fn static_to_owned(self) -> Collator {
         Collator {
-            special_primaries: if let Some(s) = self.special_primaries {
-                // `map` not available in const context
-                Some(DataPayload::from_static_ref(s))
-            } else {
-                None
-            },
+            special_primaries: DataPayload::from_static_ref(self.special_primaries),
             root: DataPayload::from_static_ref(self.root),
             tailoring: if let Some(s) = self.tailoring {
                 // `map` not available in const context
@@ -751,10 +730,6 @@ impl CollatorBorrowed<'_> {
         } else {
             // +1 so that we can use "<" and primary ignorables test out early.
             self.special_primaries
-                .as_ref()
-                // `unwrap()` is OK, because we've ensured in the constructor that value
-                // is `Some` if we have alternate handling.
-                .unwrap()
                 .last_primary_for_group(self.options.max_variable())
                 + 1
         };
@@ -763,13 +738,7 @@ impl CollatorBorrowed<'_> {
         // https://github.com/rust-lang/rust/issues/15701
         #[allow(clippy::unwrap_used)]
         let numeric_primary = if self.options.numeric() {
-            Some(
-                self.special_primaries
-                    .as_ref()
-                    // `unwrap` is OK, because we've ensured `Some` in the constructor
-                    .unwrap()
-                    .numeric_primary,
-            )
+            Some(self.special_primaries.numeric_primary)
         } else {
             None
         };
@@ -1544,3 +1513,25 @@ impl CollatorBorrowed<'_> {
         Ordering::Equal
     }
 }
+
+/// Helper for checking if a byte is compressible
+pub(crate) struct CompressibleBytes<'a> {
+    arr: &'a [u8; 32],
+}
+
+impl<'a> CompressibleBytes<'a> {
+    pub(crate) fn new(arr: &'a [u8; 32]) -> Self {
+        Self { arr }
+    }
+
+    #[allow(dead_code)]
+    pub(crate) fn is_compressible(&self, b: u8) -> bool {
+        // Indexing OK by construction and pasting this
+        // into Compiler Explorer shows that the panic
+        // is optimized away.
+        #[allow(clippy::indexing_slicing)]
+        let field = self.arr[usize::from(b >> 3)];
+        let mask = 1 << (b & 0b111);
+        (field & mask) != 0
+    }
+}

components/collator/src/provider.rs

@@ -25,6 +25,7 @@ use zerovec::ule::AsULE;
 use zerovec::ZeroVec;
 use zerovec::{zeroslice, ZeroSlice};
 
+use crate::comparison::CompressibleBytes;
 use crate::elements::CollationElement;
 use crate::elements::CollationElement32;
 use crate::elements::Tag;
@@ -64,7 +65,7 @@ const _: () = {
     impl_collation_diacritics_v1!(Baked);
     impl_collation_jamo_v1!(Baked);
     impl_collation_metadata_v1!(Baked);
-    impl_collation_special_primaries_v1!(Baked);
+    impl_collation_special_primaries_v2!(Baked);
     impl_collation_reordering_v1!(Baked);
 };
 
@@ -126,8 +127,8 @@ icu_provider::data_marker!(
 );
 icu_provider::data_marker!(
     /// Data marker for collcation special primaries data.
-    CollationSpecialPrimariesV1,
-    "collation/special/primaries/v1",
+    CollationSpecialPrimariesV2,
+    "collation/special/primaries/v2",
     CollationSpecialPrimaries<'static>,
     is_singleton = true,
 );
@@ -141,7 +142,7 @@ pub const MARKERS: &[DataMarkerInfo] = &[
     CollationJamoV1::INFO,
     CollationMetadataV1::INFO,
     CollationReorderingV1::INFO,
-    CollationSpecialPrimariesV1::INFO,
+    CollationSpecialPrimariesV2::INFO,
 ];
 
 const SINGLE_U32: &ZeroSlice<u32> =
@@ -521,7 +522,26 @@ impl CollationMetadata {
     }
 }
 
-/// Special primaries associated with the root collation
+/// Root-associated additional data that doesn't change in tailorings
+///
+/// These are the fields that logically belong to the root data but
+/// don't belong to the tailoring data and that are on this separate
+/// struct, since we have the same struct for a tailoring and the
+/// bulk of the root.
+///
+/// As a practical matter, this struct happens to only carry
+/// information about what concrete numeric values for primary
+/// weights are special in particular ways. In principle, when the
+/// root data is built, the root builder is allowed to assign the
+/// numeric values as it sees fit, which is why these aren't
+/// hard-coded.
+///
+/// Note: In 2.0.0 and prior, this struct was loaded only if
+/// it was known at collator construction time (based on options)
+/// that the data here was going to be needed. With the introduction
+/// of collation keys and the decision not to introduce a collator
+/// key generator object separate from the collator, this struct
+/// is now always loaded.
 ///
 /// <div class="stab unstable">
 /// 🚧 This code is considered unstable; it may change at any time, in breaking or non-breaking ways,
@@ -539,6 +559,15 @@ pub struct CollationSpecialPrimaries<'data> {
     /// variants in `MaxVariable`, currently 4.
     #[cfg_attr(feature = "serde", serde(borrow))]
     pub last_primaries: ZeroVec<'data, u16>,
+    /// 256 bits to classify every possible byte
+    /// into compressible or non-compressible.
+    /// The 256 bits are distributed across 32
+    /// `u8`s, since we don't have a native `u256`
+    /// and have to do manual shifts and masks,
+    /// so we might as well do them with the narrow
+    /// type that is its own `ULE`.
+    #[cfg_attr(feature = "serde", serde(borrow))]
+    pub compressible_bytes: ZeroVec<'data, u8>,
     /// The high 8 bits of the numeric primary
     pub numeric_primary: u8,
 }
@@ -557,4 +586,12 @@ impl CollationSpecialPrimaries<'_> {
         // See parse.cpp in genrb and getLastPrimaryForGroup in ICU4C.
         (u32::from(self.last_primaries.get(max_variable as usize).unwrap()) << 16) - 1
     }
+
+    #[allow(dead_code)]
+    pub(crate) fn get_compressible_bytes(&self) -> CompressibleBytes<'_> {
+        // The length has already been validated when constructing the collator.
+        debug_assert_eq!(self.compressible_bytes.len(), 32);
+        #[allow(clippy::unwrap_used)]
+        CompressibleBytes::new(<&[u8; 32]>::try_from(self.compressible_bytes.as_bytes()).unwrap())
+    }
 }

components/collator/tests/tests.rs

@@ -27,7 +27,7 @@ const _: () = {
     impl_collation_diacritics_v1!(TestingProvider);
     impl_collation_jamo_v1!(TestingProvider);
     impl_collation_metadata_v1!(TestingProvider);
-    impl_collation_special_primaries_v1!(TestingProvider);
+    impl_collation_special_primaries_v2!(TestingProvider);
     impl_collation_reordering_v1!(TestingProvider);
 
     icu_normalizer_data::impl_normalizer_nfc_v1!(TestingProvider);