More CI checks

- Check formatting
- Check clippy
- Check MSRV
- Check docs

Author: Jules-Bertholet

.github/workflows/rust.yml

@@ -7,7 +7,11 @@ on:
     branches: [ master ]
 
 env:
+  CARGO_INCREMENTAL: 0
   CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+  RUSTFLAGS: -D warnings
+  RUSTDOCFLAGS: -D warnings --cfg docsrs
 
 jobs:
   build:
@@ -20,27 +24,50 @@ jobs:
           - nightly
     steps:
     - uses: actions/checkout@v2
-    - name: Install latest nightly
+    - name: Install toolchain
       uses: actions-rs/toolchain@v1
       with:
           toolchain: ${{ matrix.rust }}
           override: true
     - name: Build
       run: cargo build --verbose
-    - name: Run tests
-      run: cargo test --verbose
+    - name: Run tests with all features
+      run: cargo test --all-features --verbose
     - name: Run tests without features
-      run: cargo test --verbose --no-default-features
+      run: cargo test --no-default-features --verbose
     - name: Package
       run: cargo package
     - name: Test package
       run: cd $(find target/package/ -maxdepth 1 -mindepth 1 -type d) && cargo test
     - name: Test package without features
       run: cd $(find target/package/ -maxdepth 1 -mindepth 1 -type d) && cargo test --no-default-features
+    - name: Build docs
+      if: matrix.rust == 'nightly'
+      run: cargo doc --all-features --verbose
+    - name: Check formatting
+      if: matrix.rust == 'stable'
+      run: cargo fmt --all --check
+    - name: Check clippy
+      if: matrix.rust == 'stable'
+      run: cargo clippy --all-features --all --verbose
+  msrv:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install msrv toolchain
+      uses: actions-rs/toolchain@v1
+      with:
+          toolchain: 1.36
+          override: true
+    - name: Build
+      run: cargo build --verbose --all-features
   regen:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
     - name: Regen
       run: cd scripts && python3 unicode.py
     - name: Diff

scripts/unicode.py

@@ -20,6 +20,7 @@
 # out-of-line and check the tables.rs and normalization_tests.rs files into git.
 import collections
 import urllib.request
+from itertools import batched
 
 UNICODE_VERSION = "15.1.0"
 UCD_URL = "https://www.unicode.org/Public/%s/ucd/" % UNICODE_VERSION
@@ -100,9 +101,9 @@ def _load_unicode_data(self):
         self.general_category_mark = []
         self.general_category_public_assigned = []
 
-        assigned_start = 0;
-        prev_char_int = -1;
-        prev_name = "";
+        assigned_start = 0
+        prev_char_int = -1
+        prev_name = ""
 
         for line in self._fetch("UnicodeData.txt").splitlines():
             # See ftp://ftp.unicode.org/Public/3.0-Update/UnicodeData-3.0.0.html
@@ -131,7 +132,7 @@ def _load_unicode_data(self):
                     self.general_category_public_assigned.append((assigned_start, prev_char_int))
                     assigned_start = char_int
                 prev_char_int = char_int
-                prev_name = name;
+                prev_name = name
 
         self.general_category_public_assigned.append((assigned_start, prev_char_int))
 
@@ -158,16 +159,16 @@ def _load_cjk_compat_ideograph_variants(self):
 
             char_int = self.name_to_char_int[description]
 
-            assert not char_int in self.combining_classes, "Unexpected: CJK compat variant with a combining class"
-            assert not char_int in self.compat_decomp, "Unexpected: CJK compat variant and compatibility decomposition"
+            assert char_int not in self.combining_classes, "Unexpected: CJK compat variant with a combining class"
+            assert char_int not in self.compat_decomp, "Unexpected: CJK compat variant and compatibility decomposition"
             assert len(self.canon_decomp[char_int]) == 1, "Unexpected: CJK compat variant and non-singleton canonical decomposition"
             # If we ever need to handle Hangul here, we'll need to handle it separately.
             assert not (S_BASE <= char_int < S_BASE + S_COUNT)
 
             cjk_compat_variant_parts = [int(c, 16) for c in variation_sequence.split()]
             for c in cjk_compat_variant_parts:
-                assert not c in self.canon_decomp, "Unexpected: CJK compat variant is unnormalized (canon)"
-                assert not c in self.compat_decomp, "Unexpected: CJK compat variant is unnormalized (compat)"
+                assert c not in self.canon_decomp, "Unexpected: CJK compat variant is unnormalized (canon)"
+                assert c not in self.compat_decomp, "Unexpected: CJK compat variant is unnormalized (compat)"
             self.cjk_compat_variants_fully_decomp[char_int] = cjk_compat_variant_parts
 
     def _load_norm_props(self):
@@ -354,20 +355,26 @@ def is_first_and_last(first, last):
         return False
     return first[1:-8] == last[1:-7]
 
-def gen_mph_data(name, d, kv_type, kv_callback):
+def gen_mph_data(name, d, kv_type, kv_callback, kv_row_width):
     (salt, keys) = minimal_perfect_hash(d)
-    out.write("pub(crate) const %s_SALT: &[u16] = &[\n" % name.upper())
-    for s in salt:
-        out.write("    0x{:x},\n".format(s))
+    out.write(f"\npub(crate) const {name.upper()}_SALT: &[u16] = &[\n")
+    for s_row in batched(salt, 13):
+        out.write("   ")
+        for s in s_row:
+            out.write(f" 0x{s:03X},")
+        out.write("\n")
+    out.write("];\n")
+    out.write(f"pub(crate) const {name.upper()}_KV: &[{kv_type}] = &[\n")
+    for k_row in batched(keys, kv_row_width):
+        out.write("   ")
+        for k in k_row:
+            out.write(f" {kv_callback(k)},")
+        out.write("\n")
     out.write("];\n")
-    out.write("pub(crate) const {}_KV: &[{}] = &[\n".format(name.upper(), kv_type))
-    for k in keys:
-        out.write("    {},\n".format(kv_callback(k)))
-    out.write("];\n\n")
 
 def gen_combining_class(combining_classes, out):
     gen_mph_data('canonical_combining_class', combining_classes, 'u32',
-        lambda k: "0x{:X}".format(int(combining_classes[k]) | (k << 8)))
+        lambda k: f"0x{int(combining_classes[k]) | (k << 8):07X}", 8)
 
 def gen_composition_table(canon_comp, out):
     table = {}
@@ -376,7 +383,7 @@ def gen_composition_table(canon_comp, out):
             table[(c1 << 16) | c2] = c3
     (salt, keys) = minimal_perfect_hash(table)
     gen_mph_data('COMPOSITION_TABLE', table, '(u32, char)',
-        lambda k: "(0x%s, '\\u{%s}')" % (hexify(k), hexify(table[k])))
+        lambda k: f"(0x{k:08X}, '\\u{{{table[k]:06X}}}')", 1)
 
     out.write("pub(crate) fn composition_table_astral(c1: char, c2: char) -> Option<char> {\n")
     out.write("    match (c1, c2) {\n")
@@ -403,7 +410,7 @@ def gen_decomposition_tables(canon_decomp, compat_decomp, cjk_compat_variants_de
         assert offset < 65536
         out.write("];\n")
         gen_mph_data(name + '_decomposed', table, "(u32, (u16, u16))",
-            lambda k: "(0x{:x}, ({}, {}))".format(k, offsets[k], len(table[k])))
+            lambda k: f"(0x{k:05X}, (0x{offsets[k]:03X}, 0x{len(table[k]):X}))", 1)
 
 def gen_qc_match(prop_table, out):
     out.write("    match c {\n")
@@ -421,7 +428,7 @@ def gen_qc_match(prop_table, out):
     out.write("    }\n")
 
 def gen_nfc_qc(prop_tables, out):
-    out.write("#[inline]\n")
+    out.write("\n#[inline]\n")
     out.write("#[allow(ellipsis_inclusive_range_patterns)]\n")
     out.write("pub fn qc_nfc(c: char) -> IsNormalized {\n")
     gen_qc_match(prop_tables['NFC_QC'], out)
@@ -450,13 +457,13 @@ def gen_nfkd_qc(prop_tables, out):
 
 def gen_combining_mark(general_category_mark, out):
     gen_mph_data('combining_mark', general_category_mark, 'u32',
-        lambda k: '0x{:04x}'.format(k))
+        lambda k: '0x{:05X}'.format(k), 10)
 
 def gen_public_assigned(general_category_public_assigned, out):
     # This could be done as a hash but the table is somewhat small.
     out.write("#[inline]\n")
     out.write("pub fn is_public_assigned(c: char) -> bool {\n")
-    out.write("    match c {\n")
+    out.write("    matches!(c,\n")
 
     start = True
     for first, last in general_category_public_assigned:
@@ -469,12 +476,9 @@ def gen_public_assigned(general_category_public_assigned, out):
             out.write("'\\u{%s}'\n" % hexify(first))
         else:
             out.write("'\\u{%s}'..='\\u{%s}'\n" % (hexify(first), hexify(last)))
-    out.write("        => true,\n")
 
-    out.write("        _ => false,\n")
-    out.write("    }\n")
+    out.write("    )\n")
     out.write("}\n")
-    out.write("\n")
 
 def gen_stream_safe(leading, trailing, out):
     # This could be done as a hash but the table is very small.
@@ -488,10 +492,9 @@ def gen_stream_safe(leading, trailing, out):
     out.write("        _ => 0,\n")
     out.write("    }\n")
     out.write("}\n")
-    out.write("\n")
 
     gen_mph_data('trailing_nonstarters', trailing, 'u32',
-        lambda k: "0x{:X}".format(int(trailing[k]) | (k << 8)))
+        lambda k: f"0x{int(trailing[k]) | (k << 8):07X}", 8)
 
 def gen_tests(tests, out):
     out.write("""#[derive(Debug)]
@@ -585,36 +588,27 @@ def minimal_perfect_hash(d):
 
         version = "(%s, %s, %s)" % tuple(UNICODE_VERSION.split("."))
         out.write("#[allow(unused)]\n")
-        out.write("pub const UNICODE_VERSION: (u8, u8, u8) = %s;\n\n" % version)
+        out.write("pub const UNICODE_VERSION: (u8, u8, u8) = %s;\n" % version)
 
         gen_combining_class(data.combining_classes, out)
-        out.write("\n")
 
         gen_composition_table(data.canon_comp, out)
-        out.write("\n")
 
         gen_decomposition_tables(data.canon_fully_decomp, data.compat_fully_decomp, data.cjk_compat_variants_fully_decomp, out)
 
         gen_combining_mark(data.general_category_mark, out)
-        out.write("\n")
 
         gen_public_assigned(data.general_category_public_assigned, out)
-        out.write("\n")
 
         gen_nfc_qc(data.norm_props, out)
-        out.write("\n")
 
         gen_nfkc_qc(data.norm_props, out)
-        out.write("\n")
 
         gen_nfd_qc(data.norm_props, out)
-        out.write("\n")
 
         gen_nfkd_qc(data.norm_props, out)
-        out.write("\n")
 
         gen_stream_safe(data.ss_leading, data.ss_trailing, out)
-        out.write("\n")
 
     with open("normalization_tests.rs", "w", newline = "\n") as out:
         out.write(PREAMBLE)

src/__test_api.rs

@@ -5,7 +5,7 @@
 // If you're caught using this outside this crates tests/, you get to clean up the mess.
 
 #[cfg(not(feature = "std"))]
-use crate::no_std_prelude::*;
+use alloc::string::String;
 
 use crate::stream_safe::StreamSafe;
 

src/decompose.rs

@@ -8,7 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 use core::fmt::{self, Write};
-use core::iter::Fuse;
+use core::iter::{Fuse, FusedIterator};
 use core::ops::Range;
 use tinyvec::TinyVec;
 
@@ -151,6 +151,8 @@ impl<I: Iterator<Item = char>> Iterator for Decompositions<I> {
     }
 }
 
+impl<I: Iterator<Item = char>> FusedIterator for Decompositions<I> {}
+
 impl<I: Iterator<Item = char> + Clone> fmt::Display for Decompositions<I> {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         for c in self.clone() {

src/lib.rs

@@ -62,12 +62,7 @@ pub use crate::recompose::Recompositions;
 pub use crate::replace::Replacements;
 pub use crate::stream_safe::StreamSafe;
 pub use crate::tables::UNICODE_VERSION;
-use core::{
-    str::Chars,
-    option,
-};
-
-mod no_std_prelude;
+use core::{option, str::Chars};
 
 mod decompose;
 mod lookups;
@@ -169,7 +164,6 @@ impl<'a> UnicodeNormalization<Chars<'a>> for &'a str {
     }
 }
 
-
 impl UnicodeNormalization<option::IntoIter<char>> for char {
     #[inline]
     fn nfd(self) -> Decompositions<option::IntoIter<char>> {

src/no_std_prelude.rs

@@ -9,7 +9,10 @@
 // except according to those terms.
 
 use crate::decompose::Decompositions;
-use core::fmt::{self, Write};
+use core::{
+    fmt::{self, Write},
+    iter::FusedIterator,
+};
 use tinyvec::TinyVec;
 
 #[derive(Clone)]
@@ -144,6 +147,8 @@ impl<I: Iterator<Item = char>> Iterator for Recompositions<I> {
     }
 }
 
+impl<I: Iterator<Item = char> + FusedIterator> FusedIterator for Recompositions<I> {}
+
 impl<I: Iterator<Item = char> + Clone> fmt::Display for Recompositions<I> {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         for c in self.clone() {

src/recompose.rs

@@ -7,7 +7,10 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
-use core::fmt::{self, Write};
+use core::{
+    fmt::{self, Write},
+    iter::FusedIterator,
+};
 use tinyvec::ArrayVec;
 
 /// External iterator for replacements for a string's characters.
@@ -51,6 +54,8 @@ impl<I: Iterator<Item = char>> Iterator for Replacements<I> {
     }
 }
 
+impl<I: Iterator<Item = char> + FusedIterator> FusedIterator for Replacements<I> {}
+
 impl<I: Iterator<Item = char> + Clone> fmt::Display for Replacements<I> {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         for c in self.clone() {

src/replace.rs

@@ -1,3 +1,5 @@
+use core::iter::FusedIterator;
+
 use crate::lookups::{
     canonical_combining_class, canonical_fully_decomposed, compatibility_fully_decomposed,
     stream_safe_trailing_nonstarters,
@@ -59,6 +61,8 @@ impl<I: Iterator<Item = char>> Iterator for StreamSafe<I> {
     }
 }
 
+impl<I: Iterator<Item = char> + FusedIterator> FusedIterator for StreamSafe<I> {}
+
 #[derive(Debug)]
 pub(crate) struct Decomposition {
     pub(crate) leading_nonstarters: usize,
@@ -110,7 +114,7 @@ mod tests {
     use crate::normalize::decompose_compatible;
 
     #[cfg(not(feature = "std"))]
-    use crate::no_std_prelude::*;
+    use alloc::{string::String, vec::Vec};
 
     use core::char;