Move to inline-stored SHAs

Author: Mark-Simulacrum

collector/src/git.rs

@@ -9,12 +9,12 @@ use std::{
 pub fn get_commit_or_fake_it(sha: &str) -> anyhow::Result<Commit> {
     Ok(get_rust_commits()?
         .iter()
-        .find(|c| c.sha == sha)
+        .find(|c| c.sha == *sha)
         .cloned()
         .unwrap_or_else(|| {
             log::warn!("utilizing fake commit!");
             Commit {
-                sha: sha.to_string(),
+                sha: sha.into(),
                 date: crate::Date::ymd_hms(2000, 01, 01, 0, 0, 0),
             }
         }))
@@ -73,7 +73,7 @@ pub fn get_rust_commits() -> anyhow::Result<Vec<Commit>> {
         let date = crate::Date(chrono::DateTime::from_utc(date, chrono::Utc));
 
         commits.push(Commit {
-            sha: hash.to_string(),
+            sha: hash.into(),
             date,
         });
     }

collector/src/lib.rs

@@ -18,9 +18,216 @@ pub mod self_profile;
 
 pub use self_profile::{QueryData, SelfProfile};
 
+#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum Sha {
+    /// Straight-up bytes of the 40-long hex-encoded sha
+    Hex([u8; 20]),
+    /// Arbitrary bytes, usually a string ID provided by the user.
+    /// Encodes 30 characters by restricting to 5 bits per character
+    /// (enough for 32 different characters, so `a-z`, `-`, and `_`).
+    Raw { length: u8, bytes: [u8; 19] },
+}
+
+impl PartialEq<str> for Sha {
+    fn eq(&self, other: &str) -> bool {
+        self.to_string() == other
+    }
+}
+
+fn hex_decode(s: &str) -> Option<[u8; 20]> {
+    let mut in_progress = 0;
+    let mut v = [0; 20];
+    for (idx, ch) in s.chars().enumerate() {
+        let offset = if idx % 2 == 0 { 4 } else { 0 };
+        in_progress |= (ch.to_digit(16)? as u8) << offset;
+        if idx % 2 != 0 {
+            v[idx / 2] = in_progress;
+            in_progress = 0;
+        }
+    }
+    Some(v)
+}
+
+struct BitView {
+    bytes: [u8; 19],
+    bit_offset: usize,
+}
+
+impl BitView {
+    fn new() -> BitView {
+        BitView {
+            bytes: [0; 19],
+            bit_offset: 0,
+        }
+    }
+
+    fn with_bytes(bytes: [u8; 19]) -> BitView {
+        BitView {
+            bytes,
+            bit_offset: 0,
+        }
+    }
+
+    fn read(&mut self) -> bool {
+        let r = (self.bytes[self.bit_offset / 8] & (1 << (self.bit_offset % 8))) != 0;
+        self.bit_offset += 1;
+
+        if self.bit_offset > 19 * 8 {
+            panic!("pushed past limit of 152 bits");
+        }
+
+        r
+    }
+
+    fn read5(&mut self) -> u8 {
+        let mut v = 0;
+        v |= (self.read() as u8) << 0;
+        v |= (self.read() as u8) << 1;
+        v |= (self.read() as u8) << 2;
+        v |= (self.read() as u8) << 3;
+        v |= (self.read() as u8) << 4;
+        v
+    }
+
+    fn push(&mut self, b: bool) {
+        if b {
+            self.bytes[self.bit_offset / 8] |= 1 << (self.bit_offset % 8);
+        }
+        self.bit_offset += 1;
+
+        if self.bit_offset > 19 * 8 {
+            panic!("pushed past limit of 152 bits");
+        }
+    }
+
+    fn push5(&mut self, v: u8) {
+        assert!(
+            v <= 32,
+            "`{}` must be less than 32 (i.e., no more than 5 bits)",
+            v
+        );
+
+        self.push(v & 0b10000 != 0);
+        self.push(v & 0b01000 != 0);
+        self.push(v & 0b00100 != 0);
+        self.push(v & 0b00010 != 0);
+        self.push(v & 0b00001 != 0);
+    }
+}
+
+impl<'a> From<&'a str> for Sha {
+    fn from(s: &'a str) -> Sha {
+        if let Some(v) = hex_decode(s) {
+            return Sha::Hex(v);
+        }
+
+        assert!(
+            s.len() <= 30,
+            "`{}` is too long ({}), can be at most 30 bytes",
+            s,
+            s.len(),
+        );
+
+        let mut v = BitView::new();
+        for b in s.as_bytes().iter() {
+            let b = *b;
+            match b {
+                b'-' => {
+                    v.push5(0);
+                }
+                b'4' => {
+                    v.push5(1);
+                }
+                b'1' => {
+                    v.push5(2);
+                }
+                b'2' => {
+                    v.push5(3);
+                }
+                b'6' => {
+                    v.push5(4);
+                }
+                b'8' => {
+                    v.push5(5);
+                }
+                b'a'..=b'z' => {
+                    v.push5(b - b'a' + 6);
+                }
+                _ => panic!(
+                    "`{}` is not a valid character for SHA-like IDs, must be in a-z, or -, or _, in `{}`.",
+                    b as char,
+                    s,
+                ),
+            }
+        }
+        Sha::Raw {
+            length: s.len() as u8,
+            bytes: v.bytes,
+        }
+    }
+}
+
+impl Serialize for Sha {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::ser::Serializer,
+    {
+        serializer.collect_str(&self)
+    }
+}
+
+impl<'de> Deserialize<'de> for Sha {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::de::Deserializer<'de>,
+    {
+        let s: &'de str = <&'de str>::deserialize(deserializer)?;
+        Ok(s.into())
+    }
+}
+
+impl fmt::Debug for Sha {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self)
+    }
+}
+
+impl fmt::Display for Sha {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match *self {
+            Sha::Hex(hex) => {
+                for &b in hex.iter() {
+                    write!(f, "{:x}{:x}", b >> 4, b & 0xf)?;
+                }
+            }
+            Sha::Raw { length, bytes } => {
+                let mut v = BitView::with_bytes(bytes);
+                let mut decoded = [0; 19];
+                for idx in 0..length as usize {
+                    decoded[idx] = match v.read5() {
+                        0 => b'-',
+                        1 => b'4',
+                        2 => b'1',
+                        3 => b'2',
+                        4 => b'6',
+                        5 => b'8',
+                        other => other - 6 + b'a',
+                    };
+                }
+                write!(
+                    f,
+                    "{}",
+                    std::str::from_utf8(&decoded[..length as usize]).unwrap()
+                )?;
+            }
+        }
+        Ok(())
+    }
+}
+
 #[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
 pub struct Commit {
-    pub sha: String,
+    pub sha: Sha,
     pub date: Date,
 }
 

site/src/api.rs

@@ -14,7 +14,7 @@
 //! The responses are calculated in the server.rs file.
 
 use crate::load::CommitData;
-use collector::{Date, Run, StatId};
+use collector::{Date, Run, Sha, StatId};
 use serde::{Deserialize, Serialize};
 use std::collections::{BTreeSet, HashMap};
 use std::result::Result as StdResult;
@@ -23,7 +23,7 @@ use std::result::Result as StdResult;
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct DateData {
     pub date: Date,
-    pub commit: String,
+    pub commit: Sha,
     pub data: HashMap<String, Vec<(String, Run, f64)>>,
 }
 
@@ -159,7 +159,7 @@ pub mod data {
 }
 
 pub mod graph {
-    use collector::Bound;
+    use collector::{Bound, Sha};
     use serde::{Deserialize, Serialize};
     use std::collections::HashMap;
 
@@ -188,7 +188,7 @@ pub mod graph {
         pub benchmarks: HashMap<String, HashMap<String, Vec<GraphData>>>,
         pub max: HashMap<String, f32>,
         pub colors: Vec<String>,
-        pub commits: Vec<String>,
+        pub commits: Vec<Sha>,
     }
 }
 

site/src/load.rs

@@ -25,7 +25,7 @@ use collector::Date;
 use crate::api::github;
 use collector;
 pub use collector::{
-    ArtifactData, Benchmark, Commit, CommitData, Patch, Run, RunId, StatId, Stats,
+    ArtifactData, Benchmark, Commit, CommitData, Patch, Run, RunId, Sha, StatId, Stats,
 };
 use log::{error, info, trace, warn};
 
@@ -102,7 +102,7 @@ pub struct Persistent {
     pub pending_try_builds: HashSet<u32>,
     // Set of commit hashes for which we've completed benchmarking.
     #[serde(default)]
-    pub posted_ends: Vec<String>,
+    pub posted_ends: Vec<Sha>,
 }
 
 lazy_static::lazy_static! {
@@ -149,7 +149,7 @@ pub struct Keys {
 pub struct Config {
     pub keys: Keys,
     #[serde(default)]
-    pub skip: HashSet<String>,
+    pub skip: HashSet<Sha>,
 }
 
 #[derive(Debug)]
@@ -169,7 +169,7 @@ pub struct InputData {
     /// The benchmarks we interpolated for a given commit.
     ///
     /// Not all commits are in this map.
-    pub interpolated: HashMap<String, Vec<Interpolation>>,
+    pub interpolated: HashMap<Sha, Vec<Interpolation>>,
 
     pub artifact_data: HashMap<String, ArtifactData>,
 
@@ -586,12 +586,14 @@ impl InputData {
                     let mut ret = Vec::new();
                     ret.push((
                         Commit {
-                            sha: sha.clone(),
+                            sha: sha.as_str().into(),
                             date: Date::ymd_hms(2001, 01, 01, 0, 0, 0),
                         },
                         MissingReason::TryCommit,
                     ));
-                    if let Some(commit) = self.commits.iter().find(|c| c.sha == *parent_sha) {
+                    if let Some(commit) =
+                        self.commits.iter().find(|c| c.sha == *parent_sha.as_str())
+                    {
                         ret.push((commit.clone(), MissingReason::TryParent));
                     } else {
                         // could not find parent SHA
@@ -632,7 +634,7 @@ struct AssociatedData<'a> {
     data: &'a [CommitData],
     commits: &'a [Commit],
     commit_map: &'a HashMap<Commit, usize>,
-    interpolated: &'a mut HashMap<String, Vec<Interpolation>>,
+    interpolated: &'a mut HashMap<Sha, Vec<Interpolation>>,
 
     // By benchmark name, mapping to the index in the data vector
     last_seen_commit: &'a [HashMap<&'a str, usize>],
@@ -669,7 +671,7 @@ fn fill_benchmark_runs(
 
         let interpolations = data
             .interpolated
-            .entry(data.commit.sha.clone())
+            .entry(data.commit.sha)
             .or_insert_with(Vec::new);
         let run = match (start, end) {
             (Some(srun), Some(erun)) => {
@@ -714,7 +716,7 @@ fn fill_benchmark_data(benchmark_name: &str, data: &mut AssociatedData<'_>) -> O
     let commit_idx = data.commit_idx;
     let interpolation_entry = data
         .interpolated
-        .entry(data.commit.sha.clone())
+        .entry(data.commit.sha)
         .or_insert_with(Vec::new);
 
     let start = if let Some(&needle) = data.last_seen_commit[commit_idx].get(benchmark_name) {