Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
146 changes: 145 additions & 1 deletion datafusion/physical-plan/src/joins/array_map.rs
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ macro_rules! downcast_supported_integer {
pub struct ArrayMap {
// data[probSideVal-offset] -> valIdxInBuildSide + 1; 0 for absent
data: Vec<u32>,
sparse: Option<Box<SparseSlots>>,
// min val in buildSide
offset: u64,
// next[buildSideIdx] -> next matching valIdxInBuildSide + 1; 0 for end of chain.
Expand All @@ -100,6 +101,48 @@ pub struct ArrayMap {
num_of_distinct_key: usize,
}

// Each word's prefix counts set bits in preceding words. The first level
// indexes occupied 64-key ranges and the second indexes individual keys.
#[derive(Debug, Default, Clone, Copy)]
struct RankedWord {
bits: u64,
prefix: u32,
}

impl RankedWord {
fn rank(&self, bit: usize) -> usize {
self.prefix as usize + (self.bits & ((1u64 << bit) - 1)).count_ones() as usize
}

fn set_prefixes(words: &mut [Self]) -> usize {
let mut total = 0;
for word in words {
word.prefix = total;
total += word.bits.count_ones();
}
total as usize
}
}

#[derive(Debug)]
struct SparseSlots {
directory: Vec<RankedWord>,
leaves: Vec<RankedWord>,
}

impl SparseSlots {
fn slot(&self, offset: usize) -> Option<usize> {
let word = offset / 64;
let top = self.directory.get(word / 64)?;
if top.bits & (1 << (word % 64)) == 0 {
return None;
}
let leaf = &self.leaves[top.rank(word % 64)];
let bit = offset % 64;
(leaf.bits & (1 << bit) != 0).then(|| leaf.rank(bit))
}
}

impl ArrayMap {
pub fn is_supported_type(data_type: &DataType) -> bool {
matches!(
Expand Down Expand Up @@ -184,12 +227,82 @@ impl ArrayMap {

Ok(Self {
data,
sparse: None,
offset: min_val,
next,
num_of_distinct_key,
})
}

pub(crate) fn sparse_memory_bound(range: u64, rows: usize) -> Option<usize> {
if !(64..=4096).contains(&rows) || range >= 1 << 24 || range / 16 < rows as u64 {
return None;
}
let words = (range as usize + 1).div_ceil(64);
Some(
size_of::<SparseSlots>()
+ (words.div_ceil(64) + words.min(rows)) * size_of::<RankedWord>()
+ rows * size_of::<u32>(),
)
}

pub(crate) fn try_sparse(
array: &ArrayRef,
lower: u64,
range: u64,
) -> Result<Option<Self>> {
downcast_supported_integer!(array.data_type() => (build_sparse, array, lower, range))
}

fn build_sparse<T: ArrowNumericType>(
array: &ArrayRef,
lower: u64,
range: u64,
) -> Result<Option<Self>>
where
T::Native: AsPrimitive<u64>,
{
let keys = array.as_primitive::<T>();
let mut directory =
vec![RankedWord::default(); (range as usize + 1).div_ceil(4096)];
for key in keys.iter().flatten() {
let offset = key.as_().wrapping_sub(lower);
if offset > range {
return Ok(None);
}
let word = offset as usize / 64;
directory[word / 64].bits |= 1 << (word % 64);
}
let mut leaves =
vec![RankedWord::default(); RankedWord::set_prefixes(&mut directory)];
for key in keys.iter().flatten() {
let offset = key.as_().wrapping_sub(lower) as usize;
let word = offset / 64;
let leaf = &mut leaves[directory[word / 64].rank(word % 64)];
let bit = 1 << (offset % 64);
if leaf.bits & bit != 0 {
return Ok(None);
}
leaf.bits |= bit;
}
let count = RankedWord::set_prefixes(&mut leaves);
let sparse = Box::new(SparseSlots { directory, leaves });
let mut data = vec![0; count];
for (row, key) in keys.iter().enumerate() {
if let Some(key) = key {
let offset = key.as_().wrapping_sub(lower) as usize;
data[sparse.slot(offset).expect("build key present")] = row as u32 + 1;
}
}
Ok(Some(Self {
data,
sparse: Some(sparse),
offset: lower,
next: vec![],
num_of_distinct_key: count,
}))
}

fn fill_data<T: ArrowNumericType>(
array: &ArrayRef,
offset_val: u64,
Expand Down Expand Up @@ -229,7 +342,12 @@ impl ArrayMap {

/// Returns the memory usage of this [`ArrayMap`] in bytes.
pub fn size(&self) -> usize {
self.data.capacity() * size_of::<u32>() + self.next.capacity() * size_of::<u32>()
(self.data.capacity() + self.next.capacity()) * size_of::<u32>()
+ self.sparse.as_ref().map_or(0, |s| {
size_of::<SparseSlots>()
+ (s.directory.capacity() + s.leaves.capacity())
* size_of::<RankedWord>()
})
}

pub fn get_matched_indices_with_limit_offset(
Expand Down Expand Up @@ -266,6 +384,10 @@ impl ArrayMap {
/// bucket, or `None` otherwise.
#[inline]
fn get_value(&self, key: u64) -> Option<u32> {
if let Some(sparse) = &self.sparse {
let offset = usize::try_from(key.wrapping_sub(self.offset)).ok()?;
return Some(self.data[sparse.slot(offset)?]);
}
let idx = Self::key_to_index(key, self.offset, self.data.len())?;
let value = self.data[idx];
(value != 0).then_some(value)
Expand Down Expand Up @@ -421,6 +543,28 @@ mod tests {
use arrow::array::UInt64Array;
use std::sync::Arc;

#[test]
fn sparse_slots_preserve_row_ids_and_reject_duplicates() -> Result<()> {
use arrow::array::Int64Array;
let keys: ArrayRef = Arc::new(Int64Array::from(vec![
Some(10000),
None,
Some(-10),
Some(10001),
]));
let lower = (-10i64) as u64;
let map = ArrayMap::try_sparse(&keys, lower, 10011)?.unwrap();
assert_eq!(map.get_value(lower), Some(3));
assert_eq!(map.get_value(10000), Some(1));
assert_eq!(map.get_value(10001), Some(4));
for absent in [lower.wrapping_sub(1), 0, 9999, 10002, u64::MAX] {
assert_eq!(map.get_value(absent), None);
}
let duplicate: ArrayRef = Arc::new(Int64Array::from(vec![2, 10000, 2]));
assert!(ArrayMap::try_sparse(&duplicate, 2, 9998)?.is_none());
Ok(())
}

#[test]
fn test_array_map_limit_offset_duplicate_elements() -> Result<()> {
let build: ArrayRef = Arc::new(Int32Array::from(vec![1, 1, 2]));
Expand Down
65 changes: 50 additions & 15 deletions datafusion/physical-plan/src/joins/hash_join/exec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -160,19 +160,14 @@ fn subtree_max_len(data: &ArrayData) -> usize {
.fold(data.len(), usize::max)
}

/// Decides whether the build side should be joined with an [`ArrayMap`]
/// (perfect hash join), returning the `(min, max)` key range to build it over.
///
/// On `Some`, the memory of the [`ArrayMap`] has already been added to
/// `reservation`; the caller creates the map with [`ArrayMap::try_new`].
fn array_map_key_range(
fn try_build_array_map(
bounds: Option<&PartitionBounds>,
keys: &[ArrayRef],
reservation: &mut MemoryReservation,
perfect_hash_join_small_build_threshold: usize,
perfect_hash_join_min_key_density: f64,
null_equality: NullEquality,
) -> Result<Option<(u64, u64)>> {
) -> Result<Option<ArrayMap>> {
let [key] = keys else {
return Ok(None);
};
Expand Down Expand Up @@ -231,13 +226,29 @@ fn array_map_key_range(
if range >= perfect_hash_join_small_build_threshold as u64
&& dense_ratio <= perfect_hash_join_min_key_density
{
return Ok(None);
if perfect_hash_join_small_build_threshold == 0
|| !perfect_hash_join_min_key_density.is_finite()
{
return Ok(None);
}
let Some(bytes) = ArrayMap::sparse_memory_bound(range, num_rows) else {
return Ok(None);
};
let hash_bytes =
estimate_memory_size::<(u32, u64)>(num_rows, size_of::<JoinHashMapU32>())?;
if bytes > hash_bytes + num_rows * size_of::<u32>() {
return Ok(None);
}
reservation.try_grow(bytes)?;
let map = ArrayMap::try_sparse(key, min_val, range)?;
reservation.shrink(bytes - map.as_ref().map_or(0, ArrayMap::size));
return Ok(map);
}

let mem_size = ArrayMap::estimate_memory_size(min_val, max_val, num_rows);
reservation.try_grow(mem_size)?;

Ok(Some((min_val, max_val)))
ArrayMap::try_new(key, min_val, max_val).map(Some)
}

/// Immutable build buffers and their durable reservation.
Expand Down Expand Up @@ -726,7 +737,7 @@ impl From<&HashJoinExec> for HashJoinExecBuilder {
/// 4. build_side.num_rows() < u32::MAX
/// 5. NullEqualsNothing || (NullEqualsNull && build side doesn't contain null)
///
/// See [`array_map_key_range`] for more details.
/// See [`try_build_array_map`] for more details.
///
/// Note that when using [`PartitionMode::Partitioned`], the build side is split into multiple
/// partitions. This can cause a dense build side to become sparse within each partition,
Expand Down Expand Up @@ -3070,7 +3081,7 @@ impl BuildSideState {
/// (perfect hash join): a single join key of a supported integer type.
///
/// Only a candidate: the final decision also depends on the observed key
/// range and density, see [`array_map_key_range`].
/// range and density, see [`try_build_array_map`].
fn is_perfect_hash_join_candidate(
on_left: &[PhysicalExprRef],
schema: &SchemaRef,
Expand Down Expand Up @@ -3379,16 +3390,14 @@ fn build_in_memory(
};
let left_values = evaluate_expressions_to_arrays(on_left, &batch)?;

let join_hash_map = if let Some((min_val, max_val)) = array_map_key_range(
let join_hash_map = if let Some(array_map) = try_build_array_map(
bounds.as_ref(),
&left_values,
&mut map_reservation,
config.execution.perfect_hash_join_small_build_threshold,
config.execution.perfect_hash_join_min_key_density,
null_equality,
)? {
let array_map = ArrayMap::try_new(&left_values[0], min_val, max_val)?;

array_map_created_count.add(1);
metrics.build_mem_used.add(array_map.size());

Expand Down Expand Up @@ -3726,6 +3735,32 @@ mod tests {
Ok(())
}

#[test]
fn sparse_map_admission_and_duplicate_fallback() -> Result<()> {
let mut keys: Vec<i64> = (0..204).map(|i| i * 980).collect();
let bounds =
PartitionBounds::new(vec![ColumnBounds::new(0_i64.into(), keys[203].into())]);
let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(8192));
for duplicate in [false, true] {
if duplicate {
keys[1] = keys[0];
}
let array: ArrayRef = Arc::new(Int64Array::from(keys.clone()));
let mut reservation = MemoryConsumer::new("sparse").register(&pool);
let map = try_build_array_map(
Some(&bounds),
&[array],
&mut reservation,
1024,
0.15,
NullEquality::NullEqualsNothing,
)?;
assert_eq!(map.is_some(), !duplicate);
assert_eq!(reservation.size(), map.as_ref().map_or(0, ArrayMap::size));
}
Ok(())
}

/// 200 keys spread over 2M: sparse enough for a hash map rather than an
/// array map, and sizing a pruning bitmap at the 128 KiB cap.
fn sparse_keys() -> Vec<i64> {
Expand Down Expand Up @@ -3883,7 +3918,7 @@ mod tests {
}

/// Dense keys build an `ArrayMap`, which is charged to the map's own
/// reservation in `array_map_key_range` rather than the build's.
/// reservation in `try_build_array_map` rather than the build's.
#[rstest]
#[tokio::test]
async fn array_map_stays_reserved_with_dynamic_filter(
Expand Down