Skip to content

Commit 31f1fbe

Browse files
committed
Fix allocation failures during sorting with spill-to-disk
This change replaces `try_resize` with `resize` in three sites, allowing memory to overshoot the configured pool size. These are sites where we don't fall back to spilling to disk when the allocation fails. Fixes: apache#12136
1 parent b10b820 commit 31f1fbe

File tree

2 files changed

+6
-3
lines changed

2 files changed

+6
-3
lines changed

datafusion/physical-plan/src/sorts/builder.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,8 @@ impl BatchBuilder {
6969

7070
/// Append a new batch in `stream_idx`
7171
pub fn push_batch(&mut self, stream_idx: usize, batch: RecordBatch) -> Result<()> {
72-
self.reservation.try_grow(batch.get_array_memory_size())?;
72+
// Allow memory to exceed the limit
73+
self.reservation.grow(batch.get_array_memory_size());
7374
let batch_idx = self.batches.len();
7475
self.batches.push((stream_idx, batch));
7576
self.cursors[stream_idx] = BatchCursor {

datafusion/physical-plan/src/sorts/sort.rs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -431,7 +431,8 @@ impl ExternalSorter {
431431
// Reserve headroom for next sort/merge
432432
self.reserve_memory_for_merge()?;
433433

434-
self.reservation.try_resize(size)?;
434+
// Allow memory to exceed limit temporarily before spill
435+
self.reservation.resize(size);
435436
self.in_mem_batches_sorted = true;
436437
Ok(())
437438
}
@@ -569,7 +570,8 @@ impl ExternalSorter {
569570
if self.runtime.disk_manager.tmp_files_enabled() {
570571
let size = self.sort_spill_reservation_bytes;
571572
if self.merge_reservation.size() != size {
572-
self.merge_reservation.try_resize(size)?;
573+
// Allow memory to exceed the limit
574+
self.merge_reservation.resize(size);
573575
}
574576
}
575577

0 commit comments

Comments
 (0)