Skip to content

Commit 2b8c4fd

Browse files
authored
feat: Add fixed size chunking and allow fine-tune of rabin chunking (#422)
Adds following extensions to the in-repo config file and implements it accordingly: - rabin chunker allows to set minimum, maximum and average chunk size - additionally possible to use fixed-size chunking with a configurable size This PR implements restic/restic#5470 which specifies these options in the restic design document.
1 parent 3790932 commit 2b8c4fd

14 files changed

+902
-335
lines changed

crates/core/src/archiver/file_archiver.rs

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
use std::io::Read;
22

3-
use rustic_cdc::Rabin64;
4-
53
use crate::{
64
archiver::{
75
parent::{ItemWithParent, ParentResult},
@@ -36,7 +34,7 @@ use crate::{
3634
pub(crate) struct FileArchiver<'a, BE: DecryptWriteBackend, I: ReadGlobalIndex> {
3735
index: &'a I,
3836
data_packer: Packer<BE>,
39-
rabin: Rabin64,
37+
config: ConfigFile,
4038
}
4139

4240
impl<'a, BE: DecryptWriteBackend, I: ReadGlobalIndex> FileArchiver<'a, BE, I> {
@@ -64,8 +62,6 @@ impl<'a, BE: DecryptWriteBackend, I: ReadGlobalIndex> FileArchiver<'a, BE, I> {
6462
indexer: SharedIndexer<BE>,
6563
config: &ConfigFile,
6664
) -> RusticResult<Self> {
67-
let poly = config.poly()?;
68-
6965
let data_packer = Packer::new(
7066
be,
7167
BlobType::Data,
@@ -74,12 +70,10 @@ impl<'a, BE: DecryptWriteBackend, I: ReadGlobalIndex> FileArchiver<'a, BE, I> {
7470
index.total_size(BlobType::Data),
7571
)?;
7672

77-
let rabin = Rabin64::new_with_polynom(6, &poly);
78-
7973
Ok(Self {
8074
index,
8175
data_packer,
82-
rabin,
76+
config: config.clone(),
8377
})
8478
}
8579

@@ -151,11 +145,11 @@ impl<'a, BE: DecryptWriteBackend, I: ReadGlobalIndex> FileArchiver<'a, BE, I> {
151145
node: Node,
152146
p: &impl Progress,
153147
) -> RusticResult<(Node, u64)> {
154-
let chunks: Vec<_> = ChunkIter::new(
148+
let chunks: Vec<_> = ChunkIter::from_config(
149+
&self.config,
155150
r,
156151
usize::try_from(node.meta.size).unwrap_or(usize::MAX),
157-
self.rabin.clone(),
158-
)
152+
)?
159153
.map(|chunk| {
160154
let chunk = chunk?;
161155
let id = hash(&chunk);

0 commit comments

Comments
 (0)