Skip to content

refactor: simplify state of fs store entry #102

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 40 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
90c23be
Add entity-manager dep
rklaehn Jul 8, 2025
90db719
Remove unused dependencies
rklaehn Jul 8, 2025
24c7f2c
Merge branch 'lose-futures-buffered-dep' into entity-manager
rklaehn Jul 8, 2025
126d834
Merge branch 'main' into entity-manager
rklaehn Jul 8, 2025
264ed84
Add import
rklaehn Jul 8, 2025
a859674
Add helpers for set and update
rklaehn Jul 8, 2025
baf4d54
Make bipolar clippy happy again
rklaehn Jul 8, 2025
66f8540
Move message sending into meta
rklaehn Jul 8, 2025
9193b08
Merge branch 'remove-db-public-send' into entity-manager
rklaehn Jul 8, 2025
8cae7cd
WIP
rklaehn Jul 8, 2025
161a70b
Replace slots with EntityHandler
rklaehn Jul 8, 2025
169261a
Get rid of the weak reference
rklaehn Jul 8, 2025
6fe2e07
Inline entity manager and streamline the dispatcher code a bit
rklaehn Jul 9, 2025
fd9faab
revert a few changes to minimize the diff
rklaehn Jul 9, 2025
8ac1b78
codespell
rklaehn Jul 9, 2025
011746d
remove debug code
rklaehn Jul 9, 2025
def9abc
more comments
rklaehn Jul 9, 2025
3d603b0
remove options from BaoFileHandleInner
rklaehn Jul 9, 2025
e65bd07
Get rid of BaoFileHandleInner and all that indirection
rklaehn Jul 9, 2025
e81b1f3
change a few fns to take HashContext directly instead of 2 args
rklaehn Jul 9, 2025
27df6da
more using HashContext instead of TaskContext
rklaehn Jul 9, 2025
47cab9c
Add property based tests for the entity manager
rklaehn Jul 10, 2025
00c54f3
Add a property based test that tests basic functionality of the entit…
rklaehn Jul 10, 2025
664beb8
clippy
rklaehn Jul 10, 2025
7b1b71b
Merge branch 'main' into entity-manager
rklaehn Jul 10, 2025
e584ad1
fmt
rklaehn Jul 10, 2025
4412be9
Add tests for the busy and dead cases which are hard to trigger witho…
rklaehn Jul 10, 2025
c26e555
clippy again
rklaehn Jul 10, 2025
002469a
Merge branch 'entity-manager' into slim-down-state
rklaehn Jul 11, 2025
8004115
Merge branch 'main' into entity-manager
rklaehn Jul 11, 2025
4020bd4
Merge branch 'entity-manager' into slim-down-state
rklaehn Jul 11, 2025
c57f259
Merge branch 'main' into entity-manager
rklaehn Jul 23, 2025
331a3aa
Merge branch 'entity-manager' into slim-down-state
rklaehn Jul 23, 2025
42bbbec
Merge branch 'main' into entity-manager
rklaehn Jul 24, 2025
b595923
Merge branch 'entity-manager' into slim-down-state
rklaehn Jul 24, 2025
91af79e
Merge branch 'main' into entity-manager
rklaehn Aug 5, 2025
21dc789
Merge branch 'entity-manager' into slim-down-state
rklaehn Aug 5, 2025
e028496
Merge branch 'main' into slim-down-state
rklaehn Aug 5, 2025
1be113a
Update comment to remark that Drop is no longer implemented.
rklaehn Aug 5, 2025
7a9be84
Change panic! into unreachable!
rklaehn Aug 5, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 19 additions & 32 deletions src/store/fs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -218,9 +218,13 @@ impl entity_manager::Params for EmParams {
type EntityState = Slot;

async fn on_shutdown(
_state: entity_manager::ActiveEntityState<Self>,
_cause: entity_manager::ShutdownCause,
state: entity_manager::ActiveEntityState<Self>,
cause: entity_manager::ShutdownCause,
) {
if let Some(mut handle) = state.state.0.lock().await.take() {
trace!("shutting down hash: {}, cause: {cause:?}", state.id);
handle.persist(&state);
}
}
}

Expand Down Expand Up @@ -291,7 +295,7 @@ impl HashContext {
.get_or_create(|| async {
let res = self.db().get(hash).await.map_err(io::Error::other)?;
let res = match res {
Some(state) => open_bao_file(&hash, state, &self.global).await,
Some(state) => open_bao_file(state, self).await,
None => Err(io::Error::new(io::ErrorKind::NotFound, "hash not found")),
};
Ok((res?, ()))
Expand All @@ -311,11 +315,8 @@ impl HashContext {
.get_or_create(|| async {
let res = self.db().get(hash).await.map_err(io::Error::other)?;
let res = match res {
Some(state) => open_bao_file(&hash, state, &self.global).await,
None => Ok(BaoFileHandle::new_partial_mem(
hash,
self.global.options.clone(),
)),
Some(state) => open_bao_file(state, self).await,
None => Ok(BaoFileHandle::new_partial_mem()),
};
Ok((res?, ()))
})
Expand All @@ -327,12 +328,9 @@ impl HashContext {
}
}

async fn open_bao_file(
hash: &Hash,
state: EntryState<Bytes>,
ctx: &TaskContext,
) -> io::Result<BaoFileHandle> {
let options = &ctx.options;
async fn open_bao_file(state: EntryState<Bytes>, ctx: &HashContext) -> io::Result<BaoFileHandle> {
let hash = &ctx.id;
let options = &ctx.global.options;
Ok(match state {
EntryState::Complete {
data_location,
Expand Down Expand Up @@ -362,9 +360,9 @@ async fn open_bao_file(
MemOrFile::File(file)
}
};
BaoFileHandle::new_complete(*hash, data, outboard, options.clone())
BaoFileHandle::new_complete(data, outboard)
}
EntryState::Partial { .. } => BaoFileHandle::new_partial_file(*hash, ctx).await?,
EntryState::Partial { .. } => BaoFileHandle::new_partial_file(ctx).await?,
})
}

Expand Down Expand Up @@ -618,12 +616,7 @@ impl Actor {
options: options.clone(),
db: meta::Db::new(db_send),
internal_cmd_tx: fs_commands_tx,
empty: BaoFileHandle::new_complete(
Hash::EMPTY,
MemOrFile::empty(),
MemOrFile::empty(),
options,
),
empty: BaoFileHandle::new_complete(MemOrFile::empty(), MemOrFile::empty()),
protect,
});
rt.spawn(db_actor.run());
Expand Down Expand Up @@ -925,18 +918,14 @@ async fn import_bao_impl(
handle: BaoFileHandle,
ctx: HashContext,
) -> api::Result<()> {
trace!(
"importing bao: {} {} bytes",
handle.hash().fmt_short(),
size
);
trace!("importing bao: {} {} bytes", ctx.id.fmt_short(), size);
let mut batch = Vec::<BaoContentItem>::new();
let mut ranges = ChunkRanges::empty();
while let Some(item) = rx.recv().await? {
// if the batch is not empty, the last item is a leaf and the current item is a parent, write the batch
if !batch.is_empty() && batch[batch.len() - 1].is_leaf() && item.is_parent() {
let bitfield = Bitfield::new_unchecked(ranges, size.into());
handle.write_batch(&batch, &bitfield, &ctx.global).await?;
handle.write_batch(&batch, &bitfield, &ctx).await?;
batch.clear();
ranges = ChunkRanges::empty();
}
Expand All @@ -952,7 +941,7 @@ async fn import_bao_impl(
}
if !batch.is_empty() {
let bitfield = Bitfield::new_unchecked(ranges, size.into());
handle.write_batch(&batch, &bitfield, &ctx.global).await?;
handle.write_batch(&batch, &bitfield, &ctx).await?;
}
Ok(())
}
Expand Down Expand Up @@ -992,7 +981,6 @@ async fn export_ranges_impl(
"export_ranges: exporting ranges: {hash} {ranges:?} size={}",
handle.current_size()?
);
debug_assert!(handle.hash() == hash, "hash mismatch");
let bitfield = handle.bitfield()?;
let data = handle.data_reader();
let size = bitfield.size();
Expand Down Expand Up @@ -1051,8 +1039,7 @@ async fn export_bao_impl(
handle: BaoFileHandle,
) -> anyhow::Result<()> {
let ExportBaoRequest { ranges, hash, .. } = cmd;
debug_assert!(handle.hash() == hash, "hash mismatch");
let outboard = handle.outboard()?;
let outboard = handle.outboard(&hash)?;
let size = outboard.tree.size();
if size == 0 && hash != Hash::EMPTY {
// we have no data whatsoever, so we stop here
Expand Down
Loading
Loading