Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions packages/blockstore-core/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -187,18 +187,19 @@
},
"dependencies": {
"@libp2p/logger": "^6.2.4",
"abort-error": "^1.0.2",
"interface-blockstore": "^6.0.0",
"interface-store": "^7.0.0",
"it-all": "^3.0.9",
"it-filter": "^3.1.4",
"it-merge": "^3.0.12",
"multiformats": "^13.4.2"
"multiformats": "^14.0.0"
},
"devDependencies": {
"aegir": "^48.0.4",
"interface-blockstore-tests": "^8.0.0",
"it-drain": "^3.0.10",
"it-to-buffer": "^4.0.10",
"uint8arrays": "^5.1.0"
"it-to-buffer": "^5.0.0",
"uint8arrays": "^6.1.1"
}
}
18 changes: 9 additions & 9 deletions packages/blockstore-core/src/base.ts
Original file line number Diff line number Diff line change
@@ -1,28 +1,28 @@
import type { AbortOptions } from 'abort-error'
import type { Blockstore, InputPair, Pair } from 'interface-blockstore'
import type { AbortOptions, Await, AwaitGenerator, AwaitIterable } from 'interface-store'
import type { CID } from 'multiformats/cid'

export class BaseBlockstore implements Blockstore {
has (key: CID, options?: AbortOptions): Await<boolean> {
has (key: CID, options?: AbortOptions): boolean | Promise<boolean> {
return Promise.reject(new Error('.has is not implemented'))
}

put (key: CID, val: Uint8Array | AwaitIterable<Uint8Array>, options?: AbortOptions): Await<CID> {
put (key: CID, val: Uint8Array | Iterable<Uint8Array> | AsyncIterable<Uint8Array>, options?: AbortOptions): CID | Promise<CID> {
return Promise.reject(new Error('.put is not implemented'))
}

async * putMany (source: AwaitIterable<InputPair>, options?: AbortOptions): AwaitGenerator<CID> {
async * putMany (source: Iterable<InputPair> | AsyncIterable<InputPair>, options?: AbortOptions): Generator<CID> | AsyncGenerator<CID> {
for await (const { cid, bytes } of source) {
await this.put(cid, bytes, options)
yield cid
}
}

get (key: CID, options?: AbortOptions): AwaitGenerator<Uint8Array> {
get (key: CID, options?: AbortOptions): Generator<Uint8Array> | AsyncGenerator<Uint8Array> {
throw new Error('.get is not implemented')
}

async * getMany (source: AwaitIterable<CID>, options?: AbortOptions): AwaitGenerator<Pair> {
async * getMany (source: Iterable<CID> | AsyncIterable<CID>, options?: AbortOptions): Generator<Pair> | AsyncGenerator<Pair> {
for await (const key of source) {
yield {
cid: key,
Expand All @@ -31,11 +31,11 @@ export class BaseBlockstore implements Blockstore {
}
}

delete (key: CID, options?: AbortOptions): Await<void> {
delete (key: CID, options?: AbortOptions): void | Promise<void> {
return Promise.reject(new Error('.delete is not implemented'))
}

async * deleteMany (source: AwaitIterable<CID>, options?: AbortOptions): AwaitGenerator<CID> {
async * deleteMany (source: Iterable<CID> | AsyncIterable<CID>, options?: AbortOptions): Generator<CID> | AsyncGenerator<CID> {
for await (const key of source) {
await this.delete(key, options)
yield key
Expand All @@ -45,7 +45,7 @@ export class BaseBlockstore implements Blockstore {
/**
* Extending classes should override `query` or implement this method
*/
async * getAll (options?: AbortOptions): AwaitGenerator<Pair> { // eslint-disable-line require-yield
async * getAll (options?: AbortOptions): Generator<Pair> | AsyncGenerator<Pair> { // eslint-disable-line require-yield
throw new Error('.getAll is not implemented')
}
}
10 changes: 5 additions & 5 deletions packages/blockstore-core/src/black-hole.ts
Original file line number Diff line number Diff line change
@@ -1,21 +1,21 @@
import { NotFoundError } from 'interface-store'
import { BaseBlockstore } from './base.ts'
import type { AbortOptions } from 'abort-error'
import type { Pair } from 'interface-blockstore'
import type { AbortOptions, Await, AwaitGenerator, AwaitIterable } from 'interface-store'
import type { CID } from 'multiformats/cid'

export class BlackHoleBlockstore extends BaseBlockstore {
put (key: CID, value: Uint8Array | AwaitIterable<Uint8Array>, options?: AbortOptions): Await<CID> {
put (key: CID, value: Uint8Array | Iterable<Uint8Array> | AsyncIterable<Uint8Array>, options?: AbortOptions): CID | Promise<CID> {
options?.signal?.throwIfAborted()
return key
}

get (key: CID, options?: AbortOptions): AwaitGenerator<Uint8Array> {
get (key: CID, options?: AbortOptions): Generator<Uint8Array> | AsyncGenerator<Uint8Array> {
options?.signal?.throwIfAborted()
throw new NotFoundError()
}

has (key: CID, options?: AbortOptions): Await<boolean> {
has (key: CID, options?: AbortOptions): boolean | Promise<boolean> {
options?.signal?.throwIfAborted()
return false
}
Expand All @@ -25,7 +25,7 @@ export class BlackHoleBlockstore extends BaseBlockstore {
}

// eslint-disable-next-line require-yield
async * getAll (options?: AbortOptions): AwaitGenerator<Pair> {
async * getAll (options?: AbortOptions): Generator<Pair> | AsyncGenerator<Pair> {
options?.signal?.throwIfAborted()
}
}
8 changes: 4 additions & 4 deletions packages/blockstore-core/src/identity.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { NotFoundError } from 'interface-store'
import { BaseBlockstore } from './base.ts'
import type { AbortOptions } from 'abort-error'
import type { Blockstore, Pair } from 'interface-blockstore'
import type { AbortOptions, Await, AwaitIterable } from 'interface-store'
import type { CID } from 'multiformats/cid'

// https://github.com/multiformats/multicodec/blob/d06fc6194710e8909bac64273c43f16b56ca4c34/table.csv#L2
Expand All @@ -27,7 +27,7 @@ export class IdentityBlockstore extends BaseBlockstore {
this.maxDigestLength = init?.maxDigestLength
}

put (key: CID, block: Uint8Array | AwaitIterable<Uint8Array>, options?: AbortOptions): Await<CID> {
put (key: CID, block: Uint8Array | Iterable<Uint8Array> | AsyncIterable<Uint8Array>, options?: AbortOptions): CID | Promise<CID> {
if (key.multihash.code === IDENTITY_CODEC) {
if (this.maxDigestLength != null && key.multihash.digest.byteLength > this.maxDigestLength) {
throw new IdentityHashDigestTooLongError(`Identity digest too long - ${key.multihash.digest.byteLength} > this.maxDigestLength`)
Expand Down Expand Up @@ -64,7 +64,7 @@ export class IdentityBlockstore extends BaseBlockstore {
yield * this.child.get(key, options)
}

has (key: CID, options?: AbortOptions): Await<boolean> {
has (key: CID, options?: AbortOptions): boolean | Promise<boolean> {
if (key.multihash.code === IDENTITY_CODEC) {
if (this.maxDigestLength != null && key.multihash.digest.byteLength > this.maxDigestLength) {
throw new IdentityHashDigestTooLongError(`Identity digest too long - ${key.multihash.digest.byteLength} > this.maxDigestLength`)
Expand All @@ -82,7 +82,7 @@ export class IdentityBlockstore extends BaseBlockstore {
return this.child.has(key, options)
}

delete (key: CID, options?: AbortOptions): Await<void> {
delete (key: CID, options?: AbortOptions): void | Promise<void> {
if (key.code === IDENTITY_CODEC) {
if (this.maxDigestLength != null && key.multihash.digest.byteLength > this.maxDigestLength) {
throw new IdentityHashDigestTooLongError(`Identity digest too long - ${key.multihash.digest.byteLength} > this.maxDigestLength`)
Expand Down
12 changes: 6 additions & 6 deletions packages/blockstore-core/src/memory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ import { CID } from 'multiformats/cid'
import * as raw from 'multiformats/codecs/raw'
import * as Digest from 'multiformats/hashes/digest'
import { BaseBlockstore } from './base.ts'
import type { AbortOptions } from 'abort-error'
import type { Pair } from 'interface-blockstore'
import type { AbortOptions, Await, AwaitGenerator, AwaitIterable } from 'interface-store'

function isPromise <T> (p?: any): p is Promise<T> {
return typeof p?.then === 'function'
Expand All @@ -21,7 +21,7 @@ export class MemoryBlockstore extends BaseBlockstore {
this.data = new Map()
}

put (key: CID, val: Uint8Array | AwaitIterable<Uint8Array>, options?: AbortOptions): Await<CID> {
put (key: CID, val: Uint8Array | Iterable<Uint8Array> | AsyncIterable<Uint8Array>, options?: AbortOptions): CID | Promise<CID> {
options?.signal?.throwIfAborted()

let buf: Uint8Array[]
Expand All @@ -43,15 +43,15 @@ export class MemoryBlockstore extends BaseBlockstore {
return this._put(key, buf, options)
}

private _put (key: CID, val: Uint8Array[], options?: AbortOptions): Await<CID> {
private _put (key: CID, val: Uint8Array[], options?: AbortOptions): CID | Promise<CID> {
options?.signal?.throwIfAborted()

this.data.set(base32.encode(key.multihash.bytes), val)

return key
}

* get (key: CID, options?: AbortOptions): AwaitGenerator<Uint8Array> {
* get (key: CID, options?: AbortOptions): Generator<Uint8Array> | AsyncGenerator<Uint8Array> {
options?.signal?.throwIfAborted()
const buf = this.data.get(base32.encode(key.multihash.bytes))

Expand All @@ -62,7 +62,7 @@ export class MemoryBlockstore extends BaseBlockstore {
yield * buf
}

has (key: CID, options?: AbortOptions): Await<boolean> {
has (key: CID, options?: AbortOptions): boolean | Promise<boolean> {
options?.signal?.throwIfAborted()
return this.data.has(base32.encode(key.multihash.bytes))
}
Expand All @@ -72,7 +72,7 @@ export class MemoryBlockstore extends BaseBlockstore {
this.data.delete(base32.encode(key.multihash.bytes))
}

* getAll (options?: AbortOptions): AwaitGenerator<Pair> {
* getAll (options?: AbortOptions): Generator<Pair> | AsyncGenerator<Pair> {
options?.signal?.throwIfAborted()

for (const [key, value] of this.data.entries()) {
Expand Down
12 changes: 6 additions & 6 deletions packages/blockstore-core/src/tiered.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ import { NotFoundError } from 'interface-store'
import filter from 'it-filter'
import merge from 'it-merge'
import { BaseBlockstore } from './base.ts'
import type { AbortOptions } from 'abort-error'
import type { Blockstore, InputPair, Pair } from 'interface-blockstore'
import type { AbortOptions, AwaitGenerator, AwaitIterable } from 'interface-store'
import type { CID } from 'multiformats/cid'

const log = logger('blockstore:core:tiered')
Expand All @@ -24,7 +24,7 @@ export class TieredBlockstore extends BaseBlockstore {
this.stores = stores.slice()
}

async put (key: CID, value: Uint8Array | AwaitIterable<Uint8Array>, options?: AbortOptions): Promise<CID> {
async put (key: CID, value: Uint8Array | Iterable<Uint8Array> | AsyncIterable<Uint8Array>, options?: AbortOptions): Promise<CID> {
await Promise.all(
this.stores.map(async store => {
await store.put(key, value, options)
Expand All @@ -34,7 +34,7 @@ export class TieredBlockstore extends BaseBlockstore {
return key
}

async * get (key: CID, options?: AbortOptions): AwaitGenerator<Uint8Array> {
async * get (key: CID, options?: AbortOptions): Generator<Uint8Array> | AsyncGenerator<Uint8Array> {
let error: Error | undefined

for (const store of this.stores) {
Expand Down Expand Up @@ -68,21 +68,21 @@ export class TieredBlockstore extends BaseBlockstore {
)
}

async * putMany (source: AwaitIterable<InputPair>, options: AbortOptions = {}): AwaitGenerator<CID> {
async * putMany (source: Iterable<InputPair> | AsyncIterable<InputPair>, options: AbortOptions = {}): Generator<CID> | AsyncGenerator<CID> {
for await (const pair of source) {
await this.put(pair.cid, pair.bytes, options)
yield pair.cid
}
}

async * deleteMany (source: AwaitIterable<CID>, options: AbortOptions = {}): AwaitGenerator<CID> {
async * deleteMany (source: Iterable<CID> | AsyncIterable<CID>, options: AbortOptions = {}): Generator<CID> | AsyncGenerator<CID> {
for await (const cid of source) {
await this.delete(cid, options)
yield cid
}
}

async * getAll (options?: AbortOptions): AwaitGenerator<Pair> {
async * getAll (options?: AbortOptions): Generator<Pair> | AsyncGenerator<Pair> {
// deduplicate yielded pairs
const seen = new Set<string>()

Expand Down
2 changes: 1 addition & 1 deletion packages/blockstore-core/test/identity.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ import { identity } from 'multiformats/hashes/identity'
import { sha256 } from 'multiformats/hashes/sha2'
import { IdentityBlockstore } from '../src/identity.ts'
import { MemoryBlockstore } from '../src/memory.ts'
import type { AbortOptions } from 'abort-error'
import type { Blockstore } from 'interface-blockstore'
import type { AbortOptions } from 'interface-store'

describe('identity', () => {
let blockstore: Blockstore
Expand Down
3 changes: 2 additions & 1 deletion packages/blockstore-fs/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -162,12 +162,13 @@
"release": "aegir release"
},
"dependencies": {
"abort-error": "^1.0.2",
"interface-blockstore": "^6.0.0",
"interface-store": "^7.0.0",
"it-glob": "^3.0.4",
"it-map": "^3.1.4",
"it-parallel-batch": "^3.0.9",
"multiformats": "^13.4.2",
"multiformats": "^14.0.0",
"race-signal": "^2.0.0",
"steno": "^4.0.2"
},
Expand Down
14 changes: 7 additions & 7 deletions packages/blockstore-fs/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,15 @@ import { raceSignal } from 'race-signal'
import { Writer } from 'steno'
import { NextToLast } from './sharding.ts'
import type { ShardingStrategy } from './sharding.ts'
import type { AbortOptions } from 'abort-error'
import type { Blockstore, Pair } from 'interface-blockstore'
import type { AbortOptions, AwaitGenerator, AwaitIterable } from 'interface-store'
import type { CID } from 'multiformats/cid'
import type { FileHandle } from 'node:fs/promises'

/**
* Write a file atomically
*/
async function writeFile (file: string, contents: Uint8Array | AwaitIterable<Uint8Array>, options?: AbortOptions): Promise<void> {
async function writeFile (file: string, contents: Uint8Array | Iterable<Uint8Array> | AsyncIterable<Uint8Array>, options?: AbortOptions): Promise<void> {
try {
options?.signal?.throwIfAborted()
await raceSignal(fs.mkdir(path.dirname(file), {
Expand Down Expand Up @@ -147,7 +147,7 @@ export class FsBlockstore implements Blockstore {
await Promise.resolve()
}

async put (key: CID, val: Uint8Array | AwaitIterable<Uint8Array>, options?: AbortOptions): Promise<CID> {
async put (key: CID, val: Uint8Array | Iterable<Uint8Array> | AsyncIterable<Uint8Array>, options?: AbortOptions): Promise<CID> {
const { dir, file } = this.shardingStrategy.encode(key)

try {
Expand All @@ -159,7 +159,7 @@ export class FsBlockstore implements Blockstore {
}
}

async * putMany (source: AwaitIterable<Pair>, options?: AbortOptions): AsyncGenerator<CID> {
async * putMany (source: Iterable<Pair> | AsyncIterable<Pair>, options?: AbortOptions): AsyncGenerator<CID> {
yield * parallelBatch(
map(source, ({ cid, bytes }) => {
return async () => {
Expand Down Expand Up @@ -191,7 +191,7 @@ export class FsBlockstore implements Blockstore {
}
}

async * getMany (source: AwaitIterable<CID>, options?: AbortOptions): AsyncGenerator<Pair> {
async * getMany (source: Iterable<CID> | AsyncIterable<CID>, options?: AbortOptions): AsyncGenerator<Pair> {
yield * parallelBatch(
map(source, key => {
return async () => {
Expand Down Expand Up @@ -220,7 +220,7 @@ export class FsBlockstore implements Blockstore {
}
}

async * deleteMany (source: AwaitIterable<CID>, options?: AbortOptions): AsyncGenerator<CID> {
async * deleteMany (source: Iterable<CID> | AsyncIterable<CID>, options?: AbortOptions): AsyncGenerator<CID> {
yield * parallelBatch(
map(source, key => {
return async () => {
Expand All @@ -245,7 +245,7 @@ export class FsBlockstore implements Blockstore {
return true
}

async * getAll (options?: AbortOptions): AwaitGenerator<Pair> {
async * getAll (options?: AbortOptions): Generator<Pair> | AsyncGenerator<Pair> {
const pattern = `**/*${this.shardingStrategy.extension}`
.split(path.sep)
.join('/')
Expand Down
5 changes: 3 additions & 2 deletions packages/blockstore-idb/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -144,13 +144,14 @@
"release": "aegir release"
},
"dependencies": {
"abort-error": "^1.0.2",
"blockstore-core": "^6.0.0",
"idb": "^8.0.3",
"interface-blockstore": "^6.0.0",
"interface-store": "^7.0.0",
"it-all": "^3.0.9",
"it-to-buffer": "^4.0.10",
"multiformats": "^13.4.2",
"it-to-buffer": "^5.0.0",
"multiformats": "^14.0.0",
"race-signal": "^2.0.0"
},
"devDependencies": {
Expand Down
Loading
Loading