Skip to content

Commit b24a1dc

Browse files
committed
support createSyncAccessHandle mode
1 parent 3ad91aa commit b24a1dc

12 files changed

+87
-64
lines changed

packages/duckdb-wasm/src/bindings/bindings_base.ts

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ import { InstantiationProgress } from './progress';
55
import { DuckDBBindings } from './bindings_interface';
66
import { DuckDBConnection } from './connection';
77
import { StatusCode } from '../status';
8-
import { dropResponseBuffers, DuckDBRuntime, readString, callSRet, copyBuffer, DuckDBDataProtocol } from './runtime';
9-
import { CSVInsertOptions, JSONInsertOptions, ArrowInsertOptions } from './insert_options';
8+
import { callSRet, copyBuffer, dropResponseBuffers, DuckDBDataProtocol, DuckDBRuntime, readString } from './runtime';
9+
import { ArrowInsertOptions, CSVInsertOptions, JSONInsertOptions } from './insert_options';
1010
import { ScriptTokens } from './tokens';
1111
import { FileStatistics } from './file_stats';
1212
import { arrowToSQLField, arrowToSQLType } from '../json_typedef';
@@ -469,9 +469,9 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings {
469469
}
470470
dropResponseBuffers(this.mod);
471471
}
472-
public async prepareFileHandle(fileName: string, protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode): Promise<void> {
472+
public async prepareFileHandle(fileName: string, protocol: DuckDBDataProtocol, accessMode: DuckDBAccessMode, multiWindowMode:boolean): Promise<void> {
473473
if (protocol === DuckDBDataProtocol.BROWSER_FSACCESS && this._runtime.prepareFileHandles) {
474-
const list = await this._runtime.prepareFileHandles([fileName], DuckDBDataProtocol.BROWSER_FSACCESS, accessMode);
474+
const list = await this._runtime.prepareFileHandles([fileName], DuckDBDataProtocol.BROWSER_FSACCESS, accessMode, multiWindowMode);
475475
for (const item of list) {
476476
const { handle, path: filePath, fromCached } = item;
477477
if (!fromCached && handle.getSize()) {
@@ -483,9 +483,9 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings {
483483
throw new Error(`prepareFileHandle: unsupported protocol ${protocol}`);
484484
}
485485
/** Prepare a file handle that could only be acquired aschronously */
486-
public async prepareDBFileHandle(path: string, protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode): Promise<void> {
486+
public async prepareDBFileHandle(path: string, protocol: DuckDBDataProtocol, accessMode: DuckDBAccessMode, multiWindowMode:boolean): Promise<void> {
487487
if (protocol === DuckDBDataProtocol.BROWSER_FSACCESS && this._runtime.prepareDBFileHandle) {
488-
const list = await this._runtime.prepareDBFileHandle(path, DuckDBDataProtocol.BROWSER_FSACCESS, accessMode);
488+
const list = await this._runtime.prepareDBFileHandle(path, DuckDBDataProtocol.BROWSER_FSACCESS, accessMode, multiWindowMode);
489489
for (const item of list) {
490490
const { handle, path: filePath, fromCached } = item;
491491
if (!fromCached && handle.getSize()) {
@@ -655,9 +655,9 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings {
655655
return copy;
656656
}
657657
/** Enable tracking of file statistics */
658-
public async registerOPFSFileName(file: string): Promise<void> {
658+
public async registerOPFSFileName(file: string, accesssMode:DuckDBAccessMode = DuckDBAccessMode.READ_WRITE, multiWindowMode:boolean = false): Promise<void> {
659659
if (file.startsWith("opfs://")) {
660-
return await this.prepareFileHandle(file, DuckDBDataProtocol.BROWSER_FSACCESS);
660+
return await this.prepareFileHandle(file, DuckDBDataProtocol.BROWSER_FSACCESS, accesssMode, multiWindowMode);
661661
} else {
662662
throw new Error("Not an OPFS file name: " + file);
663663
}

packages/duckdb-wasm/src/bindings/bindings_interface.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,15 +54,15 @@ export interface DuckDBBindings {
5454
protocol: DuckDBDataProtocol,
5555
directIO: boolean,
5656
): Promise<HandleType>;
57-
prepareFileHandle(path: string, protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode): Promise<void>;
58-
prepareDBFileHandle(path: string, protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode): Promise<void>;
57+
prepareFileHandle(path: string, protocol: DuckDBDataProtocol, accessMode: DuckDBAccessMode, multiWindowMode:boolean): Promise<void>;
58+
prepareDBFileHandle(path: string, protocol: DuckDBDataProtocol, accessMode: DuckDBAccessMode, multiWindowMode:boolean): Promise<void>;
5959
globFiles(path: string): WebFile[];
6060
dropFile(name: string): void;
6161
dropFiles(names?: string[]): void;
6262
flushFiles(): void;
6363
copyFileToPath(name: string, path: string): void;
6464
copyFileToBuffer(name: string): Uint8Array;
65-
registerOPFSFileName(file: string): Promise<void>;
65+
registerOPFSFileName(file: string, accessMode: DuckDBAccessMode, multiWindowMode:boolean): Promise<void>;
6666
collectFileStatistics(file: string, enable: boolean): void;
6767
exportFileStatistics(file: string): FileStatistics;
6868
}

packages/duckdb-wasm/src/bindings/config.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ export interface DuckDBOPFSConfig {
3636
* - "manual": Files must be manually registered and dropped.
3737
*/
3838
fileHandling?: "auto" | "manual";
39+
window?: "single" | "multi";
3940
}
4041

4142
export enum DuckDBAccessMode {

packages/duckdb-wasm/src/bindings/runtime.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -169,8 +169,8 @@ export interface DuckDBRuntime {
169169

170170
// Prepare a file handle that could only be acquired aschronously
171171
prepareFileHandle?: (path: string, protocol: DuckDBDataProtocol) => Promise<PreparedDBFileHandle[]>;
172-
prepareFileHandles?: (path: string[], protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode) => Promise<PreparedDBFileHandle[]>;
173-
prepareDBFileHandle?: (path: string, protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode) => Promise<PreparedDBFileHandle[]>;
172+
prepareFileHandles?: (path: string[], protocol: DuckDBDataProtocol, accessMode: DuckDBAccessMode, multiWindowMode:boolean) => Promise<PreparedDBFileHandle[]>;
173+
prepareDBFileHandle?: (path: string, protocol: DuckDBDataProtocol, accessMode: DuckDBAccessMode, multiWindowMode:boolean) => Promise<PreparedDBFileHandle[]>;
174174

175175
// Call a scalar UDF function
176176
callScalarUDF(

packages/duckdb-wasm/src/bindings/runtime_browser.ts

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -142,18 +142,27 @@ export const BROWSER_RUNTIME: DuckDBRuntime & {
142142
dirHandle = await dirHandle.getDirectoryHandle(folder, { create: isReadWrite });
143143
}
144144
}
145-
const fileHandle = await dirHandle.getFileHandle(fileName, { create: false }).catch(e => {
145+
let fileHandle:FileSystemFileHandle;
146+
try {
147+
fileHandle = await dirHandle.getFileHandle(fileName, { create: false });
148+
} catch (e:any) {
146149
if (e?.name === 'NotFoundError') {
147150
if (isReadWrite) {
148-
console.debug(`File ${path} does not exists yet, creating...`);
149-
return dirHandle.getFileHandle(fileName, { create: true });
151+
console.debug(`File ${ path } does not exists yet, creating...`);
152+
fileHandle = await dirHandle.getFileHandle(fileName, { create: true });
153+
} else {
154+
console.debug(`File ${ path } does not exists, aborting as we are in read-only mode`);
155+
throw e;
150156
}
151-
console.debug(`File ${path} does not exists, aborting as we are in read-only mode`);
157+
} else {
158+
throw e;
152159
}
153-
throw e;
154-
});
160+
}
155161
try {
156-
const handle = await fileHandle.createSyncAccessHandle();
162+
let syncAccessHandleMode:FileSystemSyncAccessHandleMode = isReadWrite ? "readwrite" : "readwrite-unsafe";
163+
const handle = await fileHandle.createSyncAccessHandle({
164+
mode : syncAccessHandleMode
165+
});
157166
BROWSER_RUNTIME._preparedHandles[path] = handle;
158167
return {
159168
path,
@@ -174,10 +183,10 @@ export const BROWSER_RUNTIME: DuckDBRuntime & {
174183
throw new Error(`Unsupported protocol ${protocol} for paths ${filePaths} with protocol ${protocol}`);
175184
},
176185
/** Prepare a file handle that could only be acquired asynchronously */
177-
async prepareDBFileHandle(dbPath: string, protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode): Promise<PreparedDBFileHandle[]> {
186+
async prepareDBFileHandle(dbPath: string, protocol: DuckDBDataProtocol, accessMode: DuckDBAccessMode, multiWindowMode:boolean): Promise<PreparedDBFileHandle[]> {
178187
if (protocol === DuckDBDataProtocol.BROWSER_FSACCESS && this.prepareFileHandles) {
179188
const filePaths = [dbPath, `${dbPath}.wal`];
180-
return this.prepareFileHandles(filePaths, protocol, accessMode);
189+
return this.prepareFileHandles(filePaths, protocol, accessMode, multiWindowMode);
181190
}
182191
throw new Error(`Unsupported protocol ${protocol} for path ${dbPath} with protocol ${protocol}`);
183192
},

packages/duckdb-wasm/src/parallel/async_bindings.ts

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import { AsyncDuckDBConnection } from './async_connection';
1313
import { CSVInsertOptions, JSONInsertOptions, ArrowInsertOptions } from '../bindings/insert_options';
1414
import { ScriptTokens } from '../bindings/tokens';
1515
import { FileStatistics } from '../bindings/file_stats';
16-
import { DuckDBConfig } from '../bindings/config';
16+
import { DuckDBAccessMode, DuckDBConfig } from '../bindings/config';
1717
import { InstantiationProgress } from '../bindings/progress';
1818
import { arrowToSQLField } from '../json_typedef';
1919
import { WebFile } from '../bindings/web_file';
@@ -592,10 +592,10 @@ export class AsyncDuckDB implements AsyncDuckDBBindings {
592592
}
593593

594594
/** Enable file statistics */
595-
public async registerOPFSFileName(name: string): Promise<void> {
596-
const task = new WorkerTask<WorkerRequestType.REGISTER_OPFS_FILE_NAME, [string], null>(
595+
public async registerOPFSFileName(name: string, accessMode?:DuckDBAccessMode, multiWindowMode?:boolean): Promise<void> {
596+
const task = new WorkerTask<WorkerRequestType.REGISTER_OPFS_FILE_NAME, [string, DuckDBAccessMode, boolean], null>(
597597
WorkerRequestType.REGISTER_OPFS_FILE_NAME,
598-
[name],
598+
[name, accessMode ?? DuckDBAccessMode.READ_ONLY, multiWindowMode ?? false],
599599
);
600600
await this.postTask(task, []);
601601
}
@@ -704,8 +704,8 @@ export class AsyncDuckDB implements AsyncDuckDBBindings {
704704
const result: string[] = [];
705705
for (const file of files) {
706706
try {
707-
await this.registerOPFSFileName(file);
708-
result.push(file);
707+
await this.registerOPFSFileName( file, this.config.accessMode ?? DuckDBAccessMode.READ_WRITE, this.config.opfs?.window == "multi");
708+
result.push( file );
709709
} catch (e) {
710710
console.error(e);
711711
throw new Error("File Not found:" + file);

packages/duckdb-wasm/src/parallel/worker_dispatcher.ts

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { DuckDBBindings, DuckDBDataProtocol } from '../bindings';
1+
import { DuckDBAccessMode, DuckDBBindings, DuckDBDataProtocol } from '../bindings';
22
import { WorkerResponseVariant, WorkerRequestVariant, WorkerRequestType, WorkerResponseType } from './worker_request';
33
import { Logger, LogEntryVariant } from '../log';
44
import { InstantiationProgress } from '../bindings/progress';
@@ -136,9 +136,10 @@ export abstract class AsyncDuckDBDispatcher implements Logger {
136136

137137
case WorkerRequestType.OPEN: {
138138
const path = request.data.path;
139-
const accessMode = request.data.accessMode;
139+
const accessMode = request.data.accessMode
140140
if (path?.startsWith('opfs://')) {
141-
await this._bindings.prepareDBFileHandle(path, DuckDBDataProtocol.BROWSER_FSACCESS, accessMode);
141+
const multiWindowMode = request.data.opfs?.window == "multi";
142+
await this._bindings.prepareDBFileHandle(path, DuckDBDataProtocol.BROWSER_FSACCESS, accessMode ?? DuckDBAccessMode.READ_ONLY, multiWindowMode);
142143
request.data.useDirectIO = true;
143144
}
144145
this._bindings.open(request.data);
@@ -362,7 +363,7 @@ export abstract class AsyncDuckDBDispatcher implements Logger {
362363
break;
363364

364365
case WorkerRequestType.REGISTER_OPFS_FILE_NAME:
365-
await this._bindings.registerOPFSFileName(request.data[0]);
366+
await this._bindings.registerOPFSFileName(request.data[0], request.data[1], request.data[2]);
366367
this.sendOK(request);
367368
break;
368369

packages/duckdb-wasm/src/parallel/worker_request.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import { FileStatistics } from '../bindings/file_stats';
55
import { DuckDBConfig } from '../bindings/config';
66
import { WebFile } from '../bindings/web_file';
77
import { InstantiationProgress } from '../bindings/progress';
8-
import { DuckDBDataProtocol } from '../bindings';
8+
import { DuckDBDataProtocol, DuckDBAccessMode } from '../bindings';
99

1010
export type ConnectionID = number;
1111
export type StatementID = number;
@@ -109,7 +109,7 @@ export type WorkerRequestVariant =
109109
| WorkerRequest<WorkerRequestType.CLOSE_PREPARED, [ConnectionID, StatementID]>
110110
| WorkerRequest<WorkerRequestType.CANCEL_PENDING_QUERY, number>
111111
| WorkerRequest<WorkerRequestType.COLLECT_FILE_STATISTICS, [string, boolean]>
112-
| WorkerRequest<WorkerRequestType.REGISTER_OPFS_FILE_NAME, [string]>
112+
| WorkerRequest<WorkerRequestType.REGISTER_OPFS_FILE_NAME, [string, DuckDBAccessMode, boolean]>
113113
| WorkerRequest<WorkerRequestType.CONNECT, null>
114114
| WorkerRequest<WorkerRequestType.COPY_FILE_TO_BUFFER, string>
115115
| WorkerRequest<WorkerRequestType.COPY_FILE_TO_PATH, [string, string]>
@@ -168,7 +168,7 @@ export type WorkerResponseVariant =
168168

169169
export type WorkerTaskVariant =
170170
| WorkerTask<WorkerRequestType.COLLECT_FILE_STATISTICS, [string, boolean], null>
171-
| WorkerTask<WorkerRequestType.REGISTER_OPFS_FILE_NAME, [string], null>
171+
| WorkerTask<WorkerRequestType.REGISTER_OPFS_FILE_NAME, [string, DuckDBAccessMode, boolean], null>
172172
| WorkerTask<WorkerRequestType.CLOSE_PREPARED, [number, number], null>
173173
| WorkerTask<WorkerRequestType.CONNECT, null, ConnectionID>
174174
| WorkerTask<WorkerRequestType.COPY_FILE_TO_BUFFER, string, Uint8Array>

packages/duckdb-wasm/test/filesystem.test.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,8 @@ export function testFilesystem(
237237
//);
238238
//expect(schema_script.trim()).toEqual(`CREATE TABLE foo(v BIGINT);`);
239239
//expect(csv_buffer.trim()).toEqual(`1\n2\n3\n4\n5`);
240+
241+
await conn.query('DROP TABLE foo');
240242
});
241243

242244
it('Generate Series as Parquet', async () => {
@@ -264,6 +266,8 @@ export function testFilesystem(
264266
expect(content.nullCount).toEqual(0);
265267
expect(content.numRows).toEqual(5);
266268
expect(content.getChildAt(0)?.toArray()).toEqual(new Int32Array([1, 2, 3, 4, 5]));
269+
270+
await conn.query('DROP TABLE foo');
267271
});
268272
});
269273

packages/duckdb-wasm/test/index_browser.ts

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -115,28 +115,28 @@ import { longQueries } from './long_queries.test';
115115
const baseURL = window.location.origin;
116116
const dataURL = `${baseURL}/data`;
117117

118-
testHTTPFS(() => db!);
119-
testHTTPFSAsync(() => adb!, resolveData, dataURL);
120-
testUDF(() => db!);
121-
longQueries(() => adb!);
122-
testTableNames(() => db!);
123-
testTableNamesAsync(() => adb!);
124-
testRegressionAsync(() => adb!);
125-
testAllTypes(() => db!);
126-
testAllTypesAsync(() => adb!);
127-
testBindings(() => db!, dataURL);
128-
testAsyncBindings(() => adb!, dataURL, duckdb.DuckDBDataProtocol.HTTP);
129-
testBatchStream(() => db!);
130-
testAsyncBatchStream(() => adb!);
118+
// testHTTPFS(() => db!);
119+
// testHTTPFSAsync(() => adb!, resolveData, dataURL);
120+
// testUDF(() => db!);
121+
// longQueries(() => adb!);
122+
// testTableNames(() => db!);
123+
// testTableNamesAsync(() => adb!);
124+
// testRegressionAsync(() => adb!);
125+
// testAllTypes(() => db!);
126+
// testAllTypesAsync(() => adb!);
127+
// testBindings(() => db!, dataURL);
128+
// testAsyncBindings(() => adb!, dataURL, duckdb.DuckDBDataProtocol.HTTP);
129+
// testBatchStream(() => db!);
130+
// testAsyncBatchStream(() => adb!);
131131
testFilesystem(() => adb!, resolveData, dataURL, duckdb.DuckDBDataProtocol.HTTP);
132132
testOPFS(dataURL, () => DUCKDB_BUNDLE!);
133-
testArrowInsert(() => db!);
134-
testArrowInsertAsync(() => adb!);
135-
testJSONInsert(() => db!);
136-
testJSONInsertAsync(() => adb!);
137-
testCSVInsert(() => db!);
138-
testCSVInsertAsync(() => adb!);
139-
testTokenization(() => db!);
140-
testTokenizationAsync(() => adb!);
133+
// testArrowInsert(() => db!);
134+
// testArrowInsertAsync(() => adb!);
135+
// testJSONInsert(() => db!);
136+
// testJSONInsertAsync(() => adb!);
137+
// testCSVInsert(() => db!);
138+
// testCSVInsertAsync(() => adb!);
139+
// testTokenization(() => db!);
140+
// testTokenizationAsync(() => adb!);
141141
//testEXCEL(() => db!);
142142
//testJSON(() => db!);

packages/duckdb-wasm/test/opfs.test.ts

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ export function testOPFS(baseDir: string, bundle: () => DuckDBBundle): void {
105105

106106
it('Load Parquet file that are already with empty handler', async () => {
107107
//1. write to opfs
108-
const fileHandler = await getOpfsFileHandlerFromUrl({
108+
await getOpfsFileHandlerFromUrl({
109109
url: `${ baseDir }/tpch/0_01/parquet/lineitem.parquet`,
110110
path: 'test.parquet'
111111
});
@@ -363,8 +363,10 @@ export function testOPFS(baseDir: string, bundle: () => DuckDBBundle): void {
363363

364364
it('Copy CSV to OPFS + Load CSV', async () => {
365365
//1. data preparation
366+
db.config.accessMode = DuckDBAccessMode.READ_WRITE;
366367
db.config.opfs = {
367-
fileHandling: "auto"
368+
fileHandling: "auto",
369+
window:"multi"
368370
};
369371
await conn.query(`COPY ( SELECT 32 AS value ) TO 'opfs://file.csv'`);
370372
await conn.query(`COPY ( SELECT 42 AS value ) TO 'opfs://file.csv'`);
@@ -382,7 +384,6 @@ export function testOPFS(baseDir: string, bundle: () => DuckDBBundle): void {
382384

383385
describe('Open database in OPFS', () => {
384386
it('should not open a non-existent DB file in read-only', async () => {
385-
const logger = new ConsoleLogger(LogLevel.ERROR);
386387
const worker = new Worker(bundle().mainWorker!);
387388
const db_ = new AsyncDuckDB(logger, worker);
388389
await db_.instantiate(bundle().mainModule, bundle().pthreadWorker);
@@ -402,7 +403,6 @@ export function testOPFS(baseDir: string, bundle: () => DuckDBBundle): void {
402403
});
403404

404405
it('should not open a non-existent DB file and mkdir in read-only', async () => {
405-
const logger = new ConsoleLogger(LogLevel.ERROR);
406406
const worker = new Worker(bundle().mainWorker!);
407407
const db_ = new AsyncDuckDB(logger, worker);
408408
await db_.instantiate(bundle().mainModule, bundle().pthreadWorker);
@@ -417,7 +417,6 @@ export function testOPFS(baseDir: string, bundle: () => DuckDBBundle): void {
417417
});
418418

419419
it('should open a non-existent DB file and mkdir in read-write', async () => {
420-
const logger = new ConsoleLogger(LogLevel.ERROR);
421420
const worker = new Worker(bundle().mainWorker!);
422421
const db_ = new AsyncDuckDB(logger, worker);
423422
await db_.instantiate(bundle().mainModule, bundle().pthreadWorker);
@@ -432,7 +431,6 @@ export function testOPFS(baseDir: string, bundle: () => DuckDBBundle): void {
432431
});
433432

434433
it('should open a non-existent DB file in read-write and create files', async () => {
435-
const logger = new ConsoleLogger(LogLevel.ERROR);
436434
const worker = new Worker(bundle().mainWorker!);
437435
const db_ = new AsyncDuckDB(logger, worker);
438436
await db_.instantiate(bundle().mainModule, bundle().pthreadWorker);
Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,13 @@
1-
interface FileSystemFileHandle extends FileSystemHandle {
2-
createSyncAccessHandle(mode?: 'read' | 'readwrite' | 'readwrite-unsafe'): Promise<FileSystemSyncAccessHandle>;
1+
type FileSystemSyncAccessHandleMode = 'readwrite' | 'read-only' | 'readwrite-unsafe';
2+
3+
interface FileSystemCreateSyncAccessHandleOptions {
4+
mode?: FileSystemSyncAccessHandleMode
5+
}
6+
7+
interface FileSystemFileHandle {
8+
createSyncAccessHandle(optional: FileSystemCreateSyncAccessHandleOptions = {}): Promise<FileSystemSyncAccessHandle>;
9+
}
10+
11+
interface FileSystemSyncAccessHandle {
12+
mode: FileSystemSyncAccessHandleMode;
313
}

0 commit comments

Comments
 (0)