-
Notifications
You must be signed in to change notification settings - Fork 299
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
gguf: Add ability to load local file (#656)
Being able to load a local gguf file can be useful when we want to debug a gguf file. **Without this PR**, this ability could be done by using [file-fetch](https://www.npmjs.com/package/file-fetch). However, that won't work with big models, since the whole file is loaded into RAM. This PR add a new `RangeViewLocalFile` internal class that extends `RangeView`. It redirects calls to `fetchChunk()` to `fs.createReadStream` with the appropriate byte range. This allows the library to read specific chunk from a local file. For security reason, this ability is locked under `localFile: boolean` param. By default, it is disabled (i.e. when this library is run on hub backend, this param is disabled if unspecified) - [x] Add test case to `gguf.spec.ts` - [x] Being able to build with target=browser (only build, but will throw error on browser if being used)
- Loading branch information
Showing
6 changed files
with
205 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,118 @@ | ||
| import { createReadStream } from "node:fs"; | ||
| import { open, stat } from "node:fs/promises"; | ||
| import { Readable } from "node:stream"; | ||
| import type { FileHandle } from "node:fs/promises"; | ||
| import { fileURLToPath } from "node:url"; | ||
|
|
||
| /** | ||
| * @internal | ||
| * | ||
| * A FileBlob is a replacement for the Blob class that allows to lazy read files | ||
| * in order to preserve memory. | ||
| * | ||
| * It is a drop-in replacement for the Blob class, so you can use it as a Blob. | ||
| * | ||
| * The main difference is the instantiation, which is done asynchronously using the `FileBlob.create` method. | ||
| * | ||
| * @example | ||
| * const fileBlob = await FileBlob.create("path/to/package.json"); | ||
| * | ||
| * await fetch("https://aschen.tech", { method: "POST", body: fileBlob }); | ||
| */ | ||
| export class FileBlob extends Blob { | ||
| /** | ||
| * Creates a new FileBlob on the provided file. | ||
| * | ||
| * @param path Path to the file to be lazy readed | ||
| */ | ||
| static async create(path: string | URL): Promise<FileBlob> { | ||
| path = path instanceof URL ? fileURLToPath(path) : path; | ||
|
|
||
| const { size } = await stat(path); | ||
|
|
||
| const fileBlob = new FileBlob(path, 0, size); | ||
|
|
||
| return fileBlob; | ||
| } | ||
|
|
||
| private path: string; | ||
| private start: number; | ||
| private end: number; | ||
|
|
||
| private constructor(path: string, start: number, end: number) { | ||
| super(); | ||
|
|
||
| this.path = path; | ||
| this.start = start; | ||
| this.end = end; | ||
| } | ||
|
|
||
| /** | ||
| * Returns the size of the blob. | ||
| */ | ||
| override get size(): number { | ||
| return this.end - this.start; | ||
| } | ||
|
|
||
| /** | ||
| * Returns a new instance of FileBlob that is a slice of the current one. | ||
| * | ||
| * The slice is inclusive of the start and exclusive of the end. | ||
| * | ||
| * The slice method does not supports negative start/end. | ||
| * | ||
| * @param start beginning of the slice | ||
| * @param end end of the slice | ||
| */ | ||
| override slice(start = 0, end = this.size): FileBlob { | ||
| if (start < 0 || end < 0) { | ||
| new TypeError("Unsupported negative start/end on FileBlob.slice"); | ||
| } | ||
|
|
||
| const slice = new FileBlob(this.path, this.start + start, Math.min(this.start + end, this.end)); | ||
|
|
||
| return slice; | ||
| } | ||
|
|
||
| /** | ||
| * Read the part of the file delimited by the FileBlob and returns it as an ArrayBuffer. | ||
| */ | ||
| override async arrayBuffer(): Promise<ArrayBuffer> { | ||
| const slice = await this.execute((file) => file.read(Buffer.alloc(this.size), 0, this.size, this.start)); | ||
|
|
||
| return slice.buffer; | ||
| } | ||
|
|
||
| /** | ||
| * Read the part of the file delimited by the FileBlob and returns it as a string. | ||
| */ | ||
| override async text(): Promise<string> { | ||
| const buffer = (await this.arrayBuffer()) as Buffer; | ||
|
|
||
| return buffer.toString("utf8"); | ||
| } | ||
|
|
||
| /** | ||
| * Returns a stream around the part of the file delimited by the FileBlob. | ||
| */ | ||
| override stream(): ReturnType<Blob["stream"]> { | ||
| return Readable.toWeb(createReadStream(this.path, { start: this.start, end: this.end - 1 })) as ReturnType< | ||
| Blob["stream"] | ||
| >; | ||
| } | ||
|
|
||
| /** | ||
| * We are opening and closing the file for each action to prevent file descriptor leaks. | ||
| * | ||
| * It is an intended choice of developer experience over performances. | ||
| */ | ||
| private async execute<T>(action: (file: FileHandle) => Promise<T>) { | ||
| const file = await open(this.path, "r"); | ||
|
|
||
| try { | ||
| return await action(file); | ||
| } finally { | ||
| await file.close(); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| const isBrowser = typeof window !== "undefined" && typeof window.document !== "undefined"; | ||
|
|
||
| const isWebWorker = | ||
| typeof self === "object" && self.constructor && self.constructor.name === "DedicatedWorkerGlobalScope"; | ||
|
|
||
| export const isBackend = !isBrowser && !isWebWorker; |