Skip to content

Commit ae4f989

Browse files
matttmattt
committed
Add Xet support for faster downloads
1 parent 43c25ac commit ae4f989

File tree

11 files changed

+534
-8
lines changed

11 files changed

+534
-8
lines changed

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
.DS_Store
2-
/.build
2+
.build
33
/Packages
44
xcuserdata/
55
DerivedData/

‎Example/Package.resolved‎

Lines changed: 24 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Example/Package.swift‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// swift-tools-version: 6.0
2+
import PackageDescription
3+
4+
let package = Package(
5+
name: "download-speed-test",
6+
platforms: [
7+
.macOS(.v14),
8+
.iOS(.v16),
9+
],
10+
products: [
11+
.executable(
12+
name: "download-speed-test",
13+
targets: ["DownloadSpeedTest"]
14+
)
15+
],
16+
dependencies: [
17+
.package(path: "../"),
18+
.package(url: "https://github.com/apple/swift-argument-parser", from: "1.3.0"),
19+
],
20+
targets: [
21+
.executableTarget(
22+
name: "DownloadSpeedTest",
23+
dependencies: [
24+
.product(name: "HuggingFace", package: "swift-huggingface"),
25+
.product(name: "ArgumentParser", package: "swift-argument-parser"),
26+
]
27+
)
28+
]
29+
)

‎Example/README.md‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
# Download Speed Test Example
2+
3+
This example demonstrates how to use the HuggingFace Swift package to download files from a repository and measure download performance. It's designed to compare download speeds with and without Xet support.
4+
5+
## Usage
6+
7+
### Running the Test
8+
9+
From the `Example` directory:
10+
11+
```bash
12+
swift run download-speed-test
13+
```
14+
15+
Use `--help` to see all arguments:
16+
17+
```bash
18+
swift run download-speed-test --help
19+
```
20+
21+
### Testing with Xet Enabled/Disabled
22+
23+
To compare performance with and without Xet:
24+
25+
1. **Ensure Xet is available (Swift 6.1+):**
26+
- Build the project with Swift 6.1 or later so the `[email protected]` manifest (which links the Xet binary target) is used.
27+
- Run: `swift run download-speed-test`
28+
29+
2. **Toggle Xet usage from the CLI:**
30+
- Pass `--xet` (default) to use the accelerated path or `--no-xet` to force classic LFS downloads.
31+
- Example: `swift run download-speed-test --no-xet`
32+
33+
3. **Select a different repository:**
34+
- Use `--repo owner/name` (or `-r owner/name`) to target a different model or dataset.
35+
- Example: `swift run download-speed-test --repo meta-llama/Llama-3.2-1B`
36+
37+
4. **Without the Xet binary:**
38+
- If you build with the Swift 6.0 manifest (`Package.swift`) or remove the trait from `[email protected]`, Xet won't be linked and `HubClient.isXetEnabled` will always be `false`.
39+
- Run: `swift run download-speed-test --no-xet`
40+
41+
## What It Does
42+
43+
The test:
44+
1. Connects to the Hugging Face Hub
45+
2. Lists files in the `Qwen/Qwen3-0.6B` repository
46+
3. Selects a diverse set of files (README, config, tokenizer, model files)
47+
4. Downloads each file and measures:
48+
- Download time
49+
- File size
50+
- Download speed
51+
5. Provides a summary with total time, size, and average speed
52+
53+
## Output Example
54+
55+
```
56+
🚀 Hugging Face Download Speed Test
57+
Repository: Qwen/Qwen3-0.6B
58+
============================================================
59+
60+
✅ Xet support: ENABLED
61+
62+
📋 Listing files in repository...
63+
📦 Selected 5 files for testing:
64+
• README.md (12.5 KB)
65+
• config.json (2.1 KB)
66+
• tokenizer.json (1.2 MB)
67+
• model.safetensors (1.1 GB)
68+
• generation_config.json (1.5 KB)
69+
70+
⬇️ Starting download tests...
71+
72+
✅ [1/5] README.md
73+
Time: 0.45s
74+
Size: 12.5 KB
75+
Speed: 27.8 KB/s
76+
77+
...
78+
79+
============================================================
80+
📊 Summary
81+
============================================================
82+
Total files: 5
83+
Total time: 45.23s
84+
Total size: 1.2 GB
85+
Average speed: 27.1 MB/s
86+
```
87+
88+
## Notes
89+
90+
- The test uses a temporary directory that is automatically cleaned up
91+
- Files are downloaded sequentially to get accurate timing
92+
- The test automatically selects a mix of small and large files
93+
- Progress is shown for each file download
94+

‎Example/Sources/DownloadSpeedTest/main.swift‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
import ArgumentParser
2+
import Foundation
3+
import HuggingFace
4+
5+
@main
6+
struct DownloadSpeedTest: AsyncParsableCommand {
7+
static let configuration = CommandConfiguration(
8+
commandName: "download-speed-test",
9+
abstract: "Benchmark download performance for Hugging Face repositories."
10+
)
11+
12+
@Option(
13+
name: [.short, .long],
14+
help: "Repository identifier to benchmark (e.g. owner/name)."
15+
)
16+
var repo: String = "Qwen/Qwen3-0.6B"
17+
18+
@Flag(
19+
name: .long,
20+
inversion: .prefixedNo,
21+
help: "Enable Xet acceleration (use --no-xet to force classic LFS)."
22+
)
23+
var xet: Bool = HubClient.isXetSupported
24+
25+
func run() async throws {
26+
guard let repoID = Repo.ID(rawValue: repo) else {
27+
throw ValidationError("Invalid repository identifier: \(repo). Expected format is owner/name.")
28+
}
29+
30+
let client = HubClient(enableXet: xet)
31+
32+
print("🚀 Hugging Face Download Speed Test")
33+
print("Repository: \(repoID)")
34+
print("=" * 60)
35+
print()
36+
37+
if client.isXetEnabled {
38+
print("✅ Xet support: ENABLED")
39+
} else {
40+
print("❌ Xet support: DISABLED (using LFS)")
41+
}
42+
print()
43+
44+
print("📋 Listing files in repository...")
45+
do {
46+
let files = try await client.listFiles(
47+
in: repoID,
48+
kind: .model,
49+
revision: "main",
50+
recursive: true
51+
)
52+
53+
let testFiles = Self.selectTestFiles(from: files)
54+
55+
if testFiles.isEmpty {
56+
print("⚠️ No suitable test files found")
57+
return
58+
}
59+
60+
print("📦 Selected \(testFiles.count) files for testing:")
61+
for file in testFiles {
62+
let size = file.size.map { Self.formatBytes(Int64($0)) } ?? "unknown size"
63+
print(" • \(file.path) (\(size))")
64+
}
65+
print()
66+
67+
let tempDir = FileManager.default.temporaryDirectory
68+
.appendingPathComponent("hf-speed-test-\(UUID().uuidString)")
69+
try FileManager.default.createDirectory(
70+
at: tempDir,
71+
withIntermediateDirectories: true
72+
)
73+
defer {
74+
try? FileManager.default.removeItem(at: tempDir)
75+
}
76+
77+
var totalTime: TimeInterval = 0
78+
var totalBytes: Int = 0
79+
80+
print("⬇️ Starting download tests...")
81+
print()
82+
83+
for (index, file) in testFiles.enumerated() {
84+
let destination = tempDir.appendingPathComponent(file.path)
85+
86+
try? FileManager.default.createDirectory(
87+
at: destination.deletingLastPathComponent(),
88+
withIntermediateDirectories: true
89+
)
90+
91+
let startTime = Date()
92+
93+
do {
94+
_ = try await client.downloadFile(
95+
at: file.path,
96+
from: repoID,
97+
to: destination,
98+
kind: .model,
99+
revision: "main"
100+
)
101+
102+
let elapsed = Date().timeIntervalSince(startTime)
103+
let fileSize = file.size ?? 0
104+
let speed = fileSize > 0 ? Double(fileSize) / elapsed : 0
105+
106+
totalTime += elapsed
107+
totalBytes += fileSize
108+
109+
print("✅ [\(index + 1)/\(testFiles.count)] \(file.path)")
110+
print(" Time: \(String(format: "%.2f", elapsed))s")
111+
print(" Size: \(Self.formatBytes(Int64(fileSize)))")
112+
print(" Speed: \(Self.formatBytes(Int64(speed)))/s")
113+
print()
114+
} catch {
115+
print("❌ [\(index + 1)/\(testFiles.count)] \(file.path)")
116+
print(" Error: \(error.localizedDescription)")
117+
print()
118+
}
119+
}
120+
121+
print("=" * 60)
122+
print("📊 Summary")
123+
print("=" * 60)
124+
print("Total files: \(testFiles.count)")
125+
print("Total time: \(String(format: "%.2f", totalTime))s")
126+
print("Total size: \(Self.formatBytes(Int64(totalBytes)))")
127+
if totalTime > 0 {
128+
let avgSpeed = Double(totalBytes) / totalTime
129+
print("Average speed: \(Self.formatBytes(Int64(avgSpeed)))/s")
130+
}
131+
print()
132+
print("💡 Tip: toggle Xet via --xet / --no-xet to compare backends.")
133+
134+
} catch {
135+
print("❌ Error: \(error.localizedDescription)")
136+
throw ExitCode.failure
137+
}
138+
}
139+
140+
static func selectTestFiles(from files: [Git.TreeEntry]) -> [Git.TreeEntry] {
141+
var selected: [Git.TreeEntry] = []
142+
143+
let priorities = [
144+
"README.md",
145+
"config.json",
146+
"tokenizer.json",
147+
"*.safetensors",
148+
"*.bin",
149+
]
150+
151+
for priority in priorities {
152+
if priority.contains("*") {
153+
let pattern = priority.replacingOccurrences(of: "*", with: "")
154+
if let file = files.first(where: { $0.path.contains(pattern) && $0.type == .file }) {
155+
if !selected.contains(where: { $0.path == file.path }) {
156+
selected.append(file)
157+
}
158+
}
159+
} else {
160+
if let file = files.first(where: { $0.path == priority && $0.type == .file }) {
161+
selected.append(file)
162+
}
163+
}
164+
}
165+
166+
if selected.count < 3 {
167+
let remaining = files.filter { file in
168+
file.type == .file && !selected.contains(where: { $0.path == file.path }) && (file.size ?? 0) > 0
169+
}
170+
171+
let sorted = remaining.sorted { ($0.size ?? 0) < ($1.size ?? 0) }
172+
selected.append(contentsOf: sorted.prefix(3 - selected.count))
173+
}
174+
175+
return Array(selected.prefix(5))
176+
}
177+
178+
static func formatBytes(_ bytes: Int64) -> String {
179+
let formatter = ByteCountFormatter()
180+
formatter.allowedUnits = [.useKB, .useMB, .useGB]
181+
formatter.countStyle = .file
182+
return formatter.string(fromByteCount: bytes)
183+
}
184+
}
185+
186+
extension String {
187+
static func * (lhs: String, rhs: Int) -> String {
188+
return String(repeating: lhs, count: rhs)
189+
}
190+
}

‎Package.resolved‎

Lines changed: 15 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Package.swift‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,15 @@ let package = Package(
2020
)
2121
],
2222
dependencies: [
23-
.package(url: "https://github.com/mattt/EventSource.git", from: "1.0.0")
23+
.package(url: "https://github.com/mattt/EventSource.git", from: "1.0.0"),
24+
.package(path: "../swift-xet"),
2425
],
2526
targets: [
2627
.target(
2728
name: "HuggingFace",
2829
dependencies: [
29-
.product(name: "EventSource", package: "EventSource")
30+
.product(name: "EventSource", package: "EventSource"),
31+
.product(name: "Xet", package: "swift-xet"),
3032
],
3133
path: "Sources/HuggingFace"
3234
),

‎README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1000,3 +1000,8 @@ let response = try await client.speechToText(
10001000

10011001
print("Transcription: \(response.text)")
10021002
```
1003+
1004+
## License
1005+
1006+
This project is available under the MIT license.
1007+
See the LICENSE file for more info.

0 commit comments

Comments
 (0)