Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# archives

A Go library for reading and browsing archive files in memory. Supports ZIP, TAR (with gzip, bzip2, xz compression), and Ruby gem formats.
A Go library for reading and browsing archive files in memory. Supports ZIP, TAR (with gzip, bzip2, xz, zstd compression), and Ruby gem formats.

## Installation

Expand Down Expand Up @@ -67,7 +67,7 @@ reader, _ := archives.OpenBytes("pkg.tgz", data)
```

Filename mappings are used first. When a name has no supported extension,
`Open` and `OpenBytes` detect ZIP, TAR, gzip, bzip2, and xz from the content.
`Open` and `OpenBytes` detect ZIP, TAR, gzip, bzip2, xz, and zstd from the content.
Compressed content is opened as TAR and returns a parser error when it does not
contain a TAR archive. `Open` reads at most 512 bytes before rejecting an
unsupported stream with no recognised extension.
Expand Down Expand Up @@ -115,11 +115,11 @@ for _, f := range result.Files {
## Supported formats

- `.zip`, `.jar`, `.whl`, `.nupkg`, `.egg`, `.vsix` (ZIP-based)
- `.tar`, `.tar.gz`, `.tgz`, `.crate`, `.tar.bz2`, `.tar.xz`
- `.tar`, `.tar.gz`, `.tgz`, `.crate`, `.tar.bz2`, `.tar.xz`, `.tar.zst`
- `.gem` (Ruby gems with nested data.tar.gz)
- `.apk` (routed by content: Android packages open as ZIP, Alpine packages open as gzipped tar)

Filenames without a recognised extension are opened by inspecting the first bytes for a ZIP, tar, gzip, bzip2, or xz signature.
Filenames without a recognised extension are opened by inspecting the first bytes for a ZIP, tar, gzip, bzip2, xz, or zstd signature.

## License

Expand Down
10 changes: 9 additions & 1 deletion archives.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
//
// It supports multiple archive formats including:
// - ZIP (.zip, .jar, .whl, .nupkg, .egg, .vsix)
// - TAR (.tar, .tar.gz, .tgz, .crate, .tar.bz2, .tar.xz)
// - TAR (.tar, .tar.gz, .tgz, .crate, .tar.bz2, .tar.xz, .tar.zst)
// - GEM (.gem - Ruby gems with nested tar structure)
//
// The .apk extension is routed by content since Android packages are ZIP
Expand Down Expand Up @@ -31,6 +31,7 @@ const (
formatTGZ = "tgz"
formatTarBzip2 = "tar.bz2"
formatTarXZ = "tar.xz"
formatTarZstd = "tar.zst"
formatGem = "gem"
contentSniffSize = 512
)
Expand Down Expand Up @@ -130,6 +131,8 @@ func openRaw(format string, raw []byte) (Reader, error) {
return openTar(raw, "bzip2")
case formatTarXZ:
return openTar(raw, "xz")
case formatTarZstd:
return openTar(raw, "zstd")
case formatGem:
return openGem(raw)
default:
Expand Down Expand Up @@ -157,6 +160,8 @@ func archiveFormat(detected string) string {
return formatTarBzip2
case "xz":
return formatTarXZ
case "zstd":
return formatTarZstd
default:
return ""
}
Expand Down Expand Up @@ -209,6 +214,9 @@ func detectFormat(filename string) string {
if strings.HasSuffix(filename, ".tar.xz") {
return formatTarXZ
}
if strings.HasSuffix(filename, ".tar.zst") {
return formatTarZstd
}

// Check simple extensions. .apk is deliberately absent: Alpine packages
// are gzipped tarballs and Android packages are zips, so it falls
Expand Down
19 changes: 19 additions & 0 deletions archives_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"testing"
"time"

"github.com/klauspost/compress/zstd"
"github.com/ulikunitz/xz"
)

Expand All @@ -31,6 +32,7 @@ func TestDetectFormat(t *testing.T) {
{"package.tgz", "tgz"},
{"package.tar.bz2", "tar.bz2"},
{"package.tar.xz", "tar.xz"},
{"package.tar.zst", "tar.zst"},
{"package.gem", "gem"},
{"package.vsix", "zip"},
{"package.crate", "tgz"},
Expand Down Expand Up @@ -237,6 +239,22 @@ func createTestTarBz2(t *testing.T) []byte {
return data
}

func createTestTarZst(t *testing.T) []byte {
t.Helper()
buf := new(bytes.Buffer)
w, err := zstd.NewWriter(buf)
if err != nil {
t.Fatal(err)
}
if _, err := w.Write(createTestTar()); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
return buf.Bytes()
}

func createTestTarXz(t *testing.T) []byte {
t.Helper()
buf := new(bytes.Buffer)
Expand Down Expand Up @@ -326,6 +344,7 @@ func TestOpenDetectsExtensionlessArchives(t *testing.T) {
{"gzip", createTestTarGz()},
{"bzip2", createTestTarBz2(t)},
{"xz", createTestTarXz(t)},
{"zstd", createTestTarZst(t)},
}
for _, test := range tests {
t.Run(test.name+"/Open", func(t *testing.T) {
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,6 @@ go 1.25.6

require (
github.com/git-pkgs/magic v0.2.0
github.com/klauspost/compress v1.19.2
github.com/ulikunitz/xz v0.5.16
)
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
github.com/git-pkgs/magic v0.2.0 h1:c7HqVxnP8c88EaVMH0/KraDFVTcmiXckRiSvNZEnvMQ=
github.com/git-pkgs/magic v0.2.0/go.mod h1:3ndidt+yvFaI1M0aEkkzkOlFnLPkeVQASIUojazcxCI=
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0=
github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw=
8 changes: 8 additions & 0 deletions tar.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"io/fs"
"strings"

"github.com/klauspost/compress/zstd"
"github.com/ulikunitz/xz"
)

Expand Down Expand Up @@ -49,6 +50,13 @@ func openTar(raw []byte, compression string) (*tarReader, error) {
return nil, fmt.Errorf("opening xz: %w", err)
}
r = xzReader
case "zstd":
dec, err := zstd.NewReader(content, zstd.WithDecoderConcurrency(1))
if err != nil {
return nil, fmt.Errorf("opening zstd: %w", err)
}
defer dec.Close()
Comment thread
andrew marked this conversation as resolved.
r = dec
}

tr := tar.NewReader(r)
Expand Down