Skip to content

Commit 5c256f9

Browse files
authored
Merge pull request containerd#4388 from ChengyuZhu6/zstd
commit: Add support for converting writable layers to zstdchunked blobs
2 parents ece3f19 + 2baeb05 commit 5c256f9

File tree

5 files changed

+85
-8
lines changed

5 files changed

+85
-8
lines changed

cmd/nerdctl/container/container_commit.go

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,9 @@ func CommitCommand() *cobra.Command {
4848
cmd.Flags().Int("estargz-compression-level", 9, "eStargz compression level (1-9)")
4949
cmd.Flags().Int("estargz-chunk-size", 0, "eStargz chunk size")
5050
cmd.Flags().Int("estargz-min-chunk-size", 0, "The minimal number of bytes of data must be written in one gzip stream")
51+
cmd.Flags().Bool("zstdchunked", false, "Convert the committed layer to zstd:chunked for lazy pulling")
52+
cmd.Flags().Int("zstdchunked-compression-level", 3, "zstd:chunked compression level")
53+
cmd.Flags().Int("zstdchunked-chunk-size", 0, "zstd:chunked chunk size")
5154
return cmd
5255
}
5356

@@ -107,6 +110,24 @@ func commitOptions(cmd *cobra.Command) (types.ContainerCommitOptions, error) {
107110
return types.ContainerCommitOptions{}, err
108111
}
109112

113+
zstdchunked, err := cmd.Flags().GetBool("zstdchunked")
114+
if err != nil {
115+
return types.ContainerCommitOptions{}, err
116+
}
117+
zstdchunkedCompressionLevel, err := cmd.Flags().GetInt("zstdchunked-compression-level")
118+
if err != nil {
119+
return types.ContainerCommitOptions{}, err
120+
}
121+
zstdchunkedChunkSize, err := cmd.Flags().GetInt("zstdchunked-chunk-size")
122+
if err != nil {
123+
return types.ContainerCommitOptions{}, err
124+
}
125+
126+
// estargz and zstdchunked are mutually exclusive
127+
if estargz && zstdchunked {
128+
return types.ContainerCommitOptions{}, errors.New("options --estargz and --zstdchunked lead to conflict, only one of them can be used")
129+
}
130+
110131
return types.ContainerCommitOptions{
111132
Stdout: cmd.OutOrStdout(),
112133
GOptions: globalOptions,
@@ -122,6 +143,11 @@ func commitOptions(cmd *cobra.Command) (types.ContainerCommitOptions, error) {
122143
EstargzChunkSize: estargzChunkSize,
123144
EstargzMinChunkSize: estargzMinChunkSize,
124145
},
146+
ZstdChunkedOptions: types.ZstdChunkedOptions{
147+
ZstdChunked: zstdchunked,
148+
ZstdChunkedCompressionLevel: zstdchunkedCompressionLevel,
149+
ZstdChunkedChunkSize: zstdchunkedChunkSize,
150+
},
125151
}, nil
126152
}
127153

docs/command-reference.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -782,6 +782,10 @@ Flags:
782782
- :nerd_face: `--estargz-compression-level`: eStargz compression level (1-9) (default: 9)
783783
- :nerd_face: `--estargz-chunk-size`: eStargz chunk size
784784
- :nerd_face: `--estargz-min-chunk-size`: The minimal number of bytes of data must be written in one gzip stream
785+
- :nerd_face: `--zstdchunked`: Convert the committed layer to zstd:chunked for lazy pulling
786+
support zstdchunked convert
787+
- :nerd_face: `--zstdchunked-compression-level`: zstd:chunked compression level (default: 3)
788+
- :nerd_face: `--zstdchunked-chunk-size`: zstd:chunked chunk size
785789

786790
## Image management
787791

pkg/api/types/container_types.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -398,6 +398,8 @@ type ContainerCommitOptions struct {
398398
Format ImageFormat
399399
// Embed EstargzOptions for eStargz conversion options
400400
EstargzOptions
401+
// Embed ZstdChunkedOptions for zstd:chunked conversion options
402+
ZstdChunkedOptions
401403
}
402404

403405
type CompressionType string

pkg/cmd/container/commit.go

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -44,14 +44,15 @@ func Commit(ctx context.Context, client *containerd.Client, rawRef string, req s
4444
}
4545

4646
opts := &commit.Opts{
47-
Author: options.Author,
48-
Message: options.Message,
49-
Ref: parsedReference.String(),
50-
Pause: options.Pause,
51-
Changes: changes,
52-
Compression: options.Compression,
53-
Format: options.Format,
54-
EstargzOptions: options.EstargzOptions,
47+
Author: options.Author,
48+
Message: options.Message,
49+
Ref: parsedReference.String(),
50+
Pause: options.Pause,
51+
Changes: changes,
52+
Compression: options.Compression,
53+
Format: options.Format,
54+
EstargzOptions: options.EstargzOptions,
55+
ZstdChunkedOptions: options.ZstdChunkedOptions,
5556
}
5657

5758
walker := &containerwalker.ContainerWalker{

pkg/imgutil/commit/commit.go

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
"strings"
2828
"time"
2929

30+
"github.com/klauspost/compress/zstd"
3031
"github.com/opencontainers/go-digest"
3132
"github.com/opencontainers/image-spec/identity"
3233
"github.com/opencontainers/image-spec/specs-go"
@@ -45,6 +46,7 @@ import (
4546
"github.com/containerd/platforms"
4647
"github.com/containerd/stargz-snapshotter/estargz"
4748
estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
49+
zstdchunkedconvert "github.com/containerd/stargz-snapshotter/nativeconverter/zstdchunked"
4850

4951
"github.com/containerd/nerdctl/v2/pkg/api/types"
5052
"github.com/containerd/nerdctl/v2/pkg/clientutil"
@@ -67,6 +69,7 @@ type Opts struct {
6769
Compression types.CompressionType
6870
Format types.ImageFormat
6971
types.EstargzOptions
72+
types.ZstdChunkedOptions
7073
}
7174

7275
var (
@@ -181,6 +184,9 @@ func Commit(ctx context.Context, client *containerd.Client, container containerd
181184
// Sync filesystem to make sure that all the data writes in container could be persisted to disk.
182185
Sync()
183186

187+
if opts.ZstdChunked {
188+
opts.Compression = types.Zstd
189+
}
184190
diffLayerDesc, diffID, err := createDiff(ctx, id, sn, client.ContentStore(), differ, opts.Compression, opts)
185191
if err != nil {
186192
return emptyDigest, fmt.Errorf("failed to export layer: %w", err)
@@ -478,6 +484,44 @@ func createDiff(ctx context.Context, name string, sn snapshots.Snapshotter, cs c
478484
}
479485
}
480486

487+
// Convert to zstd:chunked if requested
488+
if opts.ZstdChunked {
489+
log.G(ctx).Infof("Converting diff layer to zstd:chunked format")
490+
491+
esgzOpts := []estargz.Option{
492+
estargz.WithChunkSize(opts.ZstdChunkedChunkSize),
493+
}
494+
495+
convertFunc := zstdchunkedconvert.LayerConvertFuncWithCompressionLevel(zstd.EncoderLevelFromZstd(opts.ZstdChunkedCompressionLevel), esgzOpts...)
496+
497+
zstdchunkedDesc, err := convertFunc(ctx, cs, newDesc)
498+
if err != nil {
499+
return ocispec.Descriptor{}, digest.Digest(""), fmt.Errorf("failed to convert diff layer to zstd:chunked: %w", err)
500+
} else if zstdchunkedDesc != nil {
501+
zstdchunkedDesc.MediaType = mediaType
502+
zstdchunkedInfo, err := cs.Info(ctx, zstdchunkedDesc.Digest)
503+
if err != nil {
504+
return ocispec.Descriptor{}, digest.Digest(""), err
505+
}
506+
507+
zstdchunkedDiffIDStr, ok := zstdchunkedInfo.Labels["containerd.io/uncompressed"]
508+
if !ok {
509+
return ocispec.Descriptor{}, digest.Digest(""), fmt.Errorf("invalid differ response with no diffID")
510+
}
511+
512+
zstdchunkedDiffID, err := digest.Parse(zstdchunkedDiffIDStr)
513+
if err != nil {
514+
return ocispec.Descriptor{}, digest.Digest(""), err
515+
}
516+
return ocispec.Descriptor{
517+
MediaType: zstdchunkedDesc.MediaType,
518+
Digest: zstdchunkedDesc.Digest,
519+
Size: zstdchunkedDesc.Size,
520+
Annotations: zstdchunkedDesc.Annotations,
521+
}, zstdchunkedDiffID, nil
522+
}
523+
}
524+
481525
return ocispec.Descriptor{
482526
MediaType: mediaType,
483527
Digest: newDesc.Digest,

0 commit comments

Comments
 (0)