Merge pull request #38 from uyuni-project/add-zstd-compression-support

Add zst compression type support

get/syncer.go

@@ -12,15 +12,17 @@ import (
 	"log"
 	"net/url"
 	"path"
+	"path/filepath"
 	"strings"
 
+	"github.com/klauspost/compress/zstd"
 	"github.com/uyuni-project/minima/util"
 	"golang.org/x/crypto/openpgp"
 )
 
 // common
 
-// XMLLocation maps a <location> tag in repodata/repomd.xml or repodata/<ID>-primary.xml.gz
+// XMLLocation maps a <location> tag in repodata/repomd.xml or repodata/<ID>-primary.xml.<compression>
 type XMLLocation struct {
 	Href string `xml:"href,attr"`
 }
@@ -39,21 +41,21 @@ type XMLData struct {
 	Checksum XMLChecksum `xml:"checksum"`
 }
 
-// repodata/<ID>-primary.xml.gz
+// repodata/<ID>-primary.xml.<compression>
 
-// XMLMetaData maps a <metadata> tag in repodata/<ID>-primary.xml.gz
+// XMLMetaData maps a <metadata> tag in repodata/<ID>-primary.xml.<compression>
 type XMLMetaData struct {
 	Packages []XMLPackage `xml:"package"`
 }
 
-// XMLPackage maps a <package> tag in repodata/<ID>-primary.xml.gz
+// XMLPackage maps a <package> tag in repodata/<ID>-primary.xml.<compression>
 type XMLPackage struct {
 	Arch     string      `xml:"arch"`
 	Location XMLLocation `xml:"location"`
 	Checksum XMLChecksum `xml:"checksum"`
 }
 
-// XMLChecksum maps a <checksum> tag in repodata/<ID>-primary.xml.gz
+// XMLChecksum maps a <checksum> tag in repodata/<ID>-primary.xml.<compression>
 type XMLChecksum struct {
 	Type     string `xml:"type,attr"`
 	Checksum string `xml:",cdata"`
@@ -72,7 +74,7 @@ type RepoType struct {
 	MetadataPath string
 	PackagesType string
 	DecodeMetadata func(io.Reader) (XMLRepomd, error)
-	DecodePackages func(io.Reader) (XMLMetaData, error)
+	DecodePackages func(io.Reader, string) (XMLMetaData, error)
 	MetadataSignatureExt string
     Noarch string
 }
@@ -313,19 +315,35 @@ func (e *SignatureError) Error() string {
 	return fmt.Sprintf("Signature error: %s", e.reason)
 }
 
-func readMetaData(reader io.Reader) (primary XMLMetaData, err error) {
-	gzReader, err := gzip.NewReader(reader)
-	if err != nil {
-		return
-	}
-	defer gzReader.Close()
+// Uncompress and read primary XML
+func readMetaData(reader io.Reader, compType string) (XMLMetaData, error) {
+	var primary XMLMetaData
+	switch compType {
+	case "gz":
+		reader, err := gzip.NewReader(reader)
+		if err != nil {
+			return primary, err
+		}
+		defer reader.Close()
 
-	decoder := xml.NewDecoder(gzReader)
-	err = decoder.Decode(&primary)
+		decoder := xml.NewDecoder(reader)
+		err = decoder.Decode(&primary)
+	case "zst":
+		reader, err := zstd.NewReader(reader)
+		if err != nil {
+			return primary, err
+		}
+		defer reader.Close()
 
-	return
+		decoder := xml.NewDecoder(reader)
+		err = decoder.Decode(&primary)
+	default:
+		return primary, errors.New("Unsupported compression type")
+	}
+	return primary, nil
 }
 
+
 func (r *Syncer) readChecksumMap() (checksumMap map[string]XMLChecksum) {
 	checksumMap = make(map[string]XMLChecksum)
 	repomdReader, err := r.storage.NewReader(repomdPath, Permanent)
@@ -361,7 +379,8 @@ func (r *Syncer) readChecksumMap() (checksumMap map[string]XMLChecksum) {
 			if err != nil {
 				return
 			}
-			primary, err := repoType.DecodePackages(primaryReader)
+			compType := strings.Trim(filepath.Ext(dataHref), ".")
+			primary, err := repoType.DecodePackages(primaryReader, compType)
 			if err != nil {
 				return
 			}
@@ -380,7 +399,8 @@ func (r *Syncer) processPrimary(path string, checksumMap map[string]XMLChecksum,
 	if err != nil {
 		return
 	}
-	primary, err := repoType.DecodePackages(reader)
+	compType := strings.Trim(filepath.Ext(path), ".")
+	primary, err := repoType.DecodePackages(reader, compType)
 	if err != nil {
 		return
 	}
@@ -473,7 +493,7 @@ func decodeRelease(reader io.Reader) (repomd XMLRepomd, err error) {
 	return
 }
 
-func decodePackages(reader io.Reader) (metadata XMLMetaData, err error) {
+func decodePackages(reader io.Reader, _ string) (metadata XMLMetaData, err error) {
 	packagesEntries, err := util.ProcessPropertiesFile(reader)
 	if err != nil {
 		return

get/syncer_test.go

@@ -68,6 +68,62 @@ func TestStoreRepo(t *testing.T) {
 	}
 }
 
+func TestStoreRepoZstd(t *testing.T) {
+	directory := filepath.Join(os.TempDir(), "syncer_test")
+	err := os.RemoveAll(directory)
+	if err != nil {
+		t.Error(err)
+	}
+
+	archs := map[string]bool{
+		"x86_64": true,
+	}
+	storage := NewFileStorage(directory)
+	url, err := url.Parse("http://localhost:8080/zstrepo")
+	if err != nil {
+		t.Error(err)
+	}
+	syncer := NewSyncer(*url, archs, storage)
+
+	// first sync
+	err = syncer.StoreRepo()
+	if err != nil {
+		t.Error(err)
+	}
+
+	expectedFiles := []string{
+		filepath.Join("repodata", "106c411e443c6b97e7d547f78e32d6d2cdf8e80577999954fdb770d8a21581a0-filelists.xml.zst"),
+		filepath.Join("repodata", "d7fd4cf502d9e1ab8865bc9690c34c3cadcfda0db269f4ddc9c6f026db3f2400-primary.xml.zst"),
+		filepath.Join("repodata", "fa0767ea9359279bb6e8ec93a70cefb3863e4fa4bcbeebbfe755a5ce16c21b94-other.xml.zst"),
+		filepath.Join("repodata", "repomd.xml"),
+		filepath.Join("x86_64", "milkyway-dummy-2.0-1.1.x86_64.rpm"),
+		filepath.Join("x86_64", "orion-dummy-1.1-1.1.x86_64.rpm"),
+		filepath.Join("x86_64", "hoag-dummy-1.1-2.1.x86_64.rpm"),
+		filepath.Join("x86_64", "perseus-dummy-1.1-1.1.x86_64.rpm"),
+		filepath.Join("x86_64", "orion-dummy-sle12-1.1-4.1.x86_64.rpm"),
+	}
+
+	for _, file := range expectedFiles {
+		originalInfo, serr := os.Stat(filepath.Join("testdata", "zstrepo", file))
+		if err != nil {
+			t.Fatal(serr)
+		}
+		syncedInfo, serr := os.Stat(filepath.Join(directory, file))
+		if serr != nil {
+			t.Fatal(serr)
+		}
+		if originalInfo.Size() != syncedInfo.Size() {
+			t.Error("original and synced versions of", file, "differ:", originalInfo.Size(), "vs", syncedInfo.Size())
+		}
+	}
+
+	// second sync
+	err = syncer.StoreRepo()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
 func TestStoreDebRepo(t *testing.T) {
 	directory := filepath.Join(os.TempDir(), "syncer_test")
 	err := os.RemoveAll(directory)

get/testdata/zstrepo/repodata/repomd.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">
+  <revision>1720081452</revision>
+  <data type="primary">
+    <checksum type="sha256">d7fd4cf502d9e1ab8865bc9690c34c3cadcfda0db269f4ddc9c6f026db3f2400</checksum>
+    <open-checksum type="sha256">cff719a5cea867237259b112623d7d7e5804c863dd41dadfe52256593aaee640</open-checksum>
+    <location href="repodata/d7fd4cf502d9e1ab8865bc9690c34c3cadcfda0db269f4ddc9c6f026db3f2400-primary.xml.zst"/>
+    <timestamp>1720081452</timestamp>
+    <size>1347</size>
+    <open-size>7245</open-size>
+  </data>
+  <data type="filelists">
+    <checksum type="sha256">106c411e443c6b97e7d547f78e32d6d2cdf8e80577999954fdb770d8a21581a0</checksum>
+    <open-checksum type="sha256">dd55851e15239a7d86bf045e7729259b6c0dd884e202a651eac05c4dd014fd40</open-checksum>
+    <location href="repodata/106c411e443c6b97e7d547f78e32d6d2cdf8e80577999954fdb770d8a21581a0-filelists.xml.zst"/>
+    <timestamp>1720081452</timestamp>
+    <size>566</size>
+    <open-size>1715</open-size>
+  </data>
+  <data type="other">
+    <checksum type="sha256">fa0767ea9359279bb6e8ec93a70cefb3863e4fa4bcbeebbfe755a5ce16c21b94</checksum>
+    <open-checksum type="sha256">9285a7e58c6870843af0f7cf069592be7ea7dd087d4836c0485f5d170e997ce5</open-checksum>
+    <location href="repodata/fa0767ea9359279bb6e8ec93a70cefb3863e4fa4bcbeebbfe755a5ce16c21b94-other.xml.zst"/>
+    <timestamp>1720081452</timestamp>
+    <size>573</size>
+    <open-size>1660</open-size>
+  </data>
+</repomd>

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7 // indirect
 	github.com/jtolds/gls v4.2.1+incompatible // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/smartystreets/assertions v0.0.0-20170818220048-9c0ea8acbc1d // indirect
 	github.com/smartystreets/goconvey v0.0.0-20170825221426-e5b2b7c91115 // indirect

go.sum

@@ -13,6 +13,8 @@ github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7 h1:SMvOWPJCES
 github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=