Skip to content

Commit 7793275

Browse files
someone1someone1
authored
files: support encrypt, sign, and encrypt + sign options (#31)
* files: support encrypt, sign, and encrypt + sign options * travis: disable zfs 0.8.x * chore: update go deps
1 parent f45656c commit 7793275

19 files changed

+416
-222
lines changed

‎.travis.yml

+10-7
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@ env:
2323
- AWS_SECRET_ACCESS_KEY=minioadmin
2424
- GCS_FAKE_SERVER="https://localhost:4443"
2525
matrix:
26-
- spl=true rel=0.7.13
27-
# - spl=false rel=0.8.3
26+
- rel=0.7.13
27+
# - rel=0.8.3 # some feature preventing zpool creation (encryption)?
2828

2929
before_install:
3030
- export MAKEFLAGS=-j$(($(grep -c '^processor' /proc/cpuinfo) * 2 + 1))
@@ -34,13 +34,16 @@ before_install:
3434
- sudo apt-get install -y linux-headers-`uname -r` tree uuid-dev libattr1-dev libblkid-dev jq gnupg2 xz-utils gzip
3535
- mkdir -p $HOME/zfs
3636
- cd $HOME/zfs
37-
- "[[ -d spl-$rel.tar.gz ]] || curl -L https://github.com/zfsonlinux/zfs/releases/download/zfs-$rel/spl-$rel.tar.gz | tar xz"
37+
- "[[ -d spl-$rel.tar.gz ]] || curl -L https://github.com/zfsonlinux/zfs/releases/download/zfs-$rel/spl-$rel.tar.gz | tar xz || true"
3838
- "[[ -d zfs-$rel.tar.gz ]] || curl -L https://github.com/zfsonlinux/zfs/releases/download/zfs-$rel/zfs-$rel.tar.gz | tar xz"
39-
- (cd spl-$rel && ./configure --prefix=/usr && make && sudo make install)
39+
- (cd spl-$rel && ./configure --prefix=/usr && make && sudo make install) || true
4040
- (cd zfs-$rel && ./configure --prefix=/usr && make && sudo make install)
4141
- sudo modprobe zfs
4242
- cd $TRAVIS_BUILD_DIR
43-
- source ./travis-setup.sh
43+
- mkdir temp
44+
- export TMPDIR=$PWD/temp
45+
- export VDEV=$(mktemp)
46+
- chmod +x ./travis-setup.sh && ./travis-setup.sh
4447

4548
install:
4649
- curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $GOPATH/bin v1.24.0
@@ -51,11 +54,11 @@ install:
5154
script:
5255
- make build
5356
- chmod +x ./integration_test.sh && ./integration_test.sh
54-
- sudo -E $(which go) test -race -v -coverprofile=coverage.out -covermode=atomic -coverpkg=$(go list ./... | grep -v '/vendor/' | paste -sd, -) ./...
57+
- sudo -E TMPDIR=$TMPDIR $(which go) test -race -v -coverprofile=coverage.out -covermode=atomic -coverpkg=$(go list ./... | grep -v '/vendor/' | paste -sd, -) ./...
5558
- make lint
5659

5760
after_success:
5861
- sudo -E $GOPATH/bin/goveralls -coverprofile=coverage.out -service=travis-ci
5962

6063
after_script:
61-
- source ./travis-teardown.sh
64+
- chmod +x ./travis-teardown.sh && ./travis-teardown.sh

‎backup/backup.go

+4-2
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
package backup
2222

2323
import (
24+
"bytes"
2425
"context"
2526
"crypto/md5" //nolint:gosec // Not used for cryptography
2627
"encoding/json"
@@ -477,10 +478,11 @@ func sendStream(ctx context.Context, j *files.JobInfo, c chan<- *files.VolumeInf
477478
var group *errgroup.Group
478479
group, ctx = errgroup.WithContext(ctx)
479480

481+
buf := bytes.NewBuffer(nil)
480482
cmd := zfs.GetZFSSendCommand(ctx, j)
481483
cin, cout := io.Pipe()
482484
cmd.Stdout = cout
483-
cmd.Stderr = os.Stderr
485+
cmd.Stderr = buf
484486
counter := datacounter.NewReaderCounter(cin)
485487
usingPipe := false
486488
if j.MaxFileBuffer == 0 {
@@ -591,7 +593,7 @@ func sendStream(ctx context.Context, j *files.JobInfo, c chan<- *files.VolumeInf
591593

592594
err = group.Wait()
593595
if err != nil {
594-
log.AppLogger.Errorf("Error waiting for zfs command to finish - %v", err)
596+
log.AppLogger.Errorf("Error waiting for zfs command to finish - %v: %s", err, buf.String())
595597
return err
596598
}
597599
log.AppLogger.Infof("zfs send completed without error")

‎backup/restore.go

+9-18
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
package backup
2222

2323
import (
24+
"bytes"
2425
"context"
2526
"crypto/md5" // nolint:gosec // MD5 not used for cryptographic purposes here
2627
"errors"
@@ -244,32 +245,21 @@ func Receive(pctx context.Context, jobInfo *files.JobInfo) error {
244245
}
245246
}
246247

247-
// Compute the Manifest File
248-
tempManifest, err := files.CreateManifestVolume(ctx, jobInfo)
249-
if err != nil {
250-
log.AppLogger.Errorf("Error trying to create manifest volume - %v", err)
251-
return err
252-
}
253-
if err = tempManifest.Close(); err != nil {
254-
log.AppLogger.Warningf("Could not close temporary manifest %v", err)
255-
}
256-
if err = tempManifest.DeleteVolume(); err != nil {
257-
log.AppLogger.Warningf("Could not delete temporary manifest %v", err)
258-
}
248+
manifestObjectName := jobInfo.ManifestObjectName()
259249
// nolint:gosec // MD5 not used for cryptographic purposes here
260-
safeManifestFile := fmt.Sprintf("%x", md5.Sum([]byte(tempManifest.ObjectName)))
250+
safeManifestFile := fmt.Sprintf("%x", md5.Sum([]byte(manifestObjectName)))
261251
safeManifestPath := filepath.Join(localCachePath, safeManifestFile)
262252

263253
// Check to see if we have the manifest file locally
264254
manifest, err := readManifest(ctx, safeManifestPath, jobInfo)
265255
if err != nil {
266256
if os.IsNotExist(err) {
267-
if bErr := backend.PreDownload(ctx, []string{tempManifest.ObjectName}); bErr != nil {
268-
log.AppLogger.Errorf("Error trying to pre download manifest volume %s - %v", tempManifest.ObjectName, bErr)
257+
if bErr := backend.PreDownload(ctx, []string{manifestObjectName}); bErr != nil {
258+
log.AppLogger.Errorf("Error trying to pre download manifest volume %s - %v", manifestObjectName, bErr)
269259
return bErr
270260
}
271261
// Try and download the manifest file from the backend
272-
if dErr := downloadTo(ctx, backend, tempManifest.ObjectName, safeManifestPath); dErr != nil {
262+
if dErr := downloadTo(ctx, backend, manifestObjectName, safeManifestPath); dErr != nil {
273263
return dErr
274264
}
275265
manifest, err = readManifest(ctx, safeManifestPath, jobInfo)
@@ -460,9 +450,10 @@ func processSequence(ctx context.Context, sequence downloadSequence, backend bac
460450
}
461451

462452
func receiveStream(ctx context.Context, cmd *exec.Cmd, j *files.JobInfo, c <-chan *files.VolumeInfo, buffer <-chan interface{}) error {
453+
buf := bytes.NewBuffer(nil)
463454
cin, cout := io.Pipe()
464455
cmd.Stdin = cin
465-
cmd.Stderr = os.Stderr
456+
cmd.Stderr = buf
466457
var group *errgroup.Group
467458
var once sync.Once
468459
group, ctx = errgroup.WithContext(ctx)
@@ -533,7 +524,7 @@ func receiveStream(ctx context.Context, cmd *exec.Cmd, j *files.JobInfo, c <-cha
533524
// Wait for the command to finish
534525
err = group.Wait()
535526
if err != nil {
536-
log.AppLogger.Errorf("Error waiting for zfs command to finish - %v", err)
527+
log.AppLogger.Errorf("Error waiting for zfs command to finish - %v: %s", err, buf.String())
537528
return err
538529
}
539530
log.AppLogger.Infof("zfs receive completed without error")

‎backup/sync.go

+1
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@ func syncCache(ctx context.Context, j *files.JobInfo, localCache string, backend
139139
func validateSnapShotExists(ctx context.Context, snapshot *files.SnapshotInfo, target string, includeBookmarks bool) (bool, error) {
140140
snapshots, err := zfs.GetSnapshotsAndBookmarks(ctx, target)
141141
if err != nil {
142+
log.AppLogger.Debugf("Could not list snapshots for %s: %v", target, err)
142143
// TODO: There are some error cases that are ok to ignore!
143144
return false, nil
144145
}

‎cmd/clean.go

+1-3
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,6 @@
2121
package cmd
2222

2323
import (
24-
"context"
25-
2624
"github.com/spf13/cobra"
2725

2826
"github.com/someone1/zfsbackup-go/backup"
@@ -39,7 +37,7 @@ var cleanCmd = &cobra.Command{
3937
PreRunE: validateCleanFlags,
4038
RunE: func(cmd *cobra.Command, args []string) error {
4139
jobInfo.Destinations = []string{args[0]}
42-
return backup.Clean(context.Background(), &jobInfo, cleanLocal)
40+
return backup.Clean(cmd.Context(), &jobInfo, cleanLocal)
4341
},
4442
}
4543

‎cmd/list.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
package cmd
2222

2323
import (
24-
"context"
2524
"time"
2625

2726
"github.com/spf13/cobra"
@@ -62,7 +61,7 @@ var listCmd = &cobra.Command{
6261
}
6362

6463
jobInfo.Destinations = []string{args[0]}
65-
return backup.List(context.Background(), &jobInfo, startsWith, before, after)
64+
return backup.List(cmd.Context(), &jobInfo, startsWith, before, after)
6665
},
6766
}
6867

‎cmd/receive.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,9 @@ var receiveCmd = &cobra.Command{
4545
log.AppLogger.Infof("Limiting the number of active files to %d", jobInfo.MaxFileBuffer)
4646

4747
if jobInfo.AutoRestore {
48-
return backup.AutoRestore(context.Background(), &jobInfo)
48+
return backup.AutoRestore(cmd.Context(), &jobInfo)
4949
}
50-
return backup.Receive(context.Background(), &jobInfo)
50+
return backup.Receive(cmd.Context(), &jobInfo)
5151
},
5252
}
5353

‎cmd/root.go

+18-6
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
package cmd
2222

2323
import (
24+
"context"
2425
"errors"
2526
"fmt"
2627
"io/ioutil"
@@ -72,8 +73,8 @@ destination of your choosing.`,
7273

7374
// Execute adds all child commands to the root command sets flags appropriately.
7475
// This is called by main.main(). It only needs to happen once to the rootCmd.
75-
func Execute() {
76-
if err := RootCmd.Execute(); err != nil {
76+
func Execute(ctx context.Context) {
77+
if err := RootCmd.ExecuteContext(ctx); err != nil {
7778
os.Exit(-1)
7879
}
7980
}
@@ -243,9 +244,14 @@ func getAndDecryptPrivateKey(email string) (*openpgp.Entity, error) {
243244
}
244245

245246
func loadSendKeys() error {
246-
if jobInfo.EncryptTo != "" && publicKeyRingPath == "" {
247-
log.AppLogger.Errorf("You must specify a public keyring path if you provide an encryptTo option")
248-
return errInvalidInput
247+
if jobInfo.EncryptTo != "" {
248+
if usingSmartOption() && secretKeyRingPath == "" {
249+
log.AppLogger.Errorf("You must specify a secret keyring path if you use a smart option with encryptTo")
250+
return errInvalidInput
251+
} else if publicKeyRingPath == "" {
252+
log.AppLogger.Errorf("You must specify a public keyring path if you provide an encryptTo option")
253+
return errInvalidInput
254+
}
249255
}
250256

251257
if jobInfo.SignFrom != "" && secretKeyRingPath == "" {
@@ -254,7 +260,13 @@ func loadSendKeys() error {
254260
}
255261

256262
if jobInfo.EncryptTo != "" {
257-
if jobInfo.EncryptKey = pgp.GetPublicKeyByEmail(jobInfo.EncryptTo); jobInfo.EncryptKey == nil {
263+
if usingSmartOption() {
264+
var err error
265+
jobInfo.EncryptKey, err = getAndDecryptPrivateKey(jobInfo.EncryptTo)
266+
if err != nil {
267+
return err
268+
}
269+
} else if jobInfo.EncryptKey = pgp.GetPublicKeyByEmail(jobInfo.EncryptTo); jobInfo.EncryptKey == nil {
258270
log.AppLogger.Errorf("Could not find public key for %s", jobInfo.EncryptTo)
259271
return errInvalidInput
260272
}

‎cmd/send.go

+1-7
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ var sendCmd = &cobra.Command{
6363
log.AppLogger.Infof("Will be signed from %s", jobInfo.SignFrom)
6464
}
6565

66-
return backup.Backup(context.Background(), &jobInfo)
66+
return backup.Backup(cmd.Context(), &jobInfo)
6767
},
6868
}
6969

@@ -309,12 +309,6 @@ func validateSendFlags(cmd *cobra.Command, args []string) error {
309309
return err
310310
}
311311

312-
if usingSmartOption() {
313-
if err := loadReceiveKeys(); err != nil {
314-
return err
315-
}
316-
}
317-
318312
if jobInfo.IncrementalSnapshot.Name != "" && fullIncremental != "" {
319313
log.AppLogger.Errorf("The flags -i and -I are mutually exclusive. Please specify only one of these flags.")
320314
return errInvalidInput

‎files/jobinfo.go

+51
Original file line numberDiff line numberDiff line change
@@ -202,3 +202,54 @@ func (j *JobInfo) ValidateSendFlags() error {
202202

203203
return nil
204204
}
205+
206+
func (j *JobInfo) ManifestObjectName() string {
207+
extensions := []string{"manifest"}
208+
nameParts := []string{j.ManifestPrefix}
209+
210+
baseParts, ext := j.volumeNameParts(true)
211+
extensions = append(extensions, ext...)
212+
nameParts = append(nameParts, baseParts...)
213+
214+
return fmt.Sprintf("%s.%s", strings.Join(nameParts, j.Separator), strings.Join(extensions, "."))
215+
}
216+
217+
func (j *JobInfo) BackupVolumeObjectName(volumeNumber int64) string {
218+
extensions := []string{"zstream"}
219+
220+
nameParts, ext := j.volumeNameParts(false)
221+
extensions = append(extensions, ext...)
222+
extensions = append(extensions, fmt.Sprintf("vol%d", volumeNumber))
223+
224+
return fmt.Sprintf("%s.%s", strings.Join(nameParts, j.Separator), strings.Join(extensions, "."))
225+
}
226+
227+
func (j *JobInfo) volumeNameParts(isManifest bool) (nameParts, extensions []string) {
228+
extensions = make([]string, 0, 2)
229+
230+
if j.EncryptKey != nil || j.SignKey != nil {
231+
extensions = append(extensions, "pgp")
232+
}
233+
234+
compressorName := j.Compressor
235+
if isManifest {
236+
compressorName = InternalCompressor
237+
}
238+
239+
switch compressorName {
240+
case InternalCompressor:
241+
extensions = append([]string{"gz"}, extensions...)
242+
case "", ZfsCompressor:
243+
default:
244+
extensions = append([]string{compressorName}, extensions...)
245+
}
246+
247+
nameParts = []string{j.VolumeName}
248+
if j.IncrementalSnapshot.Name != "" {
249+
nameParts = append(nameParts, j.IncrementalSnapshot.Name, "to", j.BaseSnapshot.Name)
250+
} else {
251+
nameParts = append(nameParts, j.BaseSnapshot.Name)
252+
}
253+
254+
return nameParts, extensions
255+
}

0 commit comments

Comments
 (0)