Skip to content

Commit

Permalink
Ensures files are also deleted in the commit
Browse files Browse the repository at this point in the history
  • Loading branch information
hermanbanken committed Apr 26, 2021
1 parent a0dc614 commit 1185e69
Show file tree
Hide file tree
Showing 4 changed files with 200 additions and 53 deletions.
46 changes: 2 additions & 44 deletions cmd/sync/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
"time"

"github.com/Q42Philips/gitops-sync/pkg/gitlogic"
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-billy/v5/osfs"
"github.com/go-git/go-git/v5"
Expand Down Expand Up @@ -158,7 +157,7 @@ func Main() {
commitOpt := &git.CommitOptions{Author: signature, Committer: signature}

// Do sync & commit
obj := sync(outputRepo, inputFs, commitOpt, *commitMsg)
obj := gitlogic.Sync(outputRepo, *outputRepoPath, inputFs, commitOpt, *commitMsg)

// Commit
ref := plumbing.NewHashReference(headRefName, obj.Hash)
Expand Down Expand Up @@ -228,7 +227,7 @@ func Main() {
// Draft merge commit opts
commitOpt.Parents = []plumbing.Hash{baseMergeBeforeHash, obj.Hash}
// Then sync again by overwriting with our inputFs
obj = sync(outputRepo, inputFs, commitOpt, fmt.Sprintf("Merge %s into %s", headRefName.Short(), baseMergeRefName.Short()))
obj = gitlogic.Sync(outputRepo, *outputRepoPath, inputFs, commitOpt, fmt.Sprintf("Merge %s into %s", headRefName.Short(), baseMergeRefName.Short()))
ref := plumbing.NewHashReference(baseMergeRefName, obj.Hash)
err = outputStorer.SetReference(ref)
orFatal(err, "updating ref (merged)")
Expand Down Expand Up @@ -341,44 +340,3 @@ func firstStr(args ...string) string {
}
return ""
}

func sync(gr *git.Repository, inputFs billy.Filesystem, commitOpt *git.CommitOptions, msg string) *object.Commit {
// Do sync
w, err := gr.Worktree()
orFatal(err, "getting worktree")

outputFs := w.Filesystem
err = gitlogic.RmRecursively(outputFs, *outputRepoPath) // remove existing files
orFatal(err, "removing old artifacts")
if *outputRepoPath != "." && *outputRepoPath != "" {
outputFs, err = gitlogic.ChrootMkdir(outputFs, *outputRepoPath)
orFatal(err, "failed to go to subdirectory")
}
err = gitlogic.Copy(inputFs, outputFs)
orFatal(err, "copy files")
w.Add(*outputRepoPath)

// Print status
status, err := w.Status()
if len(status) == 0 {
log.Println("No changes. Skipping commit.")
head, err := gr.Head()
orFatal(err, "getting head")
obj, err := gr.CommitObject(head.Hash())
orFatal(err, "getting commit")
return obj
}

log.Println("Sync changes:")
orFatal(err, "status")
prefixw.New(log.Writer(), "> ").Write([]byte(status.String()))

// Commit
w.Status()
hash, err := w.Commit(msg, commitOpt)
orFatal(err, "committing")
log.Println("Created commit", hash.String())
obj, err := gr.CommitObject(hash)
orFatal(err, "getting commit")
return obj
}
11 changes: 2 additions & 9 deletions pkg/gitlogic/gitlogic_test.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package gitlogic

import (
"bytes"
"io"
"testing"

"github.com/go-git/go-billy/v5/memfs"
Expand All @@ -14,14 +12,9 @@ func TestRecursiveDelete(t *testing.T) {
fs := memfs.New()
nested, err := ChrootMkdir(fs, "level1/level2/level3")
assert.NoError(t, err)
assert.NotNil(t, nested)
f, err := nested.Create("dummy.txt")
err = writeFile(nested, "dummy.txt", "level3:foobar")
assert.NoError(t, err)
_, err = io.Copy(f, bytes.NewBufferString("level3:foobar"))
assert.NoError(t, err)
f2, err := fs.Create("level1/dummy.txt")
assert.NoError(t, err)
_, err = io.Copy(f2, bytes.NewBufferString("level1:foobar"))
err = writeFile(fs, "level1/dummy.txt", "level1:foobar")
assert.NoError(t, err)

// Test
Expand Down
78 changes: 78 additions & 0 deletions pkg/gitlogic/sync.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package gitlogic

import (
"log"

"github.com/go-git/go-billy/v5"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/koron-go/prefixw"
"github.com/pkg/errors"
)

func Sync(gr *git.Repository, outputPath string, inputFs billy.Filesystem, commitOpt *git.CommitOptions, msg string) *object.Commit {
// Do sync
w, err := gr.Worktree()
orFatal(err, "getting worktree")

outputFs := w.Filesystem
err = RmRecursively(outputFs, outputPath) // remove existing files
orFatal(err, "removing old artifacts from fs")
if outputPath != "." && outputPath != "" {
outputFs, err = ChrootMkdir(outputFs, outputPath)
orFatal(err, "failed to go to subdirectory")
}
err = Copy(inputFs, outputFs)
orFatal(err, "copy files")
err = addAllFiles(w)
orFatal(err, "git add -A")

// Print status
status, err := w.Status()
if len(status) == 0 {
log.Println("No changes. Skipping commit.")
head, err := gr.Head()
orFatal(err, "getting head")
obj, err := gr.CommitObject(head.Hash())
orFatal(err, "getting commit")
return obj
}

log.Println("Sync changes:")
orFatal(err, "status")
prefixw.New(log.Writer(), "> ").Write([]byte(status.String()))

// Commit
w.Status()
hash, err := w.Commit(msg, commitOpt)
orFatal(err, "committing")
log.Println("Created commit", hash.String())
obj, err := gr.CommitObject(hash)
orFatal(err, "getting commit")
return obj
}

func orFatal(err error, ctx string) {
if err != nil {
log.Fatal(errors.Wrap(err, ctx))
}
}

// addAllFiles adds all files. Somehow w.AddWithOptions traverses only over
// the filesystem, therefore not removing any non-existing files on the
// filesystem from the index.
func addAllFiles(w *git.Worktree) error {
files, err := w.Status()
if err != nil {
return err
}
for filename, s := range files {
if s.Staging == git.Unmodified && s.Worktree == git.Unmodified {
continue
}
if _, err = w.Add(filename); err != nil {
return err
}
}
return nil
}
118 changes: 118 additions & 0 deletions pkg/gitlogic/sync_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
package gitlogic

import (
"bytes"
"context"
"io"
"sort"
"strings"
"testing"

"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/storage/memory"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
)

func TestSync(t *testing.T) {
// Prepare git repo
fs := memfs.New()
storer := memory.NewStorage()
repo, err := git.Init(storer, fs)
assert.NoError(t, err)
w, err := repo.Worktree()
assert.NoError(t, err)

// Some files
fs.MkdirAll("bases/app1", 1444)
writeFile(fs, "bases/app1/template.yaml", "[]")
fs.MkdirAll("bases/app2", 1444)
writeFile(fs, "bases/app2/template.yaml", "[]")
writeFile(fs, "bases/app2/deprecated.md", "some markdown")

// Initial commit
err = addAllFiles(w)
assert.NoError(t, err)
hash, err := w.Commit("init", &git.CommitOptions{})
assert.NoError(t, err)
err = storer.SetReference(plumbing.NewHashReference("master", hash))
assert.NoError(t, err)

// Input fs
inputFs := memfs.New()
writeFile(inputFs, "template.yaml", "updated: true")

// Test
commit := Sync(repo, "bases/app2", inputFs, &git.CommitOptions{}, "sync")
assert.NotNil(t, commit)
changes, err := diff(repo, hash.String(), commit.Hash.String())
assert.NoError(t, err)
assert.NotNil(t, changes)

// Assert diff is correct
if assert.Len(t, changes, 2) {
sort.SliceIsSorted(changes, func(i, j int) bool {
nameI := firstDefined(changes[i].From.Name, changes[i].To.Name)
nameJ := firstDefined(changes[j].From.Name, changes[j].To.Name)
return strings.Compare(nameI, nameJ) < 0
})
assert.Equal(t, changes[0].From.Name, "bases/app2/deprecated.md")
assert.Nil(t, changes[0].To.Tree)
assert.Equal(t, changes[1].From.Name, "bases/app2/template.yaml")
}

// Assert which files there are in the commit
err = w.Checkout(&git.CheckoutOptions{Hash: commit.Hash, Keep: false, Force: true})
assert.NoError(t, err)
files, err := fs.ReadDir("bases/app2")
assert.NoError(t, err)
assert.Len(t, files, 1)
}

func writeFile(fs billy.Filesystem, file string, contents string) error {
f, err := fs.Create(file)
if err != nil {
return err
}
_, err = io.Copy(f, bytes.NewBufferString(contents))
defer f.Close()
return err
}

// diff determines the changes between two commits given their sha
func diff(repo *git.Repository, before, after string) (diffs object.Changes, err error) {
var commitA *object.Commit
var commitB *object.Commit
if commitA, err = repo.CommitObject(plumbing.NewHash(before)); err != nil {
return nil, errors.Wrapf(err, "Getting sha %q", before)
}
if commitB, err = repo.CommitObject(plumbing.NewHash(after)); err != nil {
return nil, errors.Wrapf(err, "Getting sha %q", after)
}
var treeA *object.Tree
var treeB *object.Tree
if treeA, err = commitA.Tree(); err != nil {
return nil, errors.Wrapf(err, "Getting tree of %q", before)
}
if treeB, err = commitB.Tree(); err != nil {
return nil, errors.Wrapf(err, "Getting tree of %q", after)
}
diffs, err = object.DiffTreeWithOptions(context.Background(), treeA, treeB, &object.DiffTreeOptions{DetectRenames: true})
if err != nil {
return nil, err
}
return
}

func firstDefined(strs ...string) string {
for _, s := range strs {
if s != "" {
return s
}
}
return ""
}

0 comments on commit 1185e69

Please sign in to comment.