Skip to content

Commit 2fe03c0

Browse files
committed
Add support for indexers and alternate odb packfiles
This allows for implementations of git servers written in Go. (cherry picked from commit 05bc5e3)
1 parent 686500e commit 2fe03c0

File tree

5 files changed

+356
-1
lines changed

5 files changed

+356
-1
lines changed

indexer.go

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
package git
2+
3+
/*
4+
#include <git2.h>
5+
6+
extern const git_oid * git_indexer_hash(const git_indexer *idx);
7+
extern int git_indexer_append(git_indexer *idx, const void *data, size_t size, git_transfer_progress *stats);
8+
extern int git_indexer_commit(git_indexer *idx, git_transfer_progress *stats);
9+
extern int _go_git_indexer_new(git_indexer **out, const char *path, unsigned int mode, git_odb *odb, void *progress_cb_payload);
10+
extern void git_indexer_free(git_indexer *idx);
11+
*/
12+
import "C"
13+
import (
14+
"reflect"
15+
"runtime"
16+
"unsafe"
17+
)
18+
19+
// Indexer can post-process packfiles and create an .idx file for efficient
20+
// lookup.
21+
type Indexer struct {
22+
ptr *C.git_indexer
23+
stats C.git_transfer_progress
24+
callbacks RemoteCallbacks
25+
callbacksHandle unsafe.Pointer
26+
}
27+
28+
// NewIndexer creates a new indexer instance.
29+
func NewIndexer(packfilePath string, odb *Odb, callback TransferProgressCallback) (indexer *Indexer, err error) {
30+
indexer = new(Indexer)
31+
32+
runtime.LockOSThread()
33+
defer runtime.UnlockOSThread()
34+
35+
var odbPtr *C.git_odb = nil
36+
if odb != nil {
37+
odbPtr = odb.ptr
38+
}
39+
40+
indexer.callbacks.TransferProgressCallback = callback
41+
indexer.callbacksHandle = pointerHandles.Track(&indexer.callbacks)
42+
43+
cstr := C.CString(packfilePath)
44+
defer C.free(unsafe.Pointer(cstr))
45+
46+
ret := C._go_git_indexer_new(&indexer.ptr, cstr, 0, odbPtr, indexer.callbacksHandle)
47+
runtime.KeepAlive(odb)
48+
if ret < 0 {
49+
pointerHandles.Untrack(indexer.callbacksHandle)
50+
return nil, MakeGitError(ret)
51+
}
52+
53+
runtime.SetFinalizer(indexer, (*Indexer).Free)
54+
return indexer, nil
55+
}
56+
57+
// Write adds data to the indexer.
58+
func (indexer *Indexer) Write(data []byte) (int, error) {
59+
header := (*reflect.SliceHeader)(unsafe.Pointer(&data))
60+
ptr := unsafe.Pointer(header.Data)
61+
size := C.size_t(header.Len)
62+
63+
runtime.LockOSThread()
64+
defer runtime.UnlockOSThread()
65+
66+
ret := C.git_indexer_append(indexer.ptr, ptr, size, &indexer.stats)
67+
runtime.KeepAlive(indexer)
68+
if ret < 0 {
69+
return 0, MakeGitError(ret)
70+
}
71+
72+
return len(data), nil
73+
}
74+
75+
// Commit finalizes the pack and index. It resolves any pending deltas and
76+
// writes out the index file.
77+
//
78+
// It also returns the packfile's hash. A packfile's name is derived from the
79+
// sorted hashing of all object names.
80+
func (indexer *Indexer) Commit() (*Oid, error) {
81+
runtime.LockOSThread()
82+
defer runtime.UnlockOSThread()
83+
84+
ret := C.git_indexer_commit(indexer.ptr, &indexer.stats)
85+
if ret < 0 {
86+
return nil, MakeGitError(ret)
87+
}
88+
89+
id := newOidFromC(C.git_indexer_hash(indexer.ptr))
90+
runtime.KeepAlive(indexer)
91+
return id, nil
92+
}
93+
94+
// Free frees the indexer and its resources.
95+
func (indexer *Indexer) Free() {
96+
pointerHandles.Untrack(indexer.callbacksHandle)
97+
runtime.SetFinalizer(indexer, nil)
98+
C.git_indexer_free(indexer.ptr)
99+
}

indexer_test.go

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
package git
2+
3+
import (
4+
"fmt"
5+
"io/ioutil"
6+
"os"
7+
"path"
8+
"testing"
9+
)
10+
11+
var (
12+
// This is a packfile with three objects. The second is a delta which
13+
// depends on the third, which is also a delta.
14+
outOfOrderPack = []byte{
15+
0x50, 0x41, 0x43, 0x4b, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03,
16+
0x32, 0x78, 0x9c, 0x63, 0x67, 0x00, 0x00, 0x00, 0x10, 0x00, 0x08, 0x76,
17+
0xe6, 0x8f, 0xe8, 0x12, 0x9b, 0x54, 0x6b, 0x10, 0x1a, 0xee, 0x95, 0x10,
18+
0xc5, 0x32, 0x8e, 0x7f, 0x21, 0xca, 0x1d, 0x18, 0x78, 0x9c, 0x63, 0x62,
19+
0x66, 0x4e, 0xcb, 0xcf, 0x07, 0x00, 0x02, 0xac, 0x01, 0x4d, 0x75, 0x01,
20+
0xd7, 0x71, 0x36, 0x66, 0xf4, 0xde, 0x82, 0x27, 0x76, 0xc7, 0x62, 0x2c,
21+
0x10, 0xf1, 0xb0, 0x7d, 0xe2, 0x80, 0xdc, 0x78, 0x9c, 0x63, 0x62, 0x62,
22+
0x62, 0xb7, 0x03, 0x00, 0x00, 0x69, 0x00, 0x4c, 0xde, 0x7d, 0xaa, 0xe4,
23+
0x19, 0x87, 0x58, 0x80, 0x61, 0x09, 0x9a, 0x33, 0xca, 0x7a, 0x31, 0x92,
24+
0x6f, 0xae, 0x66, 0x75,
25+
}
26+
)
27+
28+
func TestIndexerOutOfOrder(t *testing.T) {
29+
t.Parallel()
30+
31+
tmpPath, err := ioutil.TempDir("", "git2go")
32+
checkFatal(t, err)
33+
defer os.RemoveAll(tmpPath)
34+
35+
var finalStats TransferProgress
36+
idx, err := NewIndexer(tmpPath, nil, func(stats TransferProgress) ErrorCode {
37+
finalStats = stats
38+
return ErrOk
39+
})
40+
checkFatal(t, err)
41+
defer idx.Free()
42+
43+
_, err = idx.Write(outOfOrderPack)
44+
checkFatal(t, err)
45+
oid, err := idx.Commit()
46+
checkFatal(t, err)
47+
48+
// The packfile contains the hash as the last 20 bytes.
49+
expectedOid := NewOidFromBytes(outOfOrderPack[len(outOfOrderPack)-20:])
50+
if !expectedOid.Equal(oid) {
51+
t.Errorf("mismatched packfile hash, expected %v, got %v", expectedOid, oid)
52+
}
53+
if finalStats.TotalObjects != 3 {
54+
t.Errorf("mismatched transferred objects, expected 3, got %v", finalStats.TotalObjects)
55+
}
56+
if finalStats.ReceivedObjects != 3 {
57+
t.Errorf("mismatched received objects, expected 3, got %v", finalStats.ReceivedObjects)
58+
}
59+
if finalStats.IndexedObjects != 3 {
60+
t.Errorf("mismatched indexed objects, expected 3, got %v", finalStats.IndexedObjects)
61+
}
62+
63+
odb, err := NewOdb()
64+
checkFatal(t, err)
65+
defer odb.Free()
66+
67+
backend, err := NewOdbBackendOnePack(path.Join(tmpPath, fmt.Sprintf("pack-%s.idx", oid.String())))
68+
checkFatal(t, err)
69+
// Transfer the ownership of the backend to the odb, no freeing needed.
70+
err = odb.AddBackend(backend, 1)
71+
checkFatal(t, err)
72+
73+
packfileObjects := 0
74+
err = odb.ForEach(func(id *Oid) error {
75+
packfileObjects += 1
76+
return nil
77+
})
78+
checkFatal(t, err)
79+
if packfileObjects != 3 {
80+
t.Errorf("mismatched packfile objects, expected 3, got %v", packfileObjects)
81+
}
82+
83+
// Inspect one of the well-known objects in the packfile.
84+
obj, err := odb.Read(NewOidFromBytes([]byte{
85+
0x19, 0x10, 0x28, 0x15, 0x66, 0x3d, 0x23, 0xf8, 0xb7, 0x5a, 0x47, 0xe7,
86+
0xa0, 0x19, 0x65, 0xdc, 0xdc, 0x96, 0x46, 0x8c,
87+
}))
88+
checkFatal(t, err)
89+
defer obj.Free()
90+
if "foo" != string(obj.Data()) {
91+
t.Errorf("mismatched packfile object contents, expected foo, got %q", string(obj.Data()))
92+
}
93+
}

odb.go

Lines changed: 102 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,13 @@ package git
33
/*
44
#include <git2.h>
55
6+
extern int git_odb_backend_one_pack(git_odb_backend **out, const char *index_file);
67
extern int _go_git_odb_foreach(git_odb *db, void *payload);
78
extern void _go_git_odb_backend_free(git_odb_backend *backend);
9+
extern int _go_git_odb_write_pack(git_odb_writepack **out, git_odb *db, void *progress_payload);
10+
extern int _go_git_odb_writepack_append(git_odb_writepack *writepack, const void *, size_t, git_transfer_progress *);
11+
extern int _go_git_odb_writepack_commit(git_odb_writepack *writepack, git_transfer_progress *);
12+
extern void _go_git_odb_writepack_free(git_odb_writepack *writepack);
813
*/
914
import "C"
1015
import (
@@ -42,8 +47,20 @@ func NewOdbBackendFromC(ptr unsafe.Pointer) (backend *OdbBackend) {
4247
return backend
4348
}
4449

45-
func (v *Odb) AddBackend(backend *OdbBackend, priority int) (err error) {
50+
func (v *Odb) AddAlternate(backend *OdbBackend, priority int) (err error) {
51+
runtime.LockOSThread()
52+
defer runtime.UnlockOSThread()
4653

54+
ret := C.git_odb_add_alternate(v.ptr, backend.ptr, C.int(priority))
55+
runtime.KeepAlive(v)
56+
if ret < 0 {
57+
backend.Free()
58+
return MakeGitError(ret)
59+
}
60+
return nil
61+
}
62+
63+
func (v *Odb) AddBackend(backend *OdbBackend, priority int) (err error) {
4764
runtime.LockOSThread()
4865
defer runtime.UnlockOSThread()
4966

@@ -56,6 +73,21 @@ func (v *Odb) AddBackend(backend *OdbBackend, priority int) (err error) {
5673
return nil
5774
}
5875

76+
func NewOdbBackendOnePack(packfileIndexPath string) (backend *OdbBackend, err error) {
77+
runtime.LockOSThread()
78+
defer runtime.UnlockOSThread()
79+
80+
cstr := C.CString(packfileIndexPath)
81+
defer C.free(unsafe.Pointer(cstr))
82+
83+
var odbOnePack *C.git_odb_backend = nil
84+
ret := C.git_odb_backend_one_pack(&odbOnePack, cstr)
85+
if ret < 0 {
86+
return nil, MakeGitError(ret)
87+
}
88+
return NewOdbBackendFromC(unsafe.Pointer(odbOnePack)), nil
89+
}
90+
5991
func (v *Odb) ReadHeader(oid *Oid) (uint64, ObjectType, error) {
6092
runtime.LockOSThread()
6193
defer runtime.UnlockOSThread()
@@ -231,6 +263,31 @@ func (v *Odb) NewWriteStream(size int64, otype ObjectType) (*OdbWriteStream, err
231263
return stream, nil
232264
}
233265

266+
// NewWritePack opens a stream for writing a pack file to the ODB. If the ODB
267+
// layer understands pack files, then the given packfile will likely be
268+
// streamed directly to disk (and a corresponding index created). If the ODB
269+
// layer does not understand pack files, the objects will be stored in whatever
270+
// format the ODB layer uses.
271+
func (v *Odb) NewWritePack(callback TransferProgressCallback) (*OdbWritepack, error) {
272+
writepack := new(OdbWritepack)
273+
274+
runtime.LockOSThread()
275+
defer runtime.UnlockOSThread()
276+
277+
writepack.callbacks.TransferProgressCallback = callback
278+
writepack.callbacksHandle = pointerHandles.Track(&writepack.callbacks)
279+
280+
ret := C._go_git_odb_write_pack(&writepack.ptr, v.ptr, writepack.callbacksHandle)
281+
runtime.KeepAlive(v)
282+
if ret < 0 {
283+
pointerHandles.Untrack(writepack.callbacksHandle)
284+
return nil, MakeGitError(ret)
285+
}
286+
287+
runtime.SetFinalizer(writepack, (*OdbWritepack).Free)
288+
return writepack, nil
289+
}
290+
234291
func (v *OdbBackend) Free() {
235292
C._go_git_odb_backend_free(v.ptr)
236293
}
@@ -360,3 +417,47 @@ func (stream *OdbWriteStream) Free() {
360417
runtime.SetFinalizer(stream, nil)
361418
C.git_odb_stream_free(stream.ptr)
362419
}
420+
421+
// OdbWritepack is a stream to write a packfile to the ODB.
422+
type OdbWritepack struct {
423+
ptr *C.git_odb_writepack
424+
stats C.git_transfer_progress
425+
callbacks RemoteCallbacks
426+
callbacksHandle unsafe.Pointer
427+
}
428+
429+
func (writepack *OdbWritepack) Write(data []byte) (int, error) {
430+
header := (*reflect.SliceHeader)(unsafe.Pointer(&data))
431+
ptr := unsafe.Pointer(header.Data)
432+
size := C.size_t(header.Len)
433+
434+
runtime.LockOSThread()
435+
defer runtime.UnlockOSThread()
436+
437+
ret := C._go_git_odb_writepack_append(writepack.ptr, ptr, size, &writepack.stats)
438+
runtime.KeepAlive(writepack)
439+
if ret < 0 {
440+
return 0, MakeGitError(ret)
441+
}
442+
443+
return len(data), nil
444+
}
445+
446+
func (writepack *OdbWritepack) Commit() error {
447+
runtime.LockOSThread()
448+
defer runtime.UnlockOSThread()
449+
450+
ret := C._go_git_odb_writepack_commit(writepack.ptr, &writepack.stats)
451+
runtime.KeepAlive(writepack)
452+
if ret < 0 {
453+
return MakeGitError(ret)
454+
}
455+
456+
return nil
457+
}
458+
459+
func (writepack *OdbWritepack) Free() {
460+
pointerHandles.Untrack(writepack.callbacksHandle)
461+
runtime.SetFinalizer(writepack, nil)
462+
C._go_git_odb_writepack_free(writepack.ptr)
463+
}

odb_test.go

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,3 +152,37 @@ func TestOdbForeach(t *testing.T) {
152152
t.Fatalf("Odb.ForEach() did not return the expected error, got %v", err)
153153
}
154154
}
155+
156+
func TestOdbWritepack(t *testing.T) {
157+
t.Parallel()
158+
repo := createTestRepo(t)
159+
defer cleanupTestRepo(t, repo)
160+
161+
_, _ = seedTestRepo(t, repo)
162+
163+
odb, err := repo.Odb()
164+
checkFatal(t, err)
165+
166+
var finalStats TransferProgress
167+
writepack, err := odb.NewWritePack(func(stats TransferProgress) ErrorCode {
168+
finalStats = stats
169+
return ErrOk
170+
})
171+
checkFatal(t, err)
172+
defer writepack.Free()
173+
174+
_, err = writepack.Write(outOfOrderPack)
175+
checkFatal(t, err)
176+
err = writepack.Commit()
177+
checkFatal(t, err)
178+
179+
if finalStats.TotalObjects != 3 {
180+
t.Errorf("mismatched transferred objects, expected 3, got %v", finalStats.TotalObjects)
181+
}
182+
if finalStats.ReceivedObjects != 3 {
183+
t.Errorf("mismatched received objects, expected 3, got %v", finalStats.ReceivedObjects)
184+
}
185+
if finalStats.IndexedObjects != 3 {
186+
t.Errorf("mismatched indexed objects, expected 3, got %v", finalStats.IndexedObjects)
187+
}
188+
}

0 commit comments

Comments
 (0)