Add support for managed HTTP/S transports (#810) (#812)

This change uses the newly-exposed Transport interface to use Go's
implementation of http.Client instead of httpclient via libgit2.

(cherry picked from commit b983e1d)

Author: lhchavez

credentials.go

@@ -8,6 +8,8 @@ void _go_git_populate_credential_ssh_custom(git_cred_ssh_custom *cred);
 import "C"
 import (
 	"crypto/rand"
+	"errors"
+	"runtime"
 	"unsafe"
 
 	"golang.org/x/crypto/ssh"
@@ -42,6 +44,29 @@ func credFromC(ptr *C.git_cred) *Cred {
 	return &Cred{ptr: ptr}
 }
 
+// GetUserpassPlaintext returns the plaintext username/password combination stored in the Cred.
+func (o *Cred) GetUserpassPlaintext() (username, password string, err error) {
+	if o.Type() != CredTypeUserpassPlaintext {
+		err = errors.New("credential is not userpass plaintext")
+		return
+	}
+
+	plaintextCredPtr := (*C.git_cred_userpass_plaintext)(unsafe.Pointer(o.ptr))
+	username = C.GoString(plaintextCredPtr.username)
+	password = C.GoString(plaintextCredPtr.password)
+	return
+}
+
+func NewCredUsername(username string) (int, Cred) {
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	cred := Cred{}
+	cusername := C.CString(username)
+	ret := C.git_cred_username_new(&cred.ptr, cusername)
+	return int(ret), cred
+}
+
 func NewCredUserpassPlaintext(username string, password string) (int, Cred) {
 	cred := Cred{}
 	cusername := C.CString(username)

git.go

@@ -139,22 +139,28 @@ func initLibGit2() {
 	remotePointers = newRemotePointerList()
 
 	C.git_libgit2_init()
+	features := Features()
 
 	// Due to the multithreaded nature of Go and its interaction with
 	// calling C functions, we cannot work with a library that was not built
 	// with multi-threading support. The most likely outcome is a segfault
 	// or panic at an incomprehensible time, so let's make it easy by
 	// panicking right here.
-	if Features()&FeatureThreads == 0 {
+	if features&FeatureThreads == 0 {
 		panic("libgit2 was not built with threading support")
 	}
 
-	// This is not something we should be doing, as we may be
-	// stomping all over someone else's setup. The user should do
-	// this themselves or use some binding/wrapper which does it
-	// in such a way that they can be sure they're the only ones
-	// setting it up.
-	C.git_openssl_set_locking()
+	if features&FeatureHTTPS == 0 {
+		if err := registerManagedHTTP(); err != nil {
+			panic(err)
+		}
+	} else {
+		// This is not something we should be doing, as we may be stomping all over
+		// someone else's setup. The user should do this themselves or use some
+		// binding/wrapper which does it in such a way that they can be sure
+		// they're the only ones setting it up.
+		C.git_openssl_set_locking()
+	}
 }
 
 // Shutdown frees all the resources acquired by libgit2. Make sure no

git_test.go

@@ -11,6 +11,10 @@ import (
 )
 
 func TestMain(m *testing.M) {
+	if err := registerManagedHTTP(); err != nil {
+		panic(err)
+	}
+
 	ret := m.Run()
 
 	if err := unregisterManagedTransports(); err != nil {

http.go

@@ -0,0 +1,240 @@
+package git
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"sync"
+)
+
+// RegisterManagedHTTPTransport registers a Go-native implementation of an
+// HTTP/S transport that doesn't rely on any system libraries (e.g.
+// libopenssl/libmbedtls).
+//
+// If Shutdown or ReInit are called, make sure that the smart transports are
+// freed before it.
+func RegisterManagedHTTPTransport(protocol string) (*RegisteredSmartTransport, error) {
+	return NewRegisteredSmartTransport(protocol, true, httpSmartSubtransportFactory)
+}
+
+func registerManagedHTTP() error {
+	globalRegisteredSmartTransports.Lock()
+	defer globalRegisteredSmartTransports.Unlock()
+
+	for _, protocol := range []string{"http", "https"} {
+		if _, ok := globalRegisteredSmartTransports.transports[protocol]; ok {
+			continue
+		}
+		managed, err := newRegisteredSmartTransport(protocol, true, httpSmartSubtransportFactory, true)
+		if err != nil {
+			return fmt.Errorf("failed to register transport for %q: %v", protocol, err)
+		}
+		globalRegisteredSmartTransports.transports[protocol] = managed
+	}
+	return nil
+}
+
+func httpSmartSubtransportFactory(remote *Remote, transport *Transport) (SmartSubtransport, error) {
+	var proxyFn func(*http.Request) (*url.URL, error)
+	proxyOpts, err := transport.SmartProxyOptions()
+	if err != nil {
+		return nil, err
+	}
+	switch proxyOpts.Type {
+	case ProxyTypeNone:
+		proxyFn = nil
+	case ProxyTypeAuto:
+		proxyFn = http.ProxyFromEnvironment
+	case ProxyTypeSpecified:
+		parsedUrl, err := url.Parse(proxyOpts.Url)
+		if err != nil {
+			return nil, err
+		}
+
+		proxyFn = http.ProxyURL(parsedUrl)
+	}
+
+	return &httpSmartSubtransport{
+		transport: transport,
+		client: &http.Client{
+			Transport: &http.Transport{
+				Proxy: proxyFn,
+			},
+		},
+	}, nil
+}
+
+type httpSmartSubtransport struct {
+	transport *Transport
+	client    *http.Client
+}
+
+func (t *httpSmartSubtransport) Action(url string, action SmartServiceAction) (SmartSubtransportStream, error) {
+	var req *http.Request
+	var err error
+	switch action {
+	case SmartServiceActionUploadpackLs:
+		req, err = http.NewRequest("GET", url+"/info/refs?service=git-upload-pack", nil)
+
+	case SmartServiceActionUploadpack:
+		req, err = http.NewRequest("POST", url+"/git-upload-pack", nil)
+		if err != nil {
+			break
+		}
+		req.Header.Set("Content-Type", "application/x-git-upload-pack-request")
+
+	case SmartServiceActionReceivepackLs:
+		req, err = http.NewRequest("GET", url+"/info/refs?service=git-receive-pack", nil)
+
+	case SmartServiceActionReceivepack:
+		req, err = http.NewRequest("POST", url+"/info/refs?service=git-upload-pack", nil)
+		if err != nil {
+			break
+		}
+		req.Header.Set("Content-Type", "application/x-git-receive-pack-request")
+
+	default:
+		err = errors.New("unknown action")
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("User-Agent", "git/2.0 (git2go)")
+
+	stream := newManagedHttpStream(t, req)
+	if req.Method == "POST" {
+		stream.recvReply.Add(1)
+		stream.sendRequestBackground()
+	}
+
+	return stream, nil
+}
+
+func (t *httpSmartSubtransport) Close() error {
+	return nil
+}
+
+func (t *httpSmartSubtransport) Free() {
+	t.client = nil
+}
+
+type httpSmartSubtransportStream struct {
+	owner       *httpSmartSubtransport
+	req         *http.Request
+	resp        *http.Response
+	reader      *io.PipeReader
+	writer      *io.PipeWriter
+	sentRequest bool
+	recvReply   sync.WaitGroup
+	httpError   error
+}
+
+func newManagedHttpStream(owner *httpSmartSubtransport, req *http.Request) *httpSmartSubtransportStream {
+	r, w := io.Pipe()
+	return &httpSmartSubtransportStream{
+		owner:  owner,
+		req:    req,
+		reader: r,
+		writer: w,
+	}
+}
+
+func (self *httpSmartSubtransportStream) Read(buf []byte) (int, error) {
+	if !self.sentRequest {
+		self.recvReply.Add(1)
+		if err := self.sendRequest(); err != nil {
+			return 0, err
+		}
+	}
+
+	if err := self.writer.Close(); err != nil {
+		return 0, err
+	}
+
+	self.recvReply.Wait()
+
+	if self.httpError != nil {
+		return 0, self.httpError
+	}
+
+	return self.resp.Body.Read(buf)
+}
+
+func (self *httpSmartSubtransportStream) Write(buf []byte) (int, error) {
+	if self.httpError != nil {
+		return 0, self.httpError
+	}
+	return self.writer.Write(buf)
+}
+
+func (self *httpSmartSubtransportStream) Free() {
+	if self.resp != nil {
+		self.resp.Body.Close()
+	}
+}
+
+func (self *httpSmartSubtransportStream) sendRequestBackground() {
+	go func() {
+		self.httpError = self.sendRequest()
+	}()
+	self.sentRequest = true
+}
+
+func (self *httpSmartSubtransportStream) sendRequest() error {
+	defer self.recvReply.Done()
+	self.resp = nil
+
+	var resp *http.Response
+	var err error
+	var userName string
+	var password string
+	for {
+		req := &http.Request{
+			Method: self.req.Method,
+			URL:    self.req.URL,
+			Header: self.req.Header,
+		}
+		if req.Method == "POST" {
+			req.Body = self.reader
+			req.ContentLength = -1
+		}
+
+		req.SetBasicAuth(userName, password)
+		resp, err = http.DefaultClient.Do(req)
+		if err != nil {
+			return err
+		}
+
+		if resp.StatusCode == http.StatusOK {
+			break
+		}
+
+		if resp.StatusCode == http.StatusUnauthorized {
+			resp.Body.Close()
+
+			cred, err := self.owner.transport.SmartCredentials("", CredTypeUserpassPlaintext)
+			if err != nil {
+				return err
+			}
+
+			userName, password, err = cred.GetUserpassPlaintext()
+			if err != nil {
+				return err
+			}
+
+			continue
+		}
+
+		// Any other error we treat as a hard error and punt back to the caller
+		resp.Body.Close()
+		return fmt.Errorf("Unhandled HTTP error %s", resp.Status)
+	}
+
+	self.sentRequest = true
+	self.resp = resp
+	return nil
+}

remote.go

@@ -167,6 +167,13 @@ type ProxyOptions struct {
 	Url string
 }
 
+func proxyOptionsFromC(copts *C.git_proxy_options) *ProxyOptions {
+	return &ProxyOptions{
+		Type: ProxyType(copts._type),
+		Url:  C.GoString(copts.url),
+	}
+}
+
 type Remote struct {
 	doNotCompare
 	ptr       *C.git_remote

remote_test.go

@@ -232,6 +232,30 @@ func TestRemotePrune(t *testing.T) {
 	}
 }
 
+func TestRemoteCredentialsCalled(t *testing.T) {
+	t.Parallel()
+
+	repo := createTestRepo(t)
+	defer cleanupTestRepo(t, repo)
+
+	remote, err := repo.Remotes.CreateAnonymous("https://github.com/libgit2/non-existent")
+	checkFatal(t, err)
+	defer remote.Free()
+
+	fetchOpts := FetchOptions{
+		RemoteCallbacks: RemoteCallbacks{
+			CredentialsCallback: func(url, username string, allowedTypes CredType) (ErrorCode, *Cred) {
+				return ErrorCodeUser, nil
+			},
+		},
+	}
+
+	err = remote.Fetch(nil, &fetchOpts, "fetch")
+	if IsErrorCode(err, ErrorCodeUser) {
+		t.Fatalf("remote.Fetch() = %v, want %v", err, ErrorCodeUser)
+	}
+}
+
 func newChannelPipe(t *testing.T, w io.Writer, wg *sync.WaitGroup) (*os.File, error) {
 	pr, pw, err := os.Pipe()
 	if err != nil {

script/build-libgit2.sh

@@ -58,6 +58,7 @@ cd "${BUILD_PATH}/build" &&
 cmake -DTHREADSAFE=ON \
       -DBUILD_CLAR=OFF \
       -DBUILD_SHARED_LIBS"=${BUILD_SHARED_LIBS}" \
+      -DUSE_HTTPS=OFF \
       -DCMAKE_C_FLAGS=-fPIC \
       -DCMAKE_BUILD_TYPE="RelWithDebInfo" \
       -DCMAKE_INSTALL_PREFIX="${BUILD_INSTALL_PREFIX}" \