Merge branch 'git-archive' into 'master'

"git archive" download support

This needs changes in GitLab and gitlab_git too.

See merge request !2

README.md

 # gitlab-git-http-server
 
 gitlab-git-http-server was designed to unload Git HTTP traffic from
-the GitLab Rails app (Unicorn) to a separate daemon. All authentication
-and authorization logic is still handled by the GitLab Rails app.
+the GitLab Rails app (Unicorn) to a separate daemon.  It also serves
+'git archive' downloads for GitLab.  All authentication and
+authorization logic is still handled by the GitLab Rails app.
 
 Architecture: Git client -> NGINX -> gitlab-git-http-server (makes
 auth request to GitLab Rails app) -> git-upload-pack
@@ -10,7 +11,7 @@ auth request to GitLab Rails app) -> git-upload-pack
 ## Usage
 
 ```
-  gitlab-git-http-server [OPTIONS] REPO_ROOT
+  gitlab-git-http-server [OPTIONS]
 
 Options:
   -authBackend string
@@ -27,11 +28,13 @@ Options:
     	Print version and exit
 ```
 
-gitlab-git-http-server allows Git HTTP clients to push and pull to and from Git
-repositories under REPO_ROOT. Each incoming request is first replayed (with an
-empty request body) to an external authentication/authorization HTTP server:
-the 'auth backend'. The auth backend is expected to be a GitLab Unicorn
-process.
+gitlab-git-http-server allows Git HTTP clients to push and pull to
+and from Git repositories. Each incoming request is first replayed
+(with an empty request body) to an external authentication/authorization
+HTTP server: the 'auth backend'. The auth backend is expected to
+be a GitLab Unicorn process.  The 'auth response' is a JSON message
+which tells gitlab-git-http-server the path of the Git repository
+to read from/write to.
 
 gitlab-git-http-server can listen on either a TCP or a Unix domain socket. It
 can also open a second listening TCP listening socket with the Go
@@ -63,14 +66,19 @@ You can try out the Git server without authentication as follows:
 
 ```
 # Start a fake auth backend that allows everything/everybody
-go run support/say-yes.go &
+make test/data/test.git
+go run support/fake-auth-backend.go ~+/test/data/test.git &
 # Start gitlab-git-http-server
-go build && ./gitlab-git-http-server /path/to/git-repos
+make
+./gitlab-git-http-server
 ```
 
-Now if you have a Git repository in `/path/to/git-repos/my-repo.git`,
-you can push to and pull from it at the URL
-`http://localhost:8181/my-repo.git`.
+Now you can try things like:
+
+```
+git clone http://localhost:8181/test.git
+curl -JO http://localhost:8181/test/repository/archive.zip
+```
 
 ## Example request flow
 

main.go

@@ -36,22 +36,18 @@ func main() {
 	pprofListenAddr := flag.String("pprofListenAddr", "", "pprof listening address, e.g. 'localhost:6060'")
 	flag.Usage = func() {
 		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
-		fmt.Fprintf(os.Stderr, "\n  %s [OPTIONS] REPO_ROOT\n\nOptions:\n", os.Args[0])
+		fmt.Fprintf(os.Stderr, "\n  %s [OPTIONS]\n\nOptions:\n", os.Args[0])
 		flag.PrintDefaults()
 	}
 	flag.Parse()
 
+	version := fmt.Sprintf("gitlab-git-http-server %s", Version)
 	if *printVersion {
-		fmt.Printf("gitlab-git-http-server %s\n", Version)
+		fmt.Println(version)
 		os.Exit(0)
 	}
 
-	repoRoot := flag.Arg(0)
-	if repoRoot == "" {
-		flag.Usage()
-		os.Exit(1)
-	}
-	log.Printf("repoRoot: %s", repoRoot)
+	log.Printf("Starting %s", version)
 
 	// Good housekeeping for Unix sockets: unlink before binding
 	if *listenNetwork == "unix" {
@@ -81,6 +77,6 @@ func main() {
 	// Because net/http/pprof installs itself in the DefaultServeMux
 	// we create a fresh one for the Git server.
 	serveMux := http.NewServeMux()
-	serveMux.Handle("/", newGitHandler(repoRoot, *authBackend))
+	serveMux.Handle("/", newGitHandler(*authBackend))
 	log.Fatal(http.Serve(listener, serveMux))
 }

main_test.go

 package main
 
 import (
+	"bytes"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -19,9 +21,11 @@ const servWaitSleep = 100    // milliseconds sleep interval
 const scratchDir = "test/scratch"
 const testRepoRoot = "test/data"
 const testRepo = "test.git"
+const testProject = "test"
 
 var remote = fmt.Sprintf("http://%s/%s", servAddr, testRepo)
 var checkoutDir = path.Join(scratchDir, "test")
+var cacheDir = path.Join(scratchDir, "cache")
 
 func TestAllowedClone(t *testing.T) {
 	// Prepare clone directory
@@ -30,7 +34,7 @@ func TestAllowedClone(t *testing.T) {
 	}
 
 	// Prepare test server and backend
-	ts := testAuthServer(200, `{"GL_ID":"user-123"}`)
+	ts := testAuthServer(200, gitOkBody(t))
 	defer ts.Close()
 	defer cleanUpProcessGroup(startServerOrFail(t, ts))
 
@@ -68,7 +72,7 @@ func TestAllowedPush(t *testing.T) {
 	preparePushRepo(t)
 
 	// Prepare the test server and backend
-	ts := testAuthServer(200, `{"GL_ID":"user-123"}`)
+	ts := testAuthServer(200, gitOkBody(t))
 	defer ts.Close()
 	defer cleanUpProcessGroup(startServerOrFail(t, ts))
 
@@ -96,6 +100,154 @@ func TestDeniedPush(t *testing.T) {
 	}
 }
 
+func TestAllowedDownloadZip(t *testing.T) {
+	prepareDownloadDir(t)
+
+	// Prepare test server and backend
+	archiveName := "foobar.zip"
+	ts := testAuthServer(200, archiveOkBody(t, archiveName))
+	defer ts.Close()
+	defer cleanUpProcessGroup(startServerOrFail(t, ts))
+
+	downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("http://%s/%s/repository/archive.zip", servAddr, testProject))
+	downloadCmd.Dir = scratchDir
+	runOrFail(t, downloadCmd)
+
+	extractCmd := exec.Command("unzip", archiveName)
+	extractCmd.Dir = scratchDir
+	runOrFail(t, extractCmd)
+}
+
+func TestAllowedDownloadTar(t *testing.T) {
+	prepareDownloadDir(t)
+
+	// Prepare test server and backend
+	archiveName := "foobar.tar"
+	ts := testAuthServer(200, archiveOkBody(t, archiveName))
+	defer ts.Close()
+	defer cleanUpProcessGroup(startServerOrFail(t, ts))
+
+	downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("http://%s/%s/repository/archive.tar", servAddr, testProject))
+	downloadCmd.Dir = scratchDir
+	runOrFail(t, downloadCmd)
+
+	extractCmd := exec.Command("tar", "xf", archiveName)
+	extractCmd.Dir = scratchDir
+	runOrFail(t, extractCmd)
+}
+
+func TestAllowedDownloadTarGz(t *testing.T) {
+	prepareDownloadDir(t)
+
+	// Prepare test server and backend
+	archiveName := "foobar.tar.gz"
+	ts := testAuthServer(200, archiveOkBody(t, archiveName))
+	defer ts.Close()
+	defer cleanUpProcessGroup(startServerOrFail(t, ts))
+
+	downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("http://%s/%s/repository/archive.tar.gz", servAddr, testProject))
+	downloadCmd.Dir = scratchDir
+	runOrFail(t, downloadCmd)
+
+	extractCmd := exec.Command("tar", "zxf", archiveName)
+	extractCmd.Dir = scratchDir
+	runOrFail(t, extractCmd)
+}
+
+func TestAllowedDownloadTarBz2(t *testing.T) {
+	prepareDownloadDir(t)
+
+	// Prepare test server and backend
+	archiveName := "foobar.tar.bz2"
+	ts := testAuthServer(200, archiveOkBody(t, archiveName))
+	defer ts.Close()
+	defer cleanUpProcessGroup(startServerOrFail(t, ts))
+
+	downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("http://%s/%s/repository/archive.tar.bz2", servAddr, testProject))
+	downloadCmd.Dir = scratchDir
+	runOrFail(t, downloadCmd)
+
+	extractCmd := exec.Command("tar", "jxf", archiveName)
+	extractCmd.Dir = scratchDir
+	runOrFail(t, extractCmd)
+}
+
+func TestAllowedApiDownloadZip(t *testing.T) {
+	prepareDownloadDir(t)
+
+	// Prepare test server and backend
+	archiveName := "foobar.zip"
+	ts := testAuthServer(200, archiveOkBody(t, archiveName))
+	defer ts.Close()
+	defer cleanUpProcessGroup(startServerOrFail(t, ts))
+
+	downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("http://%s/api/v3/projects/123/repository/archive.zip", servAddr))
+	downloadCmd.Dir = scratchDir
+	runOrFail(t, downloadCmd)
+
+	extractCmd := exec.Command("unzip", archiveName)
+	extractCmd.Dir = scratchDir
+	runOrFail(t, extractCmd)
+}
+
+func TestDownloadCacheHit(t *testing.T) {
+	prepareDownloadDir(t)
+
+	// Prepare test server and backend
+	archiveName := "foobar.zip"
+	ts := testAuthServer(200, archiveOkBody(t, archiveName))
+	defer ts.Close()
+	defer cleanUpProcessGroup(startServerOrFail(t, ts))
+
+	if err := os.MkdirAll(cacheDir, 0755); err != nil {
+		t.Fatal(err)
+	}
+	cachedContent := []byte{'c', 'a', 'c', 'h', 'e', 'd'}
+	if err := ioutil.WriteFile(path.Join(cacheDir, archiveName), cachedContent, 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("http://%s/api/v3/projects/123/repository/archive.zip", servAddr))
+	downloadCmd.Dir = scratchDir
+	runOrFail(t, downloadCmd)
+
+	actual, err := ioutil.ReadFile(path.Join(scratchDir, archiveName))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if bytes.Compare(actual, cachedContent) != 0 {
+		t.Fatal("Unexpected file contents in download")
+	}
+}
+
+func TestDownloadCacheCreate(t *testing.T) {
+	prepareDownloadDir(t)
+
+	// Prepare test server and backend
+	archiveName := "foobar.zip"
+	ts := testAuthServer(200, archiveOkBody(t, archiveName))
+	defer ts.Close()
+	defer cleanUpProcessGroup(startServerOrFail(t, ts))
+
+	downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("http://%s/api/v3/projects/123/repository/archive.zip", servAddr))
+	downloadCmd.Dir = scratchDir
+	runOrFail(t, downloadCmd)
+
+	compareCmd := exec.Command("cmp", path.Join(cacheDir, archiveName), path.Join(scratchDir, archiveName))
+	if err := compareCmd.Run(); err != nil {
+		t.Fatalf("Comparison between downloaded file and cache item failed: %s", err)
+	}
+}
+
+func prepareDownloadDir(t *testing.T) {
+	if err := os.RemoveAll(scratchDir); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.MkdirAll(scratchDir, 0755); err != nil {
+		t.Fatal(err)
+	}
+}
+
 func preparePushRepo(t *testing.T) {
 	if err := os.RemoveAll(scratchDir); err != nil {
 		t.Fatal(err)
@@ -117,7 +269,7 @@ func testAuthServer(code int, body string) *httptest.Server {
 }
 
 func startServerOrFail(t *testing.T, ts *httptest.Server) *exec.Cmd {
-	cmd := exec.Command("go", "run", "main.go", "githandler.go", fmt.Sprintf("-authBackend=%s", ts.URL), fmt.Sprintf("-listenAddr=%s", servAddr), testRepoRoot)
+	cmd := exec.Command("go", "run", "main.go", "githandler.go", fmt.Sprintf("-authBackend=%s", ts.URL), fmt.Sprintf("-listenAddr=%s", servAddr))
 	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
@@ -154,3 +306,30 @@ func runOrFail(t *testing.T, cmd *exec.Cmd) {
 		t.Fatal(err)
 	}
 }
+
+func gitOkBody(t *testing.T) string {
+	return fmt.Sprintf(`{"GL_ID":"user-123","RepoPath":"%s"}`, repoPath(t))
+}
+
+func archiveOkBody(t *testing.T, archiveName string) string {
+	cwd, err := os.Getwd()
+	if err != nil {
+		t.Fatal(err)
+	}
+	archivePath := path.Join(cwd, cacheDir, archiveName)
+	jsonString := `{
+		"RepoPath":"%s",
+		"ArchivePath":"%s",
+		"CommitId":"c7fbe50c7c7419d9701eebe64b1fdacc3df5b9dd",
+		"ArchivePrefix":"foobar123"
+	}`
+	return fmt.Sprintf(jsonString, repoPath(t), archivePath)
+}
+
+func repoPath(t *testing.T) string {
+	cwd, err := os.Getwd()
+	if err != nil {
+		t.Fatal(err)
+	}
+	return path.Join(cwd, testRepoRoot, testRepo)
+}

support/say-yes.go → support/fake-auth-backend.go

@@ -4,11 +4,17 @@ import (
 	"fmt"
 	"log"
 	"net/http"
+	"os"
 )
 
 func main() {
+	if len(os.Args) == 1 {
+		fmt.Fprintf(os.Stderr, "Usage: %s /path/to/test-repo.git\n", os.Args[0])
+		os.Exit(1)
+	}
+
 	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprint(w, `{"GL_ID":""}`)
+		fmt.Fprintf(w, `{"RepoPath":"%s","ArchivePath":"%s"}`, os.Args[1], r.URL.Path)
 	})
 
 	log.Fatal(http.ListenAndServe("localhost:8080", nil))