Commit e6c7d974 authored by Marin Jankovski's avatar Marin Jankovski

Handle lfs upload with middleware and callback.

parent ec721eef
...@@ -5,6 +5,7 @@ In this file we handle 'git archive' downloads ...@@ -5,6 +5,7 @@ In this file we handle 'git archive' downloads
package main package main
import ( import (
"errors"
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
...@@ -13,9 +14,8 @@ import ( ...@@ -13,9 +14,8 @@ import (
"os" "os"
"os/exec" "os/exec"
"path" "path"
"time"
"path/filepath" "path/filepath"
"errors" "time"
) )
func handleGetArchive(w http.ResponseWriter, r *gitRequest) { func handleGetArchive(w http.ResponseWriter, r *gitRequest) {
......
...@@ -9,8 +9,8 @@ import ( ...@@ -9,8 +9,8 @@ import (
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
"strings"
"path/filepath" "path/filepath"
"strings"
) )
func handleGetInfoRefs(w http.ResponseWriter, r *gitRequest) { func handleGetInfoRefs(w http.ResponseWriter, r *gitRequest) {
......
...@@ -5,11 +5,11 @@ In this file we handle git lfs objects downloads and uploads ...@@ -5,11 +5,11 @@ In this file we handle git lfs objects downloads and uploads
package main package main
import ( import (
"compress/gzip"
"crypto/sha256" "crypto/sha256"
"encoding/hex" "encoding/hex"
"errors" "errors"
"io" "io"
"io/ioutil"
"log" "log"
"net/http" "net/http"
"os" "os"
...@@ -23,100 +23,102 @@ var ( ...@@ -23,100 +23,102 @@ var (
errSizeMismatch = errors.New("Content size does not match") errSizeMismatch = errors.New("Content size does not match")
) )
func handleStoreLfsObject(w http.ResponseWriter, r *gitRequest, rpc string) { func lfsAuthorizeHandler(handleFunc serviceHandleFunc) serviceHandleFunc {
var body io.ReadCloser return preAuthorizeHandler(func(w http.ResponseWriter, r *gitRequest) {
urlPath := r.URL.Path if r.StoreLFSPath == "" {
regExp := regexp.MustCompile(`([0-9a-f]{64})/([0-9]+)`) fail500(w, "lfsAuthorizeHandler", errors.New("Don't know where to store object, no store path specified."))
matches := regExp.FindStringSubmatch(urlPath) return
}
if matches == nil { handleFunc(w, r)
log.Printf("Found no object info in path: %s", urlPath) }, "")
return }
}
oid := matches[1] func handleStoreLfsObject(handleFunc serviceHandleFunc) serviceHandleFunc {
size := matches[2] return func(w http.ResponseWriter, r *gitRequest) {
log.Printf("Found oid: %s and size: %s", oid, size)
urlPath := r.URL.Path
storePath := filepath.Join(r.StoreLFSPath, transformKey(oid)) regExp := regexp.MustCompile(`([0-9a-f]{64})/([0-9]+)`)
matches := regExp.FindStringSubmatch(urlPath)
if _, err := os.Stat(storePath); os.IsNotExist(err) {
tmpPath := filepath.Join(r.StoreLFSPath, "tmp", oid) if matches == nil {
log.Printf("Found no object info in path: %s", urlPath)
if _, err := os.Stat(tmpPath); os.IsNotExist(err) { return
// TODO try removing gzip, possibly not needed
// The client request body may have been gzipped.
if r.Header.Get("Content-Encoding") == "gzip" {
body, err = gzip.NewReader(r.Body)
if err != nil {
fail500(w, "Couldn't handle LFS upload request.", err)
return
}
} else {
body = r.Body
}
defer body.Close()
// TODO maybe set dir permissions to 700
dir := filepath.Dir(tmpPath)
if err := os.MkdirAll(dir, 0750); err != nil {
fail500(w, "Couldn't create directory for storing LFS objects.", err)
return
}
// TODO use go library for creating TMP files
file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0640)
if err != nil {
fail500(w, "Couldn't open tmp file for writing.", err)
return
}
// defer os.Remove(tmpPath)
defer file.Close()
hash := sha256.New()
hw := io.MultiWriter(hash, file)
written, err := io.Copy(hw, body)
if err != nil {
fail500(w, "Failed to save received LFS object.", err)
return
}
file.Close()
sizeInt, err := strconv.ParseInt(size, 10, 64)
if err != nil {
fail500(w, "Couldn't read size: ", err)
return
}
if written != sizeInt {
fail500(w, "Inconsistent size: ", errSizeMismatch)
return
}
shaStr := hex.EncodeToString(hash.Sum(nil))
if shaStr != oid {
fail500(w, "Inconsistent size: ", errSizeMismatch)
return
}
} }
}
// if err := os.Rename(tmpPath, path); err != nil {
// fail500(w, "Failed to rename temporary LFS object.", err)
// return
// }
log.Printf("Received the LFS object from client, oid: %s", oid) oid := matches[1]
size := matches[2]
log.Printf("Found oid: %s and size: %s", oid, size)
return sha := sha256.New()
sha.Write([]byte(oid))
tmp_hash := hex.EncodeToString(sha.Sum(nil))
tmpPath := filepath.Join(r.StoreLFSPath, "tmp")
var body io.ReadCloser
body = r.Body
defer body.Close()
dir := filepath.Dir(tmpPath)
if err := os.MkdirAll(dir, 0700); err != nil {
fail500(w, "Couldn't create directory for storing LFS objects.", err)
return
}
file, err := ioutil.TempFile(tmpPath, tmp_hash)
if err != nil {
fail500(w, "Couldn't open tmp file for writing.", err)
return
}
defer os.Remove(tmpPath)
defer file.Close()
hash := sha256.New()
hw := io.MultiWriter(hash, file)
written, err := io.Copy(hw, body)
if err != nil {
fail500(w, "Failed to save received LFS object.", err)
return
}
file.Close()
sizeInt, err := strconv.ParseInt(size, 10, 64)
if err != nil {
fail500(w, "Couldn't read size: ", err)
return
}
if written != sizeInt {
fail500(w, "Inconsistent size: ", errSizeMismatch)
return
}
shaStr := hex.EncodeToString(hash.Sum(nil))
if shaStr != oid {
fail500(w, "Inconsistent size: ", errSizeMismatch)
return
}
r.Header.Set("X-GitLab-Lfs-Tmp", filepath.Base(file.Name()))
handleFunc(w, r)
}
} }
func transformKey(key string) string { func lfsCallback(w http.ResponseWriter, r *gitRequest) {
if len(key) < 5 { authReq, err := r.u.newUpstreamRequest(r.Request, nil, "/authorize")
return key if err != nil {
fail500(w, "newUpstreamRequestlfsCallback", err)
return
} }
return filepath.Join(key[0:2], key[2:4], key[4:len(key)]) authResponse, err := r.u.httpClient.Do(authReq)
if err != nil {
fail500(w, "doRequestlfsCallback", err)
return
}
defer authResponse.Body.Close()
return
} }
...@@ -45,7 +45,8 @@ type authorizationResponse struct { ...@@ -45,7 +45,8 @@ type authorizationResponse struct {
// in the GitLab Rails app and the 'time of use' in gitlab-workhorse. // in the GitLab Rails app and the 'time of use' in gitlab-workhorse.
CommitId string CommitId string
// TODO: say something about this // StoreLFSPath is provided by the GitLab Rails application
// to mark where the tmp file should be placed
StoreLFSPath string StoreLFSPath string
} }
...@@ -54,7 +55,7 @@ type authorizationResponse struct { ...@@ -54,7 +55,7 @@ type authorizationResponse struct {
type gitRequest struct { type gitRequest struct {
*http.Request *http.Request
authorizationResponse authorizationResponse
u *upstream u *upstream
} }
// Routing table // Routing table
...@@ -68,7 +69,7 @@ var gitServices = [...]gitService{ ...@@ -68,7 +69,7 @@ var gitServices = [...]gitService{
gitService{"GET", regexp.MustCompile(`/repository/archive.tar.gz\z`), repoPreAuthorizeHandler(handleGetArchive)}, gitService{"GET", regexp.MustCompile(`/repository/archive.tar.gz\z`), repoPreAuthorizeHandler(handleGetArchive)},
gitService{"GET", regexp.MustCompile(`/repository/archive.tar.bz2\z`), repoPreAuthorizeHandler(handleGetArchive)}, gitService{"GET", regexp.MustCompile(`/repository/archive.tar.bz2\z`), repoPreAuthorizeHandler(handleGetArchive)},
gitService{"GET", regexp.MustCompile(`/uploads/`), handleSendFile}, gitService{"GET", regexp.MustCompile(`/uploads/`), handleSendFile},
gitService{"PUT", regexp.MustCompile(`/gitlab-lfs/objects/([0-9a-f]{64})/([0-9]+)\z`), repoPreAuthorizeHandler(handleStoreLfsObject)}, gitService{"PUT", regexp.MustCompile(`/gitlab-lfs/objects/([0-9a-f]{64})/([0-9]+)\z`), lfsAuthorizeHandler(handleStoreLfsObject(lfsCallback))},
gitService{"GET", regexp.MustCompile(`/gitlab-lfs/objects/([0-9a-f]{64})\z`), handleSendFile}, gitService{"GET", regexp.MustCompile(`/gitlab-lfs/objects/([0-9a-f]{64})\z`), handleSendFile},
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment