Merge branch 'id-git-lfs-authenticate' into 'master'

Go implementation for LFS authenticate

Closes #163

See merge request gitlab-org/gitlab-shell!308

go/internal/command/command.go

@@ -4,6 +4,7 @@ import (
 	"gitlab.com/gitlab-org/gitlab-shell/go/internal/command/commandargs"
 	"gitlab.com/gitlab-org/gitlab-shell/go/internal/command/discover"
 	"gitlab.com/gitlab-org/gitlab-shell/go/internal/command/fallback"
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/command/lfsauthenticate"
 	"gitlab.com/gitlab-org/gitlab-shell/go/internal/command/readwriter"
 	"gitlab.com/gitlab-org/gitlab-shell/go/internal/command/receivepack"
 	"gitlab.com/gitlab-org/gitlab-shell/go/internal/command/twofactorrecover"
@@ -38,6 +39,8 @@ func buildCommand(args *commandargs.CommandArgs, config *config.Config, readWrit
 		return &discover.Command{Config: config, Args: args, ReadWriter: readWriter}
 	case commandargs.TwoFactorRecover:
 		return &twofactorrecover.Command{Config: config, Args: args, ReadWriter: readWriter}
+	case commandargs.LfsAuthenticate:
+		return &lfsauthenticate.Command{Config: config, Args: args, ReadWriter: readWriter}
 	case commandargs.ReceivePack:
 		return &receivepack.Command{Config: config, Args: args, ReadWriter: readWriter}
 	case commandargs.UploadPack:

go/internal/command/command_test.go

@@ -7,6 +7,7 @@ import (
 
 	"gitlab.com/gitlab-org/gitlab-shell/go/internal/command/discover"
 	"gitlab.com/gitlab-org/gitlab-shell/go/internal/command/fallback"
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/command/lfsauthenticate"
 	"gitlab.com/gitlab-org/gitlab-shell/go/internal/command/receivepack"
 	"gitlab.com/gitlab-org/gitlab-shell/go/internal/command/twofactorrecover"
 	"gitlab.com/gitlab-org/gitlab-shell/go/internal/command/uploadarchive"
@@ -58,6 +59,18 @@ func TestNew(t *testing.T) {
 			},
 			expectedType: &twofactorrecover.Command{},
 		},
+		{
+			desc: "it returns an LfsAuthenticate command if the feature is enabled",
+			config: &config.Config{
+				GitlabUrl: "http+unix://gitlab.socket",
+				Migration: config.MigrationConfig{Enabled: true, Features: []string{"git-lfs-authenticate"}},
+			},
+			environment: map[string]string{
+				"SSH_CONNECTION":       "1",
+				"SSH_ORIGINAL_COMMAND": "git-lfs-authenticate",
+			},
+			expectedType: &lfsauthenticate.Command{},
+		},
 		{
 			desc: "it returns a ReceivePack command if the feature is enabled",
 			config: &config.Config{

go/internal/command/commandargs/command_args.go

@@ -13,6 +13,7 @@ type CommandType string
 const (
 	Discover         CommandType = "discover"
 	TwoFactorRecover CommandType = "2fa_recovery_codes"
+	LfsAuthenticate  CommandType = "git-lfs-authenticate"
 	ReceivePack      CommandType = "git-receive-pack"
 	UploadPack       CommandType = "git-upload-pack"
 	UploadArchive    CommandType = "git-upload-archive"

go/internal/command/commandargs/command_args_test.go

@@ -90,6 +90,13 @@ func TestParseSuccess(t *testing.T) {
 				"SSH_ORIGINAL_COMMAND": "git-upload-archive 'group/repo'",
 			},
 			expectedArgs: &CommandArgs{SshArgs: []string{"git-upload-archive", "group/repo"}, CommandType: UploadArchive},
+		}, {
+			desc: "It parses git-lfs-authenticate command",
+			environment: map[string]string{
+				"SSH_CONNECTION":       "1",
+				"SSH_ORIGINAL_COMMAND": "git-lfs-authenticate 'group/repo' download",
+			},
+			expectedArgs: &CommandArgs{SshArgs: []string{"git-lfs-authenticate", "group/repo", "download"}, CommandType: LfsAuthenticate},
 		},
 	}
 

go/internal/command/lfsauthenticate/lfsauthenticate.go

+package lfsauthenticate
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/command/commandargs"
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/command/readwriter"
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/command/shared/accessverifier"
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/command/shared/disallowedcommand"
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/config"
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/gitlabnet/lfsauthenticate"
+)
+
+const (
+	downloadAction = "download"
+	uploadAction   = "upload"
+)
+
+type Command struct {
+	Config     *config.Config
+	Args       *commandargs.CommandArgs
+	ReadWriter *readwriter.ReadWriter
+}
+
+type PayloadHeader struct {
+	Auth string `json:"Authorization"`
+}
+
+type Payload struct {
+	Header    PayloadHeader `json:"header"`
+	Href      string        `json:"href"`
+	ExpiresIn int           `json:"expires_in,omitempty"`
+}
+
+func (c *Command) Execute() error {
+	args := c.Args.SshArgs
+	if len(args) < 3 {
+		return disallowedcommand.Error
+	}
+
+	repo := args[1]
+	action, err := actionToCommandType(args[2])
+	if err != nil {
+		return err
+	}
+
+	accessResponse, err := c.verifyAccess(action, repo)
+	if err != nil {
+		return err
+	}
+
+	payload, err := c.authenticate(action, repo, accessResponse.UserId)
+	if err != nil {
+		// return nothing just like Ruby's GitlabShell#lfs_authenticate does
+		return nil
+	}
+
+	fmt.Fprintf(c.ReadWriter.Out, "%s\n", payload)
+
+	return nil
+}
+
+func actionToCommandType(action string) (commandargs.CommandType, error) {
+	var accessAction commandargs.CommandType
+	switch action {
+	case downloadAction:
+		accessAction = commandargs.UploadPack
+	case uploadAction:
+		accessAction = commandargs.ReceivePack
+	default:
+		return "", disallowedcommand.Error
+	}
+
+	return accessAction, nil
+}
+
+func (c *Command) verifyAccess(action commandargs.CommandType, repo string) (*accessverifier.Response, error) {
+	cmd := accessverifier.Command{c.Config, c.Args, c.ReadWriter}
+
+	return cmd.Verify(action, repo)
+}
+
+func (c *Command) authenticate(action commandargs.CommandType, repo, userId string) ([]byte, error) {
+	client, err := lfsauthenticate.NewClient(c.Config, c.Args)
+	if err != nil {
+		return nil, err
+	}
+
+	response, err := client.Authenticate(action, repo, userId)
+	if err != nil {
+		return nil, err
+	}
+
+	basicAuth := base64.StdEncoding.EncodeToString([]byte(response.Username + ":" + response.LfsToken))
+	payload := &Payload{
+		Header:    PayloadHeader{Auth: "Basic " + basicAuth},
+		Href:      response.RepoPath + "/info/lfs",
+		ExpiresIn: response.ExpiresIn,
+	}
+
+	return json.Marshal(payload)
+}

go/internal/command/lfsauthenticate/lfsauthenticate_test.go

+package lfsauthenticate
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/command/commandargs"
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/command/readwriter"
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/config"
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/gitlabnet/accessverifier"
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/gitlabnet/lfsauthenticate"
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/gitlabnet/testserver"
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/testhelper/requesthandlers"
+)
+
+func TestFailedRequests(t *testing.T) {
+	requests := requesthandlers.BuildDisallowedByApiHandlers(t)
+	url, cleanup := testserver.StartHttpServer(t, requests)
+	defer cleanup()
+
+	testCases := []struct {
+		desc           string
+		arguments      *commandargs.CommandArgs
+		expectedOutput string
+	}{
+		{
+			desc:           "With missing arguments",
+			arguments:      &commandargs.CommandArgs{},
+			expectedOutput: "> GitLab: Disallowed command",
+		},
+		{
+			desc:           "With disallowed command",
+			arguments:      &commandargs.CommandArgs{GitlabKeyId: "1", SshArgs: []string{"git-lfs-authenticate", "group/repo", "unknown"}},
+			expectedOutput: "> GitLab: Disallowed command",
+		},
+		{
+			desc:           "With disallowed user",
+			arguments:      &commandargs.CommandArgs{GitlabKeyId: "disallowed", SshArgs: []string{"git-lfs-authenticate", "group/repo", "download"}},
+			expectedOutput: "Disallowed by API call",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			output := &bytes.Buffer{}
+			cmd := &Command{
+				Config:     &config.Config{GitlabUrl: url},
+				Args:       tc.arguments,
+				ReadWriter: &readwriter.ReadWriter{ErrOut: output, Out: output},
+			}
+
+			err := cmd.Execute()
+			require.Error(t, err)
+
+			require.Equal(t, tc.expectedOutput, err.Error())
+		})
+	}
+}
+
+func TestLfsAuthenticateRequests(t *testing.T) {
+	userId := "123"
+
+	requests := []testserver.TestRequestHandler{
+		{
+			Path: "/api/v4/internal/lfs_authenticate",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				b, err := ioutil.ReadAll(r.Body)
+				defer r.Body.Close()
+				require.NoError(t, err)
+
+				var request *lfsauthenticate.Request
+				require.NoError(t, json.Unmarshal(b, &request))
+
+				if request.UserId == userId {
+					body := map[string]interface{}{
+						"username":             "john",
+						"lfs_token":            "sometoken",
+						"repository_http_path": "https://gitlab.com/repo/path",
+						"expires_in":           1800,
+					}
+					require.NoError(t, json.NewEncoder(w).Encode(body))
+				} else {
+					w.WriteHeader(http.StatusForbidden)
+				}
+			},
+		},
+		{
+			Path: "/api/v4/internal/allowed",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				b, err := ioutil.ReadAll(r.Body)
+				defer r.Body.Close()
+				require.NoError(t, err)
+
+				var request *accessverifier.Request
+				require.NoError(t, json.Unmarshal(b, &request))
+
+				var glId string
+				if request.Username == "somename" {
+					glId = userId
+				} else {
+					glId = "100"
+				}
+
+				body := map[string]interface{}{
+					"gl_id":  glId,
+					"status": true,
+				}
+				require.NoError(t, json.NewEncoder(w).Encode(body))
+			},
+		},
+	}
+
+	url, cleanup := testserver.StartHttpServer(t, requests)
+	defer cleanup()
+
+	testCases := []struct {
+		desc           string
+		username       string
+		expectedOutput string
+	}{
+		{
+			desc:           "With successful response from API",
+			username:       "somename",
+			expectedOutput: "{\"header\":{\"Authorization\":\"Basic am9objpzb21ldG9rZW4=\"},\"href\":\"https://gitlab.com/repo/path/info/lfs\",\"expires_in\":1800}\n",
+		},
+		{
+			desc:           "With forbidden response from API",
+			username:       "anothername",
+			expectedOutput: "",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			output := &bytes.Buffer{}
+			cmd := &Command{
+				Config:     &config.Config{GitlabUrl: url},
+				Args:       &commandargs.CommandArgs{GitlabUsername: tc.username, SshArgs: []string{"git-lfs-authenticate", "group/repo", "upload"}},
+				ReadWriter: &readwriter.ReadWriter{ErrOut: output, Out: output},
+			}
+
+			err := cmd.Execute()
+			require.NoError(t, err)
+
+			require.Equal(t, tc.expectedOutput, output.String())
+		})
+	}
+}

go/internal/gitlabnet/lfsauthenticate/client.go

+package lfsauthenticate
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/command/commandargs"
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/config"
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/gitlabnet"
+)
+
+type Client struct {
+	config *config.Config
+	client *gitlabnet.GitlabClient
+	args   *commandargs.CommandArgs
+}
+
+type Request struct {
+	Action commandargs.CommandType `json:"operation"`
+	Repo   string                  `json:"project"`
+	KeyId  string                  `json:"key_id,omitempty"`
+	UserId string                  `json:"user_id,omitempty"`
+}
+
+type Response struct {
+	Username  string `json:"username"`
+	LfsToken  string `json:"lfs_token"`
+	RepoPath  string `json:"repository_http_path"`
+	ExpiresIn int    `json:"expires_in"`
+}
+
+func NewClient(config *config.Config, args *commandargs.CommandArgs) (*Client, error) {
+	client, err := gitlabnet.GetClient(config)
+	if err != nil {
+		return nil, fmt.Errorf("Error creating http client: %v", err)
+	}
+
+	return &Client{config: config, client: client, args: args}, nil
+}
+
+func (c *Client) Authenticate(action commandargs.CommandType, repo, userId string) (*Response, error) {
+	request := &Request{Action: action, Repo: repo}
+	if c.args.GitlabKeyId != "" {
+		request.KeyId = c.args.GitlabKeyId
+	} else {
+		request.UserId = strings.TrimPrefix(userId, "user-")
+	}
+
+	response, err := c.client.Post("/lfs_authenticate", request)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+
+	return parse(response)
+}
+
+func parse(hr *http.Response) (*Response, error) {
+	response := &Response{}
+	if err := gitlabnet.ParseJSON(hr, response); err != nil {
+		return nil, err
+	}
+
+	return response, nil
+}

go/internal/gitlabnet/lfsauthenticate/client_test.go

+package lfsauthenticate
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/command/commandargs"
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/config"
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/gitlabnet/testserver"
+)
+
+const (
+	keyId  = "123"
+	repo   = "group/repo"
+	action = commandargs.UploadPack
+)
+
+func setup(t *testing.T) []testserver.TestRequestHandler {
+	requests := []testserver.TestRequestHandler{
+		{
+			Path: "/api/v4/internal/lfs_authenticate",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				b, err := ioutil.ReadAll(r.Body)
+				defer r.Body.Close()
+				require.NoError(t, err)
+
+				var request *Request
+				require.NoError(t, json.Unmarshal(b, &request))
+
+				switch request.KeyId {
+				case keyId:
+					body := map[string]interface{}{
+						"username":             "john",
+						"lfs_token":            "sometoken",
+						"repository_http_path": "https://gitlab.com/repo/path",
+						"expires_in":           1800,
+					}
+					require.NoError(t, json.NewEncoder(w).Encode(body))
+				case "forbidden":
+					w.WriteHeader(http.StatusForbidden)
+				case "broken":
+					w.WriteHeader(http.StatusInternalServerError)
+				}
+			},
+		},
+	}
+
+	return requests
+}
+
+func TestFailedRequests(t *testing.T) {
+	requests := setup(t)
+	url, cleanup := testserver.StartHttpServer(t, requests)
+	defer cleanup()
+
+	testCases := []struct {
+		desc           string
+		args           *commandargs.CommandArgs
+		expectedOutput string
+	}{
+		{
+			desc:           "With bad response",
+			args:           &commandargs.CommandArgs{GitlabKeyId: "-1", CommandType: commandargs.UploadPack},
+			expectedOutput: "Parsing failed",
+		},
+		{
+			desc:           "With API returns an error",
+			args:           &commandargs.CommandArgs{GitlabKeyId: "forbidden", CommandType: commandargs.UploadPack},
+			expectedOutput: "Internal API error (403)",
+		},
+		{
+			desc:           "With API fails",
+			args:           &commandargs.CommandArgs{GitlabKeyId: "broken", CommandType: commandargs.UploadPack},
+			expectedOutput: "Internal API error (500)",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			client, err := NewClient(&config.Config{GitlabUrl: url}, tc.args)
+			require.NoError(t, err)
+
+			repo := "group/repo"
+
+			_, err = client.Authenticate(tc.args.CommandType, repo, "")
+			require.Error(t, err)
+
+			require.Equal(t, tc.expectedOutput, err.Error())
+		})
+	}
+}
+
+func TestSuccessfulRequests(t *testing.T) {
+	requests := setup(t)
+	url, cleanup := testserver.StartHttpServer(t, requests)
+	defer cleanup()
+
+	args := &commandargs.CommandArgs{GitlabKeyId: keyId, CommandType: commandargs.LfsAuthenticate}
+	client, err := NewClient(&config.Config{GitlabUrl: url}, args)
+	require.NoError(t, err)
+
+	response, err := client.Authenticate(action, repo, "")
+	require.NoError(t, err)
+
+	expectedResponse := &Response{
+		Username:  "john",
+		LfsToken:  "sometoken",
+		RepoPath:  "https://gitlab.com/repo/path",
+		ExpiresIn: 1800,
+	}
+
+	require.Equal(t, expectedResponse, response)
+}

spec/gitlab_shell_lfs_authentication_spec.rb

+require_relative 'spec_helper'
+
+require 'open3'
+
+describe 'bin/gitlab-shell git-lfs-authentication' do
+  include_context 'gitlab shell'
+
+  let(:path) { "https://gitlab.com/repo/path" }
+
+  def mock_server(server)
+    server.mount_proc('/api/v4/internal/lfs_authenticate') do |req, res|
+      res.content_type = 'application/json'
+
+      key_id = req.query['key_id'] || req.query['user_id']
+
+      unless key_id
+        body = JSON.parse(req.body)
+        key_id = body['key_id'] || body['user_id'].to_s
+      end
+
+      if key_id == '100'
+        res.status = 200
+        res.body = %{{"username":"john","lfs_token":"sometoken","repository_http_path":"#{path}","expires_in":1800}}
+      else
+        res.status = 403
+      end
+    end
+
+    server.mount_proc('/api/v4/internal/allowed') do |req, res|
+      res.content_type = 'application/json'
+
+      key_id = req.query['key_id'] || req.query['username']
+
+      unless key_id
+        body = JSON.parse(req.body)
+        key_id = body['key_id'] || body['username'].to_s
+      end
+
+      case key_id
+      when '100', 'someone' then
+        res.status = 200
+        res.body = '{"gl_id":"user-100", "status":true}'
+      when '101' then
+        res.status = 200
+        res.body = '{"gl_id":"user-101", "status":true}'
+      else
+        res.status = 403
+      end
+    end
+  end
+
+  shared_examples 'lfs authentication command' do
+    def successful_response
+      {
+        "header" => {
+          "Authorization" => "Basic am9objpzb21ldG9rZW4="
+        },
+        "href" => "#{path}/info/lfs",
+        "expires_in" => 1800
+      }.to_json + "\n"
+    end
+
+    context 'when the command is allowed' do
+      context 'when key is provided' do
+        let(:cmd) { "#{gitlab_shell_path} key-100" }
+
+        it 'lfs is successfully authenticated' do
+          output, stderr, status = Open3.capture3(env, cmd)
+
+          expect(output).to eq(successful_response)
+          expect(status).to be_success
+        end
+      end
+
+      context 'when username is provided' do
+        let(:cmd) { "#{gitlab_shell_path} username-someone" }
+
+        it 'lfs is successfully authenticated' do
+          output, stderr, status = Open3.capture3(env, cmd)
+
+          expect(output).to eq(successful_response)
+          expect(status).to be_success
+        end
+      end
+    end
+
+    context 'when a user is not allowed to perform an action' do
+      let(:cmd) { "#{gitlab_shell_path} key-102" }
+
+      it 'lfs is not authenticated' do
+        _, stderr, status = Open3.capture3(env, cmd)
+
+        expect(stderr).not_to be_empty
+        expect(status).not_to be_success
+      end
+    end
+
+    context 'when lfs authentication is forbidden for a user' do
+      let(:cmd) { "#{gitlab_shell_path} key-101" }
+
+      it 'lfs is not authenticated' do
+        output, stderr, status = Open3.capture3(env, cmd)
+
+        expect(stderr).to be_empty
+        expect(output).to be_empty
+        expect(status).to be_success
+      end
+    end
+
+    context 'when an action for lfs authentication is unknown' do
+      let(:cmd) { "#{gitlab_shell_path} key-100" }
+      let(:env) { {'SSH_CONNECTION' => 'fake', 'SSH_ORIGINAL_COMMAND' => 'git-lfs-authenticate project/repo unknown' } }
+
+      it 'the command is disallowed' do
+        _, stderr, status = Open3.capture3(env, cmd)
+
+        expect(stderr).to eq("> GitLab: Disallowed command\n")
+        expect(status).not_to be_success
+      end
+    end
+  end
+
+  let(:env) { {'SSH_CONNECTION' => 'fake', 'SSH_ORIGINAL_COMMAND' => 'git-lfs-authenticate project/repo download' } }
+
+  describe 'without go features' do
+    before(:context) do
+      write_config(
+        "gitlab_url" => "http+unix://#{CGI.escape(tmp_socket_path)}",
+      )
+    end
+
+    it_behaves_like 'lfs authentication command'
+  end
+
+  describe 'with go features' do
+    before(:context) do
+      write_config(
+        "gitlab_url" => "http+unix://#{CGI.escape(tmp_socket_path)}",
+        "migration" => { "enabled" => true,
+                        "features" => ["git-lfs-authenticate"] }
+      )
+    end
+
+    it_behaves_like 'lfs authentication command'
+  end
+end