Merge branch 'git-annex' into 'master'

Git annex support

- [x] fix auth for git-annex
- [x] enable git-annex for repository on first annex call
- [x] config option to disable it
- [x] write tests

See merge request !55

CHANGELOG

+v2.5.0
+  - Support git-annex tool
+
 v2.4.3
   - Print broadcast message if one is available
 

VERSION

-2.4.3
+2.5.0.pre

config.yml.example

@@ -46,3 +46,8 @@ log_level: INFO
 # Set to true to see real usernames in the logs instead of key ids, which is easier to follow, but
 # incurs an extra API call on every gitlab-shell command.
 audit_usernames: false
+
+# Enable git-annex support
+# git-annex allows managing files with git, without checking the file contents into git
+# See https://git-annex.branchable.com/ for documentation
+git_annex_enabled: true

lib/gitlab_config.rb

@@ -47,6 +47,10 @@ class GitlabConfig
     @config['audit_usernames'] ||= false
   end
 
+  def git_annex_enabled?
+    @config['git_annex_enabled'] ||= true
+  end
+
   # Build redis command to write update event in gitlab queue
   def redis_command
     if redis.empty?

lib/gitlab_shell.rb

@@ -22,6 +22,7 @@ class GitlabShell
         ENV['GL_ID'] = @key_id
 
         access = api.check_access(@git_cmd, @repo_name, @key_id, '_any')
+
         if access.allowed?
           process_cmd
         else
@@ -47,19 +48,45 @@ class GitlabShell
 
   def parse_cmd
     args = Shellwords.shellwords(@origin_cmd)
-    raise DisallowedCommandError unless args.count == 2
-    @git_cmd = args[0]
-    @repo_name = escape_path(args[1])
+    @git_cmd = args.first
+
+    if @git_cmd == 'git-annex-shell' && @config.git_annex_enabled?
+      @repo_name = escape_path(args[2].gsub("\/~\/", ''))
+
+      # Make sure repository has git-annex enabled
+      init_git_annex(@repo_name)
+    else
+      raise DisallowedCommandError unless args.count == 2
+      @repo_name = escape_path(args.last)
+    end
   end
 
   def git_cmds
-    %w(git-upload-pack git-receive-pack git-upload-archive)
+    %w(git-upload-pack git-receive-pack git-upload-archive git-annex-shell)
   end
 
   def process_cmd
     repo_full_path = File.join(repos_path, repo_name)
-    $logger.info "gitlab-shell: executing git command <#{@git_cmd} #{repo_full_path}> for #{log_username}."
-    exec_cmd(@git_cmd, repo_full_path)
+
+    if @git_cmd == 'git-annex-shell' && @config.git_annex_enabled?
+      args = Shellwords.shellwords(@origin_cmd)
+      parsed_args =
+        args.map do |arg|
+          # Convert /~/group/project.git to group/project.git
+          # to make git annex path compatible with gitlab-shell
+          if arg =~ /\A\/~\/.*\.git\Z/
+            repo_full_path
+          else
+            arg
+          end
+        end
+
+      $logger.info "gitlab-shell: executing git-annex command <#{parsed_args.join(' ')}> for #{log_username}."
+      exec_cmd(*parsed_args)
+    else
+      $logger.info "gitlab-shell: executing git command <#{@git_cmd} #{repo_full_path}> for #{log_username}."
+      exec_cmd(@git_cmd, repo_full_path)
+    end
   end
 
   # This method is not covered by Rspec because it ends the current Ruby process.
@@ -99,4 +126,14 @@ class GitlabShell
       abort "Wrong repository path"
     end
   end
+
+  def init_git_annex(path)
+    full_repo_path = File.join(repos_path, path)
+
+    unless File.exists?(File.join(full_repo_path, '.git', 'annex'))
+      cmd = %W(git --git-dir=#{full_repo_path} annex init "GitLab")
+      system(*cmd)
+      $logger.info "Enable git-annex for repository: #{path}."
+    end
+  end
 end

spec/gitlab_shell_spec.rb

@@ -3,6 +3,14 @@ require_relative '../lib/gitlab_shell'
 require_relative '../lib/gitlab_access_status'
 
 describe GitlabShell do
+  before do
+    FileUtils.mkdir_p(tmp_repos_path)
+  end
+
+  after do
+    FileUtils.rm_rf(tmp_repos_path)
+  end
+
   subject do
     ARGV[0] = key_id
     GitlabShell.new.tap do |shell|
@@ -10,51 +18,77 @@ describe GitlabShell do
       shell.stub(api: api)
     end
   end
+
   let(:api) do
     double(GitlabNet).tap do |api|
       api.stub(discover: { 'name' => 'John Doe' })
       api.stub(check_access: GitAccessStatus.new(true))
     end
   end
+
   let(:key_id) { "key-#{rand(100) + 100}" }
-  let(:repository_path) { "/home/git#{rand(100)}/repos" }
+  let(:tmp_repos_path) { File.join(ROOT_PATH, 'tmp', 'repositories') }
+
   before do
-    GitlabConfig.any_instance.stub(repos_path: repository_path, audit_usernames: false)
+    GitlabConfig.any_instance.stub(repos_path: tmp_repos_path, audit_usernames: false)
   end
 
   describe :initialize do
     before { ssh_cmd 'git-receive-pack' }
 
     its(:key_id) { should == key_id }
-    its(:repos_path) { should == repository_path }
+    its(:repos_path) { should == tmp_repos_path }
   end
 
   describe :parse_cmd do
-    context 'w/o namespace' do
-      before do
-        ssh_cmd 'git-upload-pack gitlab-ci.git'
-        subject.send :parse_cmd
+    describe 'git' do
+      context 'w/o namespace' do
+        before do
+          ssh_cmd 'git-upload-pack gitlab-ci.git'
+          subject.send :parse_cmd
+        end
+
+        its(:repo_name) { should == 'gitlab-ci.git' }
+        its(:git_cmd) { should == 'git-upload-pack' }
+      end
+
+      context 'namespace' do
+        before do
+          ssh_cmd 'git-upload-pack dmitriy.zaporozhets/gitlab-ci.git'
+          subject.send :parse_cmd
+        end
+
+        its(:repo_name) { should == 'dmitriy.zaporozhets/gitlab-ci.git' }
+        its(:git_cmd) { should == 'git-upload-pack' }
       end
 
-      its(:repo_name) { should == 'gitlab-ci.git' }
-      its(:git_cmd) { should == 'git-upload-pack' }
+      context 'with an invalid number of arguments' do
+        before { ssh_cmd 'foobar' }
+
+        it "should raise an DisallowedCommandError" do
+          expect { subject.send :parse_cmd }.to raise_error(GitlabShell::DisallowedCommandError)
+        end
+      end
     end
 
-    context 'namespace' do
+    describe 'git-annex' do
+      let(:repo_path) { File.join(tmp_repos_path, 'dzaporozhets/gitlab.git') }
+
       before do
-        ssh_cmd 'git-upload-pack dmitriy.zaporozhets/gitlab-ci.git'
+        # Create existing project
+        FileUtils.mkdir_p(repo_path)
+        cmd = %W(git --git-dir=#{repo_path} init --bare)
+        system(*cmd)
+
+        ssh_cmd 'git-annex-shell inannex /~/dzaporozhets/gitlab.git SHA256E'
         subject.send :parse_cmd
       end
 
-      its(:repo_name) { should == 'dmitriy.zaporozhets/gitlab-ci.git' }
-      its(:git_cmd) { should == 'git-upload-pack' }
-    end
-
-    context 'with an invalid number of arguments' do
-      before { ssh_cmd 'foobar' }
+      its(:repo_name) { should == 'dzaporozhets/gitlab.git' }
+      its(:git_cmd) { should == 'git-annex-shell' }
 
-      it "should raise an DisallowedCommandError" do
-        expect { subject.send :parse_cmd }.to raise_error(GitlabShell::DisallowedCommandError)
+      it 'should init git-annex' do
+        File.exists?(File.join(tmp_repos_path, 'dzaporozhets/gitlab.git/annex')).should be_true
       end
     end
   end
@@ -69,7 +103,7 @@ describe GitlabShell do
       end
 
       it "should execute the command" do
-        subject.should_receive(:exec_cmd).with("git-upload-pack", File.join(repository_path, 'gitlab-ci.git'))
+        subject.should_receive(:exec_cmd).with("git-upload-pack", File.join(tmp_repos_path, 'gitlab-ci.git'))
       end
 
       it "should set the GL_ID environment variable" do
@@ -78,7 +112,7 @@ describe GitlabShell do
 
       it "should log the command execution" do
         message = "gitlab-shell: executing git command "
-        message << "<git-upload-pack #{File.join(repository_path, 'gitlab-ci.git')}> "
+        message << "<git-upload-pack #{File.join(tmp_repos_path, 'gitlab-ci.git')}> "
         message << "for user with key #{key_id}."
         $logger.should_receive(:info).with(message)
       end
@@ -98,12 +132,12 @@ describe GitlabShell do
       end
 
       it "should execute the command" do
-        subject.should_receive(:exec_cmd).with("git-receive-pack", File.join(repository_path, 'gitlab-ci.git'))
+        subject.should_receive(:exec_cmd).with("git-receive-pack", File.join(tmp_repos_path, 'gitlab-ci.git'))
       end
 
       it "should log the command execution" do
         message = "gitlab-shell: executing git command "
-        message << "<git-receive-pack #{File.join(repository_path, 'gitlab-ci.git')}> "
+        message << "<git-receive-pack #{File.join(tmp_repos_path, 'gitlab-ci.git')}> "
         message << "for user with key #{key_id}."
         $logger.should_receive(:info).with(message)
       end
@@ -137,7 +171,7 @@ describe GitlabShell do
     end
 
     context "failed connection" do
-      before { 
+      before {
         ssh_cmd 'git-upload-pack gitlab-ci.git'
         api.stub(:check_access).and_raise(GitlabNet::ApiUnreachableError)
       }
@@ -151,6 +185,15 @@ describe GitlabShell do
         subject.should_not_receive(:exec_cmd)
       end
     end
+
+    describe 'git-annex' do
+      before { ssh_cmd 'git-annex-shell commit /~/gitlab-ci.git SHA256' }
+      after { subject.exec }
+
+      it "should execute the command" do
+        subject.should_receive(:exec_cmd).with("git-annex-shell", "commit", File.join(tmp_repos_path, 'gitlab-ci.git'), "SHA256")
+      end
+    end
   end
 
   describe :validate_access do
@@ -198,5 +241,4 @@ describe GitlabShell do
   def ssh_cmd(cmd)
     ENV['SSH_ORIGINAL_COMMAND'] = cmd
   end
-
 end