Merge branch 'mv-storage' into 'master'

Add command to move repositories between repository storages

Necessary for https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/533

/cc @pcarranza 

See merge request !71

CHANGELOG

+v3.3.0
+  - Track ongoing push commands
+  - Add command to move repositories between repository storages
+
 v3.2.1
   - Allow gitlab-project's fork-project command to fork projects between different repository storages
 

hooks/pre-receive

@@ -9,10 +9,16 @@ protocol = ENV.delete('GL_PROTOCOL')
 repo_path = Dir.pwd
 
 require_relative '../lib/gitlab_custom_hook'
+require_relative '../lib/gitlab_reference_counter'
 require_relative '../lib/gitlab_access'
 
+# It's important that on pre-receive `increase_reference_counter` gets executed
+# last so that it only runs if everything else succeeded. On post-receive on the
+# other hand, we run GitlabPostReceive first because the push is already done
+# and we don't want to skip it if the custom hook fails.
 if GitlabAccess.new(repo_path, key_id, refs, protocol).exec &&
-    GitlabCustomHook.new.pre_receive(refs, repo_path)
+    GitlabCustomHook.new.pre_receive(refs, repo_path) &&
+    GitlabReferenceCounter.new(repo_path).increase
   exit 0
 else
   exit 1

lib/gitlab_post_receive.rb

 require_relative 'gitlab_init'
 require_relative 'gitlab_net'
+require_relative 'gitlab_reference_counter'
 require 'json'
 require 'base64'
 require 'securerandom'
@@ -28,7 +29,7 @@ class GitlabPostReceive
       nil
     end
 
-    result
+    result && GitlabReferenceCounter.new(repo_path).decrease
   end
 
   protected

lib/gitlab_projects.rb

@@ -4,6 +4,7 @@ require 'open3'
 
 require_relative 'gitlab_config'
 require_relative 'gitlab_logger'
+require_relative 'gitlab_reference_counter'
 
 class GitlabProjects
   GLOBAL_HOOKS_DIRECTORY = File.join(ROOT_PATH, 'hooks')
@@ -55,6 +56,7 @@ class GitlabProjects
     when 'list-projects'; puts list_projects
     when 'rm-project'; rm_project
     when 'mv-project'; mv_project
+    when 'mv-storage'; mv_storage
     when 'import-project'; import_project
     when 'fork-project'; fork_project
     when 'fetch-remote'; fetch_remote
@@ -285,6 +287,42 @@ class GitlabProjects
     FileUtils.mv(full_path, new_full_path)
   end
 
+  # Move repository from one storage path to another
+  #
+  # Wont work if target namespace directory does not exist in the new storage path
+  #
+  def mv_storage
+    new_storage = ARGV.shift
+
+    unless new_storage
+      $logger.error "mv-storage failed: no destination storage path provided."
+      return false
+    end
+
+    new_full_path = File.join(new_storage, project_name)
+
+    # verify that the source repo exists
+    unless File.exists?(full_path)
+      $logger.error "mv-storage failed: source path <#{full_path}> does not exist."
+      return false
+    end
+
+    # Make sure the destination directory exists
+    FileUtils.mkdir_p(new_full_path)
+
+    # Make sure the source path ends with a slash so that rsync copies the
+    # contents of the directory, as opposed to copying the directory by name
+    source_path = File.join(full_path, '')
+
+    if wait_for_pushes
+      $logger.info "Syncing project #{@project_name} from <#{full_path}> to <#{new_full_path}>."
+      system(*%W(rsync -a --delete #{source_path} #{new_full_path}))
+    else
+      $logger.error "mv-storage failed: source path <#{full_path}> is waiting for pushes to finish."
+      false
+    end
+  end
+
   def fork_project
     destination_repos_path = ARGV.shift
 
@@ -329,4 +367,18 @@ class GitlabProjects
     cmd = %W(git --git-dir=#{full_path} gc)
     system(*cmd)
   end
+
+  def wait_for_pushes
+    # Try for 30 seconds, polling every 10
+    3.times do
+      return true if gitlab_reference_counter.value == 0
+      sleep 10
+    end
+
+    false
+  end
+
+  def gitlab_reference_counter
+    @gitlab_reference_counter ||= GitlabReferenceCounter.new(full_path)
+  end
 end

lib/gitlab_reference_counter.rb

+require_relative 'gitlab_init'
+require_relative 'gitlab_net'
+
+class GitlabReferenceCounter
+  REFERENCE_EXPIRE_TIME = 600
+
+  attr_reader :path, :key
+
+  def initialize(path)
+    @path = path
+    @key = "git-receive-pack-reference-counter:#{path}"
+  end
+
+  def value
+    (redis_client.get(key) || 0).to_i
+  end
+
+  def increase
+    redis_cmd do
+      redis_client.incr(key)
+      redis_client.expire(key, REFERENCE_EXPIRE_TIME)
+    end
+  end
+
+  def decrease
+    redis_cmd do
+      current_value = redis_client.decr(key)
+      if current_value < 0
+        $logger.warn "Reference counter for #{path} decreased when its value was less than 1. Reseting the counter."
+        redis_client.del(key)
+      end
+    end
+  end
+
+  private
+
+  def redis_client
+    @redis_client ||= GitlabNet.new.redis_client
+  end
+
+  def redis_cmd
+    begin
+      yield
+      true
+    rescue => e
+      message = "GitLab: An unexpected error occurred in writing to Redis: #{e}"
+      $stderr.puts message
+      $logger.error message
+      false
+    end
+  end
+end

spec/gitlab_post_receive_spec.rb

@@ -25,6 +25,11 @@ describe GitlabPostReceive do
 
     before do
       allow_any_instance_of(GitlabNet).to receive(:redis_client).and_return(redis_client)
+      allow_any_instance_of(GitlabReferenceCounter).to receive(:redis_client).and_return(redis_client)
+      allow(redis_client).to receive(:get).and_return(1)
+      allow(redis_client).to receive(:incr).and_return(true)
+      allow(redis_client).to receive(:decr).and_return(0)
+      allow(redis_client).to receive(:rpush).and_return(true)
     end
 
     it "prints the broadcast message" do
@@ -59,6 +64,34 @@ describe GitlabPostReceive do
       gitlab_post_receive.exec
     end
 
+    context 'reference counter' do
+      it 'decreases the reference counter for the project' do
+        expect_any_instance_of(GitlabReferenceCounter).to receive(:decrease).and_return(true)
+
+        gitlab_post_receive.exec
+      end
+
+      context "when the redis command succeeds" do
+        before do
+          allow(redis_client).to receive(:decr).and_return(0)
+        end
+
+        it "returns true" do
+          expect(gitlab_post_receive.exec).to eq(true)
+        end
+      end
+
+      context "when the redis command fails" do
+        before do
+          allow(redis_client).to receive(:decr).and_raise('Fail')
+        end
+
+        it "returns false" do
+          expect(gitlab_post_receive.exec).to eq(false)
+        end
+      end
+    end
+
     context "when the redis command succeeds" do
 
       before do

spec/gitlab_projects_spec.rb

@@ -205,6 +205,50 @@ describe GitlabProjects do
     end
   end
 
+  describe :mv_storage do
+    let(:alternative_storage_path) { File.join(ROOT_PATH, 'tmp', 'alternative') }
+    let(:gl_projects) { build_gitlab_projects('mv-storage', tmp_repos_path, repo_name, alternative_storage_path) }
+    let(:new_repo_path) { File.join(alternative_storage_path, repo_name) }
+
+    before do
+      FileUtils.mkdir_p(tmp_repo_path)
+      FileUtils.mkdir_p(alternative_storage_path)
+      allow_any_instance_of(GitlabReferenceCounter).to receive(:value).and_return(0)
+    end
+
+    after { FileUtils.rm_rf(alternative_storage_path) }
+
+    it "should rsync a repo directory" do
+      File.exists?(tmp_repo_path).should be_true
+      gl_projects.exec
+      File.exists?(new_repo_path).should be_true
+    end
+
+    it "should fail if no destination path is provided" do
+      incomplete = build_gitlab_projects('mv-storage', tmp_repos_path, repo_name)
+      $logger.should_receive(:error).with("mv-storage failed: no destination storage path provided.")
+      incomplete.exec.should be_false
+    end
+
+    it "should fail if the source path doesn't exist" do
+      bad_source = build_gitlab_projects('mv-storage', tmp_repos_path, 'bad-src.git', alternative_storage_path)
+      $logger.should_receive(:error).with("mv-storage failed: source path <#{tmp_repos_path}/bad-src.git> does not exist.")
+      bad_source.exec.should be_false
+    end
+
+    it "should fail if there are pushes ongoing" do
+      allow_any_instance_of(GitlabReferenceCounter).to receive(:value).and_return(1)
+      $logger.should_receive(:error).with("mv-storage failed: source path <#{tmp_repo_path}> is waiting for pushes to finish.")
+      gl_projects.exec.should be_false
+    end
+
+    it "should log an mv-storage event" do
+      message = "Syncing project #{repo_name} from <#{tmp_repo_path}> to <#{new_repo_path}>."
+      $logger.should_receive(:info).with(message)
+      gl_projects.exec
+    end
+  end
+
   describe :import_project do
     context 'success import' do
       let(:gl_projects) { build_gitlab_projects('import-project', tmp_repos_path, repo_name, 'https://github.com/randx/six.git') }

spec/gitlab_reference_counter_spec.rb

+# coding: utf-8
+require 'spec_helper'
+require 'gitlab_reference_counter'
+
+describe GitlabReferenceCounter do
+  let(:redis_client) { double('redis_client') }
+  let(:reference_counter_key) { "git-receive-pack-reference-counter:/test/path" }
+  let(:gitlab_reference_counter) { GitlabReferenceCounter.new('/test/path') }
+
+  before do
+    allow(gitlab_reference_counter).to receive(:redis_client).and_return(redis_client)
+    $logger = double('logger').as_null_object
+  end
+
+  it 'increases and set the expire time of a reference count for a path' do
+    expect(redis_client).to receive(:incr).with(reference_counter_key)
+    expect(redis_client).to receive(:expire).with(reference_counter_key, GitlabReferenceCounter::REFERENCE_EXPIRE_TIME)
+    expect(gitlab_reference_counter.increase).to be(true)
+  end
+
+  it 'decreases the reference count for a path' do
+    allow(redis_client).to receive(:decr).and_return(0)
+    expect(redis_client).to receive(:decr).with(reference_counter_key)
+    expect(gitlab_reference_counter.decrease).to be(true)
+  end
+
+  it 'warns if attempting to decrease a counter with a value of one or less, and resets the counter' do
+    expect(redis_client).to receive(:decr).and_return(-1)
+    expect(redis_client).to receive(:del)
+    expect($logger).to receive(:warn).with("Reference counter for /test/path decreased when its value was less than 1. Reseting the counter.")
+    expect(gitlab_reference_counter.decrease).to be(true)
+  end
+
+  it 'get the reference count for a path' do
+    allow(redis_client).to receive(:get).and_return(1)
+    expect(gitlab_reference_counter.value).to be(1)
+  end
+end