New `gitlab:workhorse:install` rake task

Signed-off-by: Rémy Coutable <remy@rymai.me>

changelogs/unreleased/22719-provide-a-new-gitlab-workhorse-install-rake-task-similar-to-gitlab-shell-install.yml

+---
+title: New `gitlab:workhorse:install` rake task
+merge_request: 6574
+author: 

doc/install/installation.md

@@ -175,7 +175,7 @@ We recommend using a PostgreSQL database. For MySQL check the
     ```bash
     sudo -u postgres psql -d template1 -c "CREATE USER git CREATEDB;"
     ```
-    
+
 1. Create the `pg_trgm` extension (required for GitLab 8.6+):
 
     ```bash
@@ -396,15 +396,25 @@ GitLab Shell is an SSH access and repository management software developed speci
 
 ### Install gitlab-workhorse
 
-GitLab-Workhorse uses [GNU Make](https://www.gnu.org/software/make/).
-If you are not using Linux you may have to run `gmake` instead of
-`make` below.
+GitLab-Workhorse uses [GNU Make](https://www.gnu.org/software/make/). The
+following command-line will install GitLab-Workhorse in `home/git/gitlab-workhorse`
+which is the recommended location.
 
-    cd /home/git
-    sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-workhorse.git
-    cd gitlab-workhorse
-    sudo -u git -H git checkout v1.0.1
-    sudo -u git -H make
+    cd /home/git/gitlab
+
+    sudo -u git -H bundle exec rake gitlab:workhorse:install[/home/git/gitlab-workhorse] RAILS_ENV=production
+
+You can specify a different Git repository by providing `GITLAB_WORKHORSE_REPO`:
+
+    cd /home/git/gitlab
+
+    sudo -u git -H bundle exec rake gitlab:workhorse:install[/home/git/gitlab-workhorse] GITLAB_WORKHORSE_REPO=https://gitlab.com/gitlab-org/gitlab-ce.git RAILS_ENV=production
+
+You can specify a different version to use by providing `GITLAB_WORKHORSE_VERSION`:
+
+    cd /home/git/gitlab
+
+    sudo -u git -H bundle exec rake gitlab:workhorse:install[/home/git/gitlab-workhorse] GITLAB_WORKHORSE_VERSION=0.8.1 RAILS_ENV=production
 
 ### Initialize Database and Activate Advanced Features
 

doc/update/8.12-to-8.13.md

@@ -166,7 +166,7 @@ See [smtp_settings.rb.sample] as an example.
 Ensure you're still up-to-date with the latest init script changes:
 
     sudo cp lib/support/init.d/gitlab /etc/init.d/gitlab
-    
+
 For Ubuntu 16.04.1 LTS:
 
     sudo systemctl daemon-reload

lib/tasks/gitlab/shell.rake

@@ -5,42 +5,23 @@ namespace :gitlab do
       warn_user_is_not_gitlab
 
       default_version = Gitlab::Shell.version_required
-      default_version_tag = 'v' + default_version
-      args.with_defaults(tag: default_version_tag, repo: "https://gitlab.com/gitlab-org/gitlab-shell.git")
+      default_version_tag = "v#{default_version}"
+      args.with_defaults(tag: default_version_tag, repo: 'https://gitlab.com/gitlab-org/gitlab-shell.git')
 
-      user = Gitlab.config.gitlab.user
-      home_dir = Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home
       gitlab_url = Gitlab.config.gitlab.url
       # gitlab-shell requires a / at the end of the url
       gitlab_url += '/' unless gitlab_url.end_with?('/')
       target_dir = Gitlab.config.gitlab_shell.path
 
-      # Clone if needed
-      if File.directory?(target_dir)
-        Dir.chdir(target_dir) do
-          system(*%W(Gitlab.config.git.bin_path} fetch --tags --quiet))
-          system(*%W(Gitlab.config.git.bin_path} checkout --quiet #{default_version_tag}))
-        end
-      else
-        system(*%W(#{Gitlab.config.git.bin_path} clone -- #{args.repo} #{target_dir}))
-      end
+      checkout_or_clone_tag(tag: default_version_tag, repo: args.repo, target_dir: target_dir)
 
       # Make sure we're on the right tag
       Dir.chdir(target_dir) do
-        # First try to checkout without fetching
-        # to avoid stalling tests if the Internet is down.
-        reseted = reset_to_commit(args)
-
-        unless reseted
-          system(*%W(#{Gitlab.config.git.bin_path} fetch origin))
-          reset_to_commit(args)
-        end
-
         config = {
-          user: user,
+          user: Gitlab.config.gitlab.user,
           gitlab_url: gitlab_url,
           http_settings: {self_signed_cert: false}.stringify_keys,
-          auth_file: File.join(home_dir, ".ssh", "authorized_keys"),
+          auth_file: File.join(user_home, ".ssh", "authorized_keys"),
           redis: {
             bin: %x{which redis-cli}.chomp,
             namespace: "resque:gitlab"
@@ -74,7 +55,7 @@ namespace :gitlab do
       # be an issue since it is more than likely that there are no "normal"
       # user accounts on a gitlab server). The alternative is for the admin to
       # install a ruby (1.9.3+) in the global path.
-      File.open(File.join(home_dir, ".ssh", "environment"), "w+") do |f|
+      File.open(File.join(user_home, ".ssh", "environment"), "w+") do |f|
         f.puts "PATH=#{ENV['PATH']}"
       end
 
@@ -142,15 +123,4 @@ namespace :gitlab do
     puts "Quitting...".color(:red)
     exit 1
   end
-
-  def reset_to_commit(args)
-    tag, status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} describe -- #{args.tag}))
-
-    unless status.zero?
-      tag, status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} describe -- origin/#{args.tag}))
-    end
-
-    tag = tag.strip
-    system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag}))
-  end
 end

lib/tasks/gitlab/task_helpers.rake

@@ -70,7 +70,7 @@ namespace :gitlab do
 
   # Runs the given command
   #
-  # Returns nil if the command was not found
+  # Returns '' if the command was not found
   # Returns the output of the command otherwise
   #
   # see also #run_and_match
@@ -137,4 +137,37 @@ namespace :gitlab do
   def repository_storage_paths_args
     Gitlab.config.repositories.storages.values
   end
+
+  def user_home
+    Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home
+  end
+
+  def checkout_or_clone_tag(tag:, repo:, target_dir:)
+    if Dir.exist?(target_dir)
+      Dir.chdir(target_dir) do
+        run_command(%W[#{Gitlab.config.git.bin_path} fetch --tags --quiet])
+        run_command(%W[#{Gitlab.config.git.bin_path} checkout --quiet #{tag}])
+      end
+    else
+      run_command(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{target_dir}])
+    end
+
+    # Make sure we're on the right tag
+    Dir.chdir(target_dir) do
+      # First try to checkout without fetching
+      # to avoid stalling tests if the Internet is down.
+      reset_to_tag(tag)
+    end
+  end
+
+  def reset_to_tag(tag_wanted)
+    tag, status = Gitlab::Popen.popen(%W[#{Gitlab.config.git.bin_path} describe -- #{tag_wanted}])
+
+    unless status.zero?
+      run_command(%W(#{Gitlab.config.git.bin_path} fetch origin))
+      tag = run_command(%W[#{Gitlab.config.git.bin_path} describe -- origin/#{tag_wanted}])
+    end
+
+    run_command(%W[#{Gitlab.config.git.bin_path} reset --hard #{tag.strip}])
+  end
 end

lib/tasks/gitlab/workhorse.rake

+namespace :gitlab do
+  namespace :workhorse do
+    desc "GitLab | Install or upgrade gitlab-workhorse"
+    task :install, [:dir] => :environment do |t, args|
+      warn_user_is_not_gitlab
+      unless args.dir.present?
+        abort "Please specify the directory where you want to install gitlab-workhorse:\n  rake gitlab:workhorse:install[/home/git/gitlab-workhorse]"
+      end
+
+      tag = "v#{ENV['GITLAB_WORKHORSE_VERSION'] || Gitlab::Workhorse.version}"
+      repo = ENV['GITLAB_WORKHORSE_REPO'] || 'https://gitlab.com/gitlab-org/gitlab-workhorse.git'
+
+      checkout_or_clone_tag(tag: tag, repo: repo, target_dir: args.dir)
+
+      _, status = Gitlab::Popen.popen(%w[which gmake])
+      command = status.zero? ? 'gmake' : 'make'
+
+      Dir.chdir(args.dir) do
+        run_command([command])
+      end
+    end
+  end
+end

spec/support/stub_configuration.rb

@@ -17,6 +17,10 @@ module StubConfiguration
     allow(Gitlab.config.gravatar).to receive_messages(messages)
   end
 
+  def stub_gitlab_workhorse_setting(messages)
+    allow(Gitlab.config.gitlab_workhorse).to receive_messages(messages)
+  end
+
   def stub_incoming_email_setting(messages)
     allow(Gitlab.config.incoming_email).to receive_messages(messages)
   end

spec/tasks/gitlab/task_helpers_spec.rb

+require 'spec_helper'
+require 'rake'
+
+describe 'gitlab:workhorse namespace rake task' do
+  before :all do
+    Rake.application.rake_require 'tasks/gitlab/task_helpers'
+
+    # empty task as env is already loaded
+    Rake::Task.define_task :environment
+  end
+
+  describe '#checkout_or_clone_tag' do
+    let(:repo) { 'https://gitlab.com/gitlab-org/gitlab-test.git' }
+    let(:clone_path) { Rails.root.join('tmp/tests/task_helpers_tests').to_s }
+    let(:tag) { 'v1.1.0' }
+    before do
+      FileUtils.rm_rf(clone_path)
+      allow_any_instance_of(Object).to receive(:run_command)
+      expect_any_instance_of(Object).to receive(:reset_to_tag).with(tag)
+    end
+
+    after do
+      FileUtils.rm_rf(clone_path)
+    end
+
+    context 'target_dir does not exist' do
+      it 'clones the repo, retrieve the tag from origin, and checkout the tag' do
+        expect(Dir).to receive(:chdir).and_call_original
+        expect_any_instance_of(Object).
+          to receive(:run_command).with(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{clone_path}]) { FileUtils.mkdir_p(clone_path) } # Fake the cloning
+
+        checkout_or_clone_tag(tag: tag, repo: repo, target_dir: clone_path)
+      end
+    end
+
+    context 'target_dir exists' do
+      before do
+        FileUtils.mkdir_p(clone_path)
+      end
+
+      it 'fetch and checkout the tag' do
+        expect(Dir).to receive(:chdir).twice.and_call_original
+        expect_any_instance_of(Object).
+          to receive(:run_command).with(%W[#{Gitlab.config.git.bin_path} fetch --tags --quiet])
+        expect_any_instance_of(Object).
+          to receive(:run_command).with(%W[#{Gitlab.config.git.bin_path} checkout --quiet #{tag}])
+
+        checkout_or_clone_tag(tag: tag, repo: repo, target_dir: clone_path)
+      end
+    end
+  end
+
+  describe '#reset_to_tag' do
+    let(:tag) { 'v1.1.0' }
+    before do
+      expect_any_instance_of(Object).
+        to receive(:run_command).with(%W[#{Gitlab.config.git.bin_path} reset --hard #{tag}])
+    end
+
+    context 'when the tag is not checked out locally' do
+      before do
+        expect(Gitlab::Popen).
+          to receive(:popen).with(%W[#{Gitlab.config.git.bin_path} describe -- #{tag}]).and_return(['', 42])
+      end
+
+      it 'fetch origin, ensure the tag exists, and resets --hard to the given tag' do
+        expect_any_instance_of(Object).
+          to receive(:run_command).with(%W[#{Gitlab.config.git.bin_path} fetch origin])
+        expect_any_instance_of(Object).
+          to receive(:run_command).with(%W[#{Gitlab.config.git.bin_path} describe -- origin/#{tag}]).and_return(tag)
+
+        reset_to_tag(tag)
+      end
+    end
+
+    context 'when the tag is checked out locally' do
+      before do
+        expect(Gitlab::Popen).
+          to receive(:popen).with(%W[#{Gitlab.config.git.bin_path} describe -- #{tag}]).and_return([tag, 0])
+      end
+
+      it 'resets --hard to the given tag' do
+        reset_to_tag(tag)
+      end
+    end
+  end
+end

spec/tasks/gitlab/workhorse_rake_spec.rb

+require 'spec_helper'
+require 'rake'
+
+describe 'gitlab:workhorse namespace rake task' do
+  before :all do
+    Rake.application.rake_require 'tasks/gitlab/task_helpers'
+    Rake.application.rake_require 'tasks/gitlab/workhorse'
+
+    # empty task as env is already loaded
+    Rake::Task.define_task :environment
+  end
+
+  def run_rake_task(task_name, *args)
+    Rake::Task[task_name].reenable
+    Rake.application.invoke_task("#{task_name}[#{args.join(',')}]")
+  end
+
+  describe 'install' do
+    let(:repo) { 'https://gitlab.com/gitlab-org/gitlab-workhorse.git' }
+    let(:clone_path) { Rails.root.join('tmp/tests/gitlab-workhorse').to_s }
+    let(:tag) { "v#{File.read(Rails.root.join(Gitlab::Workhorse::VERSION_FILE)).chomp}" }
+    before do
+      # avoid writing task output to spec progress
+      allow($stdout).to receive :write
+      allow(ENV).to receive(:[])
+    end
+
+    context 'no dir given' do
+      it 'aborts and display a help message' do
+        expect { run_rake_task('gitlab:workhorse:install') }.to raise_error /Please specify the directory where you want to install gitlab-workhorse/
+      end
+    end
+
+    context 'when an underlying Git command fail' do
+      it 'aborts and display a help message' do
+        expect_any_instance_of(Object).
+          to receive(:checkout_or_clone_tag).and_raise 'Git error'
+
+        expect { run_rake_task('gitlab:workhorse:install', clone_path) }.to raise_error 'Git error'
+      end
+    end
+
+    describe 'checkout or clone' do
+      before do
+        expect(Dir).to receive(:chdir).with(clone_path)
+      end
+
+      it 'calls checkout_or_clone_tag with the right arguments' do
+        expect_any_instance_of(Object).
+          to receive(:checkout_or_clone_tag).with(tag: tag, repo: repo, target_dir: clone_path)
+
+        run_rake_task('gitlab:workhorse:install', clone_path)
+      end
+
+      context 'given a specific repo' do
+        before do
+          expect(ENV).to receive(:[]).with('GITLAB_WORKHORSE_REPO').and_return('https://gitlab.com/user1/gitlab-workhorse.git')
+        end
+
+        it 'calls checkout_or_clone_tag with the given repo' do
+          expect_any_instance_of(Object).
+            to receive(:checkout_or_clone_tag).with(tag: tag, repo: 'https://gitlab.com/user1/gitlab-workhorse.git', target_dir: clone_path)
+
+          run_rake_task('gitlab:workhorse:install', clone_path)
+        end
+      end
+
+      context 'given a specific version' do
+        before do
+          allow(ENV).to receive(:[]).with('GITLAB_WORKHORSE_VERSION').and_return('42.42.0')
+        end
+
+        it 'calls checkout_or_clone_tag with the given repo' do
+          expect_any_instance_of(Object).
+            to receive(:checkout_or_clone_tag).with(tag: 'v42.42.0', repo: repo, target_dir: clone_path)
+
+          run_rake_task('gitlab:workhorse:install', clone_path)
+        end
+      end
+    end
+
+    describe 'gmake/make' do
+      before do
+        FileUtils.mkdir_p(clone_path)
+        expect(Dir).to receive(:chdir).with(clone_path).and_call_original
+      end
+
+      context 'gmake is available' do
+        before do
+          expect_any_instance_of(Object).to receive(:checkout_or_clone_tag)
+          allow_any_instance_of(Object).to receive(:run_command).with(['gmake']).and_return(true)
+        end
+
+        it 'calls gmake in the gitlab-workhorse directory' do
+          expect(Gitlab::Popen).to receive(:popen).with(%w[which gmake]).and_return(['/usr/bin/gmake', 0])
+          expect_any_instance_of(Object).to receive(:run_command).with(['gmake']).and_return(true)
+
+          run_rake_task('gitlab:workhorse:install', clone_path)
+        end
+      end
+
+      context 'gmake is not available' do
+        before do
+          expect_any_instance_of(Object).to receive(:checkout_or_clone_tag)
+          allow_any_instance_of(Object).to receive(:run_command).with(['make']).and_return(true)
+        end
+
+        it 'calls make in the gitlab-workhorse directory' do
+          expect(Gitlab::Popen).to receive(:popen).with(%w[which gmake]).and_return(['', 42])
+          expect_any_instance_of(Object).to receive(:run_command).with(['make']).and_return(true)
+
+          run_rake_task('gitlab:workhorse:install', clone_path)
+        end
+      end
+    end
+  end
+end