Merge branch 'gitaly-install-path' into 'master'

Remove storage path dependency of gitaly install task

See merge request gitlab-org/gitlab-ce!21101

changelogs/unreleased/gitaly-install-path.yml

+---
+title: Remove storage path dependency of gitaly install task
+merge_request: 21101
+author:
+type: changed

doc/install/installation.md

@@ -153,7 +153,7 @@ page](https://golang.org/dl).
 
     # Remove former Go installation folder
     sudo rm -rf /usr/local/go
-    
+
     curl --remote-name --progress https://dl.google.com/go/go1.10.3.linux-amd64.tar.gz
     echo 'fa1b0e45d3b647c252f51f5e1204aba049cde4af177ef9f2181f43004f901035  go1.10.3.linux-amd64.tar.gz' | shasum -a256 -c - && \
       sudo tar -C /usr/local -xzf go1.10.3.linux-amd64.tar.gz
@@ -494,11 +494,11 @@ Make GitLab start on boot:
 ### Install Gitaly
 
     # Fetch Gitaly source with Git and compile with Go
-    sudo -u git -H bundle exec rake "gitlab:gitaly:install[/home/git/gitaly]" RAILS_ENV=production
+    sudo -u git -H bundle exec rake "gitlab:gitaly:install[/home/git/gitaly,/home/git/repositories]" RAILS_ENV=production
 
 You can specify a different Git repository by providing it as an extra parameter:
 
-    sudo -u git -H bundle exec rake "gitlab:gitaly:install[/home/git/gitaly,https://example.com/gitaly.git]" RAILS_ENV=production
+    sudo -u git -H bundle exec rake "gitlab:gitaly:install[/home/git/gitaly,/home/git/repositories,https://example.com/gitaly.git]" RAILS_ENV=production
 
 Next, make sure gitaly configured:
 

lib/gitlab/setup_helper.rb

+require 'toml-rb'
+
 module Gitlab
   module SetupHelper
     class << self
@@ -9,7 +11,7 @@ module Gitlab
       # because it uses a Unix socket.
       # For development and testing purposes, an extra storage is added to gitaly,
       # which is not known to Rails, but must be explicitly stubbed.
-      def gitaly_configuration_toml(gitaly_dir, gitaly_ruby: true)
+      def gitaly_configuration_toml(gitaly_dir, storage_paths, gitaly_ruby: true)
         storages = []
         address = nil
 
@@ -24,10 +26,7 @@ module Gitlab
             address = val['gitaly_address']
           end
 
-          # https://gitlab.com/gitlab-org/gitaly/issues/1238
-          Gitlab::GitalyClient::StorageSettings.allow_disk_access do
-            storages << { name: key, path: val.legacy_disk_path }
-          end
+          storages << { name: key, path: storage_paths[key] }
         end
 
         if Rails.env.test?
@@ -44,12 +43,12 @@ module Gitlab
       end
 
       # rubocop:disable Rails/Output
-      def create_gitaly_configuration(dir, force: false)
+      def create_gitaly_configuration(dir, storage_paths, force: false)
         config_path = File.join(dir, 'config.toml')
         FileUtils.rm_f(config_path) if force
 
         File.open(config_path, File::WRONLY | File::CREAT | File::EXCL) do |f|
-          f.puts gitaly_configuration_toml(dir)
+          f.puts gitaly_configuration_toml(dir, storage_paths)
         end
       rescue Errno::EEXIST
         puts "Skipping config.toml generation:"

lib/tasks/gitlab/gitaly.rake

 namespace :gitlab do
   namespace :gitaly do
     desc "GitLab | Install or upgrade gitaly"
-    task :install, [:dir, :repo] => :gitlab_environment do |t, args|
-      require 'toml-rb'
-
+    task :install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args|
       warn_user_is_not_gitlab
 
-      unless args.dir.present?
-        abort %(Please specify the directory where you want to install gitaly:\n  rake "gitlab:gitaly:install[/home/git/gitaly]")
+      unless args.dir.present? && args.storage_path.present?
+        abort %(Please specify the directory where you want to install gitaly and the path for the default storage
+Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]")
       end
 
       args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitaly.git')
@@ -27,7 +26,8 @@ namespace :gitlab do
           "BUNDLE_PATH=#{Bundler.bundle_path}")
       end
 
-      Gitlab::SetupHelper.create_gitaly_configuration(args.dir)
+      storage_paths = { 'default' => args.storage_path }
+      Gitlab::SetupHelper.create_gitaly_configuration(args.dir, storage_paths)
       Dir.chdir(args.dir) do
         # In CI we run scripts/gitaly-test-build instead of this command
         unless ENV['CI'].present?
@@ -35,17 +35,5 @@ namespace :gitlab do
         end
       end
     end
-
-    desc "GitLab | Print storage configuration in TOML format"
-    task storage_config: :environment do
-      require 'toml-rb'
-
-      puts "# Gitaly storage configuration generated from #{Gitlab.config.source} on #{Time.current.to_s(:long)}"
-      puts "# This is in TOML format suitable for use in Gitaly's config.toml file."
-
-      # Exclude gitaly-ruby configuration because that depends on the gitaly
-      # installation directory.
-      puts Gitlab::SetupHelper.gitaly_configuration_toml('', gitaly_ruby: false)
-    end
   end
 end

spec/support/helpers/test_env.rb

@@ -67,6 +67,7 @@ module TestEnv
 
   TMP_TEST_PATH = Rails.root.join('tmp', 'tests', '**')
   REPOS_STORAGE = 'default'.freeze
+  BROKEN_STORAGE = 'broken'.freeze
 
   # Test environment
   #
@@ -157,10 +158,11 @@ module TestEnv
     component_timed_setup('Gitaly',
       install_dir: gitaly_dir,
       version: Gitlab::GitalyClient.expected_server_version,
-      task: "gitlab:gitaly:install[#{gitaly_dir}]") do
+      task: "gitlab:gitaly:install[#{gitaly_dir},#{repos_path}]") do
 
-      # Always re-create config, in case it's outdated. This is fast anyway.
-      Gitlab::SetupHelper.create_gitaly_configuration(gitaly_dir, force: true)
+      # Re-create config, to specify the broken storage path
+      storage_paths = { 'default' => repos_path, 'broken' => broken_path }
+      Gitlab::SetupHelper.create_gitaly_configuration(gitaly_dir, storage_paths, force: true)
 
       start_gitaly(gitaly_dir)
     end
@@ -256,6 +258,10 @@ module TestEnv
     @repos_path ||= Gitlab.config.repositories.storages[REPOS_STORAGE].legacy_disk_path
   end
 
+  def broken_path
+    @broken_path ||= Gitlab.config.repositories.storages[BROKEN_STORAGE].legacy_disk_path
+  end
+
   def backup_path
     Gitlab.config.backup.path
   end

spec/tasks/gitlab/gitaly_rake_spec.rb

@@ -8,13 +8,23 @@ describe 'gitlab:gitaly namespace rake task' do
   describe 'install' do
     let(:repo) { 'https://gitlab.com/gitlab-org/gitaly.git' }
     let(:clone_path) { Rails.root.join('tmp/tests/gitaly').to_s }
+    let(:storage_path) { Rails.root.join('tmp/tests/repositories').to_s }
     let(:version) { File.read(Rails.root.join(Gitlab::GitalyClient::SERVER_VERSION_FILE)).chomp }
 
+    subject { run_rake_task('gitlab:gitaly:install', clone_path, storage_path) }
+
     context 'no dir given' do
       it 'aborts and display a help message' do
         # avoid writing task output to spec progress
         allow($stderr).to receive :write
-        expect { run_rake_task('gitlab:gitaly:install') }.to raise_error /Please specify the directory where you want to install gitaly/
+        expect { run_rake_task('gitlab:gitaly:install') }.to raise_error /Please specify the directory where you want to install gitaly and the path for the default storage/
+      end
+    end
+
+    context 'no storage path given' do
+      it 'aborts and display a help message' do
+        allow($stderr).to receive :write
+        expect { run_rake_task('gitlab:gitaly:install', clone_path) }.to raise_error /Please specify the directory where you want to install gitaly and the path for the default storage/
       end
     end
 
@@ -23,7 +33,7 @@ describe 'gitlab:gitaly namespace rake task' do
         expect(main_object)
           .to receive(:checkout_or_clone_version).and_raise 'Git error'
 
-        expect { run_rake_task('gitlab:gitaly:install', clone_path) }.to raise_error 'Git error'
+        expect { subject }.to raise_error 'Git error'
       end
     end
 
@@ -36,7 +46,7 @@ describe 'gitlab:gitaly namespace rake task' do
         expect(main_object)
           .to receive(:checkout_or_clone_version).with(version: version, repo: repo, target_dir: clone_path)
 
-        run_rake_task('gitlab:gitaly:install', clone_path)
+        subject
       end
     end
 
@@ -59,7 +69,7 @@ describe 'gitlab:gitaly namespace rake task' do
           expect(Gitlab::Popen).to receive(:popen).with(%w[which gmake]).and_return(['/usr/bin/gmake', 0])
           expect(main_object).to receive(:run_command!).with(command_preamble + %w[gmake]).and_return(true)
 
-          run_rake_task('gitlab:gitaly:install', clone_path)
+          subject
         end
       end
 
@@ -72,7 +82,7 @@ describe 'gitlab:gitaly namespace rake task' do
         it 'calls make in the gitaly directory' do
           expect(main_object).to receive(:run_command!).with(command_preamble + %w[make]).and_return(true)
 
-          run_rake_task('gitlab:gitaly:install', clone_path)
+          subject
         end
 
         context 'when Rails.env is test' do
@@ -89,55 +99,10 @@ describe 'gitlab:gitaly namespace rake task' do
           it 'calls make in the gitaly directory with --no-deployment flag for bundle' do
             expect(main_object).to receive(:run_command!).with(command_preamble + command).and_return(true)
 
-            run_rake_task('gitlab:gitaly:install', clone_path)
+            subject
           end
         end
       end
     end
   end
-
-  describe 'storage_config' do
-    it 'prints storage configuration in a TOML format' do
-      config = {
-        'default' => Gitlab::GitalyClient::StorageSettings.new(
-          'path' => '/path/to/default',
-          'gitaly_address' => 'unix:/path/to/my.socket'
-        ),
-        'nfs_01' => Gitlab::GitalyClient::StorageSettings.new(
-          'path' => '/path/to/nfs_01',
-          'gitaly_address' => 'unix:/path/to/my.socket'
-        )
-      }
-      allow(Gitlab.config.repositories).to receive(:storages).and_return(config)
-      allow(Rails.env).to receive(:test?).and_return(false)
-
-      expected_output = ''
-      Timecop.freeze do
-        expected_output = <<~TOML
-          # Gitaly storage configuration generated from #{Gitlab.config.source} on #{Time.current.to_s(:long)}
-          # This is in TOML format suitable for use in Gitaly's config.toml file.
-          bin_dir = "tmp/tests/gitaly"
-          socket_path = "/path/to/my.socket"
-          [gitlab-shell]
-          dir = "#{Gitlab.config.gitlab_shell.path}"
-          [[storage]]
-          name = "default"
-          path = "/path/to/default"
-          [[storage]]
-          name = "nfs_01"
-          path = "/path/to/nfs_01"
-        TOML
-      end
-
-      expect { run_rake_task('gitlab:gitaly:storage_config')}
-        .to output(expected_output).to_stdout
-
-      parsed_output = TomlRB.parse(expected_output)
-      config.each do |name, params|
-        Gitlab::GitalyClient::StorageSettings.allow_disk_access do
-          expect(parsed_output['storage']).to include({ 'name' => name, 'path' => params.legacy_disk_path })
-        end
-      end
-    end
-  end
 end