Commit d9614e70 authored by Douwe Maan's avatar Douwe Maan

Merge branch 'gitaly-install-path' into 'master'

Remove storage path dependency of gitaly install task

See merge request gitlab-org/gitlab-ce!21101
parents 2e03eac7 eb1a3798
---
title: Remove storage path dependency of gitaly install task
merge_request: 21101
author:
type: changed
...@@ -153,7 +153,7 @@ page](https://golang.org/dl). ...@@ -153,7 +153,7 @@ page](https://golang.org/dl).
# Remove former Go installation folder # Remove former Go installation folder
sudo rm -rf /usr/local/go sudo rm -rf /usr/local/go
curl --remote-name --progress https://dl.google.com/go/go1.10.3.linux-amd64.tar.gz curl --remote-name --progress https://dl.google.com/go/go1.10.3.linux-amd64.tar.gz
echo 'fa1b0e45d3b647c252f51f5e1204aba049cde4af177ef9f2181f43004f901035 go1.10.3.linux-amd64.tar.gz' | shasum -a256 -c - && \ echo 'fa1b0e45d3b647c252f51f5e1204aba049cde4af177ef9f2181f43004f901035 go1.10.3.linux-amd64.tar.gz' | shasum -a256 -c - && \
sudo tar -C /usr/local -xzf go1.10.3.linux-amd64.tar.gz sudo tar -C /usr/local -xzf go1.10.3.linux-amd64.tar.gz
...@@ -494,11 +494,11 @@ Make GitLab start on boot: ...@@ -494,11 +494,11 @@ Make GitLab start on boot:
### Install Gitaly ### Install Gitaly
# Fetch Gitaly source with Git and compile with Go # Fetch Gitaly source with Git and compile with Go
sudo -u git -H bundle exec rake "gitlab:gitaly:install[/home/git/gitaly]" RAILS_ENV=production sudo -u git -H bundle exec rake "gitlab:gitaly:install[/home/git/gitaly,/home/git/repositories]" RAILS_ENV=production
You can specify a different Git repository by providing it as an extra parameter: You can specify a different Git repository by providing it as an extra parameter:
sudo -u git -H bundle exec rake "gitlab:gitaly:install[/home/git/gitaly,https://example.com/gitaly.git]" RAILS_ENV=production sudo -u git -H bundle exec rake "gitlab:gitaly:install[/home/git/gitaly,/home/git/repositories,https://example.com/gitaly.git]" RAILS_ENV=production
Next, make sure gitaly configured: Next, make sure gitaly configured:
......
require 'toml-rb'
module Gitlab module Gitlab
module SetupHelper module SetupHelper
class << self class << self
...@@ -9,7 +11,7 @@ module Gitlab ...@@ -9,7 +11,7 @@ module Gitlab
# because it uses a Unix socket. # because it uses a Unix socket.
# For development and testing purposes, an extra storage is added to gitaly, # For development and testing purposes, an extra storage is added to gitaly,
# which is not known to Rails, but must be explicitly stubbed. # which is not known to Rails, but must be explicitly stubbed.
def gitaly_configuration_toml(gitaly_dir, gitaly_ruby: true) def gitaly_configuration_toml(gitaly_dir, storage_paths, gitaly_ruby: true)
storages = [] storages = []
address = nil address = nil
...@@ -24,10 +26,7 @@ module Gitlab ...@@ -24,10 +26,7 @@ module Gitlab
address = val['gitaly_address'] address = val['gitaly_address']
end end
# https://gitlab.com/gitlab-org/gitaly/issues/1238 storages << { name: key, path: storage_paths[key] }
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
storages << { name: key, path: val.legacy_disk_path }
end
end end
if Rails.env.test? if Rails.env.test?
...@@ -44,12 +43,12 @@ module Gitlab ...@@ -44,12 +43,12 @@ module Gitlab
end end
# rubocop:disable Rails/Output # rubocop:disable Rails/Output
def create_gitaly_configuration(dir, force: false) def create_gitaly_configuration(dir, storage_paths, force: false)
config_path = File.join(dir, 'config.toml') config_path = File.join(dir, 'config.toml')
FileUtils.rm_f(config_path) if force FileUtils.rm_f(config_path) if force
File.open(config_path, File::WRONLY | File::CREAT | File::EXCL) do |f| File.open(config_path, File::WRONLY | File::CREAT | File::EXCL) do |f|
f.puts gitaly_configuration_toml(dir) f.puts gitaly_configuration_toml(dir, storage_paths)
end end
rescue Errno::EEXIST rescue Errno::EEXIST
puts "Skipping config.toml generation:" puts "Skipping config.toml generation:"
......
namespace :gitlab do namespace :gitlab do
namespace :gitaly do namespace :gitaly do
desc "GitLab | Install or upgrade gitaly" desc "GitLab | Install or upgrade gitaly"
task :install, [:dir, :repo] => :gitlab_environment do |t, args| task :install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args|
require 'toml-rb'
warn_user_is_not_gitlab warn_user_is_not_gitlab
unless args.dir.present? unless args.dir.present? && args.storage_path.present?
abort %(Please specify the directory where you want to install gitaly:\n rake "gitlab:gitaly:install[/home/git/gitaly]") abort %(Please specify the directory where you want to install gitaly and the path for the default storage
Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]")
end end
args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitaly.git') args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitaly.git')
...@@ -27,7 +26,8 @@ namespace :gitlab do ...@@ -27,7 +26,8 @@ namespace :gitlab do
"BUNDLE_PATH=#{Bundler.bundle_path}") "BUNDLE_PATH=#{Bundler.bundle_path}")
end end
Gitlab::SetupHelper.create_gitaly_configuration(args.dir) storage_paths = { 'default' => args.storage_path }
Gitlab::SetupHelper.create_gitaly_configuration(args.dir, storage_paths)
Dir.chdir(args.dir) do Dir.chdir(args.dir) do
# In CI we run scripts/gitaly-test-build instead of this command # In CI we run scripts/gitaly-test-build instead of this command
unless ENV['CI'].present? unless ENV['CI'].present?
...@@ -35,17 +35,5 @@ namespace :gitlab do ...@@ -35,17 +35,5 @@ namespace :gitlab do
end end
end end
end end
desc "GitLab | Print storage configuration in TOML format"
task storage_config: :environment do
require 'toml-rb'
puts "# Gitaly storage configuration generated from #{Gitlab.config.source} on #{Time.current.to_s(:long)}"
puts "# This is in TOML format suitable for use in Gitaly's config.toml file."
# Exclude gitaly-ruby configuration because that depends on the gitaly
# installation directory.
puts Gitlab::SetupHelper.gitaly_configuration_toml('', gitaly_ruby: false)
end
end end
end end
...@@ -67,6 +67,7 @@ module TestEnv ...@@ -67,6 +67,7 @@ module TestEnv
TMP_TEST_PATH = Rails.root.join('tmp', 'tests', '**') TMP_TEST_PATH = Rails.root.join('tmp', 'tests', '**')
REPOS_STORAGE = 'default'.freeze REPOS_STORAGE = 'default'.freeze
BROKEN_STORAGE = 'broken'.freeze
# Test environment # Test environment
# #
...@@ -157,10 +158,11 @@ module TestEnv ...@@ -157,10 +158,11 @@ module TestEnv
component_timed_setup('Gitaly', component_timed_setup('Gitaly',
install_dir: gitaly_dir, install_dir: gitaly_dir,
version: Gitlab::GitalyClient.expected_server_version, version: Gitlab::GitalyClient.expected_server_version,
task: "gitlab:gitaly:install[#{gitaly_dir}]") do task: "gitlab:gitaly:install[#{gitaly_dir},#{repos_path}]") do
# Always re-create config, in case it's outdated. This is fast anyway. # Re-create config, to specify the broken storage path
Gitlab::SetupHelper.create_gitaly_configuration(gitaly_dir, force: true) storage_paths = { 'default' => repos_path, 'broken' => broken_path }
Gitlab::SetupHelper.create_gitaly_configuration(gitaly_dir, storage_paths, force: true)
start_gitaly(gitaly_dir) start_gitaly(gitaly_dir)
end end
...@@ -256,6 +258,10 @@ module TestEnv ...@@ -256,6 +258,10 @@ module TestEnv
@repos_path ||= Gitlab.config.repositories.storages[REPOS_STORAGE].legacy_disk_path @repos_path ||= Gitlab.config.repositories.storages[REPOS_STORAGE].legacy_disk_path
end end
def broken_path
@broken_path ||= Gitlab.config.repositories.storages[BROKEN_STORAGE].legacy_disk_path
end
def backup_path def backup_path
Gitlab.config.backup.path Gitlab.config.backup.path
end end
......
...@@ -8,13 +8,23 @@ describe 'gitlab:gitaly namespace rake task' do ...@@ -8,13 +8,23 @@ describe 'gitlab:gitaly namespace rake task' do
describe 'install' do describe 'install' do
let(:repo) { 'https://gitlab.com/gitlab-org/gitaly.git' } let(:repo) { 'https://gitlab.com/gitlab-org/gitaly.git' }
let(:clone_path) { Rails.root.join('tmp/tests/gitaly').to_s } let(:clone_path) { Rails.root.join('tmp/tests/gitaly').to_s }
let(:storage_path) { Rails.root.join('tmp/tests/repositories').to_s }
let(:version) { File.read(Rails.root.join(Gitlab::GitalyClient::SERVER_VERSION_FILE)).chomp } let(:version) { File.read(Rails.root.join(Gitlab::GitalyClient::SERVER_VERSION_FILE)).chomp }
subject { run_rake_task('gitlab:gitaly:install', clone_path, storage_path) }
context 'no dir given' do context 'no dir given' do
it 'aborts and display a help message' do it 'aborts and display a help message' do
# avoid writing task output to spec progress # avoid writing task output to spec progress
allow($stderr).to receive :write allow($stderr).to receive :write
expect { run_rake_task('gitlab:gitaly:install') }.to raise_error /Please specify the directory where you want to install gitaly/ expect { run_rake_task('gitlab:gitaly:install') }.to raise_error /Please specify the directory where you want to install gitaly and the path for the default storage/
end
end
context 'no storage path given' do
it 'aborts and display a help message' do
allow($stderr).to receive :write
expect { run_rake_task('gitlab:gitaly:install', clone_path) }.to raise_error /Please specify the directory where you want to install gitaly and the path for the default storage/
end end
end end
...@@ -23,7 +33,7 @@ describe 'gitlab:gitaly namespace rake task' do ...@@ -23,7 +33,7 @@ describe 'gitlab:gitaly namespace rake task' do
expect(main_object) expect(main_object)
.to receive(:checkout_or_clone_version).and_raise 'Git error' .to receive(:checkout_or_clone_version).and_raise 'Git error'
expect { run_rake_task('gitlab:gitaly:install', clone_path) }.to raise_error 'Git error' expect { subject }.to raise_error 'Git error'
end end
end end
...@@ -36,7 +46,7 @@ describe 'gitlab:gitaly namespace rake task' do ...@@ -36,7 +46,7 @@ describe 'gitlab:gitaly namespace rake task' do
expect(main_object) expect(main_object)
.to receive(:checkout_or_clone_version).with(version: version, repo: repo, target_dir: clone_path) .to receive(:checkout_or_clone_version).with(version: version, repo: repo, target_dir: clone_path)
run_rake_task('gitlab:gitaly:install', clone_path) subject
end end
end end
...@@ -59,7 +69,7 @@ describe 'gitlab:gitaly namespace rake task' do ...@@ -59,7 +69,7 @@ describe 'gitlab:gitaly namespace rake task' do
expect(Gitlab::Popen).to receive(:popen).with(%w[which gmake]).and_return(['/usr/bin/gmake', 0]) expect(Gitlab::Popen).to receive(:popen).with(%w[which gmake]).and_return(['/usr/bin/gmake', 0])
expect(main_object).to receive(:run_command!).with(command_preamble + %w[gmake]).and_return(true) expect(main_object).to receive(:run_command!).with(command_preamble + %w[gmake]).and_return(true)
run_rake_task('gitlab:gitaly:install', clone_path) subject
end end
end end
...@@ -72,7 +82,7 @@ describe 'gitlab:gitaly namespace rake task' do ...@@ -72,7 +82,7 @@ describe 'gitlab:gitaly namespace rake task' do
it 'calls make in the gitaly directory' do it 'calls make in the gitaly directory' do
expect(main_object).to receive(:run_command!).with(command_preamble + %w[make]).and_return(true) expect(main_object).to receive(:run_command!).with(command_preamble + %w[make]).and_return(true)
run_rake_task('gitlab:gitaly:install', clone_path) subject
end end
context 'when Rails.env is test' do context 'when Rails.env is test' do
...@@ -89,55 +99,10 @@ describe 'gitlab:gitaly namespace rake task' do ...@@ -89,55 +99,10 @@ describe 'gitlab:gitaly namespace rake task' do
it 'calls make in the gitaly directory with --no-deployment flag for bundle' do it 'calls make in the gitaly directory with --no-deployment flag for bundle' do
expect(main_object).to receive(:run_command!).with(command_preamble + command).and_return(true) expect(main_object).to receive(:run_command!).with(command_preamble + command).and_return(true)
run_rake_task('gitlab:gitaly:install', clone_path) subject
end end
end end
end end
end end
end end
describe 'storage_config' do
it 'prints storage configuration in a TOML format' do
config = {
'default' => Gitlab::GitalyClient::StorageSettings.new(
'path' => '/path/to/default',
'gitaly_address' => 'unix:/path/to/my.socket'
),
'nfs_01' => Gitlab::GitalyClient::StorageSettings.new(
'path' => '/path/to/nfs_01',
'gitaly_address' => 'unix:/path/to/my.socket'
)
}
allow(Gitlab.config.repositories).to receive(:storages).and_return(config)
allow(Rails.env).to receive(:test?).and_return(false)
expected_output = ''
Timecop.freeze do
expected_output = <<~TOML
# Gitaly storage configuration generated from #{Gitlab.config.source} on #{Time.current.to_s(:long)}
# This is in TOML format suitable for use in Gitaly's config.toml file.
bin_dir = "tmp/tests/gitaly"
socket_path = "/path/to/my.socket"
[gitlab-shell]
dir = "#{Gitlab.config.gitlab_shell.path}"
[[storage]]
name = "default"
path = "/path/to/default"
[[storage]]
name = "nfs_01"
path = "/path/to/nfs_01"
TOML
end
expect { run_rake_task('gitlab:gitaly:storage_config')}
.to output(expected_output).to_stdout
parsed_output = TomlRB.parse(expected_output)
config.each do |name, params|
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
expect(parsed_output['storage']).to include({ 'name' => name, 'path' => params.legacy_disk_path })
end
end
end
end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment