Commit 0164f495 authored by Toon Claes's avatar Toon Claes

Merge branch 'toon-gitaly-incrementally-again' into 'master'

test: Install Gitaly incrementally (again)

See merge request gitlab-org/gitlab!78061
parents e2aaae9c 7289fa07
......@@ -2,41 +2,6 @@
namespace :gitlab do
namespace :gitaly do
desc 'Installs gitaly for running tests within gitlab-development-kit'
task :test_install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args|
inside_gdk = Rails.env.test? && File.exist?(Rails.root.join('../GDK_ROOT'))
if ENV['FORCE_GITALY_INSTALL'] || !inside_gdk
Rake::Task["gitlab:gitaly:install"].invoke(*args)
next
end
gdk_gitaly_dir = ENV.fetch('GDK_GITALY', Rails.root.join('../gitaly'))
# Our test setup expects a git repo, so clone rather than copy
clone_repo(gdk_gitaly_dir, args.dir, clone_opts: %w[--depth 1]) unless Dir.exist?(args.dir)
# We assume the GDK gitaly already compiled binaries
build_dir = File.join(gdk_gitaly_dir, '_build')
FileUtils.cp_r(build_dir, args.dir)
# We assume the GDK gitaly already ran bundle install
bundle_dir = File.join(gdk_gitaly_dir, 'ruby', '.bundle')
FileUtils.cp_r(bundle_dir, File.join(args.dir, 'ruby'))
# For completeness we copy this for gitaly's make target
ruby_bundle_file = File.join(gdk_gitaly_dir, '.ruby-bundle')
FileUtils.cp_r(ruby_bundle_file, args.dir)
gitaly_binary = File.join(build_dir, 'bin', 'gitaly')
warn_gitaly_out_of_date!(gitaly_binary, Gitlab::GitalyClient.expected_server_version)
rescue Errno::ENOENT => e
puts "Could not copy files, did you run `gdk update`? Error: #{e.message}"
raise
end
desc 'GitLab | Gitaly | Clone and checkout gitaly'
task :clone, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args|
warn_user_is_not_gitlab
......@@ -60,9 +25,6 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]")
storage_paths = { 'default' => args.storage_path }
Gitlab::SetupHelper::Gitaly.create_configuration(args.dir, storage_paths)
# In CI we run scripts/gitaly-test-build
next if ENV['CI'].present?
Dir.chdir(args.dir) do
Bundler.with_original_env do
env = { "RUBYOPT" => nil, "BUNDLE_GEMFILE" => nil }
......
......@@ -13,8 +13,6 @@ class GitalyTestBuild
include GitalySetup
def run
set_bundler_config
# If we have the binaries from the cache, we can skip building them again
if File.exist?(tmp_tests_gitaly_bin_dir)
GitalySetup::LOGGER.debug "Gitaly binary already built. Skip building...\n"
......
......@@ -9,17 +9,11 @@ class GitalyTestSpawn
include GitalySetup
def run
set_bundler_config
install_gitaly_gems if ENV['CI']
check_gitaly_config!
install_gitaly_gems
# # Uncomment line below to see all gitaly logs merged into CI trace
# spawn('sleep 1; tail -f log/gitaly-test.log')
# In local development this pid file is used by rspec.
IO.write(File.expand_path('../tmp/tests/gitaly.pid', __dir__), start_gitaly)
IO.write(File.expand_path('../tmp/tests/gitaly2.pid', __dir__), start_gitaly2)
IO.write(File.expand_path('../tmp/tests/praefect.pid', __dir__), start_praefect)
# Optionally specify the path to the gitaly config toml as first argument.
# Used by workhorse in test.
spawn_gitaly(ARGV[0])
end
end
......
......@@ -9,8 +9,13 @@
require 'securerandom'
require 'socket'
require 'logger'
require 'bundler'
module GitalySetup
extend self
REPOS_STORAGE = 'default'
LOGGER = begin
default_name = ENV['CI'] ? 'DEBUG' : 'WARN'
level_name = ENV['GITLAB_TESTING_LOG_LEVEL']&.upcase
......@@ -52,11 +57,13 @@ module GitalySetup
def env
{
'HOME' => expand_path('tmp/tests'),
'GEM_PATH' => Gem.path.join(':'),
'BUNDLE_APP_CONFIG' => File.join(gemfile_dir, '.bundle'),
'BUNDLE_INSTALL_FLAGS' => nil,
'BUNDLE_IGNORE_CONFIG' => '1',
'BUNDLE_PATH' => bundle_path,
'BUNDLE_GEMFILE' => gemfile,
'BUNDLE_JOBS' => '4',
'BUNDLE_RETRY' => '3',
'RUBYOPT' => nil,
# Git hooks can't run during tests as the internal API is not running.
......@@ -65,17 +72,20 @@ module GitalySetup
}
end
# rubocop:disable GitlabSecurity/SystemCommandInjection
def set_bundler_config
system('bundle config set --local jobs 4', chdir: gemfile_dir)
system('bundle config set --local retry 3', chdir: gemfile_dir)
def bundle_path
# Allow the user to override BUNDLE_PATH if they need to
return ENV['GITALY_TEST_BUNDLE_PATH'] if ENV['GITALY_TEST_BUNDLE_PATH']
if ENV['CI']
bundle_path = expand_path('vendor/gitaly-ruby')
system('bundle', 'config', 'set', '--local', 'path', bundle_path, chdir: gemfile_dir)
expand_path('vendor/gitaly-ruby')
else
explicit_path = Bundler.configured_bundle_path.explicit_path
return unless explicit_path
expand_path(explicit_path)
end
end
# rubocop:enable GitlabSecurity/SystemCommandInjection
def config_path(service)
case service
......@@ -88,6 +98,10 @@ module GitalySetup
end
end
def repos_path(storage = REPOS_STORAGE)
Gitlab.config.repositories.storages[REPOS_STORAGE].legacy_disk_path
end
def service_binary(service)
case service
when :gitaly, :gitaly2
......@@ -97,16 +111,20 @@ module GitalySetup
end
end
def run_command(cmd, env: {})
system(env, *cmd, exception: true, chdir: tmp_tests_gitaly_dir)
end
def install_gitaly_gems
system(env, "make #{tmp_tests_gitaly_dir}/.ruby-bundle", chdir: tmp_tests_gitaly_dir) # rubocop:disable GitlabSecurity/SystemCommandInjection
run_command(%W[make #{tmp_tests_gitaly_dir}/.ruby-bundle], env: env)
end
def build_gitaly
system(env.merge({ 'GIT_VERSION' => nil }), 'make all git', chdir: tmp_tests_gitaly_dir) # rubocop:disable GitlabSecurity/SystemCommandInjection
run_command(%w[make all git], env: env.merge('GIT_VERSION' => nil))
end
def start_gitaly
start(:gitaly)
def start_gitaly(toml = nil)
start(:gitaly, toml)
end
def start_gitaly2
......@@ -117,14 +135,20 @@ module GitalySetup
start(:praefect)
end
def start(service)
def start(service, toml = nil)
toml ||= config_path(service)
args = ["#{tmp_tests_gitaly_bin_dir}/#{service_binary(service)}"]
args.push("-config") if service == :praefect
args.push(config_path(service))
args.push(toml)
# Ensure user configuration does not affect Git
# Context: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/58776#note_547613780
env = self.env.merge('HOME' => nil, 'XDG_CONFIG_HOME' => nil)
pid = spawn(env, *args, [:out, :err] => "log/#{service}-test.log")
begin
try_connect!(service)
try_connect!(service, toml)
rescue StandardError
Process.kill('TERM', pid)
raise
......@@ -161,29 +185,37 @@ module GitalySetup
abort 'bundle check failed' unless system(env, 'bundle', 'check', out: out, chdir: gemfile_dir)
end
def read_socket_path(service)
def connect_proc(toml)
# This code needs to work in an environment where we cannot use bundler,
# so we cannot easily use the toml-rb gem. This ad-hoc parser should be
# good enough.
config_text = IO.read(config_path(service))
config_text = IO.read(toml)
config_text.lines.each do |line|
match_data = line.match(/^\s*socket_path\s*=\s*"([^"]*)"$/)
match_data = line.match(/^\s*(socket_path|listen_addr)\s*=\s*"([^"]*)"$/)
next unless match_data
return match_data[1] if match_data
case match_data[1]
when 'socket_path'
return -> { UNIXSocket.new(match_data[2]) }
when 'listen_addr'
addr, port = match_data[2].split(':')
return -> { TCPSocket.new(addr, port.to_i) }
end
end
raise "failed to find socket_path in #{config_path(service)}"
raise "failed to find socket_path or listen_addr in #{toml}"
end
def try_connect!(service)
def try_connect!(service, toml)
LOGGER.debug "Trying to connect to #{service}: "
timeout = 20
delay = 0.1
socket = read_socket_path(service)
connect = connect_proc(toml)
Integer(timeout / delay).times do
UNIXSocket.new(socket)
connect.call
LOGGER.debug " OK\n"
return
......@@ -194,6 +226,128 @@ module GitalySetup
LOGGER.warn " FAILED to connect to #{service}\n"
raise "could not connect to #{socket}"
raise "could not connect to #{service}"
end
def gitaly_socket_path
Gitlab::GitalyClient.address(REPOS_STORAGE).delete_prefix('unix:')
end
def gitaly_dir
socket_path = gitaly_socket_path
socket_path = File.expand_path(gitaly_socket_path) if expand_path_for_socket?
File.dirname(socket_path)
end
# Linux fails with "bind: invalid argument" if a UNIX socket path exceeds 108 characters:
# https://github.com/golang/go/issues/6895. We use absolute paths in CI to ensure
# that changes in the current working directory don't affect GRPC reconnections.
def expand_path_for_socket?
!!ENV['CI']
end
def setup_gitaly
unless ENV['CI']
# In CI Gitaly is built in the setup-test-env job and saved in the
# artifacts. So when tests are started, there's no need to build Gitaly.
build_gitaly
end
Gitlab::SetupHelper::Gitaly.create_configuration(
gitaly_dir,
{ 'default' => repos_path },
force: true,
options: {
prometheus_listen_addr: 'localhost:9236'
}
)
Gitlab::SetupHelper::Gitaly.create_configuration(
gitaly_dir,
{ 'default' => repos_path },
force: true,
options: {
internal_socket_dir: File.join(gitaly_dir, "internal_gitaly2"),
gitaly_socket: "gitaly2.socket",
config_filename: "gitaly2.config.toml"
}
)
Gitlab::SetupHelper::Praefect.create_configuration(gitaly_dir, { 'praefect' => repos_path }, force: true)
end
def socket_path(service)
File.join(tmp_tests_gitaly_dir, "#{service}.socket")
end
def praefect_socket_path
"unix:" + socket_path(:praefect)
end
def stop(pid)
Process.kill('KILL', pid)
rescue Errno::ESRCH
# The process can already be gone if the test run was INTerrupted.
end
def spawn_gitaly(toml = nil)
check_gitaly_config!
pids = []
if toml
pids << start_gitaly(toml)
else
pids << start_gitaly
pids << start_gitaly2
pids << start_praefect
end
Kernel.at_exit do
# In CI, this function is called by scripts/gitaly-test-spawn, triggered
# in a before_script. Gitaly needs to remain running until the container
# is stopped.
next if ENV['CI']
# In Workhorse tests (locally or in CI), this function is called by
# scripts/gitaly-test-spawn during `make test`. Gitaly needs to remain
# running until `make test` cleans it up.
next if ENV['GITALY_PID_FILE']
pids.each { |pid| stop(pid) }
end
rescue StandardError
raise gitaly_failure_message
end
def gitaly_failure_message
message = "gitaly spawn failed\n\n"
message += "- The `gitaly` binary does not exist: #{gitaly_binary}\n" unless File.exist?(gitaly_binary)
message += "- The `praefect` binary does not exist: #{praefect_binary}\n" unless File.exist?(praefect_binary)
message += "- The `git` binary does not exist: #{git_binary}\n" unless File.exist?(git_binary)
message += "\nCheck log/gitaly-test.log for errors.\n"
unless ci?
message += "\nIf binaries are missing, try running `make -C tmp/tests/gitaly build git.`\n"
message += "\nOtherwise, try running `rm -rf #{tmp_tests_gitaly_dir}`."
end
message
end
def git_binary
File.join(tmp_tests_gitaly_dir, "_build", "deps", "git", "install", "bin", "git")
end
def gitaly_binary
File.join(tmp_tests_gitaly_dir, "_build", "bin", "gitaly")
end
def praefect_binary
File.join(tmp_tests_gitaly_dir, "_build", "bin", "praefect")
end
def git_binary_exists?
File.exist?(git_binary)
end
end
# frozen_string_literal: true
require 'parallel'
require_relative 'gitaly_setup'
module TestEnv
extend self
......@@ -93,7 +94,6 @@ module TestEnv
}.freeze
TMP_TEST_PATH = Rails.root.join('tmp', 'tests').freeze
REPOS_STORAGE = 'default'
SECOND_STORAGE_PATH = Rails.root.join('tmp', 'tests', 'second_storage')
SETUP_METHODS = %i[setup_gitaly setup_gitlab_shell setup_workhorse setup_factory_repo setup_forked_repo].freeze
......@@ -128,7 +128,7 @@ module TestEnv
# Can be overriden
def post_init
start_gitaly(gitaly_dir)
start_gitaly
end
# Clean /tmp/tests
......@@ -142,7 +142,7 @@ module TestEnv
end
FileUtils.mkdir_p(
Gitlab::GitalyClient::StorageSettings.allow_disk_access { TestEnv.repos_path }
Gitlab::GitalyClient::StorageSettings.allow_disk_access { GitalySetup.repos_path }
)
FileUtils.mkdir_p(SECOND_STORAGE_PATH)
FileUtils.mkdir_p(backup_path)
......@@ -159,109 +159,28 @@ module TestEnv
def setup_gitaly
component_timed_setup('Gitaly',
install_dir: gitaly_dir,
install_dir: GitalySetup.gitaly_dir,
version: Gitlab::GitalyClient.expected_server_version,
task: "gitlab:gitaly:test_install",
task_args: [gitaly_dir, repos_path, gitaly_url].compact) do
Gitlab::SetupHelper::Gitaly.create_configuration(
gitaly_dir,
{ 'default' => repos_path },
force: true,
options: {
prometheus_listen_addr: 'localhost:9236'
}
)
Gitlab::SetupHelper::Gitaly.create_configuration(
gitaly_dir,
{ 'default' => repos_path },
force: true,
options: {
internal_socket_dir: File.join(gitaly_dir, "internal_gitaly2"),
gitaly_socket: "gitaly2.socket",
config_filename: "gitaly2.config.toml"
}
)
Gitlab::SetupHelper::Praefect.create_configuration(gitaly_dir, { 'praefect' => repos_path }, force: true)
end
task: "gitlab:gitaly:clone",
fresh_install: ENV.key?('FORCE_GITALY_INSTALL'),
task_args: [GitalySetup.gitaly_dir, GitalySetup.repos_path, gitaly_url].compact) do
GitalySetup.setup_gitaly
end
def gitaly_socket_path
Gitlab::GitalyClient.address('default').sub(/\Aunix:/, '')
end
def gitaly_dir
socket_path = gitaly_socket_path
socket_path = File.expand_path(gitaly_socket_path) if expand_path?
File.dirname(socket_path)
end
# Linux fails with "bind: invalid argument" if a UNIX socket path exceeds 108 characters:
# https://github.com/golang/go/issues/6895. We use absolute paths in CI to ensure
# that changes in the current working directory don't affect GRPC reconnections.
def expand_path?
!!ENV['CI']
end
def start_gitaly(gitaly_dir)
def start_gitaly
if ci?
# Gitaly has been spawned outside this process already
return
end
spawn_script = Rails.root.join('scripts/gitaly-test-spawn').to_s
Bundler.with_original_env do
unless system(spawn_script)
raise gitaly_failure_message
end
end
gitaly_pid = Integer(File.read(TMP_TEST_PATH.join('gitaly.pid')))
gitaly2_pid = Integer(File.read(TMP_TEST_PATH.join('gitaly2.pid')))
praefect_pid = Integer(File.read(TMP_TEST_PATH.join('praefect.pid')))
Kernel.at_exit do
pids = [gitaly_pid, gitaly2_pid, praefect_pid]
pids.each { |pid| stop(pid) }
end
wait('gitaly')
wait('praefect')
end
def stop(pid)
Process.kill('KILL', pid)
rescue Errno::ESRCH
# The process can already be gone if the test run was INTerrupted.
GitalySetup.spawn_gitaly
end
def gitaly_url
ENV.fetch('GITALY_REPO_URL', nil)
end
def socket_path(service)
TMP_TEST_PATH.join('gitaly', "#{service}.socket").to_s
end
def praefect_socket_path
"unix:" + socket_path(:praefect)
end
def wait(service)
sleep_time = 10
sleep_interval = 0.1
socket = socket_path(service)
Integer(sleep_time / sleep_interval).times do
Socket.unix(socket)
return
rescue StandardError
sleep sleep_interval
end
raise "could not connect to #{service} at #{socket.inspect} after #{sleep_time} seconds"
end
# Feature specs are run through Workhorse
def setup_workhorse
# Always rebuild the config file
......@@ -377,8 +296,7 @@ module TestEnv
def rm_storage_dir(storage, dir)
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
repos_path = Gitlab.config.repositories.storages[storage].legacy_disk_path
target_repo_refs_path = File.join(repos_path, dir)
target_repo_refs_path = File.join(GitalySetup.repos_path(storage), dir)
FileUtils.remove_dir(target_repo_refs_path)
end
rescue Errno::ENOENT
......@@ -386,8 +304,7 @@ module TestEnv
def storage_dir_exists?(storage, dir)
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
repos_path = Gitlab.config.repositories.storages[storage].legacy_disk_path
File.exist?(File.join(repos_path, dir))
File.exist?(File.join(GitalySetup.repos_path(storage), dir))
end
end
......@@ -400,7 +317,7 @@ module TestEnv
end
def repos_path
@repos_path ||= Gitlab.config.repositories.storages[REPOS_STORAGE].legacy_disk_path
@repos_path ||= GitalySetup.repos_path
end
def backup_path
......@@ -525,7 +442,7 @@ module TestEnv
end
end
def component_timed_setup(component, install_dir:, version:, task:, task_args: [])
def component_timed_setup(component, install_dir:, version:, task:, fresh_install: true, task_args: [])
start = Time.now
ensure_component_dir_name_is_correct!(component, install_dir)
......@@ -535,7 +452,7 @@ module TestEnv
if component_needs_update?(install_dir, version)
# Cleanup the component entirely to ensure we start fresh
FileUtils.rm_rf(install_dir)
FileUtils.rm_rf(install_dir) if fresh_install
if ENV['SKIP_RAILS_ENV_IN_RAKE']
# When we run `scripts/setup-test-env`, we take care of loading the necessary dependencies
......@@ -614,39 +531,6 @@ module TestEnv
expected_version == sha.chomp
end
def gitaly_failure_message
message = "gitaly spawn failed\n\n"
message += "- The `gitaly` binary does not exist: #{gitaly_binary}\n" unless File.exist?(gitaly_binary)
message += "- The `praefect` binary does not exist: #{praefect_binary}\n" unless File.exist?(praefect_binary)
message += "- The `git` binary does not exist: #{git_binary}\n" unless File.exist?(git_binary)
message += "\nCheck log/gitaly-test.log for errors.\n"
unless ci?
message += "\nIf binaries are missing, try running `make -C tmp/tests/gitaly build git.`\n"
message += "\nOtherwise, try running `rm -rf #{gitaly_dir}`."
end
message
end
def git_binary
File.join(gitaly_dir, "_build", "deps", "git", "install", "bin", "git")
end
def gitaly_binary
File.join(gitaly_dir, "_build", "bin", "gitaly")
end
def praefect_binary
File.join(gitaly_dir, "_build", "bin", "praefect")
end
def git_binary_exists?
File.exist?(git_binary)
end
end
require_relative('../../../ee/spec/support/helpers/ee/test_env') if Gitlab.ee?
......
# frozen_string_literal: true
require_relative 'helpers/test_env'
require_relative 'helpers/gitaly_setup'
RSpec.configure do |config|
config.before(:each, :praefect) do
allow(Gitlab.config.repositories.storages['default']).to receive(:[]).and_call_original
allow(Gitlab.config.repositories.storages['default']).to receive(:[]).with('gitaly_address')
.and_return(TestEnv.praefect_socket_path)
.and_return(GitalySetup.praefect_socket_path)
end
end
......@@ -106,7 +106,7 @@ run-gitaly: $(GITALY_PID_FILE)
$(GITALY_PID_FILE): gitaly.toml
$(call message, "Starting gitaly")
cd ..; GITALY_TESTING_NO_GIT_HOOKS=1 GITALY_PID_FILE=workhorse/$(GITALY_PID_FILE) $(GITALY) workhorse/gitaly.toml &
cd ..; GITALY_TESTING_NO_GIT_HOOKS=1 GITALY_PID_FILE=workhorse/$(GITALY_PID_FILE) scripts/gitaly-test-spawn workhorse/gitaly.toml
gitaly.toml: ../tmp/tests/gitaly/config.toml
sed -e 's/^socket_path.*$$/listen_addr = "0.0.0.0:8075"/;s/^\[auth\]$$//;s/^token.*$$//;s/^internal_socket_dir.*$$//' \
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment