Commit 085ff109 authored by Lin Jen-Shin's avatar Lin Jen-Shin

Merge branch '118623-remove-db-dependency-from-setup-test-env-job-rc-attempt-2' into 'master'

ci: Make TestEnv not depend on Rails nor a DB connection (take 2)

See merge request gitlab-org/gitlab!63432
parents 581f7a4b 7a85e798
...@@ -149,13 +149,16 @@ setup-test-env: ...@@ -149,13 +149,16 @@ setup-test-env:
- .rails-job-base - .rails-job-base
- .setup-test-env-cache - .setup-test-env-cache
- .rails:rules:code-backstage-qa - .rails:rules:code-backstage-qa
- .use-pg12
stage: prepare stage: prepare
variables: variables:
GITLAB_TEST_EAGER_LOAD: "0" GITLAB_TEST_EAGER_LOAD: "0"
SETUP_DB: "false"
script: script:
- run_timed_command "bundle exec ruby -I. -e 'require \"config/environment\"; TestEnv.init'" - run_timed_command "scripts/setup-test-env"
- echo -e "\e[0Ksection_start:`date +%s`:gitaly-test-build[collapsed=true]\r\e[0KCompiling Gitaly binaries"
- run_timed_command "scripts/gitaly-test-build" # Do not use 'bundle exec' here - run_timed_command "scripts/gitaly-test-build" # Do not use 'bundle exec' here
- echo -e "\e[0Ksection_end:`date +%s`:gitaly-test-build\r\e[0K"
artifacts: artifacts:
expire_in: 7d expire_in: 7d
paths: paths:
......
...@@ -2,23 +2,28 @@ ...@@ -2,23 +2,28 @@
module EE module EE
module TestEnv module TestEnv
def init(*args, &blk) extend ::Gitlab::Utils::Override
override :setup_methods
def setup_methods
(super + [:setup_indexer]).freeze
end
override :post_init
def post_init
super super
setup_indexer Settings.elasticsearch['indexer_path'] = indexer_bin_path
end end
def setup_indexer def setup_indexer
indexer_args = [indexer_path, indexer_url].compact.join(',')
component_timed_setup( component_timed_setup(
'GitLab Elasticsearch Indexer', 'GitLab Elasticsearch Indexer',
install_dir: indexer_path, install_dir: indexer_path,
version: indexer_version, version: indexer_version,
task: "gitlab:indexer:install[#{indexer_args}]" task: "gitlab:indexer:install",
task_args: [indexer_path, indexer_url].compact
) )
Settings.elasticsearch['indexer_path'] = indexer_bin_path
end end
def indexer_path def indexer_path
......
...@@ -120,7 +120,7 @@ module Gitlab ...@@ -120,7 +120,7 @@ module Gitlab
raise "storage #{storage.inspect} is missing a gitaly_address" raise "storage #{storage.inspect} is missing a gitaly_address"
end end
unless URI(address).scheme.in?(%w(tcp unix tls)) unless %w(tcp unix tls).include?(URI(address).scheme)
raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix' or 'tls'" raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix' or 'tls'"
end end
......
...@@ -52,7 +52,7 @@ module Gitlab ...@@ -52,7 +52,7 @@ module Gitlab
@legacy_disk_path = File.expand_path(storage['path'], Rails.root) if storage['path'] @legacy_disk_path = File.expand_path(storage['path'], Rails.root) if storage['path']
storage['path'] = Deprecated storage['path'] = Deprecated
@hash = storage.with_indifferent_access @hash = ActiveSupport::HashWithIndifferentAccess.new(storage)
end end
def gitaly_address def gitaly_address
......
...@@ -3,6 +3,8 @@ ...@@ -3,6 +3,8 @@
# Prevent StateMachine warnings from outputting during a cron task # Prevent StateMachine warnings from outputting during a cron task
StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON'] StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON']
task gitlab_environment: :environment do task :gitlab_environment do
Rake::Task[:environment].invoke unless ENV['SKIP_RAILS_ENV_IN_RAKE']
extend SystemCheck::Helpers extend SystemCheck::Helpers
end end
#!/usr/bin/env ruby
# frozen_string_literal: true
require 'bundler/setup'
require 'request_store'
require 'rake'
require 'active_support/dependencies'
require 'active_support/dependencies/autoload'
require 'active_support/core_ext/numeric'
require 'active_support/string_inquirer'
ENV['SKIP_RAILS_ENV_IN_RAKE'] = 'true'
module Rails
extend self
def root
Pathname.new(File.expand_path('..', __dir__))
end
def env
@_env ||= ActiveSupport::StringInquirer.new(ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "test")
end
end
ActiveSupport::Dependencies.autoload_paths << 'lib'
load File.expand_path('../lib/tasks/gitlab/helpers.rake', __dir__)
load File.expand_path('../lib/tasks/gitlab/gitaly.rake', __dir__)
# Required for config/0_inject_enterprise_edition_module.rb, lib/gitlab/access.rb
require_dependency File.expand_path('../lib/gitlab', __dir__)
require_dependency File.expand_path('../config/initializers/0_inject_enterprise_edition_module', __dir__)
# Require for lib/gitlab/gitaly_client/storage_settings.rb and config/initializers/1_settings.rb
require 'active_support/hash_with_indifferent_access'
# Required for lib/gitlab/visibility_level.rb and lib/gitlab/safe_request_store.rb
require 'active_support/concern'
require 'active_support/core_ext/module/delegation'
# Required for lib/system_check/helpers.rb
require_dependency File.expand_path('../lib/gitlab/task_helpers', __dir__)
# Required for lib/tasks/gitlab/helpers.rake
require_dependency File.expand_path('../lib/system_check/helpers', __dir__)
# Required for config/initializers/1_settings.rb
require 'omniauth'
require 'omniauth-github'
require 'etc'
require_dependency File.expand_path('../lib/gitlab/access', __dir__)
require_dependency File.expand_path('../config/initializers/1_settings', __dir__)
Gitlab.ee do
load File.expand_path('../ee/lib/tasks/gitlab/indexer.rake', __dir__)
require_dependency File.expand_path('../ee/lib/gitlab/elastic/indexer', __dir__)
require_dependency File.expand_path('../lib/gitlab/utils/override', __dir__)
end
require_dependency File.expand_path('../spec/support/helpers/test_env', __dir__)
TestEnv.init
...@@ -230,6 +230,10 @@ RSpec.configure do |config| ...@@ -230,6 +230,10 @@ RSpec.configure do |config|
Gitlab::Database.set_open_transactions_baseline Gitlab::Database.set_open_transactions_baseline
end end
config.append_before do
Thread.current[:current_example_group] = ::RSpec.current_example.metadata[:example_group]
end
config.append_after do config.append_after do
Gitlab::Database.reset_open_transactions_baseline Gitlab::Database.reset_open_transactions_baseline
end end
......
# frozen_string_literal: true # frozen_string_literal: true
require 'parallel'
module TestEnv module TestEnv
extend ActiveSupport::Concern
extend self extend self
ComponentFailedToInstallError = Class.new(StandardError) ComponentFailedToInstallError = Class.new(StandardError)
...@@ -94,50 +95,40 @@ module TestEnv ...@@ -94,50 +95,40 @@ module TestEnv
TMP_TEST_PATH = Rails.root.join('tmp', 'tests').freeze TMP_TEST_PATH = Rails.root.join('tmp', 'tests').freeze
REPOS_STORAGE = 'default' REPOS_STORAGE = 'default'
SECOND_STORAGE_PATH = Rails.root.join('tmp', 'tests', 'second_storage') SECOND_STORAGE_PATH = Rails.root.join('tmp', 'tests', 'second_storage')
SETUP_METHODS = %i[setup_gitaly setup_gitlab_shell setup_workhorse setup_factory_repo setup_forked_repo].freeze
# Can be overriden
def setup_methods
SETUP_METHODS
end
# Test environment # Test environment
# #
# See gitlab.yml.example test section for paths # See gitlab.yml.example test section for paths
# #
def init(opts = {}) def init
unless Rails.env.test? unless Rails.env.test?
puts "\nTestEnv.init can only be run if `RAILS_ENV` is set to 'test' not '#{Rails.env}'!\n" puts "\nTestEnv.init can only be run if `RAILS_ENV` is set to 'test' not '#{Rails.env}'!\n"
exit 1 exit 1
end end
start = Time.now
# Disable mailer for spinach tests # Disable mailer for spinach tests
disable_mailer if opts[:mailer] == false
clean_test_path clean_test_path
setup_gitlab_shell # Install components in parallel as most of the setup is I/O.
Parallel.each(setup_methods) do |method|
setup_gitaly public_send(method)
# Feature specs are run through Workhorse
setup_workhorse
# Create repository for FactoryBot.create(:project)
setup_factory_repo
# Create repository for FactoryBot.create(:forked_project_with_submodules)
setup_forked_repo
end
included do |config|
config.append_before do
set_current_example_group
end end
end
def disable_mailer post_init
allow_any_instance_of(NotificationService).to receive(:mailer)
.and_return(double.as_null_object) puts "\nTest environment set up in #{Time.now - start} seconds"
end end
def enable_mailer # Can be overriden
allow_any_instance_of(NotificationService).to receive(:mailer) def post_init
.and_call_original start_gitaly(gitaly_dir)
end end
# Clean /tmp/tests # Clean /tmp/tests
...@@ -164,12 +155,11 @@ module TestEnv ...@@ -164,12 +155,11 @@ module TestEnv
end end
def setup_gitaly def setup_gitaly
install_gitaly_args = [gitaly_dir, repos_path, gitaly_url].compact.join(',')
component_timed_setup('Gitaly', component_timed_setup('Gitaly',
install_dir: gitaly_dir, install_dir: gitaly_dir,
version: Gitlab::GitalyClient.expected_server_version, version: Gitlab::GitalyClient.expected_server_version,
task: "gitlab:gitaly:install[#{install_gitaly_args}]") do task: "gitlab:gitaly:install",
task_args: [gitaly_dir, repos_path, gitaly_url].compact) do
Gitlab::SetupHelper::Gitaly.create_configuration( Gitlab::SetupHelper::Gitaly.create_configuration(
gitaly_dir, gitaly_dir,
{ 'default' => repos_path }, { 'default' => repos_path },
...@@ -190,8 +180,6 @@ module TestEnv ...@@ -190,8 +180,6 @@ module TestEnv
) )
Gitlab::SetupHelper::Praefect.create_configuration(gitaly_dir, { 'praefect' => repos_path }, force: true) Gitlab::SetupHelper::Praefect.create_configuration(gitaly_dir, { 'praefect' => repos_path }, force: true)
end end
start_gitaly(gitaly_dir)
end end
def gitaly_socket_path def gitaly_socket_path
...@@ -273,19 +261,18 @@ module TestEnv ...@@ -273,19 +261,18 @@ module TestEnv
raise "could not connect to #{service} at #{socket.inspect} after #{sleep_time} seconds" raise "could not connect to #{service} at #{socket.inspect} after #{sleep_time} seconds"
end end
# Feature specs are run through Workhorse
def setup_workhorse def setup_workhorse
start = Time.now start = Time.now
return if skip_compile_workhorse? return if skip_compile_workhorse?
puts "\n==> Setting up GitLab Workhorse..."
FileUtils.rm_rf(workhorse_dir) FileUtils.rm_rf(workhorse_dir)
Gitlab::SetupHelper::Workhorse.compile_into(workhorse_dir) Gitlab::SetupHelper::Workhorse.compile_into(workhorse_dir)
Gitlab::SetupHelper::Workhorse.create_configuration(workhorse_dir, nil) Gitlab::SetupHelper::Workhorse.create_configuration(workhorse_dir, nil)
File.write(workhorse_tree_file, workhorse_tree) if workhorse_source_clean? File.write(workhorse_tree_file, workhorse_tree) if workhorse_source_clean?
puts " GitLab Workhorse set up in #{Time.now - start} seconds...\n" puts "==> GitLab Workhorse set up in #{Time.now - start} seconds...\n"
end end
def skip_compile_workhorse? def skip_compile_workhorse?
...@@ -349,10 +336,12 @@ module TestEnv ...@@ -349,10 +336,12 @@ module TestEnv
ENV.fetch('GITLAB_WORKHORSE_URL', nil) ENV.fetch('GITLAB_WORKHORSE_URL', nil)
end end
# Create repository for FactoryBot.create(:project)
def setup_factory_repo def setup_factory_repo
setup_repo(factory_repo_path, factory_repo_path_bare, factory_repo_name, BRANCH_SHA) setup_repo(factory_repo_path, factory_repo_path_bare, factory_repo_name, BRANCH_SHA)
end end
# Create repository for FactoryBot.create(:forked_project_with_submodules)
# This repo has a submodule commit that is not present in the main test # This repo has a submodule commit that is not present in the main test
# repository. # repository.
def setup_forked_repo def setup_forked_repo
...@@ -363,20 +352,18 @@ module TestEnv ...@@ -363,20 +352,18 @@ module TestEnv
clone_url = "https://gitlab.com/gitlab-org/#{repo_name}.git" clone_url = "https://gitlab.com/gitlab-org/#{repo_name}.git"
unless File.directory?(repo_path) unless File.directory?(repo_path)
puts "\n==> Setting up #{repo_name} repository in #{repo_path}..."
start = Time.now start = Time.now
system(*%W(#{Gitlab.config.git.bin_path} clone --quiet -- #{clone_url} #{repo_path})) system(*%W(#{Gitlab.config.git.bin_path} clone --quiet -- #{clone_url} #{repo_path}))
puts " #{repo_path} set up in #{Time.now - start} seconds...\n" puts "==> #{repo_path} set up in #{Time.now - start} seconds...\n"
end end
set_repo_refs(repo_path, refs) set_repo_refs(repo_path, refs)
unless File.directory?(repo_path_bare) unless File.directory?(repo_path_bare)
puts "\n==> Setting up #{repo_name} bare repository in #{repo_path_bare}..."
start = Time.now start = Time.now
# We must copy bare repositories because we will push to them. # We must copy bare repositories because we will push to them.
system(git_env, *%W(#{Gitlab.config.git.bin_path} clone --quiet --bare -- #{repo_path} #{repo_path_bare})) system(git_env, *%W(#{Gitlab.config.git.bin_path} clone --quiet --bare -- #{repo_path} #{repo_path_bare}))
puts " #{repo_path_bare} set up in #{Time.now - start} seconds...\n" puts "==> #{repo_path_bare} set up in #{Time.now - start} seconds...\n"
end end
end end
...@@ -468,10 +455,6 @@ module TestEnv ...@@ -468,10 +455,6 @@ module TestEnv
private private
def set_current_example_group
Thread.current[:current_example_group] = ::RSpec.current_example.metadata[:example_group]
end
# These are directories that should be preserved at cleanup time # These are directories that should be preserved at cleanup time
def test_dirs def test_dirs
@test_dirs ||= %w[ @test_dirs ||= %w[
...@@ -526,7 +509,7 @@ module TestEnv ...@@ -526,7 +509,7 @@ module TestEnv
end end
end end
def component_timed_setup(component, install_dir:, version:, task:) def component_timed_setup(component, install_dir:, version:, task:, task_args: [])
start = Time.now start = Time.now
ensure_component_dir_name_is_correct!(component, install_dir) ensure_component_dir_name_is_correct!(component, install_dir)
...@@ -535,17 +518,22 @@ module TestEnv ...@@ -535,17 +518,22 @@ module TestEnv
return if File.exist?(install_dir) && ci? return if File.exist?(install_dir) && ci?
if component_needs_update?(install_dir, version) if component_needs_update?(install_dir, version)
puts "\n==> Setting up #{component}..."
# Cleanup the component entirely to ensure we start fresh # Cleanup the component entirely to ensure we start fresh
FileUtils.rm_rf(install_dir) FileUtils.rm_rf(install_dir)
unless system('rake', task) if ENV['SKIP_RAILS_ENV_IN_RAKE']
raise ComponentFailedToInstallError # When we run `scripts/setup-test-env`, we take care of loading the necessary dependencies
# so we can run the rake task programmatically.
Rake::Task[task].invoke(*task_args)
else
# In other cases, we run the task via `rake` so that the environment
# and dependencies are automatically loaded.
raise ComponentFailedToInstallError unless system('rake', "#{task}[#{task_args.join(',')}]")
end end
yield if block_given? yield if block_given?
puts " #{component} set up in #{Time.now - start} seconds...\n" puts "==> #{component} set up in #{Time.now - start} seconds...\n"
end end
rescue ComponentFailedToInstallError rescue ComponentFailedToInstallError
puts "\n#{component} failed to install, cleaning up #{install_dir}!\n" puts "\n#{component} failed to install, cleaning up #{install_dir}!\n"
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment