test_env.rb 12.8 KB
Newer Older
1
require 'rspec/mocks'
Ken's avatar
Ken committed
2
require 'toml-rb'
3

4 5 6
module TestEnv
  extend self

7 8
  ComponentFailedToInstallError = Class.new(StandardError)

9 10
  # When developing the seed repository, comment out the branch you will modify.
  BRANCH_SHA = {
11
    'signed-commits'                     => '6101e87',
12 13
    'not-merged-branch'                  => 'b83d6e3',
    'branch-merged'                      => '498214d',
14
    'empty-branch'                       => '7efb185',
15
    'ends-with.json'                     => '98b0d8b',
16 17 18 19 20
    'flatten-dir'                        => 'e56497b',
    'feature'                            => '0b4bc9a',
    'feature_conflict'                   => 'bb5206f',
    'fix'                                => '48f0be4',
    'improve/awesome'                    => '5937ac0',
21
    'merged-target'                      => '21751bf',
22
    'markdown'                           => '0ed8c6c',
23
    'lfs'                                => '55bc176',
24
    'master'                             => 'b83d6e3',
25
    'merge-test'                         => '5937ac0',
26 27 28 29 30
    "'test'"                             => 'e56497b',
    'orphaned-branch'                    => '45127a9',
    'binary-encoding'                    => '7b1cf43',
    'gitattributes'                      => '5a62481',
    'expand-collapse-diffs'              => '4842455',
31
    'symlink-expand-diff'                => '81e6355',
32 33
    'expand-collapse-files'              => '025db92',
    'expand-collapse-lines'              => '238e82d',
34 35
    'pages-deploy'                       => '7897d5b',
    'pages-deploy-target'                => '7975be0',
36
    'video'                              => '8879059',
37
    'add-balsamiq-file'                  => 'b89b56d',
38
    'crlf-diff'                          => '5938907',
Sean McGivern's avatar
Sean McGivern committed
39
    'conflict-start'                     => '824be60',
40 41
    'conflict-resolvable'                => '1450cd6',
    'conflict-binary-file'               => '259a6fb',
Sean McGivern's avatar
Sean McGivern committed
42
    'conflict-contains-conflict-markers' => '78a3086',
43
    'conflict-missing-side'              => 'eb227b3',
44
    'conflict-non-utf8'                  => 'd0a293c',
45
    'conflict-too-large'                 => '39fa04f',
46
    'deleted-image-test'                 => '6c17798',
47
    'wip'                                => 'b9238ee',
48
    'csv'                                => '3dd0896',
49
    'v1.1.0'                             => 'b83d6e3',
Phil Hughes's avatar
Phil Hughes committed
50
    'add-ipython-files'                  => '93ee732',
51
    'add-pdf-file'                       => 'e774ebd',
52
    'squash-large-files'                 => '54cec52',
Felipe Artur's avatar
Felipe Artur committed
53
    'add-pdf-text-binary'                => '79faa7b',
54 55
    'add_images_and_changes'             => '010d106',
    'update-gitlab-shell-v-6-0-1'        => '2f61d70',
56
    'update-gitlab-shell-v-6-0-3'        => 'de78448',
57 58 59
    'merge-commit-analyze-before'        => '1adbdef',
    'merge-commit-analyze-side-branch'   => '8a99451',
    'merge-commit-analyze-after'         => '646ece5',
60
    '2-mb-file'                          => 'bf12d25',
61 62 63
    'before-create-delete-modify-move'   => '845009f',
    'between-create-delete-modify-move'  => '3f5f443',
    'after-create-delete-modify-move'    => 'ba3faa7',
64
    'with-codeowners'                    => '219560e',
65 66
    'submodule_inside_folder'            => 'b491b92',
    'png-lfs'                            => 'fe42f41'
Douwe Maan's avatar
Douwe Maan committed
67
  }.freeze
68

69 70 71 72
  # gitlab-test-fork is a fork of gitlab-fork, but we don't necessarily
  # need to keep all the branches in sync.
  # We currently only need a subset of the branches
  FORKED_BRANCH_SHA = {
73 74 75 76
    'add-submodule-version-bump' => '3f547c0',
    'master'                     => '5937ac0',
    'remove-submodule'           => '2a33e0c',
    'conflict-resolvable-fork'   => '404fa3f'
Douwe Maan's avatar
Douwe Maan committed
77
  }.freeze
78

79
  TMP_TEST_PATH = Rails.root.join('tmp', 'tests', '**')
80
  REPOS_STORAGE = 'default'.freeze
81

82 83
  # Test environment
  #
84
  # See gitlab.yml.example test section for paths
85
  #
86
  def init(opts = {})
87 88 89 90 91
    unless Rails.env.test?
      puts "\nTestEnv.init can only be run if `RAILS_ENV` is set to 'test' not '#{Rails.env}'!\n"
      exit 1
    end

92 93 94
    # Disable mailer for spinach tests
    disable_mailer if opts[:mailer] == false

Robert Speicher's avatar
Robert Speicher committed
95
    clean_test_path
96

97
    # Set up GitLab shell for test instance
98 99
    setup_gitlab_shell

100
    setup_gitaly
101

102
    # Create repository for FactoryBot.create(:project)
103
    setup_factory_repo
104

105
    # Create repository for FactoryBot.create(:forked_project_with_submodules)
106
    setup_forked_repo
107 108
  end

109
  def disable_mailer
110 111
    allow_any_instance_of(NotificationService).to receive(:mailer)
      .and_return(double.as_null_object)
112
  end
113

114
  def enable_mailer
115 116
    allow_any_instance_of(NotificationService).to receive(:mailer)
      .and_call_original
117 118
  end

Robert Speicher's avatar
Robert Speicher committed
119 120 121 122
  # Clean /tmp/tests
  #
  # Keeps gitlab-shell and gitlab-test
  def clean_test_path
123
    Dir[TMP_TEST_PATH].each do |entry|
124
      unless File.basename(entry) =~ /\A(gitaly|gitlab-(shell|test|test_bare|test-fork|test-fork_bare))\z/
Robert Speicher's avatar
Robert Speicher committed
125 126 127
        FileUtils.rm_rf(entry)
      end
    end
128 129 130 131

    FileUtils.mkdir_p(repos_path)
    FileUtils.mkdir_p(backup_path)
    FileUtils.mkdir_p(pages_path)
132
    FileUtils.mkdir_p(artifacts_path)
Robert Speicher's avatar
Robert Speicher committed
133 134
  end

135 136 137 138 139 140 141 142
  def clean_gitlab_test_path
    Dir[TMP_TEST_PATH].each do |entry|
      if File.basename(entry) =~ /\A(gitlab-(test|test_bare|test-fork|test-fork_bare))\z/
        FileUtils.rm_rf(entry)
      end
    end
  end

143
  def setup_gitlab_shell
144 145 146 147
    component_timed_setup('GitLab Shell',
      install_dir: Gitlab.config.gitlab_shell.path,
      version: Gitlab::Shell.version_required,
      task: 'gitlab:shell:install')
148 149 150 151 152 153 154 155 156 157

    create_fake_git_hooks
  end

  def create_fake_git_hooks
    # gitlab-shell hooks don't work in our test environment because they try to make internal API calls
    hooks_dir = File.join(Gitlab.config.gitlab_shell.path, 'hooks')
    %w[pre-receive post-receive update].each do |hook|
      File.open(File.join(hooks_dir, hook), 'w', 0755) { |f| f.puts '#!/bin/sh' }
    end
158
  end
159

160
  def setup_gitaly
Jacob Vosmaer's avatar
Jacob Vosmaer committed
161
    socket_path = Gitlab::GitalyClient.address('default').sub(/\Aunix:/, '')
162
    gitaly_dir = File.dirname(socket_path)
163

164 165 166
    component_timed_setup('Gitaly',
      install_dir: gitaly_dir,
      version: Gitlab::GitalyClient.expected_server_version,
167
      task: "gitlab:gitaly:install[#{gitaly_dir},#{repos_path}]") do
168

169 170 171
        Gitlab::SetupHelper.create_gitaly_configuration(gitaly_dir, { 'default' => repos_path }, force: true)
        start_gitaly(gitaly_dir)
      end
172 173
  end

174
  def start_gitaly(gitaly_dir)
Jacob Vosmaer's avatar
Jacob Vosmaer committed
175 176 177 178 179
    if ENV['CI'].present?
      # Gitaly has been spawned outside this process already
      return
    end

180 181
    FileUtils.mkdir_p("tmp/tests/second_storage") unless File.exist?("tmp/tests/second_storage")

Jacob Vosmaer's avatar
Jacob Vosmaer committed
182
    spawn_script = Rails.root.join('scripts/gitaly-test-spawn').to_s
183 184 185 186 187
    Bundler.with_original_env do
      raise "gitaly spawn failed" unless system(spawn_script)
    end
    @gitaly_pid = Integer(File.read('tmp/tests/gitaly.pid'))

188 189
    Kernel.at_exit { stop_gitaly }

190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207
    wait_gitaly
  end

  def wait_gitaly
    sleep_time = 10
    sleep_interval = 0.1
    socket = Gitlab::GitalyClient.address('default').sub('unix:', '')

    Integer(sleep_time / sleep_interval).times do
      begin
        Socket.unix(socket)
        return
      rescue
        sleep sleep_interval
      end
    end

    raise "could not connect to gitaly at #{socket.inspect} after #{sleep_time} seconds"
208 209 210 211 212 213
  end

  def stop_gitaly
    return unless @gitaly_pid

    Process.kill('KILL', @gitaly_pid)
214 215
  rescue Errno::ESRCH
    # The process can already be gone if the test run was INTerrupted.
216 217
  end

218
  def setup_factory_repo
219 220 221 222 223 224 225 226 227 228 229
    setup_repo(factory_repo_path, factory_repo_path_bare, factory_repo_name,
               BRANCH_SHA)
  end

  # This repo has a submodule commit that is not present in the main test
  # repository.
  def setup_forked_repo
    setup_repo(forked_repo_path, forked_repo_path_bare, forked_repo_name,
               FORKED_BRANCH_SHA)
  end

230
  def setup_repo(repo_path, repo_path_bare, repo_name, refs)
231
    clone_url = "https://gitlab.com/gitlab-org/#{repo_name}.git"
232

233
    unless File.directory?(repo_path)
234
      system(*%W(#{Gitlab.config.git.bin_path} clone -q #{clone_url} #{repo_path}))
235 236
    end

237
    set_repo_refs(repo_path, refs)
238

239 240 241 242
    unless File.directory?(repo_path_bare)
      # We must copy bare repositories because we will push to them.
      system(git_env, *%W(#{Gitlab.config.git.bin_path} clone -q --bare #{repo_path} #{repo_path_bare}))
    end
243 244
  end

245
  def copy_repo(project, bare_repo:, refs:)
246 247
    target_repo_path = File.expand_path(repos_path + "/#{project.disk_path}.git")

248
    FileUtils.mkdir_p(target_repo_path)
249
    FileUtils.cp_r("#{File.expand_path(bare_repo)}/.", target_repo_path)
250
    FileUtils.chmod_R 0755, target_repo_path
251
    set_repo_refs(target_repo_path, refs)
252 253
  end

254 255 256 257 258 259 260 261
  def create_bare_repository(path)
    FileUtils.mkdir_p(path)

    system(git_env, *%W(#{Gitlab.config.git.bin_path} -C #{path} init --bare),
           out: '/dev/null',
           err: '/dev/null')
  end

262
  def repos_path
263
    @repos_path ||= Gitlab.config.repositories.storages[REPOS_STORAGE].legacy_disk_path
264
  end
265

266 267 268 269
  def backup_path
    Gitlab.config.backup.path
  end

270 271 272 273
  def pages_path
    Gitlab.config.pages.path
  end

274
  def artifacts_path
275
    Gitlab.config.artifacts.storage_path
276 277
  end

278 279 280 281
  # When no cached assets exist, manually hit the root path to create them
  #
  # Otherwise they'd be created by the first test, often timing out and
  # causing a transient test failure
282
  def eager_load_driver_server
283 284
    return unless defined?(Capybara)

285
    puts "Starting the Capybara driver server..."
286
    Capybara.current_session.visit '/'
287 288
  end

289 290 291 292 293 294 295 296
  def factory_repo_path_bare
    "#{factory_repo_path}_bare"
  end

  def forked_repo_path_bare
    "#{forked_repo_path}_bare"
  end

297 298 299 300 301 302 303 304
  def with_empty_bare_repository(name = nil)
    path = Rails.root.join('tmp/tests', name || 'empty-bare-repository').to_s

    yield(Rugged::Repository.init_at(path, :bare))
  ensure
    FileUtils.rm_rf(path)
  end

305 306 307
  private

  def factory_repo_path
308 309 310
    @factory_repo_path ||= Rails.root.join('tmp', 'tests', factory_repo_name)
  end

311 312 313
  def factory_repo_name
    'gitlab-test'
  end
314

315 316 317 318 319 320 321 322
  def forked_repo_path
    @forked_repo_path ||= Rails.root.join('tmp', 'tests', forked_repo_name)
  end

  def forked_repo_name
    'gitlab-test-fork'
  end

323 324 325
  # Prevent developer git configurations from being persisted to test
  # repositories
  def git_env
326
    { 'GIT_TEMPLATE_DIR' => '' }
327
  end
328 329

  def set_repo_refs(repo_path, branch_sha)
330
    instructions = branch_sha.map { |branch, sha| "update refs/heads/#{branch}\x00#{sha}\x00" }.join("\x00") << "\x00"
331 332
    update_refs = %W(#{Gitlab.config.git.bin_path} update-ref --stdin -z)
    reset = proc do
333 334 335 336
      Dir.chdir(repo_path) do
        IO.popen(update_refs, "w") { |io| io.write(instructions) }
        $?.success?
      end
337 338
    end

339 340 341 342 343 344
    # Try to reset without fetching to avoid using the network.
    unless reset.call
      raise 'Could not fetch test seed repository.' unless system(*%W(#{Gitlab.config.git.bin_path} -C #{repo_path} fetch origin))

      # Before we used Git clone's --mirror option, bare repos could end up
      # with missing refs, clearing them and retrying should fix the issue.
345
      clean_gitlab_test_path && init unless reset.call
346 347
    end
  end
348

349 350 351 352 353 354
  def component_timed_setup(component, install_dir:, version:, task:)
    puts "\n==> Setting up #{component}..."
    start = Time.now

    ensure_component_dir_name_is_correct!(component, install_dir)

355 356 357
    # On CI, once installed, components never need update
    return if File.exist?(install_dir) && ENV['CI']

358 359 360
    if component_needs_update?(install_dir, version)
      # Cleanup the component entirely to ensure we start fresh
      FileUtils.rm_rf(install_dir)
361

362 363 364 365 366 367 368 369 370 371 372 373
      unless system('rake', task)
        raise ComponentFailedToInstallError
      end
    end

    yield if block_given?

  rescue ComponentFailedToInstallError
    puts "\n#{component} failed to install, cleaning up #{install_dir}!\n"
    FileUtils.rm_rf(install_dir)
    exit 1
  ensure
374
    puts "    #{component} set up in #{Time.now - start} seconds...\n"
375 376 377 378 379 380 381 382 383 384 385 386
  end

  def ensure_component_dir_name_is_correct!(component, path)
    actual_component_dir_name = File.basename(path)
    expected_component_dir_name = component.parameterize

    unless actual_component_dir_name == expected_component_dir_name
      puts "    #{component} install dir should be named '#{expected_component_dir_name}', not '#{actual_component_dir_name}' (full install path given was '#{path}')!\n"
      exit 1
    end
  end

387
  def component_needs_update?(component_folder, expected_version)
388 389 390
    # Allow local overrides of the component for tests during development
    return false if Rails.env.test? && File.symlink?(component_folder)

391
    version = File.read(File.join(component_folder, 'VERSION')).strip
392 393 394 395

    # Notice that this will always yield true when using branch versions
    # (`=branch_name`), but that actually makes sure the server is always based
    # on the latest branch revision.
396
    version != expected_version
397 398 399
  rescue Errno::ENOENT
    true
  end
400
end