test_env.rb 11.3 KB
Newer Older
1
require 'rspec/mocks'
2
require 'toml'
3

4 5 6
module TestEnv
  extend self

7 8
  ComponentFailedToInstallError = Class.new(StandardError)

9 10
  # When developing the seed repository, comment out the branch you will modify.
  BRANCH_SHA = {
11
    'signed-commits'                     => '2d1096e',
12 13
    'not-merged-branch'                  => 'b83d6e3',
    'branch-merged'                      => '498214d',
14
    'empty-branch'                       => '7efb185',
15
    'ends-with.json'                     => '98b0d8b',
16 17 18 19 20
    'flatten-dir'                        => 'e56497b',
    'feature'                            => '0b4bc9a',
    'feature_conflict'                   => 'bb5206f',
    'fix'                                => '48f0be4',
    'improve/awesome'                    => '5937ac0',
21
    'merged-target'                      => '21751bf',
22 23
    'markdown'                           => '0ed8c6c',
    'lfs'                                => 'be93687',
24
    'master'                             => 'b83d6e3',
25
    'merge-test'                         => '5937ac0',
26 27 28 29 30
    "'test'"                             => 'e56497b',
    'orphaned-branch'                    => '45127a9',
    'binary-encoding'                    => '7b1cf43',
    'gitattributes'                      => '5a62481',
    'expand-collapse-diffs'              => '4842455',
31
    'symlink-expand-diff'                => '81e6355',
32 33 34
    'expand-collapse-files'              => '025db92',
    'expand-collapse-lines'              => '238e82d',
    'video'                              => '8879059',
35
    'add-balsamiq-file'                  => 'b89b56d',
36
    'crlf-diff'                          => '5938907',
Sean McGivern's avatar
Sean McGivern committed
37
    'conflict-start'                     => '824be60',
38 39
    'conflict-resolvable'                => '1450cd6',
    'conflict-binary-file'               => '259a6fb',
Sean McGivern's avatar
Sean McGivern committed
40
    'conflict-contains-conflict-markers' => '78a3086',
41
    'conflict-missing-side'              => 'eb227b3',
42
    'conflict-non-utf8'                  => 'd0a293c',
43
    'conflict-too-large'                 => '39fa04f',
44
    'deleted-image-test'                 => '6c17798',
45
    'wip'                                => 'b9238ee',
46
    'csv'                                => '3dd0896',
47
    'v1.1.0'                             => 'b83d6e3',
Phil Hughes's avatar
Phil Hughes committed
48
    'add-ipython-files'                  => '93ee732',
49
    'add-pdf-file'                       => 'e774ebd',
Felipe Artur's avatar
Felipe Artur committed
50 51
    'add-pdf-text-binary'                => '79faa7b',
    'add_images_and_changes'             => '010d106'
Douwe Maan's avatar
Douwe Maan committed
52
  }.freeze
53

54 55 56 57
  # gitlab-test-fork is a fork of gitlab-fork, but we don't necessarily
  # need to keep all the branches in sync.
  # We currently only need a subset of the branches
  FORKED_BRANCH_SHA = {
58 59 60 61
    'add-submodule-version-bump' => '3f547c0',
    'master'                     => '5937ac0',
    'remove-submodule'           => '2a33e0c',
    'conflict-resolvable-fork'   => '404fa3f'
Douwe Maan's avatar
Douwe Maan committed
62
  }.freeze
63

64 65
  TMP_TEST_PATH = Rails.root.join('tmp', 'tests', '**')

66 67
  # Test environment
  #
68
  # See gitlab.yml.example test section for paths
69
  #
70
  def init(opts = {})
71 72 73 74 75
    unless Rails.env.test?
      puts "\nTestEnv.init can only be run if `RAILS_ENV` is set to 'test' not '#{Rails.env}'!\n"
      exit 1
    end

76 77 78
    # Disable mailer for spinach tests
    disable_mailer if opts[:mailer] == false

Robert Speicher's avatar
Robert Speicher committed
79
    clean_test_path
80

81 82 83
    # Setup GitLab shell for test instance
    setup_gitlab_shell

84
    setup_gitaly
85

86
    # Create repository for FactoryBot.create(:project)
87
    setup_factory_repo
88

89
    # Create repository for FactoryBot.create(:forked_project_with_submodules)
90
    setup_forked_repo
91 92
  end

93 94 95 96
  def cleanup
    stop_gitaly
  end

97
  def disable_mailer
98 99
    allow_any_instance_of(NotificationService).to receive(:mailer)
      .and_return(double.as_null_object)
100
  end
101

102
  def enable_mailer
103 104
    allow_any_instance_of(NotificationService).to receive(:mailer)
      .and_call_original
105 106
  end

107
  def disable_pre_receive
108
    allow_any_instance_of(Gitlab::Git::Hook).to receive(:trigger).and_return([true, nil])
109 110
  end

Robert Speicher's avatar
Robert Speicher committed
111 112 113 114
  # Clean /tmp/tests
  #
  # Keeps gitlab-shell and gitlab-test
  def clean_test_path
115
    Dir[TMP_TEST_PATH].each do |entry|
116
      unless File.basename(entry) =~ /\A(gitaly|gitlab-(shell|test|test_bare|test-fork|test-fork_bare))\z/
Robert Speicher's avatar
Robert Speicher committed
117 118 119
        FileUtils.rm_rf(entry)
      end
    end
120 121 122 123

    FileUtils.mkdir_p(repos_path)
    FileUtils.mkdir_p(backup_path)
    FileUtils.mkdir_p(pages_path)
124
    FileUtils.mkdir_p(artifacts_path)
Robert Speicher's avatar
Robert Speicher committed
125 126
  end

127 128 129 130 131 132 133 134
  def clean_gitlab_test_path
    Dir[TMP_TEST_PATH].each do |entry|
      if File.basename(entry) =~ /\A(gitlab-(test|test_bare|test-fork|test-fork_bare))\z/
        FileUtils.rm_rf(entry)
      end
    end
  end

135
  def setup_gitlab_shell
136 137 138 139
    component_timed_setup('GitLab Shell',
      install_dir: Gitlab.config.gitlab_shell.path,
      version: Gitlab::Shell.version_required,
      task: 'gitlab:shell:install')
140
  end
141

142
  def setup_gitaly
Jacob Vosmaer's avatar
Jacob Vosmaer committed
143
    socket_path = Gitlab::GitalyClient.address('default').sub(/\Aunix:/, '')
144
    gitaly_dir = File.dirname(socket_path)
145

146 147 148 149
    component_timed_setup('Gitaly',
      install_dir: gitaly_dir,
      version: Gitlab::GitalyClient.expected_server_version,
      task: "gitlab:gitaly:install[#{gitaly_dir}]") do
150

151 152 153
      # Always re-create config, in case it's outdated. This is fast anyway.
      Gitlab::SetupHelper.create_gitaly_configuration(gitaly_dir, force: true)

154
      start_gitaly(gitaly_dir)
155
    end
156 157
  end

158
  def start_gitaly(gitaly_dir)
Jacob Vosmaer's avatar
Jacob Vosmaer committed
159 160 161 162 163 164 165
    if ENV['CI'].present?
      # Gitaly has been spawned outside this process already
      return
    end

    spawn_script = Rails.root.join('scripts/gitaly-test-spawn').to_s
    @gitaly_pid = Bundler.with_original_env { IO.popen([spawn_script], &:read).to_i }
166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183
    wait_gitaly
  end

  def wait_gitaly
    sleep_time = 10
    sleep_interval = 0.1
    socket = Gitlab::GitalyClient.address('default').sub('unix:', '')

    Integer(sleep_time / sleep_interval).times do
      begin
        Socket.unix(socket)
        return
      rescue
        sleep sleep_interval
      end
    end

    raise "could not connect to gitaly at #{socket.inspect} after #{sleep_time} seconds"
184 185 186 187 188 189
  end

  def stop_gitaly
    return unless @gitaly_pid

    Process.kill('KILL', @gitaly_pid)
190 191
  rescue Errno::ESRCH
    # The process can already be gone if the test run was INTerrupted.
192 193
  end

194
  def setup_factory_repo
195 196 197 198 199 200 201 202 203 204 205
    setup_repo(factory_repo_path, factory_repo_path_bare, factory_repo_name,
               BRANCH_SHA)
  end

  # This repo has a submodule commit that is not present in the main test
  # repository.
  def setup_forked_repo
    setup_repo(forked_repo_path, forked_repo_path_bare, forked_repo_name,
               FORKED_BRANCH_SHA)
  end

206
  def setup_repo(repo_path, repo_path_bare, repo_name, refs)
207
    clone_url = "https://gitlab.com/gitlab-org/#{repo_name}.git"
208

209
    unless File.directory?(repo_path)
210
      system(*%W(#{Gitlab.config.git.bin_path} clone -q #{clone_url} #{repo_path}))
211 212
    end

213
    set_repo_refs(repo_path, refs)
214

215 216 217 218
    unless File.directory?(repo_path_bare)
      # We must copy bare repositories because we will push to them.
      system(git_env, *%W(#{Gitlab.config.git.bin_path} clone -q --bare #{repo_path} #{repo_path_bare}))
    end
219 220
  end

221
  def copy_repo(project, bare_repo:, refs:)
222
    target_repo_path = File.expand_path(project.repository_storage_path + "/#{project.disk_path}.git")
223
    FileUtils.mkdir_p(target_repo_path)
224
    FileUtils.cp_r("#{File.expand_path(bare_repo)}/.", target_repo_path)
225
    FileUtils.chmod_R 0755, target_repo_path
226
    set_repo_refs(target_repo_path, refs)
227 228
  end

229
  def repos_path
230
    Gitlab.config.repositories.storages.default['path']
231
  end
232

233 234 235 236
  def backup_path
    Gitlab.config.backup.path
  end

237 238 239 240
  def pages_path
    Gitlab.config.pages.path
  end

241 242 243 244
  def artifacts_path
    Gitlab.config.artifacts.path
  end

245 246 247 248
  # When no cached assets exist, manually hit the root path to create them
  #
  # Otherwise they'd be created by the first test, often timing out and
  # causing a transient test failure
249
  def eager_load_driver_server
250 251
    return unless defined?(Capybara)

252
    puts "Starting the Capybara driver server..."
253
    Capybara.current_session.visit '/'
254 255
  end

256 257 258 259 260 261 262 263
  def factory_repo_path_bare
    "#{factory_repo_path}_bare"
  end

  def forked_repo_path_bare
    "#{forked_repo_path}_bare"
  end

264 265 266 267 268 269 270 271
  def with_empty_bare_repository(name = nil)
    path = Rails.root.join('tmp/tests', name || 'empty-bare-repository').to_s

    yield(Rugged::Repository.init_at(path, :bare))
  ensure
    FileUtils.rm_rf(path)
  end

272 273 274
  private

  def factory_repo_path
275 276 277
    @factory_repo_path ||= Rails.root.join('tmp', 'tests', factory_repo_name)
  end

278 279 280
  def factory_repo_name
    'gitlab-test'
  end
281

282 283 284 285 286 287 288 289
  def forked_repo_path
    @forked_repo_path ||= Rails.root.join('tmp', 'tests', forked_repo_name)
  end

  def forked_repo_name
    'gitlab-test-fork'
  end

290 291 292
  # Prevent developer git configurations from being persisted to test
  # repositories
  def git_env
293
    { 'GIT_TEMPLATE_DIR' => '' }
294
  end
295 296

  def set_repo_refs(repo_path, branch_sha)
297
    instructions = branch_sha.map { |branch, sha| "update refs/heads/#{branch}\x00#{sha}\x00" }.join("\x00") << "\x00"
298 299
    update_refs = %W(#{Gitlab.config.git.bin_path} update-ref --stdin -z)
    reset = proc do
300 301 302 303
      Dir.chdir(repo_path) do
        IO.popen(update_refs, "w") { |io| io.write(instructions) }
        $?.success?
      end
304 305
    end

306 307 308 309 310 311
    # Try to reset without fetching to avoid using the network.
    unless reset.call
      raise 'Could not fetch test seed repository.' unless system(*%W(#{Gitlab.config.git.bin_path} -C #{repo_path} fetch origin))

      # Before we used Git clone's --mirror option, bare repos could end up
      # with missing refs, clearing them and retrying should fix the issue.
312
      cleanup && clean_gitlab_test_path && init unless reset.call
313 314
    end
  end
315

316 317 318 319 320 321
  def component_timed_setup(component, install_dir:, version:, task:)
    puts "\n==> Setting up #{component}..."
    start = Time.now

    ensure_component_dir_name_is_correct!(component, install_dir)

322 323 324
    # On CI, once installed, components never need update
    return if File.exist?(install_dir) && ENV['CI']

325 326 327
    if component_needs_update?(install_dir, version)
      # Cleanup the component entirely to ensure we start fresh
      FileUtils.rm_rf(install_dir)
328

329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353
      unless system('rake', task)
        raise ComponentFailedToInstallError
      end
    end

    yield if block_given?

  rescue ComponentFailedToInstallError
    puts "\n#{component} failed to install, cleaning up #{install_dir}!\n"
    FileUtils.rm_rf(install_dir)
    exit 1
  ensure
    puts "    #{component} setup in #{Time.now - start} seconds...\n"
  end

  def ensure_component_dir_name_is_correct!(component, path)
    actual_component_dir_name = File.basename(path)
    expected_component_dir_name = component.parameterize

    unless actual_component_dir_name == expected_component_dir_name
      puts "    #{component} install dir should be named '#{expected_component_dir_name}', not '#{actual_component_dir_name}' (full install path given was '#{path}')!\n"
      exit 1
    end
  end

354
  def component_needs_update?(component_folder, expected_version)
355 356 357
    # Allow local overrides of the component for tests during development
    return false if Rails.env.test? && File.symlink?(component_folder)

358
    version = File.read(File.join(component_folder, 'VERSION')).strip
359 360 361 362

    # Notice that this will always yield true when using branch versions
    # (`=branch_name`), but that actually makes sure the server is always based
    # on the latest branch revision.
363
    version != expected_version
364 365 366
  rescue Errno::ENOENT
    true
  end
367
end