Merge branch 'knapsack' into 'master'

Use Knapsack to evenly distribute specs

Try to distribute all tests evenly accross all nodes to reduce the testing time of GitLab CE.

This uses high amount of auto-scaled machines.


See merge request !4240

.gitlab-ci.yml

@@ -2,7 +2,7 @@ image: "ruby:2.1"
 
 services:
   - mysql:latest
-  - redis:latest
+  - redis:alpine
 
 cache:
   key: "ruby21"
@@ -13,234 +13,199 @@ variables:
   MYSQL_ALLOW_EMPTY_PASSWORD: "1"
   # retry tests only in CI environment
   RSPEC_RETRY_RETRY_COUNT: "3"
+  RAILS_ENV: "test"
+  SIMPLECOV: "true"
+  USE_DB: "true"
+  USE_BUNDLE_INSTALL: "true"
 
 before_script:
   - source ./scripts/prepare_build.sh
-  - ruby -v
-  - which ruby
-  - retry gem install bundler --no-ri --no-rdoc
   - cp config/gitlab.yml.example config/gitlab.yml
-  - touch log/application.log
-  - touch log/test.log
-  - retry bundle install --without postgres production --jobs $(nproc) "${FLAGS[@]}"
-  - RAILS_ENV=test bundle exec rake db:drop db:create db:schema:load db:migrate
+  - bundle --version
+  - '[ "$USE_BUNDLE_INSTALL" != "true" ] || retry bundle install --without postgres production --jobs $(nproc) "${FLAGS[@]}"'
+  - retry gem install knapsack
+  - '[ "$USE_DB" != "true" ] || bundle exec rake db:drop db:create db:schema:load db:migrate'
 
 stages:
+- prepare
 - test
-- notifications
+- post-test
 
-spec:feature:
-  stage: test
-  script:
-    - RAILS_ENV=test bundle exec rake assets:precompile 2>/dev/null
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:feature
-
-spec:api:
-  stage: test
-  script:
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:api
-
-spec:models:
-  stage: test
-  script:
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:models
-
-spec:lib:
-  stage: test
-  script:
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:lib
-
-spec:services:
-  stage: test
-  script:
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:services
-
-spec:other:
-  stage: test
-  script:
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:other
-
-spinach:project:half:
-  stage: test
-  script:
-    - RAILS_ENV=test bundle exec rake assets:precompile 2>/dev/null
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spinach:project:half
-
-spinach:project:rest:
-  stage: test
-  script:
-    - RAILS_ENV=test bundle exec rake assets:precompile 2>/dev/null
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spinach:project:rest
+# Prepare and merge knapsack tests
 
-spinach:other:
-  stage: test
-  script:
-    - RAILS_ENV=test bundle exec rake assets:precompile 2>/dev/null
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spinach:other
-
-teaspoon:
-  stage: test
-  script:
-    - RAILS_ENV=test bundle exec teaspoon
-
-rubocop:
-  stage: test
-  script:
-    - bundle exec rubocop
-
-scss-lint:
-  stage: test
-  script:
-    - bundle exec rake scss_lint
-
-license-finder:
-  stage: test
-  script:
-    - bundle exec license_finder
-
-brakeman:
-  stage: test
-  script:
-    - bundle exec rake brakeman
-
-flog:
-  stage: test
-  script:
-    - bundle exec rake flog
-
-flay:
-  stage: test
-  script:
-    - bundle exec rake flay
-
-bundler:audit:
-  stage: test
-  only:
-    - master
-  script:
-    - "bundle exec bundle-audit check --update --ignore OSVDB-115941"
-
-db-migrate-reset:
-  stage: test
-  script:
-    - RAILS_ENV=test bundle exec rake db:migrate:reset
-
-# Ruby 2.2 jobs
-
-spec:feature:ruby22:
-  stage: test
-  image: ruby:2.2
-  only:
-    - master
-  script:
-    - RAILS_ENV=test bundle exec rake assets:precompile 2>/dev/null
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:feature
+.knapsack-state: &knapsack-state
+  services: []
+  variables:
+    USE_DB: "false"
+    USE_BUNDLE_INSTALL: "false"
   cache:
-    key: "ruby22"
+    key: "knapsack"
     paths:
-    - vendor
-
-spec:api:ruby22:
-  stage: test
-  image: ruby:2.2
-  only:
-  - master
-  script:
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:api
-  cache:
-    key: "ruby22"
+    - knapsack/
+  artifacts:
     paths:
-    - vendor
+    - knapsack/
 
-spec:models:ruby22:
-  stage: test
-  image: ruby:2.2
-  only:
-  - master
+knapsack:
+  <<: *knapsack-state
+  stage: prepare
   script:
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:models
-  cache:
-    key: "ruby22"
-    paths:
-    - vendor
+    - mkdir -p knapsack/
+    - '[[ -f knapsack/rspec_report.json ]] || echo "{}" > knapsack/rspec_report.json'
+    - '[[ -f knapsack/spinach_report.json ]] || echo "{}" > knapsack/spinach_report.json'
 
-spec:lib:ruby22:
-  stage: test
-  image: ruby:2.2
-  only:
-  - master
+update-knapsack:
+  <<: *knapsack-state
+  stage: post-test
   script:
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:lib
-  cache:
-    key: "ruby22"
-    paths:
-    - vendor
+    - scripts/merge-reports knapsack/rspec_report.json knapsack/rspec_node_*.json
+    - scripts/merge-reports knapsack/spinach_report.json knapsack/spinach_node_*.json
+    - rm -f knapsack/*_node_*.json
 
-spec:services:ruby22:
-  stage: test
-  image: ruby:2.2
-  only:
-  - master
-  script:
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:services
-  cache:
-    key: "ruby22"
-    paths:
-    - vendor
+# Execute all testing suites
 
-spec:other:ruby22:
+.rspec-knapsack: &rspec-knapsack
   stage: test
-  image: ruby:2.2
-  only:
-  - master
   script:
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:other
-  cache:
-    key: "ruby22"
+    - bundle exec rake assets:precompile 2>/dev/null
+    - JOB_NAME=( $CI_BUILD_NAME )
+    - export CI_NODE_INDEX=${JOB_NAME[1]}
+    - export CI_NODE_TOTAL=${JOB_NAME[2]}
+    - export KNAPSACK_REPORT_PATH=knapsack/rspec_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
+    - export KNAPSACK_GENERATE_REPORT=true
+    - cp knapsack/rspec_report.json ${KNAPSACK_REPORT_PATH}
+    - knapsack rspec
+  artifacts:
     paths:
-    - vendor
-
-spinach:project:half:ruby22:
-  stage: test
-  image: ruby:2.2
-  only:
-  - master
-  script:
-    - RAILS_ENV=test bundle exec rake assets:precompile 2>/dev/null
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spinach:project:half
-  cache:
-    key: "ruby22"
+    - knapsack/
+
+.spinach-knapsack: &spinach-knapsack
+  stage: test
+  script:
+    - bundle exec rake assets:precompile 2>/dev/null
+    - JOB_NAME=( $CI_BUILD_NAME )
+    - export CI_NODE_INDEX=${JOB_NAME[1]}
+    - export CI_NODE_TOTAL=${JOB_NAME[2]}
+    - export KNAPSACK_REPORT_PATH=knapsack/spinach_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
+    - export KNAPSACK_GENERATE_REPORT=true
+    - cp knapsack/spinach_report.json ${KNAPSACK_REPORT_PATH}
+    - knapsack spinach "-r rerun"
+    # retry failed tests 3 times
+    - retry '[ ! -e tmp/spinach-rerun.txt ] || bin/spinach -r rerun $(cat tmp/spinach-rerun.txt)'
+  artifacts:
     paths:
-    - vendor
-
-spinach:project:rest:ruby22:
-  stage: test
-  image: ruby:2.2
+    - knapsack/
+
+rspec 0 20: *rspec-knapsack
+rspec 1 20: *rspec-knapsack
+rspec 2 20: *rspec-knapsack
+rspec 3 20: *rspec-knapsack
+rspec 4 20: *rspec-knapsack
+rspec 5 20: *rspec-knapsack
+rspec 6 20: *rspec-knapsack
+rspec 7 20: *rspec-knapsack
+rspec 8 20: *rspec-knapsack
+rspec 9 20: *rspec-knapsack
+rspec 10 20: *rspec-knapsack
+rspec 11 20: *rspec-knapsack
+rspec 12 20: *rspec-knapsack
+rspec 13 20: *rspec-knapsack
+rspec 14 20: *rspec-knapsack
+rspec 15 20: *rspec-knapsack
+rspec 16 20: *rspec-knapsack
+rspec 17 20: *rspec-knapsack
+rspec 18 20: *rspec-knapsack
+rspec 19 20: *rspec-knapsack
+
+spinach 0 10: *spinach-knapsack
+spinach 1 10: *spinach-knapsack
+spinach 2 10: *spinach-knapsack
+spinach 3 10: *spinach-knapsack
+spinach 4 10: *spinach-knapsack
+spinach 5 10: *spinach-knapsack
+spinach 6 10: *spinach-knapsack
+spinach 7 10: *spinach-knapsack
+spinach 8 10: *spinach-knapsack
+spinach 9 10: *spinach-knapsack
+
+# Execute all testing suites against Ruby 2.2
+
+.ruby-22: &ruby-22
+  image: "ruby:2.2"
   only:
     - master
-  script:
-    - RAILS_ENV=test bundle exec rake assets:precompile 2>/dev/null
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spinach:project:rest
   cache:
     key: "ruby22"
     paths:
     - vendor
 
-spinach:other:ruby22:
+.rspec-knapsack-ruby22: &rspec-knapsack-ruby22
+  <<: *rspec-knapsack
+  <<: *ruby-22
+
+.spinach-knapsack-ruby22: &spinach-knapsack-ruby22
+  <<: *rspec-knapsack
+  <<: *ruby-22
+  
+rspec 0 20 ruby22: *rspec-knapsack-ruby22
+rspec 1 20 ruby22: *rspec-knapsack-ruby22
+rspec 2 20 ruby22: *rspec-knapsack-ruby22
+rspec 3 20 ruby22: *rspec-knapsack-ruby22
+rspec 4 20 ruby22: *rspec-knapsack-ruby22
+rspec 5 20 ruby22: *rspec-knapsack-ruby22
+rspec 6 20 ruby22: *rspec-knapsack-ruby22
+rspec 7 20 ruby22: *rspec-knapsack-ruby22
+rspec 8 20 ruby22: *rspec-knapsack-ruby22
+rspec 9 20 ruby22: *rspec-knapsack-ruby22
+rspec 10 20 ruby22: *rspec-knapsack-ruby22
+rspec 11 20 ruby22: *rspec-knapsack-ruby22
+rspec 12 20 ruby22: *rspec-knapsack-ruby22
+rspec 13 20 ruby22: *rspec-knapsack-ruby22
+rspec 14 20 ruby22: *rspec-knapsack-ruby22
+rspec 15 20 ruby22: *rspec-knapsack-ruby22
+rspec 16 20 ruby22: *rspec-knapsack-ruby22
+rspec 17 20 ruby22: *rspec-knapsack-ruby22
+rspec 18 20 ruby22: *rspec-knapsack-ruby22
+rspec 19 20 ruby22: *rspec-knapsack-ruby22
+
+spinach 0 10 ruby22: *spinach-knapsack-ruby22
+spinach 1 10 ruby22: *spinach-knapsack-ruby22
+spinach 2 10 ruby22: *spinach-knapsack-ruby22
+spinach 3 10 ruby22: *spinach-knapsack-ruby22
+spinach 4 10 ruby22: *spinach-knapsack-ruby22
+spinach 5 10 ruby22: *spinach-knapsack-ruby22
+spinach 6 10 ruby22: *spinach-knapsack-ruby22
+spinach 7 10 ruby22: *spinach-knapsack-ruby22
+spinach 8 10 ruby22: *spinach-knapsack-ruby22
+spinach 9 10 ruby22: *spinach-knapsack-ruby22
+
+# Other generic tests
+
+.exec: &exec
+  stage: test
+  script:
+    - bundle exec $CI_BUILD_NAME
+
+teaspoon: *exec
+rubocop: *exec
+rake scss_lint: *exec
+rake brakeman: *exec
+rake flog: *exec
+rake flay: *exec
+rake db:migrate:reset: *exec
+license_finder: *exec
+
+bundler:audit:
   stage: test
-  image: ruby:2.2
   only:
     - master
   script:
-    - RAILS_ENV=test bundle exec rake assets:precompile 2>/dev/null
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spinach:other
-  cache:
-    key: "ruby22"
-    paths:
-    - vendor
+    - "bundle exec bundle-audit check --update --ignore OSVDB-115941"
+
+# Notify slack in the end
 
 notify:slack:
-  stage: notifications
+  stage: post-test
   script:
     - ./scripts/notify_slack.sh "#builds" "Build on \`$CI_BUILD_REF_NAME\` failed! Commit \`$(git log -1 --oneline)\` See <https://gitlab.com/gitlab-org/$(basename "$PWD")/commit/"$CI_BUILD_REF"/builds>"
   when: on_failure

Gemfile

@@ -316,6 +316,7 @@ group :test do
   gem 'webmock', '~> 1.21.0'
   gem 'test_after_commit', '~> 0.4.2'
   gem 'sham_rack'
+  gem 'knapsack'
 end
 
 group :production do

Gemfile.lock

@@ -358,6 +358,9 @@ GEM
       actionpack (>= 3.0.0)
       activesupport (>= 3.0.0)
     kgio (2.10.0)
+    knapsack (1.11.0)
+      rake
+      timecop (>= 0.1.0)
     launchy (2.4.3)
       addressable (~> 2.3)
     letter_opener (1.4.1)
@@ -735,6 +738,7 @@ GEM
     thor (0.19.1)
     thread_safe (0.3.5)
     tilt (2.0.2)
+    timecop (0.8.1)
     timfel-krb5-auth (0.8.3)
     tinder (1.10.1)
       eventmachine (~> 1.0)
@@ -882,6 +886,7 @@ DEPENDENCIES
   jquery-ui-rails (~> 5.0.0)
   jwt
   kaminari (~> 0.17.0)
+  knapsack
   letter_opener_web (~> 1.3.0)
   license_finder
   licensee (~> 8.0.0)

Rakefile

@@ -3,8 +3,11 @@
 # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
 
 require File.expand_path('../config/application', __FILE__)
+require 'knapsack'
 
 relative_url_conf = File.expand_path('../config/initializers/relative_url', __FILE__)
 require relative_url_conf if File.exist?("#{relative_url_conf}.rb")
 
 Gitlab::Application.load_tasks
+
+Knapsack.load_tasks

features/support/env.rb

@@ -11,11 +11,14 @@ ENV['RAILS_ENV'] = 'test'
 require './config/environment'
 require 'rspec/expectations'
 require 'sidekiq/testing/inline'
+require 'knapsack'
 
 require_relative 'capybara'
 require_relative 'db_cleaner'
 require_relative 'rerun'
 
+Knapsack::Adapters::SpinachAdapter.bind
+
 %w(select2_helper test_env repo_helpers).each do |f|
   require Rails.root.join('spec', 'support', f)
 end

scripts/merge-reports

+#!/usr/bin/env ruby
+
+require 'json'
+require 'yaml'
+
+main_report_file = ARGV.shift
+unless main_report_file
+  puts 'usage: merge_reports <main-report> [extra reports...]'
+  exit 1
+end
+
+puts "Loading #{main_report_file}..."
+main_report = JSON.parse(File.read(main_report_file))
+new_report = main_report.dup
+
+ARGV.each do |report_file|
+  report = JSON.parse(File.read(report_file))
+
+  # Remove existing values
+  updates = report.delete_if do |key, value|
+    main_report[key] && main_report[key] == value
+  end
+  new_report.merge!(updates)
+
+  puts "Merged #{report_file} adding #{updates.size} results."
+end
+
+File.write(main_report_file, JSON.pretty_generate(new_report))
+puts "Saved #{main_report_file}."

scripts/prepare_build.sh

 #!/bin/bash
 
 retry() {
-    for i in $(seq 1 3); do
     if eval "$@"; then
         return 0
     fi
+
+    for i in 2 1; do
         sleep 3s
-        echo "Retrying..."
+        echo "Retrying $i..."
+        if eval "$@"; then
+            return 0
+        fi
     done
     return 1
 }

spec/spec_helper.rb

@@ -15,6 +15,9 @@ require 'rspec/rails'
 require 'shoulda/matchers'
 require 'sidekiq/testing/inline'
 require 'rspec/retry'
+require 'knapsack'
+
+Knapsack::Adapters::RSpecAdapter.bind
 
 # Requires supporting ruby files with custom matchers and macros, etc,
 # in spec/support/ and its subdirectories.