Commit be60d106 authored by Robert Speicher's avatar Robert Speicher

Merge branch 'rc/speedup-rake-jobs' into 'master'

Speedup CI jobs

See merge request gitlab-org/gitlab-ce!14911
parents 2a97550d f4c3e82a
...@@ -35,8 +35,14 @@ variables: ...@@ -35,8 +35,14 @@ variables:
before_script: before_script:
- bundle --version - bundle --version
- date
- source scripts/utils.sh - source scripts/utils.sh
- date
- source scripts/prepare_build.sh - source scripts/prepare_build.sh
- date
after_script:
- date
stages: stages:
- build - build
...@@ -88,6 +94,26 @@ stages: ...@@ -88,6 +94,26 @@ stages:
- /(^docs[\/-].*|.*-docs$)/ - /(^docs[\/-].*|.*-docs$)/
- /(^qa[\/-].*|.*-qa$)/ - /(^qa[\/-].*|.*-qa$)/
# Jobs that only need to pull cache
.dedicated-no-docs-pull-cache-job: &dedicated-no-docs-pull-cache-job
<<: *dedicated-runner
<<: *except-docs-and-qa
<<: *pull-cache
dependencies:
- setup-test-env
stage: test
# Jobs that do not need a DB
.dedicated-no-docs-no-db-pull-cache-job: &dedicated-no-docs-no-db-pull-cache-job
<<: *dedicated-no-docs-pull-cache-job
variables:
SETUP_DB: "false"
.rake-exec: &rake-exec
<<: *dedicated-no-docs-no-db-pull-cache-job
script:
- bundle exec rake $CI_JOB_NAME
.rspec-metadata: &rspec-metadata .rspec-metadata: &rspec-metadata
<<: *dedicated-runner <<: *dedicated-runner
<<: *except-docs-and-qa <<: *except-docs-and-qa
...@@ -164,21 +190,23 @@ stages: ...@@ -164,21 +190,23 @@ stages:
- master@gitlab/gitlabhq - master@gitlab/gitlabhq
- master@gitlab/gitlab-ee - master@gitlab/gitlab-ee
## .gitlab-setup: &gitlab-setup
# Trigger a package build in omnibus-gitlab repository <<: *dedicated-no-docs-pull-cache-job
# <<: *use-pg
package-qa: variables:
<<: *dedicated-runner CREATE_DB_USER: "true"
image: ruby:2.4-alpine
before_script: []
stage: build
cache: {}
when: manual
script: script:
- scripts/trigger-build-omnibus # Manually clone gitlab-test and only seed this project in
only: # db/fixtures/development/04_project.rb thanks to SIZE=1 below
- //@gitlab-org/gitlab-ce - git clone https://gitlab.com/gitlab-org/gitlab-test.git
- //@gitlab-org/gitlab-ee /home/git/repositories/gitlab-org/gitlab-test.git
- scripts/gitaly-test-spawn
- force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup
artifacts:
when: on_failure
expire_in: 1d
paths:
- log/development.log
# Review docs base # Review docs base
.review-docs: &review-docs .review-docs: &review-docs
...@@ -201,6 +229,47 @@ package-qa: ...@@ -201,6 +229,47 @@ package-qa:
only: only:
- branches - branches
# DB migration, rollback, and seed jobs
.db-migrate-reset: &db-migrate-reset
<<: *dedicated-no-docs-pull-cache-job
script:
- bundle exec rake db:migrate:reset
.migration-paths: &migration-paths
<<: *dedicated-no-docs-pull-cache-job
variables:
CREATE_DB_USER: "true"
script:
- git fetch https://gitlab.com/gitlab-org/gitlab-ce.git v9.3.0
- git checkout -f FETCH_HEAD
- bundle install $BUNDLE_INSTALL_FLAGS
- date
- cp config/gitlab.yml.example config/gitlab.yml
- bundle exec rake db:drop db:create db:schema:load db:seed_fu
- date
- git checkout $CI_COMMIT_SHA
- bundle install $BUNDLE_INSTALL_FLAGS
- date
- . scripts/prepare_build.sh
- date
- bundle exec rake db:migrate
##
# Trigger a package build in omnibus-gitlab repository
#
package-qa:
<<: *dedicated-runner
image: ruby:2.4-alpine
before_script: []
stage: build
cache: {}
when: manual
script:
- scripts/trigger-build-omnibus
only:
- //@gitlab-org/gitlab-ce
- //@gitlab-org/gitlab-ee
# Trigger a docs build in gitlab-docs # Trigger a docs build in gitlab-docs
# Useful to preview the docs changes live # Useful to preview the docs changes live
review-docs-deploy: review-docs-deploy:
...@@ -265,7 +334,7 @@ update-tests-metadata: ...@@ -265,7 +334,7 @@ update-tests-metadata:
flaky-examples-check: flaky-examples-check:
<<: *dedicated-runner <<: *dedicated-runner
image: ruby:2.3-alpine image: ruby:2.4-alpine
services: [] services: []
before_script: [] before_script: []
variables: variables:
...@@ -299,7 +368,9 @@ compile-assets: ...@@ -299,7 +368,9 @@ compile-assets:
<<: *default-cache <<: *default-cache
script: script:
- node --version - node --version
- date
- yarn install --frozen-lockfile --cache-folder .yarn-cache - yarn install --frozen-lockfile --cache-folder .yarn-cache
- date
- bundle exec rake gitlab:assets:compile - bundle exec rake gitlab:assets:compile
artifacts: artifacts:
expire_in: 7d expire_in: 7d
...@@ -387,26 +458,11 @@ spinach-pg 1 2: *spinach-metadata-pg ...@@ -387,26 +458,11 @@ spinach-pg 1 2: *spinach-metadata-pg
spinach-mysql 0 2: *spinach-metadata-mysql spinach-mysql 0 2: *spinach-metadata-mysql
spinach-mysql 1 2: *spinach-metadata-mysql spinach-mysql 1 2: *spinach-metadata-mysql
# Static analysis jobs
.ruby-static-analysis: &ruby-static-analysis
variables:
SIMPLECOV: "false"
SETUP_DB: "false"
.rake-exec: &rake-exec
<<: *dedicated-runner
<<: *except-docs-and-qa
<<: *pull-cache
<<: *ruby-static-analysis
stage: test
script:
- bundle exec rake $CI_JOB_NAME
static-analysis: static-analysis:
<<: *dedicated-runner <<: *dedicated-no-docs-no-db-pull-cache-job
<<: *except-docs dependencies:
<<: *ruby-static-analysis - compile-assets
stage: test - setup-test-env
script: script:
- scripts/static-analysis - scripts/static-analysis
cache: cache:
...@@ -463,15 +519,6 @@ ee_compat_check: ...@@ -463,15 +519,6 @@ ee_compat_check:
paths: paths:
- ee_compat_check/patches/*.patch - ee_compat_check/patches/*.patch
# DB migration, rollback, and seed jobs
.db-migrate-reset: &db-migrate-reset
<<: *dedicated-runner
<<: *except-docs-and-qa
<<: *pull-cache
stage: test
script:
- bundle exec rake db:migrate:reset
db:migrate:reset-pg: db:migrate:reset-pg:
<<: *db-migrate-reset <<: *db-migrate-reset
<<: *use-pg <<: *use-pg
...@@ -486,25 +533,6 @@ db:check-schema-pg: ...@@ -486,25 +533,6 @@ db:check-schema-pg:
script: script:
- source scripts/schema_changed.sh - source scripts/schema_changed.sh
.migration-paths: &migration-paths
<<: *dedicated-runner
<<: *except-docs-and-qa
<<: *pull-cache
stage: test
variables:
SETUP_DB: "false"
CREATE_DB_USER: "true"
script:
- git fetch https://gitlab.com/gitlab-org/gitlab-ce.git v9.3.0
- git checkout -f FETCH_HEAD
- bundle install $BUNDLE_INSTALL_FLAGS
- cp config/gitlab.yml.example config/gitlab.yml
- bundle exec rake db:drop db:create db:schema:load db:seed_fu
- git checkout $CI_COMMIT_SHA
- bundle install $BUNDLE_INSTALL_FLAGS
- . scripts/prepare_build.sh
- bundle exec rake db:migrate
migration:path-pg: migration:path-pg:
<<: *migration-paths <<: *migration-paths
<<: *use-pg <<: *use-pg
...@@ -514,10 +542,7 @@ migration:path-mysql: ...@@ -514,10 +542,7 @@ migration:path-mysql:
<<: *use-mysql <<: *use-mysql
.db-rollback: &db-rollback .db-rollback: &db-rollback
<<: *dedicated-runner <<: *dedicated-no-docs-pull-cache-job
<<: *except-docs-and-qa
<<: *pull-cache
stage: test
script: script:
- bundle exec rake db:rollback STEP=119 - bundle exec rake db:rollback STEP=119
- bundle exec rake db:migrate - bundle exec rake db:migrate
...@@ -530,27 +555,6 @@ db:rollback-mysql: ...@@ -530,27 +555,6 @@ db:rollback-mysql:
<<: *db-rollback <<: *db-rollback
<<: *use-mysql <<: *use-mysql
.gitlab-setup: &gitlab-setup
<<: *dedicated-runner
<<: *except-docs-and-qa
<<: *pull-cache
stage: test
variables:
SIZE: "1"
SETUP_DB: "false"
CREATE_DB_USER: "true"
FIXTURE_PATH: db/fixtures/development
script:
- git clone https://gitlab.com/gitlab-org/gitlab-test.git
/home/git/repositories/gitlab-org/gitlab-test.git
- scripts/gitaly-test-spawn
- force=yes bundle exec rake gitlab:setup
artifacts:
when: on_failure
expire_in: 1d
paths:
- log/development.log
gitlab:setup-pg: gitlab:setup-pg:
<<: *gitlab-setup <<: *gitlab-setup
<<: *use-pg <<: *use-pg
...@@ -561,10 +565,7 @@ gitlab:setup-mysql: ...@@ -561,10 +565,7 @@ gitlab:setup-mysql:
# Frontend-related jobs # Frontend-related jobs
gitlab:assets:compile: gitlab:assets:compile:
<<: *dedicated-runner <<: *dedicated-no-docs-no-db-pull-cache-job
<<: *except-docs-and-qa
<<: *pull-cache
stage: test
dependencies: [] dependencies: []
variables: variables:
NODE_ENV: "production" NODE_ENV: "production"
...@@ -574,7 +575,9 @@ gitlab:assets:compile: ...@@ -574,7 +575,9 @@ gitlab:assets:compile:
WEBPACK_REPORT: "true" WEBPACK_REPORT: "true"
NO_COMPRESSION: "true" NO_COMPRESSION: "true"
script: script:
- date
- yarn install --frozen-lockfile --production --cache-folder .yarn-cache - yarn install --frozen-lockfile --production --cache-folder .yarn-cache
- date
- bundle exec rake gitlab:assets:compile - bundle exec rake gitlab:assets:compile
artifacts: artifacts:
name: webpack-report name: webpack-report
...@@ -583,17 +586,16 @@ gitlab:assets:compile: ...@@ -583,17 +586,16 @@ gitlab:assets:compile:
- webpack-report/ - webpack-report/
karma: karma:
<<: *dedicated-runner <<: *dedicated-no-docs-pull-cache-job
<<: *except-docs-and-qa
<<: *pull-cache
<<: *use-pg <<: *use-pg
stage: test dependencies:
variables: - compile-assets
BABEL_ENV: "coverage" - setup-test-env
CHROME_LOG_FILE: "chrome_debug.log"
script: script:
- export BABEL_ENV=coverage CHROME_LOG_FILE=chrome_debug.log
- date
- scripts/gitaly-test-spawn - scripts/gitaly-test-spawn
- bundle exec rake gettext:po_to_json - date
- bundle exec rake karma - bundle exec rake karma
coverage: '/^Statements *: (\d+\.\d+%)/' coverage: '/^Statements *: (\d+\.\d+%)/'
artifacts: artifacts:
...@@ -601,13 +603,11 @@ karma: ...@@ -601,13 +603,11 @@ karma:
expire_in: 31d expire_in: 31d
when: always when: always
paths: paths:
- chrome_debug.log - chrome_debug.log
- coverage-javascript/ - coverage-javascript/
codequality: codequality:
<<: *except-docs <<: *dedicated-no-docs-no-db-pull-cache-job
<<: *pull-cache
stage: test
image: docker:latest image: docker:latest
before_script: [] before_script: []
services: services:
...@@ -639,11 +639,7 @@ sast: ...@@ -639,11 +639,7 @@ sast:
paths: [gl-sast-report.json] paths: [gl-sast-report.json]
qa:internal: qa:internal:
<<: *dedicated-runner <<: *dedicated-no-docs-no-db-pull-cache-job
<<: *except-docs
stage: test
variables:
SETUP_DB: "false"
services: [] services: []
script: script:
- cd qa/ - cd qa/
...@@ -651,11 +647,7 @@ qa:internal: ...@@ -651,11 +647,7 @@ qa:internal:
- bundle exec rspec - bundle exec rspec
qa:selectors: qa:selectors:
<<: *dedicated-runner <<: *dedicated-no-docs-no-db-pull-cache-job
<<: *except-docs
stage: test
variables:
SETUP_DB: "false"
services: [] services: []
script: script:
- cd qa/ - cd qa/
...@@ -663,14 +655,8 @@ qa:selectors: ...@@ -663,14 +655,8 @@ qa:selectors:
- bundle exec bin/qa Test::Sanity::Selectors - bundle exec bin/qa Test::Sanity::Selectors
coverage: coverage:
<<: *dedicated-runner <<: *dedicated-no-docs-no-db-pull-cache-job
<<: *except-docs-and-qa
<<: *pull-cache
stage: post-test stage: post-test
services: []
variables:
SETUP_DB: "false"
USE_BUNDLE_INSTALL: "true"
script: script:
- bundle exec scripts/merge-simplecov - bundle exec scripts/merge-simplecov
coverage: '/LOC \((\d+\.\d+%)\) covered.$/' coverage: '/LOC \((\d+\.\d+%)\) covered.$/'
...@@ -682,26 +668,25 @@ coverage: ...@@ -682,26 +668,25 @@ coverage:
- coverage/assets/ - coverage/assets/
lint:javascript:report: lint:javascript:report:
<<: *dedicated-runner <<: *dedicated-no-docs-no-db-pull-cache-job
<<: *except-docs-and-qa
<<: *pull-cache
stage: post-test stage: post-test
dependencies: dependencies:
- compile-assets - compile-assets
- setup-test-env - setup-test-env
before_script: [] before_script: []
script: script:
- date
- find app/ spec/ -name '*.js' -exec sed --in-place 's|/\* eslint-disable .*\*/||' {} \; # run report over all files - find app/ spec/ -name '*.js' -exec sed --in-place 's|/\* eslint-disable .*\*/||' {} \; # run report over all files
- date
- yarn run eslint-report || true # ignore exit code - yarn run eslint-report || true # ignore exit code
artifacts: artifacts:
name: eslint-report name: eslint-report
expire_in: 31d expire_in: 31d
paths: paths:
- eslint-report.html - eslint-report.html
pages: pages:
<<: *dedicated-runner <<: *dedicated-no-docs-no-db-pull-cache-job
<<: *pull-cache
before_script: [] before_script: []
stage: pages stage: pages
dependencies: dependencies:
...@@ -726,10 +711,7 @@ pages: ...@@ -726,10 +711,7 @@ pages:
# Insurance in case a gem needed by one of our releases gets yanked from # Insurance in case a gem needed by one of our releases gets yanked from
# rubygems.org in the future. # rubygems.org in the future.
cache gems: cache gems:
<<: *dedicated-runner <<: *dedicated-no-docs-no-db-pull-cache-job
<<: *pull-cache
variables:
SETUP_DB: "false"
script: script:
- bundle package --all --all-platforms - bundle package --all --all-platforms
artifacts: artifacts:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment