Merge branch 'feature-branch-gather-jobs-data' into 'master'

Reduce the complexity of our CI configuration

Closes #51367 and #65702

See merge request gitlab-org/gitlab-ce!31728

.gitlab-ci.yml

 image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-golang-1.11-git-2.22-chrome-73.0-node-12.x-yarn-1.16-postgresql-9.6-graphicsmagick-1.3.33"
 
+stages:
+  - build
+  - prepare
+  - quick-test
+  - test
+  - review
+  - qa
+  - post-test
+  - pages
+
 variables:
   RAILS_ENV: "test"
   NODE_ENV: "test"
@@ -11,25 +21,9 @@ variables:
   FLAKY_RSPEC_SUITE_REPORT_PATH: rspec_flaky/report-suite.json
   BUILD_ASSETS_IMAGE: "false"
 
-before_script:
-  - date
-  - source scripts/utils.sh
-  - source scripts/prepare_build.sh
-  - date
-
 after_script:
   - date
 
-stages:
-  - build
-  - prepare
-  - merge
-  - test
-  - review
-  - qa
-  - post-test
-  - pages
-
 include:
   - local: .gitlab/ci/global.gitlab-ci.yml
   - local: .gitlab/ci/cng.gitlab-ci.yml

.gitlab/ci/cng.gitlab-ci.yml

 cloud-native-image:
   image: ruby:2.6-alpine
-  before_script: []
   dependencies: []
   stage: post-test
   allow_failure: true
   variables:
     GIT_DEPTH: "1"
-  cache: {}
   when: manual
   script:
     - install_gitlab_gem
     - CNG_PROJECT_PATH="gitlab-org/build/CNG" BUILD_TRIGGER_TOKEN=$CI_JOB_TOKEN ./scripts/trigger-build cng
   only:
+    refs:
       - tags@gitlab-org/gitlab-ce
       - tags@gitlab-org/gitlab-ee

.gitlab/ci/docs.gitlab-ci.yml

-.review-docs: &review-docs
-  extends: .single-script-job-dedicated-runner
+.review-docs:
+  extends:
+    - .default-tags
+    - .default-retry
+  image: ruby:2.6-alpine
+  stage: review
+  dependencies: []
   variables:
-    SCRIPT_NAME: trigger-build-docs
+    GIT_STRATEGY: none
   environment:
     name: review-docs/$CI_COMMIT_REF_SLUG
     # DOCS_REVIEW_APPS_DOMAIN and DOCS_GITLAB_REPO_SUFFIX are CI variables
     # Discussion: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14236/diffs#note_40140693
     url: http://$CI_ENVIRONMENT_SLUG.$DOCS_REVIEW_APPS_DOMAIN/$DOCS_GITLAB_REPO_SUFFIX
     on_stop: review-docs-cleanup
+  before_script:
+    # We don't clone the repo by using GIT_STRATEGY: none and only download the
+    # single script we need here so it's much faster than cloning.
+    - apk add --update openssl
+    - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/trigger-build-docs
+    - chmod 755 trigger-build-docs
 
 # Trigger a manual docs build in gitlab-docs only on non docs-only branches.
 # Useful to preview the docs changes live.
 review-docs-deploy-manual:
   extends:
     - .review-docs
-    - .no-docs-and-no-qa
-  stage: review
+    - .except-docs-qa
   script:
     - gem install gitlab --no-document
-    - ./$SCRIPT_NAME deploy
+    - ./trigger-build-docs deploy
   when: manual
   only:
     - branches@gitlab-org/gitlab-ce
@@ -27,39 +37,40 @@ review-docs-deploy-manual:
 # Always trigger a docs build in gitlab-docs only on docs-only branches.
 # Useful to preview the docs changes live.
 review-docs-deploy:
-  <<: *review-docs
-  stage: review
+  extends:
+    - .review-docs
+    - .except-qa
   script:
     - gem install gitlab --no-document
-    - ./$SCRIPT_NAME deploy
+    - ./trigger-build-docs deploy
   only:
     - /(^docs[\/-].+|.+-docs$)/@gitlab-org/gitlab-ce
     - /(^docs[\/-].+|.+-docs$)/@gitlab-org/gitlab-ee
-  except:
-    - /(^qa[\/-].*|.*-qa$)/
 
 # Cleanup remote environment of gitlab-docs
 review-docs-cleanup:
-  <<: *review-docs
-  stage: review
+  extends:
+    - .review-docs
+    - .except-qa
   environment:
     name: review-docs/$CI_COMMIT_REF_SLUG
     action: stop
   script:
     - gem install gitlab --no-document
-    - ./$SCRIPT_NAME cleanup
+    - ./trigger-build-docs cleanup
   when: manual
   only:
     - branches@gitlab-org/gitlab-ce
     - branches@gitlab-org/gitlab-ee
 
 docs lint:
-  extends: .dedicated-runner
+  extends:
+    - .default-tags
+    - .default-retry
+    - .except-qa
   image: "registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-docs-lint"
   stage: test
-  cache: {}
   dependencies: []
-  before_script: []
   script:
     - scripts/lint-doc.sh
     - mv doc/ /tmp/gitlab-docs/content/$DOCS_GITLAB_REPO_SUFFIX
@@ -72,5 +83,3 @@ docs lint:
     - bundle exec nanoc check internal_links
     # Check the internal anchor links
     - bundle exec nanoc check internal_anchors
-  except:
-    - /(^qa[\/-].*|.*-qa$)/

.gitlab/ci/frontend.gitlab-ci.yml

-.assets-compile-cache: &assets-compile-cache
+.assets-compile-cache:
   cache:
-    key: "assets-compile:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v6"
     paths:
       - vendor/ruby/
       - .yarn-cache/
       - tmp/cache/assets/sprockets
 
-.use-pg: &use-pg
-  services:
-    - name: postgres:9.6.14
-      command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
-    - name: redis:alpine
-
 .gitlab:assets:compile-metadata:
-  <<: *assets-compile-cache
-  extends: .dedicated-no-docs-pull-cache-job
+  extends:
+    - .default-tags
+    - .default-retry
+    - .assets-compile-cache
+    - .default-before_script
+    - .except-docs
   image: dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-git-2.22-chrome-73.0-node-12.x-yarn-1.16-graphicsmagick-1.3.33-docker-18.06.1
-  dependencies:
-    - setup-test-env
+  dependencies: ["setup-test-env"]
   services:
     - docker:19.03.0-dind
   variables:
@@ -30,6 +26,14 @@
     NODE_OPTIONS: --max_old_space_size=3584
     DOCKER_DRIVER: overlay2
     DOCKER_HOST: tcp://docker:2375
+  cache:
+    key: "assets-compile:production:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v6"
+  artifacts:
+    name: webpack-report
+    expire_in: 31d
+    paths:
+      - webpack-report/
+      - public/assets/
   script:
     - node --version
     - retry yarn install --frozen-lockfile --production --cache-folder .yarn-cache --prefer-offline
@@ -42,43 +46,41 @@
     - install_api_client_dependencies_with_apt
     - play_job "review-build-cng" || true  # this job might not exist so ignore the failure if it cannot be played
     - play_job "schedule:review-build-cng" || true  # this job might not exist so ignore the failure if it cannot be played
-  artifacts:
-    name: webpack-report
-    expire_in: 31d
-    paths:
-      - webpack-report/
-      - public/assets/
   only:
     - /.+/@gitlab-org/gitlab-ce
     - /.+/@gitlab-org/gitlab-ee
     - /.+/@gitlab/gitlabhq
     - /.+/@gitlab/gitlab-ee
   tags:
-    - docker
     - gitlab-org
+    - docker
 
 gitlab:assets:compile:
   extends: .gitlab:assets:compile-metadata
-  cache:
-    policy: pull-push
   only:
+    refs:
       - master@gitlab-org/gitlab-ce
       - master@gitlab-org/gitlab-ee
+  cache:
+    policy: pull-push
 
 gitlab:assets:compile pull-cache:
   extends: .gitlab:assets:compile-metadata
-  cache:
-    policy: pull
   except:
     refs:
       - master@gitlab-org/gitlab-ce
       - master@gitlab-org/gitlab-ee
       - /(^docs[\/-].+|.+-docs$)/
+  cache:
+    policy: pull
 
 .compile-assets-metadata:
-  extends: .dedicated-runner
-  <<: *use-pg
-  <<: *assets-compile-cache
+  extends:
+    - .default-tags
+    - .default-retry
+    - .assets-compile-cache
+    - .default-before_script
+    - .use-pg
   stage: prepare
   script:
     - node --version
@@ -89,6 +91,8 @@ gitlab:assets:compile pull-cache:
   variables:
     # we override the max_old_space_size to prevent OOM errors
     NODE_OPTIONS: --max_old_space_size=3584
+  cache:
+    key: "assets-compile:test:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v6"
   artifacts:
     expire_in: 7d
     paths:
@@ -96,30 +100,34 @@ gitlab:assets:compile pull-cache:
       - public/assets
 
 compile-assets:
-  extends: .compile-assets-metadata
-  cache:
-    policy: pull-push
+  extends:
+    - .compile-assets-metadata
   only:
+    refs:
       - master@gitlab-org/gitlab-ce
       - master@gitlab-org/gitlab-ee
+  cache:
+    policy: pull-push
 
 compile-assets pull-cache:
   extends: .compile-assets-metadata
-  cache:
-    policy: pull
   except:
     refs:
       - master@gitlab-org/gitlab-ce
       - master@gitlab-org/gitlab-ee
       - /(^docs[\/-].+|.+-docs$)/
+  cache:
+    policy: pull
 
 karma:
-  extends: .dedicated-no-docs-pull-cache-job
-  <<: *use-pg
-  dependencies:
-    - compile-assets
-    - compile-assets pull-cache
-    - setup-test-env
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .default-before_script
+    - .use-pg
+    - .except-docs
+  dependencies: ["compile-assets", "compile-assets pull-cache", "setup-test-env"]
   variables:
     # we override the max_old_space_size to prevent OOM errors
     NODE_OPTIONS: --max_old_space_size=3584
@@ -142,12 +150,14 @@ karma:
       junit: junit_karma.xml
 
 jest:
-  extends: .dedicated-no-docs-and-no-qa-pull-cache-job
-  <<: *use-pg
-  dependencies:
-    - compile-assets
-    - compile-assets pull-cache
-    - setup-test-env
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .default-before_script
+    - .use-pg
+    - .except-docs-qa
+  dependencies: ["compile-assets", "compile-assets pull-cache", "setup-test-env"]
   script:
     - scripts/gitaly-test-spawn
     - date
@@ -170,36 +180,41 @@ jest:
       - tmp/jest/jest/
     policy: pull-push
 
-qa:internal:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
-  services: []
-  script:
+.qa:
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .except-docs
+  dependencies: ["setup-test-env"]
+  variables:
+    SETUP_DB: "false"
+  before_script:
     - cd qa/
     - bundle install
+
+qa:internal:
+  extends: .qa
+  script:
     - bundle exec rspec
-  dependencies:
-    - setup-test-env
 
 qa:selectors:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
-  services: []
+  extends: .qa
   script:
-    - cd qa/
-    - bundle install
     - bundle exec bin/qa Test::Sanity::Selectors
-  dependencies:
-    - setup-test-env
 
-.qa-frontend-node: &qa-frontend-node
-  extends: .dedicated-no-docs-no-db-pull-cache-job
-  stage: test
+.qa-frontend-node:
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .except-docs
+  dependencies: []
   cache:
     key: "$CI_JOB_NAME"
     paths:
       - .yarn-cache/
     policy: pull-push
-  dependencies: []
-  before_script: []
   script:
     - date
     - yarn install --frozen-lockfile --cache-folder .yarn-cache --prefer-offline
@@ -207,23 +222,28 @@ qa:selectors:
     - yarn run webpack-prod
 
 qa-frontend-node:8:
-  <<: *qa-frontend-node
+  extends: .qa-frontend-node
   image: node:carbon
 
 qa-frontend-node:10:
-  <<: *qa-frontend-node
+  extends: .qa-frontend-node
   image: node:dubnium
 
 qa-frontend-node:latest:
-  <<: *qa-frontend-node
+  extends: .qa-frontend-node
   image: node:latest
   allow_failure: true
 
 lint:javascript:report:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .except-docs
+  variables:
+    SETUP_DB: "false"
   stage: post-test
   dependencies: []
-  before_script: []
   script:
     - date
     - yarn run eslint-report || true  # ignore exit code
@@ -234,12 +254,15 @@ lint:javascript:report:
       - eslint-report.html
 
 jsdoc:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .except-docs
+  variables:
+    SETUP_DB: "false"
   stage: post-test
-  dependencies:
-    - compile-assets
-    - compile-assets pull-cache
-  before_script: []
+  dependencies: ["compile-assets", "compile-assets pull-cache"]
   script:
     - date
     - yarn run jsdoc || true  # ignore exit code

.gitlab/ci/global.gitlab-ci.yml

-.dedicated-runner:
+.default-tags:
+  tags:
+    - gitlab-org
+
+.default-retry:
   retry:
     max: 2  # This is confusing but this means "3 runs at max".
     when:
       - unknown_failure
       - api_failure
       - runner_system_failure
-  tags:
-    - gitlab-org
 
-.default-cache: &default-cache
+.default-before_script:
+  before_script:
+    - date
+    - source scripts/utils.sh
+    - source scripts/prepare_build.sh
+    - date
+
+# Jobs that only need to pull cache
+.default-cache:
+  cache:
     key: "debian-stretch-ruby-2.6.3-node-12.x"
     paths:
       - vendor/ruby
       - .yarn-cache/
       - vendor/gitaly-ruby
-
-.dedicated-runner-default-cache:
-  extends: .dedicated-runner
-  cache:
-    <<: *default-cache
-
-# Jobs that only need to pull cache
-.dedicated-pull-cache-job:
-  extends: .dedicated-runner
-  cache:
-    <<: *default-cache
     policy: pull
-  stage: test
 
-.no-docs:
+.except-docs:
   except:
     refs:
       - /(^docs[\/-].+|.+-docs$)/
 
-.no-docs-and-no-qa:
+.except-qa:
   except:
     refs:
-      - /(^docs[\/-].+|.+-docs$)/
       - /(^qa[\/-].*|.*-qa$)/
 
-.dedicated-no-docs-pull-cache-job:
-  extends:
-    - .dedicated-pull-cache-job
-    - .no-docs
-
-.dedicated-no-docs-and-no-qa-pull-cache-job:
-  extends:
-    - .dedicated-pull-cache-job
-    - .no-docs-and-no-qa
-
-# Jobs that do not need a DB
-.dedicated-no-docs-no-db-pull-cache-job:
-  extends: .dedicated-no-docs-pull-cache-job
-  variables:
-    SETUP_DB: "false"
-
-# Jobs that need a dedicated runner, with no cache
-.dedicated-no-docs:
-  extends:
-    - .dedicated-runner
-    - .no-docs
+.except-docs-qa:
+  except:
+    refs:
+      - /(^docs[\/-].+|.+-docs$)/
+      - /(^qa[\/-].*|.*-qa$)/
 
-.single-script-job-dedicated-runner:
-  extends: .dedicated-runner
-  image: ruby:2.6-alpine
-  stage: test
-  cache: {}
-  dependencies: []
-  variables:
-    GIT_STRATEGY: none
-  before_script:
-    # We don't clone the repo by using GIT_STRATEGY: none and only download the
-    # single script we need here so it's much faster than cloning.
-    - export SCRIPT_NAME="${SCRIPT_NAME:-$CI_JOB_NAME}"
-    - apk add --update openssl
-    - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/$SCRIPT_NAME
-    - chmod 755 $(basename $SCRIPT_NAME)
+.except-docs-qa-geo:
+  except:
+    refs:
+      - /(^docs[\/-].+|.+-docs$)/
+      - /(^qa[\/-].*|.*-qa$)/
+      - /(^geo[\/-].*|.*-geo$)/
 
-.review-only: &review-only
+.review-only:
   only:
     refs:
       - branches@gitlab-org/gitlab-ce
@@ -88,3 +61,16 @@
       - master
       - /^\d+-\d+-auto-deploy-\d+$/
       - /(^docs[\/-].+|.+-docs$)/
+
+.use-pg:
+  services:
+    - name: postgres:9.6.14
+      command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
+    - name: redis:alpine
+
+.use-pg-10:
+  image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-golang-1.11-git-2.22-chrome-73.0-node-12.x-yarn-1.16-postgresql-10-graphicsmagick-1.3.33"
+  services:
+    - name: postgres:10.9
+      command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
+    - name: redis:alpine

.gitlab/ci/memory.gitlab-ci.yml

 memory-static:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .default-before_script
+    - .except-docs
+  variables:
+    SETUP_DB: "false"
   script:
     # Uses two different reports from the 'derailed_benchmars' gem.
 
@@ -23,7 +30,13 @@ memory-static:
 # The application is booted in `production` environment.
 # All tests are run without a webserver (directly using Rack::Mock by default).
 memory-on-boot:
-  extends: .rspec-metadata-pg-10
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .default-before_script
+    - .use-pg-10
+    - .except-docs-qa
   variables:
     NODE_ENV: "production"
     RAILS_ENV: "production"

.gitlab/ci/pages.gitlab-ci.yml

 pages:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
-  before_script: []
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .except-docs
+  only:
+    refs:
+      - master@gitlab-org/gitlab-ce
+      - master@gitlab-org/gitlab-ee
   stage: pages
-  dependencies:
-    - coverage
-    - karma
-    - gitlab:assets:compile
-    - lint:javascript:report
-    - jsdoc
+  dependencies: ["coverage", "karma", "gitlab:assets:compile", "lint:javascript:report", "jsdoc"]
   script:
     - mv public/ .public/
     - mkdir public/
@@ -21,6 +23,3 @@ pages:
   artifacts:
     paths:
       - public
-  only:
-    - master@gitlab-org/gitlab-ce
-    - master@gitlab-org/gitlab-ee

.gitlab/ci/qa.gitlab-ci.yml

 .package-and-qa-base:
   image: ruby:2.6-alpine
   stage: qa
-  before_script: []
   needs: ["build-qa-image", "gitlab:assets:compile pull-cache"]
   dependencies: []
-  cache: {}
   variables:
     GIT_DEPTH: "1"
   retry: 0
@@ -19,7 +17,9 @@
     - master
 
 package-and-qa-manual:
-  extends: .package-and-qa-base
+  extends:
+    - .package-and-qa-base
+    - .except-docs-qa
   when: manual
   except:
     - master

.gitlab/ci/rails.gitlab-ci.yml

-.use-pg: &use-pg
-  services:
-    - name: postgres:9.6.14
-      command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
-    - name: redis:alpine
-
-.use-pg-10: &use-pg-10
-  services:
-    - name: postgres:10.9
-      command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
-    - name: redis:alpine
-
-.only-schedules-master: &only-schedules-master
+.only-schedules-master:
   only:
+    refs:
       - schedules@gitlab-org/gitlab-ce
       - schedules@gitlab-org/gitlab-ee
       - master@gitlab-org/gitlab-ce
@@ -19,34 +8,24 @@
       - master@gitlab/gitlabhq
       - master@gitlab/gitlab-ee
 
-.gitlab-setup: &gitlab-setup
+.rake-exec:
   extends:
-    - .dedicated-no-docs-and-no-qa-pull-cache-job
-    - .use-pg
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .default-before_script
   variables:
     SETUP_DB: "false"
-  script:
-    # Manually clone gitlab-test and only seed this project in
-    # db/fixtures/development/04_project.rb thanks to SIZE=1 below
-    - git clone https://gitlab.com/gitlab-org/gitlab-test.git
-       /home/git/repositories/gitlab-org/gitlab-test.git
-    - scripts/gitaly-test-spawn
-    - force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup
-  artifacts:
-    when: on_failure
-    expire_in: 1d
-    paths:
-      - log/development.log
-
-.rake-exec: &rake-exec
-  extends: .dedicated-no-docs-no-db-pull-cache-job
   script:
     - bundle exec rake $CI_JOB_NAME
 
-.rspec-metadata: &rspec-metadata
+.rspec-base:
   extends:
-    - .dedicated-pull-cache-job
-    - .no-docs-and-no-qa
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .default-before_script
+    - .except-docs-qa
   stage: test
   script:
     - JOB_NAME=( $CI_JOB_NAME )
@@ -83,52 +62,24 @@
     reports:
       junit: junit_rspec.xml
 
-.rspec-metadata-pg: &rspec-metadata-pg
-  <<: *rspec-metadata
-  <<: *use-pg
-
-.rspec-metadata-pg-10: &rspec-metadata-pg-10
-  <<: *rspec-metadata
-  <<: *use-pg-10
-  image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-golang-1.11-git-2.22-chrome-73.0-node-12.x-yarn-1.16-postgresql-10-graphicsmagick-1.3.33"
-
-# DB migration, rollback, and seed jobs
-.db-migrate-reset: &db-migrate-reset
-  extends: .dedicated-no-docs-and-no-qa-pull-cache-job
-  script:
-    - bundle exec rake db:migrate:reset
-  dependencies:
-    - setup-test-env
+.rspec-base-pg:
+  extends:
+    - .rspec-base
+    - .use-pg
 
-.migration-paths: &migration-paths
-  extends: .dedicated-no-docs-and-no-qa-pull-cache-job
-  variables:
-    SETUP_DB: "false"
-  script:
-    - git fetch https://gitlab.com/gitlab-org/gitlab-ce.git v11.11.0
-    - git checkout -f FETCH_HEAD
-    - sed -i "s/gem 'oj', '~> 2.17.4'//" Gemfile
-    - sed -i "s/gem 'bootsnap', '~> 1.0.0'/gem 'bootsnap'/" Gemfile
-    - bundle update google-protobuf grpc bootsnap
-    - bundle install $BUNDLE_INSTALL_FLAGS
-    - date
-    - cp config/gitlab.yml.example config/gitlab.yml
-    - bundle exec rake db:drop db:create db:schema:load db:seed_fu
-    - date
-    - git checkout -f $CI_COMMIT_SHA
-    - bundle install $BUNDLE_INSTALL_FLAGS
-    - date
-    - . scripts/prepare_build.sh
-    - date
-    - bundle exec rake db:migrate
-  dependencies:
-    - setup-test-env
+.rspec-base-pg-10:
+  extends:
+    - .rspec-base
+    - .use-pg-10
 
 setup-test-env:
   extends:
-    - .dedicated-runner-default-cache
-    - .no-docs
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .default-before_script
     - .use-pg
+    - .except-docs
   stage: prepare
   script:
     - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init'
@@ -141,67 +92,72 @@ setup-test-env:
       - vendor/gitaly-ruby
 
 rspec unit pg:
-  <<: *rspec-metadata-pg
+  extends: .rspec-base-pg
   parallel: 20
 
 rspec integration pg:
-  <<: *rspec-metadata-pg
+  extends: .rspec-base-pg
   parallel: 6
 
 rspec system pg:
-  <<: *rspec-metadata-pg
+  extends: .rspec-base-pg
   parallel: 24
 
 rspec unit pg-10:
-  <<: *rspec-metadata-pg-10
-  <<: *only-schedules-master
+  extends:
+    - .rspec-base-pg-10
+    - .only-schedules-master
   parallel: 20
 
 rspec integration pg-10:
-  <<: *rspec-metadata-pg-10
-  <<: *only-schedules-master
+  extends:
+    - .rspec-base-pg-10
+    - .only-schedules-master
   parallel: 6
 
 rspec system pg-10:
-  <<: *rspec-metadata-pg-10
-  <<: *only-schedules-master
+  extends:
+    - .rspec-base-pg-10
+    - .only-schedules-master
   parallel: 24
 
 rspec-fast-spec-helper:
-  <<: *rspec-metadata-pg
+  extends: .rspec-base-pg
   script:
     - bundle exec rspec spec/fast_spec_helper.rb
 
-.rspec-quarantine: &rspec-quarantine
-  <<: *only-schedules-master
+rspec quarantine pg:
+  extends:
+    - .default-before_script
+    - .rspec-base-pg
+    - .only-schedules-master
   script:
-    - export CACHE_CLASSES=true
+    - export NO_KNAPSACK=1 CACHE_CLASSES=true
     - scripts/gitaly-test-spawn
     - bin/rspec --color --format documentation --tag quarantine -- spec/
-
-rspec quarantine pg:
-  <<: *rspec-metadata-pg
-  <<: *rspec-quarantine
   allow_failure: true
 
 static-analysis:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
-  dependencies:
-    - compile-assets
-    - compile-assets pull-cache
-    - setup-test-env
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .default-before_script
+    - .except-docs
+  dependencies: ["setup-test-env", "compile-assets", "compile-assets pull-cache"]
+  variables:
+    SETUP_DB: "false"
   script:
     - scripts/static-analysis
   cache:
-    key: "debian-stretch-ruby-2.6.3-node-12.x-and-rubocop"
+    key: "debian-stretch-ruby-2.6.3-and-rubocop"
     paths:
       - vendor/ruby
-      - .yarn-cache/
       - tmp/rubocop_cache
     policy: pull-push
 
 downtime_check:
-  <<: *rake-exec
+  extends: .rake-exec
   except:
     refs:
       - master
@@ -209,22 +165,20 @@ downtime_check:
       - /^[\d-]+-stable(-ee)?$/
       - /(^docs[\/-].+|.+-docs$)/
       - /(^qa[\/-].*|.*-qa$)/
-  dependencies:
-    - setup-test-env
+  dependencies: ["setup-test-env"]
 
 ee_compat_check:
-  <<: *rake-exec
+  extends: .rake-exec
   dependencies: []
   except:
     refs:
       - master
       - tags
-      - /[\d-]+-stable(-ee)?/
-      - /^security-/
       - branches@gitlab-org/gitlab-ee
       - branches@gitlab/gitlab-ee
+      - /^[\d-]+-stable(-ee)?$/
       - /(^docs[\/-].+|.+-docs$)/
-  retry: 0
+      - /^security-/
   artifacts:
     name: "${CI_JOB_NAME}_${CI_COMIT_REF_NAME}_${CI_COMMIT_SHA}"
     when: always
@@ -232,44 +186,106 @@ ee_compat_check:
     paths:
       - ee_compat_check/patches/*.patch
 
-db:migrate:reset-pg:
-  <<: *db-migrate-reset
-  <<: *use-pg
+# DB migration, rollback, and seed jobs
+db:migrate:reset:
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .default-before_script
+    - .use-pg
+    - .except-docs-qa
+  dependencies: ["setup-test-env"]
+  script:
+    - bundle exec rake db:migrate:reset
 
-db:check-schema-pg:
-  <<: *db-migrate-reset
-  <<: *use-pg
+db:check-schema:
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .default-before_script
+    - .use-pg
+    - .except-docs-qa
+  dependencies: ["setup-test-env"]
   script:
     - source scripts/schema_changed.sh
 
-migration:path-pg:
-  <<: *migration-paths
-  <<: *use-pg
+db:migrate-from-v11.11.0:
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .default-before_script
+    - .use-pg
+    - .except-docs-qa
+  dependencies: ["setup-test-env"]
+  variables:
+    SETUP_DB: "false"
+  script:
+    - git fetch https://gitlab.com/gitlab-org/gitlab-ce.git v11.11.0
+    - git checkout -f FETCH_HEAD
+    - sed -i "s/gem 'oj', '~> 2.17.4'//" Gemfile
+    - sed -i "s/gem 'bootsnap', '~> 1.0.0'/gem 'bootsnap'/" Gemfile
+    - bundle update google-protobuf grpc bootsnap
+    - bundle install $BUNDLE_INSTALL_FLAGS
+    - date
+    - cp config/gitlab.yml.example config/gitlab.yml
+    - bundle exec rake db:drop db:create db:schema:load db:seed_fu
+    - date
+    - git checkout -f $CI_COMMIT_SHA
+    - bundle install $BUNDLE_INSTALL_FLAGS
+    - date
+    - . scripts/prepare_build.sh
+    - date
+    - bundle exec rake db:migrate
 
-.db-rollback: &db-rollback
-  extends: .dedicated-no-docs-and-no-qa-pull-cache-job
+db:rollback:
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .default-before_script
+    - .use-pg
+    - .except-docs-qa
+  dependencies: ["setup-test-env"]
   script:
     - bundle exec rake db:migrate VERSION=20180101160629
     - bundle exec rake db:migrate SKIP_SCHEMA_VERSION_CHECK=true
-  dependencies:
-    - setup-test-env
-
-db:rollback-pg:
-  <<: *db-rollback
-  <<: *use-pg
 
-gitlab:setup-pg:
-  <<: *gitlab-setup
-  <<: *use-pg
-  dependencies:
-    - setup-test-env
+gitlab:setup:
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .default-before_script
+    - .use-pg
+    - .except-docs-qa
+  dependencies: ["setup-test-env"]
+  variables:
+    SETUP_DB: "false"
+  script:
+    # Manually clone gitlab-test and only seed this project in
+    # db/fixtures/development/04_project.rb thanks to SIZE=1 below
+    - git clone https://gitlab.com/gitlab-org/gitlab-test.git
+       /home/git/repositories/gitlab-org/gitlab-test.git
+    - scripts/gitaly-test-spawn
+    - force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup
+  artifacts:
+    when: on_failure
+    expire_in: 1d
+    paths:
+      - log/development.log
 
 coverage:
   # Don't include dedicated-no-docs-no-db-pull-cache-job here since we need to
   # download artifacts from all the rspec jobs instead of from setup-test-env only
   extends:
-    - .dedicated-runner-default-cache
-    - .no-docs-and-no-qa
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .default-before_script
+    - .except-docs-qa
   cache:
     policy: pull
   variables:

.gitlab/ci/reports.gitlab-ci.yml

@@ -4,36 +4,29 @@ include:
   - template: Security/Dependency-Scanning.gitlab-ci.yml
   - template: Security/DAST.gitlab-ci.yml
 
+.reports:
+  extends:
+    - .default-retry
+    - .except-docs
+  tags:
+    - gitlab-org
+    - docker
+
 code_quality:
-  extends: .dedicated-no-docs
-  # gitlab-org runners set `privileged: false` but we need to have it set to true
-  # since we're using Docker in Docker
-  tags: []
-  before_script: []
-  cache: {}
+  extends: .reports
 
 sast:
-  extends: .dedicated-no-docs
-  tags: []
-  before_script: []
-  cache: {}
+  extends: .reports
   variables:
     SAST_BRAKEMAN_LEVEL: 2
     SAST_EXCLUDED_PATHS: qa,spec,doc
 
 dependency_scanning:
-  extends: .dedicated-no-docs
-  tags: []
-  before_script: []
-  cache: {}
+  extends: .reports
 
 dast:
-  extends:
-    - .dedicated-runner
-    - .review-only
+  extends: .reports
   stage: qa
-  dependencies:
-    - review-deploy
+  dependencies: ["review-deploy"]
   before_script:
     - export DAST_WEBSITE="$(cat review_app_url.txt)"
-  cache: {}

.gitlab/ci/review.gitlab-ci.yml

-.review-schedules-only: &review-schedules-only
+.review-schedules-only:
   only:
     refs:
       - schedules@gitlab-org/gitlab-ce
@@ -11,39 +11,39 @@
       - tags
       - /(^docs[\/-].+|.+-docs$)/
 
-.review-base: &review-base
+.review-base:
   extends:
-    - .dedicated-runner
+    - .default-tags
+    - .default-retry
     - .review-only
   image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
-  cache: {}
   dependencies: []
   before_script:
     - source scripts/utils.sh
 
-.review-docker: &review-docker
-  <<: *review-base
+.review-docker:
+  extends: .review-base
   image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine
   services:
     - docker:19.03.0-dind
   tags:
     - gitlab-org
     - docker
-  variables: &review-docker-variables
+  variables:
     DOCKER_DRIVER: overlay2
     DOCKER_HOST: tcp://docker:2375
     LATEST_QA_IMAGE: "gitlab/${CI_PROJECT_NAME}-qa:nightly"
     QA_IMAGE: "${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab/${CI_PROJECT_NAME}-qa:${CI_COMMIT_REF_SLUG}"
 
 build-qa-image:
-  <<: *review-docker
+  extends: .review-docker
   stage: test
   script:
     - time docker build --cache-from ${LATEST_QA_IMAGE} --tag ${QA_IMAGE} --file ./qa/Dockerfile ./
     - echo "${CI_JOB_TOKEN}" | docker login --username gitlab-ci-token --password-stdin ${CI_REGISTRY}
     - time docker push ${QA_IMAGE}
 
-.review-build-cng-base: &review-build-cng-base
+.review-build-cng-base:
   image: ruby:2.6-alpine
   stage: test
   when: manual
@@ -52,20 +52,21 @@ build-qa-image:
     - install_api_client_dependencies_with_apk
     - install_gitlab_gem
   dependencies: []
-  cache: {}
   script:
     - BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng
 
 review-build-cng:
-  extends: .review-only
-  <<: *review-build-cng-base
+  extends:
+    - .review-build-cng-base
+    - .review-only
 
 schedule:review-build-cng:
-  <<: *review-schedules-only
-  <<: *review-build-cng-base
+  extends:
+    - .review-build-cng-base
+    - .review-schedules-only
 
-.review-deploy-base: &review-deploy-base
-  <<: *review-base
+review-deploy:
+  extends: .review-base
   allow_failure: true
   retry: 1
   stage: review
@@ -73,7 +74,7 @@ schedule:review-build-cng:
     HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}"
     DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}"
     GITLAB_HELM_CHART_REF: "master"
-  environment: &review-environment
+  environment:
     name: review/${CI_COMMIT_REF_NAME}
     url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}
     on_stop: review-stop
@@ -98,50 +99,45 @@ schedule:review-build-cng:
     expire_in: 2 days
     when: always
 
-review-deploy:
-  <<: *review-deploy-base
-
 schedule:review-deploy:
-  <<: *review-deploy-base
-  <<: *review-schedules-only
+  extends:
+    - review-deploy
+    - .review-schedules-only
 
 review-stop:
-  extends:
-    - .single-script-job-dedicated-runner
-    - .review-only
-  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
-  stage: review
+  extends: review-deploy
   when: manual
-  allow_failure: true
-  variables:
-    SCRIPT_NAME: review_apps/review-apps.sh
   environment:
-    <<: *review-environment
     action: stop
-  script:
+  variables:
+    GIT_STRATEGY: none
+  before_script:
+    # We don't clone the repo by using GIT_STRATEGY: none and only download the
+    # single script we need here so it's much faster than cloning.
+    - apk add --update openssl
+    - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/review_apps/review-apps.sh
     - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/utils.sh
     - source utils.sh
-    - source $(basename $SCRIPT_NAME)
+    - source review-apps.sh
+  script:
     - delete
+  artifacts: {}
 
-.review-qa-base: &review-qa-base
-  <<: *review-docker
-  allow_failure: true
+.review-qa-base:
+  extends: .review-docker
   retry: 2
   stage: qa
   variables:
-    <<: *review-docker-variables
     QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa"
     QA_CAN_TEST_GIT_PROTOCOL_V2: "false"
+    QA_DEBUG: "true"
     GITLAB_USERNAME: "root"
     GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}"
     GITLAB_ADMIN_USERNAME: "root"
     GITLAB_ADMIN_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}"
     GITHUB_ACCESS_TOKEN: "${REVIEW_APPS_QA_GITHUB_ACCESS_TOKEN}"
     EE_LICENSE: "${REVIEW_APPS_EE_LICENSE}"
-    QA_DEBUG: "true"
-  dependencies:
-    - review-deploy
+  dependencies: ["review-deploy"]
   artifacts:
     paths:
       - ./qa/gitlab-qa-run-*
@@ -156,12 +152,13 @@ review-stop:
     - gem install gitlab-qa --no-document ${GITLAB_QA_VERSION:+ --version ${GITLAB_QA_VERSION}}
 
 review-qa-smoke:
-  <<: *review-qa-base
+  extends: .review-qa-base
+  allow_failure: true
   script:
     - gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}"
 
 review-qa-all:
-  <<: *review-qa-base
+  extends: .review-qa-base
   allow_failure: true
   when: manual
   parallel: 5
@@ -172,21 +169,16 @@ review-qa-all:
 
 parallel-spec-reports:
   extends:
-    - .dedicated-runner
-    - .no-docs
-  dependencies:
-    - review-qa-all
+    - .default-tags
+    - .except-docs
   image: ruby:2.6-alpine
-  services: []
-  before_script: []
+  stage: post-test
+  dependencies: ["review-qa-all"]
   variables:
-    SETUP_DB: "false"
     NEW_PARALLEL_SPECS_REPORT: qa/report-new.html
     BASE_ARTIFACT_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/file/qa/"
-  stage: post-test
   allow_failure: true
   when: manual
-  retry: 0
   artifacts:
     when: always
     paths:
@@ -196,15 +188,15 @@ parallel-spec-reports:
       junit: qa/gitlab-qa-run-*/**/rspec-*.xml
   script:
     - apk add --update build-base libxml2-dev libxslt-dev && rm -rf /var/cache/apk/*
-    - gem install nokogiri
+    - gem install nokogiri --no-document
     - cd qa/gitlab-qa-run-*/gitlab-*
     - ARTIFACT_DIRS=$(pwd |rev| awk -F / '{print $1,$2}' | rev | sed s_\ _/_)
-    - cd ../../..
+    - cd -
     - '[[ -f $NEW_PARALLEL_SPECS_REPORT ]] || echo "{}" > ${NEW_PARALLEL_SPECS_REPORT}'
     - scripts/merge-html-reports ${NEW_PARALLEL_SPECS_REPORT} ${BASE_ARTIFACT_URL}${ARTIFACT_DIRS} qa/gitlab-qa-run-*/**/rspec.htm
 
-.review-performance-base: &review-performance-base
-  <<: *review-qa-base
+review-performance:
+  extends: .review-qa-base
   allow_failure: true
   before_script:
     - export CI_ENVIRONMENT_URL="$(cat review_app_url.txt)"
@@ -222,18 +214,16 @@ parallel-spec-reports:
     reports:
       performance: performance.json
 
-review-performance:
-  <<: *review-performance-base
-
 schedule:review-performance:
-  <<: *review-performance-base
-  <<: *review-schedules-only
-  dependencies:
-    - schedule:review-deploy
+  extends:
+    - review-performance
+    - .review-schedules-only
+  dependencies: ["schedule:review-deploy"]
 
 schedule:review-cleanup:
-  <<: *review-base
-  <<: *review-schedules-only
+  extends:
+    - .review-base
+    - .review-schedules-only
   stage: build
   allow_failure: true
   environment:
@@ -246,11 +236,13 @@ schedule:review-cleanup:
     - ruby -rrubygems scripts/review_apps/automated_cleanup.rb
 
 danger-review:
-  extends: .dedicated-pull-cache-job
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
   image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger
   stage: test
   dependencies: []
-  before_script: []
   only:
     variables:
       - $DANGER_GITLAB_API_TOKEN
@@ -259,9 +251,8 @@ danger-review:
       - master
       - /^\d+-\d+-auto-deploy-\d+$/
       - /^[\d-]+-stable(-ee)?$/
-    variables:
-      - $CI_COMMIT_REF_NAME =~ /^ce-to-ee-.*/
-      - $CI_COMMIT_REF_NAME =~ /.*-stable(-ee)?-prepare-.*/
+      - /^ce-to-ee-.*/
+      - /.*-stable(-ee)?-prepare-.*/
   script:
     - git version
     - node --version

.gitlab/ci/setup.gitlab-ci.yml

 # Insurance in case a gem needed by one of our releases gets yanked from
 # rubygems.org in the future.
 cache gems:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .default-before_script
+    - .except-docs
+  dependencies: ["setup-test-env"]
+  variables:
+    SETUP_DB: "false"
   script:
     - bundle package --all --all-platforms
   artifacts:
     paths:
       - vendor/cache
   only:
+    refs:
       - master@gitlab-org/gitlab-ce
       - master@gitlab-org/gitlab-ee
       - tags
-  dependencies:
-    - setup-test-env
 
-gitlab_git_test:
+.minimal-job:
   extends:
-    - .dedicated-runner
-    - .no-docs-and-no-qa
-  variables:
-    SETUP_DB: "false"
-  before_script: []
+    - .default-tags
+    - .default-retry
+    - .except-docs-qa
   dependencies: []
-  cache: {}
+
+gitlab_git_test:
+  extends: .minimal-job
   script:
     - spec/support/prepare-gitlab-git-test-for-commit --check-for-changes
 
 no_ee_check:
-  extends:
-    - .dedicated-runner
-    - .no-docs-and-no-qa
-  variables:
-    SETUP_DB: "false"
-  before_script: []
-  dependencies: []
-  cache: {}
+  extends: .minimal-job
   script:
     - scripts/no-ee-check
   only:
-    - /.+/@gitlab-org/gitlab-ce
+    refs:
+      - branches@gitlab-org/gitlab-ce

.gitlab/ci/test-metadata.gitlab-ci.yml

-.tests-metadata-state: &tests-metadata-state
-  extends: .dedicated-runner
+.tests-metadata-state:
   variables:
     TESTS_METADATA_S3_BUCKET: "gitlab-ce-cache"
   before_script:
@@ -14,7 +13,7 @@
 retrieve-tests-metadata:
   extends:
     - .tests-metadata-state
-    - .no-docs-and-no-qa
+    - .except-docs-qa
   stage: prepare
   cache:
     key: tests_metadata
@@ -29,7 +28,7 @@ retrieve-tests-metadata:
     - '[[ -f $FLAKY_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_SUITE_REPORT_PATH}'
 
 update-tests-metadata:
-  <<: *tests-metadata-state
+  extends: .tests-metadata-state
   stage: post-test
   cache:
     key: tests_metadata
@@ -49,24 +48,23 @@ update-tests-metadata:
     - rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json
     - scripts/insert-rspec-profiling-data
   only:
+    refs:
       - master@gitlab-org/gitlab-ce
       - master@gitlab-org/gitlab-ee
       - master@gitlab/gitlabhq
       - master@gitlab/gitlab-ee
 
 flaky-examples-check:
-  extends: .dedicated-runner
+  extends:
+    - .default-tags
+    - .default-retry
   image: ruby:2.6-alpine
-  services: []
-  before_script: []
+  stage: post-test
   variables:
-    SETUP_DB: "false"
-    USE_BUNDLE_INSTALL: "false"
     NEW_FLAKY_SPECS_REPORT: rspec_flaky/report-new.json
-  stage: post-test
   allow_failure: true
-  retry: 0
   only:
+    refs:
       - branches
   except:
     refs:

.gitlab/ci/yaml.gitlab-ci.yml

 # Yamllint of *.yml for .gitlab-ci.yml.
 # This uses rules from project root `.yamllint`.
 lint-ci-gitlab:
-  extends: .dedicated-runner
-  before_script: []
-  dependencies: []
+  extends:
+    - .default-tags
+    - .default-retry
   image: sdesbure/yamllint:latest
+  dependencies: []
   script:
     - yamllint .gitlab-ci.yml .gitlab/ci lib/gitlab/ci/templates changelogs