Merge branch 'feature-branch-gather-jobs-data' into 'master'

Reduce the complexity of our CI configuration Closes #51367 and #65702 See merge request gitlab-org/gitlab-ce!31728

Merge branch 'feature-branch-gather-jobs-data' into 'master'
Reduce the complexity of our CI configuration Closes #51367 and #65702 See merge request gitlab-org/gitlab-ce!31728
53c39cc1 · Stan Hu · be97241a · b881c772 · 53c39cc1 · 53c39cc1
Commit 53c39cc1 authored Aug 27, 2019 by Stan Hu
14 changed files
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
 image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-golang-1.11-git-2.22-chrome-73.0-node-12.x-yarn-1.16-postgresql-9.6-graphicsmagick-1.3.33"
+stages:
+  - build
+  - prepare
+  - quick-test
+  - test
+  - review
+  - qa
+  - post-test
+  - pages
 variables:
  RAILS_ENV: "test"
  NODE_ENV: "test"
@@ -11,25 +21,9 @@ variables:
  FLAKY_RSPEC_SUITE_REPORT_PATH: rspec_flaky/report-suite.json
  BUILD_ASSETS_IMAGE: "false"
-before_script:
-  - date
-  - source scripts/utils.sh
-  - source scripts/prepare_build.sh
-  - date
 after_script:
  - date
-stages:
-  - build
-  - prepare
-  - merge
-  - test
-  - review
-  - qa
-  - post-test
-  - pages
 include:
  - local: .gitlab/ci/global.gitlab-ci.yml
  - local: .gitlab/ci/cng.gitlab-ci.yml

--- a/.gitlab/ci/cng.gitlab-ci.yml
+++ b/.gitlab/ci/cng.gitlab-ci.yml
 cloud-native-image:
  image: ruby:2.6-alpine
-  before_script: []
  dependencies: []
  stage: post-test
  allow_failure: true
  variables:
    GIT_DEPTH: "1"
-  cache: {}
  when: manual
  script:
    - install_gitlab_gem
    - CNG_PROJECT_PATH="gitlab-org/build/CNG" BUILD_TRIGGER_TOKEN=$CI_JOB_TOKEN ./scripts/trigger-build cng
  only:
-    - tags@gitlab-org/gitlab-ce
+    refs:
-    - tags@gitlab-org/gitlab-ee
+      - tags@gitlab-org/gitlab-ce
+      - tags@gitlab-org/gitlab-ee
--- a/.gitlab/ci/docs.gitlab-ci.yml
+++ b/.gitlab/ci/docs.gitlab-ci.yml
-.review-docs: &review-docs
+.review-docs:
-  extends: .single-script-job-dedicated-runner
+  extends:
+    - .default-tags
+    - .default-retry
+  image: ruby:2.6-alpine
+  stage: review
+  dependencies: []
  variables:
-    SCRIPT_NAME: trigger-build-docs
+    GIT_STRATEGY: none
  environment:
    name: review-docs/$CI_COMMIT_REF_SLUG
    # DOCS_REVIEW_APPS_DOMAIN and DOCS_GITLAB_REPO_SUFFIX are CI variables
    # Discussion: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14236/diffs#note_40140693
    url: http://$CI_ENVIRONMENT_SLUG.$DOCS_REVIEW_APPS_DOMAIN/$DOCS_GITLAB_REPO_SUFFIX
    on_stop: review-docs-cleanup
+  before_script:
+    # We don't clone the repo by using GIT_STRATEGY: none and only download the
+    # single script we need here so it's much faster than cloning.
+    - apk add --update openssl
+    - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/trigger-build-docs
+    - chmod 755 trigger-build-docs
 # Trigger a manual docs build in gitlab-docs only on non docs-only branches.
 # Useful to preview the docs changes live.
 review-docs-deploy-manual:
  extends:
    - .review-docs
-    - .no-docs-and-no-qa
+    - .except-docs-qa
-  stage: review
  script:
    - gem install gitlab --no-document
-    - ./$SCRIPT_NAME deploy
+    - ./trigger-build-docs deploy
  when: manual
  only:
    - branches@gitlab-org/gitlab-ce
@@ -27,39 +37,40 @@ review-docs-deploy-manual:
 # Always trigger a docs build in gitlab-docs only on docs-only branches.
 # Useful to preview the docs changes live.
 review-docs-deploy:
-  <<: *review-docs
+  extends:
-  stage: review
+    - .review-docs
+    - .except-qa
  script:
    - gem install gitlab --no-document
-    - ./$SCRIPT_NAME deploy
+    - ./trigger-build-docs deploy
  only:
    - /(^docs[\/-].+|.+-docs$)/@gitlab-org/gitlab-ce
    - /(^docs[\/-].+|.+-docs$)/@gitlab-org/gitlab-ee
-  except:
-    - /(^qa[\/-].*|.*-qa$)/
 # Cleanup remote environment of gitlab-docs
 review-docs-cleanup:
-  <<: *review-docs
+  extends:
-  stage: review
+    - .review-docs
+    - .except-qa
  environment:
    name: review-docs/$CI_COMMIT_REF_SLUG
    action: stop
  script:
    - gem install gitlab --no-document
-    - ./$SCRIPT_NAME cleanup
+    - ./trigger-build-docs cleanup
  when: manual
  only:
    - branches@gitlab-org/gitlab-ce
    - branches@gitlab-org/gitlab-ee
 docs lint:
-  extends: .dedicated-runner
+  extends:
+    - .default-tags
+    - .default-retry
+    - .except-qa
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-docs-lint"
  stage: test
-  cache: {}
  dependencies: []
-  before_script: []
  script:
    - scripts/lint-doc.sh
    - mv doc/ /tmp/gitlab-docs/content/$DOCS_GITLAB_REPO_SUFFIX
@@ -72,5 +83,3 @@ docs lint:
    - bundle exec nanoc check internal_links
    # Check the internal anchor links
    - bundle exec nanoc check internal_anchors
-  except:
-    - /(^qa[\/-].*|.*-qa$)/
--- a/.gitlab/ci/frontend.gitlab-ci.yml
+++ b/.gitlab/ci/frontend.gitlab-ci.yml
-.assets-compile-cache: &assets-compile-cache
+.assets-compile-cache:
  cache:
-    key: "assets-compile:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v6"
    paths:
      - vendor/ruby/
      - .yarn-cache/
      - tmp/cache/assets/sprockets
-.use-pg: &use-pg
-  services:
-    - name: postgres:9.6.14
-      command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
-    - name: redis:alpine
 .gitlab:assets:compile-metadata:
-  <<: *assets-compile-cache
+  extends:
-  extends: .dedicated-no-docs-pull-cache-job
+    - .default-tags
+    - .default-retry
+    - .assets-compile-cache
+    - .default-before_script
+    - .except-docs
  image: dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-git-2.22-chrome-73.0-node-12.x-yarn-1.16-graphicsmagick-1.3.33-docker-18.06.1
-  dependencies:
+  dependencies: ["setup-test-env"]
-    - setup-test-env
  services:
    - docker:19.03.0-dind
  variables:
@@ -30,6 +26,14 @@
    NODE_OPTIONS: --max_old_space_size=3584
    DOCKER_DRIVER: overlay2
    DOCKER_HOST: tcp://docker:2375
+  cache:
+    key: "assets-compile:production:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v6"
+  artifacts:
+    name: webpack-report
+    expire_in: 31d
+    paths:
+      - webpack-report/
+      - public/assets/
  script:
    - node --version
    - retry yarn install --frozen-lockfile --production --cache-folder .yarn-cache --prefer-offline
@@ -42,43 +46,41 @@
    - install_api_client_dependencies_with_apt
    - play_job "review-build-cng" || true  # this job might not exist so ignore the failure if it cannot be played
    - play_job "schedule:review-build-cng" || true  # this job might not exist so ignore the failure if it cannot be played
-  artifacts:
-    name: webpack-report
-    expire_in: 31d
-    paths:
-      - webpack-report/
-      - public/assets/
  only:
    - /.+/@gitlab-org/gitlab-ce
    - /.+/@gitlab-org/gitlab-ee
    - /.+/@gitlab/gitlabhq
    - /.+/@gitlab/gitlab-ee
  tags:
-    - docker
    - gitlab-org
+    - docker
 gitlab:assets:compile:
  extends: .gitlab:assets:compile-metadata
+  only:
+    refs:
+      - master@gitlab-org/gitlab-ce
+      - master@gitlab-org/gitlab-ee
  cache:
    policy: pull-push
-  only:
-    - master@gitlab-org/gitlab-ce
-    - master@gitlab-org/gitlab-ee
 gitlab:assets:compile pull-cache:
  extends: .gitlab:assets:compile-metadata
-  cache:
-    policy: pull
  except:
    refs:
      - master@gitlab-org/gitlab-ce
      - master@gitlab-org/gitlab-ee
      - /(^docs[\/-].+|.+-docs$)/
+  cache:
+    policy: pull
 .compile-assets-metadata:
-  extends: .dedicated-runner
+  extends:
-  <<: *use-pg
+    - .default-tags
-  <<: *assets-compile-cache
+    - .default-retry
+    - .assets-compile-cache
+    - .default-before_script
+    - .use-pg
  stage: prepare
  script:
    - node --version
@@ -89,6 +91,8 @@ gitlab:assets:compile pull-cache:
  variables:
    # we override the max_old_space_size to prevent OOM errors
    NODE_OPTIONS: --max_old_space_size=3584
+  cache:
+    key: "assets-compile:test:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v6"
  artifacts:
    expire_in: 7d
    paths:
@@ -96,30 +100,34 @@ gitlab:assets:compile pull-cache:
      - public/assets
 compile-assets:
-  extends: .compile-assets-metadata
+  extends:
+    - .compile-assets-metadata
+  only:
+    refs:
+      - master@gitlab-org/gitlab-ce
+      - master@gitlab-org/gitlab-ee
  cache:
    policy: pull-push
-  only:
-    - master@gitlab-org/gitlab-ce
-    - master@gitlab-org/gitlab-ee
 compile-assets pull-cache:
  extends: .compile-assets-metadata
-  cache:
-    policy: pull
  except:
    refs:
      - master@gitlab-org/gitlab-ce
      - master@gitlab-org/gitlab-ee
      - /(^docs[\/-].+|.+-docs$)/
+  cache:
+    policy: pull
 karma:
-  extends: .dedicated-no-docs-pull-cache-job
+  extends:
-  <<: *use-pg
+    - .default-tags
-  dependencies:
+    - .default-retry
-    - compile-assets
+    - .default-cache
-    - compile-assets pull-cache
+    - .default-before_script
-    - setup-test-env
+    - .use-pg
+    - .except-docs
+  dependencies: ["compile-assets", "compile-assets pull-cache", "setup-test-env"]
  variables:
    # we override the max_old_space_size to prevent OOM errors
    NODE_OPTIONS: --max_old_space_size=3584
@@ -142,12 +150,14 @@ karma:
      junit: junit_karma.xml
 jest:
-  extends: .dedicated-no-docs-and-no-qa-pull-cache-job
+  extends:
-  <<: *use-pg
+    - .default-tags
-  dependencies:
+    - .default-retry
-    - compile-assets
+    - .default-cache
-    - compile-assets pull-cache
+    - .default-before_script
-    - setup-test-env
+    - .use-pg
+    - .except-docs-qa
+  dependencies: ["compile-assets", "compile-assets pull-cache", "setup-test-env"]
  script:
    - scripts/gitaly-test-spawn
    - date
@@ -170,36 +180,41 @@ jest:
      - tmp/jest/jest/
    policy: pull-push
-qa:internal:
+.qa:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
+  extends:
-  services: []
+    - .default-tags
-  script:
+    - .default-retry
+    - .default-cache
+    - .except-docs
+  dependencies: ["setup-test-env"]
+  variables:
+    SETUP_DB: "false"
+  before_script:
    - cd qa/
    - bundle install
+qa:internal:
+  extends: .qa
+  script:
    - bundle exec rspec
-  dependencies:
-    - setup-test-env
 qa:selectors:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
+  extends: .qa
-  services: []
  script:
-    - cd qa/
-    - bundle install
    - bundle exec bin/qa Test::Sanity::Selectors
-  dependencies:
-    - setup-test-env
-.qa-frontend-node: &qa-frontend-node
+.qa-frontend-node:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
+  extends:
-  stage: test
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .except-docs
+  dependencies: []
  cache:
    key: "$CI_JOB_NAME"
    paths:
      - .yarn-cache/
    policy: pull-push
-  dependencies: []
-  before_script: []
  script:
    - date
    - yarn install --frozen-lockfile --cache-folder .yarn-cache --prefer-offline
@@ -207,23 +222,28 @@ qa:selectors:
    - yarn run webpack-prod
 qa-frontend-node:8:
-  <<: *qa-frontend-node
+  extends: .qa-frontend-node
  image: node:carbon
 qa-frontend-node:10:
-  <<: *qa-frontend-node
+  extends: .qa-frontend-node
  image: node:dubnium
 qa-frontend-node:latest:
-  <<: *qa-frontend-node
+  extends: .qa-frontend-node
  image: node:latest
  allow_failure: true
 lint:javascript:report:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .except-docs
+  variables:
+    SETUP_DB: "false"
  stage: post-test
  dependencies: []
-  before_script: []
  script:
    - date
    - yarn run eslint-report || true  # ignore exit code
@@ -234,12 +254,15 @@ lint:javascript:report:
      - eslint-report.html
 jsdoc:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .except-docs
+  variables:
+    SETUP_DB: "false"
  stage: post-test
-  dependencies:
+  dependencies: ["compile-assets", "compile-assets pull-cache"]
-    - compile-assets
-    - compile-assets pull-cache
-  before_script: []
  script:
    - date
    - yarn run jsdoc || true  # ignore exit code

--- a/.gitlab/ci/global.gitlab-ci.yml
+++ b/.gitlab/ci/global.gitlab-ci.yml
-.dedicated-runner:
+.default-tags:
+  tags:
+    - gitlab-org
+.default-retry:
  retry:
    max: 2  # This is confusing but this means "3 runs at max".
    when:
      - unknown_failure
      - api_failure
      - runner_system_failure
-  tags:
-    - gitlab-org
-.default-cache: &default-cache
-  key: "debian-stretch-ruby-2.6.3-node-12.x"
-  paths:
-    - vendor/ruby
-    - .yarn-cache/
-    - vendor/gitaly-ruby
-.dedicated-runner-default-cache:
+.default-before_script:
-  extends: .dedicated-runner
+  before_script:
-  cache:
+    - date
-    <<: *default-cache
+    - source scripts/utils.sh
+    - source scripts/prepare_build.sh
+    - date
 # Jobs that only need to pull cache
-.dedicated-pull-cache-job:
+.default-cache:
-  extends: .dedicated-runner
  cache:
-    <<: *default-cache
+    key: "debian-stretch-ruby-2.6.3-node-12.x"
+    paths:
+      - vendor/ruby
+      - .yarn-cache/
+      - vendor/gitaly-ruby
    policy: pull
-  stage: test
-.no-docs:
+.except-docs:
  except:
    refs:
      - /(^docs[\/-].+|.+-docs$)/
-.no-docs-and-no-qa:
+.except-qa:
  except:
    refs:
-      - /(^docs[\/-].+|.+-docs$)/
      - /(^qa[\/-].*|.*-qa$)/
-.dedicated-no-docs-pull-cache-job:
+.except-docs-qa:
-  extends:
+  except:
-    - .dedicated-pull-cache-job
+    refs:
-    - .no-docs
+      - /(^docs[\/-].+|.+-docs$)/
+      - /(^qa[\/-].*|.*-qa$)/
-.dedicated-no-docs-and-no-qa-pull-cache-job:
-  extends:
-    - .dedicated-pull-cache-job
-    - .no-docs-and-no-qa
-# Jobs that do not need a DB
-.dedicated-no-docs-no-db-pull-cache-job:
-  extends: .dedicated-no-docs-pull-cache-job
-  variables:
-    SETUP_DB: "false"
-# Jobs that need a dedicated runner, with no cache
-.dedicated-no-docs:
-  extends:
-    - .dedicated-runner
-    - .no-docs
-.single-script-job-dedicated-runner:
+.except-docs-qa-geo:
-  extends: .dedicated-runner
+  except:
-  image: ruby:2.6-alpine
+    refs:
-  stage: test
+      - /(^docs[\/-].+|.+-docs$)/
-  cache: {}
+      - /(^qa[\/-].*|.*-qa$)/
-  dependencies: []
+      - /(^geo[\/-].*|.*-geo$)/
-  variables:
-    GIT_STRATEGY: none
-  before_script:
-    # We don't clone the repo by using GIT_STRATEGY: none and only download the
-    # single script we need here so it's much faster than cloning.
-    - export SCRIPT_NAME="${SCRIPT_NAME:-$CI_JOB_NAME}"
-    - apk add --update openssl
-    - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/$SCRIPT_NAME
-    - chmod 755 $(basename $SCRIPT_NAME)
-.review-only: &review-only
+.review-only:
  only:
    refs:
      - branches@gitlab-org/gitlab-ce
@@ -88,3 +61,16 @@
      - master
      - /^\d+-\d+-auto-deploy-\d+$/
      - /(^docs[\/-].+|.+-docs$)/
+.use-pg:
+  services:
+    - name: postgres:9.6.14
+      command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
+    - name: redis:alpine
+.use-pg-10:
+  image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-golang-1.11-git-2.22-chrome-73.0-node-12.x-yarn-1.16-postgresql-10-graphicsmagick-1.3.33"
+  services:
+    - name: postgres:10.9
+      command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
+    - name: redis:alpine
--- a/.gitlab/ci/memory.gitlab-ci.yml
+++ b/.gitlab/ci/memory.gitlab-ci.yml
 memory-static:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .default-before_script
+    - .except-docs
+  variables:
+    SETUP_DB: "false"
  script:
    # Uses two different reports from the 'derailed_benchmars' gem.
@@ -23,7 +30,13 @@ memory-static:
 # The application is booted in `production` environment.
 # All tests are run without a webserver (directly using Rack::Mock by default).
 memory-on-boot:
-  extends: .rspec-metadata-pg-10
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .default-before_script
+    - .use-pg-10
+    - .except-docs-qa
  variables:
    NODE_ENV: "production"
    RAILS_ENV: "production"

--- a/.gitlab/ci/pages.gitlab-ci.yml
+++ b/.gitlab/ci/pages.gitlab-ci.yml
 pages:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
+  extends:
-  before_script: []
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .except-docs
+  only:
+    refs:
+      - master@gitlab-org/gitlab-ce
+      - master@gitlab-org/gitlab-ee
  stage: pages
-  dependencies:
+  dependencies: ["coverage", "karma", "gitlab:assets:compile", "lint:javascript:report", "jsdoc"]
-    - coverage
-    - karma
-    - gitlab:assets:compile
-    - lint:javascript:report
-    - jsdoc
  script:
    - mv public/ .public/
    - mkdir public/
@@ -21,6 +23,3 @@ pages:
  artifacts:
    paths:
      - public
-  only:
-    - master@gitlab-org/gitlab-ce
-    - master@gitlab-org/gitlab-ee
--- a/.gitlab/ci/qa.gitlab-ci.yml
+++ b/.gitlab/ci/qa.gitlab-ci.yml
 .package-and-qa-base:
  image: ruby:2.6-alpine
  stage: qa
-  before_script: []
  needs: ["build-qa-image", "gitlab:assets:compile pull-cache"]
  dependencies: []
-  cache: {}
  variables:
    GIT_DEPTH: "1"
  retry: 0
@@ -19,7 +17,9 @@
    - master
 package-and-qa-manual:
-  extends: .package-and-qa-base
+  extends:
+    - .package-and-qa-base
+    - .except-docs-qa
  when: manual
  except:
    - master

--- a/.gitlab/ci/rails.gitlab-ci.yml
+++ b/.gitlab/ci/rails.gitlab-ci.yml
--- a/.gitlab/ci/reports.gitlab-ci.yml
+++ b/.gitlab/ci/reports.gitlab-ci.yml
@@ -4,36 +4,29 @@ include:
  - template: Security/Dependency-Scanning.gitlab-ci.yml
  - template: Security/DAST.gitlab-ci.yml
+.reports:
+  extends:
+    - .default-retry
+    - .except-docs
+  tags:
+    - gitlab-org
+    - docker
 code_quality:
-  extends: .dedicated-no-docs
+  extends: .reports
-  # gitlab-org runners set `privileged: false` but we need to have it set to true
-  # since we're using Docker in Docker
-  tags: []
-  before_script: []
-  cache: {}
 sast:
-  extends: .dedicated-no-docs
+  extends: .reports
-  tags: []
-  before_script: []
-  cache: {}
  variables:
    SAST_BRAKEMAN_LEVEL: 2
    SAST_EXCLUDED_PATHS: qa,spec,doc
 dependency_scanning:
-  extends: .dedicated-no-docs
+  extends: .reports
-  tags: []
-  before_script: []
-  cache: {}
 dast:
-  extends:
+  extends: .reports
-    - .dedicated-runner
-    - .review-only
  stage: qa
-  dependencies:
+  dependencies: ["review-deploy"]
-    - review-deploy
  before_script:
    - export DAST_WEBSITE="$(cat review_app_url.txt)"
-  cache: {}
--- a/.gitlab/ci/review.gitlab-ci.yml
+++ b/.gitlab/ci/review.gitlab-ci.yml
-.review-schedules-only: &review-schedules-only
+.review-schedules-only:
  only:
    refs:
      - schedules@gitlab-org/gitlab-ce
@@ -11,39 +11,39 @@
      - tags
      - /(^docs[\/-].+|.+-docs$)/
-.review-base: &review-base
+.review-base:
  extends:
-    - .dedicated-runner
+    - .default-tags
+    - .default-retry
    - .review-only
  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
-  cache: {}
  dependencies: []
  before_script:
    - source scripts/utils.sh
-.review-docker: &review-docker
+.review-docker:
-  <<: *review-base
+  extends: .review-base
  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine
  services:
    - docker:19.03.0-dind
  tags:
    - gitlab-org
    - docker
-  variables: &review-docker-variables
+  variables:
    DOCKER_DRIVER: overlay2
    DOCKER_HOST: tcp://docker:2375
    LATEST_QA_IMAGE: "gitlab/${CI_PROJECT_NAME}-qa:nightly"
    QA_IMAGE: "${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab/${CI_PROJECT_NAME}-qa:${CI_COMMIT_REF_SLUG}"
 build-qa-image:
-  <<: *review-docker
+  extends: .review-docker
  stage: test
  script:
    - time docker build --cache-from ${LATEST_QA_IMAGE} --tag ${QA_IMAGE} --file ./qa/Dockerfile ./
    - echo "${CI_JOB_TOKEN}" | docker login --username gitlab-ci-token --password-stdin ${CI_REGISTRY}
    - time docker push ${QA_IMAGE}
-.review-build-cng-base: &review-build-cng-base
+.review-build-cng-base:
  image: ruby:2.6-alpine
  stage: test
  when: manual
@@ -52,20 +52,21 @@ build-qa-image:
    - install_api_client_dependencies_with_apk
    - install_gitlab_gem
  dependencies: []
-  cache: {}
  script:
    - BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng
 review-build-cng:
-  extends: .review-only
+  extends:
-  <<: *review-build-cng-base
+    - .review-build-cng-base
+    - .review-only
 schedule:review-build-cng:
-  <<: *review-schedules-only
+  extends:
-  <<: *review-build-cng-base
+    - .review-build-cng-base
+    - .review-schedules-only
-.review-deploy-base: &review-deploy-base
+review-deploy:
-  <<: *review-base
+  extends: .review-base
  allow_failure: true
  retry: 1
  stage: review
@@ -73,7 +74,7 @@ schedule:review-build-cng:
    HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}"
    DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}"
    GITLAB_HELM_CHART_REF: "master"
-  environment: &review-environment
+  environment:
    name: review/${CI_COMMIT_REF_NAME}
    url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}
    on_stop: review-stop
@@ -98,50 +99,45 @@ schedule:review-build-cng:
    expire_in: 2 days
    when: always
-review-deploy:
-  <<: *review-deploy-base
 schedule:review-deploy:
-  <<: *review-deploy-base
+  extends:
-  <<: *review-schedules-only
+    - review-deploy
+    - .review-schedules-only
 review-stop:
-  extends:
+  extends: review-deploy
-    - .single-script-job-dedicated-runner
-    - .review-only
-  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
-  stage: review
  when: manual
-  allow_failure: true
-  variables:
-    SCRIPT_NAME: review_apps/review-apps.sh
  environment:
-    <<: *review-environment
    action: stop
-  script:
+  variables:
+    GIT_STRATEGY: none
+  before_script:
+    # We don't clone the repo by using GIT_STRATEGY: none and only download the
+    # single script we need here so it's much faster than cloning.
+    - apk add --update openssl
+    - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/review_apps/review-apps.sh
    - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/utils.sh
    - source utils.sh
-    - source $(basename $SCRIPT_NAME)
+    - source review-apps.sh
+  script:
    - delete
+  artifacts: {}
-.review-qa-base: &review-qa-base
+.review-qa-base:
-  <<: *review-docker
+  extends: .review-docker
-  allow_failure: true
  retry: 2
  stage: qa
  variables:
-    <<: *review-docker-variables
    QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa"
    QA_CAN_TEST_GIT_PROTOCOL_V2: "false"
+    QA_DEBUG: "true"
    GITLAB_USERNAME: "root"
    GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}"
    GITLAB_ADMIN_USERNAME: "root"
    GITLAB_ADMIN_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}"
    GITHUB_ACCESS_TOKEN: "${REVIEW_APPS_QA_GITHUB_ACCESS_TOKEN}"
    EE_LICENSE: "${REVIEW_APPS_EE_LICENSE}"
-    QA_DEBUG: "true"
+  dependencies: ["review-deploy"]
-  dependencies:
-    - review-deploy
  artifacts:
    paths:
      - ./qa/gitlab-qa-run-*
@@ -156,12 +152,13 @@ review-stop:
    - gem install gitlab-qa --no-document ${GITLAB_QA_VERSION:+ --version ${GITLAB_QA_VERSION}}
 review-qa-smoke:
-  <<: *review-qa-base
+  extends: .review-qa-base
+  allow_failure: true
  script:
    - gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}"
 review-qa-all:
-  <<: *review-qa-base
+  extends: .review-qa-base
  allow_failure: true
  when: manual
  parallel: 5
@@ -172,21 +169,16 @@ review-qa-all:
 parallel-spec-reports:
  extends:
-    - .dedicated-runner
+    - .default-tags
-    - .no-docs
+    - .except-docs
-  dependencies:
-    - review-qa-all
  image: ruby:2.6-alpine
-  services: []
+  stage: post-test
-  before_script: []
+  dependencies: ["review-qa-all"]
  variables:
-    SETUP_DB: "false"
    NEW_PARALLEL_SPECS_REPORT: qa/report-new.html
    BASE_ARTIFACT_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/file/qa/"
-  stage: post-test
  allow_failure: true
  when: manual
-  retry: 0
  artifacts:
    when: always
    paths:
@@ -196,15 +188,15 @@ parallel-spec-reports:
      junit: qa/gitlab-qa-run-*/**/rspec-*.xml
  script:
    - apk add --update build-base libxml2-dev libxslt-dev && rm -rf /var/cache/apk/*
-    - gem install nokogiri
+    - gem install nokogiri --no-document
    - cd qa/gitlab-qa-run-*/gitlab-*
    - ARTIFACT_DIRS=$(pwd |rev| awk -F / '{print $1,$2}' | rev | sed s_\ _/_)
-    - cd ../../..
+    - cd -
    - '[[ -f $NEW_PARALLEL_SPECS_REPORT ]] || echo "{}" > ${NEW_PARALLEL_SPECS_REPORT}'
    - scripts/merge-html-reports ${NEW_PARALLEL_SPECS_REPORT} ${BASE_ARTIFACT_URL}${ARTIFACT_DIRS} qa/gitlab-qa-run-*/**/rspec.htm
-.review-performance-base: &review-performance-base
+review-performance:
-  <<: *review-qa-base
+  extends: .review-qa-base
  allow_failure: true
  before_script:
    - export CI_ENVIRONMENT_URL="$(cat review_app_url.txt)"
@@ -222,18 +214,16 @@ parallel-spec-reports:
    reports:
      performance: performance.json
-review-performance:
-  <<: *review-performance-base
 schedule:review-performance:
-  <<: *review-performance-base
+  extends:
-  <<: *review-schedules-only
+    - review-performance
-  dependencies:
+    - .review-schedules-only
-    - schedule:review-deploy
+  dependencies: ["schedule:review-deploy"]
 schedule:review-cleanup:
-  <<: *review-base
+  extends:
-  <<: *review-schedules-only
+    - .review-base
+    - .review-schedules-only
  stage: build
  allow_failure: true
  environment:
@@ -246,11 +236,13 @@ schedule:review-cleanup:
    - ruby -rrubygems scripts/review_apps/automated_cleanup.rb
 danger-review:
-  extends: .dedicated-pull-cache-job
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
  image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger
  stage: test
  dependencies: []
-  before_script: []
  only:
    variables:
      - $DANGER_GITLAB_API_TOKEN
@@ -259,9 +251,8 @@ danger-review:
      - master
      - /^\d+-\d+-auto-deploy-\d+$/
      - /^[\d-]+-stable(-ee)?$/
-    variables:
+      - /^ce-to-ee-.*/
-      - $CI_COMMIT_REF_NAME =~ /^ce-to-ee-.*/
+      - /.*-stable(-ee)?-prepare-.*/
-      - $CI_COMMIT_REF_NAME =~ /.*-stable(-ee)?-prepare-.*/
  script:
    - git version
    - node --version

--- a/.gitlab/ci/setup.gitlab-ci.yml
+++ b/.gitlab/ci/setup.gitlab-ci.yml
 # Insurance in case a gem needed by one of our releases gets yanked from
 # rubygems.org in the future.
 cache gems:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .default-before_script
+    - .except-docs
+  dependencies: ["setup-test-env"]
+  variables:
+    SETUP_DB: "false"
  script:
    - bundle package --all --all-platforms
  artifacts:
    paths:
      - vendor/cache
  only:
-    - master@gitlab-org/gitlab-ce
+    refs:
-    - master@gitlab-org/gitlab-ee
+      - master@gitlab-org/gitlab-ce
-    - tags
+      - master@gitlab-org/gitlab-ee
-  dependencies:
+      - tags
-    - setup-test-env
-gitlab_git_test:
+.minimal-job:
  extends:
-    - .dedicated-runner
+    - .default-tags
-    - .no-docs-and-no-qa
+    - .default-retry
-  variables:
+    - .except-docs-qa
-    SETUP_DB: "false"
-  before_script: []
  dependencies: []
-  cache: {}
+gitlab_git_test:
+  extends: .minimal-job
  script:
    - spec/support/prepare-gitlab-git-test-for-commit --check-for-changes
 no_ee_check:
-  extends:
+  extends: .minimal-job
-    - .dedicated-runner
-    - .no-docs-and-no-qa
-  variables:
-    SETUP_DB: "false"
-  before_script: []
-  dependencies: []
-  cache: {}
  script:
    - scripts/no-ee-check
  only:
-    - /.+/@gitlab-org/gitlab-ce
+    refs:
+      - branches@gitlab-org/gitlab-ce
--- a/.gitlab/ci/test-metadata.gitlab-ci.yml
+++ b/.gitlab/ci/test-metadata.gitlab-ci.yml
-.tests-metadata-state: &tests-metadata-state
+.tests-metadata-state:
-  extends: .dedicated-runner
  variables:
    TESTS_METADATA_S3_BUCKET: "gitlab-ce-cache"
  before_script:
@@ -14,7 +13,7 @@
 retrieve-tests-metadata:
  extends:
    - .tests-metadata-state
-    - .no-docs-and-no-qa
+    - .except-docs-qa
  stage: prepare
  cache:
    key: tests_metadata
@@ -29,7 +28,7 @@ retrieve-tests-metadata:
    - '[[ -f $FLAKY_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_SUITE_REPORT_PATH}'
 update-tests-metadata:
-  <<: *tests-metadata-state
+  extends: .tests-metadata-state
  stage: post-test
  cache:
    key: tests_metadata
@@ -49,25 +48,24 @@ update-tests-metadata:
    - rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json
    - scripts/insert-rspec-profiling-data
  only:
-    - master@gitlab-org/gitlab-ce
+    refs:
-    - master@gitlab-org/gitlab-ee
+      - master@gitlab-org/gitlab-ce
-    - master@gitlab/gitlabhq
+      - master@gitlab-org/gitlab-ee
-    - master@gitlab/gitlab-ee
+      - master@gitlab/gitlabhq
+      - master@gitlab/gitlab-ee
 flaky-examples-check:
-  extends: .dedicated-runner
+  extends:
+    - .default-tags
+    - .default-retry
  image: ruby:2.6-alpine
-  services: []
+  stage: post-test
-  before_script: []
  variables:
-    SETUP_DB: "false"
-    USE_BUNDLE_INSTALL: "false"
    NEW_FLAKY_SPECS_REPORT: rspec_flaky/report-new.json
-  stage: post-test
  allow_failure: true
-  retry: 0
  only:
-    - branches
+    refs:
+      - branches
  except:
    refs:
      - master

--- a/.gitlab/ci/yaml.gitlab-ci.yml
+++ b/.gitlab/ci/yaml.gitlab-ci.yml
 # Yamllint of *.yml for .gitlab-ci.yml.
 # This uses rules from project root `.yamllint`.
 lint-ci-gitlab:
-  extends: .dedicated-runner
+  extends:
-  before_script: []
+    - .default-tags
-  dependencies: []
+    - .default-retry
  image: sdesbure/yamllint:latest
+  dependencies: []
  script:
    - yamllint .gitlab-ci.yml .gitlab/ci lib/gitlab/ci/templates changelogs