Merge branch '216730-decrease-compile-assets-pull-cache-duration-from-9-17-minutes' into 'master'

Simplify/optimize the assets compilation jobs and update caches on 2-hourly scheduled master pipelines [UPDATE CACHE] Closes #218440 See merge request gitlab-org/gitlab!32171

Merge branch '216730-decrease-compile-assets-pull-cache-duration-from-9-17-minutes' into 'master'
Simplify/optimize the assets compilation jobs and update caches on 2-hourly scheduled master pipelines [UPDATE CACHE] Closes #218440 See merge request gitlab-org/gitlab!32171
2a3ced54 · Lin Jen-Shin · a240ab20 · 862d74e1 · 2a3ced54 · 2a3ced54
Commit 2a3ced54 authored Jun 02, 2020 by Lin Jen-Shin
15 changed files
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -15,7 +15,7 @@ stages:
 # in cases where jobs require Docker-in-Docker, the job
 # definition must be extended with `.use-docker-in-docker`
 default:
-  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-9.6-graphicsmagick-1.3.34"
+  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34"
  tags:
    - gitlab-org
  # All jobs are interruptible by default

--- a/.gitlab/ci/docs.gitlab-ci.yml
+++ b/.gitlab/ci/docs.gitlab-ci.yml
@@ -62,7 +62,7 @@ docs lint:
 graphql-reference-verify:
  extends:
    - .default-retry
-    - .default-cache
+    - .rails-cache
    - .default-before_script
    - .docs:rules:graphql-reference-verify
    - .use-pg11

--- a/.gitlab/ci/frontend.gitlab-ci.yml
+++ b/.gitlab/ci/frontend.gitlab-ci.yml
-.assets-compile-cache:
+.frontend-base:
-  cache:
-    paths:
-      - vendor/ruby/
-      - public/assets/webpack/
-      - assets-hash.txt
-      - .yarn-cache/
-      - tmp/cache/assets/sprockets
-      - tmp/cache/babel-loader
-      - tmp/cache/vue-loader
-      - tmp/cache/webpack-dlls
-.gitlab:assets:compile-metadata:
  extends:
    - .default-retry
    - .default-before_script
    - .assets-compile-cache
-  image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-graphicsmagick-1.3.34-docker-19.03.1
+  variables:
+    SETUP_DB: "false"
+    # we override the max_old_space_size to prevent OOM errors
+    NODE_OPTIONS: --max_old_space_size=3584
+    WEBPACK_VENDOR_DLL: "true"
+.compile-assets-base:
+  extends: .frontend-base
+  image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-git-2.26-lfs-2.9-node-12.x-yarn-1.21-graphicsmagick-1.3.34
  stage: prepare
+  script:
+    - node --version
+    - run_timed_command "retry yarn install --frozen-lockfile"
+    - free -m
+    - run_timed_command "bin/rake gitlab:assets:compile > assets-compile.log 2>&1"
+    - run_timed_command "scripts/clean-old-cached-assets"
+compile-production-assets:
+  extends:
+    - .compile-assets-base
+    - .frontend:rules:compile-production-assets
  variables:
    NODE_ENV: "production"
    RAILS_ENV: "production"
-    SETUP_DB: "false"
    WEBPACK_REPORT: "true"
-    # we override the max_old_space_size to prevent OOM errors
-    NODE_OPTIONS: --max_old_space_size=3584
-  cache:
-    key: "assets-compile:production:v1"
  artifacts:
    name: webpack-report
    expire_in: 31d
    paths:
-      - webpack-report/
      - assets-compile.log
      # These assets are used in multiple locations:
      # - in `build-assets-image` job to create assets image for packaging systems
      # - GitLab UI for integration tests: https://gitlab.com/gitlab-org/gitlab-ui/-/blob/e88493b3c855aea30bf60baee692a64606b0eb1e/.storybook/preview-head.pug#L1
-      - public/assets
+      - public/assets/
+      - webpack-report/
    when: always
-  script:
+  after_script:
-    - node --version
-    - retry yarn install --frozen-lockfile --production --cache-folder .yarn-cache --prefer-offline
-    - free -m
-    - time bin/rake gitlab:assets:compile > assets-compile.log 2>&1
-    - scripts/clean-old-cached-assets
    - rm -f /etc/apt/sources.list.d/google*.list  # We don't need to update Chrome here
-gitlab:assets:compile pull-push-cache:
+compile-test-assets:
  extends:
-    - .gitlab:assets:compile-metadata
+    - .compile-assets-base
-    - .frontend:rules:gitlab-assets-compile-pull-push-cache
+    - .frontend:rules:compile-test-assets
-  cache:
-    policy: pull-push
-gitlab:assets:compile pull-cache:
-  extends:
-    - .gitlab:assets:compile-metadata
-    - .frontend:rules:gitlab-assets-compile-pull-cache
-  cache:
-    policy: pull
-build-assets-image:
-  extends:
-    - .use-kaniko
-    - .frontend:rules:gitlab-assets-compile-pull-cache
-  stage: build-images
-  needs: ["gitlab:assets:compile pull-cache"]
-  variables:
-    GIT_DEPTH: "1"
-  script:
-    # TODO: Change the image tag to be the MD5 of assets files and skip image building if the image exists
-    # We'll also need to pass GITLAB_ASSETS_TAG to the trigerred omnibus-gitlab pipeline similarly to how we do it for trigerred CNG pipelines
-    # https://gitlab.com/gitlab-org/gitlab/issues/208389
-    - scripts/build_assets_image
-.compile-assets-metadata:
-  extends:
-    - .default-retry
-    - .default-before_script
-    - .assets-compile-cache
-  stage: prepare
-  script:
-    - node --version
-    - retry yarn install --frozen-lockfile --cache-folder .yarn-cache --prefer-offline
-    - free -m
-    - time bin/rake gitlab:assets:compile > assets-compile.log 2>&1
-    - scripts/clean-old-cached-assets
-  variables:
-    SETUP_DB: "false"
-    # we override the max_old_space_size to prevent OOM errors
-    NODE_OPTIONS: --max_old_space_size=3584
-    WEBPACK_VENDOR_DLL: "true"
-  cache:
-    key: "assets-compile:test:v1"
  artifacts:
    expire_in: 7d
    paths:
-      - node_modules
-      - public/assets
      - assets-compile.log
+      - public/assets/
+      - node_modules/@gitlab/svgs/dist/icons.json  # app/helpers/icons_helper.rb uses this file
    when: always
-compile-assets pull-push-cache:
+compile-test-assets as-if-foss:
  extends:
-    - .compile-assets-metadata
+    - compile-test-assets
-    - .frontend:rules:compile-assets-pull-push-cache
+    - .frontend:rules:compile-test-assets-as-if-foss
-  cache:
+    - .as-if-foss
-    policy: pull-push
-compile-assets pull-push-cache as-if-foss:
+update-assets-compile-production-cache:
  extends:
-    - .compile-assets-metadata
+    - compile-production-assets
-    - .frontend:rules:compile-assets-pull-push-cache-as-if-foss
+    - .shared:rules:update-cache
-    - .as-if-foss
+  stage: prepare
+  artifacts: {}  # This job's purpose is only to update the cache.
  cache:
-    policy: pull-push
+    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
-    key: "assets-compile:test:as-if-foss:v1"
-compile-assets pull-cache:
+update-assets-compile-test-cache:
  extends:
-    - .compile-assets-metadata
+    - compile-test-assets
-    - .frontend:rules:compile-assets-pull-cache
+    - .shared:rules:update-cache
+  stage: prepare
+  artifacts: {}  # This job's purpose is only to update the cache.
  cache:
-    policy: pull
+    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
-compile-assets pull-cache as-if-foss:
+update-yarn-cache:
  extends:
-    - .compile-assets-metadata
+    - .default-retry
-    - .frontend:rules:compile-assets-pull-cache-as-if-foss
+    - .yarn-cache
-    - .as-if-foss
+    - .shared:rules:update-cache
+  stage: prepare
+  script:
+    - source scripts/utils.sh
+    - run_timed_command "retry yarn install --frozen-lockfile"
  cache:
-    policy: pull
+    policy: push
-    key: "assets-compile:test:as-if-foss:v1"
+build-assets-image:
+  extends:
+    - .use-kaniko
+    - .frontend:rules:compile-production-assets
+  stage: build-images
+  needs: ["compile-production-assets"]
+  variables:
+    GIT_DEPTH: "1"
+  script:
+    # TODO: Change the image tag to be the MD5 of assets files and skip image building if the image exists
+    # We'll also need to pass GITLAB_ASSETS_TAG to the trigerred omnibus-gitlab pipeline similarly to how we do it for trigerred CNG pipelines
+    # https://gitlab.com/gitlab-org/gitlab/issues/208389
+    - run_timed_command "scripts/build_assets_image"
 .frontend-fixtures-base:
  extends:
-    - .default-retry
+    - .frontend-base
    - .rails-cache
-    - .default-before_script
    - .use-pg11
  stage: fixtures
-  needs: ["setup-test-env", "compile-assets pull-cache"]
+  needs: ["setup-test-env", "compile-test-assets"]
+  variables:
+    SETUP_DB: "true"
  script:
    - run_timed_command "scripts/gitaly-test-build"
    - run_timed_command "scripts/gitaly-test-spawn"
-    - run_timed_command "bundle exec rake frontend:fixtures"
+    - run_timed_command "bin/rake frontend:fixtures"
  artifacts:
    name: frontend-fixtures
    expire_in: 31d
    when: always
    paths:
-      - node_modules
-      - public/assets
      - tmp/tests/frontend/
 frontend-fixtures:
@@ -164,25 +135,27 @@ frontend-fixtures-as-if-foss:
    - .frontend:rules:default-frontend-jobs-as-if-foss
    - .as-if-foss
-.frontend-job-base:
+.frontend-test-base:
  extends:
    - .default-retry
-    - .default-cache
+    - .yarn-cache
-    - .default-before_script
  variables:
    USE_BUNDLE_INSTALL: "false"
    SETUP_DB: "false"
  stage: test
+  before_script:
+    - source scripts/utils.sh
 .karma-base:
-  extends: .frontend-job-base
+  extends: .frontend-test-base
  variables:
    # we override the max_old_space_size to prevent OOM errors
    NODE_OPTIONS: --max_old_space_size=3584
  script:
+    - source scripts/utils.sh
    - export BABEL_ENV=coverage CHROME_LOG_FILE=chrome_debug.log
-    - date
+    - run_timed_command "retry yarn install --frozen-lockfile"
-    - yarn karma
+    - run_timed_command "yarn karma"
 karma:
  extends:
@@ -209,15 +182,11 @@ karma-as-if-foss:
  needs: ["frontend-fixtures-as-if-foss"]
 .jest-base:
-  extends: .frontend-job-base
+  extends: .frontend-test-base
  script:
-    - date
+    - source scripts/utils.sh
-    - yarn jest --ci --coverage --testSequencer ./scripts/frontend/parallel_ci_sequencer.js
+    - run_timed_command "retry yarn install --frozen-lockfile"
-  cache:
+    - run_timed_command "yarn jest --ci --coverage --testSequencer ./scripts/frontend/parallel_ci_sequencer.js"
-    key: jest
-    paths:
-      - tmp/cache/jest/
-    policy: pull-push
 jest:
  extends:
@@ -238,17 +207,13 @@ jest:
 jest-integration:
  extends:
-    - .frontend-job-base
+    - .frontend-test-base
    - .frontend:rules:default-frontend-jobs
  script:
-    - date
+    - source scripts/utils.sh
-    - yarn jest:integration --ci
+    - run_timed_command "retry yarn install --frozen-lockfile"
+    - run_timed_command "yarn jest:integration --ci"
  needs: ["frontend-fixtures"]
-  cache:
-    key: jest-integration
-    paths:
-      - tmp/cache/jest/
-    policy: pull-push
 jest-as-if-foss:
  extends:
@@ -256,8 +221,6 @@ jest-as-if-foss:
    - .frontend:rules:default-frontend-jobs-as-if-foss
    - .as-if-foss
  needs: ["frontend-fixtures-as-if-foss"]
-  cache:
-    policy: pull
 coverage-frontend:
  extends:
@@ -268,33 +231,26 @@ coverage-frontend:
  stage: post-test
  before_script:
    - source scripts/utils.sh
-    - retry yarn install --frozen-lockfile
+    - run_timed_command "retry yarn install --frozen-lockfile"
  script:
-    - yarn node scripts/frontend/merge_coverage_frontend.js
+    - run_timed_command "yarn node scripts/frontend/merge_coverage_frontend.js"
  artifacts:
    name: coverage-frontend
    expire_in: 31d
    paths:
      - coverage-frontend/
-  cache:
-    policy: pull
 .qa-frontend-node:
  extends:
    - .default-retry
+    - .yarn-cache
    - .frontend:rules:qa-frontend-node
  stage: test
  dependencies: []
-  cache:
-    key: "$CI_JOB_NAME"
-    paths:
-      - .yarn-cache/
-    policy: pull-push
  script:
-    - date
+    - source scripts/utils.sh
-    - yarn install --frozen-lockfile --cache-folder .yarn-cache --prefer-offline
+    - run_timed_command "yarn install --frozen-lockfile"
-    - date
+    - run_timed_command "yarn run webpack-prod"
-    - yarn run webpack-prod
 qa-frontend-node:10:
  extends: .qa-frontend-node
@@ -309,25 +265,18 @@ qa-frontend-node:latest:
 webpack-dev-server:
  extends:
    - .default-retry
+    - .yarn-cache
    - .frontend:rules:default-frontend-jobs
  stage: test
  needs: []
  variables:
    WEBPACK_MEMORY_TEST: "true"
    WEBPACK_VENDOR_DLL: "true"
-  cache:
-    key:
-      files:
-        - yarn.lock
-      prefix: "v1"
-    paths:
-      - node_modules/
-      - tmp/cache/webpack-dlls/
  script:
    - source scripts/utils.sh
-    - retry yarn install --frozen-lockfile
+    - run_timed_command "retry yarn install --frozen-lockfile"
-    - retry yarn webpack-vendor
+    - run_timed_command "retry yarn webpack-vendor"
-    - node --expose-gc node_modules/.bin/webpack-dev-server --config config/webpack.config.js
+    - run_timed_command "node --expose-gc node_modules/.bin/webpack-dev-server --config config/webpack.config.js"
  artifacts:
    name: webpack-dev-server
    expire_in: 31d
@@ -340,7 +289,7 @@ bundle-size-review:
    - .frontend:rules:bundle-size-review
  image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger
  stage: test
-  needs: ["gitlab:assets:compile pull-cache"]
+  needs: ["compile-production-assets"]
  script:
    - mkdir -p bundle-size-review
    - cp webpack-report/index.html bundle-size-review/bundle-report.html

--- a/.gitlab/ci/global.gitlab-ci.yml
+++ b/.gitlab/ci/global.gitlab-ci.yml
@@ -10,46 +10,58 @@
 .default-before_script:
  before_script:
-    - date
    - '[ "$FOSS_ONLY" = "1" ] && rm -rf ee/ qa/spec/ee/ qa/qa/specs/features/ee/ qa/qa/ee/ qa/qa/ee.rb'
    - export GOPATH=$CI_PROJECT_DIR/.go
    - mkdir -p $GOPATH
    - source scripts/utils.sh
    - source scripts/prepare_build.sh
-    - date
-# Jobs that only need to pull cache
+.rails-cache:
-.default-cache:
  cache:
-    key: "debian-stretch-ruby-2.6.6-pg11-node-12.x"
+    key: "rails-v1"
    paths:
-      - .go/pkg/mod
+      - vendor/ruby/
-      - vendor/ruby
+      - vendor/gitaly-ruby/
-      - .yarn-cache/
+      - .go/pkg/mod/
-      - vendor/gitaly-ruby
    policy: pull
-.rails-cache:
+.static-analysis-cache:
+  cache:
+    key: "static-analysis-v1"
+    paths:
+      - vendor/ruby/
+      - node_modules/
+      - tmp/rubocop_cache/
+    policy: pull
+.qa-cache:
  cache:
-    key:
+    key: "qa-v1"
-      files:
-        - Gemfile.lock
-        - GITALY_SERVER_VERSION
-      prefix: "ruby-go-cache-v1"
    paths:
-      - vendor/ruby
+      - qa/vendor/ruby/
-      - vendor/gitaly-ruby
-      - .go/pkg/mod
    policy: pull
 .yarn-cache:
  cache:
-    key:
+    key: "yarn-v1"
-      files:
-        - yarn.lock
-      prefix: "v1"
    paths:
      - node_modules/
+      - tmp/cache/webpack-dlls/
+    policy: pull
+.assets-compile-cache:
+  cache:
+    key: "assets-compile-${NODE_ENV}-v1"
+    paths:
+      - vendor/ruby/
+      - node_modules/
+      - assets-hash.txt
+      - public/assets/webpack/
+      - tmp/cache/assets/sprockets/
+      - tmp/cache/babel-loader/
+      - tmp/cache/vue-loader/
+      - tmp/cache/webpack-dlls/
+    policy: pull
 .use-pg11:
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34"
@@ -75,6 +87,7 @@
    name: gcr.io/kaniko-project/executor:debug-v0.20.0
    entrypoint: [""]
  before_script:
+    - source scripts/utils.sh
    - mkdir -p /kaniko/.docker
    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json

--- a/.gitlab/ci/memory.gitlab-ci.yml
+++ b/.gitlab/ci/memory.gitlab-ci.yml
 .only-code-memory-job-base:
  extends:
    - .default-retry
-    - .default-cache
+    - .rails-cache
    - .default-before_script
    - .memory:rules
@@ -39,7 +39,7 @@ memory-on-boot:
    - .only-code-memory-job-base
    - .use-pg11
  stage: test
-  needs: ["setup-test-env", "compile-assets pull-cache"]
+  needs: ["setup-test-env", "compile-test-assets"]
  variables:
    NODE_ENV: "production"
    RAILS_ENV: "production"

--- a/.gitlab/ci/pages.gitlab-ci.yml
+++ b/.gitlab/ci/pages.gitlab-ci.yml
@@ -7,7 +7,7 @@ pages:
    - rspec:coverage
    - coverage-frontend
    - karma
-    - gitlab:assets:compile pull-cache
+    - compile-production-assets
  script:
    - mv public/ .public/
    - mkdir public/

--- a/.gitlab/ci/qa.gitlab-ci.yml
+++ b/.gitlab/ci/qa.gitlab-ci.yml
 .qa-job-base:
  extends:
    - .default-retry
+    - .qa-cache
  stage: test
  needs: []
-  cache:
-    key: "qa-framework-jobs:v1"
-    paths:
-      - vendor/ruby
  before_script:
    - '[ "$FOSS_ONLY" = "1" ] && rm -rf ee/ qa/spec/ee/ qa/qa/specs/features/ee/ qa/qa/ee/ qa/qa/ee.rb'
    - cd qa/
@@ -22,11 +19,9 @@ qa:internal:
 qa:internal-as-if-foss:
  extends:
-    - .qa-job-base
+    - qa:internal
    - .qa:rules:as-if-foss
    - .as-if-foss
-  script:
-    - bundle exec rspec
 qa:selectors:
  extends:
@@ -41,6 +36,16 @@ qa:selectors-as-if-foss:
    - .qa:rules:as-if-foss
    - .as-if-foss
+update-qa-cache:
+  extends:
+    - .qa-job-base
+    - .shared:rules:update-cache
+  stage: prepare
+  script:
+    - echo "Cache has been updated and ready to be uploaded."
+  cache:
+    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
 .package-and-qa-base:
  image: ruby:2.6-alpine
  stage: qa

--- a/.gitlab/ci/rails.gitlab-ci.yml
+++ b/.gitlab/ci/rails.gitlab-ci.yml
-.rails:needs:setup-and-assets:
-  needs: ["setup-test-env", "compile-assets pull-cache"]
 .rails-job-base:
  extends:
    - .default-retry
@@ -35,33 +32,54 @@ setup-test-env:
      - tmp/tests/repositories
      - tmp/tests/second_storage
    when: always
+update-rails-cache:
+  extends:
+    - setup-test-env
+    - .shared:rules:update-cache
+  artifacts: {}  # This job's purpose is only to update the cache.
  cache:
+    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
+.static-analysis-base:
+  extends:
+    - .default-retry
+    - .default-before_script
+    - .static-analysis-cache
+  needs: []
+  variables:
+    SETUP_DB: "false"
+    ENABLE_SPRING: "1"
+update-static-analysis-cache:
+  extends:
+    - .static-analysis-base
+    - .shared:rules:update-cache
+  stage: prepare
+  script:
+    - rm -rf ./node_modules  # We remove node_modules because there's no mechanism to remove stall entries.
+    - run_timed_command "retry yarn install --frozen-lockfile"
+    - bundle exec rubocop --parallel  # For the moment we only cache `vendor/ruby/`, `node_modules/`, and `tmp/rubocop_cache` so we don't need to run all the tasks,
+  cache:
+    # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up but RuboCop has a mechanism
+    # for keeping only the N latest cache files, so we take advantage of it with `pull-push` and removing `node_modules` at the start of the job.
    policy: pull-push
 static-analysis:
  extends:
-    - .rails-job-base
+    - .static-analysis-base
    - .rails:rules:default-refs-code-backstage-qa
-    - .rails:needs:setup-and-assets
  stage: test
-  variables:
-    SETUP_DB: "false"
-    ENABLE_SPRING: "1"
  parallel: 4
  script:
+    - run_timed_command "retry yarn install --frozen-lockfile"
    - scripts/static-analysis
-  cache:
-    key: "ruby-2.6.6-pg11-rubocop"
-    paths:
-      - vendor/ruby
-      - tmp/rubocop_cache
-    policy: pull-push
 downtime_check:
  extends:
    - .rails-job-base
    - .rails:rules:downtime_check
-  needs: ["setup-test-env"]
+  needs: []
  stage: test
  variables:
    SETUP_DB: "false"
@@ -71,7 +89,7 @@ downtime_check:
 .rspec-base:
  extends: .rails-job-base
  stage: test
-  needs: ["setup-test-env", "retrieve-tests-metadata", "compile-assets pull-cache"]
+  needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets"]
  script:
    - run_timed_command "scripts/gitaly-test-build"
    - run_timed_command "scripts/gitaly-test-spawn"
@@ -219,8 +237,6 @@ rspec:coverage:
    - memory-on-boot
  variables:
    SETUP_DB: "false"
-  cache:
-    policy: pull
  script:
    - bundle exec scripts/merge-simplecov
    - bundle exec scripts/gather-test-memory-data
@@ -248,7 +264,7 @@ rspec:coverage:
    - .rails:rules:as-if-foss
    - .as-if-foss
    - .use-pg11
-  needs: ["setup-test-env", "retrieve-tests-metadata", "compile-assets pull-cache as-if-foss"]
+  needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss"]
 .rspec-ee-base-pg11:
  extends:

--- a/.gitlab/ci/review.gitlab-ci.yml
+++ b/.gitlab/ci/review.gitlab-ci.yml
@@ -38,7 +38,7 @@ review-build-cng:
    - install_api_client_dependencies_with_apk
    - install_gitlab_gem
  needs:
-    - job: gitlab:assets:compile pull-cache
+    - job: compile-production-assets
      artifacts: false
  script:
    - BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng
@@ -238,5 +238,3 @@ danger-review:
    - source scripts/utils.sh
    - retry yarn install --frozen-lockfile
    - danger --fail-on-errors=true --verbose
-  cache:
-    policy: pull
--- a/.gitlab/ci/rules.gitlab-ci.yml
+++ b/.gitlab/ci/rules.gitlab-ci.yml
@@ -37,6 +37,9 @@
 .if-merge-request-title-as-if-foss: &if-merge-request-title-as-if-foss
  if: '$CI_MERGE_REQUEST_TITLE =~ /RUN AS-IF-FOSS/'
+.if-merge-request-title-update-caches: &if-merge-request-title-update-caches
+  if: '$CI_MERGE_REQUEST_TITLE =~ /UPDATE CACHE/'
 .if-security-merge-request: &if-security-merge-request
  if: '$CI_PROJECT_NAMESPACE == "gitlab-org/security" && $CI_MERGE_REQUEST_IID'
@@ -183,6 +186,14 @@
  - ".dockerignore"
  - "qa/**/*"
+################
+# Shared rules #
+################
+.shared:rules:update-cache:
+  rules:
+    - <<: *if-master-schedule-2-hourly
+    - <<: *if-merge-request-title-update-caches
 ####################
 # Cache repo rules #
 ####################
@@ -244,45 +255,18 @@
 ##################
 # Frontend rules #
 ##################
-# This job only runs on `master` since it pushes to the cache.
+.frontend:rules:compile-production-assets:
-.frontend:rules:gitlab-assets-compile-pull-push-cache:
-  rules:
-    - <<: *if-not-canonical-namespace
-      when: never
-    - <<: *if-master-refs
-      changes: *code-backstage-qa-patterns
-      when: on_success
-.frontend:rules:gitlab-assets-compile-pull-cache:
  rules:
    - <<: *if-not-canonical-namespace
      when: never
    - <<: *if-default-refs
      changes: *code-backstage-qa-patterns
-      when: on_success
-.frontend:rules:compile-assets-pull-push-cache:
-  rules:
-    - <<: *if-master-refs
-      changes: *code-backstage-qa-patterns
-      when: on_success
-# This job only runs on `master` since it pushes to the cache.
-.frontend:rules:compile-assets-pull-push-cache-as-if-foss:
-  rules:
-    - <<: *if-not-ee
-      when: never
-    - <<: *if-master-push
-      changes: *code-backstage-qa-patterns
-    - <<: *if-master-schedule-2-hourly
-.frontend:rules:compile-assets-pull-cache:
+.frontend:rules:compile-test-assets:
  rules:
-    - <<: *if-default-refs
+    - changes: *code-backstage-qa-patterns
-      changes: *code-backstage-qa-patterns
-      when: on_success
-.frontend:rules:compile-assets-pull-cache-as-if-foss:
+.frontend:rules:compile-test-assets-as-if-foss:
  rules:
    - <<: *if-not-ee
      when: never
@@ -299,7 +283,6 @@
  rules:
    - <<: *if-default-refs
      changes: *code-backstage-patterns
-      when: on_success
 .frontend:rules:default-frontend-jobs-as-if-foss:
  rules:
@@ -327,10 +310,8 @@
  rules:
    - <<: *if-master-refs
      changes: *frontend-dependency-patterns
-      when: on_success
    - <<: *if-merge-request
      changes: *frontend-dependency-patterns
-      when: on_success
 .frontend:rules:qa-frontend-node-latest:
  rules:

--- a/.gitlab/ci/setup.gitlab-ci.yml
+++ b/.gitlab/ci/setup.gitlab-ci.yml
@@ -3,7 +3,7 @@
 cache gems:
  extends:
    - .default-retry
-    - .default-cache
+    - .rails-cache
    - .default-before_script
    - .setup:rules:cache-gems
  stage: test

--- a/.gitlab/ci/test-metadata.gitlab-ci.yml
+++ b/.gitlab/ci/test-metadata.gitlab-ci.yml
@@ -3,11 +3,6 @@
    TESTS_METADATA_S3_BUCKET: "gitlab-ce-cache"
  before_script:
    - source scripts/utils.sh
-  cache:
-    key: tests_metadata
-    paths:
-      - knapsack/
-      - rspec_flaky/
  artifacts:
    expire_in: 31d
    paths:
@@ -20,8 +15,6 @@ retrieve-tests-metadata:
    - .tests-metadata-state
    - .test-metadata:rules:retrieve-tests-metadata
  stage: prepare
-  cache:
-    policy: pull
  script:
    - source scripts/rspec_helpers.sh
    - retrieve_tests_metadata
@@ -44,8 +37,6 @@ update-tests-metadata:
    - rspec-ee unit pg11 geo
    - rspec-ee integration pg11 geo
    - rspec-ee system pg11 geo
-  cache:
-    policy: push
  script:
    - retry gem install fog-aws mime-types activesupport rspec_profiling postgres-copy --no-document
    - source scripts/rspec_helpers.sh

--- a/.rubocop.yml
+++ b/.rubocop.yml
@@ -27,6 +27,7 @@ AllCops:
    - 'plugins/**/*'
    - 'file_hooks/**/*'
  CacheRootDirectory: tmp
+  MaxFilesInCache: 18000
 Cop/AvoidKeywordArgumentsInSidekiqWorkers:
  Enabled: true

--- a/config/initializers/01_secret_token.rb
+++ b/config/initializers/01_secret_token.rb
@@ -61,6 +61,8 @@ def generate_new_rsa_private_key
 end
 def warn_missing_secret(secret)
+  return if Rails.env.test?
  warn "Missing Rails.application.secrets.#{secret} for #{Rails.env} environment. The secret will be generated and stored in config/secrets.yml."
 end

--- a/doc/development/pipelines.md
+++ b/doc/development/pipelines.md
@@ -11,173 +11,17 @@ We're striving to [dogfood](https://about.gitlab.com/handbook/engineering/#dogfo
 GitLab [CI/CD features and best-practices](../ci/yaml/README.md)
 as much as possible.
-## Stages
+## Overview
-The current stages are:
+### Pipeline types
- `sync`: This stage is used to synchronize changes from <https://gitlab.com/gitlab-org/gitlab> to
-  <https://gitlab.com/gitlab-org/gitlab-foss>.
- `prepare`: This stage includes jobs that prepare artifacts that are needed by
-  jobs in subsequent stages.
- `build-images`: This stage includes jobs that prepare Docker images
-  that are needed by jobs in subsequent stages or downstream pipelines.
- `fixtures`: This stage includes jobs that prepare fixtures needed by frontend tests.
- `test`: This stage includes most of the tests, DB/migration jobs, and static analysis jobs.
- `post-test`: This stage includes jobs that build reports or gather data from
-  the `test` stage's jobs (e.g. coverage, Knapsack metadata etc.).
- `review-prepare`: This stage includes a job that build the CNG images that are
-  later used by the (Helm) Review App deployment (see
-  [Review Apps](testing_guide/review_apps.md) for details).
- `review`: This stage includes jobs that deploy the GitLab and Docs Review Apps.
- `qa`: This stage includes jobs that perform QA tasks against the Review App
-  that is deployed in the previous stage.
- `post-qa`: This stage includes jobs that build reports or gather data from
-  the `qa` stage's jobs (e.g. Review App performance report).
- `pages`: This stage includes a job that deploys the various reports as
-  GitLab Pages (e.g. <https://gitlab-org.gitlab.io/gitlab/coverage-ruby/>,
-  <https://gitlab-org.gitlab.io/gitlab/coverage-javascript/>,
-  <https://gitlab-org.gitlab.io/gitlab/webpack-report/>).
-## Default image
-The default image is defined in <https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml>.
-It includes Ruby, Go, Git, Git LFS, Chrome, Node, Yarn, PostgreSQL, and Graphics Magick.
-The images used in our pipelines are configured in the
-[`gitlab-org/gitlab-build-images`](https://gitlab.com/gitlab-org/gitlab-build-images)
-project, which is push-mirrored to <https://dev.gitlab.org/gitlab/gitlab-build-images>
-for redundancy.
-The current version of the build images can be found in the
-["Used by GitLab section"](https://gitlab.com/gitlab-org/gitlab-build-images/blob/master/.gitlab-ci.yml).
-## Default variables
-In addition to the [predefined variables](../ci/variables/predefined_variables.md),
-each pipeline includes default variables defined in
-<https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml>.
-## Common job definitions
-Most of the jobs [extend from a few CI definitions](../ci/yaml/README.md#extends)
-defined in [`.gitlab/ci/global.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/global.gitlab-ci.yml)
-that are scoped to a single [configuration parameter](../ci/yaml/README.md#configuration-parameters).
-| Job definitions  | Description |
-|------------------|-------------|
-| `.default-tags` | Ensures a job has the `gitlab-org` tag to ensure it's using our dedicated runners. |
-| `.default-retry` | Allows a job to [retry](../ci/yaml/README.md#retry) upon `unknown_failure`, `api_failure`, `runner_system_failure`, `job_execution_timeout`, or `stuck_or_timeout_failure`. |
-| `.default-before_script` | Allows a job to use a default `before_script` definition suitable for Ruby/Rails tasks that may need a database running (e.g. tests). |
-| `.default-cache` | Allows a job to use a default `cache` definition suitable for Ruby/Rails and frontend tasks. |
-| `.use-pg11` | Allows a job to use the `postgres:11.6` and `redis:alpine` services. |
-| `.use-pg11-ee` | Same as `.use-pg11` but also use the `docker.elastic.co/elasticsearch/elasticsearch:6.4.2` services. |
-| `.use-kaniko` | Allows a job to use the `kaniko` tool to build Docker images. |
-| `.as-if-foss` | Simulate the FOSS project by setting the `FOSS_ONLY='1'` environment variable. |
-## `workflow:rules`
-We're using the [`workflow:rules` keyword](../ci/yaml/README.md#workflowrules) to
-define default rules to determine whether or not a pipeline is created.
-These rules are defined in <https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml>
-and are as follows:
-1. If `$FORCE_GITLAB_CI` is set, create a pipeline.
-1. For merge requests, create a pipeline.
-1. For `master` branch, create a pipeline (this includes on schedules, pushes, merges, etc.).
-1. For tags, create a pipeline.
-1. If `$GITLAB_INTERNAL` isn't set, don't create a pipeline.
-1. For stable, auto-deploy, and security branches, create a pipeline.
-1. For any other cases (e.g. when pushing a branch with no MR for it), no pipeline is created.
-## `rules`, `if:` conditions and `changes:` patterns
-We're using the [`rules` keyword](../ci/yaml/README.md#rules) extensively.
-All `rules` definitions are defined in
-<https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rules.gitlab-ci.yml>,
-then included in individual jobs via [`extends`](../ci/yaml/README.md#extends).
-The `rules` definitions are composed of `if:` conditions and `changes:` patterns,
-which are also defined in
-<https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rules.gitlab-ci.yml>
-and included in `rules` definitions via [YAML anchors](../ci/yaml/README.md#anchors)
-### `if:` conditions
-| `if:` conditions | Description | Notes |
-|------------------|-------------|-------|
-| `if-not-canonical-namespace`                                 | Matches if the project isn't in the canonical (`gitlab-org/`) or security (`gitlab-org/security`) namespace. | Use to create a job for forks (by using `when: on_success\|manual`), or **not** create a job for forks (by using `when: never`). |
-| `if-not-ee`                                                  | Matches if the project isn't EE (i.e. project name isn't `gitlab` or `gitlab-ee`). | Use to create a job only in the FOSS project (by using `when: on_success|manual`), or **not** create a job if the project is EE (by using `when: never`). |
-| `if-not-foss`                                                | Matches if the project isn't FOSS (i.e. project name isn't `gitlab-foss`, `gitlab-ce`, or `gitlabhq`). | Use to create a job only in the EE project (by using `when: on_success|manual`), or **not** create a job if the project is FOSS (by using `when: never`). |
-| `if-default-refs`                                            | Matches if the pipeline is for `master`, `/^[\d-]+-stable(-ee)?$/` (stable branches), `/^\d+-\d+-auto-deploy-\d+$/` (auto-deploy branches), `/^security\//` (security branches), merge requests, and tags. | Note that jobs won't be created for branches with this default configuration. |
-| `if-master-refs`                                             | Matches if the current branch is `master`. | |
-| `if-master-or-tag`                                           | Matches if the pipeline is for the `master` branch or for a tag. | |
-| `if-merge-request`                                           | Matches if the pipeline is for a merge request. | |
-| `if-nightly-master-schedule`                                 | Matches if the pipeline is for a `master` scheduled pipeline with `$NIGHTLY` set. | |
-| `if-dot-com-gitlab-org-schedule`                             | Limits jobs creation to scheduled pipelines for the `gitlab-org` group on GitLab.com. | |
-| `if-dot-com-gitlab-org-master`                               | Limits jobs creation to the `master` branch for the `gitlab-org` group on GitLab.com. | |
-| `if-dot-com-gitlab-org-merge-request`                        | Limits jobs creation to merge requests for the `gitlab-org` group on GitLab.com. | |
-| `if-dot-com-gitlab-org-and-security-tag`                     | Limits job creation to tags for the `gitlab-org` and `gitlab-org/security` groups on GitLab.com. | |
-| `if-dot-com-gitlab-org-and-security-merge-request`           | Limit jobs creation to merge requests for the `gitlab-org` and `gitlab-org/security` groups on GitLab.com. | |
-| `if-dot-com-ee-schedule`                                     | Limits jobs to scheduled pipelines for the `gitlab-org/gitlab` project on GitLab.com. | |
-| `if-cache-credentials-schedule`                              | Limits jobs to scheduled pipelines with the `$CI_REPO_CACHE_CREDENTIALS` variable set. | |
-### `changes:` patterns
-| `changes:` patterns          | Description                                                              |
-|------------------------------|--------------------------------------------------------------------------|
-| `ci-patterns`                | Only create job for CI config-related changes.                           |
-| `yaml-patterns`              | Only create job for YAML-related changes.                                |
-| `docs-patterns`              | Only create job for docs-related changes.                                |
-| `frontend-dependency-patterns` | Only create job when frontend dependencies are updated (i.e. `package.json`, and `yarn.lock`). changes. |
-| `frontend-patterns`          | Only create job for frontend-related changes.                           |
-| `backstage-patterns`         | Only create job for backstage-related changes (i.e. Danger, fixtures, RuboCop, specs). |
-| `code-patterns`              | Only create job for code-related changes.                                |
-| `qa-patterns`                | Only create job for QA-related changes.                                  |
-| `code-backstage-patterns`    | Combination of `code-patterns` and `backstage-patterns`.                 |
-| `code-qa-patterns`           | Combination of `code-patterns` and `qa-patterns`.                        |
-| `code-backstage-qa-patterns` | Combination of `code-patterns`, `backstage-patterns`, and `qa-patterns`. |
-## Interruptible jobs pipelines
-By default, all jobs are [interruptible](../ci/yaml/README.md#interruptible), except the
-`dont-interrupt-me` job which runs automatically on `master`, and is `manual`
-otherwise.
-If you want a running pipeline to finish even if you push new commits to a merge
-request, be sure to start the `dont-interrupt-me` job before pushing.
-## PostgreSQL versions testing
-### Current versions testing
-| Where? | PostgreSQL version |
-| ------ | ------ |
-| MRs | 11 |
-| `master` (non-scheduled pipelines) | 11 |
-| 2-hourly scheduled pipelines | 11 |
-### Long-term plan
-We follow the [PostgreSQL versions shipped with Omnibus GitLab](https://docs.gitlab.com/omnibus/package-information/postgresql_versions.html):
-| PostgreSQL version | 12.10 (April 2020) | 13.0 (May 2020) | 13.1 (June 2020) | 13.2 (July 2020) | 13.3 (August 2020) | 13.4, 13.5   | 13.6 (November 2020) | 14.0 (May 2021?) |
-| ------ | ------------------ | --------------- | ---------------- | ---------------- | ------------------ | ------------ | -------------------- | ---------------- |
-| PG9.6  | MRs/`master`/`2-hour`/`nightly` | -               | -                | -                | -                  | -            | -                    | -                |
-| PG10   | `nightly`           | -               | -                | -                | -                  | -            | -                    | -                |
-| PG11   | `master`/`2-hour` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | `nightly`              | -                |
-| PG12   | -                  | -               | -                | -                | `master`/`2-hour` | `master`/`2-hour` | MRs/`master`/`2-hour`/`nightly`         | `master`/`2-hour` |
-| PG13   | -                  | -               | -                | -                | -                  | -            | -                    | MRs/`master`/`2-hour`/`nightly`     |
-## Pipeline types
 Since we use the [`rules:`](../ci/yaml/README.md#rules) and [`needs:`](../ci/yaml/README.md#needs) keywords extensively,
 we have four main pipeline types which are described below. Note that an MR that includes multiple types of changes would
 have a pipelines that include jobs from multiple types (e.g. a combination of docs-only and code-only pipelines).
-### Docs-only MR pipeline
+#### Docs-only MR pipeline
+<details><summary>Click to view the graph</summary>
 Reference pipeline: <https://gitlab.com/gitlab-org/gitlab/pipelines/135236627>
@@ -191,7 +35,11 @@ graph LR
  end
 ```
-### Code-only MR pipeline
+</details>
+#### Code-only MR pipeline
+<details><summary>Click to view the graph</summary>
 Reference pipeline: <https://gitlab.com/gitlab-org/gitlab/pipelines/136295694>
@@ -204,11 +52,11 @@ graph RL;
    click 1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8100542&udv=0"
    1-2["build-qa-image (3.4 minutes)"];
    click 1-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914325&udv=0"
-    1-3["compile-assets pull-cache (9.06 minutes)"];
+    1-3["compile-test-assets (9.06 minutes)"];
    click 1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914317&udv=0"
-    1-4["compile-assets pull-cache as-if-foss (8.35 minutes)"];
+    1-4["compile-test-assets as-if-foss (8.35 minutes)"];
    click 1-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356616&udv=0"
-    1-5["gitlab:assets:compile pull-cache (22 minutes)"];
+    1-5["compile-production-assets (22 minutes)"];
    click 1-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914312&udv=0"
    1-6["setup-test-env (8.22 minutes)"];
    click 1-6 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914315&udv=0"
@@ -224,6 +72,8 @@ graph RL;
    1-18["kubesec-sast"];
    1-19["nodejs-scan-sast"];
    1-20["secrets-sast"];
+    1-21["static-analysis (17 minutes)"];
+    click 1-21 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914471&udv=0"
    class 1-3 criticalPath;
    class 1-6 criticalPath;
@@ -241,8 +91,6 @@ graph RL;
    2_1-1 & 2_1-2 & 2_1-3 & 2_1-4 --> 1-6;
  end
-  2_2-1["static-analysis (17 minutes)"];
-  click 2_2-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914471&udv=0"
  2_2-2["frontend-fixtures (17.2 minutes)"];
  class 2_2-2 criticalPath;
  click 2_2-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=7910143&udv=0"
@@ -252,13 +100,13 @@ graph RL;
  click 2_2-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356727&udv=0"
  2_2-5["webpack-dev-server (6.1 minutes)"];
  click 2_2-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8404303&udv=0"
-  subgraph "Needs `setup-test-env` & `compile-assets`";
+  subgraph "Needs `setup-test-env` & `compile-test-assets`";
-    2_2-1 & 2_2-2 & 2_2-4 & 2_2-5 --> 1-6 & 1-3;
+    2_2-2 & 2_2-4 & 2_2-5 --> 1-6 & 1-3;
    2_2-3 --> 1-6 & 1-4;
  end
  2_3-1["build-assets-image (2.5 minutes)"];
-  subgraph "Needs `gitlab:assets:compile`";
+  subgraph "Needs `compile-production-assets`";
    2_3-1 --> 1-5
  end
@@ -269,7 +117,7 @@ graph RL;
  end
  2_5-1["rspec & db jobs (12-22 minutes)"];
-  subgraph "Needs `compile-assets`, `setup-test-env`, & `retrieve-tests-metadata`";
+  subgraph "Needs `compile-test-assets`, `setup-test-env`, & `retrieve-tests-metadata`";
    2_5-1 --> 1-3 & 1-6 & 1-14;
    class 2_5-1 criticalPath;
    click 2_5-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations"
@@ -304,7 +152,11 @@ graph RL;
  end
 ```
-### Frontend-only MR pipeline
+</details>
+#### Frontend-only MR pipeline
+<details><summary>Click to view the graph</summary>
 Reference pipeline: <https://gitlab.com/gitlab-org/gitlab/pipelines/134661039>
@@ -317,11 +169,11 @@ graph RL;
    click 1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8100542&udv=0"
    1-2["build-qa-image (3.4 minutes)"];
    click 1-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914325&udv=0"
-    1-3["compile-assets pull-cache (9.06 minutes)"];
+    1-3["compile-test-assets (9.06 minutes)"];
    click 1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914317&udv=0"
-    1-4["compile-assets pull-cache as-if-foss (8.35 minutes)"];
+    1-4["compile-test-assets as-if-foss (8.35 minutes)"];
    click 1-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356616&udv=0"
-    1-5["gitlab:assets:compile pull-cache (22 minutes)"];
+    1-5["compile-production-assets (22 minutes)"];
    click 1-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914312&udv=0"
    1-6["setup-test-env (8.22 minutes)"];
    click 1-6 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914315&udv=0"
@@ -337,6 +189,8 @@ graph RL;
    1-18["kubesec-sast"];
    1-19["nodejs-scan-sast"];
    1-20["secrets-sast"];
+    1-21["static-analysis (17 minutes)"];
+    click 1-21 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914471&udv=0"
    class 1-3 criticalPath;
    class 1-5 criticalPath;
@@ -355,8 +209,6 @@ graph RL;
    2_1-1 & 2_1-2 & 2_1-3 & 2_1-4 --> 1-6;
  end
-  2_2-1["static-analysis (17 minutes)"];
-  click 2_2-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914471&udv=0"
  2_2-2["frontend-fixtures (17.2 minutes)"];
  class 2_2-2 criticalPath;
  click 2_2-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=7910143&udv=0"
@@ -366,14 +218,14 @@ graph RL;
  click 2_2-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356727&udv=0"
  2_2-5["webpack-dev-server (6.1 minutes)"];
  click 2_2-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8404303&udv=0"
-  subgraph "Needs `setup-test-env` & `compile-assets`";
+  subgraph "Needs `setup-test-env` & `compile-test-assets`";
-    2_2-1 & 2_2-2 & 2_2-4 & 2_2-5 --> 1-6 & 1-3;
+    2_2-2 & 2_2-4 & 2_2-5 --> 1-6 & 1-3;
    2_2-3 --> 1-6 & 1-4;
  end
  2_3-1["build-assets-image (2.5 minutes)"];
  class 2_3-1 criticalPath;
-  subgraph "Needs `gitlab:assets:compile`";
+  subgraph "Needs `compile-production-assets`";
    2_3-1 --> 1-5
  end
@@ -384,7 +236,7 @@ graph RL;
  end
  2_5-1["rspec & db jobs (12-22 minutes)"];
-  subgraph "Needs `compile-assets`, `setup-test-env, & `retrieve-tests-metadata`";
+  subgraph "Needs `compile-test-assets`, `setup-test-env, & `retrieve-tests-metadata`";
    2_5-1 --> 1-3 & 1-6 & 1-14;
    class 2_5-1 criticalPath;
    click 2_5-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations"
@@ -444,7 +296,11 @@ graph RL;
  end
 ```
-### QA-only MR pipeline
+</details>
+#### QA-only MR pipeline
+<details><summary>Click to view the graph</summary>
 Reference pipeline: <https://gitlab.com/gitlab-org/gitlab/pipelines/134645109>
@@ -457,11 +313,11 @@ graph RL;
    click 1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8100542&udv=0"
    1-2["build-qa-image (3.4 minutes)"];
    click 1-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914325&udv=0"
-    1-3["compile-assets pull-cache (9.06 minutes)"];
+    1-3["compile-test-assets (9.06 minutes)"];
    click 1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914317&udv=0"
-    1-4["compile-assets pull-cache as-if-foss (8.35 minutes)"];
+    1-4["compile-test-assets as-if-foss (8.35 minutes)"];
    click 1-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356616&udv=0"
-    1-5["gitlab:assets:compile pull-cache (22 minutes)"];
+    1-5["compile-production-assets (22 minutes)"];
    click 1-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914312&udv=0"
    1-6["setup-test-env (8.22 minutes)"];
    click 1-6 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914315&udv=0"
@@ -477,6 +333,8 @@ graph RL;
    1-18["kubesec-sast"];
    1-19["nodejs-scan-sast"];
    1-20["secrets-sast"];
+    1-21["static-analysis (17 minutes)"];
+    click 1-21 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914471&udv=0"
    class 1-5 criticalPath;
  end
@@ -487,14 +345,8 @@ graph RL;
    2_1-1 --> 1-6;
  end
-  2_2-1["static-analysis (17 minutes)"];
-  click 2_2-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914471&udv=0"
-  subgraph "Needs `setup-test-env` & `compile-assets`";
-    2_2-1 --> 1-6 & 1-3;
-  end
  2_3-1["build-assets-image (2.5 minutes)"];
-  subgraph "Needs `gitlab:assets:compile`";
+  subgraph "Needs `compile-production-assets`";
    2_3-1 --> 1-5
    class 2_3-1 criticalPath;
  end
@@ -507,16 +359,56 @@ graph RL;
  end
 ```
-## Test jobs
+</details>
+### `workflow:rules`
+We're using the [`workflow:rules` keyword](../ci/yaml/README.md#workflowrules) to
+define default rules to determine whether or not a pipeline is created.
+These rules are defined in <https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml>
+and are as follows:
+1. If `$FORCE_GITLAB_CI` is set, create a pipeline.
+1. For merge requests, create a pipeline.
+1. For `master` branch, create a pipeline (this includes on schedules, pushes, merges, etc.).
+1. For tags, create a pipeline.
+1. If `$GITLAB_INTERNAL` isn't set, don't create a pipeline.
+1. For stable, auto-deploy, and security branches, create a pipeline.
+1. For any other cases (e.g. when pushing a branch with no MR for it), no pipeline is created.
+### PostgreSQL versions testing
+#### Current versions testing
+| Where? | PostgreSQL version |
+| ------ | ------ |
+| MRs | 11 |
+| `master` (non-scheduled pipelines) | 11 |
+| 2-hourly scheduled pipelines | 11 |
+#### Long-term plan
+We follow the [PostgreSQL versions shipped with Omnibus GitLab](https://docs.gitlab.com/omnibus/package-information/postgresql_versions.html):
+| PostgreSQL version | 12.10 (April 2020) | 13.0 (May 2020) | 13.1 (June 2020) | 13.2 (July 2020) | 13.3 (August 2020) | 13.4, 13.5   | 13.6 (November 2020) | 14.0 (May 2021?) |
+| ------ | ------------------ | --------------- | ---------------- | ---------------- | ------------------ | ------------ | -------------------- | ---------------- |
+| PG9.6  | MRs/`master`/`2-hour`/`nightly` | -               | -                | -                | -                  | -            | -                    | -                |
+| PG10   | `nightly`           | -               | -                | -                | -                  | -            | -                    | -                |
+| PG11   | `master`/`2-hour` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | `nightly`              | -                |
+| PG12   | -                  | -               | -                | -                | `master`/`2-hour` | `master`/`2-hour` | MRs/`master`/`2-hour`/`nightly`         | `master`/`2-hour` |
+| PG13   | -                  | -               | -                | -                | -                  | -            | -                    | MRs/`master`/`2-hour`/`nightly`     |
+### Test jobs
 Consult [GitLab tests in the Continuous Integration (CI) context](testing_guide/ci.md)
 for more information.
-## Review app jobs
+### Review app jobs
 Consult the [Review Apps](testing_guide/review_apps.md) dedicated page for more information.
-## As-if-FOSS jobs
+### As-if-FOSS jobs
 The `* as-if-foss` jobs allows to run GitLab's test suite "as-if-FOSS", meaning as if the jobs would run in the context
 of the `gitlab-org/gitlab-foss` project. These jobs are only created in the following cases:
@@ -532,7 +424,39 @@ folders removed before the tests start running.
 The intent is to ensure that a change won't introduce a failure once the `gitlab-org/gitlab` project will be synced to
 the `gitlab-org/gitlab-foss` project.
-## Pre-clone step
+## Performance
+### Interruptible pipelines
+By default, all jobs are [interruptible](../ci/yaml/README.md#interruptible), except the
+`dont-interrupt-me` job which runs automatically on `master`, and is `manual`
+otherwise.
+If you want a running pipeline to finish even if you push new commits to a merge
+request, be sure to start the `dont-interrupt-me` job before pushing.
+### Caching strategy
+1. All jobs must only pull caches by default.
+1. All jobs must be able to pass with an empty cache. In other words, caches are only there to speed up jobs.
+1. We currently have 6 different caches defined in
+   [`.gitlab/ci/global.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/global.gitlab-ci.yml),
+   with fixed keys:
+   - `.rails-cache`.
+   - `.static-analysis-cache`.
+   - `.qa-cache`
+   - `.yarn-cache`.
+   - `.assets-compile-cache` (the key includes `${NODE_ENV}` so it's actually two different caches).
+1. Only 6 specific jobs, running in 2-hourly scheduled pipelines, are pushing (i.e. updating) to the caches:
+   - `update-rails-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
+   - `update-static-analysis-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
+   - `update-qa-cache`, defined in [`.gitlab/ci/qa.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/qa.gitlab-ci.yml).
+   - `update-assets-compile-production-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/frontend.gitlab-ci.yml).
+   - `update-assets-compile-test-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/frontend.gitlab-ci.yml).
+   - `update-yarn-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/frontend.gitlab-ci.yml).
+1. These jobs will run in merge requests whose title include `UPDATE CACHE`.
+### Pre-clone step
 The `gitlab-org/gitlab` project on GitLab.com uses a [pre-clone step](https://gitlab.com/gitlab-org/gitlab/-/issues/39134)
 to seed the project with a recent archive of the repository. This is done for
@@ -597,6 +521,124 @@ credentials are stored in the 1Password GitLab.com Production vault.
 Note that this bucket should be located in the same continent as the
 runner, or [network egress charges will apply](https://cloud.google.com/storage/pricing).
+## CI configuration internals
+### Stages
+The current stages are:
+- `sync`: This stage is used to synchronize changes from <https://gitlab.com/gitlab-org/gitlab> to
+  <https://gitlab.com/gitlab-org/gitlab-foss>.
+- `prepare`: This stage includes jobs that prepare artifacts that are needed by
+  jobs in subsequent stages.
+- `build-images`: This stage includes jobs that prepare Docker images
+  that are needed by jobs in subsequent stages or downstream pipelines.
+- `fixtures`: This stage includes jobs that prepare fixtures needed by frontend tests.
+- `test`: This stage includes most of the tests, DB/migration jobs, and static analysis jobs.
+- `post-test`: This stage includes jobs that build reports or gather data from
+  the `test` stage's jobs (e.g. coverage, Knapsack metadata etc.).
+- `review-prepare`: This stage includes a job that build the CNG images that are
+  later used by the (Helm) Review App deployment (see
+  [Review Apps](testing_guide/review_apps.md) for details).
+- `review`: This stage includes jobs that deploy the GitLab and Docs Review Apps.
+- `qa`: This stage includes jobs that perform QA tasks against the Review App
+  that is deployed in the previous stage.
+- `post-qa`: This stage includes jobs that build reports or gather data from
+  the `qa` stage's jobs (e.g. Review App performance report).
+- `pages`: This stage includes a job that deploys the various reports as
+  GitLab Pages (e.g. <https://gitlab-org.gitlab.io/gitlab/coverage-ruby/>,
+  <https://gitlab-org.gitlab.io/gitlab/coverage-javascript/>,
+  <https://gitlab-org.gitlab.io/gitlab/webpack-report/>).
+### Default image
+The default image is defined in <https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml>.
+It includes Ruby, Go, Git, Git LFS, Chrome, Node, Yarn, PostgreSQL, and Graphics Magick.
+The images used in our pipelines are configured in the
+[`gitlab-org/gitlab-build-images`](https://gitlab.com/gitlab-org/gitlab-build-images)
+project, which is push-mirrored to <https://dev.gitlab.org/gitlab/gitlab-build-images>
+for redundancy.
+The current version of the build images can be found in the
+["Used by GitLab section"](https://gitlab.com/gitlab-org/gitlab-build-images/blob/master/.gitlab-ci.yml).
+### Default variables
+In addition to the [predefined variables](../ci/variables/predefined_variables.md),
+each pipeline includes default variables defined in
+<https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml>.
+### Common job definitions
+Most of the jobs [extend from a few CI definitions](../ci/yaml/README.md#extends)
+defined in [`.gitlab/ci/global.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/global.gitlab-ci.yml)
+that are scoped to a single [configuration parameter](../ci/yaml/README.md#configuration-parameters).
+| Job definitions  | Description |
+|------------------|-------------|
+| `.default-tags` | Ensures a job has the `gitlab-org` tag to ensure it's using our dedicated runners. |
+| `.default-retry` | Allows a job to [retry](../ci/yaml/README.md#retry) upon `unknown_failure`, `api_failure`, `runner_system_failure`, `job_execution_timeout`, or `stuck_or_timeout_failure`. |
+| `.default-before_script` | Allows a job to use a default `before_script` definition suitable for Ruby/Rails tasks that may need a database running (e.g. tests). |
+| `.rails-cache` | Allows a job to use a default `cache` definition suitable for Ruby/Rails tasks. |
+| `.static-analysis-cache` | Allows a job to use a default `cache` definition suitable for static analysis tasks. |
+| `.yarn-cache` | Allows a job to use a default `cache` definition suitable for frontend jobs that do a `yarn install`. |
+| `.assets-compile-cache` | Allows a job to use a default `cache` definition suitable for frontend jobs that compile assets. |
+| `.use-pg11` | Allows a job to use the `postgres:11.6` and `redis:alpine` services. |
+| `.use-pg11-ee` | Same as `.use-pg11` but also use the `docker.elastic.co/elasticsearch/elasticsearch:6.4.2` services. |
+| `.use-kaniko` | Allows a job to use the `kaniko` tool to build Docker images. |
+| `.as-if-foss` | Simulate the FOSS project by setting the `FOSS_ONLY='1'` environment variable. |
+### `rules`, `if:` conditions and `changes:` patterns
+We're using the [`rules` keyword](../ci/yaml/README.md#rules) extensively.
+All `rules` definitions are defined in
+<https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rules.gitlab-ci.yml>,
+then included in individual jobs via [`extends`](../ci/yaml/README.md#extends).
+The `rules` definitions are composed of `if:` conditions and `changes:` patterns,
+which are also defined in
+<https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rules.gitlab-ci.yml>
+and included in `rules` definitions via [YAML anchors](../ci/yaml/README.md#anchors)
+#### `if:` conditions
+| `if:` conditions | Description | Notes |
+|------------------|-------------|-------|
+| `if-not-canonical-namespace`                                 | Matches if the project isn't in the canonical (`gitlab-org/`) or security (`gitlab-org/security`) namespace. | Use to create a job for forks (by using `when: on_success\|manual`), or **not** create a job for forks (by using `when: never`). |
+| `if-not-ee`                                                  | Matches if the project isn't EE (i.e. project name isn't `gitlab` or `gitlab-ee`). | Use to create a job only in the FOSS project (by using `when: on_success|manual`), or **not** create a job if the project is EE (by using `when: never`). |
+| `if-not-foss`                                                | Matches if the project isn't FOSS (i.e. project name isn't `gitlab-foss`, `gitlab-ce`, or `gitlabhq`). | Use to create a job only in the EE project (by using `when: on_success|manual`), or **not** create a job if the project is FOSS (by using `when: never`). |
+| `if-default-refs`                                            | Matches if the pipeline is for `master`, `/^[\d-]+-stable(-ee)?$/` (stable branches), `/^\d+-\d+-auto-deploy-\d+$/` (auto-deploy branches), `/^security\//` (security branches), merge requests, and tags. | Note that jobs won't be created for branches with this default configuration. |
+| `if-master-refs`                                             | Matches if the current branch is `master`. | |
+| `if-master-or-tag`                                           | Matches if the pipeline is for the `master` branch or for a tag. | |
+| `if-merge-request`                                           | Matches if the pipeline is for a merge request. | |
+| `if-nightly-master-schedule`                                 | Matches if the pipeline is for a `master` scheduled pipeline with `$NIGHTLY` set. | |
+| `if-dot-com-gitlab-org-schedule`                             | Limits jobs creation to scheduled pipelines for the `gitlab-org` group on GitLab.com. | |
+| `if-dot-com-gitlab-org-master`                               | Limits jobs creation to the `master` branch for the `gitlab-org` group on GitLab.com. | |
+| `if-dot-com-gitlab-org-merge-request`                        | Limits jobs creation to merge requests for the `gitlab-org` group on GitLab.com. | |
+| `if-dot-com-gitlab-org-and-security-tag`                     | Limits job creation to tags for the `gitlab-org` and `gitlab-org/security` groups on GitLab.com. | |
+| `if-dot-com-gitlab-org-and-security-merge-request`           | Limit jobs creation to merge requests for the `gitlab-org` and `gitlab-org/security` groups on GitLab.com. | |
+| `if-dot-com-ee-schedule`                                     | Limits jobs to scheduled pipelines for the `gitlab-org/gitlab` project on GitLab.com. | |
+| `if-cache-credentials-schedule`                              | Limits jobs to scheduled pipelines with the `$CI_REPO_CACHE_CREDENTIALS` variable set. | |
+#### `changes:` patterns
+| `changes:` patterns          | Description                                                              |
+|------------------------------|--------------------------------------------------------------------------|
+| `ci-patterns`                | Only create job for CI config-related changes.                           |
+| `yaml-patterns`              | Only create job for YAML-related changes.                                |
+| `docs-patterns`              | Only create job for docs-related changes.                                |
+| `frontend-dependency-patterns` | Only create job when frontend dependencies are updated (i.e. `package.json`, and `yarn.lock`). changes. |
+| `frontend-patterns`          | Only create job for frontend-related changes.                           |
+| `backstage-patterns`         | Only create job for backstage-related changes (i.e. Danger, fixtures, RuboCop, specs). |
+| `code-patterns`              | Only create job for code-related changes.                                |
+| `qa-patterns`                | Only create job for QA-related changes.                                  |
+| `code-backstage-patterns`    | Combination of `code-patterns` and `backstage-patterns`.                 |
+| `code-qa-patterns`           | Combination of `code-patterns` and `qa-patterns`.                        |
+| `code-backstage-qa-patterns` | Combination of `code-patterns`, `backstage-patterns`, and `qa-patterns`. |
 ---
 [Return to Development documentation](README.md)