Merge branch...

Merge branch '34322-investigate-how-to-reduce-the-duration-of-the-jest-job-to-under-15-minutes' into 'master'

Extract frontend fixture creation from jest & karma

See merge request gitlab-org/gitlab!24568

.gitlab-ci.yml

@@ -3,6 +3,7 @@ image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.5-golang-1.1
 stages:
   - sync
   - prepare
+  - fixtures
   - test
   - post-test
   - review-prepare

.gitlab/ci/dev-fixtures.gitlab-ci.yml

@@ -7,7 +7,6 @@
     - .use-pg9
   stage: test
   needs: ["setup-test-env"]
-  dependencies: ["setup-test-env"]
   variables:
     FIXTURE_PATH: "db/fixtures/development"
     SEED_CYCLE_ANALYTICS: "true"

.gitlab/ci/docs.gitlab-ci.yml

@@ -46,7 +46,9 @@ docs lint:
     - .docs:rules:docs-lint
   image: "registry.gitlab.com/gitlab-org/gitlab-docs:docs-lint"
   stage: test
-  dependencies: []
+  needs:
+    - job: "retrieve-tests-metadata"
+      artifacts: false
   script:
     - scripts/lint-doc.sh
     # Lint Markdown

.gitlab/ci/frontend.gitlab-ci.yml

@@ -120,15 +120,54 @@ compile-assets pull-cache as-if-foss:
     policy: pull
     key: "assets-compile:v9:foss"
 
-.frontend-job-base:
+.frontend-fixtures-base:
   extends:
     - .default-tags
     - .default-retry
     - .default-cache
     - .default-before_script
     - .use-pg9
+  stage: fixtures
+  needs:
+    - job: "setup-test-env"
+      artifacts: true
+    - job: "compile-assets pull-cache"
+      artifacts: true
+  script:
+    - date
+    - scripts/gitaly-test-spawn
+    - date
+    - bundle exec rake frontend:fixtures
+  artifacts:
+    name: frontend-fixtures
+    expire_in: 31d
+    when: always
+    paths:
+      - node_modules
+      - public/assets
+      - tmp/tests/frontend/
+
+frontend-fixtures:
+  extends:
+    - .frontend-fixtures-base
+    - .frontend:rules:default-frontend-jobs
+
+frontend-fixtures-as-if-foss:
+  extends:
+    - .frontend-fixtures-base
+    - .frontend:rules:default-frontend-jobs-as-if-foss
+    - .as-if-foss
+
+.frontend-job-base:
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .default-before_script
+  variables:
+    USE_BUNDLE_INSTALL: "false"
+    SETUP_DB: "false"
   stage: test
-  needs: ["setup-test-env", "compile-assets pull-cache"]
 
 .karma-base:
   extends: .frontend-job-base
@@ -138,14 +177,13 @@ compile-assets pull-cache as-if-foss:
   script:
     - export BABEL_ENV=coverage CHROME_LOG_FILE=chrome_debug.log
     - date
-    - scripts/gitaly-test-spawn
-    - date
-    - bundle exec rake karma
+    - yarn karma
 
 karma:
   extends:
     - .karma-base
     - .frontend:rules:default-frontend-jobs
+  needs: ["frontend-fixtures"]
   coverage: '/^Statements *: (\d+\.\d+%)/'
   artifacts:
     name: coverage-javascript
@@ -163,13 +201,11 @@ karma-as-if-foss:
     - .karma-base
     - .frontend:rules:default-frontend-jobs-as-if-foss
     - .as-if-foss
+  needs: ["frontend-fixtures-as-if-foss"]
 
 .jest-base:
   extends: .frontend-job-base
   script:
-    - scripts/gitaly-test-spawn
-    - date
-    - bundle exec rake frontend:fixtures
     - date
     - yarn jest --ci --coverage
   cache:
@@ -182,6 +218,7 @@ jest:
   extends:
     - .jest-base
     - .frontend:rules:default-frontend-jobs
+  needs: ["frontend-fixtures"]
   artifacts:
     name: coverage-frontend
     expire_in: 31d
@@ -198,6 +235,7 @@ jest-as-if-foss:
     - .jest-base
     - .frontend:rules:default-frontend-jobs-as-if-foss
     - .as-if-foss
+  needs: ["frontend-fixtures-as-if-foss"]
   cache:
     policy: pull
 

.gitlab/ci/qa.gitlab-ci.yml

@@ -3,7 +3,9 @@
     - .default-tags
     - .default-retry
   stage: test
-  dependencies: []
+  needs:
+    - job: "retrieve-tests-metadata"
+      artifacts: false
   cache:
     key: "qa-framework-jobs:v1"
     paths:

.gitlab/ci/reports.gitlab-ci.yml

@@ -12,6 +12,9 @@ code_quality:
     - .default-retry
     - .reports:rules:code_quality
   stage: test
+  needs:
+    - job: "retrieve-tests-metadata"
+      artifacts: false
   image: docker:stable
   allow_failure: true
   services:
@@ -39,7 +42,6 @@ code_quality:
     paths:
       - gl-code-quality-report.json  # GitLab-specific
     expire_in: 1 week  # GitLab-specific
-  dependencies: []
 
 # We need to duplicate this job's definition because it seems it's impossible to
 # override an included `only.refs`.
@@ -52,7 +54,9 @@ sast:
     - .reports:rules:sast
   stage: test
   allow_failure: true
-  dependencies: []  # GitLab-specific
+  needs:
+    - job: "retrieve-tests-metadata"
+      artifacts: false
   artifacts:
     paths:
       - gl-sast-report.json  # GitLab-specific
@@ -90,6 +94,9 @@ dependency_scanning:
     - .default-retry
     - .reports:rules:dependency_scanning
   stage: test
+  needs:
+    - job: "retrieve-tests-metadata"
+      artifacts: false
   image: docker:stable
   variables:
     DOCKER_DRIVER: overlay2
@@ -148,7 +155,6 @@ dependency_scanning:
     reports:
       dependency_scanning: gl-dependency-scanning-report.json
     expire_in: 1 week  # GitLab-specific
-  dependencies: []
 
 # We need to duplicate this job's definition because it seems it's impossible to
 # override an included `only.refs`.

.gitlab/ci/review.gitlab-ci.yml

@@ -248,7 +248,9 @@ danger-review:
     - .review:rules:danger
   image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger
   stage: test
-  dependencies: []
+  needs:
+    - job: "retrieve-tests-metadata"
+      artifacts: false
   script:
     - git version
     - node --version

doc/development/pipelines.md

@@ -136,17 +136,23 @@ graph RL;
   E[review-build-cng];
   F[build-qa-image];
   G[review-deploy];
-  I["karma, jest, webpack-dev-server, static-analysis"];
+  I["karma, jest"];
   I2["karma-as-if-foss, jest-as-if-foss<br/>(EE default refs only)"];
   J["compile-assets pull-push-cache<br/>(master only)"];
   J2["compile-assets pull-push-cache as-if-foss<br/>(EE master only)"];
   K[compile-assets pull-cache];
   K2["compile-assets pull-cache as-if-foss<br/>(EE default refs only)"];
+  U[frontend-fixtures];
+  U2["frontend-fixtures-as-if-foss<br/>(EE default refs only)"];
+  V["webpack-dev-server, static-analysis"];
   M[coverage];
   N["pages (master only)"];
   Q[package-and-qa];
   S["RSpec<br/>(e.g. rspec unit pg9)"]
   T[retrieve-tests-metadata];
+  QA["qa:internal, qa:selectors"];
+  QA2["qa:internal-as-if-foss, qa:selectors-as-if-foss<br/>(EE default refs only)"];
+  X["docs lint, code_quality, sast, dependency_scanning, danger-review"];
 
 subgraph "`prepare` stage"
     A
@@ -160,17 +166,26 @@ subgraph "`prepare` stage"
     T
     end
 
+subgraph "`fixture` stage"
+    U -.-> |needs and depends on| A;
+    U -.-> |needs and depends on| K;
+    U2 -.-> |needs and depends on| A;
+    U2 -.-> |needs and depends on| K2;
+    end
+
 subgraph "`test` stage"
     D -.-> |needs| A;
-    I -.-> |needs and depends on| A;
-    I -.-> |needs and depends on| K;
-    I2 -.-> |needs and depends on| A;
-    I2 -.-> |needs and depends on| K;
+    I -.-> |needs and depends on| U;
+    I2 -.-> |needs and depends on| U2;
     L -.-> |needs and depends on| A;
     S -.-> |needs and depends on| A;
     S -.-> |needs and depends on| K;
     S -.-> |needs and depends on| T;
     L["db:*, gitlab:setup, graphql-docs-verify, downtime_check"] -.-> |needs| A;
+    V -.-> |needs and depends on| K;
+    X -.-> |needs| T;
+    QA -.-> |needs| T;
+    QA2 -.-> |needs| T;
     end
 
 subgraph "`post-test` stage"