Merge branch 'move-long-static-analysis-tasks-to-dedicated-jobs' into 'master'

ci: Move long static-analysis tasks to dedicated jobs

See merge request gitlab-org/gitlab!72305

.gitlab-ci.yml

@@ -3,6 +3,7 @@ stages:
   - prepare
   - build-images
   - fixtures
+  - lint
   - test
   - post-test
   - review

.gitlab/ci/docs.gitlab-ci.yml

@@ -45,7 +45,7 @@ docs-lint markdown:
     - .docs:rules:docs-lint
   # When updating the image version here, update it in /scripts/lint-doc.sh too.
   image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-markdown:alpine-3.14-vale-2.10.4-markdownlint-0.28.1
-  stage: test
+  stage: lint
   needs: []
   script:
     - scripts/lint-doc.sh
@@ -54,7 +54,7 @@ docs-lint links:
   extends:
     - .docs:rules:docs-lint
   image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-html:alpine-3.14-ruby-2.7.4-db71f027
-  stage: test
+  stage: lint
   needs: []
   script:
     # Prepare docs for build
@@ -71,7 +71,7 @@ ui-docs-links lint:
   extends:
     - .docs:rules:docs-lint
     - .static-analysis-base
-  stage: test
+  stage: lint
   needs: []
   script:
     - bundle exec haml-lint -i DocumentationLinks

.gitlab/ci/frontend.gitlab-ci.yml

@@ -197,15 +197,6 @@ graphql-schema-dump:
     - *yarn-install
   stage: test
 
-eslint-as-if-foss:
-  extends:
-    - .frontend-test-base
-    - .frontend:rules:eslint-as-if-foss
-    - .as-if-foss
-  needs: []
-  script:
-    - run_timed_command "yarn run lint:eslint:all"
-
 .jest-base:
   extends: .frontend-test-base
   script:

.gitlab/ci/global.gitlab-ci.yml

@@ -140,6 +140,10 @@
     - <<: *gitaly-binaries-cache
       policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
 
+.ruby-cache:
+  cache:
+    - *ruby-gems-cache
+
 .rails-cache:
   cache:
     - *ruby-gems-cache
@@ -151,7 +155,12 @@
     - *node-modules-cache
     - *rubocop-cache
 
-.static-analysis-cache-push:
+.rubocop-job-cache:
+  cache:
+    - *ruby-gems-cache
+    - *rubocop-cache
+
+.rubocop-job-cache-push:
   cache:
     - *ruby-gems-cache  # We don't push this cache as it's already rebuilt by `update-setup-test-env-cache`
     - *rubocop-cache-push

.gitlab/ci/rails.gitlab-ci.yml

@@ -507,7 +507,11 @@ rspec:feature-flags:
     - .coverage-base
     - .rails:rules:rspec-feature-flags
   stage: post-test
-  needs: ["static-analysis"]
+  needs:
+    - job: "feature-flags-usage"
+    - job: "haml-lint foss"
+    - job: "haml-lint ee"
+      optional: true
   script:
     - !reference [.minimal-bundle-install, script]
     - if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then

.gitlab/ci/reports.gitlab-ci.yml

@@ -9,6 +9,7 @@ code_quality:
   extends:
     - .default-retry
     - .use-docker-in-docker
+  stage: lint
   artifacts:
     paths:
       - gl-code-quality-report.json  # GitLab-specific
@@ -20,6 +21,7 @@ code_quality:
   extends:
     - .default-retry
     - sast
+  stage: lint
   needs: []
   artifacts:
     paths:
@@ -54,6 +56,7 @@ gosec-sast:
 
 .secret-analyzer:
   extends: .default-retry
+  stage: lint
   needs: []
   artifacts:
     paths:
@@ -68,6 +71,7 @@ secret_detection:
   extends:
     - .default-retry
     - dependency_scanning
+  stage: lint
   needs: []
   variables:
     DS_EXCLUDED_PATHS: "qa/qa/ee/fixtures/secure_premade_reports, spec, ee/spec, tmp"  # GitLab-specific
@@ -134,6 +138,7 @@ package_hunter-bundler:
 
 license_scanning:
   extends: .default-retry
+  stage: lint
   needs: []
   artifacts:
     expire_in: 1 week  # GitLab-specific

.gitlab/ci/rules.gitlab-ci.yml

@@ -1354,6 +1354,12 @@
   rules:
     - changes: *code-backstage-qa-patterns
 
+.static-analysis:rules:ee:
+  rules:
+    - <<: *if-not-ee
+      when: never
+    - changes: *code-backstage-qa-patterns
+
 .static-analysis:rules:as-if-foss:
   rules:
     - <<: *if-not-ee

.gitlab/ci/static-analysis.gitlab-ci.yml

@@ -2,7 +2,7 @@
   extends:
     - .default-retry
     - .default-before_script
-    - .static-analysis-cache
+  stage: lint
   needs: []
   variables:
     SETUP_DB: "false"
@@ -14,7 +14,7 @@
 update-static-analysis-cache:
   extends:
     - .static-analysis-base
-    - .static-analysis-cache-push
+    - .rubocop-job-cache-push
     - .shared:rules:update-cache
   stage: prepare
   script:
@@ -23,31 +23,87 @@ update-static-analysis-cache:
 static-analysis:
   extends:
     - .static-analysis-base
+    - .static-analysis-cache
     - .static-analysis:rules:ee-and-foss
-  stage: test
-  parallel: 4
+  parallel: 2
   script:
     - run_timed_command "retry yarn install --frozen-lockfile"
     - scripts/static-analysis
-  artifacts:
-    expire_in: 31d
-    when: always
-    paths:
-      - tmp/feature_flags/
 
-static-analysis-with-database:
+static-analysis as-if-foss:
+  extends:
+    - static-analysis
+    - .static-analysis:rules:as-if-foss
+    - .as-if-foss
+
+static-verification-with-database:
   extends:
     - .static-analysis-base
+    - .rubocop-job-cache
     - .static-analysis:rules:ee-and-foss
     - .use-pg12
-  stage: test
   script:
     - bundle exec rake lint:static_verification_with_database
   variables:
     SETUP_DB: "true"
 
-static-analysis as-if-foss:
+eslint:
   extends:
-    - static-analysis
-    - .static-analysis:rules:as-if-foss
+    - .static-analysis-base
+    - .yarn-cache
+    - .static-analysis:rules:ee
+  needs: []
+  variables:
+    USE_BUNDLE_INSTALL: "false"
+  script:
+    - run_timed_command "retry yarn install --frozen-lockfile"
+    - run_timed_command "yarn run lint:eslint:all"
+
+eslint as-if-foss:
+  extends:
+    - eslint
+    - .frontend:rules:eslint-as-if-foss
     - .as-if-foss
+
+haml-lint foss:
+  extends:
+    - .static-analysis-base
+    - .ruby-cache
+    - .static-analysis:rules:ee-and-foss
+  script:
+    - run_timed_command "bin/rake 'haml_lint[app/views]'"
+  artifacts:
+    expire_in: 31d
+    when: always
+    paths:
+      - tmp/feature_flags/
+
+haml-lint ee:
+  extends:
+    - "haml-lint foss"
+    - .static-analysis:rules:ee
+  script:
+    - run_timed_command "bin/rake 'haml_lint[ee/app/views]'"
+
+rubocop:
+  extends:
+    - .static-analysis-base
+    - .rubocop-job-cache
+    - .static-analysis:rules:ee-and-foss
+  script:
+    - run_timed_command "bundle exec rubocop --parallel"
+
+feature-flags-usage:
+  extends:
+    - .static-analysis-base
+    - .rubocop-job-cache
+    - .static-analysis:rules:ee-and-foss
+  script:
+    # We need to disable the cache for this cop since it creates files under tmp/feature_flags/*.used,
+    # the cache would prevent these files from being created.
+    - run_timed_command "bundle exec rubocop --only Gitlab/MarkUsedFeatureFlags --cache false"
+  artifacts:
+    expire_in: 31d
+    when: always
+    paths:
+      - tmp/feature_flags/

.gitlab/ci/yaml.gitlab-ci.yml

@@ -5,7 +5,7 @@ lint-yaml:
     - .default-retry
     - .yaml-lint:rules
   image: pipelinecomponents/yamllint:latest
-  stage: test
+  stage: lint
   needs: []
   variables:
     LINT_PATHS: .gitlab-ci.yml .gitlab/ci lib/gitlab/ci/templates

.haml-lint.yml

@@ -6,6 +6,7 @@ skip_frontmatter: false
 exclude:
   - 'vendor/**/*'
   - 'spec/**/*'
+  - 'ee/spec/**/*'
 require:
   - './haml_lint/linter/no_plain_nodes.rb'
   - './haml_lint/linter/documentation_links.rb'

doc/development/pipelines.md

@@ -590,7 +590,8 @@ The current stages are:
 - `build-images`: This stage includes jobs that prepare Docker images
   that are needed by jobs in subsequent stages or downstream pipelines.
 - `fixtures`: This stage includes jobs that prepare fixtures needed by frontend tests.
-- `test`: This stage includes most of the tests, DB/migration jobs, and static analysis jobs.
+- `lint`: This stage includes linting and static analysis jobs.
+- `test`: This stage includes most of the tests, and DB/migration jobs.
 - `post-test`: This stage includes jobs that build reports or gather data from
   the `test` stage's jobs (for example, coverage, Knapsack metadata, and so on).
 - `review-prepare`: This stage includes a job that build the CNG images that are
@@ -741,8 +742,10 @@ request, be sure to start the `dont-interrupt-me` job before pushing.
    [`.gitlab/ci/global.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/global.gitlab-ci.yml),
    with fixed keys:
    - `.setup-test-env-cache`
+   - `.ruby-cache`
    - `.rails-cache`
    - `.static-analysis-cache`
+   - `.rubocop-cache`
    - `.coverage-cache`
    - `.danger-review-cache`
    - `.qa-cache`
@@ -752,7 +755,7 @@ request, be sure to start the `dont-interrupt-me` job before pushing.
 1. Only the following jobs, running in 2-hourly scheduled pipelines, are pushing (that is, updating) to the caches:
    - `update-setup-test-env-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
    - `update-gitaly-binaries-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
-   - `update-static-analysis-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
+   - `update-rubocop-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
    - `update-qa-cache`, defined in [`.gitlab/ci/qa.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/qa.gitlab-ci.yml).
    - `update-assets-compile-production-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/frontend.gitlab-ci.yml).
    - `update-assets-compile-test-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/frontend.gitlab-ci.yml).

lib/tasks/haml-lint.rake

@@ -4,16 +4,5 @@ unless Rails.env.production?
   require 'haml_lint/rake_task'
   require Rails.root.join('haml_lint/inline_javascript')
 
-  # Workaround for warnings from parser/current
-  # Keep it even if it no longer emits any warnings,
-  # because we'll still see warnings in console/server anyway,
-  # and we don't need to break static-analysis for this.
-  task :haml_lint do
-    require 'parser'
-    def Parser.warn(*args)
-      puts(*args) # static-analysis ignores stdout if status is 0
-    end
-  end
-
   HamlLint::RakeTask.new
 end

scripts/static-analysis

@@ -43,13 +43,7 @@ class StaticAnalysis
   # contain values that a FOSS installation won't find. To work
   # around this we will only enable this task on EE installations.
   TASKS_WITH_DURATIONS_SECONDS = [
-    Task.new(%w[bin/rake lint:haml], 562),
-    # We need to disable the cache for this cop since it creates files under tmp/feature_flags/*.used,
-    # the cache would prevent these files from being created.
-    Task.new(%w[bundle exec rubocop --only Gitlab/MarkUsedFeatureFlags --cache false], 400),
     (Gitlab.ee? ? Task.new(%w[bin/rake gettext:updated_check], 360) : nil),
-    Task.new(%w[yarn run lint:eslint:all], 312),
-    Task.new(%w[bundle exec rubocop --parallel], 60),
     Task.new(%w[yarn run lint:prettier], 160),
     Task.new(%w[bin/rake gettext:lint], 85),
     Task.new(%W[bundle exec license_finder --decisions-file config/dependency_decisions.yml --project-path #{project_path}], 20),