Commit 3334b89d authored by Albert Salim's avatar Albert Salim

Merge branch 'move-long-static-analysis-tasks-to-dedicated-jobs' into 'master'

ci: Move long static-analysis tasks to dedicated jobs

See merge request gitlab-org/gitlab!72305
parents 3050da47 7f76703f
......@@ -3,6 +3,7 @@ stages:
- prepare
- build-images
- fixtures
- lint
- test
- post-test
- review
......
......@@ -45,7 +45,7 @@ docs-lint markdown:
- .docs:rules:docs-lint
# When updating the image version here, update it in /scripts/lint-doc.sh too.
image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-markdown:alpine-3.14-vale-2.10.4-markdownlint-0.28.1
stage: test
stage: lint
needs: []
script:
- scripts/lint-doc.sh
......@@ -54,7 +54,7 @@ docs-lint links:
extends:
- .docs:rules:docs-lint
image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-html:alpine-3.14-ruby-2.7.4-db71f027
stage: test
stage: lint
needs: []
script:
# Prepare docs for build
......@@ -71,7 +71,7 @@ ui-docs-links lint:
extends:
- .docs:rules:docs-lint
- .static-analysis-base
stage: test
stage: lint
needs: []
script:
- bundle exec haml-lint -i DocumentationLinks
......@@ -197,15 +197,6 @@ graphql-schema-dump:
- *yarn-install
stage: test
eslint-as-if-foss:
extends:
- .frontend-test-base
- .frontend:rules:eslint-as-if-foss
- .as-if-foss
needs: []
script:
- run_timed_command "yarn run lint:eslint:all"
.jest-base:
extends: .frontend-test-base
script:
......
......@@ -140,6 +140,10 @@
- <<: *gitaly-binaries-cache
policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
.ruby-cache:
cache:
- *ruby-gems-cache
.rails-cache:
cache:
- *ruby-gems-cache
......@@ -151,7 +155,12 @@
- *node-modules-cache
- *rubocop-cache
.static-analysis-cache-push:
.rubocop-job-cache:
cache:
- *ruby-gems-cache
- *rubocop-cache
.rubocop-job-cache-push:
cache:
- *ruby-gems-cache # We don't push this cache as it's already rebuilt by `update-setup-test-env-cache`
- *rubocop-cache-push
......
......@@ -507,7 +507,11 @@ rspec:feature-flags:
- .coverage-base
- .rails:rules:rspec-feature-flags
stage: post-test
needs: ["static-analysis"]
needs:
- job: "feature-flags-usage"
- job: "haml-lint foss"
- job: "haml-lint ee"
optional: true
script:
- !reference [.minimal-bundle-install, script]
- if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then
......
......@@ -9,6 +9,7 @@ code_quality:
extends:
- .default-retry
- .use-docker-in-docker
stage: lint
artifacts:
paths:
- gl-code-quality-report.json # GitLab-specific
......@@ -20,6 +21,7 @@ code_quality:
extends:
- .default-retry
- sast
stage: lint
needs: []
artifacts:
paths:
......@@ -54,6 +56,7 @@ gosec-sast:
.secret-analyzer:
extends: .default-retry
stage: lint
needs: []
artifacts:
paths:
......@@ -68,6 +71,7 @@ secret_detection:
extends:
- .default-retry
- dependency_scanning
stage: lint
needs: []
variables:
DS_EXCLUDED_PATHS: "qa/qa/ee/fixtures/secure_premade_reports, spec, ee/spec, tmp" # GitLab-specific
......@@ -134,6 +138,7 @@ package_hunter-bundler:
license_scanning:
extends: .default-retry
stage: lint
needs: []
artifacts:
expire_in: 1 week # GitLab-specific
......
......@@ -1354,6 +1354,12 @@
rules:
- changes: *code-backstage-qa-patterns
.static-analysis:rules:ee:
rules:
- <<: *if-not-ee
when: never
- changes: *code-backstage-qa-patterns
.static-analysis:rules:as-if-foss:
rules:
- <<: *if-not-ee
......
......@@ -2,7 +2,7 @@
extends:
- .default-retry
- .default-before_script
- .static-analysis-cache
stage: lint
needs: []
variables:
SETUP_DB: "false"
......@@ -14,7 +14,7 @@
update-static-analysis-cache:
extends:
- .static-analysis-base
- .static-analysis-cache-push
- .rubocop-job-cache-push
- .shared:rules:update-cache
stage: prepare
script:
......@@ -23,31 +23,87 @@ update-static-analysis-cache:
static-analysis:
extends:
- .static-analysis-base
- .static-analysis-cache
- .static-analysis:rules:ee-and-foss
stage: test
parallel: 4
parallel: 2
script:
- run_timed_command "retry yarn install --frozen-lockfile"
- scripts/static-analysis
artifacts:
expire_in: 31d
when: always
paths:
- tmp/feature_flags/
static-analysis-with-database:
static-analysis as-if-foss:
extends:
- static-analysis
- .static-analysis:rules:as-if-foss
- .as-if-foss
static-verification-with-database:
extends:
- .static-analysis-base
- .rubocop-job-cache
- .static-analysis:rules:ee-and-foss
- .use-pg12
stage: test
script:
- bundle exec rake lint:static_verification_with_database
variables:
SETUP_DB: "true"
static-analysis as-if-foss:
eslint:
extends:
- static-analysis
- .static-analysis:rules:as-if-foss
- .static-analysis-base
- .yarn-cache
- .static-analysis:rules:ee
needs: []
variables:
USE_BUNDLE_INSTALL: "false"
script:
- run_timed_command "retry yarn install --frozen-lockfile"
- run_timed_command "yarn run lint:eslint:all"
eslint as-if-foss:
extends:
- eslint
- .frontend:rules:eslint-as-if-foss
- .as-if-foss
haml-lint foss:
extends:
- .static-analysis-base
- .ruby-cache
- .static-analysis:rules:ee-and-foss
script:
- run_timed_command "bin/rake 'haml_lint[app/views]'"
artifacts:
expire_in: 31d
when: always
paths:
- tmp/feature_flags/
haml-lint ee:
extends:
- "haml-lint foss"
- .static-analysis:rules:ee
script:
- run_timed_command "bin/rake 'haml_lint[ee/app/views]'"
rubocop:
extends:
- .static-analysis-base
- .rubocop-job-cache
- .static-analysis:rules:ee-and-foss
script:
- run_timed_command "bundle exec rubocop --parallel"
feature-flags-usage:
extends:
- .static-analysis-base
- .rubocop-job-cache
- .static-analysis:rules:ee-and-foss
script:
# We need to disable the cache for this cop since it creates files under tmp/feature_flags/*.used,
# the cache would prevent these files from being created.
- run_timed_command "bundle exec rubocop --only Gitlab/MarkUsedFeatureFlags --cache false"
artifacts:
expire_in: 31d
when: always
paths:
- tmp/feature_flags/
......@@ -5,7 +5,7 @@ lint-yaml:
- .default-retry
- .yaml-lint:rules
image: pipelinecomponents/yamllint:latest
stage: test
stage: lint
needs: []
variables:
LINT_PATHS: .gitlab-ci.yml .gitlab/ci lib/gitlab/ci/templates
......
......@@ -6,6 +6,7 @@ skip_frontmatter: false
exclude:
- 'vendor/**/*'
- 'spec/**/*'
- 'ee/spec/**/*'
require:
- './haml_lint/linter/no_plain_nodes.rb'
- './haml_lint/linter/documentation_links.rb'
......
......@@ -590,7 +590,8 @@ The current stages are:
- `build-images`: This stage includes jobs that prepare Docker images
that are needed by jobs in subsequent stages or downstream pipelines.
- `fixtures`: This stage includes jobs that prepare fixtures needed by frontend tests.
- `test`: This stage includes most of the tests, DB/migration jobs, and static analysis jobs.
- `lint`: This stage includes linting and static analysis jobs.
- `test`: This stage includes most of the tests, and DB/migration jobs.
- `post-test`: This stage includes jobs that build reports or gather data from
the `test` stage's jobs (for example, coverage, Knapsack metadata, and so on).
- `review-prepare`: This stage includes a job that build the CNG images that are
......@@ -741,8 +742,10 @@ request, be sure to start the `dont-interrupt-me` job before pushing.
[`.gitlab/ci/global.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/global.gitlab-ci.yml),
with fixed keys:
- `.setup-test-env-cache`
- `.ruby-cache`
- `.rails-cache`
- `.static-analysis-cache`
- `.rubocop-cache`
- `.coverage-cache`
- `.danger-review-cache`
- `.qa-cache`
......@@ -752,7 +755,7 @@ request, be sure to start the `dont-interrupt-me` job before pushing.
1. Only the following jobs, running in 2-hourly scheduled pipelines, are pushing (that is, updating) to the caches:
- `update-setup-test-env-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
- `update-gitaly-binaries-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
- `update-static-analysis-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
- `update-rubocop-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
- `update-qa-cache`, defined in [`.gitlab/ci/qa.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/qa.gitlab-ci.yml).
- `update-assets-compile-production-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/frontend.gitlab-ci.yml).
- `update-assets-compile-test-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/frontend.gitlab-ci.yml).
......
......@@ -4,16 +4,5 @@ unless Rails.env.production?
require 'haml_lint/rake_task'
require Rails.root.join('haml_lint/inline_javascript')
# Workaround for warnings from parser/current
# Keep it even if it no longer emits any warnings,
# because we'll still see warnings in console/server anyway,
# and we don't need to break static-analysis for this.
task :haml_lint do
require 'parser'
def Parser.warn(*args)
puts(*args) # static-analysis ignores stdout if status is 0
end
end
HamlLint::RakeTask.new
end
......@@ -43,13 +43,7 @@ class StaticAnalysis
# contain values that a FOSS installation won't find. To work
# around this we will only enable this task on EE installations.
TASKS_WITH_DURATIONS_SECONDS = [
Task.new(%w[bin/rake lint:haml], 562),
# We need to disable the cache for this cop since it creates files under tmp/feature_flags/*.used,
# the cache would prevent these files from being created.
Task.new(%w[bundle exec rubocop --only Gitlab/MarkUsedFeatureFlags --cache false], 400),
(Gitlab.ee? ? Task.new(%w[bin/rake gettext:updated_check], 360) : nil),
Task.new(%w[yarn run lint:eslint:all], 312),
Task.new(%w[bundle exec rubocop --parallel], 60),
Task.new(%w[yarn run lint:prettier], 160),
Task.new(%w[bin/rake gettext:lint], 85),
Task.new(%W[bundle exec license_finder --decisions-file config/dependency_decisions.yml --project-path #{project_path}], 20),
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment