Merge branch 'allow-more-parallelization-for-static-analysis' into 'master'

Allow more parallelization for static analysis

See merge request gitlab-org/gitlab!32684

.gitlab/ci/rails.gitlab-ci.yml

@@ -46,7 +46,8 @@ static-analysis:
   stage: test
   variables:
     SETUP_DB: "false"
-  parallel: 2
+    ENABLE_SPRING: "1"
+  parallel: 4
   script:
     - scripts/static-analysis
   cache:

scripts/static-analysis

@@ -5,103 +5,118 @@ require_relative '../lib/gitlab'
 require_relative '../lib/gitlab/popen'
 require_relative '../lib/gitlab/popen/runner'
 
-def emit_warnings(static_analysis)
-  static_analysis.warned_results.each do |result|
-    puts
-    puts "**** #{result.cmd.join(' ')} had the following warning(s):"
-    puts
-    puts result.stderr
-    puts
-  end
-end
+class StaticAnalysis
+  ALLOWED_WARNINGS = [
+    # https://github.com/browserslist/browserslist/blob/d0ec62eb48c41c218478cd3ac28684df051cc865/node.js#L329
+    # warns if caniuse-lite package is older than 6 months. Ignore this
+    # warning message so that GitLab backports don't fail.
+    "Browserslist: caniuse-lite is outdated. Please run next command `yarn upgrade`"
+  ].freeze
+
+  # `gettext:updated_check` and `gitlab:sidekiq:sidekiq_queues_yml:check` will fail on FOSS installations
+  # (e.g. gitlab-org/gitlab-foss) since they test against a single
+  # file that is generated by an EE installation, which can
+  # contain values that a FOSS installation won't find. To work
+  # around this we will only enable this task on EE installations.
+  TASKS_BY_DURATIONS_SECONDS_DESC = {
+    %w[bin/rake lint:haml] => 338,
+    (Gitlab.ee? ? %w[bin/rake gettext:updated_check] : nil) => 308,
+    # Most of the time, RuboCop finishes in 30 seconds, but sometimes it can take around 1200 seconds so we set a
+    # duration of 300 to lower the likelihood that it will run in the same job as another long task...
+    %w[bundle exec rubocop --parallel] => 300,
+    %w[yarn run eslint] => 197,
+    %w[yarn run prettier-all] => 124,
+    %w[bin/rake gettext:lint] => 96,
+    %w[bundle exec license_finder] => 49,
+    %w[bin/rake scss_lint] => 38,
+    %w[bin/rake lint:static_verification] => 22,
+    %w[bin/rake gitlab:sidekiq:all_queues_yml:check] => 13,
+    (Gitlab.ee? ? %w[bin/rake gitlab:sidekiq:sidekiq_queues_yml:check] : nil) => 13,
+    %w[bin/rake config_lint] => 11,
+    %w[yarn run stylelint] => 9,
+    %w[scripts/lint-conflicts.sh] => 0.59,
+    %w[yarn run block-dependencies] => 0.35,
+    %w[scripts/lint-rugged] => 0.23,
+    %w[scripts/gemfile_lock_changed.sh] => 0.02,
+    %w[scripts/frontend/check_no_partial_karma_jest.sh] => 0.01,
+    %w[scripts/lint-changelog-filenames] => 0.01
+  }.reject { |k| k.nil? }.sort_by { |a| -a[1] }.to_h.keys.freeze
+
+  def run_tasks!
+    tasks = tasks_to_run((ENV['CI_NODE_INDEX'] || 1).to_i, (ENV['CI_NODE_TOTAL'] || 1).to_i)
+
+    static_analysis = Gitlab::Popen::Runner.new
+
+    static_analysis.run(tasks) do |cmd, &run|
+      puts
+      puts "$ #{cmd.join(' ')}"
+
+      result = run.call
+
+      puts "==> Finished in #{result.duration} seconds"
+      puts
+    end
 
-def emit_errors(static_analysis)
-  static_analysis.failed_results.each do |result|
     puts
-    puts "**** #{result.cmd.join(' ')} failed with the following error(s):"
+    puts '==================================================='
     puts
-    puts result.stdout
-    puts result.stderr
     puts
-  end
-end
 
-ALLOWED_WARNINGS = [
-  # https://github.com/browserslist/browserslist/blob/d0ec62eb48c41c218478cd3ac28684df051cc865/node.js#L329
-  # warns if caniuse-lite package is older than 6 months. Ignore this
-  # warning message so that GitLab backports don't fail.
-  "Browserslist: caniuse-lite is outdated. Please run next command `yarn upgrade`"
-].freeze
+    if static_analysis.all_success_and_clean?
+      puts 'All static analyses passed successfully.'
+    elsif static_analysis.all_success?
+      puts 'All static analyses passed successfully, but we have warnings:'
+      puts
 
-def warning_count(static_analysis)
-  static_analysis.warned_results
-    .count { |result| !ALLOWED_WARNINGS.include?(result.stderr.strip) }
-end
+      emit_warnings(static_analysis)
 
-def jobs_to_run(node_index, node_total)
-  all_tasks = [
-    %w[bin/rake lint:all],
-    %w[bundle exec license_finder],
-    %w[yarn run eslint],
-    %w[yarn run stylelint],
-    %w[yarn run prettier-all],
-    %w[yarn run block-dependencies],
-    %w[bundle exec rubocop --parallel],
-    %w[scripts/lint-conflicts.sh],
-    %w[scripts/lint-rugged],
-    %w[scripts/frontend/check_no_partial_karma_jest.sh],
-    %w[scripts/lint-changelog-filenames],
-    %w[scripts/gemfile_lock_changed.sh]
-  ]
-
-  case node_total
-  when 1
-    all_tasks
-  when 2
-    rake_lint_all, *rest_jobs = all_tasks
-    case node_index
-    when 1
-      [rake_lint_all]
+      exit 2 if warning_count(static_analysis).nonzero?
     else
-      rest_jobs
-    end
-  else
-    raise "Parallelization > 2 (currently set to #{node_total}) isn't supported yet!"
-  end
-end
-
-tasks = jobs_to_run((ENV['CI_NODE_INDEX'] || 1).to_i, (ENV['CI_NODE_TOTAL'] || 1).to_i)
-static_analysis = Gitlab::Popen::Runner.new
+      puts 'Some static analyses failed:'
 
-static_analysis.run(tasks) do |cmd, &run|
-  puts
-  puts "$ #{cmd.join(' ')}"
+      emit_warnings(static_analysis)
+      emit_errors(static_analysis)
 
-  result = run.call
-
-  puts "==> Finished in #{result.duration} seconds"
-  puts
-end
+      exit 1
+    end
+  end
 
-puts
-puts '==================================================='
-puts
-puts
+  def emit_warnings(static_analysis)
+    static_analysis.warned_results.each do |result|
+      puts
+      puts "**** #{result.cmd.join(' ')} had the following warning(s):"
+      puts
+      puts result.stderr
+      puts
+    end
+  end
 
-if static_analysis.all_success_and_clean?
-  puts 'All static analyses passed successfully.'
-elsif static_analysis.all_success?
-  puts 'All static analyses passed successfully, but we have warnings:'
-  puts
+  def emit_errors(static_analysis)
+    static_analysis.failed_results.each do |result|
+      puts
+      puts "**** #{result.cmd.join(' ')} failed with the following error(s):"
+      puts
+      puts result.stdout
+      puts result.stderr
+      puts
+    end
+  end
 
-  emit_warnings(static_analysis)
+  def warning_count(static_analysis)
+    static_analysis.warned_results
+      .count { |result| !ALLOWED_WARNINGS.include?(result.stderr.strip) }
+  end
 
-  exit 2 if warning_count(static_analysis).nonzero?
-else
-  puts 'Some static analyses failed:'
+  def tasks_to_run(node_index, node_total)
+    tasks = []
+    TASKS_BY_DURATIONS_SECONDS_DESC.each_with_index do |task, i|
+      tasks << task if i % node_total == (node_index - 1)
+    end
 
-  emit_warnings(static_analysis)
-  emit_errors(static_analysis)
+    tasks
+  end
+end
 
-  exit 1
+if $0 == __FILE__
+  StaticAnalysis.new.run_tasks!
 end