Automatic merge of gitlab-org/gitlab master

Dangerfile

@@ -17,6 +17,14 @@ end
 
 anything_to_post = status_report.values.any? { |data| data.any? }
 
-if helper.ci? && anything_to_post
+return unless helper.ci?
+
+if project_helper.labels_to_add.any?
+  gitlab.api.update_merge_request(gitlab.mr_json['project_id'],
+                                  gitlab.mr_json['iid'],
+                                  add_labels: project_helper.labels_to_add.join(','))
+end
+
+if anything_to_post
   markdown("**If needed, you can retry the [`danger-review` job](#{ENV['CI_JOB_URL']}) that generated this comment.**")
 end

app/controllers/admin/hook_logs_controller.rb

@@ -9,6 +9,7 @@ class Admin::HookLogsController < Admin::ApplicationController
   respond_to :html
 
   feature_category :integrations
+  urgency :low, [:retry]
 
   def show
   end

app/controllers/admin/hooks_controller.rb

@@ -6,6 +6,7 @@ class Admin::HooksController < Admin::ApplicationController
   before_action :hook_logs, only: :edit
 
   feature_category :integrations
+  urgency :low, [:test]
 
   def index
     @hooks = SystemHook.all

app/controllers/concerns/integrations_actions.rb

@@ -8,6 +8,8 @@ module IntegrationsActions
     include IntegrationsHelper
 
     before_action :integration, only: [:edit, :update, :overrides, :test]
+
+    urgency :low, [:test]
   end
 
   def edit

app/controllers/projects/hook_logs_controller.rb

@@ -13,6 +13,7 @@ class Projects::HookLogsController < Projects::ApplicationController
   layout 'project_settings'
 
   feature_category :integrations
+  urgency :low, [:retry]
 
   def show
   end

app/controllers/projects/hooks_controller.rb

@@ -13,6 +13,7 @@ class Projects::HooksController < Projects::ApplicationController
   layout "project_settings"
 
   feature_category :integrations
+  urgency :low, [:test]
 
   def index
     @hooks = @project.hooks

app/controllers/projects/services_controller.rb

@@ -18,6 +18,7 @@ class Projects::ServicesController < Projects::ApplicationController
   layout "project_settings"
 
   feature_category :integrations
+  urgency :low, [:test]
 
   def edit
   end

app/models/analytics/cycle_analytics/issue_stage_event.rb

@@ -8,6 +8,9 @@ module Analytics
 
       validates(*%i[stage_event_hash_id issue_id group_id project_id start_event_timestamp], presence: true)
 
+      alias_attribute :state, :state_id
+      enum state: Issue.available_states, _suffix: true
+
       def self.issuable_id_column
         :issue_id
       end

app/models/analytics/cycle_analytics/merge_request_stage_event.rb

@@ -8,6 +8,9 @@ module Analytics
 
       validates(*%i[stage_event_hash_id merge_request_id group_id project_id start_event_timestamp], presence: true)
 
+      alias_attribute :state, :state_id
+      enum state: MergeRequest.available_states, _suffix: true
+
       def self.issuable_id_column
         :merge_request_id
       end

app/models/concerns/analytics/cycle_analytics/stage_event_model.rb

@@ -15,6 +15,7 @@ module Analytics
               :project_id,
               :author_id,
               :milestone_id,
+              :state_id,
               :start_event_timestamp,
               :end_event_timestamp
             )
@@ -31,6 +32,7 @@ module Analytics
             project_id,
             milestone_id,
             author_id,
+            state_id,
             start_event_timestamp,
             end_event_timestamp
           )
@@ -39,10 +41,11 @@ module Analytics
           DO UPDATE SET
             group_id = excluded.group_id,
             project_id = excluded.project_id,
-            start_event_timestamp = excluded.start_event_timestamp,
-            end_event_timestamp = excluded.end_event_timestamp,
             milestone_id = excluded.milestone_id,
-            author_id = excluded.author_id
+            author_id = excluded.author_id,
+            state_id = excluded.state_id,
+            start_event_timestamp = excluded.start_event_timestamp,
+            end_event_timestamp = excluded.end_event_timestamp
           SQL
 
           result = connection.execute(query)

config/application.rb

@@ -28,6 +28,7 @@ module Gitlab
     require_dependency Rails.root.join('lib/gitlab/redis/sessions')
     require_dependency Rails.root.join('lib/gitlab/current_settings')
     require_dependency Rails.root.join('lib/gitlab/middleware/read_only')
+    require_dependency Rails.root.join('lib/gitlab/middleware/compressed_json')
     require_dependency Rails.root.join('lib/gitlab/middleware/basic_health_check')
     require_dependency Rails.root.join('lib/gitlab/middleware/same_site_cookies')
     require_dependency Rails.root.join('lib/gitlab/middleware/handle_ip_spoof_attack_error')
@@ -319,6 +320,8 @@ module Gitlab
 
     config.middleware.insert_after Rack::Sendfile, ::Gitlab::Middleware::RackMultipartTempfileFactory
 
+    config.middleware.insert_before Rack::Runtime, ::Gitlab::Middleware::CompressedJson
+
     # Allow access to GitLab API from other domains
     config.middleware.insert_before Warden::Manager, Rack::Cors do
       headers_to_expose = %w[Link X-Total X-Total-Pages X-Per-Page X-Page X-Next-Page X-Prev-Page X-Gitlab-Blob-Id X-Gitlab-Commit-Id X-Gitlab-Content-Sha256 X-Gitlab-Encoding X-Gitlab-File-Name X-Gitlab-File-Path X-Gitlab-Last-Commit-Id X-Gitlab-Ref X-Gitlab-Size]

danger/database/Dangerfile

@@ -66,8 +66,6 @@ if gitlab.mr_labels.include?('database') || db_paths_to_review.any?
   end
 
   unless helper.has_database_scoped_labels?
-    gitlab.api.update_merge_request(gitlab.mr_json['project_id'],
-                                    gitlab.mr_json['iid'],
-                                    add_labels: 'database::review pending')
+    project_helper.labels_to_add << 'database::review pending'
   end
 end

danger/feature_flag/Dangerfile

@@ -58,9 +58,7 @@ def message_for_feature_flag_with_group!(feature_flag:, mr_group_label:)
   return if feature_flag.group_match_mr_label?(mr_group_label)
 
   if mr_group_label.nil?
-    gitlab.api.update_merge_request(gitlab.mr_json['project_id'],
-                                    gitlab.mr_json['iid'],
-                                    add_labels: feature_flag.group)
+    project_helper.labels_to_add << feature_flag.group
   else
     fail %(`group` is set to ~"#{feature_flag.group}" in #{gitlab.html_link(feature_flag.path)}, which does not match ~"#{mr_group_label}" set on the MR!)
   end

danger/product_intelligence/Dangerfile

@@ -13,12 +13,10 @@ MSG
 
 # exit if not matching files or if no product intelligence labels
 product_intelligence_paths_to_review = project_helper.changes_by_category[:product_intelligence]
-labels = product_intelligence.missing_labels
+labels_to_add = product_intelligence.missing_labels
 
-return if product_intelligence_paths_to_review.empty? || labels.empty?
+return if product_intelligence_paths_to_review.empty? || labels_to_add.empty?
 
 warn format(CHANGED_FILES_MESSAGE, changed_files: helper.markdown_list(product_intelligence_paths_to_review))
 
-gitlab.api.update_merge_request(gitlab.mr_json['project_id'],
-                                    gitlab.mr_json['iid'],
-                                    add_labels: labels)
+project_helper.labels_to_add.concat(labels_to_add)

danger/specialization_labels/Dangerfile

@@ -19,8 +19,4 @@ labels_to_add = project_helper.changes_by_category.each_with_object([]) do |(cat
   memo << label if label && !gitlab.mr_labels.include?(label)
 end
 
-if labels_to_add.any?
-  gitlab.api.update_merge_request(gitlab.mr_json['project_id'],
-                                  gitlab.mr_json['iid'],
-                                  add_labels: labels_to_add.join(','))
-end
+project_helper.labels_to_add.concat(labels_to_add) if labels_to_add.any?

danger/metadata/Dangerfile → danger/z_metadata/Dangerfile

@@ -18,7 +18,7 @@ if gitlab.mr_body.size < 5
   fail "Please provide a proper merge request description."
 end
 
-if (TYPE_LABELS & gitlab.mr_labels).empty?
+if (TYPE_LABELS & (gitlab.mr_labels + project_helper.labels_to_add)).empty?
   warn 'Please add a [merge request type](https://about.gitlab.com/handbook/engineering/metrics/#work-type-classification) to this merge request.'
 end
 

db/migrate/20211025103744_add_state_id_to_vsa_issue_stage_events.rb

+# frozen_string_literal: true
+
+class AddStateIdToVsaIssueStageEvents < Gitlab::Database::Migration[1.0]
+  def change
+    add_column :analytics_cycle_analytics_issue_stage_events, :state_id, :smallint, default: 1, null: false
+  end
+end

db/migrate/20211025103758_add_state_id_to_vsa_merge_request_stage_events.rb

+# frozen_string_literal: true
+
+class AddStateIdToVsaMergeRequestStageEvents < Gitlab::Database::Migration[1.0]
+  def change
+    add_column :analytics_cycle_analytics_merge_request_stage_events, :state_id, :smallint, default: 1, null: false
+  end
+end

db/schema_migrations/20211025103744

+56f86f2a20509f2052d0eab918a6d2bbb679d785cf5349916ba695aa92f0609f
\ No newline at end of file

db/schema_migrations/20211025103758

+1722382c5141157497f48b209c0fa60c5959fa16c53d09fb6e8e6b2cb6e6ab9b
\ No newline at end of file

doc/development/dangerbot.md

@@ -118,6 +118,23 @@ However, you can speed these cycles up somewhat by emptying the
 `.gitlab/ci/rails.gitlab-ci.yml` file in your merge request. Just don't forget
 to revert the change before merging!
 
+#### Adding labels via Danger
+
+NOTE:
+This is currently applicable to the [`gitlab-org/gitlab`](https://gitlab.com/gitlab-org/gitlab)
+project only.
+
+Danger is often used to improve MR hygiene by adding labels. Instead of calling the
+API directly in your `Dangerfile`, add the labels to the `project_helper.labels_to_add` array.
+The main `Dangerfile` will then take care of adding the labels to the MR with a single API call.
+
+#### Shared rules and plugins
+
+If the rule or plugin you implement can be useful for other projects, think about
+upstreaming them to the [`gitlab-org/gitlab-dangerfiles`](https://gitlab.com/gitlab-org/gitlab-dangerfiles) project.
+
+#### Enable Danger on a project
+
 To enable the Dangerfile on another existing GitLab project, run the following
 extra steps:
 

ee/app/controllers/groups/hooks_controller.rb

@@ -15,6 +15,7 @@ class Groups::HooksController < Groups::ApplicationController
   layout 'group_settings'
 
   feature_category :integrations
+  urgency :low, [:test]
 
   def index
     @hooks = @group.hooks

ee/app/services/analytics/cycle_analytics/data_loader_service.rb

@@ -126,6 +126,7 @@ module Analytics
               project_id: record['project_id'],
               author_id: record['author_id'],
               milestone_id: record['milestone_id'],
+              state_id: record['state_id'],
               start_event_timestamp: record[event_column_name(stage.start_event)],
               end_event_timestamp: record[event_column_name(stage.end_event)]
             }
@@ -146,6 +147,7 @@ module Analytics
           model.arel_table[project_column].as('project_id'),
           model.arel_table[:milestone_id],
           model.arel_table[:author_id],
+          model.arel_table[:state_id],
           Project.arel_table[:parent_id].as('group_id')
         ]
       end

ee/spec/services/analytics/cycle_analytics/data_loader_service_spec.rb

@@ -103,7 +103,8 @@ RSpec.describe Analytics::CycleAnalytics::DataLoaderService do
             mr.project.parent_id,
             mr.project_id,
             mr.created_at,
-            mr.metrics.merged_at
+            mr.metrics.merged_at,
+            mr.state_id
           ]
         end
 
@@ -116,7 +117,8 @@ RSpec.describe Analytics::CycleAnalytics::DataLoaderService do
             event.group_id,
             event.project_id,
             event.start_event_timestamp,
-            event.end_event_timestamp
+            event.end_event_timestamp,
+            Analytics::CycleAnalytics::MergeRequestStageEvent.states[event.state_id]
           ]
         end
 
@@ -191,7 +193,8 @@ RSpec.describe Analytics::CycleAnalytics::DataLoaderService do
             issue.project.parent_id,
             issue.project_id,
             issue.created_at,
-            issue.closed_at
+            issue.closed_at,
+            issue.state_id
           ]
         end
 
@@ -204,7 +207,8 @@ RSpec.describe Analytics::CycleAnalytics::DataLoaderService do
             event.group_id,
             event.project_id,
             event.start_event_timestamp,
-            event.end_event_timestamp
+            event.end_event_timestamp,
+            Analytics::CycleAnalytics::IssueStageEvent.states[event.state_id]
           ]
         end
 

lib/gitlab/middleware/compressed_json.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Middleware
+    class CompressedJson
+      COLLECTOR_PATH = '/api/v4/error_tracking/collector'
+      MAXIMUM_BODY_SIZE = 200.kilobytes.to_i
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        if compressed_et_request?(env)
+          input = extract(env['rack.input'])
+
+          if input.length > MAXIMUM_BODY_SIZE
+            return too_large
+          end
+
+          env.delete('HTTP_CONTENT_ENCODING')
+          env['CONTENT_LENGTH'] = input.length
+          env['rack.input'] = StringIO.new(input)
+        end
+
+        @app.call(env)
+      end
+
+      def compressed_et_request?(env)
+        env['REQUEST_METHOD'] == 'POST' &&
+          env['HTTP_CONTENT_ENCODING'] == 'gzip' &&
+          env['CONTENT_TYPE'] == 'application/json' &&
+          env['PATH_INFO'].start_with?((File.join(relative_url, COLLECTOR_PATH)))
+      end
+
+      def too_large
+        [413, { 'Content-Type' => 'text/plain' }, ['Payload Too Large']]
+      end
+
+      def relative_url
+        File.join('', Gitlab.config.gitlab.relative_url_root).chomp('/')
+      end
+
+      def extract(input)
+        Zlib::GzipReader.new(input).read(MAXIMUM_BODY_SIZE + 1)
+      end
+    end
+  end
+end

lib/gitlab/project_template.rb

@@ -66,7 +66,7 @@ module Gitlab
           ProjectTemplate.new('nfhexo', 'Netlify/Hexo', _('A Hexo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features'), 'https://gitlab.com/pages/nfhexo', 'illustrations/logos/netlify.svg'),
           ProjectTemplate.new('salesforcedx', 'SalesforceDX', _('A project boilerplate for Salesforce App development with Salesforce Developer tools'), 'https://gitlab.com/gitlab-org/project-templates/salesforcedx'),
           ProjectTemplate.new('serverless_framework', 'Serverless Framework/JS', _('A basic page and serverless function that uses AWS Lambda, AWS API Gateway, and GitLab Pages'), 'https://gitlab.com/gitlab-org/project-templates/serverless-framework', 'illustrations/logos/serverless_framework.svg'),
-          ProjectTemplate.new('tencent_serverless_framework', 'Tencent Serverless Framework/NextjsSSR', _('A project boilerplate for Tencent Serverless Framework that uses Next.js SSR'), 'https://gitlab.com/project-templates/nextjsssr_demo', 'illustrations/logos/tencent_serverless_framework.svg'),
+          ProjectTemplate.new('tencent_serverless_framework', 'Tencent Serverless Framework/NextjsSSR', _('A project boilerplate for Tencent Serverless Framework that uses Next.js SSR'), 'https://gitlab.com/gitlab-org/project-templates/nextjsssr_demo', 'illustrations/logos/tencent_serverless_framework.svg'),
           ProjectTemplate.new('jsonnet', 'Jsonnet for Dynamic Child Pipelines', _('An example showing how to use Jsonnet with GitLab dynamic child pipelines'), 'https://gitlab.com/gitlab-org/project-templates/jsonnet'),
           ProjectTemplate.new('cluster_management', 'GitLab Cluster Management', _('An example project for managing Kubernetes clusters integrated with GitLab'), 'https://gitlab.com/gitlab-org/project-templates/cluster-management'),
           ProjectTemplate.new('kotlin_native_linux', 'Kotlin Native Linux', _('A basic template for developing Linux programs using Kotlin Native'), 'https://gitlab.com/gitlab-org/project-templates/kotlin-native-linux')

scripts/utils.sh

@@ -172,5 +172,5 @@ function danger_as_local() {
   # Force danger to skip CI source GitLab and fallback to "local only git repo".
   unset GITLAB_CI
   # We need to base SHA to help danger determine the base commit for this shallow clone.
-  bundle exec danger dry_run --fail-on-errors=true --verbose --base="${CI_MERGE_REQUEST_DIFF_BASE_SHA}"
+  bundle exec danger dry_run --fail-on-errors=true --verbose --base="${CI_MERGE_REQUEST_DIFF_BASE_SHA}" --head="${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}"
 }

spec/db/schema_spec.rb

@@ -18,8 +18,8 @@ RSpec.describe 'Database schema' do
     approvals: %w[user_id],
     approver_groups: %w[target_id],
     approvers: %w[target_id user_id],
-    analytics_cycle_analytics_merge_request_stage_events: %w[author_id group_id merge_request_id milestone_id project_id stage_event_hash_id],
-    analytics_cycle_analytics_issue_stage_events: %w[author_id group_id issue_id milestone_id project_id stage_event_hash_id],
+    analytics_cycle_analytics_merge_request_stage_events: %w[author_id group_id merge_request_id milestone_id project_id stage_event_hash_id state_id],
+    analytics_cycle_analytics_issue_stage_events: %w[author_id group_id issue_id milestone_id project_id stage_event_hash_id state_id],
     audit_events: %w[author_id entity_id target_id],
     award_emoji: %w[awardable_id user_id],
     aws_roles: %w[role_external_id],

spec/lib/gitlab/middleware/compressed_json_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Middleware::CompressedJson do
+  let_it_be(:decompressed_input) { '{"foo": "bar"}' }
+  let_it_be(:input) { ActiveSupport::Gzip.compress(decompressed_input) }
+
+  let(:app) { double(:app) }
+  let(:middleware) { described_class.new(app) }
+  let(:env) do
+    {
+      'HTTP_CONTENT_ENCODING' => 'gzip',
+      'REQUEST_METHOD' => 'POST',
+      'CONTENT_TYPE' => 'application/json',
+      'PATH_INFO' => path,
+      'rack.input' => StringIO.new(input)
+    }
+  end
+
+  shared_examples 'decompress middleware' do
+    it 'replaces input with a decompressed content' do
+      expect(app).to receive(:call)
+
+      middleware.call(env)
+
+      expect(env['rack.input'].read).to eq(decompressed_input)
+      expect(env['CONTENT_LENGTH']).to eq(decompressed_input.length)
+      expect(env['HTTP_CONTENT_ENCODING']).to be_nil
+    end
+  end
+
+  describe '#call' do
+    context 'with collector route' do
+      let(:path) { '/api/v4/error_tracking/collector/1/store'}
+
+      it_behaves_like 'decompress middleware'
+    end
+
+    context 'with collector route under relative url' do
+      let(:path) { '/gitlab/api/v4/error_tracking/collector/1/store'}
+
+      before do
+        stub_config_setting(relative_url_root: '/gitlab')
+      end
+
+      it_behaves_like 'decompress middleware'
+    end
+
+    context 'with some other route' do
+      let(:path) { '/api/projects/123' }
+
+      it 'keeps the original input' do
+        expect(app).to receive(:call)
+
+        middleware.call(env)
+
+        expect(env['rack.input'].read).to eq(input)
+        expect(env['HTTP_CONTENT_ENCODING']).to eq('gzip')
+      end
+    end
+
+    context 'payload is too large' do
+      let(:body_limit) { Gitlab::Middleware::CompressedJson::MAXIMUM_BODY_SIZE }
+      let(:decompressed_input) { 'a' * (body_limit + 100) }
+      let(:input) { ActiveSupport::Gzip.compress(decompressed_input) }
+      let(:path) { '/api/v4/error_tracking/collector/1/envelope'}
+
+      it 'reads only limited size' do
+        expect(middleware.call(env))
+          .to eq([413, { 'Content-Type' => 'text/plain' }, ['Payload Too Large']])
+      end
+    end
+  end
+end

spec/models/analytics/cycle_analytics/issue_stage_event_spec.rb

@@ -9,5 +9,9 @@ RSpec.describe Analytics::CycleAnalytics::IssueStageEvent do
   it { is_expected.to validate_presence_of(:project_id) }
   it { is_expected.to validate_presence_of(:start_event_timestamp) }
 
+  it 'has state enum' do
+    expect(described_class.states).to eq(Issue.available_states)
+  end
+
   it_behaves_like 'StageEventModel'
 end

spec/models/analytics/cycle_analytics/merge_request_stage_event_spec.rb

@@ -9,5 +9,9 @@ RSpec.describe Analytics::CycleAnalytics::MergeRequestStageEvent do
   it { is_expected.to validate_presence_of(:project_id) }
   it { is_expected.to validate_presence_of(:start_event_timestamp) }
 
+  it 'has state enum' do
+    expect(described_class.states).to eq(MergeRequest.available_states)
+  end
+
   it_behaves_like 'StageEventModel'
 end

spec/requests/api/error_tracking/collector_spec.rb

@@ -122,6 +122,20 @@ RSpec.describe API::ErrorTracking::Collector do
       it_behaves_like 'bad request'
     end
 
+    context 'gzip body' do
+      let(:headers) do
+        {
+          'X-Sentry-Auth' => "Sentry sentry_key=#{client_key.public_key}",
+          'HTTP_CONTENT_ENCODING' => 'gzip',
+          'CONTENT_TYPE' => 'application/json'
+        }
+      end
+
+      let(:raw_event) { ActiveSupport::Gzip.compress(fixture_file('error_tracking/parsed_event.json')) }
+
+      it_behaves_like 'successful request'
+    end
+
     context 'sentry_key as param and empty headers' do
       let(:url) { "/error_tracking/collector/api/#{project.id}/store?sentry_key=#{sentry_key}" }
       let(:headers) { {} }

spec/support/shared_examples/models/concerns/analytics/cycle_analytics/stage_event_model_examples.rb

@@ -12,6 +12,7 @@ RSpec.shared_examples 'StageEventModel' do
           project_id: 4,
           author_id: 5,
           milestone_id: 6,
+          state_id: 1,
           start_event_timestamp: time,
           end_event_timestamp: time
         },
@@ -22,6 +23,7 @@ RSpec.shared_examples 'StageEventModel' do
           project_id: 11,
           author_id: 12,
           milestone_id: 13,
+          state_id: 1,
           start_event_timestamp: time,
           end_event_timestamp: time
         }
@@ -36,6 +38,7 @@ RSpec.shared_examples 'StageEventModel' do
         :project_id,
         :milestone_id,
         :author_id,
+        :state_id,
         :start_event_timestamp,
         :end_event_timestamp
       ]
@@ -59,7 +62,13 @@ RSpec.shared_examples 'StageEventModel' do
       upsert_data
 
       output_data = described_class.all.map do |record|
-        column_order.map { |column| record[column] }
+        column_order.map do |column|
+          if column == :state_id
+            described_class.states[record[column]]
+          else
+            record[column]
+          end
+        end
       end.sort
 
       expect(input_data.map(&:values).sort).to eq(output_data)

tooling/danger/project_helper.rb

@@ -22,12 +22,12 @@ module Tooling
         ce_ee_vue_templates
         ci_templates
         datateam
-        metadata
         feature_flag
         roulette
         sidekiq_queues
         specialization_labels
         specs
+        z_metadata
       ].freeze
 
       MESSAGE_PREFIX = '==>'
@@ -189,6 +189,10 @@ module Tooling
         read_file(filename).lines(chomp: true)
       end
 
+      def labels_to_add
+        @labels_to_add ||= []
+      end
+
       private
 
       def read_file(filename)