Commit 966d5af0 authored by Chad Woolley's avatar Chad Woolley

Create query_limiting_report.rb and update issue links

- See https://gitlab.com/gitlab-org/gitlab/-/issues/325640
- See https://gitlab.com/groups/gitlab-org/-/epics/5670
parent 3048d77a
......@@ -191,7 +191,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
end
def disable_query_limiting
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/63107')
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/29418')
end
def application_setting_params
......
......@@ -43,7 +43,7 @@ class Import::GitlabProjectsController < Import::BaseController
end
def disable_query_limiting
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42437')
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20823')
end
def uploader_class
......
......@@ -116,6 +116,6 @@ class Import::ManifestController < Import::BaseController
end
def disable_query_limiting
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/48939')
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/23147')
end
end
......@@ -119,7 +119,7 @@ class Projects::ForksController < Projects::ApplicationController
end
def disable_query_limiting
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42335')
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20783')
end
def load_namespaces_with_associations
......
......@@ -356,10 +356,10 @@ class Projects::IssuesController < Projects::ApplicationController
def disable_query_limiting
# Also see the following issues:
#
# 1. https://gitlab.com/gitlab-org/gitlab-foss/issues/42423
# 2. https://gitlab.com/gitlab-org/gitlab-foss/issues/42424
# 3. https://gitlab.com/gitlab-org/gitlab-foss/issues/42426
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42422')
# 1. https://gitlab.com/gitlab-org/gitlab/-/issues/20815
# 2. https://gitlab.com/gitlab-org/gitlab/-/issues/20816
# 3. https://gitlab.com/gitlab-org/gitlab/-/issues/21068
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20814')
end
private
......
......@@ -134,7 +134,7 @@ class Projects::MergeRequests::CreationsController < Projects::MergeRequests::Ap
# rubocop: enable CodeReuse/ActiveRecord
def disable_query_limiting
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42384')
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20801')
end
def incr_count_webide_merge_request
......
......@@ -470,8 +470,8 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
end
def disable_query_limiting
# Also see https://gitlab.com/gitlab-org/gitlab-foss/issues/42441
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42438')
# Also see https://gitlab.com/gitlab-org/gitlab/-/issues/20827
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20824')
end
def reports_response(report_comparison, pipeline = nil)
......
......@@ -43,6 +43,6 @@ class Projects::NetworkController < Projects::ApplicationController
end
def disable_query_limiting
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42333')
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20782')
end
end
......@@ -88,6 +88,6 @@ class Projects::NotesController < Projects::ApplicationController
end
def disable_query_limiting
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42383')
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20800')
end
end
......@@ -272,8 +272,8 @@ class Projects::PipelinesController < Projects::ApplicationController
end
def disable_query_limiting
# Also see https://gitlab.com/gitlab-org/gitlab-foss/issues/42343
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42339')
# Also see https://gitlab.com/gitlab-org/gitlab/-/issues/20785
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20784')
end
def authorize_update_pipeline!
......
......@@ -163,7 +163,7 @@ class RegistrationsController < Devise::RegistrationsController
end
def disable_query_limiting
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42380')
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20798')
end
def load_recaptcha
......
......@@ -11,7 +11,7 @@ module Mutations
description: 'The project to move the issue to.'
def resolve(project_path:, iid:, target_project_path:)
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/267762')
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20816')
issue = authorized_find!(project_path: project_path, iid: iid)
source_project = issue.project
......
......@@ -42,7 +42,7 @@ module Mutations
description: 'Squash commits on the source branch before merge.'
def resolve(project_path:, iid:, **args)
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42317')
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/4796')
merge_request = authorized_find!(project_path: project_path, iid: iid)
project = merge_request.target_project
......
......@@ -48,7 +48,7 @@ class PostReceiveService
end
def process_mr_push_options(push_options, changes)
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/61359')
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/28494')
return unless repository
unless repository.repo_type.project?
......
......@@ -70,7 +70,7 @@ module API
optional :variables, Array, desc: 'Array of variables available in the pipeline'
end
post ':id/pipeline' do
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42124')
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20711')
authorize! :create_pipeline, user_project
......
......@@ -112,7 +112,7 @@ module API
end
def delete_group(group)
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/46285')
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/22226')
destroy_conditionally!(group) do |group|
::Groups::DestroyService.new(group, current_user).async_execute
end
......
......@@ -242,7 +242,7 @@ module API
use :issue_params
end
post ':id/issues' do
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42320')
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20773')
check_rate_limit! :issues_create, [current_user]
......@@ -288,7 +288,7 @@ module API
end
# rubocop: disable CodeReuse/ActiveRecord
put ':id/issues/:issue_iid' do
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42322')
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20775')
issue = user_project.issues.find_by!(iid: params.delete(:issue_iid))
authorize! :update_issue, issue
......@@ -346,7 +346,7 @@ module API
end
# rubocop: disable CodeReuse/ActiveRecord
post ':id/issues/:issue_iid/move' do
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42323')
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20776')
issue = user_project.issues.find_by(iid: params[:issue_iid])
not_found!('Issue') unless issue
......
......@@ -207,7 +207,7 @@ module API
use :optional_params
end
post ":id/merge_requests" do
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42316')
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20770')
authorize! :create_merge_request_from, user_project
......@@ -416,7 +416,7 @@ module API
at_least_one_of(*::API::MergeRequests.update_params_at_least_one_of)
end
put ':id/merge_requests/:merge_request_iid' do
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42318')
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20772')
merge_request = find_merge_request_with_access(params.delete(:merge_request_iid), :update_merge_request)
......@@ -445,7 +445,7 @@ module API
optional :squash, type: Grape::API::Boolean, desc: 'When true, the commits will be squashed into a single commit on merge'
end
put ':id/merge_requests/:merge_request_iid/merge' do
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42317')
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/4796')
merge_request = find_project_merge_request(params[:merge_request_iid])
......
......@@ -67,7 +67,7 @@ module API
check_rate_limit! :project_import, [current_user, :project_import]
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42437')
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20823')
validate_file!
......
......@@ -310,7 +310,7 @@ module API
optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the fork'
end
post ':id/fork', feature_category: :source_code_management do
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42284')
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20759')
not_found! unless can?(current_user, :fork_project, user_project)
......
......@@ -21,7 +21,7 @@ module API
optional :variables, type: Hash, desc: 'The list of variables to be injected into build'
end
post ":id/(ref/:ref/)trigger/pipeline", requirements: { ref: /.+/ } do
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42283')
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20758')
forbidden! if gitlab_pipeline_hook_request?
......
#!/usr/bin/env ruby
# frozen_string_literal: true
####
# Prints a report which helps reconcile occurrences of the `QueryLimiting.disable(ISSUE_LINK)`
# allowlist block against the corresponding open issues.
#
# If everything is consistent, the script should ideally not report any issues or code lines,
# other than possibly remaining "calls with no issue iid" which use variables/etc.
#
# - See https://gitlab.com/gitlab-org/gitlab/-/issues/325640
# - See https://gitlab.com/groups/gitlab-org/-/epics/5670
require 'rubygems'
require 'gitlab'
require 'optparse'
class QueryLimitingReport
GITLAB_PROJECT_ID = 278964 # gitlab-org/gitlab project
ISSUES_SEARCH_LABEL = 'querylimiting-disable'
CODE_LINES_SEARCH_STRING = 'QueryLimiting.disable'
PAGINATION_LIMIT = 500
DEFAULT_OPTIONS = {
api_token: ENV['API_TOKEN']
}.freeze
def initialize(options)
@options = options
Gitlab.configure do |config|
config.endpoint = 'https://gitlab.com/api/v4'
config.private_token = options.fetch(:api_token)
end
end
def execute
# PLAN:
# Read all issues matching criteria and extract array of issue iids
# Find all code references and extract issue iids
# Print list of all issues without code references
# Print list of all code references issue iids that don't have search label
# Print list of all code references with no issue iids (i.e. dynamic or variable argument)
total_issues = find_issues_by_label(ISSUES_SEARCH_LABEL)
issues = total_issues.select { |issue| issue[:state] == 'opened' }
code_lines = find_code_lines
code_lines_grouped = code_lines.group_by { |code_line| code_line[:has_issue_iid] }
code_lines_without_issue_iid = code_lines_grouped[false]
code_lines_with_issue_iid = code_lines_grouped[true]
all_issue_iids_in_code_lines = code_lines_with_issue_iid.map { |line| line[:issue_iid] }
issues_without_code_references = issues.reject do |issue|
all_issue_iids_in_code_lines.include?(issue[:iid])
end
all_issue_iids = issues.map { |issue| issue[:iid] }
code_lines_with_missing_issues = code_lines_with_issue_iid.reject do |code_line|
all_issue_iids.include?(code_line[:issue_iid])
end
puts "\n\n\nREPORT:"
puts "\n\nFound #{total_issues.length} total issues with '#{ISSUES_SEARCH_LABEL}' search label, #{issues.length} are still opened..."
puts "\n\nFound #{code_lines.length} total occurrences of '#{CODE_LINES_SEARCH_STRING}' in code..."
puts "\n" + '-' * 80
puts "\n\nIssues without any '#{CODE_LINES_SEARCH_STRING}' code references (#{issues_without_code_references.length} total):"
pp issues_without_code_references
puts "\n" + '-' * 80
puts "\n\n'#{CODE_LINES_SEARCH_STRING}' calls with references to an issue which doesn't have '#{ISSUES_SEARCH_LABEL}' search label (#{code_lines_with_missing_issues.length} total):"
pp code_lines_with_missing_issues
puts "\n" + '-' * 80
puts "\n\n'#{CODE_LINES_SEARCH_STRING}' calls with no issue iid (#{code_lines_without_issue_iid&.length || 0} total):"
pp code_lines_without_issue_iid
end
private
attr_reader :options
def find_issues_by_label(label)
issues = []
puts("Finding issues by label #{label}...")
paginated_issues = Gitlab.issues(GITLAB_PROJECT_ID, 'labels' => label)
paginated_issues.paginate_with_limit(PAGINATION_LIMIT) do |item|
item_hash = item.to_hash
issue_iid = item_hash.fetch('iid')
issue = {
iid: issue_iid,
state: item_hash.fetch('state'),
title: item_hash.fetch('title'),
issue_url: "https://gitlab.com/gitlab-org/gitlab/issues/#{issue_iid}"
}
issues << issue
end
issues
end
def find_code_lines
code_lines = []
puts("Finding code lines...")
paginated_blobs = Gitlab.search_in_project(GITLAB_PROJECT_ID, 'blobs', CODE_LINES_SEARCH_STRING)
paginated_blobs.paginate_with_limit(PAGINATION_LIMIT) do |item|
item_hash = item.to_hash
filename = item_hash.fetch('filename')
next if filename !~ /\.rb\Z/
file_contents = Gitlab.file_contents(GITLAB_PROJECT_ID, filename)
file_lines = file_contents.split("\n")
file_lines.each_index do |index|
line = file_lines[index]
if line =~ /#{CODE_LINES_SEARCH_STRING}/
issue_iid = line.slice(%r{issues/(\d+)\D}, 1)
line_number = index + 1
code_line = {
file_location: "#{filename}:#{line_number}",
filename: filename,
line_number: line_number,
line: line,
issue_iid: issue_iid.to_i,
has_issue_iid: !issue_iid.nil?
}
code_lines << code_line
end
end
end
code_lines.sort_by! { |line| "#{line[:filename]}-#{line[:line_number].to_s.rjust(4, '0')}" }
code_lines.map do |line|
line.delete(:filename)
line.delete(:line_number)
line
end
end
end
if $0 == __FILE__
options = QueryLimitingReport::DEFAULT_OPTIONS.dup
OptionParser.new do |opts|
opts.on("-t", "--api-token API_TOKEN", String, "A value API token with the `read_api` scope. Can be set as an env variable 'API_TOKEN'.") do |value|
options[:api_token] = value
end
opts.on("-h", "--help", "Prints this help") do
puts opts
exit
end
end.parse!
QueryLimitingReport.new(options).execute
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment