Skip to content

G
gitlab-ce
Commit b2151a31 authored by Heinrich Lee Yu's avatar Heinrich Lee Yu
Browse files
Options

Merge branch 'remove-postgres-hll-batch-counting-ff' into 'master' 

Remove postgres hll batch counting ff [RUN ALL RSPEC] [RUN AS-IF-FOSS]

See merge request gitlab-org/gitlab!62813
parents 11a68aa4 38f718f8
Hide whitespace changes
Inline Side-by-side
Showing with 23 additions and 278 deletions
config/feature_flags/development/postgres_hll_batch_counting.yml deleted 100644 → 0
View file @ 11a68aa4
---
name: postgres_hll_batch_counting
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/48233
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/285485
milestone: '13.7'
type: development
group: group::product analytics
default_enabled: false
ee/lib/ee/gitlab/usage_data.rb
View file @ b2151a31
......@@ -393,7 +393,6 @@ module EE
# rubocop:disable CodeReuse/ActiveRecord
def count_secure_user_scans(time_period)
return {} if time_period.blank?
return {} unless ::Feature.enabled?(:postgres_hll_batch_counting)
user_scans = {}
start_id, finish_id = min_max_security_scan_id(time_period)
......@@ -435,53 +434,33 @@ module EE
pipelines_with_secure_jobs = {}
# HLL batch counting always iterate over pkey of
# given relation, while ordinary batch count
# iterated over counted attribute, one-to-many joins
# can break batch size limitation, and lead to
# time outing batch queries, to avoid that
# different join strategy is used for HLL counter
if ::Feature.enabled?(:postgres_hll_batch_counting)
start_id, finish_id = min_max_security_scan_id(time_period)
start_id, finish_id = min_max_security_scan_id(time_period)
::Security::Scan.scan_types.each do |name, scan_type|
relation = ::Security::Scan
.latest_successful_by_build
.by_scan_types(scan_type)
.where(security_scans: time_period)
metric_name = "#{name}_pipeline"
aggregated_metrics_params = {
metric_name: metric_name,
recorded_at_timestamp: recorded_at,
time_period: time_period
}
pipelines_with_secure_jobs[metric_name.to_sym] =
if start_id && finish_id
estimate_batch_distinct_count(relation, :commit_id, batch_size: 1000, start: start_id, finish: finish_id) do |result|
::Gitlab::Usage::Metrics::Aggregates::Sources::PostgresHll
.save_aggregated_metrics(**aggregated_metrics_params.merge({ data: result }))
end
else
::Security::Scan.scan_types.each do |name, scan_type|
relation = ::Security::Scan
.latest_successful_by_build
.by_scan_types(scan_type)
.where(security_scans: time_period)
metric_name = "#{name}_pipeline"
aggregated_metrics_params = {
metric_name: metric_name,
recorded_at_timestamp: recorded_at,
time_period: time_period
}
pipelines_with_secure_jobs[metric_name.to_sym] =
if start_id && finish_id
estimate_batch_distinct_count(relation, :commit_id, batch_size: 1000, start: start_id, finish: finish_id) do |result|
::Gitlab::Usage::Metrics::Aggregates::Sources::PostgresHll
.save_aggregated_metrics(**aggregated_metrics_params.merge({ data: ::Gitlab::Database::PostgresHll::Buckets.new }))
0
.save_aggregated_metrics(**aggregated_metrics_params.merge({ data: result }))
end
end
else
start = minimum_id(::Ci::Pipeline)
finish = maximum_id(::Ci::Pipeline)
::Security::Scan.scan_types.each do |name, scan_type|
relation = ::Ci::Build.joins(:security_scans)
.where(status: 'success', retried: [nil, false])
.where(security_scans: { scan_type: scan_type })
.where(time_period)
pipelines_with_secure_jobs["#{name}_pipeline".to_sym] = distinct_count(relation, :commit_id, start: start, finish: finish, batch: false)
end
else
::Gitlab::Usage::Metrics::Aggregates::Sources::PostgresHll
.save_aggregated_metrics(**aggregated_metrics_params.merge({ data: ::Gitlab::Database::PostgresHll::Buckets.new }))
0
end
end
pipelines_with_secure_jobs
end
......
ee/spec/lib/ee/gitlab/usage_data_spec.rb
View file @ b2151a31
......@@ -775,232 +775,6 @@ RSpec.describe Gitlab::UsageData do
user_secret_detection_scans: be_within(error_rate).percent_of(1)
)
end
context 'with feature flag: postgres_hll_batch_counting is disabled' do
before do
stub_feature_flags(postgres_hll_batch_counting: false)
end
it 'does not count users who have run scans' do
for_defined_days_back do
create(:ee_ci_build, :api_fuzzing, :success, user: user3)
create(:ee_ci_build, :dast, :success, user: user2)
create(:ee_ci_build, :container_scanning, :success, user: user3)
create(:ee_ci_build, :coverage_fuzzing, :success, user: user)
create(:ee_ci_build, :dependency_scanning, :success, user: user)
create(:ee_ci_build, :sast, :success, user: user2)
create(:ee_ci_build, :secret_detection, :success, user: user)
create(:ee_ci_build, :secret_detection, :running, user: user2)
create(:ee_ci_build, :secret_detection, :failed, user: user3)
end
expect(described_class.usage_activity_by_stage_secure(described_class.monthly_time_range_db_params)).not_to include(
:user_api_fuzzing_scans,
:user_container_scanning_scans,
:user_coverage_fuzzing_scans,
:user_dast_scans,
:user_dependency_scanning_scans,
:user_sast_scans,
:user_secret_detection_scans
)
end
it 'includes accurate usage_activity_by_stage data' do
expect(described_class.usage_activity_by_stage_secure(described_class.monthly_time_range_db_params)).to include(
user_preferences_group_overview_security_dashboard: 3,
user_container_scanning_jobs: 1,
user_api_fuzzing_jobs: 1,
user_api_fuzzing_dnd_jobs: 1,
user_coverage_fuzzing_jobs: 1,
user_dast_jobs: 1,
user_dependency_scanning_jobs: 1,
user_license_management_jobs: 1,
user_sast_jobs: 1,
user_secret_detection_jobs: 1,
sast_pipeline: 0,
sast_scans: 0,
dependency_scanning_pipeline: 0,
dependency_scanning_scans: 0,
container_scanning_pipeline: 0,
container_scanning_scans: 0,
dast_pipeline: 0,
dast_scans: 0,
secret_detection_pipeline: 0,
secret_detection_scans: 0,
coverage_fuzzing_pipeline: 0,
coverage_fuzzing_scans: 0,
api_fuzzing_pipeline: 0,
api_fuzzing_scans: 0,
user_unique_users_all_secure_scanners: 1
)
end
it 'counts pipelines that have security jobs' do
for_defined_days_back do
ds_build = create(:ci_build, name: 'retirejs', user: user, status: 'success')
ds_bundler_audit_build = create(:ci_build, :failed, user: user, name: 'retirejs')
ds_bundler_build = create(:ci_build, name: 'bundler-audit', user: user, commit_id: ds_build.pipeline.id, status: 'success')
secret_detection_build = create(:ci_build, name: 'secret', user: user, commit_id: ds_build.pipeline.id, status: 'success')
cs_build = create(:ci_build, name: 'klar', user: user, status: 'success')
sast_build = create(:ci_build, name: 'sast', user: user, status: 'success', retried: true)
create(:security_scan, build: ds_build, scan_type: 'dependency_scanning' )
create(:security_scan, build: ds_bundler_build, scan_type: 'dependency_scanning')
create(:security_scan, build: secret_detection_build, scan_type: 'secret_detection')
create(:security_scan, build: cs_build, scan_type: 'container_scanning')
create(:security_scan, build: sast_build, scan_type: 'sast')
create(:security_scan, build: ds_bundler_audit_build, scan_type: 'dependency_scanning')
end
expect(described_class.usage_activity_by_stage_secure({})).to include(
user_preferences_group_overview_security_dashboard: 3,
user_container_scanning_jobs: 1,
user_dast_jobs: 1,
user_dependency_scanning_jobs: 1,
user_license_management_jobs: 1,
user_sast_jobs: 1,
user_secret_detection_jobs: 1,
user_unique_users_all_secure_scanners: 1,
sast_scans: 0,
dependency_scanning_scans: 4,
container_scanning_scans: 2,
dast_scans: 0,
secret_detection_scans: 2,
coverage_fuzzing_scans: 0
)
expect(described_class.usage_activity_by_stage_secure(described_class.monthly_time_range_db_params)).to include(
user_preferences_group_overview_security_dashboard: 3,
user_api_fuzzing_jobs: 1,
user_api_fuzzing_dnd_jobs: 1,
user_container_scanning_jobs: 1,
user_coverage_fuzzing_jobs: 1,
user_dast_jobs: 1,
user_dependency_scanning_jobs: 1,
user_license_management_jobs: 1,
user_sast_jobs: 1,
user_secret_detection_jobs: 1,
sast_pipeline: 0,
dependency_scanning_pipeline: 1,
container_scanning_pipeline: 1,
dast_pipeline: 0,
secret_detection_pipeline: 1,
coverage_fuzzing_pipeline: 0,
api_fuzzing_pipeline: 0,
user_unique_users_all_secure_scanners: 1,
sast_scans: 0,
dependency_scanning_scans: 2,
container_scanning_scans: 1,
dast_scans: 0,
secret_detection_scans: 1,
coverage_fuzzing_scans: 0,
api_fuzzing_scans: 0
)
end
it 'counts unique users correctly across multiple scanners' do
for_defined_days_back do
create(:ci_build, name: 'sast', user: user2)
create(:ci_build, name: 'dast', user: user2)
create(:ci_build, name: 'dast', user: user3)
end
expect(described_class.usage_activity_by_stage_secure(described_class.monthly_time_range_db_params)).to include(
user_preferences_group_overview_security_dashboard: 3,
user_api_fuzzing_jobs: 1,
user_api_fuzzing_dnd_jobs: 1,
user_container_scanning_jobs: 1,
user_coverage_fuzzing_jobs: 1,
user_dast_jobs: 3,
user_dependency_scanning_jobs: 1,
user_license_management_jobs: 1,
user_sast_jobs: 2,
user_secret_detection_jobs: 1,
sast_pipeline: 0,
sast_scans: 0,
dependency_scanning_pipeline: 0,
dependency_scanning_scans: 0,
container_scanning_pipeline: 0,
container_scanning_scans: 0,
dast_pipeline: 0,
dast_scans: 0,
secret_detection_pipeline: 0,
secret_detection_scans: 0,
coverage_fuzzing_pipeline: 0,
coverage_fuzzing_scans: 0,
api_fuzzing_pipeline: 0,
api_fuzzing_scans: 0,
user_unique_users_all_secure_scanners: 3
)
end
it 'combines license_scanning into license_management' do
for_defined_days_back do
create(:ci_build, name: 'license_scanning', user: user)
end
expect(described_class.usage_activity_by_stage_secure(described_class.monthly_time_range_db_params)).to include(
user_preferences_group_overview_security_dashboard: 3,
user_api_fuzzing_jobs: 1,
user_api_fuzzing_dnd_jobs: 1,
user_container_scanning_jobs: 1,
user_coverage_fuzzing_jobs: 1,
user_dast_jobs: 1,
user_dependency_scanning_jobs: 1,
user_license_management_jobs: 2,
user_sast_jobs: 1,
user_secret_detection_jobs: 1,
sast_pipeline: 0,
sast_scans: 0,
dependency_scanning_pipeline: 0,
dependency_scanning_scans: 0,
container_scanning_pipeline: 0,
container_scanning_scans: 0,
dast_pipeline: 0,
dast_scans: 0,
secret_detection_pipeline: 0,
secret_detection_scans: 0,
coverage_fuzzing_pipeline: 0,
coverage_fuzzing_scans: 0,
api_fuzzing_pipeline: 0,
api_fuzzing_scans: 0,
user_unique_users_all_secure_scanners: 1
)
end
it 'has to resort to 0 for counting license scan' do
allow(Gitlab::Database::BatchCount).to receive(:batch_distinct_count).and_raise(ActiveRecord::StatementInvalid)
allow(Gitlab::Database::BatchCount).to receive(:batch_count).and_raise(ActiveRecord::StatementInvalid)
allow(::Ci::Build).to receive(:distinct_count_by).and_raise(ActiveRecord::StatementInvalid)
expect(described_class.usage_activity_by_stage_secure(described_class.monthly_time_range_db_params)).to eq(
user_preferences_group_overview_security_dashboard: -1,
user_api_fuzzing_jobs: -1,
user_api_fuzzing_dnd_jobs: -1,
user_container_scanning_jobs: -1,
user_coverage_fuzzing_jobs: -1,
user_dast_jobs: -1,
user_dependency_scanning_jobs: -1,
user_license_management_jobs: -1,
user_sast_jobs: -1,
user_secret_detection_jobs: -1,
sast_pipeline: -1,
sast_scans: -1,
dependency_scanning_pipeline: -1,
dependency_scanning_scans: -1,
container_scanning_pipeline: -1,
container_scanning_scans: -1,
dast_pipeline: -1,
dast_scans: -1,
secret_detection_pipeline: -1,
secret_detection_scans: -1,
coverage_fuzzing_pipeline: -1,
coverage_fuzzing_scans: -1,
api_fuzzing_pipeline: -1,
api_fuzzing_scans: -1,
user_unique_users_all_secure_scanners: -1
)
end
end
end
describe 'usage_activity_by_stage_verify' do
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7