Skip to content

G
gitlab-ce
Commit bd93f764 authored by Mehmet Emin INAC's avatar Mehmet Emin INAC
Browse files
Options

Change the migration to do not depend on repor files

parent e649fc11
Hide whitespace changes
Inline Side-by-side
Showing with 18 additions and 208 deletions
ee/lib/ee/gitlab/background_migration/populate_resolved_on_default_branch_column.rb
View file @ bd93f764
......@@ -87,30 +87,21 @@ module EE
has_one :route, as: :source
has_many :vulnerabilities
has_many :vulnerability_findings
has_many :vulnerability_identifiers
has_many :vulnerability_scanners
def self.polymorphic_name
'Project'
end
def reports
return [] unless default_branch
def resolved_vulnerabilities
return Vulnerability.none unless latest_pipeline_id
@reports ||= artifacts.flat_map(&:reports)
vulnerabilities.not_found_in_pipeline_id(latest_pipeline_id)
end
private
delegate :connection, to: :'self.class', private: true
def artifacts
return [] unless latest_pipeline_id
JobArtifact.for_pipeline(latest_pipeline_id).each { |artifact| artifact.project = self }
end
def latest_pipeline_id
strong_memoize(:latest_pipeline_id) { pipeline_with_reports&.fetch('id') }
end
......@@ -204,140 +195,17 @@ module EE
end
end
class JobArtifact < ActiveRecord::Base
ARTIFACTS_SQL = <<~SQL
SELECT
"ci_job_artifacts".*
FROM "ci_job_artifacts"
INNER JOIN "ci_builds" ON "ci_job_artifacts"."job_id" = "ci_builds"."id"
AND "ci_builds"."commit_id" = %{commit_id}
AND "ci_builds"."type" = 'Ci::Build'
AND ("ci_builds"."retried" IS FALSE OR "ci_builds"."retried" IS NULL)
WHERE
"ci_job_artifacts"."file_type" IN (%{file_types})
SQL
FILE_FORMAT_ADAPTERS = {
gzip: ::Gitlab::Ci::Build::Artifacts::Adapters::GzipStream,
raw: ::Gitlab::Ci::Build::Artifacts::Adapters::RawStream
}.freeze
self.table_name = 'ci_job_artifacts'
enum file_format: {
raw: 1,
zip: 2,
gzip: 3
}, _suffix: true
enum file_location: {
legacy_path: 1,
hashed_path: 2
}
enum file_type: {
archive: 1,
metadata: 2,
trace: 3,
junit: 4,
sast: 5, ## EE-specific
dependency_scanning: 6, ## EE-specific
container_scanning: 7, ## EE-specific
dast: 8, ## EE-specific
codequality: 9, ## EE-specific
license_management: 10, ## EE-specific
license_scanning: 101, ## EE-specific till 13.0
performance: 11, ## EE-specific till 13.2
metrics: 12, ## EE-specific
metrics_referee: 13, ## runner referees
network_referee: 14, ## runner referees
lsif: 15, # LSIF data for code navigation
dotenv: 16,
cobertura: 17,
terraform: 18, # Transformed json
accessibility: 19,
cluster_applications: 20,
secret_detection: 21, ## EE-specific
requirements: 22, ## EE-specific
coverage_fuzzing: 23, ## EE-specific
browser_performance: 24, ## EE-specific
load_performance: 25 ## EE-specific
}
mount_uploader :file, JobArtifactUploader
attr_accessor :project
delegate :namespace, to: :project
def self.for_pipeline(pipeline_id)
find_by_sql(artifacts_sql_for(pipeline_id))
end
def self.artifacts_sql_for(pipeline_id)
format(ARTIFACTS_SQL, commit_id: pipeline_id, file_types: Project::FILE_TYPES.join(', '))
end
def reports
reports = []
each_blob do |blob|
report = ::Gitlab::Ci::Reports::Security::Report.new(file_type, nil, created_at)
parse_security_artifact_blob(report, blob)
reports << report
end
reports
end
def hashed_path?
super || file_location.nil?
end
private
def each_blob(&blk)
unless file_format_adapter_class
raise NotSupportedAdapterError, 'This file format requires a dedicated adapter'
end
file.open do |stream|
file_format_adapter_class.new(stream).each_blob(&blk)
end
end
def file_format_adapter_class
FILE_FORMAT_ADAPTERS[file_format.to_sym]
end
def parse_security_artifact_blob(security_report, blob)
report_clone = security_report.clone_as_blank
::Gitlab::Ci::Parsers.fabricate!(security_report.type).parse!(blob, report_clone)
security_report.merge!(report_clone)
end
end
class Route < ActiveRecord::Base; end
class Vulnerability < ActiveRecord::Base
include EachBatch
scope :id_not_in, -> (ids) { where.not(id: ids) }
end
class VulnerabilityFinding < ActiveRecord::Base
self.table_name = 'vulnerability_occurrences'
attribute(:project_fingerprint, ::Gitlab::Database::ShaAttribute.new)
attribute(:location_fingerprint, ::Gitlab::Database::ShaAttribute.new)
belongs_to :scanner, class_name: 'VulnerabilityScanner'
belongs_to :primary_identifier, class_name: 'VulnerabilityIdentifier'
end
class VulnerabilityScanner < ActiveRecord::Base
scope :by_external_id, -> (external_ids) { where(external_id: external_ids) }
end
class VulnerabilityIdentifier < ActiveRecord::Base
attribute(:fingerprint, ::Gitlab::Database::ShaAttribute.new)
scope :by_fingerprint, -> (fingerprints) { where(fingerprint: fingerprints) }
scope :not_found_in_pipeline_id, -> (pipeline_id) { where.not(id: found_in_pipeline(pipeline_id)) }
scope :found_in_pipeline, -> (pipeline_id) do
joins(<<~SQL)
INNER JOIN vulnerability_occurrences vo ON vo.vulnerability_id = vulnerabilities.id
INNER JOIN vulnerability_occurrence_pipelines vop ON vop.occurrence_id = vo.id AND vop.pipeline_id = #{pipeline_id}
SQL
end
end
# This class depends on following classes
......@@ -410,11 +278,8 @@ module EE
attr_accessor :project_id
delegate :reports, to: :project, private: true
def update_vulnerabilities
@updated_count ||= project.vulnerabilities
.id_not_in(existing_vulnerability_ids)
@updated_count ||= project.resolved_vulnerabilities
.update_all(resolved_on_default_branch: true)
end
......@@ -422,20 +287,11 @@ module EE
::Gitlab::BackgroundMigration::Logger.info(
migrator: 'PopulateResolvedOnDefaultBranchColumnForProject',
message: 'Project migrated',
stats: stats,
updated_count: @updated_count,
project_id: project_id
)
end
def stats
{
all_count: findings.length,
valid_count: all_valid_findings.length,
existing_count: existing_vulnerability_ids.length,
updated_count: @updated_count
}
end
def log_error(error)
::Gitlab::BackgroundMigration::Logger.error(
migrator: 'PopulateResolvedOnDefaultBranchColumnForProject',
......@@ -447,52 +303,6 @@ module EE
def project
@project ||= Project.find(project_id)
end
def existing_vulnerability_ids
@existing_vulnerability_ids ||= all_valid_findings.map { |finding| find_saved_finding_for(finding)&.vulnerability_id }.compact
end
def all_valid_findings
@all_valid_findings ||= findings.select(&:scanner)
.select(&:primary_identifier)
.select(&:location)
end
def findings
@findings ||= reports.flat_map(&:findings)
end
def find_saved_finding_for(finding)
project.vulnerability_findings.find_by({
scanner: scanner_objects[finding.scanner.key],
primary_identifier: identifier_objects[finding.primary_identifier.key],
location_fingerprint: finding.location.fingerprint
})
end
def scanner_objects
@scanner_objects ||= project.vulnerability_scanners.by_external_id(all_scanner_external_ids).group_by(&:external_id)
end
def all_scanner_external_ids
all_scanners.map(&:external_id).uniq
end
def all_scanners
reports.map(&:scanners).flat_map(&:values)
end
def identifier_objects
@identifier_objects ||= project.vulnerability_identifiers.by_fingerprint(all_identifier_fingerprints).group_by(&:fingerprint)
end
def all_identifier_fingerprints
all_identifiers.map(&:fingerprint).uniq
end
def all_identifiers
reports.map(&:identifiers).flat_map(&:values)
end
end
end
end
......
ee/spec/lib/ee/gitlab/background_migration/populate_resolved_on_default_branch_column_spec.rb
View file @ bd93f764
......@@ -9,7 +9,9 @@ RSpec.describe ::Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchCol
let(:pipelines) { table(:ci_pipelines) }
let(:vulnerabilities) { table(:vulnerabilities) }
let(:findings) { table(:vulnerability_occurrences) }
let(:finding_pipelines) { table(:vulnerability_occurrence_pipelines) }
let(:builds) { table(:ci_builds) }
let(:artifacts) { table(:ci_job_artifacts) }
let(:scanners) { table(:vulnerability_scanners) }
let(:vulnerability_identifiers) { table(:vulnerability_identifiers) }
......@@ -60,8 +62,6 @@ RSpec.describe ::Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchCol
let(:pipeline) { pipelines.create!(project_id: project.id, ref: 'master', sha: 'adf43c3a', status: 'success') }
let(:utility_object) { described_class.new(project.id) }
let(:scanner) { scanners.create!(project_id: project.id, external_id: 'bandit', name: 'Bandit') }
let(:artifact_model) { EE::Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn::JobArtifact }
let(:artifact_fixture_path) { Rails.root.join('ee/spec/fixtures/security_reports/master/gl-sast-report.json') }
let(:sha_attribute) { Gitlab::Database::ShaAttribute.new }
let(:vulnerability_identifier) do
vulnerability_identifiers.create!(
......@@ -98,11 +98,9 @@ RSpec.describe ::Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchCol
before do
build = builds.create!(commit_id: pipeline.id, retried: false, type: 'Ci::Build')
artifact = artifact_model.new(project_id: project.id, job_id: build.id, file_type: 5, file_format: 1)
artifact.file = fixture_file_upload(artifact_fixture_path, 'application/json')
artifact.save!
artifacts.create!(project_id: project.id, job_id: build.id, file_type: 5, file_format: 1)
findings.create!(
finding = findings.create!(
project_id: project.id,
vulnerability_id: existing_vulnerability.id,
severity: 5,
......@@ -117,6 +115,8 @@ RSpec.describe ::Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchCol
metadata_version: '1',
raw_metadata: '')
finding_pipelines.create!(occurrence_id: finding.id, pipeline_id: pipeline.id)
allow(::Gitlab::CurrentSettings).to receive(:default_branch_name).and_return(:master)
end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7