Commit 8ce11592 authored by Tetiana Chupryna's avatar Tetiana Chupryna

Merge branch 'move-POROs-into-CE_part2' into 'master'

Move ee/lib/gitlab/ci/reports/security POROs to CE part2 [RUN AS-IF-FOSS]

See merge request gitlab-org/gitlab!65959
parents ffef544d 12f892d7
......@@ -2,6 +2,35 @@
module VulnerabilityFindingHelpers
extend ActiveSupport::Concern
end
def matches_signatures(other_signatures, other_uuid)
other_signature_types = other_signatures.index_by(&:algorithm_type)
# highest first
match_result = nil
signatures.sort_by(&:priority).reverse_each do |signature|
matching_other_signature = other_signature_types[signature.algorithm_type]
next if matching_other_signature.nil?
match_result = matching_other_signature == signature
break
end
VulnerabilityFindingHelpers.prepend_mod_with('VulnerabilityFindingHelpers')
if match_result.nil?
[uuid, *signature_uuids].include?(other_uuid)
else
match_result
end
end
def signature_uuids
signatures.map do |signature|
hex_sha = signature.signature_hex
::Security::VulnerabilityUUID.generate(
report_type: report_type,
location_fingerprint: hex_sha,
primary_identifier_fingerprint: primary_identifier&.fingerprint,
project_id: project_id
)
end
end
end
......@@ -2,6 +2,30 @@
module VulnerabilityFindingSignatureHelpers
extend ActiveSupport::Concern
end
# If the location object describes a physical location within a file
# (filename + line numbers), the 'location' algorithm_type should be used
# If the location object describes arbitrary data, then the 'hash'
# algorithm_type should be used.
ALGORITHM_TYPES = { hash: 1, location: 2, scope_offset: 3 }.with_indifferent_access.freeze
class_methods do
def priority(algorithm_type)
raise ArgumentError, "No priority for #{algorithm_type.inspect}" unless ALGORITHM_TYPES.key?(algorithm_type)
ALGORITHM_TYPES[algorithm_type]
end
VulnerabilityFindingSignatureHelpers.prepend_mod_with('VulnerabilityFindingSignatureHelpers')
def algorithm_types
ALGORITHM_TYPES
end
end
def priority
self.class.priority(algorithm_type)
end
def algorithm_types
self.class.algorithm_types
end
end
# frozen_string_literal: true
module EE
module VulnerabilityFindingHelpers
extend ActiveSupport::Concern
def matches_signatures(other_signatures, other_uuid)
other_signature_types = other_signatures.index_by(&:algorithm_type)
# highest first
match_result = nil
signatures.sort_by(&:priority).reverse_each do |signature|
matching_other_signature = other_signature_types[signature.algorithm_type]
next if matching_other_signature.nil?
match_result = matching_other_signature == signature
break
end
if match_result.nil?
[uuid, *signature_uuids].include?(other_uuid)
else
match_result
end
end
def signature_uuids
signatures.map do |signature|
hex_sha = signature.signature_hex
::Security::VulnerabilityUUID.generate(
report_type: report_type,
location_fingerprint: hex_sha,
primary_identifier_fingerprint: primary_identifier&.fingerprint,
project_id: project_id
)
end
end
end
end
# frozen_string_literal: true
module EE
module VulnerabilityFindingSignatureHelpers
extend ActiveSupport::Concern
# If the location object describes a physical location within a file
# (filename + line numbers), the 'location' algorithm_type should be used
#
# If the location object describes arbitrary data, then the 'hash'
# algorithm_type should be used.
PRIORITIES = {
scope_offset: 3,
location: 2,
hash: 1
}.with_indifferent_access.freeze
class_methods do
def priority(algorithm_type)
raise ArgumentError, "No priority for #{algorithm_type.inspect}" unless PRIORITIES.key?(algorithm_type)
PRIORITIES[algorithm_type]
end
end
def priority
self.class.priority(algorithm_type)
end
end
end
......@@ -8,9 +8,7 @@ module Vulnerabilities
include VulnerabilityFindingSignatureHelpers
belongs_to :finding, foreign_key: 'finding_id', inverse_of: :signatures, class_name: 'Vulnerabilities::Finding'
enum algorithm_type: { hash: 1, location: 2, scope_offset: 3 }, _prefix: :algorithm
enum algorithm_type: VulnerabilityFindingSignatureHelpers::ALGORITHM_TYPES, _prefix: :algorithm
validates :finding, presence: true
def signature_hex
......
......@@ -14,10 +14,8 @@ module EE
container_scanning: ::Gitlab::Ci::Parsers::Security::ContainerScanning,
cluster_image_scanning: ::Gitlab::Ci::Parsers::Security::ContainerScanning,
dast: ::Gitlab::Ci::Parsers::Security::Dast,
sast: ::Gitlab::Ci::Parsers::Security::Sast,
api_fuzzing: ::Gitlab::Ci::Parsers::Security::Dast,
coverage_fuzzing: ::Gitlab::Ci::Parsers::Security::CoverageFuzzing,
secret_detection: ::Gitlab::Ci::Parsers::Security::SecretDetection,
metrics: ::Gitlab::Ci::Parsers::Metrics::Generic,
requirements: ::Gitlab::Ci::Parsers::RequirementsManagement::Requirement
})
......
# frozen_string_literal: true
module EE
module Gitlab
module Ci
module Parsers
module Security
module Common
extend ::Gitlab::Utils::Override
private
# map remediations to relevant vulnerabilities
def collate_remediations
return report_data["vulnerabilities"] || [] unless report_data["remediations"]
report_data["vulnerabilities"].map do |vulnerability|
remediation = fixes[vulnerability['id']] || fixes[vulnerability['cve']]
vulnerability.merge("remediations" => [remediation])
end
end
def fixes
@fixes ||= report_data['remediations'].each_with_object({}) do |item, memo|
item['fixes'].each do |fix|
id = fix['id'] || fix['cve']
memo[id] = item if id
end
memo
end
end
def create_remediations(remediations_data)
remediations_data.to_a.compact.map do |remediation_data|
::Gitlab::Ci::Reports::Security::Remediation.new(remediation_data['summary'], remediation_data['diff'])
end
end
override :create_vulnerabilities
def create_vulnerabilities
collate_remediations.each { |vulnerability| create_vulnerability(vulnerability, create_remediations(report_data['remediations'])) }
end
end
end
end
end
end
end
# frozen_string_literal: true
module EE
module Gitlab
module Ci
module Parsers
module Security
module Validators
module SchemaValidator
module Schema
extend ::Gitlab::Utils::Override
override :root_path
def root_path
if [:sast, :secret_detection].include?(report_type)
super
else
File.join(__dir__, 'schemas')
end
end
end
end
end
end
end
end
end
end
......@@ -2,16 +2,6 @@
FactoryBot.define do
factory :ee_ci_job_artifact, class: '::Ci::JobArtifact', parent: :ci_job_artifact do
trait :sast do
file_type { :sast }
file_format { :raw }
after(:build) do |artifact, _|
artifact.file = fixture_file_upload(
Rails.root.join('spec/fixtures/security_reports/master/gl-sast-report.json'), 'application/json')
end
end
trait :with_exceeding_identifiers do
file_type { :sast }
file_format { :raw }
......@@ -22,16 +12,6 @@ FactoryBot.define do
end
end
trait :secret_detection do
file_type { :secret_detection }
file_format { :raw }
after(:build) do |artifact, _|
artifact.file = fixture_file_upload(
Rails.root.join('spec/fixtures/security_reports/master/gl-secret-detection-report.json'), 'application/json')
end
end
trait :dast do
file_format { :raw }
file_type { :dast }
......@@ -149,56 +129,6 @@ FactoryBot.define do
end
end
trait :sast_feature_branch do
file_format { :raw }
file_type { :sast }
after(:build) do |artifact, _|
artifact.file = fixture_file_upload(
Rails.root.join('ee/spec/fixtures/security_reports/feature-branch/gl-sast-report.json'), 'application/json')
end
end
trait :secret_detection_feature_branch do
file_format { :raw }
file_type { :secret_detection }
after(:build) do |artifact, _|
artifact.file = fixture_file_upload(
Rails.root.join('ee/spec/fixtures/security_reports/feature-branch/gl-secret-detection-report.json'), 'application/json')
end
end
trait :sast_deprecated do
file_type { :sast }
file_format { :raw }
after(:build) do |artifact, _|
artifact.file = fixture_file_upload(
Rails.root.join('ee/spec/fixtures/security_reports/deprecated/gl-sast-report.json'), 'application/json')
end
end
trait :sast_with_corrupted_data do
file_type { :sast }
file_format { :raw }
after(:build) do |artifact, _|
artifact.file = fixture_file_upload(
Rails.root.join('spec/fixtures/trace/sample_trace'), 'application/json')
end
end
trait :sast_with_missing_scanner do
file_type { :sast }
file_format { :raw }
after(:build) do |artifact, _|
artifact.file = fixture_file_upload(
Rails.root.join('ee/spec/fixtures/security_reports/master/gl-sast-missing-scanner.json'), 'application/json')
end
end
trait :license_scanning do
file_type { :license_scanning }
file_format { :raw }
......@@ -319,26 +249,6 @@ FactoryBot.define do
end
end
trait :common_security_report do
file_format { :raw }
file_type { :dependency_scanning }
after(:build) do |artifact, _|
artifact.file = fixture_file_upload(
Rails.root.join('ee/spec/fixtures/security_reports/master/gl-common-scanning-report.json'), 'application/json')
end
end
trait :common_security_report_with_blank_names do
file_format { :raw }
file_type { :dependency_scanning }
after(:build) do |artifact, _|
artifact.file = fixture_file_upload(
Rails.root.join('ee/spec/fixtures/security_reports/master/gl-common-scanning-report-names.json'), 'application/json')
end
end
trait :container_scanning_feature_branch do
file_format { :raw }
file_type { :container_scanning }
......
......@@ -8,7 +8,7 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Common do
with_them do
let_it_be(:pipeline) { create(:ci_pipeline) }
let(:artifact) { build(:ee_ci_job_artifact, :common_security_report) }
let(:artifact) { build(:ci_job_artifact, :common_security_report) }
let(:report) { Gitlab::Ci::Reports::Security::Report.new(artifact.file_type, pipeline, 2.weeks.ago) }
let(:location) { ::Gitlab::Ci::Reports::Security::Locations::DependencyScanning.new(file_path: 'yarn/yarn.lock', package_version: 'v2', package_name: 'saml2') }
let(:tracking_data) { nil }
......@@ -94,7 +94,7 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Common do
end
describe 'parsing finding.name' do
let(:artifact) { build(:ee_ci_job_artifact, :common_security_report_with_blank_names) }
let(:artifact) { build(:ci_job_artifact, :common_security_report_with_blank_names) }
context 'when message is provided' do
it 'sets message from the report as a finding name' do
......
......@@ -11,8 +11,6 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator do
:coverage_fuzzing | ['root is missing required keys: vulnerabilities'] | { 'version' => '10.0.0', 'vulnerabilities' => [] }
:dast | ['root is missing required keys: vulnerabilities'] | { 'version' => '10.0.0', 'vulnerabilities' => [] }
:dependency_scanning | ['root is missing required keys: dependency_files, vulnerabilities'] | { 'version' => '10.0.0', 'vulnerabilities' => [], 'dependency_files' => [] }
:sast | ['root is missing required keys: vulnerabilities'] | { 'version' => '10.0.0', 'vulnerabilities' => [] }
:secret_detection | ['root is missing required keys: vulnerabilities'] | { 'version' => '10.0.0', 'vulnerabilities' => [] }
end
with_them do
......
......@@ -11,7 +11,9 @@ module Gitlab
cobertura: ::Gitlab::Ci::Parsers::Coverage::Cobertura,
terraform: ::Gitlab::Ci::Parsers::Terraform::Tfplan,
accessibility: ::Gitlab::Ci::Parsers::Accessibility::Pa11y,
codequality: ::Gitlab::Ci::Parsers::Codequality::CodeClimate
codequality: ::Gitlab::Ci::Parsers::Codequality::CodeClimate,
sast: ::Gitlab::Ci::Parsers::Security::Sast,
secret_detection: ::Gitlab::Ci::Parsers::Security::SecretDetection
}
end
......
......@@ -27,7 +27,8 @@ module Gitlab
create_scan
create_analyzer
set_report_version
collate_remediations.each { |vulnerability| create_vulnerability(vulnerability) }
create_vulnerabilities
report_data
rescue JSON::ParserError
......@@ -73,35 +74,20 @@ module Gitlab
@analyzer_data ||= report_data.dig('scan', 'analyzer')
end
# map remediations to relevant vulnerabilities
def collate_remediations
return report_data["vulnerabilities"] || [] unless report_data["remediations"]
report_data["vulnerabilities"].map do |vulnerability|
remediation = fixes[vulnerability['id']] || fixes[vulnerability['cve']]
vulnerability.merge("remediations" => [remediation])
end
def tracking_data(data)
data['tracking']
end
def fixes
@fixes ||= report_data['remediations'].each_with_object({}) do |item, memo|
item['fixes'].each do |fix|
id = fix['id'] || fix['cve']
memo[id] = item if id
end
memo
def create_vulnerabilities
if report_data["vulnerabilities"]
report_data["vulnerabilities"].each { |vulnerability| create_vulnerability(vulnerability) }
end
end
def tracking_data(data)
data['tracking']
end
def create_vulnerability(data)
def create_vulnerability(data, remediations = [])
identifiers = create_identifiers(data['identifiers'])
links = create_links(data['links'])
location = create_location(data['location'] || {})
remediations = create_remediations(data['remediations'])
signatures = create_signatures(tracking_data(data))
if @vulnerability_finding_signatures_enabled && !signatures.empty?
......@@ -231,12 +217,6 @@ module Gitlab
::Gitlab::Ci::Reports::Security::Link.new(name: link['name'], url: link['url'])
end
def create_remediations(remediations_data)
remediations_data.to_a.compact.map do |remediation_data|
::Gitlab::Ci::Reports::Security::Remediation.new(remediation_data['summary'], remediation_data['diff'])
end
end
def parse_severity_level(input)
input&.downcase.then { |value| ::Enums::Vulnerability.severity_levels.key?(value) ? value : 'unknown' }
end
......@@ -282,3 +262,5 @@ module Gitlab
end
end
end
Gitlab::Ci::Parsers::Security::Common.prepend_mod_with("Gitlab::Ci::Parsers::Security::Common")
......@@ -7,7 +7,9 @@ module Gitlab
module Validators
class SchemaValidator
class Schema
ROOT_PATH = File.join(__dir__, 'schemas')
def root_path
File.join(__dir__, 'schemas')
end
def initialize(report_type)
@report_type = report_type
......@@ -28,7 +30,7 @@ module Gitlab
end
def schema_path
File.join(ROOT_PATH, file_name)
File.join(root_path, file_name)
end
def file_name
......@@ -62,3 +64,5 @@ module Gitlab
end
end
end
Gitlab::Ci::Parsers::Security::Validators::SchemaValidator::Schema.prepend_mod_with("Gitlab::Ci::Parsers::Security::Validators::SchemaValidator::Schema")
......@@ -14,10 +14,6 @@ module Gitlab
@signature_value = params.dig(:signature_value)
end
def priority
::Vulnerabilities::FindingSignature.priority(algorithm_type)
end
def signature_sha
Digest::SHA1.digest(signature_value)
end
......@@ -34,7 +30,7 @@ module Gitlab
end
def valid?
::Vulnerabilities::FindingSignature.algorithm_types.key?(algorithm_type)
algorithm_types.key?(algorithm_type)
end
def eql?(other)
......
......@@ -287,6 +287,76 @@ FactoryBot.define do
end
end
trait :common_security_report do
file_format { :raw }
file_type { :dependency_scanning }
after(:build) do |artifact, _|
artifact.file = fixture_file_upload(
Rails.root.join('spec/fixtures/security_reports/master/gl-common-scanning-report.json'), 'application/json')
end
end
trait :common_security_report_with_blank_names do
file_format { :raw }
file_type { :dependency_scanning }
after(:build) do |artifact, _|
artifact.file = fixture_file_upload(
Rails.root.join('spec/fixtures/security_reports/master/gl-common-scanning-report-names.json'), 'application/json')
end
end
trait :sast_deprecated do
file_type { :sast }
file_format { :raw }
after(:build) do |artifact, _|
artifact.file = fixture_file_upload(
Rails.root.join('spec/fixtures/security_reports/deprecated/gl-sast-report.json'), 'application/json')
end
end
trait :sast_with_corrupted_data do
file_type { :sast }
file_format { :raw }
after(:build) do |artifact, _|
artifact.file = fixture_file_upload(
Rails.root.join('spec/fixtures/trace/sample_trace'), 'application/json')
end
end
trait :sast_feature_branch do
file_format { :raw }
file_type { :sast }
after(:build) do |artifact, _|
artifact.file = fixture_file_upload(
Rails.root.join('spec/fixtures/security_reports/feature-branch/gl-sast-report.json'), 'application/json')
end
end
trait :secret_detection_feature_branch do
file_format { :raw }
file_type { :secret_detection }
after(:build) do |artifact, _|
artifact.file = fixture_file_upload(
Rails.root.join('spec/fixtures/security_reports/feature-branch/gl-secret-detection-report.json'), 'application/json')
end
end
trait :sast_with_missing_scanner do
file_type { :sast }
file_format { :raw }
after(:build) do |artifact, _|
artifact.file = fixture_file_upload(
Rails.root.join('spec/fixtures/security_reports/master/gl-sast-missing-scanner.json'), 'application/json')
end
end
trait :secret_detection do
file_type { :secret_detection }
file_format { :raw }
......
......@@ -5,6 +5,8 @@ FactoryBot.define do
reports { FactoryBot.build_list(:ci_reports_security_report, 1) }
findings { FactoryBot.build_list(:ci_reports_security_finding, 1) }
skip_create
initialize_with do
::Gitlab::Ci::Reports::Security::AggregatedReport.new(reports, findings)
end
......
This diff is collapsed.
......@@ -18,7 +18,7 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Sast do
with_them do
let(:report) { Gitlab::Ci::Reports::Security::Report.new(artifact.file_type, pipeline, created_at) }
let(:artifact) { create(:ee_ci_job_artifact, report_format) }
let(:artifact) { create(:ci_job_artifact, report_format) }
before do
artifact.each_blob { |blob| described_class.parse!(blob, report) }
......
......@@ -13,7 +13,7 @@ RSpec.describe Gitlab::Ci::Parsers::Security::SecretDetection do
with_them do
let(:report) { Gitlab::Ci::Reports::Security::Report.new(artifact.file_type, pipeline, created_at) }
let(:artifact) { create(:ee_ci_job_artifact, report_format) }
let(:artifact) { create(:ci_job_artifact, report_format) }
before do
artifact.each_blob { |blob| described_class.parse!(blob, report) }
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator do
using RSpec::Parameterized::TableSyntax
where(:report_type, :expected_errors, :valid_data) do
:sast | ['root is missing required keys: vulnerabilities'] | { 'version' => '10.0.0', 'vulnerabilities' => [] }
:secret_detection | ['root is missing required keys: vulnerabilities'] | { 'version' => '10.0.0', 'vulnerabilities' => [] }
end
with_them do
let(:validator) { described_class.new(report_type, report_data) }
describe '#valid?' do
subject { validator.valid? }
context 'when given data is invalid according to the schema' do
let(:report_data) { {} }
it { is_expected.to be_falsey }
end
context 'when given data is valid according to the schema' do
let(:report_data) { valid_data }
it { is_expected.to be_truthy }
end
end
describe '#errors' do
let(:report_data) { { 'version' => '10.0.0' } }
subject { validator.errors }
it { is_expected.to eq(expected_errors) }
end
end
end
......@@ -4,7 +4,7 @@ require 'spec_helper'
RSpec.describe Gitlab::Ci::Reports::Security::Reports do
let_it_be(:pipeline) { create(:ci_pipeline) }
let_it_be(:artifact) { create(:ee_ci_job_artifact, :sast) }
let_it_be(:artifact) { create(:ci_job_artifact, :sast) }
let(:security_reports) { described_class.new(pipeline) }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment