Commit b8056cf8 authored by Dmitriy Zaporozhets's avatar Dmitriy Zaporozhets

Merge branch '7062-format-dast-output' into 'master'

Store DAST scan results into the database

Closes #7062

See merge request gitlab-org/gitlab-ee!9192
parents 954fa1ca 32941e05
...@@ -12,7 +12,8 @@ module EE ...@@ -12,7 +12,8 @@ module EE
LICENSED_PARSER_FEATURES = { LICENSED_PARSER_FEATURES = {
sast: :sast, sast: :sast,
dependency_scanning: :dependency_scanning, dependency_scanning: :dependency_scanning,
container_scanning: :container_scanning container_scanning: :container_scanning,
dast: :dast
}.with_indifferent_access.freeze }.with_indifferent_access.freeze
prepended do prepended do
...@@ -63,6 +64,9 @@ module EE ...@@ -63,6 +64,9 @@ module EE
next if file_type == "container_scanning" && next if file_type == "container_scanning" &&
::Feature.disabled?(:parse_container_scanning_reports, default_enabled: false) ::Feature.disabled?(:parse_container_scanning_reports, default_enabled: false)
next if file_type == "dast" &&
::Feature.disabled?(:parse_dast_reports, default_enabled: false)
security_reports.get_report(file_type).tap do |security_report| security_reports.get_report(file_type).tap do |security_report|
begin begin
next unless project.feature_available?(LICENSED_PARSER_FEATURES.fetch(file_type)) next unless project.feature_available?(LICENSED_PARSER_FEATURES.fetch(file_type))
......
---
title: Store DAST scan results in the database
merge_request: 9192
author:
type: added
...@@ -12,6 +12,7 @@ module EE ...@@ -12,6 +12,7 @@ module EE
license_management: ::Gitlab::Ci::Parsers::LicenseManagement::LicenseManagement, license_management: ::Gitlab::Ci::Parsers::LicenseManagement::LicenseManagement,
dependency_scanning: ::Gitlab::Ci::Parsers::Security::DependencyScanning, dependency_scanning: ::Gitlab::Ci::Parsers::Security::DependencyScanning,
container_scanning: ::Gitlab::Ci::Parsers::Security::ContainerScanning, container_scanning: ::Gitlab::Ci::Parsers::Security::ContainerScanning,
dast: ::Gitlab::Ci::Parsers::Security::Dast,
sast: ::Gitlab::Ci::Parsers::Security::Sast sast: ::Gitlab::Ci::Parsers::Security::Sast
}) })
end end
......
...@@ -89,6 +89,10 @@ module Gitlab ...@@ -89,6 +89,10 @@ module Gitlab
def generate_identifier_fingerprint(identifier) def generate_identifier_fingerprint(identifier)
Digest::SHA1.hexdigest("#{identifier['type']}:#{identifier['value']}") Digest::SHA1.hexdigest("#{identifier['type']}:#{identifier['value']}")
end end
def generate_location_fingerprint(location)
raise NotImplementedError
end
end end
end end
end end
......
# frozen_string_literal: true
module Gitlab
module Ci
module Parsers
module Security
class Dast < Common
FORMAT_VERSION = '2.0'.freeze
protected
def parse_report(json_data)
report = super
format_report(report)
end
private
def format_report(data)
{
'vulnerabilities' => extract_vulnerabilities_from(data),
'version' => FORMAT_VERSION
}
end
def extract_vulnerabilities_from(data)
site = data['site']
results = []
if site
host = site['@name']
site['alerts'].each do |vulnerability|
results += flatten_vulnerabilities(vulnerability, host)
end
end
results
end
def flatten_vulnerabilities(vulnerability, host)
common_vulnerability = format_vulnerability(vulnerability)
vulnerability['instances'].map do |instance|
common_vulnerability.merge('location' => location(instance, host))
end
end
def format_vulnerability(vulnerability)
{
'category' => 'dast',
'message' => vulnerability['name'],
'description' => sanitize(vulnerability['desc']),
'cve' => vulnerability['pluginid'],
'severity' => severity(vulnerability['riskcode']),
'solution' => sanitize(vulnerability['solution']),
'confidence' => confidence(vulnerability['confidence']),
'scanner' => { 'id' => 'zaproxy', 'name' => 'ZAProxy' },
'identifiers' => [
{
'type' => 'ZAProxy_PluginId',
'name' => vulnerability['name'],
'value' => vulnerability['pluginid'],
'url' => "https://github.com/zaproxy/zaproxy/blob/w2019-01-14/docs/scanners.md"
},
{
'type' => 'CWE',
'name' => "CWE-#{vulnerability['cweid']}",
'value' => vulnerability['cweid'],
'url' => "https://cwe.mitre.org/data/definitions/#{vulnerability['cweid']}.html"
},
{
'type' => 'WASC',
'name' => "WASC-#{vulnerability['wascid']}",
'value' => vulnerability['wascid'],
'url' => "http://projects.webappsec.org/w/page/13246974/Threat%20Classification%20Reference%20Grid"
}
],
'links' => links(vulnerability['reference'])
}
end
def generate_location_fingerprint(location)
Digest::SHA1.hexdigest("#{location['param']} #{location['method']} #{location['path']}")
end
# https://github.com/zaproxy/zaproxy/blob/cfb44f7e29f490d95b03830d90aadaca51a72a6a/src/scripts/templates/passive/Passive%20default%20template.js#L25
# NOTE: ZAProxy levels: 0: info, 1: low, 2: medium, 3: high
def severity(value)
case Integer(value)
when 0
'ignore'
when 1
'low'
when 2
'medium'
when 3
'high'
else
'unknown'
end
rescue ArgumentError
'unknown'
end
# NOTE: ZAProxy levels: 0: falsePositive, 1: low, 2: medium, 3: high, 4: confirmed
def confidence(value)
case Integer(value)
when 0
'ignore'
when 1
'low'
when 2
'medium'
when 3
'high'
when 4
'critical'
else
'unknown'
end
rescue ArgumentError
'unknown'
end
def links(reference)
urls_from(reference).each_with_object([]) do |url, links|
next if url.blank?
links << { 'url' => url }
end
end
def urls_from(reference)
tags = reference.lines('</p>')
tags.map { |tag| sanitize(tag) }
end
def location(instance, hostname)
{
'param' => instance['param'],
'method' => instance['method'],
'hostname' => hostname,
'path' => instance['uri'].sub(hostname, '')
}
end
def sanitize(html_str)
ActionView::Base.full_sanitizer.sanitize(html_str)
end
end
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Ci::Parsers::Security::Dast do
let(:parser) { described_class.new }
describe '#parse!' do
let(:project) { artifact.project }
let(:pipeline) { artifact.job.pipeline }
let(:artifact) { create(:ee_ci_job_artifact, :dast) }
let(:report) { Gitlab::Ci::Reports::Security::Report.new(artifact.file_type) }
before do
artifact.each_blob do |blob|
parser.parse!(blob, report)
end
end
it 'parses all identifiers and occurrences' do
expect(report.occurrences.length).to eq(2)
expect(report.identifiers.length).to eq(3)
expect(report.scanners.length).to eq(1)
end
it 'generates expected location fingerprint' do
expected1 = Digest::SHA1.hexdigest('X-Content-Type-Options GET ')
expected2 = Digest::SHA1.hexdigest('X-Content-Type-Options GET /')
expect(report.occurrences.first[:location_fingerprint]).to eq(expected1)
expect(report.occurrences.last[:location_fingerprint]).to eq(expected2)
end
describe 'occurrence properties' do
using RSpec::Parameterized::TableSyntax
where(:attribute, :value) do
:report_type | 'dast'
:severity | 'low'
:confidence | 'medium'
end
with_them do
it 'saves properly occurrence' do
occurrence = report.occurrences.last
expect(occurrence[attribute]).to eq(value)
end
end
end
end
describe '#format_vulnerability' do
let(:parsed_report) do
JSON.parse!(
File.read(
Rails.root.join('spec/fixtures/security-reports/master/gl-dast-report.json')
)
)
end
let(:file_vulnerability) { parsed_report['site']['alerts'][0] }
let(:sanitized_desc) { file_vulnerability['desc'].gsub('<p>', '').gsub('</p>', '') }
let(:sanitized_solution) { file_vulnerability['solution'].gsub('<p>', '').gsub('</p>', '') }
let(:version) { parsed_report['@version'] }
it 'format ZAProxy vulnerability into common format' do
data = parser.send(:format_vulnerability, file_vulnerability)
expect(data['category']).to eq('dast')
expect(data['message']).to eq('X-Content-Type-Options Header Missing')
expect(data['description']).to eq(sanitized_desc)
expect(data['cve']).to eq('10021')
expect(data['severity']).to eq('low')
expect(data['confidence']).to eq('medium')
expect(data['solution']).to eq(sanitized_solution)
expect(data['scanner']).to eq({ 'id' => 'zaproxy', 'name' => 'ZAProxy' })
expect(data['links']).to eq([{ 'url' => 'http://msdn.microsoft.com/en-us/library/ie/gg622941%28v=vs.85%29.aspx' },
{ 'url' => 'https://www.owasp.org/index.php/List_of_useful_HTTP_headers' }])
expect(data['identifiers'][0]).to eq({
'type' => 'ZAProxy_PluginId',
'name' => 'X-Content-Type-Options Header Missing',
'value' => '10021',
'url' => "https://github.com/zaproxy/zaproxy/blob/w2019-01-14/docs/scanners.md"
})
expect(data['identifiers'][1]).to eq({
'type' => 'CWE',
'name' => "CWE-16",
'value' => '16',
'url' => "https://cwe.mitre.org/data/definitions/16.html"
})
expect(data['identifiers'][2]).to eq({
'type' => 'WASC',
'name' => "WASC-15",
'value' => '15',
'url' => "http://projects.webappsec.org/w/page/13246974/Threat%20Classification%20Reference%20Grid"
})
end
end
describe '#location' do
let(:file_vulnerability) do
JSON.parse!(
File.read(
Rails.root.join('spec/fixtures/security-reports/master/gl-dast-report.json')
)
)['site']['alerts'][0]
end
let(:instance) { file_vulnerability['instances'][1] }
let(:host) { 'http://bikebilly-spring-auto-devops-review-feature-br-3y2gpb.35.192.176.43.xip.io' }
it 'format location struct' do
data = parser.send(:location, instance, host)
expect(data['param']).to eq('X-Content-Type-Options')
expect(data['method']).to eq('GET')
expect(data['hostname']).to eq(host)
expect(data['path']).to eq('/')
end
end
describe '#severity' do
using RSpec::Parameterized::TableSyntax
where(:severity, :expected) do
'0' | 'ignore'
'1' | 'low'
'2' | 'medium'
'3' | 'high'
'42' | 'unknown'
'' | 'unknown'
end
with_them do
it 'substitutes with right values' do
expect(parser.send(:severity, severity)).to eq(expected)
end
end
end
describe '#confidence' do
using RSpec::Parameterized::TableSyntax
where(:confidence, :expected) do
'0' | 'ignore'
'1' | 'low'
'2' | 'medium'
'3' | 'high'
'4' | 'critical'
'42' | 'unknown'
'' | 'unknown'
end
with_them do
it 'substitutes with right values' do
expect(parser.send(:confidence, confidence)).to eq(expected)
end
end
end
end
...@@ -157,7 +157,7 @@ describe Ci::Build do ...@@ -157,7 +157,7 @@ describe Ci::Build do
subject { job.collect_security_reports!(security_reports) } subject { job.collect_security_reports!(security_reports) }
before do before do
stub_licensed_features(sast: true, dependency_scanning: true, container_scanning: true) stub_licensed_features(sast: true, dependency_scanning: true, container_scanning: true, dast: true)
end end
context 'when build has a security report' do context 'when build has a security report' do
...@@ -178,6 +178,7 @@ describe Ci::Build do ...@@ -178,6 +178,7 @@ describe Ci::Build do
create(:ee_ci_job_artifact, :sast, job: job, project: job.project) create(:ee_ci_job_artifact, :sast, job: job, project: job.project)
create(:ee_ci_job_artifact, :dependency_scanning, job: job, project: job.project) create(:ee_ci_job_artifact, :dependency_scanning, job: job, project: job.project)
create(:ee_ci_job_artifact, :container_scanning, job: job, project: job.project) create(:ee_ci_job_artifact, :container_scanning, job: job, project: job.project)
create(:ee_ci_job_artifact, :dast, job: job, project: job.project)
end end
it 'parses blobs and add the results to the reports' do it 'parses blobs and add the results to the reports' do
...@@ -186,6 +187,7 @@ describe Ci::Build do ...@@ -186,6 +187,7 @@ describe Ci::Build do
expect(security_reports.get_report('sast').occurrences.size).to eq(33) expect(security_reports.get_report('sast').occurrences.size).to eq(33)
expect(security_reports.get_report('dependency_scanning').occurrences.size).to eq(4) expect(security_reports.get_report('dependency_scanning').occurrences.size).to eq(4)
expect(security_reports.get_report('container_scanning').occurrences.size).to eq(8) expect(security_reports.get_report('container_scanning').occurrences.size).to eq(8)
expect(security_reports.get_report('dast').occurrences.size).to eq(2)
end end
end end
...@@ -217,6 +219,20 @@ describe Ci::Build do ...@@ -217,6 +219,20 @@ describe Ci::Build do
end end
end end
context 'when Feature flag is disabled for DAST reports parsing' do
before do
stub_feature_flags(parse_dast_reports: false)
create(:ee_ci_job_artifact, :sast, job: job, project: job.project)
create(:ee_ci_job_artifact, :dast, job: job, project: job.project)
end
it 'does NOT parse dast report' do
subject
expect(security_reports.reports.keys).to contain_exactly('sast')
end
end
context 'when there is a corrupted sast report' do context 'when there is a corrupted sast report' do
before do before do
create(:ee_ci_job_artifact, :sast_with_corrupted_data, job: job, project: job.project) create(:ee_ci_job_artifact, :sast_with_corrupted_data, job: job, project: job.project)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment