Commit 7be313f5 authored by Olivier Gonzalez's avatar Olivier Gonzalez Committed by Kamil Trzciński

Parse and store dependency scanning results in database

parent 6b2e23a5
# frozen_string_literal: true
class Groups::Security::VulnerabilitiesController < Groups::Security::ApplicationController
def index
@vulnerabilities = group.latest_vulnerabilities.ordered
@vulnerabilities = group.latest_vulnerabilities
.sast # FIXME: workaround until https://gitlab.com/gitlab-org/gitlab-ee/issues/6240
.ordered
.page(params[:page])
respond_to do |format|
......
......@@ -8,7 +8,8 @@ module EE
extend ActiveSupport::Concern
LICENSED_PARSER_FEATURES = {
sast: :sast
sast: :sast,
dependency_scanning: :dependency_scanning
}.with_indifferent_access.freeze
prepended do
......
---
title: Parse and store dependency scanning reports in database
merge_request: 8642
author:
type: added
......@@ -7,7 +7,8 @@ module Gitlab
ParserNotFoundError = Class.new(StandardError)
PARSERS = {
sast: ::Gitlab::Ci::Parsers::Security::Sast
sast: ::Gitlab::Ci::Parsers::Security::Common,
dependency_scanning: ::Gitlab::Ci::Parsers::Security::Common
}.freeze
def self.fabricate!(file_type)
......
......@@ -4,8 +4,8 @@ module Gitlab
module Ci
module Parsers
module Security
class Sast
SastParserError = Class.new(StandardError)
class Common
SecurityReportParserError = Class.new(StandardError)
METADATA_VERSION = '1.2'
......@@ -16,9 +16,9 @@ module Gitlab
create_vulnerability(report, vulnerability)
end
rescue JSON::ParserError
raise SastParserError, 'JSON parsing failed'
raise SecurityReportParserError, 'JSON parsing failed'
rescue
raise SastParserError, 'SAST report parsing failed'
raise SecurityReportParserError, "#{report.type} security report parsing failed"
end
protected
......
......@@ -103,18 +103,40 @@ describe Groups::Security::VulnerabilitiesController do
end
end
context 'whith multiple report types' do
before do
projects.each do |project|
create_vulnerabilities(2, project_guest, { report_type: :sast })
create_vulnerabilities(1, project_dev, { report_type: :dependency_scanning })
end
end
# FIXME: we only support SAST in group dashboard until https://gitlab.com/gitlab-org/gitlab-ee/issues/6240
# and https://gitlab.com/gitlab-org/gitlab-ee/issues/8481
it "returns a list of vulnerabilities but only for SAST report type" do
subject
expect(response).to have_gitlab_http_status(200)
expect(json_response).to be_an(Array)
expect(json_response.length).to eq 2
expect(json_response.map { |v| v['report_type'] }.uniq).to contain_exactly('sast')
expect(response).to match_response_schema('vulnerabilities/occurrence_list', dir: 'ee')
end
end
def create_vulnerabilities(count, project, options = {})
report_type = options[:report_type] || :sast
pipeline = create(:ci_pipeline, :success, project: project)
vulnerabilities = create_list(:vulnerabilities_occurrence, count, pipelines: [pipeline], project: project)
vulnerabilities = create_list(:vulnerabilities_occurrence, count, report_type: report_type, pipelines: [pipeline], project: project)
return vulnerabilities unless options[:with_feedback]
vulnerabilities.each do |occurrence|
create(:vulnerability_feedback, :sast, :dismissal,
create(:vulnerability_feedback, report_type, :dismissal,
pipeline: pipeline,
project: project_dev,
project_fingerprint: occurrence.project_fingerprint)
create(:vulnerability_feedback, :sast, :issue,
create(:vulnerability_feedback, report_type, :issue,
pipeline: pipeline,
issue: create(:issue, project: project),
project: project_dev,
......
......@@ -2,24 +2,37 @@
require 'spec_helper'
describe Gitlab::Ci::Parsers::Security::Sast do
describe Gitlab::Ci::Parsers::Security::Common do
describe '#parse!' do
let(:artifact) { create(:ee_ci_job_artifact, :sast) }
let(:project) { artifact.project }
let(:pipeline) { artifact.job.pipeline }
let(:report) { Gitlab::Ci::Reports::Security::Report.new(artifact.file_type) }
let(:sast) { described_class.new }
let(:parser) { described_class.new }
before do
artifact.each_blob do |blob|
sast.parse!(blob, report)
parser.parse!(blob, report)
end
end
it "parses all identifiers and occurrences" do
expect(report.occurrences.length).to eq(3)
expect(report.identifiers.length).to eq(4)
expect(report.scanners.length).to eq(3)
context 'sast report' do
let(:artifact) { create(:ee_ci_job_artifact, :sast) }
it "parses all identifiers and occurrences" do
expect(report.occurrences.length).to eq(3)
expect(report.identifiers.length).to eq(4)
expect(report.scanners.length).to eq(3)
end
end
context 'dependency_scanning report' do
let(:artifact) { create(:ee_ci_job_artifact, :dependency_scanning) }
it "parses all identifiers and occurrences" do
expect(report.occurrences.length).to eq(4)
expect(report.identifiers.length).to eq(7)
expect(report.scanners.length).to eq(2)
end
end
end
end
......@@ -155,7 +155,7 @@ describe Ci::Build do
subject { job.collect_security_reports!(security_reports) }
before do
stub_licensed_features(sast: true)
stub_licensed_features(sast: true, dependency_scanning: true)
end
context 'when build has a security report' do
......@@ -171,6 +171,20 @@ describe Ci::Build do
end
end
context 'when there are multiple report' do
before do
create(:ee_ci_job_artifact, :sast, job: job, project: job.project)
create(:ee_ci_job_artifact, :dependency_scanning, job: job, project: job.project)
end
it 'parses blobs and add the results to the reports' do
subject
expect(security_reports.get_report('sast').occurrences.size).to eq(3)
expect(security_reports.get_report('dependency_scanning').occurrences.size).to eq(4)
end
end
context 'when there is a corrupted sast report' do
before do
create(:ee_ci_job_artifact, :sast_with_corrupted_data, job: job, project: job.project)
......
......@@ -223,29 +223,32 @@ describe Ci::Pipeline do
subject { pipeline.security_reports }
before do
stub_licensed_features(sast: true)
stub_licensed_features(sast: true, dependency_scanning: true)
end
context 'when pipeline has multiple builds with security reports' do
let!(:build_sast_1) { create(:ci_build, :success, name: 'sast_1', pipeline: pipeline, project: project) }
let!(:build_sast_2) { create(:ci_build, :success, name: 'sast_2', pipeline: pipeline, project: project) }
let(:build_sast_1) { create(:ci_build, :success, name: 'sast_1', pipeline: pipeline, project: project) }
let(:build_sast_2) { create(:ci_build, :success, name: 'sast_2', pipeline: pipeline, project: project) }
let(:build_ds_1) { create(:ci_build, :success, name: 'ds_1', pipeline: pipeline, project: project) }
before do
create(:ee_ci_job_artifact, :sast, job: build_sast_1, project: project)
create(:ee_ci_job_artifact, :sast, job: build_sast_2, project: project)
create(:ee_ci_job_artifact, :dependency_scanning, job: build_ds_1, project: project)
end
it 'returns security reports with collected data grouped as expected' do
expect(subject.reports.keys).to eq(%w(sast))
expect(subject.reports.keys).to contain_exactly('sast', 'dependency_scanning')
expect(subject.get_report('sast').occurrences.size).to eq(6)
expect(subject.get_report('dependency_scanning').occurrences.size).to eq(4)
end
context 'when builds are retried' do
let!(:build_sast_1) { create(:ci_build, :retried, name: 'sast_1', pipeline: pipeline, project: project) }
let!(:build_sast_2) { create(:ci_build, :retried, name: 'sast_2', pipeline: pipeline, project: project) }
let(:build_sast_1) { create(:ci_build, :retried, name: 'sast_1', pipeline: pipeline, project: project) }
it 'does not take retried builds into account' do
expect(subject.reports).to eq({})
expect(subject.get_report('sast').occurrences.size).to eq(3)
expect(subject.get_report('dependency_scanning').occurrences.size).to eq(4)
end
end
end
......
......@@ -3,36 +3,45 @@
require 'spec_helper'
describe Security::StoreReportService, '#execute' do
let(:artifact) { create(:ee_ci_job_artifact, :sast) }
let(:artifact) { create(:ee_ci_job_artifact, report_type) }
let(:project) { artifact.project }
let(:pipeline) { artifact.job.pipeline }
let(:report) { pipeline.security_reports.get_report('sast') }
let(:report) { pipeline.security_reports.get_report(report_type.to_s) }
before do
stub_licensed_features(sast: true)
stub_licensed_features(sast: true, dependency_scanning: true)
end
subject { described_class.new(pipeline, report).execute }
context 'without existing data' do
it 'inserts all scanners' do
expect { subject }.to change { Vulnerabilities::Scanner.count }.by(3)
end
using RSpec::Parameterized::TableSyntax
it 'inserts all identifiers' do
expect { subject }.to change { Vulnerabilities::Identifier.count }.by(4)
where(:case_name, :report_type, :scanners, :identifiers, :occurrences, :occurrence_identifiers, :occurrence_pipelines) do
'with SAST report' | :sast | 3 | 4 | 3 | 5 | 3
'with Dependency Scanning report' | :dependency_scanning | 2 | 7 | 4 | 7 | 4
end
it 'inserts all occurrences' do
expect { subject }.to change { Vulnerabilities::Occurrence.count }.by(3)
end
with_them do
it 'inserts all scanners' do
expect { subject }.to change { Vulnerabilities::Scanner.count }.by(scanners)
end
it 'inserts all occurrence identifiers (join model)' do
expect { subject }.to change { Vulnerabilities::OccurrenceIdentifier.count }.by(5)
end
it 'inserts all identifiers' do
expect { subject }.to change { Vulnerabilities::Identifier.count }.by(identifiers)
end
it 'inserts all occurrences' do
expect { subject }.to change { Vulnerabilities::Occurrence.count }.by(occurrences)
end
it 'inserts all occurrence identifiers (join model)' do
expect { subject }.to change { Vulnerabilities::OccurrenceIdentifier.count }.by(occurrence_identifiers)
end
it 'inserts all occurrence pipelines (join model)' do
expect { subject }.to change { Vulnerabilities::OccurrencePipeline.count }.by(3)
it 'inserts all occurrence pipelines (join model)' do
expect { subject }.to change { Vulnerabilities::OccurrencePipeline.count }.by(occurrence_pipelines)
end
end
end
......@@ -42,7 +51,8 @@ describe Security::StoreReportService, '#execute' do
let!(:new_artifact) { create(:ee_ci_job_artifact, :sast, job: new_build) }
let(:new_build) { create(:ci_build, pipeline: new_pipeline) }
let(:new_pipeline) { create(:ci_pipeline, project: project) }
let(:new_report) { new_pipeline.security_reports.get_report('sast') }
let(:new_report) { new_pipeline.security_reports.get_report(report_type.to_s) }
let(:report_type) { :sast }
let!(:occurrence) do
create(:vulnerabilities_occurrence,
......@@ -75,6 +85,7 @@ describe Security::StoreReportService, '#execute' do
context 'with existing data from same pipeline' do
let!(:occurrence) { create(:vulnerabilities_occurrence, project: project, pipelines: [pipeline]) }
let(:report_type) { :sast }
it 'skips report' do
expect(subject).to eq({
......
......@@ -9,16 +9,19 @@ describe Security::StoreReportsService, '#execute' do
context 'when there are reports' do
before do
stub_licensed_features(sast: true)
stub_licensed_features(sast: true, dependency_scanning: true)
create(:ee_ci_build, :sast, pipeline: pipeline)
create(:ee_ci_build, :dependency_scanning, pipeline: pipeline)
end
it 'initializes a new StoreReportService and execute it' do
it 'initializes and execute a StoreReportService for each report' do
expect(Security::StoreReportService).to receive(:new)
.with(pipeline, instance_of(::Gitlab::Ci::Reports::Security::Report)).and_call_original
expect_any_instance_of(Security::StoreReportService).to receive(:execute)
.once.and_call_original
.twice.with(pipeline, instance_of(::Gitlab::Ci::Reports::Security::Report))
.and_wrap_original do |method, *original_args|
method.call(*original_args).tap do |store_service|
expect(store_service).to receive(:execute).once.and_call_original
end
end
subject
end
......
[
{
"priority": "Unknown",
"file": "pom.xml",
"cve": "CVE-2012-4387",
"url": "http://struts.apache.org/docs/s2-011.html",
"message": "Long parameter name DoS for org.apache.struts/struts2-core",
"tools": [
"gemnasium"
"category": "dependency_scanning",
"name": "io.netty/netty - CVE-2014-3488",
"message": "DoS by CPU exhaustion when using malicious SSL packets",
"cve": "app/pom.xml:io.netty/netty@3.9.1.Final:CVE-2014-3488",
"severity": "Unknown",
"solution": "Upgrade to the latest version",
"scanner": {
"id": "gemnasium",
"name": "Gemnasium"
},
"location": {
"file": "app/pom.xml"
},
"identifiers": [
{
"type": "gemnasium",
"name": "Gemnasium-d1bf36d9-9f07-46cd-9cfc-8675338ada8f",
"value": "d1bf36d9-9f07-46cd-9cfc-8675338ada8f",
"url": "https://deps.sec.gitlab.com/packages/maven/io.netty/netty/versions/3.9.1.Final/advisories"
},
{
"type": "cve",
"name": "CVE-2014-3488",
"value": "CVE-2014-3488",
"url": "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2014-3488"
}
],
"links": [
{
"url": "https://bugzilla.redhat.com/CVE-2014-3488"
},
{
"url": "http://netty.io/news/2014/06/11/3.html"
},
{
"url": "https://github.com/netty/netty/issues/2562"
}
],
"priority": "Unknown",
"file": "app/pom.xml",
"url": "https://bugzilla.redhat.com/CVE-2014-3488",
"tool": "gemnasium"
},
{
"priority": "Unknown",
"file": "pom.xml",
"cve": "CVE-2013-1966",
"url": "http://struts.apache.org/docs/s2-014.html",
"message": "Remote command execution due to flaw in the includeParams attribute of URL and Anchor tags for org.apache.struts/struts2-core",
"tools": [
"gemnasium"
"category": "dependency_scanning",
"name": "Django - CVE-2017-12794",
"message": "Possible XSS in traceback section of technical 500 debug page",
"cve": "app/requirements.txt:Django@1.11.3:CVE-2017-12794",
"severity": "Unknown",
"solution": "Upgrade to latest version or apply patch.",
"scanner": {
"id": "gemnasium",
"name": "Gemnasium"
},
"location": {
"file": "app/requirements.txt"
},
"identifiers": [
{
"type": "gemnasium",
"name": "Gemnasium-6162a015-8635-4a15-8d7c-dc9321db366f",
"value": "6162a015-8635-4a15-8d7c-dc9321db366f",
"url": "https://deps.sec.gitlab.com/packages/pypi/Django/versions/1.11.3/advisories"
},
{
"type": "cve",
"name": "CVE-2017-12794",
"value": "CVE-2017-12794",
"url": "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-12794"
}
],
"links": [
{
"url": "https://www.djangoproject.com/weblog/2017/sep/05/security-releases/"
}
],
"priority": "Unknown",
"file": "app/requirements.txt",
"url": "https://www.djangoproject.com/weblog/2017/sep/05/security-releases/",
"tool": "gemnasium"
},
{
"priority": "Unknown",
"file": "pom.xml",
"cve": "CVE-2013-2115",
"url": "http://struts.apache.org/docs/s2-014.html",
"message": "Remote command execution due to flaw in the includeParams attribute of URL and Anchor tags for org.apache.struts/struts2-core",
"tools": [
"gemnasium"
"category": "dependency_scanning",
"name": "nokogiri - USN-3424-1",
"message": "Vulnerabilities in libxml2",
"cve": "rails/Gemfile.lock:nokogiri@1.8.0:USN-3424-1",
"severity": "Unknown",
"solution": "Upgrade to latest version.",
"scanner": {
"id": "gemnasium",
"name": "Gemnasium"
},
"location": {
"file": "rails/Gemfile.lock"
},
"identifiers": [
{
"type": "gemnasium",
"name": "Gemnasium-06565b64-486d-4326-b906-890d9915804d",
"value": "06565b64-486d-4326-b906-890d9915804d",
"url": "https://deps.sec.gitlab.com/packages/gem/nokogiri/versions/1.8.0/advisories"
},
{
"type": "usn",
"name": "USN-3424-1",
"value": "USN-3424-1",
"url": "https://usn.ubuntu.com/3424-1/"
}
],
"links": [
{
"url": "https://github.com/sparklemotion/nokogiri/issues/1673"
}
],
"priority": "Unknown",
"file": "rails/Gemfile.lock",
"url": "https://github.com/sparklemotion/nokogiri/issues/1673",
"tool": "gemnasium"
},
{
"priority": "Unknown",
"file": "pom.xml",
"cve": "CVE-2013-2134",
"url": "http://struts.apache.org/docs/s2-015.html",
"message": "Arbitrary OGNL code execution via unsanitized wildcard matching for org.apache.struts/struts2-core",
"tools": [
"gemnasium"
"category": "dependency_scanning",
"name": "ffi - CVE-2018-1000201",
"message": "ruby-ffi DDL loading issue on Windows OS",
"cve": "ffi:1.9.18:CVE-2018-1000201",
"severity": "High",
"solution": "upgrade to \u003e= 1.9.24",
"scanner": {
"id": "bundler_audit",
"name": "bundler-audit"
},
"location": {
"file": "sast-sample-rails/Gemfile.lock"
},
"identifiers": [
{
"type": "cve",
"name": "CVE-2018-1000201",
"value": "CVE-2018-1000201",
"url": "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-1000201"
}
],
"tool": "gemnasium"
"links": [
{
"url": "https://github.com/ffi/ffi/releases/tag/1.9.24"
}
],
"priority": "High",
"file": "sast-sample-rails/Gemfile.lock",
"url": "https://github.com/ffi/ffi/releases/tag/1.9.24",
"tool": "bundler_audit"
}
]
[
{
"priority": "Unknown",
"file": "pom.xml",
"cve": "CVE-2012-4386",
"url": "http://struts.apache.org/docs/s2-010.html",
"message": "CSRF protection bypass for org.apache.struts/struts2-core",
"tools": [
"gemnasium"
"category": "dependency_scanning",
"name": "io.netty/netty - CVE-2014-3488",
"message": "DoS by CPU exhaustion when using malicious SSL packets",
"cve": "app/pom.xml:io.netty/netty@3.9.1.Final:CVE-2014-3488",
"severity": "Unknown",
"solution": "Upgrade to the latest version",
"scanner": {
"id": "gemnasium",
"name": "Gemnasium"
},
"location": {
"file": "app/pom.xml"
},
"identifiers": [
{
"type": "gemnasium",
"name": "Gemnasium-d1bf36d9-9f07-46cd-9cfc-8675338ada8f",
"value": "d1bf36d9-9f07-46cd-9cfc-8675338ada8f",
"url": "https://deps.sec.gitlab.com/packages/maven/io.netty/netty/versions/3.9.1.Final/advisories"
},
{
"type": "cve",
"name": "CVE-2014-3488",
"value": "CVE-2014-3488",
"url": "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2014-3488"
}
],
"links": [
{
"url": "https://bugzilla.redhat.com/CVE-2014-3488"
},
{
"url": "http://netty.io/news/2014/06/11/3.html"
},
{
"url": "https://github.com/netty/netty/issues/2562"
}
],
"priority": "Unknown",
"file": "app/pom.xml",
"url": "https://bugzilla.redhat.com/CVE-2014-3488",
"tool": "gemnasium"
},
{
"priority": "Unknown",
"file": "pom.xml",
"cve": "CVE-2012-4387",
"url": "http://struts.apache.org/docs/s2-011.html",
"message": "Long parameter name DoS for org.apache.struts/struts2-core",
"tools": [
"gemnasium"
"category": "dependency_scanning",
"name": "Django - CVE-2017-12794",
"message": "Possible XSS in traceback section of technical 500 debug page",
"cve": "app/requirements.txt:Django@1.11.3:CVE-2017-12794",
"severity": "Unknown",
"solution": "Upgrade to latest version or apply patch.",
"scanner": {
"id": "gemnasium",
"name": "Gemnasium"
},
"location": {
"file": "app/requirements.txt"
},
"identifiers": [
{
"type": "gemnasium",
"name": "Gemnasium-6162a015-8635-4a15-8d7c-dc9321db366f",
"value": "6162a015-8635-4a15-8d7c-dc9321db366f",
"url": "https://deps.sec.gitlab.com/packages/pypi/Django/versions/1.11.3/advisories"
},
{
"type": "cve",
"name": "CVE-2017-12794",
"value": "CVE-2017-12794",
"url": "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-12794"
}
],
"links": [
{
"url": "https://www.djangoproject.com/weblog/2017/sep/05/security-releases/"
}
],
"priority": "Unknown",
"file": "app/requirements.txt",
"url": "https://www.djangoproject.com/weblog/2017/sep/05/security-releases/",
"tool": "gemnasium"
},
{
"priority": "Unknown",
"file": "pom.xml",
"cve": "CVE-2013-1966",
"url": "http://struts.apache.org/docs/s2-014.html",
"message": "Remote command execution due to flaw in the includeParams attribute of URL and Anchor tags for org.apache.struts/struts2-core",
"tools": [
"gemnasium"
"category": "dependency_scanning",
"name": "nokogiri - USN-3424-1",
"message": "Vulnerabilities in libxml2",
"cve": "rails/Gemfile.lock:nokogiri@1.8.0:USN-3424-1",
"severity": "Unknown",
"solution": "Upgrade to latest version.",
"scanner": {
"id": "gemnasium",
"name": "Gemnasium"
},
"location": {
"file": "rails/Gemfile.lock"
},
"identifiers": [
{
"type": "gemnasium",
"name": "Gemnasium-06565b64-486d-4326-b906-890d9915804d",
"value": "06565b64-486d-4326-b906-890d9915804d",
"url": "https://deps.sec.gitlab.com/packages/gem/nokogiri/versions/1.8.0/advisories"
},
{
"type": "usn",
"name": "USN-3424-1",
"value": "USN-3424-1",
"url": "https://usn.ubuntu.com/3424-1/"
}
],
"links": [
{
"url": "https://github.com/sparklemotion/nokogiri/issues/1673"
}
],
"priority": "Unknown",
"file": "rails/Gemfile.lock",
"url": "https://github.com/sparklemotion/nokogiri/issues/1673",
"tool": "gemnasium"
},
{
"category": "dependency_scanning",
"name": "ffi - CVE-2018-1000201",
"message": "ruby-ffi DDL loading issue on Windows OS",
"cve": "ffi:1.9.18:CVE-2018-1000201",
"severity": "High",
"solution": "upgrade to \u003e= 1.9.24",
"scanner": {
"id": "bundler_audit",
"name": "bundler-audit"
},
"location": {
"file": "sast-sample-rails/Gemfile.lock"
},
"identifiers": [
{
"type": "cve",
"name": "CVE-2018-1000201",
"value": "CVE-2018-1000201",
"url": "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-1000201"
}
],
"links": [
{
"url": "https://github.com/ffi/ffi/releases/tag/1.9.24"
}
],
"priority": "High",
"file": "sast-sample-rails/Gemfile.lock",
"url": "https://github.com/ffi/ffi/releases/tag/1.9.24",
"tool": "bundler_audit"
}
]
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment