Merge branch '7061_store_container_scanning_reports_in_db_be' into 'master'

Store container scanning results in the database

Closes #7061

See merge request gitlab-org/gitlab-ee!8797

ee/app/models/ee/ci/build.rb

@@ -11,7 +11,8 @@ module EE
 
       LICENSED_PARSER_FEATURES = {
         sast: :sast,
-        dependency_scanning: :dependency_scanning
+        dependency_scanning: :dependency_scanning,
+        container_scanning: :container_scanning
       }.with_indifferent_access.freeze
 
       prepended do

ee/changelogs/unreleased/7061_store_container_scanning_reports_in_db_be.yml

+---
+title: Store container scanning CI jobs results into the database
+merge_request: 8797
+author:
+type: added

ee/lib/ee/gitlab/ci/parsers.rb

@@ -11,6 +11,7 @@ module EE
             super.merge({
                 license_management: ::Gitlab::Ci::Parsers::LicenseManagement::LicenseManagement,
                 dependency_scanning: ::Gitlab::Ci::Parsers::Security::DependencyScanning,
+                container_scanning: ::Gitlab::Ci::Parsers::Security::ContainerScanning,
                 sast: ::Gitlab::Ci::Parsers::Security::Sast
             })
           end

ee/lib/gitlab/ci/parsers/security/container_scanning.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Parsers
+      module Security
+        class ContainerScanning < Common
+          extend ::Gitlab::Utils::Override
+
+          DEPRECATED_REPORT_VERSION = "1.3".freeze
+
+          override :parse_report
+          def parse_report(json_data)
+            report = super
+
+            if report.is_a?(Array)
+              report = {
+                  "version" => DEPRECATED_REPORT_VERSION,
+                  "vulnerabilities" => report
+              }
+            end
+
+            report
+          end
+
+          def parse!(json_data, report)
+            vulnerabilities = format_report(JSON.parse!(json_data))
+
+            vulnerabilities.each do |vulnerability|
+              create_vulnerability(report, vulnerability, DEPRECATED_REPORT_VERSION)
+            end
+          rescue JSON::ParserError
+            raise SecurityReportParserError, 'JSON parsing failed'
+          rescue
+            raise SecurityReportParserError, "#{report.type} security report parsing failed"
+          end
+
+          private
+
+          # Transforms the Clair JSON report into the expected format
+          def format_report(data)
+            vulnerabilities = data['vulnerabilities']
+            results = []
+
+            vulnerabilities.each do |vulnerability|
+              results.append(format_vulnerability(vulnerability))
+            end
+
+            results
+          end
+
+          def format_vulnerability(vulnerability)
+            {
+                'category' => 'container_scanning',
+                'message' => name(vulnerability),
+                'description' => vulnerability['description'],
+                'cve' => vulnerability['vulnerability'],
+                'severity' => translate_severity(vulnerability['severity']),
+                'solution' => solution(vulnerability),
+                'confidence' => 'Medium',
+                'scanner' => { 'id' => 'clair', 'name' => 'Clair' },
+                'identifiers' => [
+                  {
+                      'type' => 'cve',
+                      'name' => vulnerability['vulnerability'],
+                      'value' => vulnerability['vulnerability'],
+                      'url' => vulnerability['link']
+                  }
+                ],
+                'links' => [{ 'url' => vulnerability['link'] }],
+                'priority' => 'Unknown',
+                'url' => vulnerability['link'],
+                'tool' => 'clair'
+            }
+          end
+
+          def translate_severity(severity)
+            case severity
+            when 'Negligible'
+              'low'
+            when 'Unknown', 'Low', 'Medium', 'High', 'Critical'
+              severity.downcase
+            when 'Defcon1'
+              'critical'
+            else
+              safe_severity = ERB::Util.html_escape(severity)
+              raise SecurityReportParserError, "Unknown severity in container scanning report: #{safe_severity}"
+            end
+          end
+
+          def solution(vulnerability)
+            if vulnerability['fixedby'].present?
+              "Upgrade to version #{vulnerability['fixedby']}"
+            end
+          end
+
+          def name(vulnerability)
+            # Name is package name and the CVE is is affected by.
+            "#{vulnerability['featurename']} - #{vulnerability['vulnerability']}"
+          end
+
+          def metadata_version(vulnerability)
+            '1.3'
+          end
+
+          def generate_location_fingerprint(location)
+            # Location is irrelevant for Clair vulnerabilities.
+            # SHA1 value for 'clair'
+            'cb750fa5a7a31c527d5c15388a432c4ba3338457'
+          end
+        end
+      end
+    end
+  end
+end

ee/spec/lib/gitlab/ci/parsers/security/container_scanning_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Ci::Parsers::Security::ContainerScanning do
+  let(:parser) { described_class.new }
+
+  let(:zap_vulnerabilities) do
+    JSON.parse!(
+      File.read(
+        Rails.root.join('spec/fixtures/security-reports/master/gl-container-scanning-report.json')
+      )
+    )['vulnerabilities']
+  end
+
+  describe '#parse!' do
+    let(:project) { artifact.project }
+    let(:pipeline) { artifact.job.pipeline }
+    let(:artifact) { create(:ee_ci_job_artifact, :container_scanning) }
+    let(:report) { Gitlab::Ci::Reports::Security::Report.new(artifact.file_type) }
+
+    before do
+      artifact.each_blob do |blob|
+        parser.parse!(blob, report)
+      end
+    end
+
+    it "parses all identifiers and occurrences" do
+      expect(report.occurrences.length).to eq(8)
+      expect(report.identifiers.length).to eq(8)
+      expect(report.scanners.length).to eq(1)
+    end
+
+    it "generates expected location fingerprint" do
+      expect(report.occurrences.first[:location_fingerprint]).to eq('cb750fa5a7a31c527d5c15388a432c4ba3338457')
+    end
+
+    it "generates expected metadata_version" do
+      expect(report.occurrences.first[:metadata_version]).to eq('1.3')
+    end
+  end
+
+  describe '#format_vulnerability' do
+    it 'format ZAP vulnerability into the 1.3 format' do
+      expect(parser.send(:format_vulnerability, zap_vulnerabilities[0])).to eq( {
+        'category' => 'container_scanning',
+        'message' => 'glibc - CVE-2017-18269',
+        'confidence' => 'Medium',
+        'cve' => 'CVE-2017-18269',
+        'identifiers' => [
+          {
+            'type' => 'cve',
+            'name' => 'CVE-2017-18269',
+            'value' => 'CVE-2017-18269',
+            'url' => 'https://security-tracker.debian.org/tracker/CVE-2017-18269'
+          }
+        ],
+        'links' => [{ 'url' => 'https://security-tracker.debian.org/tracker/CVE-2017-18269' }],
+        'description' => 'SSE2-optimized memmove implementation problem.',
+        'priority' => 'Unknown',
+        'scanner' => { 'id' => 'clair', 'name' => 'Clair' },
+        'severity' => 'critical',
+        'solution' => 'Upgrade to version 2.24-11+deb9u4',
+        'tool' => 'clair',
+        'url' => 'https://security-tracker.debian.org/tracker/CVE-2017-18269'
+      } )
+    end
+  end
+
+  describe '#translate_severity' do
+    context 'with recognised values' do
+      using RSpec::Parameterized::TableSyntax
+
+      where(:severity, :expected) do
+        'Unknown'    | 'unknown'
+        'Negligible' | 'low'
+        'Low'        | 'low'
+        'Medium'     | 'medium'
+        'High'       | 'high'
+        'Critical'   | 'critical'
+        'Defcon1'    | 'critical'
+      end
+
+      with_them do
+        it "translate severity from Clair" do
+          expect(parser.send(:translate_severity, severity)).to eq(expected)
+        end
+      end
+    end
+
+    context 'with a wrong value' do
+      it 'throws an exception' do
+        expect { parser.send(:translate_severity, 'abcd<efg>') }.to raise_error(
+          ::Gitlab::Ci::Parsers::Security::Common::SecurityReportParserError,
+          'Unknown severity in container scanning report: abcd&lt;efg&gt;'
+        )
+      end
+    end
+  end
+
+  describe '#solution' do
+    context 'without a fixedby value' do
+      it 'returns nil' do
+        expect(parser.send(:solution, zap_vulnerabilities[1])).to be_nil
+      end
+    end
+
+    context 'with a fixedby value' do
+      it 'returns a solution' do
+        expect(parser.send(:solution, zap_vulnerabilities[0])).to eq('Upgrade to version 2.24-11+deb9u4')
+      end
+    end
+  end
+end

ee/spec/models/ci/build_spec.rb

+# frozen_string_literal: true
+
 require 'spec_helper'
 
 describe Ci::Build do
@@ -155,7 +157,7 @@ describe Ci::Build do
     subject { job.collect_security_reports!(security_reports) }
 
     before do
-      stub_licensed_features(sast: true, dependency_scanning: true)
+      stub_licensed_features(sast: true, dependency_scanning: true, container_scanning: true)
     end
 
     context 'when build has a security report' do
@@ -175,6 +177,7 @@ describe Ci::Build do
         before do
           create(:ee_ci_job_artifact, :sast, job: job, project: job.project)
           create(:ee_ci_job_artifact, :dependency_scanning, job: job, project: job.project)
+          create(:ee_ci_job_artifact, :container_scanning, job: job, project: job.project)
         end
 
         it 'parses blobs and add the results to the reports' do
@@ -182,6 +185,7 @@ describe Ci::Build do
 
           expect(security_reports.get_report('sast').occurrences.size).to eq(33)
           expect(security_reports.get_report('dependency_scanning').occurrences.size).to eq(4)
+          expect(security_reports.get_report('container_scanning').occurrences.size).to eq(8)
         end
       end
 

ee/spec/models/ci/pipeline_spec.rb

+# frozen_string_literal: true
+
 require 'spec_helper'
 
 describe Ci::Pipeline do
@@ -223,24 +225,27 @@ describe Ci::Pipeline do
     subject { pipeline.security_reports }
 
     before do
-      stub_licensed_features(sast: true, dependency_scanning: true)
+      stub_licensed_features(sast: true, dependency_scanning: true, container_scanning: true)
     end
 
     context 'when pipeline has multiple builds with security reports' do
       let(:build_sast_1) { create(:ci_build, :success, name: 'sast_1', pipeline: pipeline, project: project) }
       let(:build_sast_2) { create(:ci_build, :success, name: 'sast_2', pipeline: pipeline, project: project) }
       let(:build_ds_1) { create(:ci_build, :success, name: 'ds_1', pipeline: pipeline, project: project) }
+      let(:build_cs_1) { create(:ci_build, :success, name: 'cs_1', pipeline: pipeline, project: project) }
 
       before do
         create(:ee_ci_job_artifact, :sast, job: build_sast_1, project: project)
         create(:ee_ci_job_artifact, :sast, job: build_sast_2, project: project)
         create(:ee_ci_job_artifact, :dependency_scanning, job: build_ds_1, project: project)
+        create(:ee_ci_job_artifact, :container_scanning, job: build_cs_1, project: project)
       end
 
       it 'returns security reports with collected data grouped as expected' do
-        expect(subject.reports.keys).to contain_exactly('sast', 'dependency_scanning')
+        expect(subject.reports.keys).to contain_exactly('sast', 'dependency_scanning', 'container_scanning')
         expect(subject.get_report('sast').occurrences.size).to eq(66)
         expect(subject.get_report('dependency_scanning').occurrences.size).to eq(4)
+        expect(subject.get_report('container_scanning').occurrences.size).to eq(8)
       end
 
       context 'when builds are retried' do
@@ -249,6 +254,7 @@ describe Ci::Pipeline do
         it 'does not take retried builds into account' do
           expect(subject.get_report('sast').occurrences.size).to eq(33)
           expect(subject.get_report('dependency_scanning').occurrences.size).to eq(4)
+          expect(subject.get_report('container_scanning').occurrences.size).to eq(8)
         end
       end
     end

ee/spec/services/security/store_report_service_spec.rb

@@ -9,7 +9,7 @@ describe Security::StoreReportService, '#execute' do
   let(:report) { pipeline.security_reports.get_report(report_type.to_s) }
 
   before do
-    stub_licensed_features(sast: true, dependency_scanning: true)
+    stub_licensed_features(sast: true, dependency_scanning: true, container_scanning: true)
   end
 
   subject { described_class.new(pipeline, report).execute }
@@ -19,7 +19,8 @@ describe Security::StoreReportService, '#execute' do
 
     where(:case_name, :report_type, :scanners, :identifiers, :occurrences, :occurrence_identifiers, :occurrence_pipelines) do
       'with SAST report'                | :sast                | 3 | 14 | 33 | 35 | 33
-      'with Dependency Scanning report' | :dependency_scanning | 2 | 7 | 4 | 7 | 4
+      'with Dependency Scanning report' | :dependency_scanning | 2 | 7  | 4  | 7  | 4
+      'with Container Scanning report'  | :container_scanning  | 1 | 8  | 8  | 8  | 8
     end
 
     with_them do

ee/spec/services/security/store_reports_service_spec.rb

@@ -9,14 +9,15 @@ describe Security::StoreReportsService, '#execute' do
 
   context 'when there are reports' do
     before do
-      stub_licensed_features(sast: true, dependency_scanning: true)
+      stub_licensed_features(sast: true, dependency_scanning: true, container_scanning: true)
       create(:ee_ci_build, :sast, pipeline: pipeline)
       create(:ee_ci_build, :dependency_scanning, pipeline: pipeline)
+      create(:ee_ci_build, :container_scanning, pipeline: pipeline)
     end
 
     it 'initializes and execute a StoreReportService for each report' do
       expect(Security::StoreReportService).to receive(:new)
-        .twice.with(pipeline, instance_of(::Gitlab::Ci::Reports::Security::Report))
+        .exactly(3).times.with(pipeline, instance_of(::Gitlab::Ci::Reports::Security::Report))
         .and_wrap_original do |method, *original_args|
           method.call(*original_args).tap do |store_service|
             expect(store_service).to receive(:execute).once.and_call_original

spec/fixtures/security-reports/master/gl-container-scanning-report.json

 {
-    "image": "registry.gitlab.com/bikebilly/auto-devops-10-6/feature-branch:e7315ba964febb11bac8f5cd6ec433db8a3a1583",
+    "image": "registry.gitlab.com/groulot/container-scanning-test/master:5f21de6956aee99ddb68ae49498662d9872f50ff",
     "unapproved": [
-        "CVE-2017-15651"
+        "CVE-2017-18018",
+        "CVE-2016-2781",
+        "CVE-2017-12424",
+        "CVE-2007-5686",
+        "CVE-2013-4235"
     ],
     "vulnerabilities": [
         {
-            "featurename": "musl",
-            "featureversion": "1.1.14-r15",
-            "vulnerability": "CVE-2017-15651",
-            "namespace": "alpine:v3.4",
-            "description": "",
-            "link": "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-15651",
+            "featurename": "glibc",
+            "featureversion": "2.24-11+deb9u3",
+            "vulnerability": "CVE-2017-18269",
+            "namespace": "debian:9",
+            "description": "SSE2-optimized memmove implementation problem.",
+            "link": "https://security-tracker.debian.org/tracker/CVE-2017-18269",
+            "severity": "Defcon1",
+            "fixedby": "2.24-11+deb9u4"
+        },
+        {
+            "featurename": "glibc",
+            "featureversion": "2.24-11+deb9u3",
+            "vulnerability": "CVE-2017-16997",
+            "namespace": "debian:9",
+            "description": "elf/dl-load.c in the GNU C Library (aka glibc or libc6) 2.19 through 2.26 mishandles RPATH and RUNPATH containing $ORIGIN for a privileged (setuid or AT_SECURE) program, which allows local users to gain privileges via a Trojan horse library in the current working directory, related to the fillin_rpath and decompose_rpath functions. This is associated with misinterpretion of an empty RPATH/RUNPATH token as the \"./\" directory. NOTE: this configuration of RPATH/RUNPATH for a privileged program is apparently very uncommon; most likely, no such program is shipped with any common Linux distribution.",
+            "link": "https://security-tracker.debian.org/tracker/CVE-2017-16997",
+            "severity": "Critical",
+            "fixedby": ""
+        },
+        {
+            "featurename": "glibc",
+            "featureversion": "2.24-11+deb9u3",
+            "vulnerability": "CVE-2018-1000001",
+            "namespace": "debian:9",
+            "description": "In glibc 2.26 and earlier there is confusion in the usage of getcwd() by realpath() which can be used to write before the destination buffer leading to a buffer underflow and potential code execution.",
+            "link": "https://security-tracker.debian.org/tracker/CVE-2018-1000001",
+            "severity": "High",
+            "fixedby": ""
+        },
+        {
+            "featurename": "glibc",
+            "featureversion": "2.24-11+deb9u3",
+            "vulnerability": "CVE-2016-10228",
+            "namespace": "debian:9",
+            "description": "The iconv program in the GNU C Library (aka glibc or libc6) 2.25 and earlier, when invoked with the -c option, enters an infinite loop when processing invalid multi-byte input sequences, leading to a denial of service.",
+            "link": "https://security-tracker.debian.org/tracker/CVE-2016-10228",
             "severity": "Medium",
-            "fixedby": "1.1.14-r16"
+            "fixedby": ""
+        },
+        {
+            "featurename": "elfutils",
+            "featureversion": "0.168-1",
+            "vulnerability": "CVE-2018-18520",
+            "namespace": "debian:9",
+            "description": "An Invalid Memory Address Dereference exists in the function elf_end in libelf in elfutils through v0.174. Although eu-size is intended to support ar files inside ar files, handle_ar in size.c closes the outer ar file before handling all inner entries. The vulnerability allows attackers to cause a denial of service (application crash) with a crafted ELF file.",
+            "link": "https://security-tracker.debian.org/tracker/CVE-2018-18520",
+            "severity": "Low",
+            "fixedby": ""
+        },
+        {
+            "featurename": "glibc",
+            "featureversion": "2.24-11+deb9u3",
+            "vulnerability": "CVE-2010-4052",
+            "namespace": "debian:9",
+            "description": "Stack consumption vulnerability in the regcomp implementation in the GNU C Library (aka glibc or libc6) through 2.11.3, and 2.12.x through 2.12.2, allows context-dependent attackers to cause a denial of service (resource exhaustion) via a regular expression containing adjacent repetition operators, as demonstrated by a {10,}{10,}{10,}{10,} sequence in the proftpd.gnu.c exploit for ProFTPD.",
+            "link": "https://security-tracker.debian.org/tracker/CVE-2010-4052",
+            "severity": "Negligible",
+            "fixedby": ""
+        },
+        {
+            "featurename": "nettle",
+            "featureversion": "3.3-1",
+            "vulnerability": "CVE-2018-16869",
+            "namespace": "debian:9",
+            "description": "A Bleichenbacher type side-channel based padding oracle attack was found in the way nettle handles endian conversion of RSA decrypted PKCS#1 v1.5 data. An attacker who is able to run a process on the same physical core as the victim process, could use this flaw extract plaintext or in some cases downgrade any TLS connections to a vulnerable server.",
+            "link": "https://security-tracker.debian.org/tracker/CVE-2018-16869",
+            "severity": "Unknown",
+            "fixedby": ""
+        },
+        {
+            "featurename": "perl",
+            "featureversion": "5.24.1-3+deb9u4",
+            "vulnerability": "CVE-2018-18311",
+            "namespace": "debian:9",
+            "description": "Perl before 5.26.3 and 5.28.x before 5.28.1 has a buffer overflow via a crafted regular expression that triggers invalid write operations.",
+            "link": "https://security-tracker.debian.org/tracker/CVE-2018-18311",
+            "severity": "Unknown",
+            "fixedby": "5.24.1-3+deb9u5"
         }
     ]
-}
+}
\ No newline at end of file