Merge branch '7062-format-dast-output' into 'master'

Store DAST scan results into the database

Closes #7062

See merge request gitlab-org/gitlab-ee!9192

ee/app/models/ee/ci/build.rb

@@ -12,7 +12,8 @@ module EE
       LICENSED_PARSER_FEATURES = {
         sast: :sast,
         dependency_scanning: :dependency_scanning,
-        container_scanning: :container_scanning
+        container_scanning: :container_scanning,
+        dast: :dast
       }.with_indifferent_access.freeze
 
       prepended do
@@ -63,6 +64,9 @@ module EE
           next if file_type == "container_scanning" &&
               ::Feature.disabled?(:parse_container_scanning_reports, default_enabled: false)
 
+          next if file_type == "dast" &&
+              ::Feature.disabled?(:parse_dast_reports, default_enabled: false)
+
           security_reports.get_report(file_type).tap do |security_report|
             begin
               next unless project.feature_available?(LICENSED_PARSER_FEATURES.fetch(file_type))

ee/changelogs/unreleased/7062-format-dast-output.yml

+---
+title: Store DAST scan results in the database
+merge_request: 9192
+author:
+type: added

ee/lib/ee/gitlab/ci/parsers.rb

@@ -12,6 +12,7 @@ module EE
                 license_management: ::Gitlab::Ci::Parsers::LicenseManagement::LicenseManagement,
                 dependency_scanning: ::Gitlab::Ci::Parsers::Security::DependencyScanning,
                 container_scanning: ::Gitlab::Ci::Parsers::Security::ContainerScanning,
+                dast: ::Gitlab::Ci::Parsers::Security::Dast,
                 sast: ::Gitlab::Ci::Parsers::Security::Sast
             })
           end

ee/lib/gitlab/ci/parsers/security/common.rb

@@ -89,6 +89,10 @@ module Gitlab
           def generate_identifier_fingerprint(identifier)
             Digest::SHA1.hexdigest("#{identifier['type']}:#{identifier['value']}")
           end
+
+          def generate_location_fingerprint(location)
+            raise NotImplementedError
+          end
         end
       end
     end

ee/lib/gitlab/ci/parsers/security/dast.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Parsers
+      module Security
+        class Dast < Common
+          FORMAT_VERSION = '2.0'.freeze
+
+          protected
+
+          def parse_report(json_data)
+            report = super
+
+            format_report(report)
+          end
+
+          private
+
+          def format_report(data)
+            {
+              'vulnerabilities' => extract_vulnerabilities_from(data),
+              'version' => FORMAT_VERSION
+            }
+          end
+
+          def extract_vulnerabilities_from(data)
+            site = data['site']
+            results = []
+
+            if site
+              host = site['@name']
+
+              site['alerts'].each do |vulnerability|
+                results += flatten_vulnerabilities(vulnerability, host)
+              end
+            end
+
+            results
+          end
+
+          def flatten_vulnerabilities(vulnerability, host)
+            common_vulnerability = format_vulnerability(vulnerability)
+
+            vulnerability['instances'].map do |instance|
+              common_vulnerability.merge('location' => location(instance, host))
+            end
+          end
+
+          def format_vulnerability(vulnerability)
+            {
+              'category' => 'dast',
+              'message' => vulnerability['name'],
+              'description' => sanitize(vulnerability['desc']),
+              'cve' => vulnerability['pluginid'],
+              'severity' => severity(vulnerability['riskcode']),
+              'solution' => sanitize(vulnerability['solution']),
+              'confidence' => confidence(vulnerability['confidence']),
+              'scanner' => { 'id' => 'zaproxy', 'name' => 'ZAProxy' },
+              'identifiers' => [
+                {
+                  'type' => 'ZAProxy_PluginId',
+                  'name' => vulnerability['name'],
+                  'value' => vulnerability['pluginid'],
+                  'url' => "https://github.com/zaproxy/zaproxy/blob/w2019-01-14/docs/scanners.md"
+                },
+                {
+                  'type' => 'CWE',
+                  'name' => "CWE-#{vulnerability['cweid']}",
+                  'value' => vulnerability['cweid'],
+                  'url' => "https://cwe.mitre.org/data/definitions/#{vulnerability['cweid']}.html"
+                },
+                {
+                  'type' => 'WASC',
+                  'name' => "WASC-#{vulnerability['wascid']}",
+                  'value' => vulnerability['wascid'],
+                  'url' => "http://projects.webappsec.org/w/page/13246974/Threat%20Classification%20Reference%20Grid"
+                }
+              ],
+              'links' => links(vulnerability['reference'])
+            }
+          end
+
+          def generate_location_fingerprint(location)
+            Digest::SHA1.hexdigest("#{location['param']} #{location['method']} #{location['path']}")
+          end
+
+          # https://github.com/zaproxy/zaproxy/blob/cfb44f7e29f490d95b03830d90aadaca51a72a6a/src/scripts/templates/passive/Passive%20default%20template.js#L25
+          # NOTE: ZAProxy levels: 0: info, 1: low, 2: medium, 3: high
+          def severity(value)
+            case Integer(value)
+            when 0
+              'ignore'
+            when 1
+              'low'
+            when 2
+              'medium'
+            when 3
+              'high'
+            else
+              'unknown'
+            end
+          rescue ArgumentError
+            'unknown'
+          end
+
+          # NOTE: ZAProxy levels: 0: falsePositive, 1: low, 2: medium, 3: high, 4: confirmed
+          def confidence(value)
+            case Integer(value)
+            when 0
+              'ignore'
+            when 1
+              'low'
+            when 2
+              'medium'
+            when 3
+              'high'
+            when 4
+              'critical'
+            else
+              'unknown'
+            end
+          rescue ArgumentError
+            'unknown'
+          end
+
+          def links(reference)
+            urls_from(reference).each_with_object([]) do |url, links|
+              next if url.blank?
+
+              links << { 'url' => url }
+            end
+          end
+
+          def urls_from(reference)
+            tags = reference.lines('</p>')
+            tags.map { |tag| sanitize(tag) }
+          end
+
+          def location(instance, hostname)
+            {
+              'param' => instance['param'],
+              'method' => instance['method'],
+              'hostname' => hostname,
+              'path' => instance['uri'].sub(hostname, '')
+            }
+          end
+
+          def sanitize(html_str)
+            ActionView::Base.full_sanitizer.sanitize(html_str)
+          end
+        end
+      end
+    end
+  end
+end

ee/spec/lib/gitlab/ci/parsers/security/dast_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Ci::Parsers::Security::Dast do
+  let(:parser) { described_class.new }
+
+  describe '#parse!' do
+    let(:project) { artifact.project }
+    let(:pipeline) { artifact.job.pipeline }
+    let(:artifact) { create(:ee_ci_job_artifact, :dast) }
+    let(:report) { Gitlab::Ci::Reports::Security::Report.new(artifact.file_type) }
+
+    before do
+      artifact.each_blob do |blob|
+        parser.parse!(blob, report)
+      end
+    end
+
+    it 'parses all identifiers and occurrences' do
+      expect(report.occurrences.length).to eq(2)
+      expect(report.identifiers.length).to eq(3)
+      expect(report.scanners.length).to eq(1)
+    end
+
+    it 'generates expected location fingerprint' do
+      expected1 = Digest::SHA1.hexdigest('X-Content-Type-Options GET ')
+      expected2 = Digest::SHA1.hexdigest('X-Content-Type-Options GET /')
+
+      expect(report.occurrences.first[:location_fingerprint]).to eq(expected1)
+      expect(report.occurrences.last[:location_fingerprint]).to eq(expected2)
+    end
+
+    describe 'occurrence properties' do
+      using RSpec::Parameterized::TableSyntax
+
+      where(:attribute, :value) do
+        :report_type | 'dast'
+        :severity | 'low'
+        :confidence | 'medium'
+      end
+
+      with_them do
+        it 'saves properly occurrence' do
+          occurrence = report.occurrences.last
+
+          expect(occurrence[attribute]).to eq(value)
+        end
+      end
+    end
+  end
+
+  describe '#format_vulnerability' do
+    let(:parsed_report) do
+      JSON.parse!(
+        File.read(
+          Rails.root.join('spec/fixtures/security-reports/master/gl-dast-report.json')
+        )
+      )
+    end
+
+    let(:file_vulnerability) { parsed_report['site']['alerts'][0] }
+    let(:sanitized_desc) { file_vulnerability['desc'].gsub('<p>', '').gsub('</p>', '') }
+    let(:sanitized_solution) { file_vulnerability['solution'].gsub('<p>', '').gsub('</p>', '') }
+    let(:version) { parsed_report['@version'] }
+
+    it 'format ZAProxy vulnerability into common format' do
+      data = parser.send(:format_vulnerability, file_vulnerability)
+
+      expect(data['category']).to eq('dast')
+      expect(data['message']).to eq('X-Content-Type-Options Header Missing')
+      expect(data['description']).to eq(sanitized_desc)
+      expect(data['cve']).to eq('10021')
+      expect(data['severity']).to eq('low')
+      expect(data['confidence']).to eq('medium')
+      expect(data['solution']).to eq(sanitized_solution)
+      expect(data['scanner']).to eq({ 'id' => 'zaproxy', 'name' => 'ZAProxy' })
+      expect(data['links']).to eq([{ 'url' => 'http://msdn.microsoft.com/en-us/library/ie/gg622941%28v=vs.85%29.aspx' },
+                                   { 'url' => 'https://www.owasp.org/index.php/List_of_useful_HTTP_headers' }])
+      expect(data['identifiers'][0]).to eq({
+                                             'type'  => 'ZAProxy_PluginId',
+                                             'name'  => 'X-Content-Type-Options Header Missing',
+                                             'value' => '10021',
+                                             'url'   => "https://github.com/zaproxy/zaproxy/blob/w2019-01-14/docs/scanners.md"
+                                           })
+      expect(data['identifiers'][1]).to eq({
+                                             'type'  => 'CWE',
+                                             'name'  => "CWE-16",
+                                             'value' => '16',
+                                             'url'   => "https://cwe.mitre.org/data/definitions/16.html"
+                                           })
+      expect(data['identifiers'][2]).to eq({
+                                             'type'  => 'WASC',
+                                             'name'  => "WASC-15",
+                                             'value' => '15',
+                                             'url'   => "http://projects.webappsec.org/w/page/13246974/Threat%20Classification%20Reference%20Grid"
+                                           })
+    end
+  end
+
+  describe '#location' do
+    let(:file_vulnerability) do
+      JSON.parse!(
+        File.read(
+          Rails.root.join('spec/fixtures/security-reports/master/gl-dast-report.json')
+        )
+      )['site']['alerts'][0]
+    end
+
+    let(:instance) { file_vulnerability['instances'][1] }
+    let(:host) { 'http://bikebilly-spring-auto-devops-review-feature-br-3y2gpb.35.192.176.43.xip.io' }
+
+    it 'format location struct' do
+      data = parser.send(:location, instance, host)
+
+      expect(data['param']).to eq('X-Content-Type-Options')
+      expect(data['method']).to eq('GET')
+      expect(data['hostname']).to eq(host)
+      expect(data['path']).to eq('/')
+    end
+  end
+
+  describe '#severity' do
+    using RSpec::Parameterized::TableSyntax
+
+    where(:severity, :expected) do
+      '0'  | 'ignore'
+      '1'  | 'low'
+      '2'  | 'medium'
+      '3'  | 'high'
+      '42' | 'unknown'
+      ''   | 'unknown'
+    end
+
+    with_them do
+      it 'substitutes with right values' do
+        expect(parser.send(:severity, severity)).to eq(expected)
+      end
+    end
+  end
+
+  describe '#confidence' do
+    using RSpec::Parameterized::TableSyntax
+
+    where(:confidence, :expected) do
+      '0'  | 'ignore'
+      '1'  | 'low'
+      '2'  | 'medium'
+      '3'  | 'high'
+      '4'  | 'critical'
+      '42' | 'unknown'
+      ''   | 'unknown'
+    end
+
+    with_them do
+      it 'substitutes with right values' do
+        expect(parser.send(:confidence, confidence)).to eq(expected)
+      end
+    end
+  end
+end

ee/spec/models/ci/build_spec.rb

@@ -157,7 +157,7 @@ describe Ci::Build do
     subject { job.collect_security_reports!(security_reports) }
 
     before do
-      stub_licensed_features(sast: true, dependency_scanning: true, container_scanning: true)
+      stub_licensed_features(sast: true, dependency_scanning: true, container_scanning: true, dast: true)
     end
 
     context 'when build has a security report' do
@@ -178,6 +178,7 @@ describe Ci::Build do
           create(:ee_ci_job_artifact, :sast, job: job, project: job.project)
           create(:ee_ci_job_artifact, :dependency_scanning, job: job, project: job.project)
           create(:ee_ci_job_artifact, :container_scanning, job: job, project: job.project)
+          create(:ee_ci_job_artifact, :dast, job: job, project: job.project)
         end
 
         it 'parses blobs and add the results to the reports' do
@@ -186,6 +187,7 @@ describe Ci::Build do
           expect(security_reports.get_report('sast').occurrences.size).to eq(33)
           expect(security_reports.get_report('dependency_scanning').occurrences.size).to eq(4)
           expect(security_reports.get_report('container_scanning').occurrences.size).to eq(8)
+          expect(security_reports.get_report('dast').occurrences.size).to eq(2)
         end
       end
 
@@ -217,6 +219,20 @@ describe Ci::Build do
         end
       end
 
+      context 'when Feature flag is disabled for DAST reports parsing' do
+        before do
+          stub_feature_flags(parse_dast_reports: false)
+          create(:ee_ci_job_artifact, :sast, job: job, project: job.project)
+          create(:ee_ci_job_artifact, :dast, job: job, project: job.project)
+        end
+
+        it 'does NOT parse dast report' do
+          subject
+
+          expect(security_reports.reports.keys).to contain_exactly('sast')
+        end
+      end
+
       context 'when there is a corrupted sast report' do
         before do
           create(:ee_ci_job_artifact, :sast_with_corrupted_data, job: job, project: job.project)