Extend exported attributes of vulnerabilities

This MR adds 2 new columns to the vulnerability export CSV file which are called `CWE` and `Other Identifiers`.

doc/api/vulnerability_exports.md

@@ -198,18 +198,18 @@ The response will be `404 Not Found` if the vulnerability export is not finished
 Example response:
 
 ```csv
-Group Name,Project Name,Scanner Type,Scanner Name,Status,Vulnerability,Details,Additional Info,Severity,CVE
-Gitlab.org,Defend,container_scanning,Clair,confirmed,CVE-2017-16997 in glibc,,CVE-2017-16997 in glibc,critical,CVE-2017-16997
+Group Name,Project Name,Scanner Type,Scanner Name,Status,Vulnerability,Details,Additional Info,Severity,CVE,CWE,Other Identifiers
+Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2017-16997 in glibc,,CVE-2017-16997 in glibc,critical,CVE-2017-16997
 Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2017-18269 in glibc,,CVE-2017-18269 in glibc,critical,CVE-2017-18269
 Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2018-1000001 in glibc,,CVE-2018-1000001 in glibc,high,CVE-2018-1000001
 Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2016-10228 in glibc,,CVE-2016-10228 in glibc,medium,CVE-2016-10228
-Gitlab.org,Defend,container_scanning,Clair,confirmed,CVE-2010-4052 in glibc,,CVE-2010-4052 in glibc,low,CVE-2010-4052
+Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2010-4052 in glibc,,CVE-2010-4052 in glibc,low,CVE-2010-4052
 Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2018-18520 in elfutils,,CVE-2018-18520 in elfutils,low,CVE-2018-18520
-Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2018-16869 in nettle,,CVE-2018-16869 in nettle,unknown,CVE-2018-16869
-Gitlab.org,Defend,dependency_scanning,Gemnasium,detected,Regular Expression Denial of Service in debug,,Regular Expression Denial of Service in debug,unknown,yarn.lock:debug:gemnasium:37283ed4-0380-40d7-ada7-2d994afcc62a
-Gitlab.org,Defend,dependency_scanning,Gemnasium,detected,Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js,,Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js,unknown,yarn.lock:saml2-js:gemnasium:9952e574-7b5b-46fa-a270-aeb694198a98
-Gitlab.org,Defend,sast,Find Security Bugs,detected,Predictable pseudorandom number generator,,Predictable pseudorandom number generator,medium,818bf5dacb291e15d9e6dc3c5ac32178:PREDICTABLE_RANDOM:src/main/java/com/gitlab/security_products/tests/App.java:47
-Gitlab.org,Defend,sast,Find Security Bugs,detected,Cipher with no integrity,,Cipher with no integrity,medium,e6449b89335daf53c0db4c0219bc1634:CIPHER_INTEGRITY:src/main/java/com/gitlab/security_products/tests/App.java:29
-Gitlab.org,Defend,sast,Find Security Bugs,detected,Predictable pseudorandom number generator,,Predictable pseudorandom number generator,medium,e8ff1d01f74cd372f78da8f5247d3e73:PREDICTABLE_RANDOM:src/main/java/com/gitlab/security_products/tests/App.java:41
-Gitlab.org,Defend,sast,Find Security Bugs,confirmed,ECB mode is insecure 2,,ECB mode is insecure,medium,ea0f905fc76f2739d5f10a1fd1e37a10:ECB_MODE:src/main/java/com/gitlab/security_products/tests/App.java:29
-Gitlab.org,Defend,```
+Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2018-16869 in nettle,,CVE-2018-16869 in nettle,unknown,CVE-2018-16869,CWE-1
+Gitlab.org,Defend,dependency_scanning,Gemnasium,detected,Regular Expression Denial of Service in debug,,Regular Expression Denial of Service in debug,unknown,CVE-2021-1234,CWE-2,"""yarn.lock:debug:gemnasium:37283ed4-0380-40d7-ada7-2d994afcc62a"""
+Gitlab.org,Defend,dependency_scanning,Gemnasium,detected,Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js,,Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js,unknown,,,"""yarn.lock:saml2-js:gemnasium:9952e574-7b5b-46fa-a270-aeb694198a98"""
+Gitlab.org,Defend,sast,Find Security Bugs,detected,Predictable pseudorandom number generator,,Predictable pseudorandom number generator,medium,,,"""818bf5dacb291e15d9e6dc3c5ac32178:PREDICTABLE_RANDOM:src/main/java/com/gitlab/security_products/tests/App.java:47"""
+Gitlab.org,Defend,sast,Find Security Bugs,detected,Cipher with no integrity,,Cipher with no integrity,medium,,,"""e6449b89335daf53c0db4c0219bc1634:CIPHER_INTEGRITY:src/main/java/com/gitlab/security_products/tests/App.java:29"""
+Gitlab.org,Defend,sast,Find Security Bugs,detected,Predictable pseudorandom number generator,,Predictable pseudorandom number generator,medium,,,"""e8ff1d01f74cd372f78da8f5247d3e73:PREDICTABLE_RANDOM:src/main/java/com/gitlab/security_products/tests/App.java:41"""
+Gitlab.org,Defend,sast,Find Security Bugs,detected,ECB mode is insecure,,ECB mode is insecure,medium,,,"""ea0f905fc76f2739d5f10a1fd1e37a10:ECB_MODE:src/main/java/com/gitlab/security_products/tests/App.java:29"""
+```

ee/app/models/ee/vulnerability.rb

@@ -107,13 +107,15 @@ module EE
 
       scope :with_limit, -> (maximum) { limit(maximum) }
 
-      delegate :scanner_name, :scanner_external_id, :metadata, :message, :cve, :description,
+      delegate :scanner_name, :scanner_external_id, :metadata, :message, :description,
                to: :finding, prefix: true, allow_nil: true
 
       delegate :default_branch, :name, to: :project, prefix: true, allow_nil: true
       delegate :name, to: :group, prefix: true, allow_nil: true
 
-      delegate :solution, :identifiers, :links, :remediations, :file, to: :finding, allow_nil: true
+      delegate :solution, :identifiers, :links, :remediations, :file,
+               :cve_value, :cwe_value, :other_identifier_values,
+               to: :finding, allow_nil: true
 
       def to_reference(from = nil, full: false)
         reference = "#{self.class.reference_prefix}#{id}"

ee/app/models/vulnerabilities/finding.rb

@@ -280,8 +280,16 @@ module Vulnerabilities
       metadata.dig('message')
     end
 
-    def cve
-      metadata.dig('cve')
+    def cve_value
+      identifiers.find(&:cve?)&.name
+    end
+
+    def cwe_value
+      identifiers.find(&:cwe?)&.name
+    end
+
+    def other_identifier_values
+      identifiers.select(&:other?).map(&:name)
     end
 
     alias_method :==, :eql? # eql? is necessary in some cases like array intersection

ee/app/models/vulnerabilities/identifier.rb

@@ -25,5 +25,17 @@ module Vulnerabilities
     validates :name, presence: true
 
     scope :with_fingerprint, -> (fingerprints) { where(fingerprint: fingerprints) }
+
+    def cve?
+      external_type.casecmp?('cve')
+    end
+
+    def cwe?
+      external_type.casecmp?('cwe')
+    end
+
+    def other?
+      !(cve? || cwe?)
+    end
   end
 end

ee/app/services/vulnerability_exports/export_service.rb

@@ -55,7 +55,7 @@ module VulnerabilityExports
     end
 
     def vulnerabilities
-      Security::VulnerabilitiesFinder.new(exportable).execute.with_findings_and_scanner
+      Security::VulnerabilitiesFinder.new(exportable).execute.with_findings_scanner_and_identifiers
     end
 
     def schedule_export_deletion

ee/app/services/vulnerability_exports/exporters/csv_service.rb

@@ -3,6 +3,8 @@
 module VulnerabilityExports
   module Exporters
     class CsvService
+      IDENTIFIER_DELIMITER = '; '
+      IDENTIFIER_FORMATTER = -> (v) { v.other_identifier_values.to_csv(col_sep: IDENTIFIER_DELIMITER, row_sep: '') }
       MAPPING = {
         'Group Name' => 'group_name',
         'Project Name' => 'project_name',
@@ -13,7 +15,9 @@ module VulnerabilityExports
         'Details' => 'finding_description',
         'Additional Info' => 'finding_message',
         'Severity' => 'severity',
-        'CVE' => 'finding_cve'
+        'CVE' => 'cve_value',
+        'CWE' => 'cwe_value',
+        'Other Identifiers' => IDENTIFIER_FORMATTER
       }.freeze
 
       attr_reader :vulnerabilities

ee/changelogs/unreleased/214224_extend_vulnerability_exports.yml

+---
+title: Add `cwe` and `Other Identifiers` columns into vulnerability export files
+merge_request: 43179
+author:
+type: added

ee/spec/fixtures/vulnerabilities/exports/root-security-reports_vulnerabilities_2020-03-12T1235.csv

-Group Name,Project Name,Scanner Type,Scanner Name,Status,Vulnerability,Details,Additional Info,Severity,CVE
+Group Name,Project Name,Scanner Type,Scanner Name,Status,Vulnerability,Details,Additional Info,Severity,CVE,CWE,Other Identifiers
 Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2017-16997 in glibc,,CVE-2017-16997 in glibc,critical,CVE-2017-16997
 Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2017-18269 in glibc,,CVE-2017-18269 in glibc,critical,CVE-2017-18269
 Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2018-1000001 in glibc,,CVE-2018-1000001 in glibc,high,CVE-2018-1000001
 Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2016-10228 in glibc,,CVE-2016-10228 in glibc,medium,CVE-2016-10228
 Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2010-4052 in glibc,,CVE-2010-4052 in glibc,low,CVE-2010-4052
 Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2018-18520 in elfutils,,CVE-2018-18520 in elfutils,low,CVE-2018-18520
-Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2018-16869 in nettle,,CVE-2018-16869 in nettle,unknown,CVE-2018-16869
-Gitlab.org,Defend,dependency_scanning,Gemnasium,detected,Regular Expression Denial of Service in debug,,Regular Expression Denial of Service in debug,unknown,yarn.lock:debug:gemnasium:37283ed4-0380-40d7-ada7-2d994afcc62a
-Gitlab.org,Defend,dependency_scanning,Gemnasium,detected,Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js,,Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js,unknown,yarn.lock:saml2-js:gemnasium:9952e574-7b5b-46fa-a270-aeb694198a98
-Gitlab.org,Defend,sast,Find Security Bugs,detected,Predictable pseudorandom number generator,,Predictable pseudorandom number generator,medium,818bf5dacb291e15d9e6dc3c5ac32178:PREDICTABLE_RANDOM:src/main/java/com/gitlab/security_products/tests/App.java:47
-Gitlab.org,Defend,sast,Find Security Bugs,detected,Cipher with no integrity,,Cipher with no integrity,medium,e6449b89335daf53c0db4c0219bc1634:CIPHER_INTEGRITY:src/main/java/com/gitlab/security_products/tests/App.java:29
-Gitlab.org,Defend,sast,Find Security Bugs,detected,Predictable pseudorandom number generator,,Predictable pseudorandom number generator,medium,e8ff1d01f74cd372f78da8f5247d3e73:PREDICTABLE_RANDOM:src/main/java/com/gitlab/security_products/tests/App.java:41
-Gitlab.org,Defend,sast,Find Security Bugs,detected,ECB mode is insecure,,ECB mode is insecure,medium,ea0f905fc76f2739d5f10a1fd1e37a10:ECB_MODE:src/main/java/com/gitlab/security_products/tests/App.java:29
+Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2018-16869 in nettle,,CVE-2018-16869 in nettle,unknown,CVE-2018-16869,CWE-1
+Gitlab.org,Defend,dependency_scanning,Gemnasium,detected,Regular Expression Denial of Service in debug,,Regular Expression Denial of Service in debug,unknown,CVE-2021-1234,CWE-2,"""yarn.lock:debug:gemnasium:37283ed4-0380-40d7-ada7-2d994afcc62a"""
+Gitlab.org,Defend,dependency_scanning,Gemnasium,detected,Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js,,Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js,unknown,,,"""yarn.lock:saml2-js:gemnasium:9952e574-7b5b-46fa-a270-aeb694198a98"""
+Gitlab.org,Defend,sast,Find Security Bugs,detected,Predictable pseudorandom number generator,,Predictable pseudorandom number generator,medium,,,"""818bf5dacb291e15d9e6dc3c5ac32178:PREDICTABLE_RANDOM:src/main/java/com/gitlab/security_products/tests/App.java:47"""
+Gitlab.org,Defend,sast,Find Security Bugs,detected,Cipher with no integrity,,Cipher with no integrity,medium,,,"""e6449b89335daf53c0db4c0219bc1634:CIPHER_INTEGRITY:src/main/java/com/gitlab/security_products/tests/App.java:29"""
+Gitlab.org,Defend,sast,Find Security Bugs,detected,Predictable pseudorandom number generator,,Predictable pseudorandom number generator,medium,,,"""e8ff1d01f74cd372f78da8f5247d3e73:PREDICTABLE_RANDOM:src/main/java/com/gitlab/security_products/tests/App.java:41"""
+Gitlab.org,Defend,sast,Find Security Bugs,detected,ECB mode is insecure,,ECB mode is insecure,medium,,,"""ea0f905fc76f2739d5f10a1fd1e37a10:ECB_MODE:src/main/java/com/gitlab/security_products/tests/App.java:29"""

ee/spec/models/ee/vulnerability_spec.rb

@@ -546,7 +546,9 @@ RSpec.describe Vulnerability do
     it { is_expected.to delegate_method(:scanner_name).to(:finding).with_prefix.allow_nil }
     it { is_expected.to delegate_method(:metadata).to(:finding).with_prefix.allow_nil }
     it { is_expected.to delegate_method(:message).to(:finding).with_prefix.allow_nil }
-    it { is_expected.to delegate_method(:cve).to(:finding).with_prefix.allow_nil }
+    it { is_expected.to delegate_method(:cve_value).to(:finding).allow_nil }
+    it { is_expected.to delegate_method(:cwe_value).to(:finding).allow_nil }
+    it { is_expected.to delegate_method(:other_identifier_values).to(:finding).allow_nil }
     it { is_expected.to delegate_method(:default_branch).to(:project).with_prefix.allow_nil }
     it { is_expected.to delegate_method(:name).to(:project).with_prefix.allow_nil }
     it { is_expected.to delegate_method(:name).to(:group).with_prefix.allow_nil }

ee/spec/models/vulnerabilities/finding_spec.rb

@@ -669,15 +669,46 @@ RSpec.describe Vulnerabilities::Finding do
     it { is_expected.to eql(expected_message) }
   end
 
-  describe '#cve' do
+  describe '#cve_value' do
     let(:finding) { build(:vulnerabilities_finding) }
-    let(:expected_cve) { finding.metadata['cve'] }
+    let(:expected_cve) { 'CVE-2020-0000' }
 
-    subject { finding.cve }
+    subject { finding.cve_value }
+
+    before do
+      finding.identifiers << build(:vulnerabilities_identifier, external_type: 'cve', name: expected_cve)
+    end
 
     it { is_expected.to eql(expected_cve) }
   end
 
+  describe '#cwe_value' do
+    let(:finding) { build(:vulnerabilities_finding) }
+    let(:expected_cwe) { 'CWE-0000' }
+
+    subject { finding.cwe_value }
+
+    before do
+      finding.identifiers << build(:vulnerabilities_identifier, external_type: 'cwe', name: expected_cwe)
+    end
+
+    it { is_expected.to eql(expected_cwe) }
+  end
+
+  describe '#other_identifier_values' do
+    let(:finding) { build(:vulnerabilities_finding) }
+    let(:expected_values) { ['ID 1', 'ID 2'] }
+
+    subject { finding.other_identifier_values }
+
+    before do
+      finding.identifiers << build(:vulnerabilities_identifier, external_type: 'foo', name: expected_values.first)
+      finding.identifiers << build(:vulnerabilities_identifier, external_type: 'bar', name: expected_values.second)
+    end
+
+    it { is_expected.to match_array(expected_values) }
+  end
+
   describe "#metadata" do
     let(:finding) { build(:vulnerabilities_finding) }
 

ee/spec/models/vulnerabilities/identifier_spec.rb

@@ -3,6 +3,8 @@
 require 'spec_helper'
 
 RSpec.describe Vulnerabilities::Identifier do
+  using RSpec::Parameterized::TableSyntax
+
   describe 'associations' do
     it { is_expected.to have_many(:finding_identifiers).class_name('Vulnerabilities::FindingIdentifier') }
     it { is_expected.to have_many(:findings).class_name('Vulnerabilities::Finding') }
@@ -43,4 +45,52 @@ RSpec.describe Vulnerabilities::Identifier do
       end
     end
   end
+
+  describe 'type check methods' do
+    shared_examples_for 'type check method' do |method:|
+      with_them do
+        let(:identifier) { build_stubbed(:vulnerabilities_identifier, external_type: external_type) }
+
+        subject { identifier.public_send(method) }
+
+        it { is_expected.to be(expected_value) }
+      end
+    end
+
+    describe '#cve?' do
+      it_behaves_like 'type check method', method: :cve? do
+        where(:external_type, :expected_value) do
+          'CVE' | true
+          'cve' | true
+          'CWE' | false
+          'cwe' | false
+          'foo' | false
+        end
+      end
+    end
+
+    describe '#cwe?' do
+      it_behaves_like 'type check method', method: :cwe? do
+        where(:external_type, :expected_value) do
+          'CWE' | true
+          'cwe' | true
+          'CVE' | false
+          'cve' | false
+          'foo' | false
+        end
+      end
+    end
+
+    describe '#other?' do
+      it_behaves_like 'type check method', method: :other? do
+        where(:external_type, :expected_value) do
+          'CWE' | false
+          'cwe' | false
+          'CVE' | false
+          'cve' | false
+          'foo' | true
+        end
+      end
+    end
+  end
 end

ee/spec/services/vulnerability_exports/export_service_spec.rb

@@ -100,7 +100,7 @@ RSpec.describe VulnerabilityExports::ExportService do
 
       context 'when the export format is csv' do
         let(:vulnerabilities) { Vulnerability.none }
-        let(:mock_relation) { double(:relation, with_findings_and_scanner: vulnerabilities) }
+        let(:mock_relation) { double(:relation, with_findings_scanner_and_identifiers: vulnerabilities) }
         let(:mock_vulnerability_finder_service_object) { instance_double(Security::VulnerabilitiesFinder, execute: mock_relation) }
         let(:exportable_full_path) { 'foo' }
         let(:time_suffix) { Time.current.utc.strftime('%FT%H%M') }

ee/spec/services/vulnerability_exports/exporters/csv_service_spec.rb

@@ -21,7 +21,13 @@ RSpec.describe VulnerabilityExports::Exporters::CsvService do
 
   it 'includes the columns required for import' do
     expect(csv.headers).to contain_exactly('Group Name', 'Project Name', 'Scanner Type', 'Scanner Name', 'Status',
-                                           'Vulnerability', 'Details', 'Additional Info', 'Severity', 'CVE')
+                                           'Vulnerability', 'Details', 'Additional Info', 'Severity', 'CVE', 'CWE', 'Other Identifiers')
+  end
+
+  describe 'CSV content' do
+    before do
+      vulnerability.finding.identifiers << create(:vulnerabilities_identifier, external_type: 'GSO', name: 'GSO-1234;1234')
+      vulnerability.finding.identifiers << create(:vulnerabilities_identifier, external_type: 'TSO', name: 'TSO-1234')
     end
 
     context 'when a project belongs to a group' do
@@ -39,7 +45,9 @@ RSpec.describe VulnerabilityExports::Exporters::CsvService do
         expect(csv[0]['Details']).to eq vulnerability.finding_description
         expect(csv[0]['Additional Info']).to eq vulnerability.finding_message
         expect(csv[0]['Severity']).to eq vulnerability.severity
-      expect(csv[0]['CVE']).to eq vulnerability.finding_cve
+        expect(csv[0]['CVE']).to eq vulnerability.cve_value
+        expect(csv[0]['CWE']).to eq vulnerability.cwe_value
+        expect(csv[0]['Other Identifiers']).to eq '"GSO-1234;1234"; TSO-1234'
       end
     end
 
@@ -58,7 +66,10 @@ RSpec.describe VulnerabilityExports::Exporters::CsvService do
         expect(csv[0]['Details']).to eq vulnerability.finding_description
         expect(csv[0]['Additional Info']).to eq vulnerability.finding_message
         expect(csv[0]['Severity']).to eq vulnerability.severity
-      expect(csv[0]['CVE']).to eq vulnerability.finding_cve
+        expect(csv[0]['CVE']).to eq vulnerability.cve_value
+        expect(csv[0]['CWE']).to eq vulnerability.cwe_value
+        expect(csv[0]['Other Identifiers']).to eq '"GSO-1234;1234"; TSO-1234'
+      end
     end
   end
 end