Merge branch '7046_vulnerabilities_data_model' into 'master'

Add data model and migration for vulnerabilities

See merge request gitlab-org/gitlab-ee!6896

db/schema.rb

@@ -2949,6 +2949,63 @@ ActiveRecord::Schema.define(version: 20180917214204) do
   add_index "vulnerability_feedback", ["pipeline_id"], name: "index_vulnerability_feedback_on_pipeline_id", using: :btree
   add_index "vulnerability_feedback", ["project_id", "category", "feedback_type", "project_fingerprint"], name: "vulnerability_feedback_unique_idx", unique: true, using: :btree
 
+  create_table "vulnerability_identifiers", id: :bigserial, force: :cascade do |t|
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.integer "project_id", null: false
+    t.binary "fingerprint", null: false
+    t.string "external_type", null: false
+    t.string "external_id", null: false
+    t.string "name", null: false
+    t.text "url"
+  end
+
+  add_index "vulnerability_identifiers", ["project_id", "fingerprint"], name: "index_vulnerability_identifiers_on_project_id_and_fingerprint", unique: true, using: :btree
+
+  create_table "vulnerability_occurrence_identifiers", id: :bigserial, force: :cascade do |t|
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.integer "occurrence_id", limit: 8, null: false
+    t.integer "identifier_id", limit: 8, null: false
+    t.boolean "primary", default: false, null: false
+  end
+
+  add_index "vulnerability_occurrence_identifiers", ["identifier_id"], name: "index_vulnerability_occurrence_identifiers_on_identifier_id", using: :btree
+  add_index "vulnerability_occurrence_identifiers", ["occurrence_id", "identifier_id"], name: "index_vulnerability_occurrence_identifiers_on_unique_keys", unique: true, using: :btree
+
+  create_table "vulnerability_occurrences", id: :bigserial, force: :cascade do |t|
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.integer "severity", limit: 2, null: false
+    t.integer "confidence", limit: 2, null: false
+    t.integer "report_type", limit: 2, null: false
+    t.integer "pipeline_id", null: false
+    t.integer "project_id", null: false
+    t.integer "scanner_id", limit: 8, null: false
+    t.binary "first_seen_in_commit_sha", null: false
+    t.binary "project_fingerprint", null: false
+    t.binary "location_fingerprint", null: false
+    t.binary "primary_identifier_fingerprint", null: false
+    t.string "ref", null: false
+    t.string "name", null: false
+    t.string "metadata_version", null: false
+    t.text "raw_metadata", null: false
+  end
+
+  add_index "vulnerability_occurrences", ["pipeline_id"], name: "index_vulnerability_occurrences_on_pipeline_id", using: :btree
+  add_index "vulnerability_occurrences", ["project_id", "ref", "scanner_id", "primary_identifier_fingerprint", "location_fingerprint"], name: "index_vulnerability_occurrences_on_unique_keys", unique: true, using: :btree
+  add_index "vulnerability_occurrences", ["scanner_id"], name: "index_vulnerability_occurrences_on_scanner_id", using: :btree
+
+  create_table "vulnerability_scanners", id: :bigserial, force: :cascade do |t|
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.integer "project_id", null: false
+    t.string "external_id", null: false
+    t.string "name", null: false
+  end
+
+  add_index "vulnerability_scanners", ["project_id", "external_id"], name: "index_vulnerability_scanners_on_project_id_and_external_id", unique: true, using: :btree
+
   create_table "web_hook_logs", force: :cascade do |t|
     t.integer "web_hook_id", null: false
     t.string "trigger"
@@ -3242,6 +3299,13 @@ ActiveRecord::Schema.define(version: 20180917214204) do
   add_foreign_key "vulnerability_feedback", "issues", on_delete: :nullify
   add_foreign_key "vulnerability_feedback", "projects", on_delete: :cascade
   add_foreign_key "vulnerability_feedback", "users", column: "author_id", on_delete: :cascade
+  add_foreign_key "vulnerability_identifiers", "projects", on_delete: :cascade
+  add_foreign_key "vulnerability_occurrence_identifiers", "vulnerability_identifiers", column: "identifier_id", on_delete: :cascade
+  add_foreign_key "vulnerability_occurrence_identifiers", "vulnerability_occurrences", column: "occurrence_id", on_delete: :cascade
+  add_foreign_key "vulnerability_occurrences", "ci_pipelines", column: "pipeline_id", on_delete: :cascade
+  add_foreign_key "vulnerability_occurrences", "projects", on_delete: :cascade
+  add_foreign_key "vulnerability_occurrences", "vulnerability_scanners", column: "scanner_id", on_delete: :cascade
+  add_foreign_key "vulnerability_scanners", "projects", on_delete: :cascade
   add_foreign_key "web_hook_logs", "web_hooks", on_delete: :cascade
   add_foreign_key "web_hooks", "projects", name: "fk_0c8ca6d9d1", on_delete: :cascade
 end

ee/app/models/ee/project.rb

@@ -35,6 +35,9 @@ module EE
       has_many :audit_events, as: :entity
       has_many :path_locks
       has_many :vulnerability_feedback
+      has_many :vulnerabilities, class_name: 'Vulnerabilities::Occurrence'
+      has_many :vulnerability_identifiers, class_name: 'Vulnerabilities::Identifier'
+      has_many :vulnerability_scanners, class_name: 'Vulnerabilities::Scanner'
       has_many :protected_environments
       has_many :software_license_policies, inverse_of: :project, class_name: 'SoftwareLicensePolicy'
       accepts_nested_attributes_for :software_license_policies, allow_destroy: true

ee/app/models/vulnerabilities/identifier.rb

+# frozen_string_literal: true
+
+module Vulnerabilities
+  class Identifier < ActiveRecord::Base
+    include ShaAttribute
+
+    self.table_name = "vulnerability_identifiers"
+
+    sha_attribute :fingerprint
+
+    has_many :occurrence_identifiers, class_name: 'Vulnerabilities::OccurrenceIdentifier'
+    has_many :occurrences, through: :occurrence_identifiers, class_name: 'Vulnerabilities::Occurrence'
+    has_many :primary_occurrences,  -> { where(vulnerability_occurrences: { primary_identifier_fingerprint: fingerprint }) },
+      through: :occurrence_identifiers, class_name: 'Vulnerabilities::Occurrence', source: :occurrence
+
+    belongs_to :project
+
+    validates :project, presence: true
+    validates :external_type, presence: true
+    validates :external_id, presence: true
+    validates :fingerprint, presence: true
+    # Uniqueness validation doesn't work with binary columns, so save this useless query. It is enforce by DB constraint anyway.
+    # TODO: find out why it fails
+    # validates :fingerprint, presence: true, uniqueness: { scope: :project_id }
+    validates :name, presence: true
+  end
+end

ee/app/models/vulnerabilities/occurrence.rb

+# frozen_string_literal: true
+
+module Vulnerabilities
+  class Occurrence < ActiveRecord::Base
+    include ShaAttribute
+
+    self.table_name = "vulnerability_occurrences"
+
+    # Used for both severity and confidence
+    LEVELS = {
+      undefined: 0,
+      ignore: 1,
+      unknown: 2,
+      experimental: 3,
+      low: 4,
+      medium: 5,
+      high: 6,
+      critical: 7
+    }.with_indifferent_access.freeze
+
+    sha_attribute :first_seen_in_commit_sha
+    sha_attribute :project_fingerprint
+    sha_attribute :primary_identifier_fingerprint
+    sha_attribute :location_fingerprint
+
+    belongs_to :project
+    belongs_to :pipeline, class_name: 'Ci::Pipeline'
+    belongs_to :scanner, class_name: 'Vulnerabilities::Scanner'
+
+    has_many :occurrence_identifiers, class_name: 'Vulnerabilities::OccurrenceIdentifier'
+    has_many :identifiers, through: :occurrence_identifiers, class_name: 'Vulnerabilities::Identifier'
+
+    enum report_type: {
+      sast: 0,
+      dependency_scanning: 1,
+      container_scanning: 2,
+      dast: 3
+    }
+
+    validates :scanner, presence: true
+    validates :project, presence: true
+    validates :pipeline, presence: true
+    validates :ref, presence: true
+
+    validates :first_seen_in_commit_sha, presence: true
+    validates :project_fingerprint, presence: true
+    validates :primary_identifier_fingerprint, presence: true
+    validates :location_fingerprint, presence: true
+    # Uniqueness validation doesn't work with binary columns, so save this useless query. It is enforce by DB constraint anyway.
+    # TODO: find out why it fails
+    # validates :location_fingerprint, presence: true, uniqueness: { scope: [:primary_identifier_fingerprint, :scanner_id, :ref, :project_id] }
+    validates :name, presence: true
+    validates :report_type, presence: true
+    validates :severity, presence: true, inclusion: { in: LEVELS.keys }
+    validates :confidence, presence: true, inclusion: { in: LEVELS.keys }
+
+    validates :metadata_version, presence: true
+    validates :raw_metadata, presence: true
+
+    # Override getter and setter for :severity as we can't use enum (it conflicts with :confidence)
+    # To be replaced with enum using _prefix when migrating to rails 5
+    def severity
+      LEVELS.key(read_attribute(:severity))
+    end
+
+    def severity=(severity)
+      write_attribute(:severity, LEVELS[severity])
+    end
+
+    # Override getter and setter for :confidence as we can't use enum (it conflicts with :severity)
+    # To be replaced with enum using _prefix when migrating to rails 5
+    def confidence
+      LEVELS.key(read_attribute(:confidence))
+    end
+
+    def confidence=(confidence)
+      write_attribute(:confidence, LEVELS[confidence])
+    end
+  end
+end

ee/app/models/vulnerabilities/occurrence_identifier.rb

+# frozen_string_literal: true
+
+module Vulnerabilities
+  class OccurrenceIdentifier < ActiveRecord::Base
+    self.table_name = "vulnerability_occurrence_identifiers"
+
+    belongs_to :occurrence, class_name: 'Vulnerabilities::Occurrence'
+    belongs_to :identifier, class_name: 'Vulnerabilities::Identifier'
+
+    validates :occurrence, presence: true
+    validates :identifier, presence: true
+    validates :identifier_id, uniqueness: { scope: [:occurrence_id] }
+    validates :occurrence_id, uniqueness: true, if: :primary
+  end
+end

ee/app/models/vulnerabilities/scanner.rb

+# frozen_string_literal: true
+
+module Vulnerabilities
+  class Scanner < ActiveRecord::Base
+    self.table_name = "vulnerability_scanners"
+
+    has_many :occurrences, class_name: 'Vulnerabilities::Occurrence'
+
+    belongs_to :project
+
+    validates :project, presence: true
+    validates :external_id, presence: true, uniqueness: { scope: :project_id }
+    validates :name, presence: true
+  end
+end

ee/changelogs/unreleased/7046_vulnerabilities_data_model.yml

+---
+title: Add data model and migration for vulnerabilities
+merge_request:
+author:
+type: added

ee/db/migrate/20180917171038_create_vulnerability_scanners.rb

+# frozen_string_literal: true
+
+class CreateVulnerabilityScanners < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :vulnerability_scanners, id: :bigserial do |t|
+      t.timestamps_with_timezone null: false
+
+      t.references :project, null: false, foreign_key: { on_delete: :cascade }
+
+      t.string :external_id, null: false
+      t.string :name, null: false
+
+      t.index [:project_id, :external_id], unique: true
+    end
+  end
+end

ee/db/migrate/20180917171533_create_vulnerability_occurrences.rb

+# frozen_string_literal: true
+
+class CreateVulnerabilityOccurrences < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :vulnerability_occurrences, id: :bigserial  do |t|
+      t.timestamps_with_timezone null: false
+
+      t.integer :severity, null: false, limit: 2
+      t.integer :confidence, null: false, limit: 2
+      t.integer :report_type, null: false, limit: 2
+
+      t.integer :pipeline_id, null: false
+      t.foreign_key :ci_pipelines, column: :pipeline_id, on_delete: :cascade
+      t.references :project, null: false, foreign_key: { on_delete: :cascade }
+
+      t.bigint :scanner_id, null: false
+      t.foreign_key :vulnerability_scanners, column: :scanner_id, on_delete: :cascade
+
+      t.binary :first_seen_in_commit_sha, null: false, limit: 20
+      t.binary :project_fingerprint, null: false, limit: 20
+      t.binary :location_fingerprint, null: false, limit: 20
+      t.binary :primary_identifier_fingerprint, null: false, limit: 20
+
+      t.string :ref, null: false
+      t.string :name, null: false
+      t.string :metadata_version, null: false
+      t.text :raw_metadata, null: false
+
+      t.index :pipeline_id
+      t.index :scanner_id
+      t.index [:project_id, :ref, :scanner_id, :primary_identifier_fingerprint, :location_fingerprint],
+        unique: true,
+        name: 'index_vulnerability_occurrences_on_unique_keys',
+        length: { location_fingerprint: 20, primary_identifier_fingerprint: 20 }
+    end
+  end
+end

ee/db/migrate/20180917171534_create_vulnerability_identifiers.rb

+# frozen_string_literal: true
+
+class CreateVulnerabilityIdentifiers < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :vulnerability_identifiers, id: :bigserial do |t|
+      t.timestamps_with_timezone null: false
+
+      t.references :project, null: false, foreign_key: { on_delete: :cascade }
+
+      t.binary :fingerprint, null: false, limit: 20
+      t.string :external_type, null: false
+      t.string :external_id, null: false
+      t.string :name, null: false
+      t.text :url, null: true
+
+      t.index [:project_id, :fingerprint], unique: true
+    end
+  end
+end

ee/db/migrate/20180917171535_create_vulnerability_occurrence_identifiers.rb

+# frozen_string_literal: true
+
+class CreateVulnerabilityOccurrenceIdentifiers < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :vulnerability_occurrence_identifiers, id: :bigserial do |t|
+      t.timestamps_with_timezone null: false
+
+      t.bigint :occurrence_id, null: false
+      t.foreign_key :vulnerability_occurrences, column: :occurrence_id, on_delete: :cascade
+      t.bigint :identifier_id, null: false
+      t.foreign_key :vulnerability_identifiers, column: :identifier_id, on_delete: :cascade
+
+      t.boolean :primary, null: false, default: false
+
+      t.index :identifier_id
+      t.index [:occurrence_id, :identifier_id],
+        unique: true,
+        name: 'index_vulnerability_occurrence_identifiers_on_unique_keys'
+    end
+  end
+end

ee/spec/factories/vulnerabilities/identifiers.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :vulnerabilities_identifier, class: Vulnerabilities::Identifier do
+    external_type 'CVE'
+    external_id 'CVE-2018-1234'
+    fingerprint '52d084cede3db8fafcd6b8ae382ddf1970da3b7f'
+    name 'CVE-2018-1234'
+    url 'http://cve.mitre.org/cgi-bin/cvename.cgi?name=2018-1234'
+    project
+  end
+end

ee/spec/factories/vulnerabilities/occurrence_identifiers.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :vulnerabilities_occurrence_identifier, class: Vulnerabilities::OccurrenceIdentifier do
+    occurrence factory: :vulnerabilities_occurrence
+    identifier factory: :vulnerabilities_identifier
+  end
+end

ee/spec/factories/vulnerabilities/occurrences.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :vulnerabilities_occurrence, class: Vulnerabilities::Occurrence do
+    name 'Cipher with no integrity'
+    project
+    pipeline factory: :ci_pipeline
+    ref 'master'
+    first_seen_in_commit_sha '52d084cede3db8fafcd6b8ae382ddf1970da3b7f'
+    project_fingerprint '4e5b6966dd100170b4b1ad599c7058cce91b57b4'
+    primary_identifier_fingerprint '4e5b6966dd100170b4b1ad599c7058cce91b57b4'
+    location_fingerprint '4e5b6966dd100170b4b1ad599c7058cce91b57b4'
+    report_type :sast
+    severity :high
+    confidence :medium
+    scanner factory: :vulnerabilities_scanner
+    metadata_version 'sast:1.0'
+    raw_metadata 'raw_metadata'
+  end
+end

ee/spec/factories/vulnerabilities/scanners.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :vulnerabilities_scanner, class: Vulnerabilities::Scanner do
+    external_id 'find_sec_bugs'
+    name 'Find Security Bugs'
+    project
+  end
+end

ee/spec/lib/gitlab/import_export/all_models.yml

@@ -52,7 +52,10 @@ project:
 - source_pipelines
 - sourced_pipelines
 - prometheus_metrics
+- vulnerabilities
 - vulnerability_feedback
+- vulnerability_identifiers
+- vulnerability_scanners
 - prometheus_alerts
 - software_license_policies
 - project_registry

ee/spec/models/vulnerabilities/identifier_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Vulnerabilities::Identifier do
+  describe 'associations' do
+    it { is_expected.to have_many(:occurrence_identifiers).class_name('Vulnerabilities::OccurrenceIdentifier') }
+    it { is_expected.to have_many(:occurrences).class_name('Vulnerabilities::Occurrence') }
+    it { is_expected.to have_many(:primary_occurrences).class_name('Vulnerabilities::Occurrence') }
+    it { is_expected.to belong_to(:project) }
+  end
+
+  describe 'validations' do
+    let!(:identifier) { create(:vulnerabilities_identifier) }
+
+    it { is_expected.to validate_presence_of(:name) }
+    it { is_expected.to validate_presence_of(:project) }
+    it { is_expected.to validate_presence_of(:external_type) }
+    it { is_expected.to validate_presence_of(:external_id) }
+    it { is_expected.to validate_presence_of(:fingerprint) }
+    # Uniqueness validation doesn't work with binary columns. See TODO in class file
+    # it { is_expected.to validate_uniqueness_of(:fingerprint).scoped_to(:project_id) }
+  end
+end

ee/spec/models/vulnerabilities/occurrence_identifier_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Vulnerabilities::OccurrenceIdentifier do
+  describe 'associations' do
+    it { is_expected.to belong_to(:identifier).class_name('Vulnerabilities::Identifier') }
+    it { is_expected.to belong_to(:occurrence).class_name('Vulnerabilities::Occurrence') }
+  end
+
+  describe 'validations' do
+    let!(:occurrence_identifier) { create(:vulnerabilities_occurrence_identifier) }
+
+    it { is_expected.to validate_presence_of(:occurrence) }
+    it { is_expected.to validate_presence_of(:identifier) }
+    it { is_expected.to validate_uniqueness_of(:identifier_id).scoped_to(:occurrence_id) }
+
+    context 'when primary' do
+      before do
+        allow_any_instance_of(described_class).to receive(:primary).and_return(true)
+      end
+
+      it { is_expected.to validate_uniqueness_of(:occurrence_id) }
+    end
+
+    context 'when not primary' do
+      before do
+        allow_any_instance_of(described_class).to receive(:primary).and_return(false)
+      end
+
+      it { is_expected.not_to validate_uniqueness_of(:occurrence_id) }
+    end
+  end
+end

ee/spec/models/vulnerabilities/occurrence_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Vulnerabilities::Occurrence do
+  it { is_expected.to define_enum_for(:report_type) }
+
+  describe 'associations' do
+    it { is_expected.to belong_to(:project) }
+    it { is_expected.to belong_to(:pipeline) }
+    it { is_expected.to belong_to(:scanner).class_name('Vulnerabilities::Scanner') }
+    it { is_expected.to have_many(:identifiers).class_name('Vulnerabilities::Identifier') }
+    it { is_expected.to have_many(:occurrence_identifiers).class_name('Vulnerabilities::OccurrenceIdentifier') }
+  end
+
+  describe 'validations' do
+    let(:occurrence) { build(:vulnerabilities_occurrence) }
+
+    it { is_expected.to validate_presence_of(:scanner) }
+    it { is_expected.to validate_presence_of(:project) }
+    it { is_expected.to validate_presence_of(:pipeline) }
+    it { is_expected.to validate_presence_of(:ref) }
+    it { is_expected.to validate_presence_of(:first_seen_in_commit_sha) }
+    it { is_expected.to validate_presence_of(:project_fingerprint) }
+    it { is_expected.to validate_presence_of(:primary_identifier_fingerprint) }
+    it { is_expected.to validate_presence_of(:location_fingerprint) }
+    it { is_expected.to validate_presence_of(:name) }
+    it { is_expected.to validate_presence_of(:report_type) }
+    it { is_expected.to validate_presence_of(:metadata_version) }
+    it { is_expected.to validate_presence_of(:raw_metadata) }
+    it { is_expected.to validate_presence_of(:severity) }
+    it { is_expected.to validate_inclusion_of(:severity).in_array(described_class::LEVELS.keys) }
+    it { is_expected.to validate_presence_of(:confidence) }
+    it { is_expected.to validate_inclusion_of(:confidence).in_array(described_class::LEVELS.keys) }
+    # Uniqueness validation doesn't work with binary columns. See TODO in class file
+    # it { is_expected.to validate_uniqueness_of(:ref).scoped_to(:primary_identifier_fingerprint, :location_fingerprint, :scanner_id, :ref, :project_id) }
+  end
+
+  context 'database uniqueness' do
+    let(:occurrence) { create(:vulnerabilities_occurrence) }
+    let(:new_occurrence) { occurrence.dup }
+
+    it "when all index attributes are identical" do
+      expect { new_occurrence.save! }.to raise_error(ActiveRecord::RecordNotUnique)
+    end
+
+    describe 'when some parameters are changed' do
+      using RSpec::Parameterized::TableSyntax
+
+      # we use block to delay object creations
+      where(:key, :value_block) do
+        :primary_identifier_fingerprint | -> { '005b6966dd100170b4b1ad599c7058cce91b57b4' }
+        :ref | -> { 'another_ref' }
+        :scanner | -> { create(:vulnerabilities_scanner) }
+        :project | -> { create(:project) }
+      end
+
+      with_them do
+        it "is valid" do
+          expect { new_occurrence.update!({ key => value_block.call }) }.not_to raise_error
+        end
+      end
+    end
+  end
+end

ee/spec/models/vulnerabilities/scanner_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Vulnerabilities::Scanner do
+  describe 'associations' do
+    it { is_expected.to have_many(:occurrences).class_name('Vulnerabilities::Occurrence') }
+    it { is_expected.to belong_to(:project) }
+  end
+
+  describe 'validations' do
+    let!(:scanner) { create(:vulnerabilities_scanner) }
+
+    it { is_expected.to validate_presence_of(:name) }
+    it { is_expected.to validate_presence_of(:project) }
+    it { is_expected.to validate_presence_of(:project) }
+    it { is_expected.to validate_uniqueness_of(:external_id).scoped_to(:project_id) }
+  end
+end