Apply model layer validations on ingestion process

With this change, the ingestion service classes are now using the
`BulkInsertSafe` utility module which validates the records before
executing the UPSERT queries.

Some of the validations require additional DB calls like the uniqueness
and presence of associated records therefore we disable them to prevent
running N+1 queries.

ee/app/models/vulnerabilities/finding.rb

@@ -11,10 +11,6 @@ module Vulnerabilities
     # https://gitlab.com/gitlab-org/gitlab/-/issues/214563#note_370782508 is why the table names are not renamed
     self.table_name = "vulnerability_occurrences"
 
-    # This is necessary to prevent updating the
-    # created_at attribute with upsert queries.
-    attr_readonly(:created_at)
-
     FINDINGS_PER_PAGE = 20
     MAX_NUMBER_OF_IDENTIFIERS = 20
     REPORT_TYPES_WITH_LOCATION_IMAGE = %w[container_scanning cluster_image_scanning].freeze

ee/app/models/vulnerabilities/finding_identifier.rb

@@ -4,10 +4,6 @@ module Vulnerabilities
   class FindingIdentifier < ApplicationRecord
     self.table_name = "vulnerability_occurrence_identifiers"
 
-    # This is necessary to prevent updating the
-    # created_at attribute with upsert queries.
-    attr_readonly(:created_at)
-
     alias_attribute :finding_id, :occurrence_id
 
     belongs_to :finding, class_name: 'Vulnerabilities::Finding', inverse_of: :finding_identifiers, foreign_key: 'occurrence_id'

ee/app/models/vulnerabilities/finding_remediation.rb

@@ -5,10 +5,6 @@ module Vulnerabilities
   class FindingRemediation < ApplicationRecord
     self.table_name = 'vulnerability_findings_remediations'
 
-    # This is necessary to prevent updating the
-    # created_at attribute with upsert queries.
-    attr_readonly(:created_at)
-
     belongs_to :finding, class_name: 'Vulnerabilities::Finding', inverse_of: :finding_remediations, foreign_key: 'vulnerability_occurrence_id', optional: false
     belongs_to :remediation, class_name: 'Vulnerabilities::Remediation', inverse_of: :finding_remediations, foreign_key: 'vulnerability_remediation_id', optional: false
 

ee/app/models/vulnerabilities/finding_signature.rb

@@ -7,10 +7,6 @@ module Vulnerabilities
 
     self.table_name = 'vulnerability_finding_signatures'
 
-    # This is necessary to prevent updating the
-    # created_at attribute with upsert queries.
-    attr_readonly(:created_at)
-
     belongs_to :finding, foreign_key: 'finding_id', inverse_of: :signatures, class_name: 'Vulnerabilities::Finding'
     enum algorithm_type: VulnerabilityFindingSignatureHelpers::ALGORITHM_TYPES, _prefix: :algorithm
     validates :finding, presence: true

ee/app/models/vulnerabilities/flag.rb

@@ -4,10 +4,6 @@ module Vulnerabilities
   class Flag < ApplicationRecord
     self.table_name = 'vulnerability_flags'
 
-    # This is necessary to prevent updating the
-    # created_at attribute with upsert queries.
-    attr_readonly(:created_at)
-
     belongs_to :finding, class_name: 'Vulnerabilities::Finding', foreign_key: 'vulnerability_occurrence_id', inverse_of: :vulnerability_flags, optional: false
 
     validates :origin, length: { maximum: 255 }

ee/app/services/security/ingestion/bulk_insertable_task.rb

@@ -28,31 +28,72 @@ module Security
       include Gitlab::Utils::StrongMemoize
 
       def self.included(base)
-        base.singleton_class.attr_accessor :model, :unique_by, :uses
+        base.singleton_class.attr_accessor(:model, :unique_by, :uses)
+        base.extend(ClassMethods)
+      end
+
+      module ClassMethods
+        # Creates a proxy class to be used in UPSERT queries
+        # which will also run the model layer validations except
+        # the uniquness and presence of associations validations.
+        def klass
+          @klass ||= Class.new(model).tap do |klass|
+            remove_validations(klass)
+          end.include(BulkInsertSafe)
+        end
+
+        private
+
+        def remove_validations(klass)
+          klass.validators.each do |validator|
+            remove_validation_if_necessary(klass, validator)
+          end
+        end
+
+        def remove_validation_if_necessary(klass, validator)
+          return unless uniqunesss_validator?(validator) || presence_of_association_validator?(klass, validator)
+
+          klass.skip_callback(:validate, :before, validator)
+        end
+
+        def uniqunesss_validator?(validator)
+          validator.instance_of?(ActiveRecord::Validations::UniquenessValidator)
+        end
+
+        def presence_of_association_validator?(klass, validator)
+          validator.instance_of?(ActiveRecord::Validations::PresenceValidator) &&
+            (klass.reflections.keys & validator.attributes.map(&:to_s)).any?
+        end
       end
 
       def execute
-        result_set
+        return_data
 
         after_ingest if uses
       end
 
       private
 
-      delegate :unique_by, :model, :uses, :cast_values, to: :'self.class', private: true
+      delegate :unique_by, :model, :klass, :uses, :cast_values, to: :'self.class', private: true
 
       def return_data
-        @return_data ||= result_set&.cast_values(model.attribute_types).to_a
-      end
-
-      def result_set
-        strong_memoize(:result_set) do
-          if insert_attributes.present?
-            ActiveRecord::InsertAll.new(model, insert_attributes, on_duplicate: on_duplicate, returning: uses, unique_by: unique_by).execute
+        strong_memoize(:return_data) do
+          if insert_objects.present?
+            unique_by.present? ? bulk_upsert : bulk_insert
+          else
+            []
           end
         end
       end
 
+      def bulk_insert
+        klass.bulk_insert!(insert_objects, skip_duplicates: true, returns: uses)
+      end
+
+      def bulk_upsert
+        klass.bulk_upsert!(insert_objects, unique_by: unique_by, returns: uses, &method(:slice_attributes))
+      end
+
       def after_ingest
         raise "Implement the `after_ingest` template method!"
       end
@@ -61,16 +102,28 @@ module Security
         raise "Implement the `attributes` template method!"
       end
 
+      def insert_objects
+        @insert_objects ||= insert_attributes.map { |attributes| klass.new(attributes) }
+      end
+
       def insert_attributes
         @insert_attributes ||= attributes.map { |values| values.merge(timestamps) }
       end
 
-      def timestamps
-        @timestamps ||= Time.zone.now.then { |time| { created_at: time, updated_at: time } }
+      # `BulkInsertSafe` module is trying to update all the attributes
+      # of a record which overrides the columns with NULL values if the
+      # attribute is not provided. For this reason, we need to slice the
+      # attributes with this callback.
+      def slice_attributes(item_attributes)
+        item_attributes.slice!(*attribute_names)
       end
 
-      def on_duplicate
-        unique_by.present? ? :update : :skip
+      def attribute_names
+        @attribute_names ||= insert_attributes.first.keys.map(&:to_s)
+      end
+
+      def timestamps
+        @timestamps ||= Time.zone.now.then { |time| { created_at: time, updated_at: time } }
       end
     end
   end