Merge branch '276498_apply_model_layer_validations_on_report_ingestion' into 'master'

Apply model layer validations on report ingestion

See merge request gitlab-org/gitlab!72145

ee/app/models/vulnerabilities/finding.rb

@@ -11,10 +11,6 @@ module Vulnerabilities
     # https://gitlab.com/gitlab-org/gitlab/-/issues/214563#note_370782508 is why the table names are not renamed
     self.table_name = "vulnerability_occurrences"
 
-    # This is necessary to prevent updating the
-    # created_at attribute with upsert queries.
-    attr_readonly(:created_at)
-
     FINDINGS_PER_PAGE = 20
     MAX_NUMBER_OF_IDENTIFIERS = 20
     REPORT_TYPES_WITH_LOCATION_IMAGE = %w[container_scanning cluster_image_scanning].freeze

ee/app/models/vulnerabilities/finding_identifier.rb

@@ -4,10 +4,6 @@ module Vulnerabilities
   class FindingIdentifier < ApplicationRecord
     self.table_name = "vulnerability_occurrence_identifiers"
 
-    # This is necessary to prevent updating the
-    # created_at attribute with upsert queries.
-    attr_readonly(:created_at)
-
     alias_attribute :finding_id, :occurrence_id
 
     belongs_to :finding, class_name: 'Vulnerabilities::Finding', inverse_of: :finding_identifiers, foreign_key: 'occurrence_id'

ee/app/models/vulnerabilities/finding_remediation.rb

@@ -5,10 +5,6 @@ module Vulnerabilities
   class FindingRemediation < ApplicationRecord
     self.table_name = 'vulnerability_findings_remediations'
 
-    # This is necessary to prevent updating the
-    # created_at attribute with upsert queries.
-    attr_readonly(:created_at)
-
     belongs_to :finding, class_name: 'Vulnerabilities::Finding', inverse_of: :finding_remediations, foreign_key: 'vulnerability_occurrence_id', optional: false
     belongs_to :remediation, class_name: 'Vulnerabilities::Remediation', inverse_of: :finding_remediations, foreign_key: 'vulnerability_remediation_id', optional: false
 

ee/app/models/vulnerabilities/finding_signature.rb

@@ -7,10 +7,6 @@ module Vulnerabilities
 
     self.table_name = 'vulnerability_finding_signatures'
 
-    # This is necessary to prevent updating the
-    # created_at attribute with upsert queries.
-    attr_readonly(:created_at)
-
     belongs_to :finding, foreign_key: 'finding_id', inverse_of: :signatures, class_name: 'Vulnerabilities::Finding'
     enum algorithm_type: VulnerabilityFindingSignatureHelpers::ALGORITHM_TYPES, _prefix: :algorithm
     validates :finding, presence: true

ee/app/models/vulnerabilities/flag.rb

@@ -4,10 +4,6 @@ module Vulnerabilities
   class Flag < ApplicationRecord
     self.table_name = 'vulnerability_flags'
 
-    # This is necessary to prevent updating the
-    # created_at attribute with upsert queries.
-    attr_readonly(:created_at)
-
     belongs_to :finding, class_name: 'Vulnerabilities::Finding', foreign_key: 'vulnerability_occurrence_id', inverse_of: :vulnerability_flags, optional: false
 
     validates :origin, length: { maximum: 255 }

ee/app/services/security/ingestion/bulk_insertable_task.rb

@@ -28,29 +28,70 @@ module Security
       include Gitlab::Utils::StrongMemoize
 
       def self.included(base)
-        base.singleton_class.attr_accessor :model, :unique_by, :uses
+        base.singleton_class.attr_accessor(:model, :unique_by, :uses)
+        base.extend(ClassMethods)
+      end
+
+      module ClassMethods
+        # Creates a proxy class to be used in UPSERT queries
+        # which will also run the model layer validations except
+        # the uniquness and presence of associations validations.
+        def klass
+          @klass ||= Class.new(model).tap do |klass|
+            remove_validations(klass)
+          end.include(BulkInsertSafe)
+        end
+
+        private
+
+        def remove_validations(klass)
+          klass.validators.each do |validator|
+            remove_validation_if_necessary(klass, validator)
+          end
+        end
+
+        def remove_validation_if_necessary(klass, validator)
+          return unless uniqunesss_validator?(validator) || presence_of_association_validator?(klass, validator)
+
+          klass.skip_callback(:validate, :before, validator)
+        end
+
+        def uniqunesss_validator?(validator)
+          validator.instance_of?(ActiveRecord::Validations::UniquenessValidator)
+        end
+
+        def presence_of_association_validator?(klass, validator)
+          validator.instance_of?(ActiveRecord::Validations::PresenceValidator) &&
+            (klass.reflections.keys & validator.attributes.map(&:to_s)).any?
+        end
       end
 
       def execute
-        result_set
+        return_data
 
         after_ingest if uses
       end
 
       private
 
-      delegate :unique_by, :model, :uses, :cast_values, to: :'self.class', private: true
+      delegate :unique_by, :model, :klass, :uses, :cast_values, to: :'self.class', private: true
 
       def return_data
-        @return_data ||= result_set&.cast_values(model.attribute_types).to_a
+        strong_memoize(:return_data) do
+          if insert_objects.present?
+            unique_by.present? ? bulk_upsert : bulk_insert
+          else
+            []
           end
-
-      def result_set
-        strong_memoize(:result_set) do
-          if insert_attributes.present?
-            ActiveRecord::InsertAll.new(model, insert_attributes, on_duplicate: on_duplicate, returning: uses, unique_by: unique_by).execute
         end
       end
+
+      def bulk_insert
+        klass.bulk_insert!(insert_objects, skip_duplicates: true, returns: uses)
+      end
+
+      def bulk_upsert
+        klass.bulk_upsert!(insert_objects, unique_by: unique_by, returns: uses, &method(:slice_attributes))
       end
 
       def after_ingest
@@ -61,16 +102,28 @@ module Security
         raise "Implement the `attributes` template method!"
       end
 
+      def insert_objects
+        @insert_objects ||= insert_attributes.map { |attributes| klass.new(attributes) }
+      end
+
       def insert_attributes
         @insert_attributes ||= attributes.map { |values| values.merge(timestamps) }
       end
 
-      def timestamps
-        @timestamps ||= Time.zone.now.then { |time| { created_at: time, updated_at: time } }
+      # `BulkInsertSafe` module is trying to update all the attributes
+      # of a record which overrides the columns with NULL values if the
+      # attribute is not provided. For this reason, we need to slice the
+      # attributes with this callback.
+      def slice_attributes(item_attributes)
+        item_attributes.slice!(*attribute_names)
+      end
+
+      def attribute_names
+        @attribute_names ||= insert_attributes.first.keys.map(&:to_s)
       end
 
-      def on_duplicate
-        unique_by.present? ? :update : :skip
+      def timestamps
+        @timestamps ||= Time.zone.now.then { |time| { created_at: time, updated_at: time } }
       end
     end
   end