verifiable_replicator.rb 7.91 KB
Newer Older
1 2 3 4 5 6 7 8
# frozen_string_literal: true

module Geo
  module VerifiableReplicator
    extend ActiveSupport::Concern

    include Delay

9
    DEFAULT_VERIFICATION_BATCH_SIZE = 10
10
    DEFAULT_REVERIFICATION_BATCH_SIZE = 1000
11

12 13 14 15
    included do
      event :checksum_succeeded
    end

16
    class_methods do
17 18
      extend Gitlab::Utils::Override

19 20 21 22 23 24 25 26
      delegate :verification_pending_batch,
               :verification_failed_batch,
               :needs_verification_count,
               :needs_reverification_count,
               :fail_verification_timeouts,
               :reverifiable_batch,
               :reverify_batch,
               to: :verification_query_class
27

28
      # If replication is disabled, then so is verification.
29
      override :verification_enabled?
30 31 32 33
      def verification_enabled?
        enabled? && verification_feature_flag_enabled?
      end

34
      # Override this to check a feature flag
35
      def verification_feature_flag_enabled?
36
        false
37 38
      end

39 40 41 42
      # Called every minute by VerificationCronWorker
      def trigger_background_verification
        return false unless verification_enabled?

43
        ::Geo::VerificationBatchWorker.perform_with_capacity(replicable_name)
Mike Kozono's avatar
Mike Kozono committed
44 45

        ::Geo::VerificationTimeoutWorker.perform_async(replicable_name)
46 47 48 49

        # Secondaries don't need to run this since they will receive an event for each
        # rechecksummed resource: https://gitlab.com/gitlab-org/gitlab/-/issues/13842
        ::Geo::ReverificationBatchWorker.perform_async(replicable_name) if ::Gitlab::Geo.primary?
50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72
      end

      # Called by VerificationBatchWorker.
      #
      # - Gets next batch of records that need to be verified
      # - Verifies them
      #
      def verify_batch
        self.replicator_batch_to_verify.each(&:verify)
      end

      # Called by VerificationBatchWorker.
      #
      # - Asks the DB how many things still need to be verified (with a limit)
      # - Converts that to a number of batches
      #
      # @return [Integer] number of batches of verification work remaining, up to the given maximum
      def remaining_verification_batch_count(max_batch_count:)
        needs_verification_count(limit: max_batch_count * verification_batch_size)
          .fdiv(verification_batch_size)
          .ceil
      end

73 74 75 76 77 78 79 80 81 82 83 84
      # Called by ReverificationBatchWorker.
      #
      # - Asks the DB how many things still need to be reverified (with a limit)
      # - Converts that to a number of batches
      #
      # @return [Integer] number of batches of reverification work remaining, up to the given maximum
      def remaining_reverification_batch_count(max_batch_count:)
        needs_reverification_count(limit: max_batch_count * reverification_batch_size)
          .fdiv(reverification_batch_size)
          .ceil
      end

85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104
      # @return [Array<Gitlab::Geo::Replicator>] batch of replicators which need to be verified
      def replicator_batch_to_verify
        model_record_id_batch_to_verify.map do |id|
          self.new(model_record_id: id)
        end
      end

      # @return [Array<Integer>] list of IDs for this replicator's model which need to be verified
      def model_record_id_batch_to_verify
        ids = verification_pending_batch(batch_size: verification_batch_size)

        remaining_batch_size = verification_batch_size - ids.size

        if remaining_batch_size > 0
          ids += verification_failed_batch(batch_size: remaining_batch_size)
        end

        ids
      end

105 106 107 108 109
      # @return [Integer] number of records set to be re-verified
      def reverify_batch!
        reverify_batch(batch_size: reverification_batch_size)
      end

110 111 112 113 114 115 116 117 118
      # If primary, query the model table.
      # If secondary, query the registry table.
      def verification_query_class
        Gitlab::Geo.secondary? ? registry_class : model
      end

      # @return [Integer] number of records to verify per batch job
      def verification_batch_size
        DEFAULT_VERIFICATION_BATCH_SIZE
119 120 121 122 123
      end

      # @return [Integer] number of records to reverify per batch job
      def reverification_batch_size
        DEFAULT_REVERIFICATION_BATCH_SIZE
124 125
      end

126
      def checksummed_count
127 128 129 130
        # When verification is disabled, this returns nil.
        # Bonus: This causes the progress bar to be hidden.
        return unless verification_enabled?

131
        model.verification_succeeded.count
132 133 134
      end

      def checksum_failed_count
135 136 137 138
        # When verification is disabled, this returns nil.
        # Bonus: This causes the progress bar to be hidden.
        return unless verification_enabled?

139 140 141 142 143 144 145 146 147
        model.verification_failed.count
      end

      def checksum_total_count
        # When verification is disabled, this returns nil.
        # Bonus: This causes the progress bar to be hidden.
        return unless verification_enabled?

        model.available_verifiables.count
148
      end
149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164

      def verified_count
        # When verification is disabled, this returns nil.
        # Bonus: This causes the progress bar to be hidden.
        return unless verification_enabled?

        registry_class.synced.verification_succeeded.count
      end

      def verification_failed_count
        # When verification is disabled, this returns nil.
        # Bonus: This causes the progress bar to be hidden.
        return unless verification_enabled?

        registry_class.synced.verification_failed.count
      end
165 166 167 168 169 170 171 172

      def verification_total_count
        # When verification is disabled, this returns nil.
        # Bonus: This causes the progress bar to be hidden.
        return unless verification_enabled?

        registry_class.synced.available_verifiables.count
      end
173 174
    end

175 176 177 178 179 180 181 182 183 184
    def handle_after_checksum_succeeded
      return false unless Gitlab::Geo.primary?
      return unless self.class.verification_enabled?

      publish(:checksum_succeeded, **event_params)
    end

    # Called by Gitlab::Geo::Replicator#consume
    def consume_event_checksum_succeeded(**params)
      return unless Gitlab::Geo.secondary?
185
      return unless registry.persisted?
186 187 188 189

      registry.verification_pending!
    end

190
    # Schedules a verification job after a model record is created/updated
191
    def after_verifiable_update
192
      verify_async if should_primary_verify?
193 194
    end

195
    def verify_async
196 197 198 199 200
      # Marking started prevents backfill (VerificationBatchWorker) from picking
      # this up too.
      # Also, if another verification job is running, this will make that job
      # set state to pending after it finishes, since the calculated checksum
      # is already invalidated.
201
      verification_state_tracker.verification_started!
202

203 204 205
      Geo::VerificationWorker.perform_async(replicable_name, model_record.id)
    end

206
    # Calculates checksum and asks the model/registry to manage verification
207
    # state.
208
    def verify
209
      verification_state_tracker.track_checksum_attempt! do
210
        calculate_checksum
211
      end
212 213 214 215 216 217 218
    end

    # Check if given checksum matches known one
    #
    # @param [String] checksum
    # @return [Boolean] whether checksum matches
    def matches_checksum?(checksum)
219
      primary_checksum == checksum
220 221 222 223 224 225 226 227 228 229 230 231
    end

    # Checksum value from the main database
    #
    # @abstract
    def primary_checksum
      model_record.verification_checksum
    end

    def secondary_checksum
      registry.verification_checksum
    end
232 233 234 235

    def verification_state_tracker
      Gitlab::Geo.secondary? ? registry : model_record
    end
236

237 238 239 240 241 242
    # @abstract
    # @return [String] a checksum representing the data
    def calculate_checksum
      raise NotImplementedError, "#{self.class} does not implement #{__method__}"
    end

243 244 245 246 247
    private

    def should_primary_verify?
      self.class.verification_enabled? &&
       primary_checksum.nil? && # Some models may populate this as part of creating the record
248 249 250 251 252 253 254
       checksummable?
    end

    # @abstract
    # @return [Boolean] whether the replicable is capable of checksumming itself
    def checksummable?
      raise NotImplementedError, "#{self.class} does not implement #{__method__}"
255
    end
256 257
  end
end