Added a generic Blob Retriever

Blob retriever relies on replicator to find the model and associated
carrierwave uploader and files.

ee/app/models/concerns/geo/blob_replicator_strategy.rb

@@ -27,6 +27,10 @@ module Geo
       download
     end
 
+    # Return the carrierwave uploader instance scoped to current model
+    #
+    # @abstract
+    # @return [Carrierwave::Uploader]
     def carrierwave_uploader
       raise NotImplementedError
     end
@@ -39,6 +43,14 @@ module Geo
       update_verification_state!(failure: e.message)
     end
 
+    # Check if given checksum matches known one
+    #
+    # @param [String] checksum
+    # @return [Boolean] whether checksum matches
+    def matches_checksum?(checksum)
+      model_record.verification_checksum == checksum
+    end
+
     private
 
     def update_verification_state!(checksum: nil, failure: nil)

ee/app/services/geo/blob_upload_service.rb

@@ -5,13 +5,15 @@ module Geo
     include ExclusiveLeaseGuard
     include ::Gitlab::Geo::LogHelpers
 
-    attr_reader :replicable_name, :blob_id, :decoded_params, :replicator
+    attr_reader :replicable_name, :blob_id, :checksum, :replicator
 
     def initialize(replicable_name:, blob_id:, decoded_params:)
       @replicable_name = replicable_name
       @blob_id = blob_id
-      @decoded_params = decoded_params
-      @replicator = Gitlab::Geo::Replicator.for_replicable_name(params[:replicable_name])
+      @checksum = decoded_params.delete(:checksum)
+
+      replicator_klass = Gitlab::Geo::Replicator.for_replicable_name(replicable_name)
+      @replicator = replicator_klass.new(model_record_id: blob_id)
 
       fail_unimplemented!(replicable_name) unless @replicator
     end
@@ -21,7 +23,7 @@ module Geo
     end
 
     def retriever
-      retriever = Gitlab::Geo::Replication::BlobRetriever.new(replicable_name, blob_id)
+      Gitlab::Geo::Replication::BlobRetriever.new(replicator: replicator, checksum: checksum)
     end
 
     private

ee/lib/gitlab/geo/jwt_request_decoder.rb

@@ -19,7 +19,7 @@ module Gitlab
         @auth_header = auth_header
       end
 
-      # Return decoded attributes from informed header
+      # Return decoded attributes from given header
       #
       # @return [Hash] decoded attributes
       def decode

ee/lib/gitlab/geo/replication/blob_retriever.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Geo
+    module Replication
+      class BlobRetriever < BaseRetriever
+        attr_reader :replicator, :checksum
+
+        # @param [Gitlab::Geo::Replicator] replicator
+        # @param [String] checksum
+        def initialize(replicator:, checksum:)
+          raise ArgumentError, 'Invalid replicator given' unless replicator.is_a?(Gitlab::Geo::Replicator)
+
+          @replicator = replicator
+          @checksum = checksum
+        end
+
+        def execute
+          return error("#{replicator.replicable_name} not found") unless recorded_file
+          return file_not_found(recorded_file) unless recorded_file.file_exist?
+          return error('Checksum mismatch') unless matches_checksum?
+
+          success(replicator.carrierwave_uploader.file)
+        end
+
+        private
+
+        def recorded_file
+          strong_memoize(:recorded_file) do
+            replicator.model_record
+          rescue ActiveRecord::RecordNotFound
+            nil
+          end
+        end
+
+        # Check if given checksum matches known good one
+        #
+        # We skip the check if no checksum is given to the retriever
+        #
+        # @return [Boolean]
+        def matches_checksum?
+          return true unless checksum
+
+          replicator.matches_checksum?(checksum)
+        end
+      end
+    end
+  end
+end

ee/lib/gitlab/geo/replication/job_artifact_retriever.rb

@@ -11,7 +11,6 @@ module Gitlab
         def execute
           return error('Job artifact not found') unless job_artifact.present?
 
-
           unless job_artifact.file.present? && job_artifact.file.exists?
             log_error("Could not upload job artifact because it does not have a file", id: job_artifact.id)
 

ee/spec/factories/packages.rb

@@ -175,6 +175,8 @@ FactoryBot.define do
       file { fixture_file_upload('ee/spec/fixtures/npm/foo-1.0.1.tgz') }
       file_name { 'foo-1.0.1.tgz' }
       file_sha1 { 'be93151dc23ac34a82752444556fe79b32c7a1ad' }
+      verified_at { Date.current }
+      verification_checksum { '4437b5775e61455588a7e5187a2e5c58c680694260bbe5501c235ec690d17f83' }
       size { 400.kilobytes }
     end
 

ee/spec/lib/gitlab/geo/jwt_request_decoder_spec.rb

@@ -62,15 +62,15 @@ describe Gitlab::Geo::JwtRequestDecoder do
   end
 
   describe '#valid_attributes?' do
-    it 'returns true when all informed attributes and decoded data are all the same' do
+    it 'returns true when all given attributes and decoded data are all the same' do
       expect(subject.valid_attributes?(input: 123, other_input: 'string value')).to be_truthy
     end
 
-    it 'returns true when informed attributes is a slice of decoded data' do
+    it 'returns true when given attributes is a slice of decoded data' do
       expect(subject.valid_attributes?(input: 123)).to be_truthy
     end
 
-    it 'returns false when one informed data doesnt match its corresponding decoded one' do
+    it 'returns false when one given data doesnt match its corresponding decoded one' do
       expect(subject.valid_attributes?(input: 123, other_input: 'wrong value')).to be_falsey
     end
   end

ee/spec/lib/gitlab/geo/replication/blob_retriever_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+describe Gitlab::Geo::Replication::BlobRetriever, :aggregate_failures do
+  let(:package_file) { create(:package_file, :npm) }
+  let(:package_checksum) { package_file.class.hexdigest(package_file.file.path) }
+  let(:replicator_class) { Geo::PackageFileReplicator }
+  let(:replicator) { replicator_class.new(model_record_id: package_file.id) }
+
+  describe '#initialize' do
+    it 'errors out with an invalid replicator' do
+      expect { described_class.new(replicator: Object.new, checksum: nil) }.to raise_error(ArgumentError)
+    end
+
+    it 'accepts valid attributes' do
+      expect { described_class.new(replicator: replicator, checksum: nil) }.not_to raise_error
+      expect { described_class.new(replicator: replicator, checksum: package_checksum) }.not_to raise_error
+    end
+  end
+
+  describe '#execute' do
+    subject { described_class.new(replicator: replicator, checksum: package_checksum) }
+
+    it 'returns model not found error if record cant be found' do
+      subject = described_class.new(replicator: replicator_class.new(model_record_id: 1234567890), checksum: nil)
+      response = subject.execute
+
+      expect(response).to include(code: :not_found)
+      expect(response).to include(message: /package_file not found/)
+    end
+
+    it 'returns file not found if file cant be found' do
+      subject
+      File.unlink(package_file.file.path)
+
+      response = subject.execute
+
+      expect(response).to include(code: :not_found)
+      expect(response).to include(message: /file not found/)
+    end
+
+    it 'returns checksum mismatch if sending an invalid checksum' do
+      subject = described_class.new(replicator: replicator, checksum: 'invalid')
+      response = subject.execute
+
+      expect(response).to include(code: :not_found)
+      expect(response).to include(message: 'Checksum mismatch')
+    end
+
+    it 'works with valid attributes' do
+      response = subject.execute
+
+      expect(response).to include(code: :ok)
+      expect(response).to include(message: 'Success')
+      expect(response[:file].path).to eq(package_file.file.path)
+    end
+  end
+end

ee/spec/services/geo/blob_upload_service_spec.rb

@@ -3,10 +3,37 @@
 require 'spec_helper'
 
 describe Geo::BlobUploadService do
-  include ::EE::GeoHelpers
+  let(:package_file) { create(:package_file, :npm) }
 
-  let_it_be(:primary) { create(:geo_node, :primary) }
-  let_it_be(:secondary) { create(:geo_node) }
+  subject { described_class.new(replicable_name: 'package_file', blob_id: package_file.id, decoded_params: {}) }
 
-  subject { described_class.new() }
+  describe '#initialize' do
+    it 'initializes with valid attributes' do
+      expect { subject }.not_to raise_error
+    end
+  end
+
+  describe '#execute' do
+    it 'works with valid attributes' do
+      expect { subject.execute }.not_to raise_error
+    end
+
+    it 'errors with an invalid attributes' do
+      service = described_class.new(replicable_name: 'package_file', blob_id: 1234567890, decoded_params: {})
+
+      response = service.execute
+
+      expect(response).to include(code: :not_found)
+    end
+
+    it 'returns a file with valid attributes' do
+      service = described_class.new(replicable_name: 'package_file', blob_id: package_file.id,
+                                    decoded_params: { checksum: package_file.verification_checksum })
+
+      response = service.execute
+
+      expect(response).to include(code: :ok)
+      expect(response[:file].path).to eq(package_file.file.path)
+    end
+  end
 end

spec/support/shared_examples/models/concerns/blob_replicator_strategy_shared_examples.rb

@@ -52,6 +52,7 @@ RSpec.shared_examples 'a blob replicator' do
       replicator.calculate_checksum!
 
       expect(model_record.reload.verification_checksum).not_to be_nil
+      expect(model_record.reload.verified_at).not_to be_nil
     end
 
     it 'saves the error message and increments retry counter' do