Commit 79a1bdec authored by Gabriel Mazetto's avatar Gabriel Mazetto

Move downloaders, transfers and uploaders to same namespace

Every class related to blob replication will live in
`Gitlab::Geo::Replication`
parent 96cd3545
......@@ -27,9 +27,10 @@ module Gitlab
return fail_before_transfer unless upload.present?
return missing_on_primary if upload.model.nil?
transfer = ::Gitlab::Geo::FileTransfer.new(object_type.to_sym, upload)
transfer = ::Gitlab::Geo::Replication::FileTransfer.new(object_type.to_sym, upload)
Result.from_transfer_result(transfer.download_from_primary)
end
# rubocop: enable CodeReuse/ActiveRecord
class Result
......
......@@ -30,6 +30,7 @@ module Gitlab
success(CarrierWave::SanitizedFile.new(recorded_file.absolute_path))
end
# rubocop: enable CodeReuse/ActiveRecord
private
......
......@@ -7,7 +7,7 @@ module Gitlab
# * Returning the necessary response data to send the file back
#
# TODO: Rearrange things so this class does not inherit from FileUploader
class JobArtifactUploader < ::Gitlab::Geo::FileUploader
class JobArtifactUploader < FileUploader
# rubocop: disable CodeReuse/ActiveRecord
def execute
job_artifact = ::Ci::JobArtifact.find_by(id: object_db_id)
......
......@@ -14,7 +14,7 @@ module Gitlab
lfs_object = LfsObject.find_by(id: object_db_id)
return fail_before_transfer unless lfs_object.present?
transfer = ::Gitlab::Geo::LfsTransfer.new(lfs_object)
transfer = ::Gitlab::Geo::Replication::LfsTransfer.new(lfs_object)
Result.from_transfer_result(transfer.download_from_primary)
end
# rubocop: enable CodeReuse/ActiveRecord
......
......@@ -117,7 +117,7 @@ module Gitlab
if response.code == 404 && body.present?
begin
json_response = JSON.parse(body)
return json_response['geo_code'] == Gitlab::Geo::FileUploader::FILE_NOT_FOUND_GEO_CODE
return code_file_not_found?(json_response['geo_code'])
rescue JSON::ParserError
end
end
......@@ -125,6 +125,10 @@ module Gitlab
false
end
def code_file_not_found?(geo_code)
geo_code == Gitlab::Geo::FileUploader::FILE_NOT_FOUND_GEO_CODE
end
def default_permissions
0666 - File.umask
end
......@@ -141,6 +145,7 @@ module Gitlab
nil
end
# @param [String] file_path disk location to compare checksum mismatch
def checksum_mismatch?(file_path)
# Skip checksum check if primary didn't generate one because, for
# example, large attachments are checksummed asynchronously, and most
......
......@@ -2,11 +2,10 @@
require 'spec_helper'
describe Gitlab::Geo::FileDownloader, :geo do
describe Gitlab::Geo::Replication::FileDownloader, :geo do
include EE::GeoHelpers
set(:primary_node) { create(:geo_node, :primary) }
set(:secondary_node) { create(:geo_node) }
subject { downloader.execute }
......@@ -25,6 +24,9 @@ describe Gitlab::Geo::FileDownloader, :geo do
end
context 'when in a secondary geo node' do
context 'with local storage only' do
let(:secondary_node) { create(:geo_node, :local_storage_only) }
before do
stub_current_geo_node(secondary_node)
......@@ -36,6 +38,7 @@ describe Gitlab::Geo::FileDownloader, :geo do
expect(subject.primary_missing_file).to be_falsey
end
end
end
def stub_geo_file_transfer(file_type, upload)
url = primary_node.geo_transfers_url(file_type, upload.id.to_s)
......@@ -43,7 +46,7 @@ describe Gitlab::Geo::FileDownloader, :geo do
stub_request(:get, url).to_return(status: 200, body: upload.build_uploader.file.read, headers: {})
end
def stub_geo_file_transfer_object_storage
def stub_geo_file_transfer_object_storage(file_type, upload)
url = primary_node.geo_transfers_url(file_type, upload.id.to_s)
stub_request(:get, url).to_return(status: 307, body: upload.build_uploader.url, headers: {})
......
require 'spec_helper'
describe Gitlab::Geo::FileTransfer do
describe Gitlab::Geo::Replication::FileTransfer do
include ::EE::GeoHelpers
set(:primary_node) { create(:geo_node, :primary) }
......@@ -65,7 +65,7 @@ describe Gitlab::Geo::FileTransfer do
it 'returns a failed result indicating primary_missing_file' do
expect(FileUtils).not_to receive(:mv).with(anything, upload.absolute_path).and_call_original
response = double(:response, success?: false, code: 404, msg: "No such file")
expect(File).to receive(:read).and_return("{\"geo_code\":\"#{Gitlab::Geo::FileUploader::FILE_NOT_FOUND_GEO_CODE}\"}")
expect(File).to receive(:read).and_return("{\"geo_code\":\"#{Gitlab::Geo::Replication::FileUploader::FILE_NOT_FOUND_GEO_CODE}\"}")
expect(Gitlab::HTTP).to receive(:get).and_return(response)
result = subject.download_from_primary
......
require 'spec_helper'
describe Gitlab::Geo::FileUploader, :geo do
describe Gitlab::Geo::Replication::FileUploader, :geo do
shared_examples_for 'returns necessary params for sending a file from an API endpoint' do
subject { @subject ||= uploader.execute }
......
require 'spec_helper'
describe Gitlab::Geo::JobArtifactDownloader, :geo do
describe Gitlab::Geo::Replication::JobArtifactDownloader, :geo do
let(:job_artifact) { create(:ci_job_artifact) }
context '#execute' do
context 'with job artifact' do
it 'returns a FileDownloader::Result object' do
downloader = described_class.new(:job_artifact, job_artifact.id)
result = Gitlab::Geo::Transfer::Result.new(success: true, bytes_downloaded: 1)
result = Gitlab::Geo::Replication::Transfer::Result.new(success: true, bytes_downloaded: 1)
allow_any_instance_of(Gitlab::Geo::JobArtifactTransfer)
.to receive(:download_from_primary).and_return(result)
expect(downloader.execute).to be_a(Gitlab::Geo::FileDownloader::Result)
expect(downloader.execute).to be_a(Gitlab::Geo::Replication::FileDownloader::Result)
end
end
......@@ -20,7 +20,7 @@ describe Gitlab::Geo::JobArtifactDownloader, :geo do
let(:downloader) { described_class.new(:job_artifact, 10000) }
it 'returns a FileDownloader::Result object' do
expect(downloader.execute).to be_a(Gitlab::Geo::FileDownloader::Result)
expect(downloader.execute).to be_a(Gitlab::Geo::Replication::FileDownloader::Result)
end
it 'returns a result indicating a failure before a transfer was attempted' do
......
......@@ -71,7 +71,7 @@ describe Gitlab::Geo::JobArtifactTransfer, :geo do
it 'returns a failed result indicating primary_missing_file' do
expect(FileUtils).not_to receive(:mv).with(anything, job_artifact.file.path).and_call_original
response = double(:response, success?: false, code: 404, msg: "No such file")
expect(File).to receive(:read).and_return("{\"geo_code\":\"#{Gitlab::Geo::FileUploader::FILE_NOT_FOUND_GEO_CODE}\"}")
expect(File).to receive(:read).and_return("{\"geo_code\":\"#{Gitlab::Geo::Replication::FileUploader::FILE_NOT_FOUND_GEO_CODE}\"}")
expect(Gitlab::HTTP).to receive(:get).and_return(response)
result = subject.download_from_primary
......
require 'spec_helper'
describe Gitlab::Geo::JobArtifactUploader, :geo do
describe Gitlab::Geo::Replication::JobArtifactUploader, :geo do
context '#execute' do
let(:uploader) { described_class.new(job_artifact.id, {}) }
subject { uploader.execute }
......
require 'spec_helper'
describe Gitlab::Geo::LfsDownloader, :geo do
describe Gitlab::Geo::Replication::LfsDownloader, :geo do
let(:lfs_object) { create(:lfs_object) }
context '#execute' do
context 'with LFS object' do
it 'returns a FileDownloader::Result object' do
downloader = described_class.new(:lfs, lfs_object.id)
result = Gitlab::Geo::Transfer::Result.new(success: true, bytes_downloaded: 1)
result = Gitlab::Geo::Replication::Transfer::Result.new(success: true, bytes_downloaded: 1)
allow_any_instance_of(Gitlab::Geo::LfsTransfer)
allow_any_instance_of(Gitlab::Geo::Replication::LfsTransfer)
.to receive(:download_from_primary).and_return(result)
expect(downloader.execute).to be_a(Gitlab::Geo::FileDownloader::Result)
expect(downloader.execute).to be_a(Gitlab::Geo::Replication::FileDownloader::Result)
end
end
......@@ -20,7 +20,7 @@ describe Gitlab::Geo::LfsDownloader, :geo do
let(:downloader) { described_class.new(:lfs, 10000) }
it 'returns a FileDownloader::Result object' do
expect(downloader.execute).to be_a(Gitlab::Geo::FileDownloader::Result)
expect(downloader.execute).to be_a(Gitlab::Geo::Replication::FileDownloader::Result)
end
it 'returns a result indicating a failure before a transfer was attempted' do
......
require 'spec_helper'
describe Gitlab::Geo::LfsTransfer do
describe Gitlab::Geo::Replication::LfsTransfer do
include ::EE::GeoHelpers
set(:primary_node) { create(:geo_node, :primary) }
......@@ -49,7 +49,7 @@ describe Gitlab::Geo::LfsTransfer do
it 'returns a failed result indicating primary_missing_file' do
expect(FileUtils).not_to receive(:mv).with(anything, lfs_object.file.path).and_call_original
response = double(:response, success?: false, code: 404, msg: "No such file")
expect(File).to receive(:read).and_return("{\"geo_code\":\"#{Gitlab::Geo::FileUploader::FILE_NOT_FOUND_GEO_CODE}\"}")
expect(File).to receive(:read).and_return("{\"geo_code\":\"#{Gitlab::Geo::Replication::FileUploader::FILE_NOT_FOUND_GEO_CODE}\"}")
expect(Gitlab::HTTP).to receive(:get).and_return(response)
result = subject.download_from_primary
......
require 'spec_helper'
describe Gitlab::Geo::LfsUploader, :geo do
describe Gitlab::Geo::Replication::LfsUploader, :geo do
context '#execute' do
subject { uploader.execute }
......
......@@ -41,7 +41,7 @@ describe API::Geo do
describe 'allowed IPs' do
let(:note) { create(:note, :with_attachment) }
let(:upload) { Upload.find_by(model: note, uploader: 'AttachmentUploader') }
let(:transfer) { Gitlab::Geo::FileTransfer.new(:attachment, upload) }
let(:transfer) { Gitlab::Geo::Replication::FileTransfer.new(:attachment, upload) }
let(:req_header) { Gitlab::Geo::TransferRequest.new(transfer.request_data).headers }
it 'responds with 401 when IP is not allowed' do
......@@ -64,7 +64,7 @@ describe API::Geo do
describe 'GET /geo/transfers/attachment/1' do
let(:note) { create(:note, :with_attachment) }
let(:upload) { Upload.find_by(model: note, uploader: 'AttachmentUploader') }
let(:transfer) { Gitlab::Geo::FileTransfer.new(:attachment, upload) }
let(:transfer) { Gitlab::Geo::Replication::FileTransfer.new(:attachment, upload) }
let(:req_header) { Gitlab::Geo::TransferRequest.new(transfer.request_data).headers }
before do
......@@ -103,7 +103,7 @@ describe API::Geo do
describe 'GET /geo/transfers/avatar/1' do
let(:user) { create(:user, avatar: fixture_file_upload('spec/fixtures/dk.png', 'image/png')) }
let(:upload) { Upload.find_by(model: user, uploader: 'AvatarUploader') }
let(:transfer) { Gitlab::Geo::FileTransfer.new(:avatar, upload) }
let(:transfer) { Gitlab::Geo::Replication::FileTransfer.new(:avatar, upload) }
let(:req_header) { Gitlab::Geo::TransferRequest.new(transfer.request_data).headers }
before do
......@@ -142,7 +142,7 @@ describe API::Geo do
describe 'GET /geo/transfers/file/1' do
let(:project) { create(:project) }
let(:upload) { Upload.find_by(model: project, uploader: 'FileUploader') }
let(:transfer) { Gitlab::Geo::FileTransfer.new(:file, upload) }
let(:transfer) { Gitlab::Geo::Replication::FileTransfer.new(:file, upload) }
let(:req_header) { Gitlab::Geo::TransferRequest.new(transfer.request_data).headers }
before do
......@@ -178,7 +178,7 @@ describe API::Geo do
get api("/geo/transfers/file/#{upload.id}"), headers: req_header
expect(response).to have_gitlab_http_status(404)
expect(json_response['geo_code']).to eq(Gitlab::Geo::FileUploader::FILE_NOT_FOUND_GEO_CODE)
expect(json_response['geo_code']).to eq(Gitlab::Geo::Replication::FileUploader::FILE_NOT_FOUND_GEO_CODE)
end
end
end
......@@ -194,7 +194,7 @@ describe API::Geo do
describe 'GET /geo/transfers/lfs/1' do
let(:lfs_object) { create(:lfs_object, :with_file) }
let(:transfer) { Gitlab::Geo::LfsTransfer.new(lfs_object) }
let(:transfer) { Gitlab::Geo::Replication::LfsTransfer.new(lfs_object) }
let(:req_header) { Gitlab::Geo::TransferRequest.new(transfer.request_data).headers }
before do
......@@ -229,7 +229,7 @@ describe API::Geo do
get api("/geo/transfers/lfs/#{lfs_object.id}"), headers: req_header
expect(response).to have_gitlab_http_status(404)
expect(json_response['geo_code']).to eq(Gitlab::Geo::FileUploader::FILE_NOT_FOUND_GEO_CODE)
expect(json_response['geo_code']).to eq(Gitlab::Geo::Replication::FileUploader::FILE_NOT_FOUND_GEO_CODE)
end
end
end
......
......@@ -12,13 +12,25 @@ describe Geo::FileDownloadService do
end
describe '#downloader' do
Gitlab::Geo::FileReplication::USER_UPLOADS_OBJECT_TYPES.each do |object_type|
subject {described_class.new(object_type, 1) }
Gitlab::Geo::Replication::USER_UPLOADS_OBJECT_TYPES.each do |object_type|
it "returns a FileDownloader given object_type is #{object_type}" do
subject = described_class.new(object_type, 1)
it "returns FileDownloader given object_type is #{object_type}" do
expect(subject.downloader).to eq(Gitlab::Geo::FileDownloader)
expect(subject.downloader).to be_a(Gitlab::Geo::Replication::FileDownloader)
end
end
it "returns a LfsDownloader given object_type is lfs" do
subject = described_class.new('lfs', 1)
expect(subject.downloader).to be_a(Gitlab::Geo::Replication::LfsDownloader)
end
it "returns a JobArtifactDownloader given object_type is job_artifact" do
subject = described_class.new('job_artifact', 1)
expect(subject.downloader).to be_a(Gitlab::Geo::Replication::JobArtifactDownloader)
end
end
context 'retry time' do
......@@ -434,6 +446,6 @@ describe Geo::FileDownloadService do
success: success,
primary_missing_file: primary_missing_file)
instance = double("(instance of Gitlab::Geo::Transfer)", download_from_primary: result)
allow(Gitlab::Geo::Transfer).to receive(:new).and_return(instance)
allow(Gitlab::Geo::Replication::Transfer).to receive(:new).and_return(instance)
end
end
......@@ -12,6 +12,28 @@ describe Geo::FileUploadService do
stub_current_geo_node(node)
end
describe '#uploader' do
Gitlab::Geo::Replication::USER_UPLOADS_OBJECT_TYPES.each do |file_type|
it "returns a FileUploader given type is #{file_type}" do
subject = described_class.new({ type: file_type, id: 1 }, 'request-data')
expect(subject.uploader).to be_a(Gitlab::Geo::Replication::FileUploader)
end
end
it "returns a LfsUploader given object_type is lfs" do
subject = described_class.new({ type: 'lfs', id: 1 }, 'request-data')
expect(subject.uploader).to be_a(Gitlab::Geo::Replication::LfsUploader)
end
it "returns a JobArtifactUploader given object_type is job_artifact" do
subject = described_class.new({ type: 'job_artifact', id: 1 }, 'request-data')
expect(subject.uploader).to be_a(Gitlab::Geo::Replication::JobArtifactUploader)
end
end
shared_examples 'no authorization header' do
it 'returns nil' do
service = described_class.new(params, nil)
......@@ -37,7 +59,7 @@ describe Geo::FileUploadService do
let(:user) { create(:user, avatar: fixture_file_upload('spec/fixtures/dk.png', 'image/png')) }
let(:upload) { Upload.find_by(model: user, uploader: 'AvatarUploader') }
let(:params) { { id: upload.id, type: 'avatar' } }
let(:request_data) { Gitlab::Geo::FileTransfer.new(:avatar, upload).request_data }
let(:request_data) { Gitlab::Geo::Replication::FileTransfer.new(:avatar, upload).request_data }
it 'sends avatar file' do
service = described_class.new(params, req_header)
......@@ -56,7 +78,7 @@ describe Geo::FileUploadService do
let(:group) { create(:group, avatar: fixture_file_upload('spec/fixtures/dk.png', 'image/png')) }
let(:upload) { Upload.find_by(model: group, uploader: 'AvatarUploader') }
let(:params) { { id: upload.id, type: 'avatar' } }
let(:request_data) { Gitlab::Geo::FileTransfer.new(:avatar, upload).request_data }
let(:request_data) { Gitlab::Geo::Replication::FileTransfer.new(:avatar, upload).request_data }
it 'sends avatar file' do
service = described_class.new(params, req_header)
......@@ -75,7 +97,7 @@ describe Geo::FileUploadService do
let(:project) { create(:project, avatar: fixture_file_upload('spec/fixtures/dk.png', 'image/png')) }
let(:upload) { Upload.find_by(model: project, uploader: 'AvatarUploader') }
let(:params) { { id: upload.id, type: 'avatar' } }
let(:request_data) { Gitlab::Geo::FileTransfer.new(:avatar, upload).request_data }
let(:request_data) { Gitlab::Geo::Replication::FileTransfer.new(:avatar, upload).request_data }
it 'sends avatar file' do
service = described_class.new(params, req_header)
......@@ -94,7 +116,7 @@ describe Geo::FileUploadService do
let(:note) { create(:note, :with_attachment) }
let(:upload) { Upload.find_by(model: note, uploader: 'AttachmentUploader') }
let(:params) { { id: upload.id, type: 'attachment' } }
let(:request_data) { Gitlab::Geo::FileTransfer.new(:attachment, upload).request_data }
let(:request_data) { Gitlab::Geo::Replication::FileTransfer.new(:attachment, upload).request_data }
it 'sends attachment file' do
service = described_class.new(params, req_header)
......@@ -113,7 +135,7 @@ describe Geo::FileUploadService do
let(:project) { create(:project) }
let(:upload) { Upload.find_by(model: project, uploader: 'FileUploader') }
let(:params) { { id: upload.id, type: 'file' } }
let(:request_data) { Gitlab::Geo::FileTransfer.new(:file, upload).request_data }
let(:request_data) { Gitlab::Geo::Replication::FileTransfer.new(:file, upload).request_data }
let(:file) { fixture_file_upload('spec/fixtures/dk.png', 'image/png') }
before do
......@@ -137,7 +159,7 @@ describe Geo::FileUploadService do
let(:group) { create(:group) }
let(:upload) { Upload.find_by(model: group, uploader: 'NamespaceFileUploader') }
let(:params) { { id: upload.id, type: 'file' } }
let(:request_data) { Gitlab::Geo::FileTransfer.new(:file, upload).request_data }
let(:request_data) { Gitlab::Geo::Replication::FileTransfer.new(:file, upload).request_data }
let(:file) { fixture_file_upload('spec/fixtures/dk.png', 'image/png') }
before do
......@@ -160,7 +182,7 @@ describe Geo::FileUploadService do
context 'LFS Object' do
let(:lfs_object) { create(:lfs_object, :with_file) }
let(:params) { { id: lfs_object.id, type: 'lfs' } }
let(:request_data) { Gitlab::Geo::LfsTransfer.new(lfs_object).request_data }
let(:request_data) { Gitlab::Geo::Replication::LfsTransfer.new(lfs_object).request_data }
it 'sends LFS file' do
service = described_class.new(params, req_header)
......@@ -197,7 +219,7 @@ describe Geo::FileUploadService do
let(:project) { create(:project) }
let(:upload) { Upload.find_by(model: project, uploader: 'ImportExportUploader') }
let(:params) { { id: upload.id, type: 'import_export' } }
let(:request_data) { Gitlab::Geo::FileTransfer.new(:import_export, upload).request_data }
let(:request_data) { Gitlab::Geo::Replication::FileTransfer.new(:import_export, upload).request_data }
let(:file) { fixture_file_upload('spec/fixtures/project_export.tar.gz') }
before do
......
......@@ -312,7 +312,7 @@ describe Geo::FileDownloadDispatchWorker, :geo, :geo_fdw do
stub_const('Geo::Scheduler::SchedulerWorker::DB_RETRIEVE_BATCH_SIZE', 5)
secondary.update!(files_max_capacity: 2)
result_object = double(:result, success: true, bytes_downloaded: 100, primary_missing_file: false)
allow_any_instance_of(::Gitlab::Geo::Transfer).to receive(:download_from_primary).and_return(result_object)
allow_any_instance_of(::Gitlab::Geo::Replication::Transfer).to receive(:download_from_primary).and_return(result_object)
avatar = fixture_file_upload('spec/fixtures/dk.png')
create_list(:lfs_object, 2, :with_file)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment