Merge branch 'dz-migrate-packages-to-object-storage' into 'master'

Add gitlab:packages:migrate task to migrate package files to object storage

See merge request gitlab-org/gitlab-ee!7810

ee/app/models/packages/package_file.rb

@@ -7,6 +7,7 @@ class Packages::PackageFile < ActiveRecord::Base
   validates :file_name, presence: true
 
   scope :recent, -> { order(id: :desc) }
+  scope :with_files_stored_locally, -> { where(file_store: ::Packages::PackageFileUploader::Store::LOCAL) }
 
   mount_uploader :file, Packages::PackageFileUploader
 
@@ -17,4 +18,9 @@ class Packages::PackageFile < ActiveRecord::Base
     # which happens after object is inserted/updated
     self.update_column(:file_store, file.object_store)
   end
+
+  def log_geo_deleted_event
+    # Keep empty for now. Should be addressed in future
+    # by https://gitlab.com/gitlab-org/gitlab-ee/issues/7891
+  end
 end

ee/app/uploaders/packages/package_file_uploader.rb

@@ -5,6 +5,8 @@ class Packages::PackageFileUploader < GitlabUploader
 
   storage_options Gitlab.config.packages
 
+  alias_method :upload, :model
+
   def filename
     model.file_name
   end

ee/lib/tasks/gitlab/packages/migrate.rake

+require 'logger'
+
+desc "GitLab | Migrate packages files to remote storage"
+namespace :gitlab do
+  namespace :packages do
+    task migrate: :environment do
+      logger = Logger.new(STDOUT)
+      logger.info('Starting transfer of package files to object storage')
+
+      unless ::Packages::PackageFileUploader.object_store_enabled?
+        raise 'Object store is disabled for packages feature'
+      end
+
+      ::Packages::PackageFile.with_files_stored_locally.find_each(batch_size: 10) do |package_file|
+        begin
+          package_file.file.migrate!(::Packages::PackageFileUploader::Store::REMOTE)
+
+          logger.info("Transferred package file #{package_file.id} of size #{package_file.size.to_i.bytes} to object storage")
+        rescue => e
+          logger.error("Failed to transfer package file #{package_file.id} with error: #{e.message}")
+        end
+      end
+    end
+  end
+end

ee/spec/tasks/gitlab/packages/migrate_rake_spec.rb

+require 'rake_helper'
+
+describe 'gitlab:packages namespace rake task' do
+  before :all do
+    Rake.application.rake_require 'tasks/gitlab/packages/migrate'
+  end
+
+  describe 'migrate' do
+    let(:local) { ObjectStorage::Store::LOCAL }
+    let(:remote) { ObjectStorage::Store::REMOTE }
+    let!(:package_file) { create(:package_file, :pom, file_store: local) }
+
+    def packages_migrate
+      run_rake_task('gitlab:packages:migrate')
+    end
+
+    context 'object storage disabled' do
+      before do
+        stub_packages_object_storage(enabled: false)
+      end
+
+      it "doesn't migrate files" do
+        expect { packages_migrate }.to raise_error('Object store is disabled for packages feature')
+      end
+    end
+
+    context 'object storage enabled' do
+      before do
+        stub_packages_object_storage
+      end
+
+      it 'migrates local file to object storage' do
+        expect { packages_migrate }.to change { package_file.reload.file_store }.from(local).to(remote)
+      end
+    end
+  end
+
+  def stub_packages_object_storage(**params)
+    stub_object_storage_uploader(config: Gitlab.config.packages.object_store,
+                                 uploader: ::Packages::PackageFileUploader,
+                                 remote_directory: 'packages',
+                                 **params)
+  end
+end