Commit 7a122dc2 authored by James Fargher's avatar James Fargher

Merge branch 'feature/dependency-proxy-migrate-rake' into 'master'

Add dependency proxy migrate rake task

See merge request gitlab-org/gitlab!79663
parents 9082915f 007224aa
...@@ -14,6 +14,8 @@ class DependencyProxy::Blob < ApplicationRecord ...@@ -14,6 +14,8 @@ class DependencyProxy::Blob < ApplicationRecord
validates :file, presence: true validates :file, presence: true
validates :file_name, presence: true validates :file_name, presence: true
scope :with_files_stored_locally, -> { where(file_store: ::DependencyProxy::FileUploader::Store::LOCAL) }
mount_file_store_uploader DependencyProxy::FileUploader mount_file_store_uploader DependencyProxy::FileUploader
def self.total_size def self.total_size
...@@ -24,3 +26,5 @@ class DependencyProxy::Blob < ApplicationRecord ...@@ -24,3 +26,5 @@ class DependencyProxy::Blob < ApplicationRecord
find_or_initialize_by(file_name: file_name) find_or_initialize_by(file_name: file_name)
end end
end end
DependencyProxy::Blob.prepend_mod_with('DependencyProxy::Blob')
...@@ -17,6 +17,7 @@ class DependencyProxy::Manifest < ApplicationRecord ...@@ -17,6 +17,7 @@ class DependencyProxy::Manifest < ApplicationRecord
validates :digest, presence: true validates :digest, presence: true
scope :order_id_desc, -> { reorder(id: :desc) } scope :order_id_desc, -> { reorder(id: :desc) }
scope :with_files_stored_locally, -> { where(file_store: ::DependencyProxy::FileUploader::Store::LOCAL) }
mount_file_store_uploader DependencyProxy::FileUploader mount_file_store_uploader DependencyProxy::FileUploader
...@@ -24,3 +25,5 @@ class DependencyProxy::Manifest < ApplicationRecord ...@@ -24,3 +25,5 @@ class DependencyProxy::Manifest < ApplicationRecord
find_by(file_name: file_name) || find_by(digest: digest) find_by(file_name: file_name) || find_by(digest: digest)
end end
end end
DependencyProxy::Manifest.prepend_mod_with('DependencyProxy::Manifest')
...@@ -526,7 +526,7 @@ To migrate existing local data to object storage see the following guides: ...@@ -526,7 +526,7 @@ To migrate existing local data to object storage see the following guides:
- [Uploads](raketasks/uploads/migrate.md#migrate-to-object-storage) - [Uploads](raketasks/uploads/migrate.md#migrate-to-object-storage)
- [Merge request diffs](merge_request_diffs.md#using-object-storage) - [Merge request diffs](merge_request_diffs.md#using-object-storage)
- [Packages](packages/index.md#migrating-local-packages-to-object-storage) (optional feature) - [Packages](packages/index.md#migrating-local-packages-to-object-storage) (optional feature)
- Dependency Proxy - [migration not yet supported](https://gitlab.com/gitlab-org/gitlab/-/issues/343064) - [Dependency Proxy](packages/dependency_proxy.md#migrate-local-dependency-proxy-blobs-and-manifests-to-object-storage)
- [Terraform state files](terraform_state.md#migrate-to-object-storage) - [Terraform state files](terraform_state.md#migrate-to-object-storage)
- [Pages content](pages/index.md#migrate-pages-deployments-to-object-storage) - [Pages content](pages/index.md#migrate-pages-deployments-to-object-storage)
......
...@@ -199,20 +199,52 @@ This section describes the earlier configuration format. ...@@ -199,20 +199,52 @@ This section describes the earlier configuration format.
1. [Restart GitLab](../restart_gitlab.md#installations-from-source "How to restart GitLab") for the changes to take effect. 1. [Restart GitLab](../restart_gitlab.md#installations-from-source "How to restart GitLab") for the changes to take effect.
### Migrate from local storage to object storage #### Migrate local Dependency Proxy blobs and manifests to object storage
GitLab currently [doesn't support](https://gitlab.com/gitlab-org/gitlab/-/issues/343064) After [configuring object storage](#using-object-storage),
migrating the existing cache to object storage. use the following task to migrate existing Dependency Proxy blobs and manifests from local storage
to remote storage. The processing is done in a background worker and requires no downtime.
To switch from local storage to object storage: For Omnibus GitLab:
1. [Reconfigure GitLab to use object storage](#using-object-storage). ```shell
1. Purge the dependency proxy cache (on both local and object storage) sudo gitlab-rake "gitlab:dependency_proxy:migrate"
using the [Dependency Proxy API](../../api/dependency_proxy.md#purge-the-dependency-proxy-for-a-group). ```
For installations from source:
```shell
RAILS_ENV=production sudo -u git -H bundle exec rake gitlab:dependency_proxy:migrate
```
You can optionally track progress and verify that all packages migrated successfully using the
[PostgreSQL console](https://docs.gitlab.com/omnibus/settings/database.html#connecting-to-the-bundled-postgresql-database):
- For Omnibus GitLab instances: `sudo gitlab-rails dbconsole`
- For installations from source: `sudo -u git -H psql -d gitlabhq_production`
Verify that `objectstg` (where `file_store = '2'`) has the count of all Dependency Proxy blobs and
manifests for each respective query:
Object storage now caches container images when they're requested. This also ```shell
[reduces the size of dependency proxy storage](../../user/packages/dependency_proxy/reduce_dependency_proxy_storage.md) gitlabhq_production=# SELECT count(*) AS total, sum(case when file_store = '1' then 1 else 0 end) AS filesystem, sum(case when file_store = '2' then 1 else 0 end) AS objectstg FROM dependency_proxy_blobs;
by removing unused images.
total | filesystem | objectstg
------+------------+-----------
22 | 0 | 22
gitlabhq_production=# SELECT count(*) AS total, sum(case when file_store = '1' then 1 else 0 end) AS filesystem, sum(case when file_store = '2' then 1 else 0 end) AS objectstg FROM dependency_proxy_manifests;
total | filesystem | objectstg
------+------------+-----------
10 | 0 | 10
```
Verify that there are no files on disk in the `packages` folder:
```shell
sudo find /var/opt/gitlab/gitlab-rails/shared/dependency_proxy -type f | grep -v tmp | wc -l
```
## Disabling Authentication ## Disabling Authentication
......
# frozen_string_literal: true
module EE
module DependencyProxy
module Blob
extend ActiveSupport::Concern
def log_geo_deleted_event
# Keep empty for now. Should be addressed in future
# by https://gitlab.com/gitlab-org/gitlab/-/issues/259694
end
end
end
end
# frozen_string_literal: true
module EE
module DependencyProxy
module Manifest
extend ActiveSupport::Concern
def log_geo_deleted_event
# Keep empty for now. Should be addressed in future
# by https://gitlab.com/gitlab-org/gitlab/-/issues/259694
end
end
end
end
# frozen_string_literal: true
require 'logger'
desc "GitLab | Dependency Proxy | Migrate dependency proxy files to remote storage"
namespace :gitlab do
namespace :dependency_proxy do
task migrate: :environment do
logger = Logger.new($stdout)
logger.info('Starting transfer of dependency proxy files to object storage')
unless ::DependencyProxy::FileUploader.object_store_enabled?
raise 'Object store is disabled for dependency proxy feature'
end
::DependencyProxy::Blob.with_files_stored_locally.find_each(batch_size: 10) do |blob_file|
blob_file.file.migrate!(::DependencyProxy::FileUploader::Store::REMOTE)
logger.info("Transferred dependency proxy blob file #{blob_file.id} of size #{blob_file.size.to_i.bytes} to object storage")
rescue StandardError => e
logger.error("Failed to transfer dependency proxy blob file #{blob_file.id} with error: #{e.message}")
end
::DependencyProxy::Manifest.with_files_stored_locally.find_each(batch_size: 10) do |manifest_file|
manifest_file.file.migrate!(::DependencyProxy::FileUploader::Store::REMOTE)
logger.info("Transferred dependency proxy manifest file #{manifest_file.id} of size #{manifest_file.size.to_i.bytes} to object storage")
rescue StandardError => e
logger.error("Failed to transfer dependency proxy manifest file #{manifest_file.id} with error: #{e.message}")
end
end
end
end
# frozen_string_literal: true
require 'rake_helper'
RSpec.describe 'gitlab:dependency_proxy namespace rake task', :silence_stdout do
before :all do
Rake.application.rake_require 'tasks/gitlab/dependency_proxy/migrate'
end
describe 'migrate' do
let(:local) { ObjectStorage::Store::LOCAL }
let(:remote) { ObjectStorage::Store::REMOTE }
let!(:blob) { create(:dependency_proxy_blob) }
let!(:manifest) { create(:dependency_proxy_manifest) }
def dependency_proxy_migrate
run_rake_task('gitlab:dependency_proxy:migrate')
end
context 'object storage disabled' do
before do
stub_dependency_proxy_object_storage(enabled: false)
end
it "doesn't migrate files" do
expect { dependency_proxy_migrate }.to raise_error('Object store is disabled for dependency proxy feature')
end
end
context 'object storage enabled' do
before do
stub_dependency_proxy_object_storage
end
it 'migrates local file to object storage' do
expect { dependency_proxy_migrate }.to change { blob.reload.file_store }.from(local).to(remote)
.and change { manifest.reload.file_store }.from(local).to(remote)
end
end
context 'an error is raised while migrating' do
let(:blob_error) { 'Failed to transfer dependency proxy blob file' }
let(:manifest_error) { 'Failed to transfer dependency proxy manifest file' }
let!(:blob_non_existent) { create(:dependency_proxy_blob) }
let!(:manifest_non_existent) { create(:dependency_proxy_manifest) }
before do
stub_dependency_proxy_object_storage
blob_non_existent.file.file.delete
manifest_non_existent.file.file.delete
end
it 'fails to migrate a local file that does not exist' do
expect { dependency_proxy_migrate }.to output(include(blob_error, manifest_error)).to_stdout
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment