Commit ca0ae617 authored by Michael Kozono's avatar Michael Kozono

Merge branch 'terraform-states-migrate-task' into 'master'

Add rake task for migrating Terraform files to object storage

See merge request gitlab-org/gitlab!50740
parents d99a5c53 ddc31df1
...@@ -9,6 +9,8 @@ module Terraform ...@@ -9,6 +9,8 @@ module Terraform
belongs_to :build, class_name: 'Ci::Build', optional: true, foreign_key: :ci_build_id belongs_to :build, class_name: 'Ci::Build', optional: true, foreign_key: :ci_build_id
scope :ordered_by_version_desc, -> { order(version: :desc) } scope :ordered_by_version_desc, -> { order(version: :desc) }
scope :with_files_stored_locally, -> { where(file_store: Terraform::StateUploader::Store::LOCAL) }
scope :preload_state, -> { includes(:terraform_state) }
default_value_for(:file_store) { StateUploader.default_store } default_value_for(:file_store) { StateUploader.default_store }
......
...@@ -6,6 +6,10 @@ module Terraform ...@@ -6,6 +6,10 @@ module Terraform
storage_options Gitlab.config.terraform_state storage_options Gitlab.config.terraform_state
# TODO: Remove this line
# See https://gitlab.com/gitlab-org/gitlab/-/issues/232917
alias_method :upload, :model
delegate :terraform_state, :project_id, to: :model delegate :terraform_state, :project_id, to: :model
# Use Lockbox to encrypt/decrypt the stored file (registers CarrierWave callbacks) # Use Lockbox to encrypt/decrypt the stored file (registers CarrierWave callbacks)
......
---
title: Add rake task to migrate Terraform states to object storage
merge_request: 50740
author:
type: added
...@@ -100,6 +100,11 @@ See [the available connection settings for different providers](object_storage.m ...@@ -100,6 +100,11 @@ See [the available connection settings for different providers](object_storage.m
``` ```
1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect.
1. Migrate any existing local states to the object storage (GitLab 13.9 and later):
```shell
gitlab-rake gitlab:terraform_states:migrate
```
**In installations from source:** **In installations from source:**
...@@ -120,3 +125,8 @@ See [the available connection settings for different providers](object_storage.m ...@@ -120,3 +125,8 @@ See [the available connection settings for different providers](object_storage.m
``` ```
1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. 1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect.
1. Migrate any existing local states to the object storage (GitLab 13.9 and later):
```shell
sudo -u git -H bundle exec rake gitlab:terraform_states:migrate RAILS_ENV=production
```
...@@ -9,7 +9,6 @@ module EE ...@@ -9,7 +9,6 @@ module EE
include ::Gitlab::Geo::ReplicableModel include ::Gitlab::Geo::ReplicableModel
with_replicator Geo::TerraformStateVersionReplicator with_replicator Geo::TerraformStateVersionReplicator
scope :with_files_stored_locally, -> { where(file_store: ::ObjectStorage::Store::LOCAL) }
scope :project_id_in, ->(ids) { joins(:terraform_state).where('terraform_states.project_id': ids) } scope :project_id_in, ->(ids) { joins(:terraform_state).where('terraform_states.project_id': ids) }
end end
......
...@@ -9,22 +9,6 @@ RSpec.describe Terraform::StateVersion do ...@@ -9,22 +9,6 @@ RSpec.describe Terraform::StateVersion do
let_it_be(:group) { create(:group) } let_it_be(:group) { create(:group) }
let_it_be(:project) { create(:project, group: group) } let_it_be(:project) { create(:project, group: group) }
describe '.with_files_stored_locally' do
it 'includes states with local storage' do
create_list(:terraform_state_version, 5)
expect(described_class.with_files_stored_locally).to have_attributes(count: 5)
end
it 'excludes states without local storage' do
stub_terraform_state_object_storage
create_list(:terraform_state_version, 5)
expect(described_class.with_files_stored_locally).to have_attributes(count: 0)
end
end
describe '.replicables_for_current_secondary' do describe '.replicables_for_current_secondary' do
where(:selective_sync_enabled, :object_storage_sync_enabled, :terraform_object_storage_enabled, :synced_states) do where(:selective_sync_enabled, :object_storage_sync_enabled, :terraform_object_storage_enabled, :synced_states) do
true | true | true | 5 true | true | true | 5
......
# frozen_string_literal: true
module Gitlab
module Terraform
class StateMigrationHelper
class << self
def migrate_to_remote_storage(&block)
migrate_in_batches(
::Terraform::StateVersion.with_files_stored_locally.preload_state,
::Terraform::StateUploader::Store::REMOTE,
&block
)
end
private
def batch_size
ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i
end
def migrate_in_batches(versions, store, &block)
versions.find_each(batch_size: batch_size) do |version| # rubocop:disable CodeReuse/ActiveRecord
version.file.migrate!(store)
yield version if block_given?
end
end
end
end
end
end
# frozen_string_literal: true
require 'logger'
desc "GitLab | Terraform | Migrate Terraform states to remote storage"
namespace :gitlab do
namespace :terraform_states do
task migrate: :environment do
logger = Logger.new(STDOUT)
logger.info('Starting transfer of Terraform states to object storage')
begin
Gitlab::Terraform::StateMigrationHelper.migrate_to_remote_storage do |state_version|
message = "Transferred Terraform state version ID #{state_version.id} (#{state_version.terraform_state.name}/#{state_version.version}) to object storage"
logger.info(message)
end
rescue => e
logger.error("Failed to migrate: #{e.message}")
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Terraform::StateMigrationHelper do
before do
stub_terraform_state_object_storage
end
describe '.migrate_to_remote_storage' do
let!(:local_version) { create(:terraform_state_version, file_store: Terraform::StateUploader::Store::LOCAL) }
subject { described_class.migrate_to_remote_storage }
it 'migrates remote files to remote storage' do
subject
expect(local_version.reload.file_store).to eq(Terraform::StateUploader::Store::REMOTE)
end
end
end
...@@ -24,6 +24,24 @@ RSpec.describe Terraform::StateVersion do ...@@ -24,6 +24,24 @@ RSpec.describe Terraform::StateVersion do
it { expect(subject.map(&:version)).to eq(versions.sort.reverse) } it { expect(subject.map(&:version)).to eq(versions.sort.reverse) }
end end
describe '.with_files_stored_locally' do
subject { described_class.with_files_stored_locally }
it 'includes states with local storage' do
create_list(:terraform_state_version, 5)
expect(subject).to have_attributes(count: 5)
end
it 'excludes states without local storage' do
stub_terraform_state_object_storage
create_list(:terraform_state_version, 5)
expect(subject).to have_attributes(count: 0)
end
end
end end
context 'file storage' do context 'file storage' do
......
# frozen_string_literal: true
require 'rake_helper'
RSpec.describe 'gitlab:terraform_states' do
let_it_be(:version) { create(:terraform_state_version) }
let(:logger) { instance_double(Logger) }
let(:helper) { double }
before(:all) do
Rake.application.rake_require 'tasks/gitlab/terraform/migrate'
end
before do
allow(Logger).to receive(:new).with(STDOUT).and_return(logger)
end
describe 'gitlab:terraform_states:migrate' do
subject { run_rake_task('gitlab:terraform_states:migrate') }
it 'invokes the migration helper to move files to object storage' do
expect(Gitlab::Terraform::StateMigrationHelper).to receive(:migrate_to_remote_storage).and_yield(version)
expect(logger).to receive(:info).with('Starting transfer of Terraform states to object storage')
expect(logger).to receive(:info).with(/Transferred Terraform state version ID #{version.id}/)
subject
end
context 'an error is raised while migrating' do
let(:error_message) { 'Something went wrong' }
before do
allow(Gitlab::Terraform::StateMigrationHelper).to receive(:migrate_to_remote_storage).and_raise(StandardError, error_message)
end
it 'logs the error' do
expect(logger).to receive(:info).with('Starting transfer of Terraform states to object storage')
expect(logger).to receive(:error).with("Failed to migrate: #{error_message}")
subject
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment