Merge branch 'hchouraria-backup-terraform-state-files' into 'master'

Add Terraform States to Backup and Restore tasks

See merge request gitlab-org/gitlab!63347

.gitlab/ci/rails.gitlab-ci.yml

@@ -463,7 +463,7 @@ db:backup_and_restore:
   script:
     - . scripts/prepare_build.sh
     - bundle exec rake db:drop db:create db:structure:load db:seed_fu
-    - mkdir -p tmp/tests/public/uploads tmp/tests/{artifacts,pages,lfs-objects,registry}
+    - mkdir -p tmp/tests/public/uploads tmp/tests/{artifacts,pages,lfs-objects,terraform_state,registry}
     - bundle exec rake gitlab:backup:create
     - date
     - bundle exec rake gitlab:backup:restore

db/fixtures/development/31_terraform_state.rb

+# frozen_string_literal: true
+
+TERRAFORM_FILE_VERSION = 1
+
+# Create sample terraform states in existing projects
+Gitlab::Seeder.quiet do
+  tfdata = {terraform_version: '0.14.1'}.to_json
+
+  Project.not_mass_generated.find_each do |project|
+    # Create as the project's creator
+    user = project.creator
+    # Set a build job source, if one exists for the project
+    build = project.builds.last
+
+    remote_state_handler = ::Terraform::RemoteStateHandler.new(project, user, name: project.path, lock_id: nil)
+
+    remote_state_handler.handle_with_lock do |state|
+      # Upload a file if a version does not already exist
+      state.update_file!(CarrierWaveStringFile.new(tfdata), version: TERRAFORM_FILE_VERSION, build: build) if state.latest_version.nil?
+    end
+
+    # rubocop:disable Rails/Output
+    print '.'
+    # rubocop:enable Rails/Output
+  end
+end

doc/raketasks/backup_restore.md

@@ -58,6 +58,7 @@ including:
 - CI/CD job output logs
 - CI/CD job artifacts
 - LFS objects
+- Terraform states
 - Container Registry images
 - GitLab Pages content
 - Snippets
@@ -65,7 +66,6 @@ including:
 
 Backups do not include:
 
-- [Terraform state files](../administration/terraform_state.md)
 - [Package registry files](../administration/packages/index.md)
 - [Mattermost data](https://docs.mattermost.com/administration/config-settings.html#file-storage)
 
@@ -276,6 +276,7 @@ You can exclude specific directories from the backup by adding the environment v
 - `builds` (CI job output logs)
 - `artifacts` (CI job artifacts)
 - `lfs` (LFS objects)
+- `terraform_state` (Terraform states)
 - `registry` (Container Registry images)
 - `pages` (Pages content)
 - `repositories` (Git repositories data)

lib/backup/manager.rb

@@ -2,7 +2,7 @@
 
 module Backup
   class Manager
-    ARCHIVES_TO_BACKUP = %w[uploads builds artifacts pages lfs registry].freeze
+    ARCHIVES_TO_BACKUP = %w[uploads builds artifacts pages lfs terraform_state registry].freeze
     FOLDERS_TO_BACKUP = %w[repositories db].freeze
     FILE_NAME_SUFFIX = '_gitlab_backup.tar'
 

lib/backup/terraform_state.rb

+# frozen_string_literal: true
+
+module Backup
+  class TerraformState < Backup::Files
+    attr_reader :progress
+
+    def initialize(progress)
+      @progress = progress
+
+      super('terraform_state', Settings.terraform_state.storage_path, excludes: ['tmp'])
+    end
+  end
+end

lib/tasks/gitlab/backup.rake

@@ -16,6 +16,7 @@ namespace :gitlab do
       Rake::Task['gitlab:backup:artifacts:create'].invoke
       Rake::Task['gitlab:backup:pages:create'].invoke
       Rake::Task['gitlab:backup:lfs:create'].invoke
+      Rake::Task['gitlab:backup:terraform_state:create'].invoke
       Rake::Task['gitlab:backup:registry:create'].invoke
 
       backup = Backup::Manager.new(progress)
@@ -83,6 +84,7 @@ namespace :gitlab do
       Rake::Task['gitlab:backup:artifacts:restore'].invoke unless backup.skipped?('artifacts')
       Rake::Task['gitlab:backup:pages:restore'].invoke unless backup.skipped?('pages')
       Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs')
+      Rake::Task['gitlab:backup:terraform_state:restore'].invoke unless backup.skipped?('terraform_state')
       Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry')
       Rake::Task['gitlab:shell:setup'].invoke
       Rake::Task['cache:clear'].invoke
@@ -254,6 +256,25 @@ namespace :gitlab do
       end
     end
 
+    namespace :terraform_state do
+      task create: :gitlab_environment do
+        puts_time "Dumping terraform states ... ".color(:blue)
+
+        if ENV["SKIP"] && ENV["SKIP"].include?("terraform_state")
+          puts_time "[SKIPPED]".color(:cyan)
+        else
+          Backup::TerraformState.new(progress).dump
+          puts_time "done".color(:green)
+        end
+      end
+
+      task restore: :gitlab_environment do
+        puts_time "Restoring terraform states ... ".color(:blue)
+        Backup::TerraformState.new(progress).restore
+        puts_time "done".color(:green)
+      end
+    end
+
     namespace :registry do
       task create: :gitlab_environment do
         puts_time "Dumping container registry images ... ".color(:blue)

spec/lib/backup/artifacts_spec.rb

@@ -12,7 +12,7 @@ RSpec.describe Backup::Artifacts do
       Dir.mktmpdir do |tmpdir|
         allow(JobArtifactUploader).to receive(:root) { "#{tmpdir}" }
 
-        expect(backup.app_files_dir).to eq("#{tmpdir}")
+        expect(backup.app_files_dir).to eq("#{File.realpath(tmpdir)}")
       end
     end
   end

spec/lib/backup/lfs_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Backup::Lfs do
+  let(:progress) { StringIO.new }
+
+  subject(:backup) { described_class.new(progress) }
+
+  describe '#dump' do
+    before do
+      allow(File).to receive(:realpath).and_call_original
+      allow(File).to receive(:realpath).with('/var/lfs-objects').and_return('/var/lfs-objects')
+      allow(File).to receive(:realpath).with('/var/lfs-objects/..').and_return('/var')
+      allow(Settings.lfs).to receive(:storage_path).and_return('/var/lfs-objects')
+    end
+
+    it 'uses the correct lfs dir in tar command', :aggregate_failures do
+      expect(backup.app_files_dir).to eq('/var/lfs-objects')
+      expect(backup).to receive(:tar).and_return('blabla-tar')
+      expect(backup).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found -C /var/lfs-objects -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], ''])
+      expect(backup).to receive(:pipeline_succeeded?).and_return(true)
+
+      backup.dump
+    end
+  end
+end

spec/lib/backup/manager_spec.rb

@@ -15,7 +15,7 @@ RSpec.describe Backup::Manager do
   end
 
   describe '#pack' do
-    let(:expected_backup_contents) { %w(repositories db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz backup_information.yml) }
+    let(:expected_backup_contents) { %w(repositories db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz terraform_state.tar.gz backup_information.yml) }
     let(:tar_file) { '1546300800_2019_01_01_12.3_gitlab_backup.tar' }
     let(:tar_system_options) { { out: [tar_file, 'w', Gitlab.config.backup.archive_permissions] } }
     let(:tar_cmdline) { ['tar', '-cf', '-', *expected_backup_contents, tar_system_options] }
@@ -57,7 +57,7 @@ RSpec.describe Backup::Manager do
     end
 
     context 'when skipped is set in backup_information.yml' do
-      let(:expected_backup_contents) { %w{db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz backup_information.yml} }
+      let(:expected_backup_contents) { %w{db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz terraform_state.tar.gz backup_information.yml} }
       let(:backup_information) do
         {
           backup_created_at: Time.zone.parse('2019-01-01'),
@@ -74,7 +74,7 @@ RSpec.describe Backup::Manager do
     end
 
     context 'when a directory does not exist' do
-      let(:expected_backup_contents) { %w{db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz backup_information.yml} }
+      let(:expected_backup_contents) { %w{db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz terraform_state.tar.gz backup_information.yml} }
 
       before do
         expect(Dir).to receive(:exist?).with(File.join(Gitlab.config.backup.path, 'repositories')).and_return(false)

spec/lib/backup/terraform_state_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Backup::TerraformState do
+  let(:progress) { StringIO.new }
+
+  subject(:backup) { described_class.new(progress) }
+
+  describe '#dump' do
+    before do
+      allow(File).to receive(:realpath).and_call_original
+      allow(File).to receive(:realpath).with('/var/terraform_state').and_return('/var/terraform_state')
+      allow(File).to receive(:realpath).with('/var/terraform_state/..').and_return('/var')
+      allow(Settings.terraform_state).to receive(:storage_path).and_return('/var/terraform_state')
+    end
+
+    it 'uses the correct storage dir in tar command and excludes tmp', :aggregate_failures do
+      expect(backup.app_files_dir).to eq('/var/terraform_state')
+      expect(backup).to receive(:tar).and_return('blabla-tar')
+      expect(backup).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./tmp -C /var/terraform_state -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], ''])
+      expect(backup).to receive(:pipeline_succeeded?).and_return(true)
+
+      backup.dump
+    end
+  end
+end

spec/lib/backup/uploads_spec.rb

@@ -14,13 +14,14 @@ RSpec.describe Backup::Uploads do
 
         allow(Gitlab.config.uploads).to receive(:storage_path) { tmpdir }
 
-        expect(backup.app_files_dir).to eq("#{tmpdir}/uploads")
+        expect(backup.app_files_dir).to eq("#{File.realpath(tmpdir)}/uploads")
       end
     end
   end
 
   describe '#dump' do
     before do
+      allow(File).to receive(:realpath).and_call_original
       allow(File).to receive(:realpath).with('/var/uploads').and_return('/var/uploads')
       allow(File).to receive(:realpath).with('/var/uploads/..').and_return('/var')
       allow(Gitlab.config.uploads).to receive(:storage_path) { '/var' }

spec/support/helpers/test_env.rb

@@ -148,6 +148,8 @@ module TestEnv
     FileUtils.mkdir_p(backup_path)
     FileUtils.mkdir_p(pages_path)
     FileUtils.mkdir_p(artifacts_path)
+    FileUtils.mkdir_p(lfs_path)
+    FileUtils.mkdir_p(terraform_state_path)
   end
 
   def setup_gitlab_shell
@@ -414,6 +416,14 @@ module TestEnv
     Gitlab.config.artifacts.storage_path
   end
 
+  def lfs_path
+    Gitlab.config.lfs.storage_path
+  end
+
+  def terraform_state_path
+    Gitlab.config.terraform_state.storage_path
+  end
+
   # When no cached assets exist, manually hit the root path to create them
   #
   # Otherwise they'd be created by the first test, often timing out and

spec/tasks/gitlab/backup_rake_spec.rb

@@ -4,6 +4,7 @@ require 'rake_helper'
 
 RSpec.describe 'gitlab:app namespace rake task', :delete do
   let(:enable_registry) { true }
+  let(:backup_types) { %w{db repo uploads builds artifacts pages lfs terraform_state registry} }
 
   def tars_glob
     Dir.glob(File.join(Gitlab.config.backup.path, '*_gitlab_backup.tar'))
@@ -14,7 +15,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
   end
 
   def backup_files
-    %w(backup_information.yml artifacts.tar.gz builds.tar.gz lfs.tar.gz pages.tar.gz)
+    %w(backup_information.yml artifacts.tar.gz builds.tar.gz lfs.tar.gz terraform_state.tar.gz pages.tar.gz)
   end
 
   def backup_directories
@@ -47,7 +48,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
   end
 
   def reenable_backup_sub_tasks
-    %w{db repo uploads builds artifacts pages lfs registry}.each do |subtask|
+    backup_types.each do |subtask|
       Rake::Task["gitlab:backup:#{subtask}:create"].reenable
     end
   end
@@ -71,14 +72,9 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
           allow(YAML).to receive(:load_file)
             .and_return({ gitlab_version: gitlab_version })
           expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
-          expect(Rake::Task['gitlab:backup:db:restore']).to receive(:invoke)
-          expect(Rake::Task['gitlab:backup:repo:restore']).to receive(:invoke)
-          expect(Rake::Task['gitlab:backup:builds:restore']).to receive(:invoke)
-          expect(Rake::Task['gitlab:backup:uploads:restore']).to receive(:invoke)
-          expect(Rake::Task['gitlab:backup:artifacts:restore']).to receive(:invoke)
-          expect(Rake::Task['gitlab:backup:pages:restore']).to receive(:invoke)
-          expect(Rake::Task['gitlab:backup:lfs:restore']).to receive(:invoke)
-          expect(Rake::Task['gitlab:backup:registry:restore']).to receive(:invoke)
+          backup_types.each do |subtask|
+            expect(Rake::Task["gitlab:backup:#{subtask}:restore"]).to receive(:invoke)
+          end
           expect(Rake::Task['gitlab:shell:setup']).to receive(:invoke)
         end
 
@@ -95,7 +91,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
     context 'when the restore directory is not empty' do
       before do
         # We only need a backup of the repositories for this test
-        stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry')
+        stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,terraform_state,registry')
 
         create(:project, :repository)
       end
@@ -139,11 +135,9 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         expect(Rake::Task['gitlab:backup:artifacts:restore']).to receive(:invoke)
         expect(Rake::Task['gitlab:backup:pages:restore']).to receive(:invoke)
         expect(Rake::Task['gitlab:backup:lfs:restore']).to receive(:invoke)
+        expect(Rake::Task['gitlab:backup:terraform_state:restore']).to receive(:invoke)
         expect(Rake::Task['gitlab:backup:registry:restore']).to receive(:invoke)
         expect(Rake::Task['gitlab:shell:setup']).to receive(:invoke)
-
-        # We only need a backup of the repositories for this test
-        stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry')
       end
 
       it 'restores the data' do
@@ -202,10 +196,8 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
       end
 
       context 'specific backup tasks' do
-        let(:task_list) { %w(db repo uploads builds artifacts pages lfs registry) }
-
         it 'prints a progress message to stdout' do
-          task_list.each do |task|
+          backup_types.each do |task|
             expect { run_rake_task("gitlab:backup:#{task}:create") }.to output(/Dumping /).to_stdout_from_any_process
           end
         end
@@ -219,10 +211,11 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
           expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping artifacts ... ")
           expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping pages ... ")
           expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping lfs objects ... ")
+          expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping terraform states ... ")
           expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping container registry images ... ")
-          expect(Gitlab::BackupLogger).to receive(:info).with(message: "done").exactly(7).times
+          expect(Gitlab::BackupLogger).to receive(:info).with(message: "done").exactly(8).times
 
-          task_list.each do |task|
+          backup_types.each do |task|
             run_rake_task("gitlab:backup:#{task}:create")
           end
         end
@@ -255,7 +248,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
 
         tar_contents, exit_status = Gitlab::Popen.popen(
-          %W{tar -tvf #{backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz registry.tar.gz}
+          %W{tar -tvf #{backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz terraform_state.tar.gz registry.tar.gz}
         )
 
         expect(exit_status).to eq(0)
@@ -266,6 +259,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         expect(tar_contents).to match('artifacts.tar.gz')
         expect(tar_contents).to match('pages.tar.gz')
         expect(tar_contents).to match('lfs.tar.gz')
+        expect(tar_contents).to match('terraform_state.tar.gz')
         expect(tar_contents).to match('registry.tar.gz')
         expect(tar_contents).not_to match(%r{^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|pages.tar.gz|artifacts.tar.gz|registry.tar.gz)/$})
       end
@@ -274,7 +268,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
 
         temp_dirs = Dir.glob(
-          File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts,pages,lfs,registry}')
+          File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts,pages,lfs,terraform_state,registry}')
         )
 
         expect(temp_dirs).to be_empty
@@ -304,7 +298,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
 
       before do
         # We only need a backup of the repositories for this test
-        stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry')
+        stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,terraform_state,registry')
         stub_storage_settings( second_storage_name => {
           'gitaly_address' => Gitlab.config.repositories.storages.default.gitaly_address,
           'path' => TestEnv::SECOND_STORAGE_PATH
@@ -378,7 +372,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
     context 'concurrency settings' do
       before do
         # We only need a backup of the repositories for this test
-        stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry')
+        stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,terraform_state,registry')
 
         create(:project, :repository)
       end
@@ -425,31 +419,33 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
   end
   # backup_create task
 
-  describe "Skipping items" do
+  describe "Skipping items in a backup" do
     before do
-      stub_env('SKIP', 'repositories,uploads')
+      stub_env('SKIP', 'an-unknown-type,repositories,uploads,anotherunknowntype')
 
       create(:project, :repository)
     end
 
-    it "does not contain skipped item" do
+    it "does not contain repositories and uploads" do
       expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
 
       tar_contents, _exit_status = Gitlab::Popen.popen(
-        %W{tar -tvf #{backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz registry.tar.gz}
+        %W{tar -tvf #{backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz terraform_state.tar.gz registry.tar.gz}
       )
 
       expect(tar_contents).to match('db/')
-      expect(tar_contents).to match('uploads.tar.gz')
+      expect(tar_contents).to match('uploads.tar.gz: Not found in archive')
       expect(tar_contents).to match('builds.tar.gz')
       expect(tar_contents).to match('artifacts.tar.gz')
       expect(tar_contents).to match('lfs.tar.gz')
+      expect(tar_contents).to match('terraform_state.tar.gz')
       expect(tar_contents).to match('pages.tar.gz')
       expect(tar_contents).to match('registry.tar.gz')
       expect(tar_contents).not_to match('repositories/')
+      expect(tar_contents).to match('repositories: Not found in archive')
     end
 
-    it 'does not invoke repositories restore' do
+    it 'does not invoke restore of repositories and uploads' do
       expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
 
       allow(Rake::Task['gitlab:shell:setup'])
@@ -463,6 +459,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
       expect(Rake::Task['gitlab:backup:artifacts:restore']).to receive :invoke
       expect(Rake::Task['gitlab:backup:pages:restore']).to receive :invoke
       expect(Rake::Task['gitlab:backup:lfs:restore']).to receive :invoke
+      expect(Rake::Task['gitlab:backup:terraform_state:restore']).to receive :invoke
       expect(Rake::Task['gitlab:backup:registry:restore']).to receive :invoke
       expect(Rake::Task['gitlab:shell:setup']).to receive :invoke
       expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout_from_any_process
@@ -488,6 +485,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         'builds.tar.gz',
         'artifacts.tar.gz',
         'lfs.tar.gz',
+        'terraform_state.tar.gz',
         'pages.tar.gz',
         'registry.tar.gz',
         'repositories'
@@ -501,14 +499,9 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         .to receive(:invoke).and_return(true)
 
       expect(Rake::Task['gitlab:db:drop_tables']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:db:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:repo:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:uploads:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:builds:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:artifacts:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:pages:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:lfs:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:registry:restore']).to receive :invoke
+      backup_types.each do |subtask|
+        expect(Rake::Task["gitlab:backup:#{subtask}:restore"]).to receive :invoke
+      end
       expect(Rake::Task['gitlab:shell:setup']).to receive :invoke
       expect { run_rake_task("gitlab:backup:restore") }.to output.to_stdout_from_any_process
     end