Commit 7b047094 authored by Douwe Maan's avatar Douwe Maan

Merge branch '42593-restore-rake-task' into 'master'

Tweak restore rake task to provide an option to set alternate temporary backup directory

Closes #42593

See merge request gitlab-org/gitlab-ce!17516
parents 1246b9c1 9d27fb29
...@@ -5,9 +5,5 @@ module Backup ...@@ -5,9 +5,5 @@ module Backup
def initialize def initialize
super('artifacts', JobArtifactUploader.root) super('artifacts', JobArtifactUploader.root)
end end
def create_files_dir
Dir.mkdir(app_files_dir, 0700)
end
end end
end end
...@@ -5,9 +5,5 @@ module Backup ...@@ -5,9 +5,5 @@ module Backup
def initialize def initialize
super('builds', Settings.gitlab_ci.builds_path) super('builds', Settings.gitlab_ci.builds_path)
end end
def create_files_dir
Dir.mkdir(app_files_dir, 0700)
end
end end
end end
require 'open3' require 'open3'
require_relative 'helper'
module Backup module Backup
class Files class Files
include Backup::Helper
attr_reader :name, :app_files_dir, :backup_tarball, :files_parent_dir attr_reader :name, :app_files_dir, :backup_tarball, :files_parent_dir
def initialize(name, app_files_dir) def initialize(name, app_files_dir)
...@@ -35,15 +38,22 @@ module Backup ...@@ -35,15 +38,22 @@ module Backup
def restore def restore
backup_existing_files_dir backup_existing_files_dir
create_files_dir
run_pipeline!([%w(gzip -cd), %W(tar -C #{app_files_dir} -xf -)], in: backup_tarball) run_pipeline!([%w(gzip -cd), %W(tar --unlink-first --recursive-unlink -C #{app_files_dir} -xf -)], in: backup_tarball)
end end
def backup_existing_files_dir def backup_existing_files_dir
timestamped_files_path = File.join(files_parent_dir, "#{name}.#{Time.now.to_i}") timestamped_files_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}.#{Time.now.to_i}")
if File.exist?(app_files_dir) if File.exist?(app_files_dir)
FileUtils.mv(app_files_dir, File.expand_path(timestamped_files_path)) # Move all files in the existing repos directory except . and .. to
# repositories.old.<timestamp> directory
FileUtils.mkdir_p(timestamped_files_path, mode: 0700)
files = Dir.glob(File.join(app_files_dir, "*"), File::FNM_DOTMATCH) - [File.join(app_files_dir, "."), File.join(app_files_dir, "..")]
begin
FileUtils.mv(files, timestamped_files_path)
rescue Errno::EACCES
access_denied_error(app_files_dir)
end
end end
end end
......
module Backup
module Helper
def access_denied_error(path)
message = <<~EOS
### NOTICE ###
As part of restore, the task tried to move existing content from #{path}.
However, it seems that directory contains files/folders that are not owned
by the user #{Gitlab.config.gitlab.user}. To proceed, please move the files
or folders inside #{path} to a secure location so that #{path} is empty and
run restore task again.
EOS
raise message
end
end
end
...@@ -5,9 +5,5 @@ module Backup ...@@ -5,9 +5,5 @@ module Backup
def initialize def initialize
super('lfs', Settings.lfs.storage_path) super('lfs', Settings.lfs.storage_path)
end end
def create_files_dir
Dir.mkdir(app_files_dir, 0700)
end
end end
end end
...@@ -5,9 +5,5 @@ module Backup ...@@ -5,9 +5,5 @@ module Backup
def initialize def initialize
super('pages', Gitlab.config.pages.path) super('pages', Gitlab.config.pages.path)
end end
def create_files_dir
Dir.mkdir(app_files_dir, 0700)
end
end end
end end
...@@ -5,9 +5,5 @@ module Backup ...@@ -5,9 +5,5 @@ module Backup
def initialize def initialize
super('registry', Settings.registry.path) super('registry', Settings.registry.path)
end end
def create_files_dir
Dir.mkdir(app_files_dir, 0700)
end
end end
end end
require 'yaml' require 'yaml'
require_relative 'helper'
module Backup module Backup
class Repository class Repository
include Backup::Helper
# rubocop:disable Metrics/AbcSize # rubocop:disable Metrics/AbcSize
def dump def dump
prepare prepare
...@@ -63,18 +66,27 @@ module Backup ...@@ -63,18 +66,27 @@ module Backup
end end
end end
def restore def prepare_directories
Gitlab.config.repositories.storages.each do |name, repository_storage| Gitlab.config.repositories.storages.each do |name, repository_storage|
path = repository_storage.legacy_disk_path path = repository_storage.legacy_disk_path
next unless File.exist?(path) next unless File.exist?(path)
# Move repos dir to 'repositories.old' dir # Move all files in the existing repos directory except . and .. to
bk_repos_path = File.join(path, '..', 'repositories.old.' + Time.now.to_i.to_s) # repositories.old.<timestamp> directory
FileUtils.mv(path, bk_repos_path) bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s)
# This is expected from gitlab:check FileUtils.mkdir_p(bk_repos_path, mode: 0700)
FileUtils.mkdir_p(path, mode: 02770) files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")]
begin
FileUtils.mv(files, bk_repos_path)
rescue Errno::EACCES
access_denied_error(path)
end
end
end end
def restore
prepare_directories
Project.find_each(batch_size: 1000) do |project| Project.find_each(batch_size: 1000) do |project|
progress.print " * #{display_repo_path(project)} ... " progress.print " * #{display_repo_path(project)} ... "
path_to_project_repo = path_to_repo(project) path_to_project_repo = path_to_repo(project)
......
...@@ -5,9 +5,5 @@ module Backup ...@@ -5,9 +5,5 @@ module Backup
def initialize def initialize
super('uploads', Rails.root.join('public/uploads')) super('uploads', Rails.root.join('public/uploads'))
end end
def create_files_dir
Dir.mkdir(app_files_dir)
end
end end
end end
require 'spec_helper'
describe Backup::Files do
let(:progress) { StringIO.new }
let!(:project) { create(:project) }
before do
allow(progress).to receive(:puts)
allow(progress).to receive(:print)
allow(FileUtils).to receive(:mkdir_p).and_return(true)
allow(FileUtils).to receive(:mv).and_return(true)
allow(File).to receive(:exist?).and_return(true)
allow(File).to receive(:realpath).with("/var/gitlab-registry").and_return("/var/gitlab-registry")
allow(File).to receive(:realpath).with("/var/gitlab-registry/..").and_return("/var")
allow_any_instance_of(String).to receive(:color) do |string, _color|
string
end
allow_any_instance_of(described_class).to receive(:progress).and_return(progress)
end
describe '#restore' do
subject { described_class.new('registry', '/var/gitlab-registry') }
let(:timestamp) { Time.utc(2017, 3, 22) }
around do |example|
Timecop.freeze(timestamp) { example.run }
end
describe 'folders with permission' do
before do
allow(subject).to receive(:run_pipeline!).and_return(true)
allow(subject).to receive(:backup_existing_files).and_return(true)
allow(Dir).to receive(:glob).with("/var/gitlab-registry/*", File::FNM_DOTMATCH).and_return(["/var/gitlab-registry/.", "/var/gitlab-registry/..", "/var/gitlab-registry/sample1"])
end
it 'moves all necessary files' do
allow(subject).to receive(:backup_existing_files).and_call_original
expect(FileUtils).to receive(:mv).with(["/var/gitlab-registry/sample1"], File.join(Gitlab.config.backup.path, "tmp", "registry.#{Time.now.to_i}"))
subject.restore
end
it 'raises no errors' do
expect { subject.restore }.not_to raise_error
end
it 'calls tar command with unlink' do
expect(subject).to receive(:run_pipeline!).with([%w(gzip -cd), %w(tar --unlink-first --recursive-unlink -C /var/gitlab-registry -xf -)], any_args)
subject.restore
end
end
describe 'folders without permissions' do
before do
allow(FileUtils).to receive(:mv).and_raise(Errno::EACCES)
allow(subject).to receive(:run_pipeline!).and_return(true)
end
it 'shows error message' do
expect(subject).to receive(:access_denied_error).with("/var/gitlab-registry")
subject.restore
end
end
end
end
...@@ -7,6 +7,8 @@ describe Backup::Repository do ...@@ -7,6 +7,8 @@ describe Backup::Repository do
before do before do
allow(progress).to receive(:puts) allow(progress).to receive(:puts)
allow(progress).to receive(:print) allow(progress).to receive(:print)
allow(FileUtils).to receive(:mkdir_p).and_return(true)
allow(FileUtils).to receive(:mv).and_return(true)
allow_any_instance_of(String).to receive(:color) do |string, _color| allow_any_instance_of(String).to receive(:color) do |string, _color|
string string
...@@ -68,6 +70,17 @@ describe Backup::Repository do ...@@ -68,6 +70,17 @@ describe Backup::Repository do
end end
end end
end end
describe 'folders without permissions' do
before do
allow(FileUtils).to receive(:mv).and_raise(Errno::EACCES)
end
it 'shows error message' do
expect(subject).to receive(:access_denied_error)
subject.restore
end
end
end end
describe '#empty_repo?' do describe '#empty_repo?' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment