Commit 24b048db authored by Dmitriy Zaporozhets's avatar Dmitriy Zaporozhets

Merge branch 'backup_skip_option' into 'master'

Ability to skip some items in backup

https://dev.gitlab.org/gitlab/gitlabhq/issues/1936

Now it is possible to run

```
sudo gitlab-rake gitlab:backup:create SKIP=repositories,uploads
```

available options: repositories,uploads,db

See merge request !470
parents 2a0d38c0 67c83489
...@@ -52,6 +52,7 @@ v 7.10.0 (unreleased) ...@@ -52,6 +52,7 @@ v 7.10.0 (unreleased)
- Don't show commit comment button when user is not signed in. - Don't show commit comment button when user is not signed in.
- Fix admin user projects lists. - Fix admin user projects lists.
- Don't leak private group existence by redirecting from namespace controller to group controller. - Don't leak private group existence by redirecting from namespace controller to group controller.
- Ability to skip some items from backup (database, respositories or uploads)
v 7.9.2 v 7.9.2
- Contains no changes - Contains no changes
......
...@@ -17,6 +17,13 @@ sudo gitlab-rake gitlab:backup:create ...@@ -17,6 +17,13 @@ sudo gitlab-rake gitlab:backup:create
sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production
``` ```
Also you can choose what should be backed up by adding environment variable SKIP. Available options: db,
uploads (attachments), repositories. Use a comma to specify several options at the same time.
```
sudo gitlab-rake gitlab:backup:create SKIP=db,uploads
```
Example output: Example output:
``` ```
......
module Backup module Backup
class Manager class Manager
BACKUP_CONTENTS = %w{repositories/ db/ uploads/ backup_information.yml}
def pack def pack
# saving additional informations # saving additional informations
s = {} s = {}
...@@ -9,6 +7,7 @@ module Backup ...@@ -9,6 +7,7 @@ module Backup
s[:backup_created_at] = Time.now s[:backup_created_at] = Time.now
s[:gitlab_version] = Gitlab::VERSION s[:gitlab_version] = Gitlab::VERSION
s[:tar_version] = tar_version s[:tar_version] = tar_version
s[:skipped] = ENV["SKIP"]
tar_file = "#{s[:backup_created_at].to_i}_gitlab_backup.tar" tar_file = "#{s[:backup_created_at].to_i}_gitlab_backup.tar"
Dir.chdir(Gitlab.config.backup.path) do Dir.chdir(Gitlab.config.backup.path) do
...@@ -17,12 +16,12 @@ module Backup ...@@ -17,12 +16,12 @@ module Backup
file << s.to_yaml.gsub(/^---\n/,'') file << s.to_yaml.gsub(/^---\n/,'')
end end
FileUtils.chmod(0700, %w{db uploads repositories}) FileUtils.chmod(0700, folders_to_backup)
# create archive # create archive
$progress.print "Creating backup archive: #{tar_file} ... " $progress.print "Creating backup archive: #{tar_file} ... "
orig_umask = File.umask(0077) orig_umask = File.umask(0077)
if Kernel.system('tar', '-cf', tar_file, *BACKUP_CONTENTS) if Kernel.system('tar', '-cf', tar_file, *backup_contents)
$progress.puts "done".green $progress.puts "done".green
else else
puts "creating archive #{tar_file} failed".red puts "creating archive #{tar_file} failed".red
...@@ -46,6 +45,7 @@ module Backup ...@@ -46,6 +45,7 @@ module Backup
connection = ::Fog::Storage.new(connection_settings) connection = ::Fog::Storage.new(connection_settings)
directory = connection.directories.get(remote_directory) directory = connection.directories.get(remote_directory)
if directory.files.create(key: tar_file, body: File.open(tar_file), public: false) if directory.files.create(key: tar_file, body: File.open(tar_file), public: false)
$progress.puts "done".green $progress.puts "done".green
else else
...@@ -56,7 +56,10 @@ module Backup ...@@ -56,7 +56,10 @@ module Backup
def cleanup def cleanup
$progress.print "Deleting tmp directories ... " $progress.print "Deleting tmp directories ... "
BACKUP_CONTENTS.each do |dir|
backup_contents.each do |dir|
next unless File.exist?(File.join(Gitlab.config.backup.path, dir))
if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir)) if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir))
$progress.puts "done".green $progress.puts "done".green
else else
...@@ -73,6 +76,7 @@ module Backup ...@@ -73,6 +76,7 @@ module Backup
if keep_time > 0 if keep_time > 0
removed = 0 removed = 0
Dir.chdir(Gitlab.config.backup.path) do Dir.chdir(Gitlab.config.backup.path) do
file_list = Dir.glob('*_gitlab_backup.tar') file_list = Dir.glob('*_gitlab_backup.tar')
file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_backup.tar/ } file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_backup.tar/ }
...@@ -84,6 +88,7 @@ module Backup ...@@ -84,6 +88,7 @@ module Backup
end end
end end
end end
$progress.puts "done. (#{removed} removed)".green $progress.puts "done. (#{removed} removed)".green
else else
$progress.puts "skipping".yellow $progress.puts "skipping".yellow
...@@ -96,6 +101,7 @@ module Backup ...@@ -96,6 +101,7 @@ module Backup
# check for existing backups in the backup dir # check for existing backups in the backup dir
file_list = Dir.glob("*_gitlab_backup.tar").each.map { |f| f.split(/_/).first.to_i } file_list = Dir.glob("*_gitlab_backup.tar").each.map { |f| f.split(/_/).first.to_i }
puts "no backups found" if file_list.count == 0 puts "no backups found" if file_list.count == 0
if file_list.count > 1 && ENV["BACKUP"].nil? if file_list.count > 1 && ENV["BACKUP"].nil?
puts "Found more than one backup, please specify which one you want to restore:" puts "Found more than one backup, please specify which one you want to restore:"
puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup" puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup"
...@@ -110,6 +116,7 @@ module Backup ...@@ -110,6 +116,7 @@ module Backup
end end
$progress.print "Unpacking backup ... " $progress.print "Unpacking backup ... "
unless Kernel.system(*%W(tar -xf #{tar_file})) unless Kernel.system(*%W(tar -xf #{tar_file}))
puts "unpacking backup failed".red puts "unpacking backup failed".red
exit 1 exit 1
...@@ -117,7 +124,6 @@ module Backup ...@@ -117,7 +124,6 @@ module Backup
$progress.puts "done".green $progress.puts "done".green
end end
settings = YAML.load_file("backup_information.yml")
ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
# restoring mismatching backups can lead to unexpected problems # restoring mismatching backups can lead to unexpected problems
...@@ -136,5 +142,29 @@ module Backup ...@@ -136,5 +142,29 @@ module Backup
tar_version, _ = Gitlab::Popen.popen(%W(tar --version)) tar_version, _ = Gitlab::Popen.popen(%W(tar --version))
tar_version.force_encoding('locale').split("\n").first tar_version.force_encoding('locale').split("\n").first
end end
def skipped?(item)
settings[:skipped] && settings[:skipped].include?(item)
end
private
def backup_contents
folders_to_backup + ["backup_information.yml"]
end
def folders_to_backup
folders = %w{repositories db uploads}
if ENV["SKIP"]
return folders.reject{ |folder| ENV["SKIP"].include?(folder) }
end
folders
end
def settings
@settings ||= YAML.load_file("backup_information.yml")
end
end end
end end
...@@ -27,9 +27,9 @@ namespace :gitlab do ...@@ -27,9 +27,9 @@ namespace :gitlab do
backup = Backup::Manager.new backup = Backup::Manager.new
backup.unpack backup.unpack
Rake::Task["gitlab:backup:db:restore"].invoke Rake::Task["gitlab:backup:db:restore"].invoke unless backup.skipped?("db")
Rake::Task["gitlab:backup:repo:restore"].invoke Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories")
Rake::Task["gitlab:backup:uploads:restore"].invoke Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads")
Rake::Task["gitlab:shell:setup"].invoke Rake::Task["gitlab:shell:setup"].invoke
backup.cleanup backup.cleanup
...@@ -38,9 +38,14 @@ namespace :gitlab do ...@@ -38,9 +38,14 @@ namespace :gitlab do
namespace :repo do namespace :repo do
task create: :environment do task create: :environment do
$progress.puts "Dumping repositories ...".blue $progress.puts "Dumping repositories ...".blue
if ENV["SKIP"] && ENV["SKIP"].include?("repositories")
$progress.puts "[SKIPPED]".cyan
else
Backup::Repository.new.dump Backup::Repository.new.dump
$progress.puts "done".green $progress.puts "done".green
end end
end
task restore: :environment do task restore: :environment do
$progress.puts "Restoring repositories ...".blue $progress.puts "Restoring repositories ...".blue
...@@ -52,9 +57,14 @@ namespace :gitlab do ...@@ -52,9 +57,14 @@ namespace :gitlab do
namespace :db do namespace :db do
task create: :environment do task create: :environment do
$progress.puts "Dumping database ... ".blue $progress.puts "Dumping database ... ".blue
if ENV["SKIP"] && ENV["SKIP"].include?("db")
$progress.puts "[SKIPPED]".cyan
else
Backup::Database.new.dump Backup::Database.new.dump
$progress.puts "done".green $progress.puts "done".green
end end
end
task restore: :environment do task restore: :environment do
$progress.puts "Restoring database ... ".blue $progress.puts "Restoring database ... ".blue
...@@ -66,9 +76,14 @@ namespace :gitlab do ...@@ -66,9 +76,14 @@ namespace :gitlab do
namespace :uploads do namespace :uploads do
task create: :environment do task create: :environment do
$progress.puts "Dumping uploads ... ".blue $progress.puts "Dumping uploads ... ".blue
if ENV["SKIP"] && ENV["SKIP"].include?("uploads")
$progress.puts "[SKIPPED]".cyan
else
Backup::Uploads.new.dump Backup::Uploads.new.dump
$progress.puts "done".green $progress.puts "done".green
end end
end
task restore: :environment do task restore: :environment do
$progress.puts "Restoring uploads ... ".blue $progress.puts "Restoring uploads ... ".blue
......
...@@ -98,4 +98,55 @@ describe 'gitlab:app namespace rake task' do ...@@ -98,4 +98,55 @@ describe 'gitlab:app namespace rake task' do
expect(temp_dirs).to be_empty expect(temp_dirs).to be_empty
end end
end # backup_create task end # backup_create task
describe "Skipping items" do
def tars_glob
Dir.glob(File.join(Gitlab.config.backup.path, '*_gitlab_backup.tar'))
end
before :all do
@origin_cd = Dir.pwd
Rake::Task["gitlab:backup:db:create"].reenable
Rake::Task["gitlab:backup:repo:create"].reenable
Rake::Task["gitlab:backup:uploads:create"].reenable
# Record the existing backup tars so we don't touch them
existing_tars = tars_glob
# Redirect STDOUT and run the rake task
orig_stdout = $stdout
$stdout = StringIO.new
ENV["SKIP"] = "repositories"
run_rake_task('gitlab:backup:create')
$stdout = orig_stdout
@backup_tar = (tars_glob - existing_tars).first
end
after :all do
FileUtils.rm(@backup_tar)
Dir.chdir @origin_cd
end
it "does not contain skipped item" do
tar_contents, exit_status = Gitlab::Popen.popen(
%W{tar -tvf #{@backup_tar} db uploads repositories}
)
expect(tar_contents).to match('db/')
expect(tar_contents).to match('uploads/')
expect(tar_contents).not_to match('repositories/')
end
it 'does not invoke repositories restore' do
Rake::Task["gitlab:shell:setup"].stub invoke: true
allow($stdout).to receive :write
expect(Rake::Task["gitlab:backup:db:restore"]).to receive :invoke
expect(Rake::Task["gitlab:backup:repo:restore"]).not_to receive :invoke
expect(Rake::Task["gitlab:shell:setup"]).to receive :invoke
expect { run_rake_task('gitlab:backup:restore') }.to_not raise_error
end
end
end # gitlab:app namespace end # gitlab:app namespace
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment