Allow user to skip archive creation on backup

Don't create a (tar) archive if the user specified SKIP=tar when
creating a backup.

Allow restore from untarred backups (they don't have easily accesible
timestamps, so just give such a backup priority if it's present).

Document the festure.

Addresses https://gitlab.com/gitlab-org/gitlab/issues/15179

doc/raketasks/backup_restore.md

@@ -271,6 +271,33 @@ For installations from source:
 sudo -u git -H bundle exec rake gitlab:backup:create SKIP=db,uploads RAILS_ENV=production
 ```
 
+### Skipping tar creation
+
+The last part of backup creation is generation of a tar file with all
+the parts in. In some cases (e.g. if the backup is picked up by other
+backup software) that might be wasted or even directly harmful, you can
+skip this step by adding `tar` to the `SKIP` environment variable.
+
+This will leave the files and directories containing the backup in the
+directory used for the intermediate files, and thus the will be
+overwritten when a new backup is created, i.e. you can't have more than
+one backup on the system, so you should make sure it's copied elsewhere.
+
+For Omnibus GitLab packages:
+
+```shell
+sudo gitlab-backup create SKIP=tar
+```
+
+NOTE: **Note**
+For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`.
+
+For installations from source:
+
+```shell
+sudo -u git -H bundle exec rake gitlab:backup:create SKIP=tar RAILS_ENV=production
+```
+
 ### Uploading backups to a remote (cloud) storage
 
 Starting with GitLab 7.4 you can let the backup script upload the '.tar' file it creates.
@@ -658,6 +685,9 @@ lose access to your GitLab server.
 
 You may also want to restore any TLS keys, certificates, or [SSH host keys](https://superuser.com/questions/532040/copy-ssh-keys-from-one-server-to-another-server/532079#532079).
 
+If an untarred backup (like the ones made with `SKIP=tar`) is found, and
+no backup is chosen with `BACKUP=<timestamp>`, the untarred is used.
+
 Depending on your case, you might want to run the restore command with one or
 more of the following options:
 

lib/backup/manager.rb

@@ -12,7 +12,7 @@ module Backup
       @progress = progress
     end
 
-    def pack
+    def write_info
       # Make sure there is a connection
       ActiveRecord::Base.connection.reconnect!
 
@@ -20,7 +20,14 @@ module Backup
         File.open("#{backup_path}/backup_information.yml", "w+") do |file|
           file << backup_information.to_yaml.gsub(/^---\n/, '')
         end
+      end
+    end
+
+    def pack
+      # Make sure there is a connection
+      ActiveRecord::Base.connection.reconnect!
 
+      Dir.chdir(backup_path) do
         # create archive
         progress.print "Creating backup archive: #{tar_file} ... "
         # Set file permissions on open to prevent chmod races.
@@ -31,8 +38,6 @@ module Backup
           puts "creating archive #{tar_file} failed".color(:red)
           raise Backup::Error, 'Backup failed'
         end
-
-        upload
       end
     end
 
@@ -107,39 +112,45 @@ module Backup
 
     # rubocop: disable Metrics/AbcSize
     def unpack
+      cleanup_required = true
       Dir.chdir(backup_path) do
         # check for existing backups in the backup dir
-        if backup_file_list.empty?
-          progress.puts "No backups found in #{backup_path}"
-          progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}"
-          exit 1
-        elsif backup_file_list.many? && ENV["BACKUP"].nil?
-          progress.puts 'Found more than one backup:'
-          # print list of available backups
-          progress.puts " " + available_timestamps.join("\n ")
-          progress.puts 'Please specify which one you want to restore:'
-          progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup'
-          exit 1
-        end
+        if File.exist?(File.join(backup_path, 'backup_information.yml')) && !ENV['BACKUP'].present?
+          progress.puts "Non tarred backup found in #{backup_path}, using that"
+          cleanup_required = false
+        else
+          if backup_file_list.empty?
+            progress.puts "No backups found in #{backup_path}"
+            progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}"
+            exit 1
+          elsif backup_file_list.many? && ENV["BACKUP"].nil?
+            progress.puts 'Found more than one backup:'
+            # print list of available backups
+            progress.puts " " + available_timestamps.join("\n ")
+            progress.puts 'Please specify which one you want to restore:'
+            progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup'
+            exit 1
+          end
 
-        tar_file = if ENV['BACKUP'].present?
-                     File.basename(ENV['BACKUP']) + FILE_NAME_SUFFIX
-                   else
-                     backup_file_list.first
-                   end
+          tar_file = if ENV['BACKUP'].present?
+                       File.basename(ENV['BACKUP']) + FILE_NAME_SUFFIX
+                     else
+                       backup_file_list.first
+                     end
 
-        unless File.exist?(tar_file)
-          progress.puts "The backup file #{tar_file} does not exist!"
-          exit 1
-        end
+          unless File.exist?(tar_file)
+            progress.puts "The backup file #{tar_file} does not exist!"
+            exit 1
+          end
 
-        progress.print 'Unpacking backup ... '
+          progress.print 'Unpacking backup ... '
 
-        if Kernel.system(*%W(tar -xf #{tar_file}))
-          progress.puts 'done'.color(:green)
-        else
-          progress.puts 'unpacking backup failed'.color(:red)
-          exit 1
+          if Kernel.system(*%W(tar -xf #{tar_file}))
+            progress.puts 'done'.color(:green)
+          else
+            progress.puts 'unpacking backup failed'.color(:red)
+            exit 1
+          end
         end
 
         ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
@@ -157,6 +168,7 @@ module Backup
           exit 1
         end
       end
+      cleanup_required
     end
 
     def tar_version

lib/tasks/gitlab/backup.rake

@@ -17,9 +17,16 @@ namespace :gitlab do
       Rake::Task['gitlab:backup:registry:create'].invoke
 
       backup = Backup::Manager.new(progress)
-      backup.pack
-      backup.cleanup
-      backup.remove_old
+      backup.write_info
+
+      if !(ENV['SKIP'] && ENV['SKIP'].include?('tar'))
+        backup.pack
+        backup.upload
+        backup.cleanup
+        backup.remove_old
+      else
+        backup.upload
+      end
 
       progress.puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \
            "and are not included in this backup. You will need these files to restore a backup.\n" \
@@ -33,7 +40,7 @@ namespace :gitlab do
       warn_user_is_not_gitlab
 
       backup = Backup::Manager.new(progress)
-      backup.unpack
+      cleanup_required = backup.unpack
 
       unless backup.skipped?('db')
         begin
@@ -72,7 +79,10 @@ namespace :gitlab do
       Rake::Task['gitlab:shell:setup'].invoke
       Rake::Task['cache:clear'].invoke
 
-      backup.cleanup
+      if cleanup_required
+        backup.cleanup
+      end
+
       puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \
            "and are not included in this backup. You will need to restore these files manually.".color(:red)
       puts "Restore task is done."