Merge branch 'zj-remove-git-rake-tasks' into 'master'

Remove git rake tasks

Closes #41699 and gitaly#768

See merge request gitlab-org/gitlab-ce!20806

GITALY_SERVER_VERSION

-0.113.0
+0.114.0

changelogs/unreleased/zj-remove-git-rake-tasks.yml

+---
+title: Remove gitlab:user:check_repos, gitlab:check_repo, gitlab:git:prune, gitlab:git:gc, and gitlab:git:repack
+merge_request: 20806
+author:
+type: removed

doc/administration/raketasks/check.md

-# Check Rake Tasks
+# Integrity Check Rake Task
 
 ## Repository Integrity
 
@@ -28,14 +28,8 @@ exactly which repositories are causing the trouble.
 
 ### Check all GitLab repositories
 
->**Note:**
->
->  - `gitlab:repo:check` has been deprecated in favor of `gitlab:git:fsck`
->  - [Deprecated][ce-15931] in GitLab 10.4.
->  - `gitlab:repo:check` will be removed in the future. [Removal issue][ce-41699]
-
 This task loops through all repositories on the GitLab server and runs the
-3 integrity checks described previously.
+integrity check described previously.
 
 **Omnibus Installation**
 
@@ -49,33 +43,6 @@ sudo gitlab-rake gitlab:git:fsck
 sudo -u git -H bundle exec rake gitlab:git:fsck RAILS_ENV=production
 ```
 
-### Check repositories for a specific user
-
-This task checks all repositories that a specific user has access to. This is important
-because sometimes you know which user is experiencing trouble but you don't know
-which project might be the cause.
-
-If the rake task is executed without brackets at the end, you will be prompted
-to enter a username.
-
-**Omnibus Installation**
-
-```bash
-sudo gitlab-rake gitlab:user:check_repos
-sudo gitlab-rake gitlab:user:check_repos[<username>]
-```
-
-**Source Installation**
-
-```bash
-sudo -u git -H bundle exec rake gitlab:user:check_repos RAILS_ENV=production
-sudo -u git -H bundle exec rake gitlab:user:check_repos[<username>] RAILS_ENV=production
-```
-
-Example output:
-
-![gitlab:user:check_repos output](../img/raketasks/check_repos_output.png)
-
 ## Uploaded Files Integrity
 
 Various types of files can be uploaded to a GitLab installation by users.
@@ -167,5 +134,4 @@ The LDAP check Rake task will test the bind_dn and password credentials
 executed as part of the `gitlab:check` task, but can run independently.
 See [LDAP Rake Tasks - LDAP Check](ldap.md#check) for details.
 
-[ce-15931]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/15931
-[ce-41699]: https://gitlab.com/gitlab-org/gitlab-ce/issues/41699
+[git-fsck]: https://git-scm.com/docs/git-fsck

lib/tasks/gitlab/check.rake

@@ -385,14 +385,6 @@ namespace :gitlab do
     end
   end
 
-  namespace :repo do
-    desc "GitLab | Check the integrity of the repositories managed by GitLab"
-    task check: :gitlab_environment do
-      puts "This task is deprecated. Please use gitlab:git:fsck instead".color(:red)
-      Rake::Task["gitlab:git:fsck"].execute
-    end
-  end
-
   namespace :orphans do
     desc 'Gitlab | Check for orphaned namespaces and repositories'
     task check: :gitlab_environment do
@@ -422,23 +414,6 @@ namespace :gitlab do
     end
   end
 
-  namespace :user do
-    desc "GitLab | Check the integrity of a specific user's repositories"
-    task :check_repos, [:username] => :gitlab_environment do |t, args|
-      username = args[:username] || prompt("Check repository integrity for username? ".color(:blue))
-      user = User.find_by(username: username)
-      if user
-        repo_dirs = user.authorized_projects.map do |p|
-          p.repository.path_to_repo
-        end
-
-        repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
-      else
-        puts "\nUser '#{username}' not found".color(:red)
-      end
-    end
-  end
-
   # Helper methods
   ##########################
 

lib/tasks/gitlab/git.rake

 namespace :gitlab do
   namespace :git do
-    desc "GitLab | Git | Repack"
-    task repack: :gitlab_environment do
-      failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} repack -a --quiet), "Repacking repo")
-      if failures.empty?
-        puts "Done".color(:green)
-      else
-        output_failures(failures)
-      end
-    end
-
-    desc "GitLab | Git | Run garbage collection on all repos"
-    task gc: :gitlab_environment do
-      failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} gc --auto --quiet), "Garbage Collecting")
-      if failures.empty?
-        puts "Done".color(:green)
-      else
-        output_failures(failures)
-      end
-    end
-
-    desc "GitLab | Git | Prune all repos"
-    task prune: :gitlab_environment do
-      failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} prune), "Git Prune")
-      if failures.empty?
-        puts "Done".color(:green)
-      else
-        output_failures(failures)
-      end
-    end
-
     desc 'GitLab | Git | Check all repos integrity'
     task fsck: :gitlab_environment do
-      failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} fsck --name-objects --no-progress), "Checking integrity") do |repo|
-        check_config_lock(repo)
-        check_ref_locks(repo)
-      end
-
-      if failures.empty?
-        puts "Done".color(:green)
-      else
-        output_failures(failures)
-      end
-    end
-
-    def perform_git_cmd(cmd, message)
-      puts "Starting #{message} on all repositories"
-
       failures = []
-      all_repos do |repo|
-        if system(*cmd, chdir: repo)
-          puts "Performed #{message} at #{repo}"
-        else
-          failures << repo
-        end
+      Project.find_each(batch_size: 100) do |project|
+        begin
+          project.repository.fsck
 
-        yield(repo) if block_given?
+        rescue => e
+          failures << "#{project.full_path} on #{project.repository_storage}: #{e}"
         end
 
-      failures
+        puts "Performed integrity check for #{project.repository.full_path}"
       end
 
-    def output_failures(failures)
+      if failures.empty?
+        puts "Done".color(:green)
+      else
         puts "The following repositories reported errors:".color(:red)
         failures.each { |f| puts "- #{f}" }
       end
-
-    def check_config_lock(repo_dir)
-      config_exists = File.exist?(File.join(repo_dir, 'config.lock'))
-      config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green)
-
-      puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}"
-    end
-
-    def check_ref_locks(repo_dir)
-      lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock'))
-
-      if lock_files.present?
-        puts "Ref lock files exist:".color(:red)
-
-        lock_files.each { |lock_file| puts "  #{lock_file}" }
-      else
-        puts "No ref lock files exist".color(:green)
-      end
     end
   end
 end

spec/tasks/gitlab/git_rake_spec.rb

@@ -2,45 +2,19 @@ require 'rake_helper'
 
 describe 'gitlab:git rake tasks' do
   let(:base_path) { 'tmp/tests/default_storage' }
-
-  before(:all) do
-    @default_storage_hash = Gitlab.config.repositories.storages.default.to_h
-  end
+  let!(:project) { create(:project, :repository) }
 
   before do
     Rake.application.rake_require 'tasks/gitlab/git'
-    storages = { 'default' => Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => base_path)) }
-
-    path = Settings.absolute("#{base_path}/@hashed/1/2/test.git")
-    FileUtils.mkdir_p(path)
-    Gitlab::Popen.popen(%W[git -C #{path} init --bare])
 
-    allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
     allow_any_instance_of(String).to receive(:color) { |string, _color| string }
 
     stub_warn_user_is_not_gitlab
   end
 
-  after do
-    FileUtils.rm_rf(Settings.absolute(base_path))
-  end
-
   describe 'fsck' do
     it 'outputs the integrity check for a repo' do
-      expect { run_rake_task('gitlab:git:fsck') }.to output(%r{Performed Checking integrity at .*@hashed/1/2/test.git}).to_stdout
-    end
-
-    it 'errors out about config.lock issues' do
-      FileUtils.touch(Settings.absolute("#{base_path}/@hashed/1/2/test.git/config.lock"))
-
-      expect { run_rake_task('gitlab:git:fsck') }.to output(/file exists\? ... yes/).to_stdout
-    end
-
-    it 'errors out about ref lock issues' do
-      FileUtils.mkdir_p(Settings.absolute("#{base_path}/@hashed/1/2/test.git/refs/heads"))
-      FileUtils.touch(Settings.absolute("#{base_path}/@hashed/1/2/test.git/refs/heads/blah.lock"))
-
-      expect { run_rake_task('gitlab:git:fsck') }.to output(/Ref lock files exist:/).to_stdout
+      expect { run_rake_task('gitlab:git:fsck') }.to output(/Performed integrity check for/).to_stdout
     end
   end
 end