Merge branch 'ajk-prune-hashed-projects' into 'master'

Development rake task to prune hashed projects

See merge request gitlab-org/gitlab!51603

lib/gitlab/hashed_storage/rake_helper.rb

@@ -65,6 +65,7 @@ module Gitlab
       def self.projects_list(relation_name, relation)
         listing(relation_name, relation.with_route) do |project|
           $stdout.puts "  - #{project.full_path} (id: #{project.id})".color(:red)
+          $stdout.puts "    #{project.repository.disk_path}"
         end
       end
 
@@ -92,6 +93,37 @@ module Gitlab
         end
       end
       # rubocop: enable CodeReuse/ActiveRecord
+
+      def self.prune(relation_name, relation, dry_run: true, root: nil)
+        root ||= '../repositories'
+
+        known_paths = Set.new
+        listing(relation_name, relation) { |p| known_paths << "#{root}/#{p.repository.disk_path}" }
+
+        marked_for_deletion = Set.new(Dir["#{root}/@hashed/*/*/*"])
+        marked_for_deletion.reject! do |path|
+          base = path.gsub(/\.(\w+\.)?git$/, '')
+          known_paths.include?(base)
+        end
+
+        if marked_for_deletion.empty?
+          $stdout.puts "No orphaned directories found. Nothing to do!"
+        else
+          n = marked_for_deletion.size
+          $stdout.puts "Found #{n} orphaned #{'directory'.pluralize(n)}"
+          $stdout.puts "Dry run. (Run again with FORCE=1 to delete). We would have deleted:" if dry_run
+        end
+
+        marked_for_deletion.each do |p|
+          p = Pathname.new(p)
+          if dry_run
+            $stdout.puts " - #{p}"
+          else
+            $stdout.puts "Removing #{p}"
+            p.rmtree
+          end
+        end
+      end
     end
   end
 end

lib/tasks/gitlab/storage.rake

@@ -116,6 +116,21 @@ namespace :gitlab do
       helper.projects_list('projects using Hashed Storage', Project.with_storage_feature(:repository))
     end
 
+    desc 'Gitlab | Storage | Prune projects using Hashed Storage. Remove all hashed directories that do not have a project associated'
+    task prune_hashed_projects: [:environment, :gitlab_environment] do
+      if Rails.env.production?
+        abort('This destructive action may only be run in development')
+      end
+
+      helper = Gitlab::HashedStorage::RakeHelper
+      name = 'projects using Hashed Storage'
+      relation = Project.with_storage_feature(:repository)
+      root = Gitlab.config.repositories.storages['default'].legacy_disk_path
+      dry_run = !ENV['FORCE'].present?
+
+      helper.prune(name, relation, dry_run: dry_run, root: root)
+    end
+
     desc 'Gitlab | Storage | Summary of project attachments using Legacy Storage'
     task legacy_attachments: :environment do
       helper = Gitlab::HashedStorage::RakeHelper