Merge branch...

Merge branch '259605-repositories-gone-after-update-broken-migration-to-hashed-storage' into 'master'

Hashed Storage: make migration and rollback resilient to exceptions

See merge request gitlab-org/gitlab!46178

app/services/projects/hashed_storage/migrate_repository_service.rb

@@ -26,11 +26,27 @@ module Projects
           project.set_repository_writable!
         end
 
-        if result && block_given?
-          yield
-        end
-
         result
+      rescue Gitlab::Git::CommandError => e
+        logger.error("Repository #{project.full_path} failed to upgrade (PROJECT_ID=#{project.id}). Git operation failed: #{e.inspect}")
+
+        rollback_migration!
+
+        false
+      rescue OpenSSL::Cipher::CipherError => e
+        logger.error("Repository #{project.full_path} failed to upgrade (PROJECT_ID=#{project.id}). There is a problem with encrypted attributes: #{e.inspect}")
+
+        rollback_migration!
+
+        false
+      end
+
+      private
+
+      def rollback_migration!
+        rollback_folder_move
+        project.storage_version = nil
+        project.set_repository_writable!
       end
     end
   end

app/services/projects/hashed_storage/rollback_repository_service.rb

@@ -21,14 +21,32 @@ module Projects
           project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:repository]
         end
 
-        project.repository_read_only = false
-        project.save!(validate: false)
-
-        if result && block_given?
-          yield
+        project.transaction do
+          project.save!(validate: false)
+          project.set_repository_writable!
         end
 
         result
+      rescue Gitlab::Git::CommandError => e
+        logger.error("Repository #{project.full_path} failed to rollback (PROJECT_ID=#{project.id}). Git operation failed: #{e.inspect}")
+
+        rollback_migration!
+
+        false
+      rescue OpenSSL::Cipher::CipherError => e
+        logger.error("Repository #{project.full_path} failed to rollback (PROJECT_ID=#{project.id}). There is a problem with encrypted attributes: #{e.inspect}")
+
+        rollback_migration!
+
+        false
+      end
+
+      private
+
+      def rollback_migration!
+        rollback_folder_move
+        project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:repository]
+        project.set_repository_writable!
       end
     end
   end

changelogs/unreleased/259605-repositories-gone-after-update-broken-migration-to-hashed-storage.yml

+---
+title: "Hashed Storage: make migration and rollback resilient to exceptions"
+merge_request: 46178
+author:
+type: fixed

ee/app/services/ee/projects/hashed_storage/migrate_repository_service.rb

@@ -8,7 +8,9 @@ module EE
 
         override :execute
         def execute
-          super do
+          result = super
+
+          if result
             ::Geo::HashedStorageMigratedEventStore.new(
               project,
               old_storage_version: old_storage_version,
@@ -17,6 +19,8 @@ module EE
               old_design_disk_path: old_design_disk_path
             ).create!
           end
+
+          result
         end
       end
     end

spec/services/projects/hashed_storage/migrate_repository_service_spec.rb

@@ -77,6 +77,42 @@ RSpec.describe Projects::HashedStorage::MigrateRepositoryService do
       end
     end
 
+    context 'when exception happens' do
+      it 'handles OpenSSL::Cipher::CipherError' do
+        expect(project).to receive(:ensure_runners_token).and_raise(OpenSSL::Cipher::CipherError)
+
+        expect { service.execute }.not_to raise_exception
+      end
+
+      it 'ensures rollback when OpenSSL::Cipher::CipherError' do
+        expect(project).to receive(:ensure_runners_token).and_raise(OpenSSL::Cipher::CipherError)
+        expect(service).to receive(:rollback_folder_move).and_call_original
+
+        service.execute
+        project.reload
+
+        expect(project.legacy_storage?).to be_truthy
+        expect(project.repository_read_only?).to be_falsey
+      end
+
+      it 'handles Gitlab::Git::CommandError' do
+        expect(project).to receive(:write_repository_config).and_raise(Gitlab::Git::CommandError)
+
+        expect { service.execute }.not_to raise_exception
+      end
+
+      it 'ensures rollback when Gitlab::Git::CommandError' do
+        expect(project).to receive(:write_repository_config).and_raise(Gitlab::Git::CommandError)
+        expect(service).to receive(:rollback_folder_move).and_call_original
+
+        service.execute
+        project.reload
+
+        expect(project.legacy_storage?).to be_truthy
+        expect(project.repository_read_only?).to be_falsey
+      end
+    end
+
     context 'when one move fails' do
       it 'rollsback repositories to original name' do
         allow(service).to receive(:move_repository).and_call_original

spec/services/projects/hashed_storage/rollback_repository_service_spec.rb

@@ -77,6 +77,42 @@ RSpec.describe Projects::HashedStorage::RollbackRepositoryService, :clean_gitlab
       end
     end
 
+    context 'when exception happens' do
+      it 'handles OpenSSL::Cipher::CipherError' do
+        expect(project).to receive(:ensure_runners_token).and_raise(OpenSSL::Cipher::CipherError)
+
+        expect { service.execute }.not_to raise_exception
+      end
+
+      it 'ensures rollback when OpenSSL::Cipher::CipherError' do
+        expect(project).to receive(:ensure_runners_token).and_raise(OpenSSL::Cipher::CipherError)
+        expect(service).to receive(:rollback_folder_move).and_call_original
+
+        service.execute
+        project.reload
+
+        expect(project.hashed_storage?(:repository)).to be_truthy
+        expect(project.repository_read_only?).to be_falsey
+      end
+
+      it 'handles Gitlab::Git::CommandError' do
+        expect(project).to receive(:write_repository_config).and_raise(Gitlab::Git::CommandError)
+
+        expect { service.execute }.not_to raise_exception
+      end
+
+      it 'ensures rollback when Gitlab::Git::CommandError' do
+        expect(project).to receive(:write_repository_config).and_raise(Gitlab::Git::CommandError)
+        expect(service).to receive(:rollback_folder_move).and_call_original
+
+        service.execute
+        project.reload
+
+        expect(project.hashed_storage?(:repository)).to be_truthy
+        expect(project.repository_read_only?).to be_falsey
+      end
+    end
+
     context 'when one move fails' do
       it 'rolls repositories back to original name' do
         allow(service).to receive(:move_repository).and_call_original