Merge branch 'zj-timeouts-everywhere' into 'master'

Add timeouts for each RPC call

See merge request gitlab-org/gitlab!16926

changelogs/unreleased/zj-timeouts-gitaly-rpcs.yml

+---
+title: Add timeouts for each RPC call
+merge_request: 31766
+author:
+type: changed

lib/gitlab/gitaly_client.rb

@@ -142,13 +142,13 @@ module Gitlab
     #   kwargs.merge(deadline: Time.now + 10)
     # end
     #
-    def self.call(storage, service, rpc, request, remote_storage: nil, timeout: nil)
+    def self.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout)
       start = Gitlab::Metrics::System.monotonic_time
       request_hash = request.is_a?(Google::Protobuf::MessageExts) ? request.to_h : {}
 
       enforce_gitaly_request_limits(:call)
 
-      kwargs = request_kwargs(storage, timeout, remote_storage: remote_storage)
+      kwargs = request_kwargs(storage, timeout: timeout.to_f, remote_storage: remote_storage)
       kwargs = yield(kwargs) if block_given?
 
       stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend
@@ -200,7 +200,7 @@ module Gitlab
     end
     private_class_method :authorization_token
 
-    def self.request_kwargs(storage, timeout, remote_storage: nil)
+    def self.request_kwargs(storage, timeout:, remote_storage: nil)
       metadata = {
         'authorization' => "Bearer #{authorization_token(storage)}",
         'client_name' => CLIENT_NAME
@@ -216,14 +216,7 @@ module Gitlab
 
       result = { metadata: metadata }
 
-      # nil timeout indicates that we should use the default
-      timeout = default_timeout if timeout.nil?
-
-      return result unless timeout > 0
-
-      deadline = real_time + timeout
-      result[:deadline] = deadline
-
+      result[:deadline] = real_time + timeout if timeout > 0
       result
     end
 
@@ -357,8 +350,6 @@ module Gitlab
 
     # The default timeout on all Gitaly calls
     def self.default_timeout
-      return no_timeout if Sidekiq.server?
-
       timeout(:gitaly_timeout_default)
     end
 
@@ -370,8 +361,12 @@ module Gitlab
       timeout(:gitaly_timeout_medium)
     end
 
-    def self.no_timeout
-      0
+    def self.long_timeout
+      if Sidekiq.server?
+        6.hours
+      else
+        55.seconds
+      end
     end
 
     def self.storage_metadata_file_path(storage)

lib/gitlab/gitaly_client/cleanup_service.rb

@@ -18,7 +18,7 @@ module Gitlab
           :cleanup_service,
           :apply_bfg_object_map_stream,
           build_object_map_enum(io),
-          timeout: GitalyClient.no_timeout
+          timeout: GitalyClient.long_timeout
         )
 
         responses.each(&blk)

lib/gitlab/gitaly_client/commit_service.rb

@@ -254,7 +254,7 @@ module Gitlab
 
       def languages(ref = nil)
         request = Gitaly::CommitLanguagesRequest.new(repository: @gitaly_repo, revision: ref || '')
-        response = GitalyClient.call(@repository.storage, :commit_service, :commit_languages, request)
+        response = GitalyClient.call(@repository.storage, :commit_service, :commit_languages, request, timeout: GitalyClient.long_timeout)
 
         response.languages.map { |l| { value: l.share.round(2), label: l.name, color: l.color, highlight: l.color } }
       end
@@ -360,7 +360,7 @@ module Gitlab
 
       def extract_signature(commit_id)
         request = Gitaly::ExtractCommitSignatureRequest.new(repository: @gitaly_repo, commit_id: commit_id)
-        response = GitalyClient.call(@repository.storage, :commit_service, :extract_commit_signature, request)
+        response = GitalyClient.call(@repository.storage, :commit_service, :extract_commit_signature, request, timeout: GitalyClient.fast_timeout)
 
         signature = +''.b
         signed_text = +''.b

lib/gitlab/gitaly_client/conflicts_service.rb

@@ -20,7 +20,7 @@ module Gitlab
           our_commit_oid: @our_commit_oid,
           their_commit_oid: @their_commit_oid
         )
-        response = GitalyClient.call(@repository.storage, :conflicts_service, :list_conflict_files, request)
+        response = GitalyClient.call(@repository.storage, :conflicts_service, :list_conflict_files, request, timeout: GitalyClient.long_timeout)
 
         GitalyClient::ConflictFilesStitcher.new(response)
       end

lib/gitlab/gitaly_client/namespace_service.rb

@@ -22,7 +22,7 @@ module Gitlab
       def remove(name)
         request = Gitaly::RemoveNamespaceRequest.new(storage_name: @storage, name: name)
 
-        gitaly_client_call(:remove_namespace, request, timeout: nil)
+        gitaly_client_call(:remove_namespace, request, timeout: GitalyClient.long_timeout)
       end
 
       def rename(from, to)

lib/gitlab/gitaly_client/object_pool_service.rb

@@ -15,13 +15,15 @@ module Gitlab
           object_pool: object_pool,
           origin: repository.gitaly_repository)
 
-        GitalyClient.call(storage, :object_pool_service, :create_object_pool, request)
+        GitalyClient.call(storage, :object_pool_service, :create_object_pool,
+                          request, timeout: GitalyClient.medium_timeout)
       end
 
       def delete
         request = Gitaly::DeleteObjectPoolRequest.new(object_pool: object_pool)
 
-        GitalyClient.call(storage, :object_pool_service, :delete_object_pool, request)
+        GitalyClient.call(storage, :object_pool_service, :delete_object_pool,
+                          request, timeout: GitalyClient.long_timeout)
       end
 
       def link_repository(repository)
@@ -40,7 +42,8 @@ module Gitlab
           origin: repository.gitaly_repository
         )
 
-        GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool, request)
+        GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool,
+                          request, timeout: GitalyClient.long_timeout)
       end
     end
   end

lib/gitlab/gitaly_client/operation_service.rb

@@ -19,7 +19,7 @@ module Gitlab
           user: Gitlab::Git::User.from_gitlab(user).to_gitaly
         )
 
-        response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request, timeout: GitalyClient.medium_timeout)
+        response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request, timeout: GitalyClient.long_timeout)
 
         if pre_receive_error = response.pre_receive_error.presence
           raise Gitlab::Git::PreReceiveError, pre_receive_error
@@ -35,7 +35,7 @@ module Gitlab
           message: encode_binary(message.to_s)
         )
 
-        response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request, timeout: GitalyClient.medium_timeout)
+        response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request, timeout: GitalyClient.long_timeout)
         if pre_receive_error = response.pre_receive_error.presence
           raise Gitlab::Git::PreReceiveError, pre_receive_error
         elsif response.exists
@@ -55,7 +55,7 @@ module Gitlab
           start_point: encode_binary(start_point)
         )
         response = GitalyClient.call(@repository.storage, :operation_service,
-          :user_create_branch, request)
+                                     :user_create_branch, request, timeout: GitalyClient.long_timeout)
 
         if response.pre_receive_error.present?
           raise Gitlab::Git::PreReceiveError.new(response.pre_receive_error)
@@ -79,7 +79,8 @@ module Gitlab
           oldrev: encode_binary(oldrev)
         )
 
-        response = GitalyClient.call(@repository.storage, :operation_service, :user_update_branch, request)
+        response = GitalyClient.call(@repository.storage, :operation_service,
+                                     :user_update_branch, request, timeout: GitalyClient.long_timeout)
 
         if pre_receive_error = response.pre_receive_error.presence
           raise Gitlab::Git::PreReceiveError, pre_receive_error
@@ -93,7 +94,8 @@ module Gitlab
           user: Gitlab::Git::User.from_gitlab(user).to_gitaly
         )
 
-        response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_branch, request)
+        response = GitalyClient.call(@repository.storage, :operation_service,
+                                     :user_delete_branch, request, timeout: GitalyClient.long_timeout)
 
         if pre_receive_error = response.pre_receive_error.presence
           raise Gitlab::Git::PreReceiveError, pre_receive_error
@@ -111,7 +113,8 @@ module Gitlab
           first_parent_ref: encode_binary(first_parent_ref)
         )
 
-        response = GitalyClient.call(@repository.storage, :operation_service, :user_merge_to_ref, request)
+        response = GitalyClient.call(@repository.storage, :operation_service,
+                                     :user_merge_to_ref, request, timeout: GitalyClient.long_timeout)
 
         if pre_receive_error = response.pre_receive_error.presence
           raise Gitlab::Git::PreReceiveError, pre_receive_error
@@ -126,7 +129,8 @@ module Gitlab
           @repository.storage,
           :operation_service,
           :user_merge_branch,
-          request_enum.each
+          request_enum.each,
+          timeout: GitalyClient.long_timeout
         )
 
         request_enum.push(
@@ -170,7 +174,8 @@ module Gitlab
           @repository.storage,
           :operation_service,
           :user_ff_branch,
-          request
+          request,
+          timeout: GitalyClient.long_timeout
         )
 
         Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update)
@@ -215,6 +220,7 @@ module Gitlab
           :operation_service,
           :user_rebase,
           request,
+          timeout: GitalyClient.long_timeout,
           remote_storage: remote_repository.storage
         )
 
@@ -236,6 +242,7 @@ module Gitlab
           :operation_service,
           :user_rebase_confirmable,
           request_enum.each,
+          timeout: GitalyClient.long_timeout,
           remote_storage: remote_repository.storage
         )
 
@@ -286,7 +293,8 @@ module Gitlab
           @repository.storage,
           :operation_service,
           :user_squash,
-          request
+          request,
+          timeout: GitalyClient.long_timeout
         )
 
         if response.git_error.presence
@@ -310,7 +318,8 @@ module Gitlab
           @repository.storage,
           :operation_service,
           :user_update_submodule,
-          request
+          request,
+          timeout: GitalyClient.long_timeout
         )
 
         if response.pre_receive_error.present?
@@ -352,7 +361,8 @@ module Gitlab
         end
 
         response = GitalyClient.call(@repository.storage, :operation_service,
-          :user_commit_files, req_enum, remote_storage: start_repository.storage)
+                                     :user_commit_files, req_enum, timeout: GitalyClient.long_timeout,
+                                     remote_storage: start_repository.storage)
 
         if (pre_receive_error = response.pre_receive_error.presence)
           raise Gitlab::Git::PreReceiveError, pre_receive_error
@@ -384,7 +394,8 @@ module Gitlab
           end
         end
 
-        response = GitalyClient.call(@repository.storage, :operation_service, :user_apply_patch, chunks)
+        response = GitalyClient.call(@repository.storage, :operation_service,
+                                     :user_apply_patch, chunks, timeout: GitalyClient.long_timeout)
 
         Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update)
       end
@@ -424,7 +435,7 @@ module Gitlab
           :"user_#{rpc}",
           request,
           remote_storage: start_repository.storage,
-          timeout: GitalyClient.medium_timeout
+          timeout: GitalyClient.long_timeout
         )
 
         handle_cherry_pick_or_revert_response(response)

lib/gitlab/gitaly_client/ref_service.rb

@@ -21,7 +21,7 @@ module Gitlab
 
       def remote_branches(remote_name)
         request = Gitaly::FindAllRemoteBranchesRequest.new(repository: @gitaly_repo, remote_name: remote_name)
-        response = GitalyClient.call(@repository.storage, :ref_service, :find_all_remote_branches, request)
+        response = GitalyClient.call(@repository.storage, :ref_service, :find_all_remote_branches, request, timeout: GitalyClient.medium_timeout)
 
         consume_find_all_remote_branches_response(remote_name, response)
       end
@@ -158,7 +158,7 @@ module Gitlab
           start_point: encode_binary(start_point)
         )
 
-        response = GitalyClient.call(@repository.storage, :ref_service, :create_branch, request)
+        response = GitalyClient.call(@repository.storage, :ref_service, :create_branch, request, timeout: GitalyClient.medium_timeout)
 
         case response.status
         when :OK
@@ -182,7 +182,7 @@ module Gitlab
           name: encode_binary(branch_name)
         )
 
-        GitalyClient.call(@repository.storage, :ref_service, :delete_branch, request)
+        GitalyClient.call(@repository.storage, :ref_service, :delete_branch, request, timeout: GitalyClient.medium_timeout)
       end
 
       def delete_refs(refs: [], except_with_prefixes: [])
@@ -192,7 +192,7 @@ module Gitlab
           except_with_prefix: except_with_prefixes.map { |r| encode_binary(r) }
         )
 
-        response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request, timeout: GitalyClient.default_timeout)
+        response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request, timeout: GitalyClient.medium_timeout)
 
         raise Gitlab::Git::Repository::GitError, response.git_error if response.git_error.present?
       end
@@ -242,7 +242,7 @@ module Gitlab
       def pack_refs
         request = Gitaly::PackRefsRequest.new(repository: @gitaly_repo)
 
-        GitalyClient.call(@storage, :ref_service, :pack_refs, request)
+        GitalyClient.call(@storage, :ref_service, :pack_refs, request, timeout: GitalyClient.long_timeout)
       end
 
       private

lib/gitlab/gitaly_client/remote_service.rb

@@ -38,9 +38,7 @@ module Gitlab
       def remove_remote(name)
         request = Gitaly::RemoveRemoteRequest.new(repository: @gitaly_repo, name: name)
 
-        response = GitalyClient.call(@storage, :remote_service, :remove_remote, request)
-
-        response.result
+        GitalyClient.call(@storage, :remote_service, :remove_remote, request, timeout: GitalyClient.long_timeout).result
       end
 
       def fetch_internal_remote(repository)
@@ -51,6 +49,7 @@ module Gitlab
 
         response = GitalyClient.call(@storage, :remote_service,
                                      :fetch_internal_remote, request,
+                                     timeout: GitalyClient.medium_timeout,
                                      remote_storage: repository.storage)
 
         response.result
@@ -63,7 +62,7 @@ module Gitlab
         )
 
         response = GitalyClient.call(@storage, :remote_service,
-                                     :find_remote_root_ref, request)
+                                     :find_remote_root_ref, request, timeout: GitalyClient.medium_timeout)
 
         encode_utf8(response.ref)
       end
@@ -95,7 +94,7 @@ module Gitlab
           end
         end
 
-        GitalyClient.call(@storage, :remote_service, :update_remote_mirror, req_enum)
+        GitalyClient.call(@storage, :remote_service, :update_remote_mirror, req_enum, timeout: GitalyClient.long_timeout)
       end
     end
   end

lib/gitlab/gitaly_client/repository_service.rb

@@ -28,17 +28,17 @@ module Gitlab
 
       def garbage_collect(create_bitmap)
         request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap)
-        GitalyClient.call(@storage, :repository_service, :garbage_collect, request)
+        GitalyClient.call(@storage, :repository_service, :garbage_collect, request, timeout: GitalyClient.long_timeout)
       end
 
       def repack_full(create_bitmap)
         request = Gitaly::RepackFullRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap)
-        GitalyClient.call(@storage, :repository_service, :repack_full, request)
+        GitalyClient.call(@storage, :repository_service, :repack_full, request, timeout: GitalyClient.long_timeout)
       end
 
       def repack_incremental
         request = Gitaly::RepackIncrementalRequest.new(repository: @gitaly_repo)
-        GitalyClient.call(@storage, :repository_service, :repack_incremental, request)
+        GitalyClient.call(@storage, :repository_service, :repack_incremental, request, timeout: GitalyClient.long_timeout)
       end
 
       def repository_size
@@ -86,12 +86,12 @@ module Gitlab
           end
         end
 
-        GitalyClient.call(@storage, :repository_service, :fetch_remote, request)
+        GitalyClient.call(@storage, :repository_service, :fetch_remote, request, timeout: GitalyClient.long_timeout)
       end
 
       def create_repository
         request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo)
-        GitalyClient.call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.medium_timeout)
+        GitalyClient.call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.fast_timeout)
       end
 
       def has_local_branches?
@@ -189,6 +189,7 @@ module Gitlab
           :repository_service,
           :fetch_source_branch,
           request,
+          timeout: GitalyClient.default_timeout,
           remote_storage: source_repository.storage
         )
 
@@ -197,7 +198,7 @@ module Gitlab
 
       def fsck
         request = Gitaly::FsckRequest.new(repository: @gitaly_repo)
-        response = GitalyClient.call(@storage, :repository_service, :fsck, request, timeout: GitalyClient.no_timeout)
+        response = GitalyClient.call(@storage, :repository_service, :fsck, request, timeout: GitalyClient.long_timeout)
 
         if response.error.empty?
           return "", 0
@@ -211,7 +212,7 @@ module Gitlab
           save_path,
           :create_bundle,
           Gitaly::CreateBundleRequest,
-          GitalyClient.no_timeout
+          GitalyClient.long_timeout
         )
       end
 
@@ -229,7 +230,7 @@ module Gitlab
           bundle_path,
           :create_repository_from_bundle,
           Gitaly::CreateRepositoryFromBundleRequest,
-          GitalyClient.no_timeout
+          GitalyClient.long_timeout
         )
       end
 
@@ -254,7 +255,7 @@ module Gitlab
           :repository_service,
           :create_repository_from_snapshot,
           request,
-          timeout: GitalyClient.no_timeout
+          timeout: GitalyClient.long_timeout
         )
       end
 
@@ -333,7 +334,7 @@ module Gitlab
 
       def search_files_by_content(ref, query)
         request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query)
-        response = GitalyClient.call(@storage, :repository_service, :search_files_by_content, request)
+        response = GitalyClient.call(@storage, :repository_service, :search_files_by_content, request, timeout: GitalyClient.default_timeout)
 
         search_results_from_response(response)
       end
@@ -343,7 +344,7 @@ module Gitlab
           repository: @gitaly_repo
         )
 
-        GitalyClient.call(@storage, :object_pool_service, :disconnect_git_alternates, request)
+        GitalyClient.call(@storage, :object_pool_service, :disconnect_git_alternates, request, timeout: GitalyClient.long_timeout)
       end
 
       private

lib/gitlab/gitaly_client/storage_service.rb

@@ -11,14 +11,14 @@ module Gitlab
       def list_directories(depth: 1)
         request = Gitaly::ListDirectoriesRequest.new(storage_name: @storage, depth: depth)
 
-        GitalyClient.call(@storage, :storage_service, :list_directories, request)
+        GitalyClient.call(@storage, :storage_service, :list_directories, request, timeout: GitalyClient.medium_timeout)
           .flat_map(&:paths)
       end
 
       # Delete all repositories in the storage. This is a slow and VERY DESTRUCTIVE operation.
       def delete_all_repositories
         request = Gitaly::DeleteAllRepositoriesRequest.new(storage_name: @storage)
-        GitalyClient.call(@storage, :storage_service, :delete_all_repositories, request)
+        GitalyClient.call(@storage, :storage_service, :delete_all_repositories, request, timeout: GitalyClient.long_timeout)
       end
     end
   end

lib/gitlab/gitaly_client/wiki_service.rb

@@ -34,7 +34,7 @@ module Gitlab
           end
         end
 
-        response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_write_page, enum)
+        response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_write_page, enum, timeout: GitalyClient.medium_timeout)
         if error = response.duplicate_error.presence
           raise Gitlab::Git::Wiki::DuplicatePageError, error
         end
@@ -61,7 +61,7 @@ module Gitlab
           end
         end
 
-        GitalyClient.call(@repository.storage, :wiki_service, :wiki_update_page, enum)
+        GitalyClient.call(@repository.storage, :wiki_service, :wiki_update_page, enum, timeout: GitalyClient.medium_timeout)
       end
 
       def delete_page(page_path, commit_details)
@@ -187,7 +187,7 @@ module Gitlab
           directory: encode_binary(dir)
         )
 
-        response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_formatted_data, request)
+        response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_formatted_data, request, timeout: GitalyClient.medium_timeout)
         response.reduce([]) { |memo, msg| memo << msg.data }.join
       end
 

spec/lib/gitlab/gitaly_client_spec.rb

@@ -182,24 +182,24 @@ describe Gitlab::GitalyClient do
       end
 
       it 'sets the gitaly-session-id in the metadata' do
-        results = described_class.request_kwargs('default', nil)
+        results = described_class.request_kwargs('default', timeout: 1)
         expect(results[:metadata]).to include('gitaly-session-id')
       end
 
       context 'when RequestStore is not enabled' do
         it 'sets a different gitaly-session-id per request' do
-          gitaly_session_id = described_class.request_kwargs('default', nil)[:metadata]['gitaly-session-id']
+          gitaly_session_id = described_class.request_kwargs('default', timeout: 1)[:metadata]['gitaly-session-id']
 
-          expect(described_class.request_kwargs('default', nil)[:metadata]['gitaly-session-id']).not_to eq(gitaly_session_id)
+          expect(described_class.request_kwargs('default', timeout: 1)[:metadata]['gitaly-session-id']).not_to eq(gitaly_session_id)
         end
       end
 
       context 'when RequestStore is enabled', :request_store do
         it 'sets the same gitaly-session-id on every outgoing request metadata' do
-          gitaly_session_id = described_class.request_kwargs('default', nil)[:metadata]['gitaly-session-id']
+          gitaly_session_id = described_class.request_kwargs('default', timeout: 1)[:metadata]['gitaly-session-id']
 
           3.times do
-            expect(described_class.request_kwargs('default', nil)[:metadata]['gitaly-session-id']).to eq(gitaly_session_id)
+            expect(described_class.request_kwargs('default', timeout: 1)[:metadata]['gitaly-session-id']).to eq(gitaly_session_id)
           end
         end
       end

spec/workers/object_pool/destroy_worker_spec.rb

@@ -16,7 +16,9 @@ describe ObjectPool::DestroyWorker do
       subject { described_class.new }
 
       it 'requests Gitaly to remove the object pool' do
-        expect(Gitlab::GitalyClient).to receive(:call).with(pool.shard_name, :object_pool_service, :delete_object_pool, Object)
+        expect(Gitlab::GitalyClient).to receive(:call)
+          .with(pool.shard_name, :object_pool_service, :delete_object_pool,
+                Object, timeout: Gitlab::GitalyClient.long_timeout)
 
         subject.perform(pool.id)
       end