Merge branch 'ce-rd-gitaly-client-for-list-new-objects-rpc' into 'master'

Backport of gitlab-ee!6767

See merge request gitlab-org/gitlab-ce!21037

GITALY_SERVER_VERSION

-0.115.0
+0.116.0

Gemfile

@@ -423,7 +423,7 @@ group :ed25519 do
 end
 
 # Gitaly GRPC client
-gem 'gitaly-proto', '~> 0.109.0', require: 'gitaly'
+gem 'gitaly-proto', '~> 0.112.0', require: 'gitaly'
 gem 'grpc', '~> 1.11.0'
 
 # Locked until https://github.com/google/protobuf/issues/4210 is closed

Gemfile.lock

@@ -284,7 +284,7 @@ GEM
       gettext_i18n_rails (>= 0.7.1)
       po_to_json (>= 1.0.0)
       rails (>= 3.2.0)
-    gitaly-proto (0.109.0)
+    gitaly-proto (0.112.0)
       google-protobuf (~> 3.1)
       grpc (~> 1.10)
     github-linguist (5.3.3)
@@ -1048,7 +1048,7 @@ DEPENDENCIES
   gettext (~> 3.2.2)
   gettext_i18n_rails (~> 1.8.0)
   gettext_i18n_rails_js (~> 1.3)
-  gitaly-proto (~> 0.109.0)
+  gitaly-proto (~> 0.112.0)
   github-linguist (~> 5.3.3)
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-gollum-lib (~> 4.2)

Gemfile.rails5.lock

@@ -287,7 +287,7 @@ GEM
       gettext_i18n_rails (>= 0.7.1)
       po_to_json (>= 1.0.0)
       rails (>= 3.2.0)
-    gitaly-proto (0.109.0)
+    gitaly-proto (0.112.0)
       google-protobuf (~> 3.1)
       grpc (~> 1.10)
     github-linguist (5.3.3)
@@ -1058,7 +1058,7 @@ DEPENDENCIES
   gettext (~> 3.2.2)
   gettext_i18n_rails (~> 1.8.0)
   gettext_i18n_rails_js (~> 1.3)
-  gitaly-proto (~> 0.109.0)
+  gitaly-proto (~> 0.112.0)
   github-linguist (~> 5.3.3)
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-gollum-lib (~> 4.2)

lib/gitlab/checks/lfs_integrity.rb

 module Gitlab
   module Checks
     class LfsIntegrity
-      REV_LIST_OBJECT_LIMIT = 2_000
-
       def initialize(project, newrev)
         @project = project
         @newrev = newrev
@@ -11,7 +9,8 @@ module Gitlab
       def objects_missing?
         return false unless @newrev && @project.lfs_enabled?
 
-        new_lfs_pointers = Gitlab::Git::LfsChanges.new(@project.repository, @newrev).new_pointers(object_limit: REV_LIST_OBJECT_LIMIT)
+        new_lfs_pointers = Gitlab::Git::LfsChanges.new(@project.repository, @newrev)
+                                                  .new_pointers(object_limit: ::Gitlab::Git::Repository::REV_LIST_COMMIT_LIMIT)
 
         return false unless new_lfs_pointers.present?
 

lib/gitlab/git/repository.rb

@@ -19,6 +19,7 @@ module Gitlab
         GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE
       ].freeze
       SEARCH_CONTEXT_LINES = 3
+      REV_LIST_COMMIT_LIMIT = 2_000
       # In https://gitlab.com/gitlab-org/gitaly/merge_requests/698
       # We copied these two prefixes into gitaly-go, so don't change these
       # or things will break! (REBASE_WORKTREE_PREFIX and SQUASH_WORKTREE_PREFIX)
@@ -380,6 +381,16 @@ module Gitlab
         end
       end
 
+      def new_blobs(newrev)
+        return [] if newrev == ::Gitlab::Git::BLANK_SHA
+
+        strong_memoize("new_blobs_#{newrev}") do
+          wrapped_gitaly_errors do
+            gitaly_ref_client.list_new_blobs(newrev, REV_LIST_COMMIT_LIMIT)
+          end
+        end
+      end
+
       def count_commits(options)
         options = process_count_commits_options(options.dup)
 

lib/gitlab/gitaly_client/ref_service.rb

@@ -82,6 +82,23 @@ module Gitlab
         commits
       end
 
+      def list_new_blobs(newrev, limit = 0)
+        request = Gitaly::ListNewBlobsRequest.new(
+          repository: @gitaly_repo,
+          commit_id: newrev,
+          limit: limit
+        )
+
+        response = GitalyClient
+          .call(@storage, :ref_service, :list_new_blobs, request, timeout: GitalyClient.medium_timeout)
+
+        response.flat_map do |msg|
+          # Returns an Array of Gitaly::NewBlobObject objects
+          # Available methods are: #size, #oid and #path
+          msg.new_blob_objects
+        end
+      end
+
       def count_tag_names
         tag_names.count
       end

spec/support/helpers/test_env.rb

@@ -51,7 +51,8 @@ module TestEnv
     'add-pdf-text-binary'                => '79faa7b',
     'add_images_and_changes'             => '010d106',
     'update-gitlab-shell-v-6-0-1'        => '2f61d70',
-    'update-gitlab-shell-v-6-0-3'        => 'de78448'
+    'update-gitlab-shell-v-6-0-3'        => 'de78448',
+    '2-mb-file'                          => 'bf12d25'
   }.freeze
 
   # gitlab-test-fork is a fork of gitlab-fork, but we don't necessarily