Merge branch 'elasticsearch-via-gitaly' into 'master'

Elasticsearch: Indexing via gitaly

See merge request gitlab-org/gitlab-ee!7434

ee/changelogs/unreleased/elasticsearch-via-gitaly.yml

+---
+title: 'Elasticsearch: Support for Gitaly'
+merge_request: 7434
+author:
+type: added

ee/lib/gitlab/elastic/indexer.rb

@@ -6,6 +6,8 @@
 module Gitlab
   module Elastic
     class Indexer
+      include Gitlab::Utils::StrongMemoize
+
       EXPERIMENTAL_INDEXER = 'gitlab-elasticsearch-indexer'.freeze
 
       Error = Class.new(StandardError)
@@ -25,6 +27,12 @@ module Gitlab
           'ELASTIC_CONNECTION_INFO' => Gitlab::CurrentSettings.elasticsearch_config.to_json,
           'RAILS_ENV'               => Rails.env
         }
+
+        if use_experimental_indexer?
+          @vars['GITALY_CONNECTION_INFO'] = {
+            storage: project.repository_storage
+          }.merge(Gitlab::GitalyClient.connection_data(project.repository_storage)).to_json
+        end
       end
 
       def run(from_sha = nil, to_sha = nil)
@@ -50,17 +58,22 @@ module Gitlab
       end
 
       def path_to_indexer
-        if Gitlab::CurrentSettings.elasticsearch_experimental_indexer? && self.class.experimental_indexer_present?
+        if use_experimental_indexer?
           EXPERIMENTAL_INDEXER
         else
           Rails.root.join('bin', 'elastic_repo_indexer').to_s
         end
       end
 
-      def run_indexer!(from_sha, to_sha)
-        command = ::Gitlab::GitalyClient::StorageSettings.allow_disk_access do
-          [path_to_indexer, project.id.to_s, repository.path_to_repo]
+      def use_experimental_indexer?
+        strong_memoize(:use_experimental_indexer) do
+          Gitlab::CurrentSettings.elasticsearch_experimental_indexer? && self.class.experimental_indexer_present?
+        end
       end
+
+      def run_indexer!(from_sha, to_sha)
+        command = [path_to_indexer, project.id.to_s, repository_path]
+
         vars = @vars.merge('FROM_SHA' => from_sha, 'TO_SHA' => to_sha)
 
         output, status = Gitlab::Popen.popen(command, nil, vars)
@@ -68,6 +81,15 @@ module Gitlab
         raise Error, output unless status&.zero?
       end
 
+      def repository_path
+        # Go indexer needs relative path while ruby indexer needs absolute one
+        if use_experimental_indexer?
+          "#{repository.disk_path}.git"
+        else
+          ::Gitlab::GitalyClient::StorageSettings.allow_disk_access { repository.path_to_repo }
+        end
+      end
+
       # rubocop: disable CodeReuse/ActiveRecord
       def update_index_status(to_sha)
         head_commit = repository.try(:commit)

ee/spec/lib/gitlab/elastic/indexer_spec.rb

@@ -104,6 +104,35 @@ describe Gitlab::Elastic::Indexer do
 
       indexer.run
     end
+
+    context 'Gitaly support' do
+      let(:project) { create(:project, :repository) }
+
+      it 'passes Gitaly parameters when it is enabled' do
+        expect(described_class).to receive(:experimental_indexer_present?).and_return(true)
+        gitaly_connection_data = {
+          storage: project.repository_storage
+        }.merge(Gitlab::GitalyClient.connection_data(project.repository_storage))
+
+        expect_popen.with(
+          [
+            'gitlab-elasticsearch-indexer',
+            project.id.to_s,
+            "#{project.repository.disk_path}.git"
+          ],
+          nil,
+          hash_including(
+            'GITALY_CONNECTION_INFO'  => gitaly_connection_data.to_json,
+            'ELASTIC_CONNECTION_INFO' => Gitlab::CurrentSettings.elasticsearch_config.to_json,
+            'RAILS_ENV'               => Rails.env,
+            'FROM_SHA'                => from_sha,
+            'TO_SHA'                  => to_sha
+          )
+        ).and_return(popen_success)
+
+        indexer.run(from_sha, to_sha)
+      end
+    end
   end
 
   def expect_popen

lib/gitlab/gitaly_client.rb

@@ -133,7 +133,11 @@ module Gitlab
     end
 
     def self.address_metadata(storage)
-      Base64.strict_encode64(JSON.dump({ storage => { 'address' => address(storage), 'token' => token(storage) } }))
+      Base64.strict_encode64(JSON.dump(storage => connection_data(storage)))
+    end
+
+    def self.connection_data(storage)
+      { 'address' => address(storage), 'token' => token(storage) }
     end
 
     # All Gitaly RPC call sites should use GitalyClient.call. This method

spec/lib/gitlab/gitaly_client_spec.rb

@@ -119,6 +119,15 @@ describe Gitlab::GitalyClient do
     end
   end
 
+  describe '.connection_data' do
+    it 'returns connection data' do
+      address = 'tcp://localhost:9876'
+      stub_repos_storages address
+
+      expect(described_class.connection_data('default')).to eq({ 'address' => address, 'token' => 'secret' })
+    end
+  end
+
   describe 'allow_n_plus_1_calls' do
     context 'when RequestStore is enabled', :request_store do
       it 'returns the result of the allow_n_plus_1_calls block' do