Merge branch 'feature/improve-logging' into 'master'

Improve ES logging



See merge request !725

CHANGELOG-EE

@@ -6,6 +6,7 @@ v 8.12.0 (Unreleased)
   - [ES] Instrument other Gitlab::Elastic classes
   - [ES] Fix: Elasticsearch does not find partial matches in project names
   - [ES] Global code search
+  - [ES] Improve logging
 
 v 8.11.6
   - Exclude blocked users from potential MR approvers

Gemfile

@@ -108,7 +108,7 @@ gem 'seed-fu', '~> 2.3.5'
 # Search
 gem 'elasticsearch-model'
 gem 'elasticsearch-rails'
-gem 'gitlab-elasticsearch-git', '~> 1.0.0', require: "elasticsearch/git"
+gem 'gitlab-elasticsearch-git', '~> 1.0.1', require: "elasticsearch/git"
 
 # Markdown and HTML processing
 gem 'html-pipeline', '~> 1.11.0'

Gemfile.lock

@@ -283,7 +283,7 @@ GEM
       mime-types (>= 1.19)
       rugged (>= 0.23.0b)
     github-markup (1.4.0)
-    gitlab-elasticsearch-git (1.0.0)
+    gitlab-elasticsearch-git (1.0.1)
       activemodel (~> 4.2)
       activesupport (~> 4.2)
       charlock_holmes (~> 0.7)
@@ -886,7 +886,7 @@ DEPENDENCIES
   gemojione (~> 3.0)
   github-linguist (~> 4.7.0)
   github-markup (~> 1.4)
-  gitlab-elasticsearch-git (~> 1.0.0)
+  gitlab-elasticsearch-git (~> 1.0.1)
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-license (~> 1.0)
   gitlab_git (~> 10.6.3)

bin/elastic_repo_indexer

@@ -17,7 +17,7 @@ FROM_SHA = ENV['FROM_SHA']
 TO_SHA = ENV['TO_SHA']
 RAILS_ENV = ENV['RAILS_ENV']
 
-LOGGER.info("[ES indexer]: Has been scheduled for project #{REPO_PATH} with SHA range #{FROM_SHA}:#{TO_SHA}")
+LOGGER.info("Has been scheduled for project #{REPO_PATH} with SHA range #{FROM_SHA}:#{TO_SHA}")
 
 elastic_connection_info = JSON.parse ENV['ELASTIC_CONNECTION_INFO']
 ELASTIC_HOST = elastic_connection_info['host']
@@ -54,22 +54,26 @@ repo = Repository.new
 
 params = { from_rev: FROM_SHA, to_rev: TO_SHA }.compact
 
-print "Indexing commits..."
+LOGGER.info("Indexing commits started")
 
 timings = Benchmark.measure do
-  repo.index_commits(params)
+  indexed = 0
+  repo.index_commits(params) do |batch, total_count|
+    indexed += batch.length
+    LOGGER.info("Indexed #{indexed}/#{total_count} commits")
+  end
 end
 
-puts "Done"
-
-LOGGER.info("[ES indexer]: Commits for #{REPO_PATH} are indexed. Time elapsed: #{timings.real}")
+LOGGER.info("Commits for #{REPO_PATH} are indexed. Time elapsed: #{timings.real}")
 
-print "Indexing blobs..."
+LOGGER.info("Indexing blobs started")
 
 timings = Benchmark.measure do
-  repo.index_blobs(params)
+  indexed = 0
+  repo.index_blobs(params) do |batch, total_count|
+    indexed += batch.length
+    LOGGER.info("Indexed #{indexed}/#{total_count} blobs")
+  end
 end
 
-puts "Done"
-
-LOGGER.info("[ES indexer]: Blobs for #{REPO_PATH} are indexed. Time elapsed: #{timings.real}")
+LOGGER.info("Blobs for #{REPO_PATH} are indexed. Time elapsed: #{timings.real}")