Merge branch...

Merge branch '932-es-elastic-workers-should-check-settings-each-time-when-they-are-running' into 'master'

[ES] Elastic workers should check settings each time when they are running

Closes #932

See merge request !693

CHANGELOG-EE

@@ -4,6 +4,7 @@ v 8.12.0 (Unreleased)
 v 8.11.3 (Unreleased)
   - [ES] Add logging to indexer
   - Set the correct `GL_PROTOCOL` when rebasing !691
+  - [ES] Elasticsearch workers checks ES settings before running
 
 v 8.11.2
   - Additional documentation on protected branches for EE

app/workers/elastic_commit_indexer_worker.rb

 class ElasticCommitIndexerWorker
   include Sidekiq::Worker
+  include Gitlab::CurrentSettings
 
   sidekiq_options queue: :elasticsearch, retry: 2
 
   def perform(project_id, oldrev = nil, newrev = nil)
+    return true unless current_application_settings.elasticsearch_indexing?
+
     project = Project.find(project_id)
     repository = project.repository
 

app/workers/elastic_indexer_worker.rb

 class ElasticIndexerWorker
   include Sidekiq::Worker
   include Elasticsearch::Model::Client::ClassMethods
+  include Gitlab::CurrentSettings
 
   sidekiq_options queue: :elasticsearch, retry: 2
 
   ISSUE_TRACKED_FIELDS = %w(assignee_id author_id confidential)
 
   def perform(operation, class_name, record_id, options = {})
+    return true unless current_application_settings.elasticsearch_indexing?
+
     klass = class_name.constantize
 
     case operation.to_s

spec/workers/elastic_commit_indexer_worker_spec.rb

@@ -6,6 +6,10 @@ describe ElasticCommitIndexerWorker do
   subject { described_class.new }
 
   describe '#perform' do
+    before do
+      stub_application_setting(elasticsearch_indexing: true)
+    end
+
     it 'runs indexer' do
       expect_any_instance_of(Gitlab::Elastic::Indexer).to receive(:run)
       subject.perform(project.id, '0000', '0000')
@@ -27,5 +31,13 @@ describe ElasticCommitIndexerWorker do
 
       expect(subject.perform(project.id)).to be_truthy
     end
+
+    it 'returns true if ES disabled' do
+      stub_application_setting(elasticsearch_indexing: false)
+
+      expect_any_instance_of(Gitlab::Elastic::Indexer).not_to receive(:run)
+
+      expect(subject.perform(1)).to be_truthy
+    end
   end
 end

spec/workers/elastic_indexer_worker_spec.rb

@@ -11,119 +11,171 @@ describe ElasticIndexerWorker, elastic: true do
     )
 
     Gitlab::Elastic::Helper.create_empty_index
+
+    stub_application_setting(elasticsearch_indexing: true)
   end
 
   after do
     Gitlab::Elastic::Helper.delete_index
   end
 
-  Sidekiq::Testing.disable! do
-    describe 'Indexing new records' do
-      it 'indexes a project' do
-        project = create :empty_project
+  it 'returns true if ES disabled' do
+    stub_application_setting(elasticsearch_indexing: false)
+
+    expect_any_instance_of(Elasticsearch::Model).not_to receive(:__elasticsearch__)
+
+    expect(subject.perform("index", "Milestone", 1)).to be_truthy
+  end
+
+  describe 'Indexing new records' do
+    it 'indexes a project' do
+      project = nil
 
-        expect do
-          subject.perform("index", "Project", project.id)
-          Gitlab::Elastic::Helper.refresh_index
-        end.to change{ Elasticsearch::Model.search('*').records.size }.by(1)
+      Sidekiq::Testing.disable! do
+        project = create :empty_project
       end
 
-      it 'indexes an issue' do
-        issue = create :issue
+      expect do
+        subject.perform("index", "Project", project.id)
+        Gitlab::Elastic::Helper.refresh_index
+      end.to change{ Elasticsearch::Model.search('*').records.size }.by(1)
+    end
+
+    it 'indexes an issue' do
+      issue = nil
 
-        expect do
-          subject.perform("index", "Issue", issue.id)
-          Gitlab::Elastic::Helper.refresh_index
-        end.to change{ Elasticsearch::Model.search('*').records.size }.by(1)
+      Sidekiq::Testing.disable! do
+        issue = create :issue
       end
 
-      it 'indexes a note' do
-        note = create :note
+      expect do
+        subject.perform("index", "Issue", issue.id)
+        Gitlab::Elastic::Helper.refresh_index
+      end.to change{ Elasticsearch::Model.search('*').records.size }.by(1)
+    end
+
+    it 'indexes a note' do
+      note = nil
 
-        expect do
-          subject.perform("index", "Note", note.id)
-          Gitlab::Elastic::Helper.refresh_index
-        end.to change{ Elasticsearch::Model.search('*').records.size }.by(1)
+      Sidekiq::Testing.disable! do
+        note = create :note
       end
 
-      it 'indexes a milestone' do
-        milestone = create :milestone
+      expect do
+        subject.perform("index", "Note", note.id)
+        Gitlab::Elastic::Helper.refresh_index
+      end.to change{ Elasticsearch::Model.search('*').records.size }.by(1)
+    end
 
-        expect do
-          subject.perform("index", "Milestone", milestone.id)
-          Gitlab::Elastic::Helper.refresh_index
-        end.to change{ Elasticsearch::Model.search('*').records.size }.by(1)
+    it 'indexes a milestone' do
+      milestone = nil
+
+      Sidekiq::Testing.disable! do
+        milestone = create :milestone
       end
 
-      it 'indexes a merge request' do
-        merge_request = create :merge_request
+      expect do
+        subject.perform("index", "Milestone", milestone.id)
+        Gitlab::Elastic::Helper.refresh_index
+      end.to change{ Elasticsearch::Model.search('*').records.size }.by(1)
+    end
 
-        expect do
-          subject.perform("index", "MergeRequest", merge_request.id)
-          Gitlab::Elastic::Helper.refresh_index
-        end.to change{ Elasticsearch::Model.search('*').records.size }.by(1)
+    it 'indexes a merge request' do
+      merge_request = nil
+
+      Sidekiq::Testing.disable! do
+        merge_request = create :merge_request
       end
+
+      expect do
+        subject.perform("index", "MergeRequest", merge_request.id)
+        Gitlab::Elastic::Helper.refresh_index
+      end.to change{ Elasticsearch::Model.search('*').records.size }.by(1)
     end
+  end
+
+  describe 'Updating index' do
+    it 'updates a project' do
+      project = nil
 
-    describe 'Updating index' do
-      it 'updates a project' do
+      Sidekiq::Testing.disable! do
         project = create :empty_project
         subject.perform("index", "Project", project.id)
         project.update(name: "new")
-
-        expect do
-          subject.perform("update", "Project", project.id)
-          Gitlab::Elastic::Helper.refresh_index
-        end.to change{ Elasticsearch::Model.search('new').records.size }.by(1)
       end
 
-      it 'updates an issue' do
+      expect do
+        subject.perform("update", "Project", project.id)
+        Gitlab::Elastic::Helper.refresh_index
+      end.to change{ Elasticsearch::Model.search('new').records.size }.by(1)
+    end
+
+    it 'updates an issue' do
+      issue = nil
+
+      Sidekiq::Testing.disable! do
         issue = create :issue
         subject.perform("index", "Issue", issue.id)
         issue.update(title: "new")
-
-        expect do
-          subject.perform("update", "Issue", issue.id)
-          Gitlab::Elastic::Helper.refresh_index
-        end.to change{ Elasticsearch::Model.search('new').records.size }.by(1)
       end
 
-      it 'updates a note' do
+      expect do
+        subject.perform("update", "Issue", issue.id)
+        Gitlab::Elastic::Helper.refresh_index
+      end.to change{ Elasticsearch::Model.search('new').records.size }.by(1)
+    end
+
+    it 'updates a note' do
+      note = nil
+
+      Sidekiq::Testing.disable! do
         note = create :note
         subject.perform("index", "Note", note.id)
         note.update(note: 'new')
-
-        expect do
-          subject.perform("update", "Note", note.id)
-          Gitlab::Elastic::Helper.refresh_index
-        end.to change{ Elasticsearch::Model.search('new').records.size }.by(1)
       end
 
-      it 'updates a milestone' do
+      expect do
+        subject.perform("update", "Note", note.id)
+        Gitlab::Elastic::Helper.refresh_index
+      end.to change{ Elasticsearch::Model.search('new').records.size }.by(1)
+    end
+
+    it 'updates a milestone' do
+      milestone = nil
+
+      Sidekiq::Testing.disable! do
         milestone = create :milestone
         subject.perform("index", "Milestone", milestone.id)
         milestone.update(title: 'new')
-
-        expect do
-          subject.perform("update", "Milestone", milestone.id)
-          Gitlab::Elastic::Helper.refresh_index
-        end.to change{ Elasticsearch::Model.search('new').records.size }.by(1)
       end
 
-      it 'updates a merge request' do
+      expect do
+        subject.perform("update", "Milestone", milestone.id)
+        Gitlab::Elastic::Helper.refresh_index
+      end.to change{ Elasticsearch::Model.search('new').records.size }.by(1)
+    end
+
+    it 'updates a merge request' do
+      merge_request = nil
+
+      Sidekiq::Testing.disable! do
         merge_request = create :merge_request
         subject.perform("index", "MergeRequest", merge_request.id)
         merge_request.update(title: 'new')
-
-        expect do
-          subject.perform("index", "MergeRequest", merge_request.id)
-          Gitlab::Elastic::Helper.refresh_index
-        end.to change{ Elasticsearch::Model.search('new').records.size }.by(1)
       end
+
+      expect do
+        subject.perform("index", "MergeRequest", merge_request.id)
+        Gitlab::Elastic::Helper.refresh_index
+      end.to change{ Elasticsearch::Model.search('new').records.size }.by(1)
     end
+  end
+
+  describe 'Delete' do
+    it 'deletes a project with all nested objects' do
+      project, issue, milestone, note, merge_request = nil
 
-    describe 'Delete' do
-      it 'deletes a project with all nested objects' do
+      Sidekiq::Testing.disable! do
         project = create :project
         subject.perform("index", "Project", project.id)
 
@@ -138,70 +190,86 @@ describe ElasticIndexerWorker, elastic: true do
 
         merge_request = create :merge_request, target_project: project, source_project: project
         subject.perform("index", "MergeRequest", merge_request.id)
+      end
 
-        ElasticCommitIndexerWorker.new.perform(project.id)
-        Gitlab::Elastic::Helper.refresh_index
+      ElasticCommitIndexerWorker.new.perform(project.id)
+      Gitlab::Elastic::Helper.refresh_index
 
-        ## All database objects + data from repository. The absolute value does not matter
-        expect(Elasticsearch::Model.search('*').total_count).to be > 40
+      ## All database objects + data from repository. The absolute value does not matter
+      expect(Elasticsearch::Model.search('*').total_count).to be > 40
 
-        subject.perform("delete", "Project", project.id)
-        Gitlab::Elastic::Helper.refresh_index
+      subject.perform("delete", "Project", project.id)
+      Gitlab::Elastic::Helper.refresh_index
 
-        expect(Elasticsearch::Model.search('*').total_count).to be(0)
-      end
+      expect(Elasticsearch::Model.search('*').total_count).to be(0)
+    end
+
+    it 'deletes an issue' do
+      issue, project_id = nil
 
-      it 'deletes an issue' do
+      Sidekiq::Testing.disable! do
         issue = create :issue
         subject.perform("index", "Issue", issue.id)
         Gitlab::Elastic::Helper.refresh_index
-
+        project_id = issue.project_id
         issue.destroy
-
-        expect do
-          subject.perform("delete", "Issue", issue.id, "project_id" => issue.project_id)
-          Gitlab::Elastic::Helper.refresh_index
-        end.to change{ Elasticsearch::Model.search('*').total_count }.by(-1)
       end
 
-      it 'deletes a note' do
+      expect do
+        subject.perform("delete", "Issue", issue.id, "project_id" => project_id)
+        Gitlab::Elastic::Helper.refresh_index
+      end.to change{ Elasticsearch::Model.search('*').total_count }.by(-1)
+    end
+
+    it 'deletes a note' do
+      note, project_id = nil
+
+      Sidekiq::Testing.disable! do
         note = create :note
         subject.perform("index", "Note", note.id)
         Gitlab::Elastic::Helper.refresh_index
-
+        project_id = note.project_id
         note.destroy
-
-        expect do
-          subject.perform("delete", "Note", note.id, "project_id" => note.project_id)
-          Gitlab::Elastic::Helper.refresh_index
-        end.to change{ Elasticsearch::Model.search('*').total_count }.by(-1)
       end
 
-      it 'deletes a milestone' do
+      expect do
+        subject.perform("delete", "Note", note.id, "project_id" => project_id)
+        Gitlab::Elastic::Helper.refresh_index
+      end.to change{ Elasticsearch::Model.search('*').total_count }.by(-1)
+    end
+
+    it 'deletes a milestone' do
+      milestone, project_id = nil
+
+      Sidekiq::Testing.disable! do
         milestone = create :milestone
         subject.perform("index", "Milestone", milestone.id)
         Gitlab::Elastic::Helper.refresh_index
-
+        project_id = milestone.project_id
         milestone.destroy
-
-        expect do
-          subject.perform("delete", "Milestone", milestone.id, "project_id" => milestone.project_id)
-          Gitlab::Elastic::Helper.refresh_index
-        end.to change{ Elasticsearch::Model.search('*').total_count }.by(-1)
       end
 
-      it 'deletes a merge request' do
+      expect do
+        subject.perform("delete", "Milestone", milestone.id, "project_id" => project_id)
+        Gitlab::Elastic::Helper.refresh_index
+      end.to change{ Elasticsearch::Model.search('*').total_count }.by(-1)
+    end
+
+    it 'deletes a merge request' do
+      merge_request, project_id = nil
+
+      Sidekiq::Testing.disable! do
         merge_request = create :merge_request
         subject.perform("index", "MergeRequest", merge_request.id)
         Gitlab::Elastic::Helper.refresh_index
-
+        project_id = merge_request.target_project_id
         merge_request.destroy
-
-        expect do
-          subject.perform("delete", "MergeRequest", merge_request.id, "project_id" => merge_request.target_project_id)
-          Gitlab::Elastic::Helper.refresh_index
-        end.to change{ Elasticsearch::Model.search('*').total_count }.by(-1)
       end
+
+      expect do
+        subject.perform("delete", "MergeRequest", merge_request.id, "project_id" => project_id)
+        Gitlab::Elastic::Helper.refresh_index
+      end.to change{ Elasticsearch::Model.search('*').total_count }.by(-1)
     end
   end
 end