Revert "Cache ES enabled namespaces and projects"

This reverts commit aa4f1cf4.

changelogs/unreleased/209854-cache-es-check.yml

----
-title: Cache ES enabled namespaces and projects
-merge_request: 27348
-author:
-type: performance

ee/app/models/concerns/elasticsearch_indexed_container.rb

@@ -3,16 +3,9 @@
 module ElasticsearchIndexedContainer
   extend ActiveSupport::Concern
 
-  CACHE_EXPIRES_IN = 10.minutes
-
   included do
     after_commit :index, on: :create
     after_commit :delete_from_index, on: :destroy
-    after_commit :drop_limited_ids_cache!, on: [:create, :destroy]
-  end
-
-  def drop_limited_ids_cache!
-    self.class.drop_limited_ids_cache!
   end
 
   class_methods do
@@ -20,28 +13,6 @@ module ElasticsearchIndexedContainer
       pluck(target_attr_name)
     end
 
-    def limited_ids
-      limited.pluck(:id)
-    end
-
-    def limited_ids_cache_key
-      [self.name.underscore.to_sym, :limited_ids]
-    end
-
-    def limited_ids_cached
-      Rails.cache.fetch(limited_ids_cache_key, expires_in: CACHE_EXPIRES_IN) do
-        limited_ids
-      end
-    end
-
-    def drop_limited_ids_cache!
-      Rails.cache.delete(limited_ids_cache_key)
-    end
-
-    def limited_include?(namespace_id)
-      limited_ids_cached.include?(namespace_id)
-    end
-
     def remove_all(except: [])
       self.where.not(target_attr_name => except).each_batch do |batch, _index|
         batch.destroy_all # #rubocop:disable Cop/DestroyAll

ee/app/models/ee/application_setting.rb

@@ -131,26 +131,37 @@ module EE
       ElasticsearchIndexedProject.target_ids
     end
 
-    def elasticsearch_limited_namespaces(ignore_descendants: false)
-      ElasticsearchIndexedNamespace.limited(ignore_descendants: ignore_descendants)
-    end
-
-    def elasticsearch_limited_projects(ignore_namespaces: false)
-      ElasticsearchIndexedProject.limited(ignore_namespaces: ignore_namespaces)
-    end
-
     def elasticsearch_indexes_project?(project)
       return false unless elasticsearch_indexing?
       return true unless elasticsearch_limit_indexing?
 
-      ElasticsearchIndexedProject.limited_include?(project.id)
+      elasticsearch_limited_projects.exists?(project.id)
     end
 
     def elasticsearch_indexes_namespace?(namespace)
       return false unless elasticsearch_indexing?
       return true unless elasticsearch_limit_indexing?
 
-      ElasticsearchIndexedNamespace.limited_include?(namespace.id)
+      elasticsearch_limited_namespaces.exists?(namespace.id)
+    end
+
+    def elasticsearch_limited_projects(ignore_namespaces = false)
+      return ::Project.where(id: ElasticsearchIndexedProject.select(:project_id)) if ignore_namespaces
+
+      union = ::Gitlab::SQL::Union.new([
+                                         ::Project.where(namespace_id: elasticsearch_limited_namespaces.select(:id)),
+                                         ::Project.where(id: ElasticsearchIndexedProject.select(:project_id))
+                                       ]).to_sql
+
+      ::Project.from("(#{union}) projects")
+    end
+
+    def elasticsearch_limited_namespaces(ignore_descendants = false)
+      namespaces = ::Namespace.where(id: ElasticsearchIndexedNamespace.select(:namespace_id))
+
+      return namespaces if ignore_descendants
+
+      ::Gitlab::ObjectHierarchy.new(namespaces).base_and_descendants
     end
 
     def pseudonymizer_available?

ee/app/models/elasticsearch_indexed_namespace.rb

@@ -18,19 +18,6 @@ class ElasticsearchIndexedNamespace < ApplicationRecord
     :namespace_id
   end
 
-  def self.limited(ignore_descendants: false)
-    namespaces = Namespace.with_route.where(id: target_ids)
-
-    return namespaces if ignore_descendants
-
-    Gitlab::ObjectHierarchy.new(namespaces).base_and_descendants
-  end
-
-  def self.drop_limited_ids_cache!
-    ElasticsearchIndexedProject.drop_limited_ids_cache!
-    super
-  end
-
   def self.index_first_n_namespaces_of_plan(plan, number_of_namespaces)
     indexed_namespaces = self.select(:namespace_id)
     now = Time.now

ee/app/models/elasticsearch_indexed_project.rb

@@ -14,17 +14,6 @@ class ElasticsearchIndexedProject < ApplicationRecord
     :project_id
   end
 
-  def self.limited(ignore_namespaces: false)
-    return Project.inc_routes.where(id: target_ids) if ignore_namespaces
-
-    Project.inc_routes.from_union(
-      [
-        Project.where(namespace_id: ElasticsearchIndexedNamespace.limited.select(:id)),
-        Project.where(id: target_ids)
-      ]
-    )
-  end
-
   private
 
   def index

ee/app/views/admin/application_settings/_elasticsearch_form.html.haml

@@ -92,14 +92,14 @@
             - if elasticsearch_too_many_namespaces?
               %p= _('Too many namespaces enabled. You will need to manage them via the console or the API.')
             - else
-              = f.text_field :elasticsearch_namespace_ids, class: 'js-elasticsearch-namespaces', value: elasticsearch_namespace_ids, data: { selected: elasticsearch_objects_options(@application_setting.elasticsearch_limited_namespaces(ignore_descendants: true)).to_json }
+              = f.text_field :elasticsearch_namespace_ids, class: 'js-elasticsearch-namespaces', value: elasticsearch_namespace_ids, data: { selected: elasticsearch_objects_options(@application_setting.elasticsearch_limited_namespaces(true)).to_json }
 
           .form-group.js-limit-projects{ class: ('hidden' unless @application_setting.elasticsearch_limit_indexing) }
             = f.label :elasticsearch_project_ids, _('Projects to index'), class: 'label-bold'
             - if elasticsearch_too_many_projects?
               %p= _('Too many projects enabled. You will need to manage them via the console or the API.')
             - else
-              = f.text_field :elasticsearch_project_ids, class: 'js-elasticsearch-projects', value: elasticsearch_project_ids, data: { selected: elasticsearch_objects_options(@application_setting.elasticsearch_limited_projects(ignore_namespaces: true)).to_json }
+              = f.text_field :elasticsearch_project_ids, class: 'js-elasticsearch-projects', value: elasticsearch_project_ids, data: { selected: elasticsearch_objects_options(@application_setting.elasticsearch_limited_projects(true)).to_json }
 
         .sub-section
           %h4= _('Elasticsearch AWS IAM credentials')

ee/spec/models/application_setting_spec.rb

@@ -286,7 +286,7 @@ describe ApplicationSetting do
 
           expect(setting.elasticsearch_limited_namespaces).to match_array(
             [namespaces.last, child_namespace, child_namespace_indexed_through_parent])
-          expect(setting.elasticsearch_limited_namespaces(ignore_descendants: true)).to match_array(
+          expect(setting.elasticsearch_limited_namespaces(true)).to match_array(
             [namespaces.last, child_namespace])
         end
       end

ee/spec/models/elasticsearch_indexed_namespace_spec.rb

@@ -28,17 +28,6 @@ describe ElasticsearchIndexedNamespace do
     end
   end
 
-  context 'caching' do
-    it 'invalidates indexed project cache' do
-      expect(ElasticsearchIndexedProject).to receive(:drop_limited_ids_cache!).and_call_original.twice
-      expect(ElasticsearchIndexedNamespace).to receive(:drop_limited_ids_cache!).and_call_original.twice
-
-      n = create(:elasticsearch_indexed_namespace)
-
-      n.destroy
-    end
-  end
-
   context 'with plans' do
     Plan::PAID_HOSTED_PLANS.each do |plan|
       plan_factory = "#{plan}_plan"

ee/spec/support/shared_examples/models/elasticsearch_indexed_container_shared_examples.rb

@@ -9,25 +9,6 @@ RSpec.shared_examples 'an elasticsearch indexed container' do
     end
   end
 
-  describe '.limited_ids_cached', :use_clean_rails_memory_store_caching do
-    subject { create container }
-
-    it 'returns correct values' do
-      initial_ids = subject.class.limited_ids
-
-      expect(initial_ids).not_to be_empty
-      expect(subject.class.limited_ids_cached).to match_array(initial_ids)
-
-      new_container = create container
-
-      expect(subject.class.limited_ids_cached).to match_array(initial_ids + [new_container.id])
-
-      new_container.destroy
-
-      expect(subject.class.limited_ids_cached).to match_array(initial_ids)
-    end
-  end
-
   describe 'callbacks' do
     subject { build container }
 
@@ -43,12 +24,6 @@ RSpec.shared_examples 'an elasticsearch indexed container' do
 
         subject.save!
       end
-
-      it 'invalidates limited_ids cache' do
-        is_expected.to receive(:drop_limited_ids_cache!)
-
-        subject.save!
-      end
     end
 
     describe 'on destroy' do
@@ -65,12 +40,6 @@ RSpec.shared_examples 'an elasticsearch indexed container' do
 
         subject.destroy!
       end
-
-      it 'invalidates limited_ids cache' do
-        is_expected.to receive(:drop_limited_ids_cache!)
-
-        subject.destroy!
-      end
     end
   end
 end