Fix performance of GroupsWithTemplatesFinder query

- Introduce OptimizedGroupsWithTemplatesFinder that queries group
templates more efficiently.
- Introduce a feature flag to enable the optimization:
`optimized_groups_with_templates_finder`

ee/app/finders/groups_with_templates_finder.rb

@@ -21,15 +21,30 @@ class GroupsWithTemplatesFinder
 
   attr_reader :group_id
 
+  # Cleanup issue: https://gitlab.com/gitlab-org/gitlab/issues/35733
   def extended_group_search
-    groups = Group.with_feature_available_in_plan(:group_project_templates)
-    Gitlab::ObjectHierarchy.new(groups).base_and_descendants
+    if ::Feature.enabled?(:optimized_groups_with_templates_finder)
+      groups = Group.with_project_templates_optimized
+      groups_with_plan = Gitlab::ObjectHierarchy
+        .new(groups)
+        .base_and_ancestors
+        .with_feature_available_in_plan(:group_project_templates)
+
+      Gitlab::ObjectHierarchy.new(groups_with_plan).base_and_descendants
+    else
+      groups = Group.with_feature_available_in_plan(:group_project_templates)
+      Gitlab::ObjectHierarchy.new(groups).base_and_descendants
+    end
   end
 
   def simple_group_search(groups)
     groups = group_id ? groups.find_by(id: group_id)&.self_and_ancestors : groups # rubocop: disable CodeReuse/ActiveRecord
     return Group.none unless groups
 
-    groups.with_project_templates
+    if ::Feature.enabled?(:optimized_groups_with_templates_finder)
+      groups.with_project_templates_optimized
+    else
+      groups.with_project_templates
+    end
   end
 end

ee/app/models/ee/group.rb

@@ -59,9 +59,11 @@ module EE
 
       scope :with_project_templates, -> do
         joins("INNER JOIN projects ON projects.namespace_id = namespaces.custom_project_templates_group_id")
-        .distinct
+          .distinct
       end
 
+      scope :with_project_templates_optimized, -> { where.not(custom_project_templates_group_id: nil) }
+
       scope :with_custom_file_templates, -> do
         preload(
           file_template_project: :route,

ee/app/models/ee/user.rb

@@ -207,14 +207,24 @@ module EE
     end
 
     def available_subgroups_with_custom_project_templates(group_id = nil)
-      groups = GroupsWithTemplatesFinder.new(group_id).execute
-
-      GroupsFinder.new(self, min_access_level: ::Gitlab::Access::DEVELOPER)
-                  .execute
-                  .where(id: groups.select(:custom_project_templates_group_id))
-                  .includes(:projects)
-                  .reorder(nil)
-                  .distinct
+      found_groups = GroupsWithTemplatesFinder.new(group_id).execute
+
+      if ::Feature.enabled?(:optimized_groups_with_templates_finder)
+        GroupsFinder.new(self, min_access_level: ::Gitlab::Access::DEVELOPER)
+          .execute
+          .where(id: found_groups.select(:custom_project_templates_group_id))
+          .preload(:projects)
+          .joins(:projects)
+          .reorder(nil)
+          .distinct
+      else
+        GroupsFinder.new(self, min_access_level: ::Gitlab::Access::DEVELOPER)
+          .execute
+          .where(id: found_groups.select(:custom_project_templates_group_id))
+          .includes(:projects)
+          .reorder(nil)
+          .distinct
+      end
     end
 
     def roadmap_layout

ee/changelogs/unreleased/maybe-fix-projects-newperformance.yml

+---
+title: Fix new project page load performance
+merge_request: 18180
+author:
+type: performance

ee/spec/features/projects/new_project_spec.rb

@@ -269,7 +269,7 @@ describe 'New project' do
         end
       end
 
-      context 'when custom project group template is set' do
+      shared_context 'when custom project group template is set' do
         let(:group1) { create(:group) }
         let(:group2) { create(:group) }
         let(:group3) { create(:group) }
@@ -429,6 +429,23 @@ describe 'New project' do
         end
       end
 
+      # Cleanup issue: https://gitlab.com/gitlab-org/gitlab/issues/35733
+      context 'when `optimized_groups_with_templates_finder` feature flag is enabled' do
+        before do
+          stub_feature_flags(optimized_groups_with_templates_finder: true)
+        end
+
+        include_context 'when custom project group template is set'
+      end
+
+      context 'when `optimized_groups_with_templates_finder` feature flag is disabled' do
+        before do
+          stub_feature_flags(optimized_groups_with_templates_finder: false)
+        end
+
+        include_context 'when custom project group template is set'
+      end
+
       context 'when group template is not set' do
         it_behaves_like 'group templates displayed' do
           let(:template_number) { 0 }

ee/spec/finders/groups_with_templates_finder_spec.rb

@@ -26,94 +26,112 @@ describe GroupsWithTemplatesFinder do
     create(:gitlab_subscription, :silver, namespace: group_2)
   end
 
-  describe 'without group id' do
-    it 'returns all groups' do
-      expect(described_class.new.execute).to contain_exactly(group_1, group_2, group_3)
-    end
-
-    context 'when namespace checked' do
-      before do
-        allow(Gitlab::CurrentSettings).to receive(:should_check_namespace_plan?) { true }
+  shared_examples 'GroupsWithTemplatesFinder examples' do
+    describe 'without group id' do
+      it 'returns all groups' do
+        expect(described_class.new.execute).to contain_exactly(group_1, group_2, group_3)
       end
 
-      it 'returns all groups before cut-off date' do
-        Timecop.freeze(described_class::CUT_OFF_DATE - 1.day) do
-          expect(described_class.new.execute).to contain_exactly(group_1, group_2, group_3)
-        end
-      end
-
-      it 'returns groups on gold/silver plan after cut-off date' do
-        Timecop.freeze(described_class::CUT_OFF_DATE + 1.day) do
-          expect(described_class.new.execute).to contain_exactly(group_1, group_2)
+      context 'when namespace checked' do
+        before do
+          allow(Gitlab::CurrentSettings).to receive(:should_check_namespace_plan?) { true }
         end
-      end
 
-      context 'with subgroup with template' do
-        before do
-          subgroup_4.update!(custom_project_templates_group_id: subgroup_5.id)
-          create(:project, namespace: subgroup_5)
+        it 'returns all groups before cut-off date' do
+          Timecop.freeze(described_class::CUT_OFF_DATE - 1.day) do
+            expect(described_class.new.execute).to contain_exactly(group_1, group_2, group_3)
+          end
         end
 
         it 'returns groups on gold/silver plan after cut-off date' do
           Timecop.freeze(described_class::CUT_OFF_DATE + 1.day) do
-            expect(described_class.new.execute).to contain_exactly(group_1, group_2, subgroup_4)
+            expect(described_class.new.execute).to contain_exactly(group_1, group_2)
           end
         end
-      end
-    end
-  end
-
-  describe 'with group id' do
-    it 'returns given group with it descendants' do
-      expect(described_class.new(group_1.id).execute).to contain_exactly(group_1)
-    end
 
-    context 'with subgroup with template' do
-      before do
-        subgroup_4.update!(custom_project_templates_group_id: subgroup_5.id)
-        create(:project, namespace: subgroup_5)
-      end
+        context 'with subgroup with template' do
+          before do
+            subgroup_4.update!(custom_project_templates_group_id: subgroup_5.id)
+            create(:project, namespace: subgroup_5)
+          end
 
-      it 'returns only chosen group' do
-        expect(described_class.new(group_1.id).execute).to contain_exactly(group_1)
+          it 'returns groups on gold/silver plan after cut-off date' do
+            Timecop.freeze(described_class::CUT_OFF_DATE + 1.day) do
+              expect(described_class.new.execute).to contain_exactly(group_1, group_2, subgroup_4)
+            end
+          end
+        end
       end
     end
 
-    context 'when namespace checked' do
-      before do
-        allow(Gitlab::CurrentSettings).to receive(:should_check_namespace_plan?) { true }
+    describe 'with group id' do
+      it 'returns given group with it descendants' do
+        expect(described_class.new(group_1.id).execute).to contain_exactly(group_1)
       end
 
-      it 'returns given group with it descendants before cut-off date' do
-        Timecop.freeze(described_class::CUT_OFF_DATE - 1.day) do
-          expect(described_class.new(group_3.id).execute).to contain_exactly(group_3)
+      context 'with subgroup with template' do
+        before do
+          subgroup_4.update!(custom_project_templates_group_id: subgroup_5.id)
+          create(:project, namespace: subgroup_5)
         end
-      end
 
-      it 'does not return the group after the cut-off date' do
-        Timecop.freeze(described_class::CUT_OFF_DATE + 1.day) do
-          expect(described_class.new(group_3.id).execute).to be_empty
+        it 'returns only chosen group' do
+          expect(described_class.new(group_1.id).execute).to contain_exactly(group_1)
         end
       end
 
-      context 'with subgroup with template' do
+      context 'when namespace checked' do
         before do
-          subgroup_4.update!(custom_project_templates_group_id: subgroup_5.id)
-          create(:project, namespace: subgroup_5)
+          allow(Gitlab::CurrentSettings).to receive(:should_check_namespace_plan?) { true }
         end
 
-        it 'returns only chosen group' do
-          Timecop.freeze(described_class::CUT_OFF_DATE + 1.day) do
-            expect(described_class.new(group_1.id).execute).to contain_exactly(group_1)
+        it 'returns given group with it descendants before cut-off date' do
+          Timecop.freeze(described_class::CUT_OFF_DATE - 1.day) do
+            expect(described_class.new(group_3.id).execute).to contain_exactly(group_3)
           end
         end
 
-        it 'returns only chosen subgroup' do
+        it 'does not return the group after the cut-off date' do
           Timecop.freeze(described_class::CUT_OFF_DATE + 1.day) do
-            expect(described_class.new(subgroup_4.id).execute).to contain_exactly(group_1, subgroup_4)
+            expect(described_class.new(group_3.id).execute).to be_empty
+          end
+        end
+
+        context 'with subgroup with template' do
+          before do
+            subgroup_4.update!(custom_project_templates_group_id: subgroup_5.id)
+            create(:project, namespace: subgroup_5)
+          end
+
+          it 'returns only chosen group' do
+            Timecop.freeze(described_class::CUT_OFF_DATE + 1.day) do
+              expect(described_class.new(group_1.id).execute).to contain_exactly(group_1)
+            end
+          end
+
+          it 'returns only chosen subgroup' do
+            Timecop.freeze(described_class::CUT_OFF_DATE + 1.day) do
+              expect(described_class.new(subgroup_4.id).execute).to contain_exactly(group_1, subgroup_4)
+            end
           end
         end
       end
     end
   end
+
+  context 'when `optimized_groups_with_templates_finder` feature flag is enabled' do
+    before do
+      stub_feature_flags(optimized_groups_with_templates_finder: true)
+    end
+
+    include_examples 'GroupsWithTemplatesFinder examples'
+  end
+
+  context 'when `optimized_groups_with_templates_finder` feature flag is disabled' do
+    before do
+      stub_feature_flags(optimized_groups_with_templates_finder: false)
+    end
+
+    include_examples 'GroupsWithTemplatesFinder examples'
+  end
 end