Merge branch '325274-use-cte-to-optimize-packages-query' into 'master'

Fix maven package query performance [RUN ALL RSPEC] [RUN AS-IF-FOSS]

See merge request gitlab-org/gitlab!57041

app/finders/packages/maven/package_finder.rb

@@ -33,7 +33,7 @@ module Packages
       end
 
       def packages_with_path
-        matching_packages = base.only_maven_packages_with_path(path)
+        matching_packages = base.only_maven_packages_with_path(path, use_cte: group.present?)
         matching_packages = matching_packages.order_by_package_file if versionless_package?(matching_packages)
 
         matching_packages

app/models/packages/package.rb

@@ -141,8 +141,19 @@ class Packages::Package < ApplicationRecord
     where(project_id: projects)
   end
 
-  def self.only_maven_packages_with_path(path)
-    joins(:maven_metadatum).where(packages_maven_metadata: { path: path })
+  def self.only_maven_packages_with_path(path, use_cte: false)
+    if use_cte && Feature.enabled?(:maven_metadata_by_path_with_optimization_fence)
+      # This is an optimization fence which assumes that looking up the Metadatum record by path (globally)
+      # and then filter down the packages (by project or by group and subgroups) will be cheaper than
+      # looking up all packages within a project or group and filter them by path.
+
+      inner_query = Packages::Maven::Metadatum.where(path: path).select(:id, :package_id)
+      cte = Gitlab::SQL::CTE.new(:maven_metadata_by_path, inner_query)
+      with(cte.to_arel)
+        .joins('INNER JOIN maven_metadata_by_path ON maven_metadata_by_path.package_id=packages_packages.id')
+    else
+      joins(:maven_metadatum).where(packages_maven_metadata: { path: path })
+    end
   end
 
   def self.by_name_and_file_name(name, file_name)

config/feature_flags/development/maven_metadata_by_path_with_optimization_fence.yml

+---
+name: maven_metadata_by_path_with_optimization_fence
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/57041
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/325460
+milestone: '13.11'
+type: development
+group: group::optimize
+default_enabled: false

spec/finders/packages/maven/package_finder_spec.rb

@@ -17,65 +17,89 @@ RSpec.describe ::Packages::Maven::PackageFinder do
     group.add_developer(user)
   end
 
-  describe '#execute!' do
-    subject { finder.execute! }
+  shared_examples 'Packages::Maven::PackageFinder examples' do
+    describe '#execute!' do
+      subject { finder.execute! }
 
-    shared_examples 'handling valid and invalid paths' do
-      context 'with a valid path' do
-        let(:param_path) { package.maven_metadatum.path }
+      shared_examples 'handling valid and invalid paths' do
+        context 'with a valid path' do
+          let(:param_path) { package.maven_metadatum.path }
 
-        it { is_expected.to eq(package) }
+          it { is_expected.to eq(package) }
+        end
+
+        context 'with an invalid path' do
+          let(:param_path) { 'com/example/my-app/1.0-SNAPSHOT' }
+
+          it 'raises an error' do
+            expect { subject }.to raise_error(ActiveRecord::RecordNotFound)
+          end
+        end
+      end
+
+      context 'within the project' do
+        let(:param_project) { project }
+
+        it_behaves_like 'handling valid and invalid paths'
       end
 
-      context 'with an invalid path' do
-        let(:param_path) { 'com/example/my-app/1.0-SNAPSHOT' }
+      context 'within a group' do
+        let(:param_group) { group }
 
+        it_behaves_like 'handling valid and invalid paths'
+      end
+
+      context 'across all projects' do
         it 'raises an error' do
           expect { subject }.to raise_error(ActiveRecord::RecordNotFound)
         end
       end
-    end
 
-    context 'within the project' do
-      let(:param_project) { project }
+      context 'versionless maven-metadata.xml package' do
+        let_it_be(:sub_group1) { create(:group, parent: group) }
+        let_it_be(:sub_group2)   { create(:group, parent: group) }
+        let_it_be(:project1) { create(:project, group: sub_group1) }
+        let_it_be(:project2) { create(:project, group: sub_group2) }
+        let_it_be(:project3) { create(:project, group: sub_group1) }
+        let_it_be(:package_name) { 'foo' }
+        let_it_be(:package1) { create(:maven_package, project: project1, name: package_name, version: nil) }
+        let_it_be(:package2) { create(:maven_package, project: project2, name: package_name, version: nil) }
+        let_it_be(:package3) { create(:maven_package, project: project3, name: package_name, version: nil) }
+
+        let(:param_group) { group }
+        let(:param_path) { package_name }
+
+        before do
+          sub_group1.add_developer(user)
+          sub_group2.add_developer(user)
+          # the package with the most recently published file should be returned
+          create(:package_file, :xml, package: package2)
+        end
 
-      it_behaves_like 'handling valid and invalid paths'
+        it { is_expected.to eq(package2) }
+      end
     end
+  end
 
-    context 'within a group' do
-      let(:param_group) { group }
-
-      it_behaves_like 'handling valid and invalid paths'
+  context 'when the maven_metadata_by_path_with_optimization_fence feature flag is off' do
+    before do
+      stub_feature_flags(maven_metadata_by_path_with_optimization_fence: false)
     end
 
-    context 'across all projects' do
-      it 'raises an error' do
-        expect { subject }.to raise_error(ActiveRecord::RecordNotFound)
-      end
+    it_behaves_like 'Packages::Maven::PackageFinder examples'
+  end
+
+  context 'when the maven_metadata_by_path_with_optimization_fence feature flag is on' do
+    before do
+      stub_feature_flags(maven_metadata_by_path_with_optimization_fence: true)
     end
 
-    context 'versionless maven-metadata.xml package' do
-      let_it_be(:sub_group1) { create(:group, parent: group) }
-      let_it_be(:sub_group2)   { create(:group, parent: group) }
-      let_it_be(:project1) { create(:project, group: sub_group1) }
-      let_it_be(:project2) { create(:project, group: sub_group2) }
-      let_it_be(:project3) { create(:project, group: sub_group1) }
-      let_it_be(:package_name) { 'foo' }
-      let_it_be(:package1) { create(:maven_package, project: project1, name: package_name, version: nil) }
-      let_it_be(:package2) { create(:maven_package, project: project2, name: package_name, version: nil) }
-      let_it_be(:package3) { create(:maven_package, project: project3, name: package_name, version: nil) }
-
-      let(:param_group) { group }
-      let(:param_path) { package_name }
-
-      before do
-        sub_group1.add_developer(user)
-        sub_group2.add_developer(user)
-        # the package with the most recently published file should be returned
-        create(:package_file, :xml, package: package2)
-      end
+    it_behaves_like 'Packages::Maven::PackageFinder examples'
+
+    it 'uses CTE in the query' do
+      sql = described_class.new('some_path', user, group: group).send(:packages_with_path).to_sql
 
-      it { is_expected.to eq(package2) }
+      expect(sql).to include('WITH "maven_metadata_by_path" AS')
     end
   end
 end