Skip to content

G
gitlab-ce
Commit def42f65 authored by Dylan Griffith's avatar Dylan Griffith
Browse files
Options

Merge branch... 

Merge branch '300843-change-project-group-notes-es-searches-to-use-new-denormalized-fields' into 'master'

Change notes ES searches to use new denormalized fields

See merge request gitlab-org/gitlab!53294
parents ffbcbcfd 98bc2786
Show whitespace changes
Inline Side-by-side
Showing with 683 additions and 305 deletions
ee/lib/elastic/latest/note_class_proxy.rb
View file @ def42f65
......@@ -13,6 +13,7 @@ module Elastic
options[:in] = ['note']
query_hash = basic_query_hash(%w[note], query, count_only: options[:count_only])
options[:no_join_project] = Elastic::DataMigrationService.migration_has_finished?(:add_permissions_data_to_notes_documents)
context.name(:note) do
query_hash = context.name(:authorized) { project_ids_filter(query_hash, options) }
query_hash = context.name(:confidentiality) { confidentiality_filter(query_hash, options) }
......@@ -100,6 +101,9 @@ module Elastic
override :project_ids_filter
def project_ids_filter(query_hash, options)
# support for not using project joins in the query
no_join_project = options[:no_join_project]
query_hash[:query][:bool][:filter] ||= []
project_query = context.name(:project) do
......@@ -107,7 +111,8 @@ module Elastic
options[:current_user],
options[:project_ids],
options[:public_and_internal_projects],
options[:features]
options[:features],
no_join_project
)
end
......@@ -124,9 +129,17 @@ module Elastic
project_query[:should].flatten.each do |condition|
noteable_type = condition.delete(:noteable_type).to_s
filters[:bool][:should] << {
should_filter = {
bool: {
must: [
{ term: { noteable_type: { _name: context.name(:noteable, :is_a, noteable_type), value: noteable_type } } }
]
}
}
should_filter[:bool][:must] << if no_join_project
condition
else
{
has_parent: {
parent_type: "project",
......@@ -136,13 +149,12 @@ module Elastic
}
}
}
},
{ term: { noteable_type: { _name: context.name(:noteable, :is_a, noteable_type), value: noteable_type } } }
]
}
}
end
filters[:bool][:should] << should_filter
end
query_hash[:query][:bool][:filter] << filters
query_hash
end
......@@ -150,18 +162,18 @@ module Elastic
# Query notes based on the various feature permission of the noteable_type.
# Appends `noteable_type` (which will be removed in project_ids_filter)
# for base model filtering.
# We do not implement `no_join_project` argument for notes class yet
# as this is not supported. This will need to be fixed when we move
# notes to a new index.
override :pick_projects_by_membership
def pick_projects_by_membership(project_ids, user, _, _ = nil)
def pick_projects_by_membership(project_ids, user, no_join_project, _ = nil)
# support for not using project joins in the query
project_id_key = no_join_project ? :project_id : :id
noteable_type_to_feature.map do |noteable_type, feature|
context.name(feature) do
condition =
if project_ids == :any
{ term: { visibility_level: { _name: context.name(:any), value: Project::PRIVATE } } }
else
{ terms: { _name: context.name(:membership, :id), id: filter_ids_by_feature(project_ids, user, feature) } }
{ terms: { _name: context.name(:membership, :id), project_id_key => filter_ids_by_feature(project_ids, user, feature) } }
end
limit =
......
ee/spec/services/search/global_service_spec.rb
View file @ def42f65
......@@ -18,248 +18,259 @@ RSpec.describe Search::GlobalService do
let(:service) { described_class.new(user, params) }
end
context 'issue search' do
let(:results) { described_class.new(nil, search: '*').execute.objects('issues') }
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).and_call_original
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?)
.with(:migrate_issues_to_separate_index)
.and_return(false)
end
it_behaves_like 'search query applies joins based on migrations shared examples', :add_new_data_to_issues_documents
end
context 'notes search' do
let(:results) { described_class.new(nil, search: '*').execute.objects('notes') }
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).and_call_original
end
it_behaves_like 'search query applies joins based on migrations shared examples', :add_permissions_data_to_notes_documents
end
context 'visibility', :elastic, :sidekiq_inline do
include_context 'ProjectPolicyTable context'
shared_examples 'search respects visibility' do
it 'respects visibility' do
enable_admin_mode!(user) if admin_mode
update_feature_access_level(project, feature_access_level)
ensure_elasticsearch_index!
expect_search_results(user, scope, expected_count: expected_count) do |user|
described_class.new(user, search: search).execute
end
end
end
let_it_be(:group) { create(:group) }
let(:project) { create(:project, project_level, namespace: group) }
let(:user) { create_user_from_membership(project, membership) }
context 'merge request' do
let!(:merge_request) { create :merge_request, target_project: project, source_project: project }
let!(:note) { create :note, project: project, noteable: merge_request }
let(:scope) { 'merge_requests' }
let(:search) { merge_request.title }
where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
permission_table_for_reporter_feature_access
end
with_them do
it "respects visibility" do
enable_admin_mode!(user) if admin_mode
update_feature_access_level(project, feature_access_level)
ensure_elasticsearch_index!
expect_search_results(user, 'merge_requests', expected_count: expected_count) do |user|
described_class.new(user, search: merge_request.title).execute
end
expect_search_results(user, 'notes', expected_count: expected_count) do |user|
described_class.new(user, search: note.note).execute
end
end
it_behaves_like 'search respects visibility'
end
end
context 'code' do
context 'blob and commit' do
let!(:project) { create(:project, project_level, :repository, namespace: group ) }
let!(:note) { create :note_on_commit, project: project }
where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
permission_table_for_guest_feature_access_and_non_private_project_only
end
with_them do
it "respects visibility" do
enable_admin_mode!(user) if admin_mode
update_feature_access_level(project, feature_access_level)
ElasticCommitIndexerWorker.new.perform(project.id)
ensure_elasticsearch_index!
expect_search_results(user, 'commits', expected_count: expected_count) do |user|
described_class.new(user, search: 'initial').execute
before do
project.repository.index_commits_and_blobs
end
expect_search_results(user, 'blobs', expected_count: expected_count) do |user|
described_class.new(user, search: '.gitmodules').execute
it_behaves_like 'search respects visibility' do
let(:scope) { 'commits' }
let(:search) { 'initial' }
end
expect_search_results(user, 'notes', expected_count: expected_count) do |user|
described_class.new(user, search: note.note).execute
end
it_behaves_like 'search respects visibility' do
let(:scope) { 'blobs' }
let(:search) { '.gitmodules' }
end
end
end
context 'issue' do
let(:scope) { 'issues' }
context 'note' do
let(:scope) { 'notes' }
let(:search) { note.note }
context 'visibility' do
let!(:issue) { create :issue, project: project }
let!(:note) { create :note, project: project, noteable: issue }
context 'on issues' do
let!(:note) { create :note_on_issue, project: project }
where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
permission_table_for_guest_feature_access
end
with_them do
it "respects visibility" do
enable_admin_mode!(user) if admin_mode
update_feature_access_level(project, feature_access_level)
ensure_elasticsearch_index!
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).and_call_original
end
expect_search_results(user, 'issues', expected_count: expected_count) do |user|
described_class.new(user, search: issue.title).execute
context 'when add_permissions_data_to_notes_documents migration is finished' do
it_behaves_like 'search respects visibility'
end
expect_search_results(user, 'notes', expected_count: expected_count) do |user|
described_class.new(user, search: note.note).execute
context 'when add_permissions_data_to_notes_documents migration is not finished' do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).with(:add_permissions_data_to_notes_documents).and_return(false)
end
it_behaves_like 'search respects visibility'
end
end
end
# Since newly created indices automatically have all migrations as
# finished we need a test to verify the old style searches work for
# instances which haven't finished the migration yet
context 'when add_new_data_to_issues_documents migration is not finished' do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).and_call_original
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?)
.with(:add_new_data_to_issues_documents)
.and_return(false)
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?)
.with(:migrate_issues_to_separate_index)
.and_return(false)
end
# issue cannot be defined prior to the migration mocks because it
# will cause the incorrect value to be passed to `use_separate_indices` when creating
# the proxy
let!(:issue) { create(:issue, project: project) }
context 'on merge requests' do
let!(:note) { create :note_on_merge_request, project: project }
where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
permission_table_for_guest_feature_access
permission_table_for_reporter_feature_access
end
with_them do
it "respects visibility" do
enable_admin_mode!(user) if admin_mode
update_feature_access_level(project, feature_access_level)
ensure_elasticsearch_index!
expect_search_results(user, 'issues', expected_count: expected_count) do |user|
described_class.new(user, search: issue.title).execute
end
end
end
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).and_call_original
end
context 'ordering' do
let_it_be(:project) { create(:project, :public) }
let!(:old_result) { create(:issue, project: project, title: 'sorted old', created_at: 1.month.ago) }
let!(:new_result) { create(:issue, project: project, title: 'sorted recent', created_at: 1.day.ago) }
let!(:very_old_result) { create(:issue, project: project, title: 'sorted very old', created_at: 1.year.ago) }
let!(:old_updated) { create(:issue, project: project, title: 'updated old', updated_at: 1.month.ago) }
let!(:new_updated) { create(:issue, project: project, title: 'updated recent', updated_at: 1.day.ago) }
let!(:very_old_updated) { create(:issue, project: project, title: 'updated very old', updated_at: 1.year.ago) }
context 'when add_permissions_data_to_notes_documents migration is finished' do
it_behaves_like 'search respects visibility'
end
context 'when add_permissions_data_to_notes_documents migration is not finished' do
before do
ensure_elasticsearch_index!
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).with(:add_permissions_data_to_notes_documents).and_return(false)
end
include_examples 'search results sorted' do
let(:results_created) { described_class.new(nil, search: 'sorted', sort: sort).execute }
let(:results_updated) { described_class.new(nil, search: 'updated', sort: sort).execute }
it_behaves_like 'search respects visibility'
end
end
end
context 'using joins for global permission checks' do
let(:results) { described_class.new(nil, search: '*').execute.objects('issues') }
let(:es_host) { Gitlab::CurrentSettings.elasticsearch_url[0] }
let(:search_url) { Addressable::Template.new("#{es_host}/{index}/doc/_search{?params*}") }
context 'on commits' do
let!(:project) { create(:project, project_level, :repository, namespace: group ) }
let!(:note) { create :note_on_commit, project: project }
where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
permission_table_for_guest_feature_access_and_non_private_project_only
end
with_them do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).and_call_original
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?)
.with(:migrate_issues_to_separate_index)
.and_return(false)
project.repository.index_commits_and_blobs
end
ensure_elasticsearch_index!
context 'when add_permissions_data_to_notes_documents migration is finished' do
it_behaves_like 'search respects visibility'
end
context 'when add_new_data_to_issues_documents migration is finished' do
context 'when add_permissions_data_to_notes_documents migration is not finished' do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?)
.with(:add_new_data_to_issues_documents)
.and_return(true)
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).with(:add_permissions_data_to_notes_documents).and_return(false)
end
it 'does not use joins to apply permissions' do
request = a_request(:get, search_url).with do |req|
expect(req.body).not_to include("has_parent")
it_behaves_like 'search respects visibility'
end
end
end
results
context 'on snippets' do
let!(:note) { create :note_on_project_snippet, project: project }
expect(request).to have_been_made
end
where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
permission_table_for_guest_feature_access
end
context 'when add_new_data_to_issues_documents migration is not finished' do
with_them do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?)
.with(:add_new_data_to_issues_documents)
.and_return(false)
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).and_call_original
end
it 'uses joins to apply permissions' do
request = a_request(:get, search_url).with do |req|
expect(req.body).to include("has_parent")
context 'when add_permissions_data_to_notes_documents migration is finished' do
it_behaves_like 'search respects visibility'
end
results
context 'when add_permissions_data_to_notes_documents migration is not finished' do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).with(:add_permissions_data_to_notes_documents).and_return(false)
end
expect(request).to have_been_made
it_behaves_like 'search respects visibility'
end
end
end
end
context 'merge_request' do
let(:scope) { 'merge_requests' }
context 'issue' do
let(:scope) { 'issues' }
let(:search) { issue.title }
context 'sorting' do
let!(:project) { create(:project, :public) }
context 'when add_new_data_to_issues_documents migration is finished' do
let!(:issue) { create :issue, project: project }
let!(:old_result) { create(:merge_request, :opened, source_project: project, source_branch: 'old-1', title: 'sorted old', created_at: 1.month.ago) }
let!(:new_result) { create(:merge_request, :opened, source_project: project, source_branch: 'new-1', title: 'sorted recent', created_at: 1.day.ago) }
let!(:very_old_result) { create(:merge_request, :opened, source_project: project, source_branch: 'very-old-1', title: 'sorted very old', created_at: 1.year.ago) }
where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
permission_table_for_guest_feature_access
end
let!(:old_updated) { create(:merge_request, :opened, source_project: project, source_branch: 'updated-old-1', title: 'updated old', updated_at: 1.month.ago) }
let!(:new_updated) { create(:merge_request, :opened, source_project: project, source_branch: 'updated-new-1', title: 'updated recent', updated_at: 1.day.ago) }
let!(:very_old_updated) { create(:merge_request, :opened, source_project: project, source_branch: 'updated-very-old-1', title: 'updated very old', updated_at: 1.year.ago) }
with_them do
it_behaves_like 'search respects visibility'
end
end
# Since newly created indices automatically have all migrations as
# finished we need a test to verify the old style searches work for
# instances which haven't finished the migration yet
context 'when add_new_data_to_issues_documents migration is not finished' do
before do
ensure_elasticsearch_index!
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).and_call_original
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?)
.with(:add_new_data_to_issues_documents)
.and_return(false)
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?)
.with(:migrate_issues_to_separate_index)
.and_return(false)
end
include_examples 'search results sorted' do
let(:results_created) { described_class.new(nil, search: 'sorted', sort: sort).execute }
let(:results_updated) { described_class.new(nil, search: 'updated', sort: sort).execute }
# issue cannot be defined prior to the migration mocks because it
# will cause the incorrect value to be passed to `use_separate_indices` when creating
# the proxy
let!(:issue) { create(:issue, project: project) }
where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
permission_table_for_guest_feature_access
end
with_them do
it_behaves_like 'search respects visibility'
end
end
end
context 'wiki' do
let!(:project) { create(:project, project_level, :wiki_repo) }
let(:scope) { 'wiki_blobs' }
let(:search) { 'term' }
where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
permission_table_for_guest_feature_access
end
with_them do
it "respects visibility" do
enable_admin_mode!(user) if admin_mode
project.wiki.create_page('test.md', '# term')
before do
project.wiki.create_page('test.md', "# #{search}")
project.wiki.index_wiki_blobs
update_feature_access_level(project, feature_access_level)
ensure_elasticsearch_index!
expect_search_results(user, 'wiki_blobs', expected_count: expected_count) do |user|
described_class.new(user, search: 'term').execute
end
it_behaves_like 'search respects visibility' do
let(:projects) { [project] }
end
end
end
......@@ -312,6 +323,52 @@ RSpec.describe Search::GlobalService do
end
end
context 'sorting', :elastic, :sidekiq_inline do
context 'issue' do
let(:scope) { 'issues' }
let_it_be(:project) { create(:project, :public) }
let!(:old_result) { create(:issue, project: project, title: 'sorted old', created_at: 1.month.ago) }
let!(:new_result) { create(:issue, project: project, title: 'sorted recent', created_at: 1.day.ago) }
let!(:very_old_result) { create(:issue, project: project, title: 'sorted very old', created_at: 1.year.ago) }
let!(:old_updated) { create(:issue, project: project, title: 'updated old', updated_at: 1.month.ago) }
let!(:new_updated) { create(:issue, project: project, title: 'updated recent', updated_at: 1.day.ago) }
let!(:very_old_updated) { create(:issue, project: project, title: 'updated very old', updated_at: 1.year.ago) }
before do
ensure_elasticsearch_index!
end
include_examples 'search results sorted' do
let(:results_created) { described_class.new(nil, search: 'sorted', sort: sort).execute }
let(:results_updated) { described_class.new(nil, search: 'updated', sort: sort).execute }
end
end
context 'merge request' do
let(:scope) { 'merge_requests' }
let(:project) { create(:project, :public) }
let!(:old_result) { create(:merge_request, :opened, source_project: project, source_branch: 'old-1', title: 'sorted old', created_at: 1.month.ago) }
let!(:new_result) { create(:merge_request, :opened, source_project: project, source_branch: 'new-1', title: 'sorted recent', created_at: 1.day.ago) }
let!(:very_old_result) { create(:merge_request, :opened, source_project: project, source_branch: 'very-old-1', title: 'sorted very old', created_at: 1.year.ago) }
let!(:old_updated) { create(:merge_request, :opened, source_project: project, source_branch: 'updated-old-1', title: 'updated old', updated_at: 1.month.ago) }
let!(:new_updated) { create(:merge_request, :opened, source_project: project, source_branch: 'updated-new-1', title: 'updated recent', updated_at: 1.day.ago) }
let!(:very_old_updated) { create(:merge_request, :opened, source_project: project, source_branch: 'updated-very-old-1', title: 'updated very old', updated_at: 1.year.ago) }
before do
ensure_elasticsearch_index!
end
include_examples 'search results sorted' do
let(:results_created) { described_class.new(nil, search: 'sorted', sort: sort).execute }
let(:results_updated) { described_class.new(nil, search: 'updated', sort: sort).execute }
end
end
end
describe '#allowed_scopes' do
context 'when ES is used' do
it 'includes ES-specific scopes' do
......@@ -396,23 +453,81 @@ RSpec.describe Search::GlobalService do
end
context 'confidential notes' do
let(:project) { create(:project, :public) }
let(:project) { create(:project, :public, :repository) }
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).and_call_original
end
context 'with notes on issues' do
it_behaves_like 'search notes shared examples' do
let(:noteable) { create :issue, project: project }
context 'when add_permissions_data_to_notes_documents migration has not finished' do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?)
.with(:add_permissions_data_to_notes_documents)
.and_return(false)
end
it_behaves_like 'search notes shared examples', :note_on_issue
end
context 'when add_permissions_data_to_notes_documents migration has finished' do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?)
.with(:add_permissions_data_to_notes_documents)
.and_return(true)
end
it_behaves_like 'search notes shared examples', :note_on_issue
end
end
context 'with notes on merge requests' do
it_behaves_like 'search notes shared examples' do
let(:noteable) { create :merge_request, target_project: project, source_project: project }
context 'when add_permissions_data_to_notes_documents migration has not finished' do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?)
.with(:add_permissions_data_to_notes_documents)
.and_return(false)
end
it_behaves_like 'search notes shared examples', :note_on_merge_request
end
context 'when add_permissions_data_to_notes_documents migration has finished' do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?)
.with(:add_permissions_data_to_notes_documents)
.and_return(true)
end
it_behaves_like 'search notes shared examples', :note_on_merge_request
end
end
context 'with notes on commits' do
it_behaves_like 'search notes shared examples' do
let(:noteable) { create(:commit, project: project) }
context 'when add_permissions_data_to_notes_documents migration has not finished' do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?)
.with(:add_permissions_data_to_notes_documents)
.and_return(false)
end
it_behaves_like 'search notes shared examples', :note_on_commit
end
context 'when add_permissions_data_to_notes_documents migration has finished' do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?)
.with(:add_permissions_data_to_notes_documents)
.and_return(true)
end
it_behaves_like 'search notes shared examples', :note_on_commit
end
end
end
......
ee/spec/services/search/group_service_spec.rb
View file @ def42f65
......@@ -67,124 +67,233 @@ RSpec.describe Search::GroupService, :elastic do
end
end
context 'notes search' do
let_it_be(:group) { create(:group) }
let_it_be(:project) { create(:project, namespace: group) }
let(:results) { described_class.new(nil, group, search: 'test').execute.objects('notes') }
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).and_call_original
end
it_behaves_like 'search query applies joins based on migrations shared examples', :add_permissions_data_to_notes_documents
end
context 'visibility', :sidekiq_inline do
include_context 'ProjectPolicyTable context'
shared_examples 'search respects visibility' do
it 'respects visibility' do
enable_admin_mode!(user) if admin_mode
projects.each do |project|
update_feature_access_level(project, feature_access_level)
end
ensure_elasticsearch_index!
expect_search_results(user, scope, expected_count: expected_count) do |user|
described_class.new(user, group, search: search).execute
end
end
end
let_it_be(:group) { create(:group) }
let!(:project) { create(:project, project_level, namespace: group) }
let!(:project2) { create(:project, project_level) }
let(:user) { create_user_from_membership(project, membership) }
let(:projects) { [project, project2] }
context 'merge request' do
let!(:merge_request) { create :merge_request, target_project: project, source_project: project }
let!(:merge_request2) { create :merge_request, target_project: project2, source_project: project2, title: merge_request.title }
let!(:note) { create :note, project: project, noteable: merge_request }
let!(:note2) { create :note, project: project2, noteable: merge_request2, note: note.note }
let(:scope) { 'merge_requests' }
let(:search) { merge_request.title }
where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
permission_table_for_reporter_feature_access
end
with_them do
it "respects visibility" do
enable_admin_mode!(user) if admin_mode
[project, project2].each do |project|
update_feature_access_level(project, feature_access_level)
it_behaves_like 'search respects visibility'
end
ensure_elasticsearch_index!
end
context 'blob and commit' do
let!(:project) { create(:project, project_level, :repository, namespace: group ) }
let!(:project2) { create(:project, project_level, :repository) }
where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
permission_table_for_guest_feature_access_and_non_private_project_only
end
with_them do
before do
project.repository.index_commits_and_blobs
project2.repository.index_commits_and_blobs
end
it_behaves_like 'search respects visibility' do
let(:scope) { 'commits' }
let(:search) { 'initial' }
end
it_behaves_like 'search respects visibility' do
let(:scope) { 'blobs' }
let(:search) { '.gitmodules' }
end
end
end
context 'note' do
let(:scope) { 'notes' }
let(:search) { note.note }
context 'on issues' do
let!(:note) { create :note_on_issue, project: project }
let!(:note2) { create :note_on_issue, project: project2, note: note.note }
where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
permission_table_for_guest_feature_access
end
with_them do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).and_call_original
end
context 'when add_permissions_data_to_notes_documents migration is finished' do
it_behaves_like 'search respects visibility'
end
context 'when add_permissions_data_to_notes_documents migration is not finished' do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).with(:add_permissions_data_to_notes_documents).and_return(false)
end
it_behaves_like 'search respects visibility'
end
end
end
context 'on merge requests' do
let!(:note) { create :note_on_merge_request, project: project }
let!(:note2) { create :note_on_merge_request, project: project2, note: note.note }
expect_search_results(user, 'merge_requests', expected_count: expected_count) do |user|
described_class.new(user, group, search: merge_request.title).execute
where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
permission_table_for_reporter_feature_access
end
expect_search_results(user, 'notes', expected_count: expected_count) do |user|
described_class.new(user, group, search: note.note).execute
with_them do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).and_call_original
end
context 'when add_permissions_data_to_notes_documents migration is finished' do
it_behaves_like 'search respects visibility'
end
context 'when add_permissions_data_to_notes_documents migration is not finished' do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).with(:add_permissions_data_to_notes_documents).and_return(false)
end
it_behaves_like 'search respects visibility'
end
end
end
context 'code' do
context 'on commits' do
let!(:project) { create(:project, project_level, :repository, namespace: group ) }
let!(:project2) { create(:project, project_level, :repository) }
let!(:note) { create :note_on_commit, project: project }
let!(:note2) { create :note_on_commit, project: project2, note: note.note }
where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
permission_table_for_guest_feature_access_and_non_private_project_only
end
with_them do
it "respects visibility" do
enable_admin_mode!(user) if admin_mode
[project, project2].each do |project|
update_feature_access_level(project, feature_access_level)
ElasticCommitIndexerWorker.new.perform(project.id)
end
ElasticCommitIndexerWorker.new.perform(project.id)
ensure_elasticsearch_index!
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).and_call_original
expect_search_results(user, 'commits', expected_count: expected_count) do |user|
described_class.new(user, group, search: 'initial').execute
project.repository.index_commits_and_blobs
project2.repository.index_commits_and_blobs
end
expect_search_results(user, 'blobs', expected_count: expected_count) do |user|
described_class.new(user, group, search: '.gitmodules').execute
context 'when add_permissions_data_to_notes_documents migration is finished' do
it_behaves_like 'search respects visibility'
end
expect_search_results(user, 'notes', expected_count: expected_count) do |user|
described_class.new(user, group, search: note.note).execute
context 'when add_permissions_data_to_notes_documents migration is not finished' do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).with(:add_permissions_data_to_notes_documents).and_return(false)
end
it_behaves_like 'search respects visibility'
end
end
end
context 'issue' do
let!(:issue) { create :issue, project: project }
let!(:issue2) { create :issue, project: project2, title: issue.title }
let!(:note) { create :note, project: project, noteable: issue }
let!(:note2) { create :note, project: project2, noteable: issue2, note: note.note }
context 'on snippets' do
let!(:note) { create :note_on_project_snippet, project: project }
let!(:note2) { create :note_on_project_snippet, project: project2, note: note.note }
where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
permission_table_for_guest_feature_access
end
with_them do
it "respects visibility" do
enable_admin_mode!(user) if admin_mode
[project, project2].each do |project|
update_feature_access_level(project, feature_access_level)
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).and_call_original
end
context 'when add_permissions_data_to_notes_documents migration is finished' do
it_behaves_like 'search respects visibility'
end
ensure_elasticsearch_index!
expect_search_results(user, 'issues', expected_count: expected_count) do |user|
described_class.new(user, group, search: issue.title).execute
context 'when add_permissions_data_to_notes_documents migration is not finished' do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).with(:add_permissions_data_to_notes_documents).and_return(false)
end
expect_search_results(user, 'notes', expected_count: expected_count) do |user|
described_class.new(user, group, search: note.note).execute
it_behaves_like 'search respects visibility'
end
end
end
end
context 'issue' do
let!(:issue) { create :issue, project: project }
let!(:issue2) { create :issue, project: project2, title: issue.title }
let(:scope) { 'issues' }
let(:search) { issue.title }
where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
permission_table_for_guest_feature_access
end
with_them do
it_behaves_like 'search respects visibility'
end
end
context 'wiki' do
let!(:project) { create(:project, project_level, :wiki_repo) }
let(:group) { project.namespace }
let(:projects) { [project] }
let(:scope) { 'wiki_blobs' }
let(:search) { 'term' }
where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
permission_table_for_guest_feature_access
end
with_them do
it "respects visibility" do
enable_admin_mode!(user) if admin_mode
project.wiki.create_page('test.md', '# term')
before do
project.wiki.create_page('test.md', "# term")
project.wiki.index_wiki_blobs
update_feature_access_level(project, feature_access_level)
ensure_elasticsearch_index!
expect_search_results(user, 'wiki_blobs', expected_count: expected_count) do |user|
described_class.new(user, project.namespace, search: 'term').execute
end
end
it_behaves_like 'search respects visibility'
end
end
......@@ -211,7 +320,7 @@ RSpec.describe Search::GroupService, :elastic do
end
end
context 'project search' do
context 'project' do
let(:project) { create(:project, project_level, namespace: group) }
where(:project_level, :membership, :expected_count) do
......@@ -238,10 +347,9 @@ RSpec.describe Search::GroupService, :elastic do
end
end
context 'sorting' do
context 'issues' do
let(:scope) { 'issues' }
context 'sorting' do
let_it_be(:group) { create(:group) }
let_it_be(:project) { create(:project, :public, group: group) }
......@@ -262,12 +370,9 @@ RSpec.describe Search::GroupService, :elastic do
let(:results_updated) { described_class.new(nil, group, search: 'updated', sort: sort).execute }
end
end
end
context 'merge requests' do
let(:scope) { 'merge_requests' }
context 'sorting' do
let!(:project) { create(:project, :public, group: group) }
let!(:old_result) { create(:merge_request, :opened, source_project: project, source_branch: 'old-1', title: 'sorted old', created_at: 1.month.ago) }
......
ee/spec/services/search/project_service_spec.rb
View file @ def42f65
......@@ -34,44 +34,140 @@ RSpec.describe Search::ProjectService do
end
end
context 'notes search' do
let_it_be(:project) { create(:project) }
let(:results) { described_class.new(project, nil, search: 'test').execute.objects('notes') }
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).and_call_original
end
it_behaves_like 'search query applies joins based on migrations shared examples', :add_permissions_data_to_notes_documents
end
context 'visibility', :elastic, :sidekiq_inline do
include_context 'ProjectPolicyTable context'
shared_examples 'search respects visibility' do
it 'respects visibility' do
enable_admin_mode!(user) if admin_mode
projects.each do |project|
update_feature_access_level(project, feature_access_level)
end
ensure_elasticsearch_index!
expect_search_results(user, scope, expected_count: expected_count) do |user|
described_class.new(project, user, search: search).execute
end
end
end
let_it_be(:group) { create(:group) }
let!(:project) { create(:project, project_level, namespace: group) }
let!(:project2) { create(:project, project_level) }
let(:user) { create_user_from_membership(project, membership) }
let(:projects) { [project, project2] }
context 'merge request' do
let!(:merge_request) { create :merge_request, target_project: project, source_project: project }
let!(:merge_request2) { create :merge_request, target_project: project2, source_project: project2, title: merge_request.title }
let!(:note) { create :note, project: project, noteable: merge_request }
let!(:note2) { create :note, project: project2, noteable: merge_request2, note: note.note }
let(:scope) { 'merge_requests' }
let(:search) { merge_request.title }
where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
permission_table_for_reporter_feature_access
end
with_them do
it "respects visibility" do
enable_admin_mode!(user) if admin_mode
[project, project2].each do |project|
update_feature_access_level(project, feature_access_level)
it_behaves_like 'search respects visibility'
end
ensure_elasticsearch_index!
end
context 'blob and commit' do
let!(:project) { create(:project, project_level, :repository, namespace: group ) }
let!(:project2) { create(:project, project_level, :repository) }
expect_search_results(user, 'merge_requests', expected_count: expected_count) do |user|
described_class.new(project, user, search: merge_request.title).execute
where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
permission_table_for_guest_feature_access_and_non_private_project_only
end
expect_search_results(user, 'notes', expected_count: expected_count) do |user|
described_class.new(project, user, search: note.note).execute
with_them do
before do
project.repository.index_commits_and_blobs
project2.repository.index_commits_and_blobs
end
it_behaves_like 'search respects visibility' do
let(:scope) { 'commits' }
let(:search) { 'initial' }
end
it_behaves_like 'search respects visibility' do
let(:scope) { 'blobs' }
let(:search) { '.gitmodules' }
end
end
end
context 'note' do
let(:scope) { 'notes' }
let(:search) { note.note }
context 'on issues' do
let!(:note) { create :note_on_issue, project: project }
let!(:note2) { create :note_on_issue, project: project2, note: note.note }
where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
permission_table_for_guest_feature_access
end
with_them do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).and_call_original
end
context 'when add_permissions_data_to_notes_documents migration is finished' do
it_behaves_like 'search respects visibility'
end
context 'when add_permissions_data_to_notes_documents migration is not finished' do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).with(:add_permissions_data_to_notes_documents).and_return(false)
end
it_behaves_like 'search respects visibility'
end
end
end
context 'code' do
context 'on merge requests' do
let!(:note) { create :note_on_merge_request, project: project }
let!(:note2) { create :note_on_merge_request, project: project2, note: note.note }
where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
permission_table_for_reporter_feature_access
end
with_them do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).and_call_original
end
context 'when add_permissions_data_to_notes_documents migration is finished' do
it_behaves_like 'search respects visibility'
end
context 'when add_permissions_data_to_notes_documents migration is not finished' do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).with(:add_permissions_data_to_notes_documents).and_return(false)
end
it_behaves_like 'search respects visibility'
end
end
end
context 'on commits' do
let!(:project) { create(:project, project_level, :repository, namespace: group ) }
let!(:project2) { create(:project, project_level, :repository) }
let!(:note) { create :note_on_commit, project: project }
......@@ -82,77 +178,87 @@ RSpec.describe Search::ProjectService do
end
with_them do
it "respects visibility" do
enable_admin_mode!(user) if admin_mode
[project, project2].each do |project|
update_feature_access_level(project, feature_access_level)
ElasticCommitIndexerWorker.new.perform(project.id)
end
ensure_elasticsearch_index!
before do
project.repository.index_commits_and_blobs
project2.repository.index_commits_and_blobs
expect_search_results(user, 'commits', expected_count: expected_count) do |user|
described_class.new(project, user, search: 'initial').execute
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).and_call_original
end
expect_search_results(user, 'blobs', expected_count: expected_count) do |user|
described_class.new(project, user, search: '.gitmodules').execute
context 'when add_permissions_data_to_notes_documents migration is finished' do
it_behaves_like 'search respects visibility'
end
expect_search_results(user, 'notes', expected_count: expected_count) do |user|
described_class.new(project, user, search: note.note).execute
context 'when add_permissions_data_to_notes_documents migration is not finished' do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).with(:add_permissions_data_to_notes_documents).and_return(false)
end
it_behaves_like 'search respects visibility'
end
end
end
context 'issue' do
let!(:issue) { create :issue, project: project }
let!(:issue2) { create :issue, project: project2, title: issue.title }
let!(:note) { create :note, project: project, noteable: issue }
let!(:note2) { create :note, project: project2, noteable: issue2, note: note.note }
context 'on snippets' do
let!(:note) { create :note_on_project_snippet, project: project }
let!(:note2) { create :note_on_project_snippet, project: project2, note: note.note }
where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
permission_table_for_guest_feature_access
end
with_them do
it "respects visibility" do
enable_admin_mode!(user) if admin_mode
[project, project2].each do |project|
update_feature_access_level(project, feature_access_level)
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).and_call_original
end
context 'when add_permissions_data_to_notes_documents migration is finished' do
it_behaves_like 'search respects visibility'
end
ensure_elasticsearch_index!
expect_search_results(user, 'issues', expected_count: expected_count) do |user|
described_class.new(project, user, search: issue.title).execute
context 'when add_permissions_data_to_notes_documents migration is not finished' do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).with(:add_permissions_data_to_notes_documents).and_return(false)
end
expect_search_results(user, 'notes', expected_count: expected_count) do |user|
described_class.new(project, user, search: note.note).execute
it_behaves_like 'search respects visibility'
end
end
end
end
context 'wiki' do
let!(:project) { create(:project, project_level, :wiki_repo) }
context 'issue' do
let!(:issue) { create :issue, project: project }
let!(:issue2) { create :issue, project: project2, title: issue.title }
let(:scope) { 'issues' }
let(:search) { issue.title }
where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
permission_table_for_guest_feature_access
end
with_them do
it "respects visibility" do
enable_admin_mode!(user) if admin_mode
project.wiki.create_page('test.md', '# term')
project.wiki.index_wiki_blobs
update_feature_access_level(project, feature_access_level)
ensure_elasticsearch_index!
it_behaves_like 'search respects visibility'
end
end
context 'wiki' do
let!(:project) { create(:project, project_level, :wiki_repo) }
let(:projects) { [project] }
let(:scope) { 'wiki_blobs' }
let(:search) { 'term' }
expect_search_results(user, 'wiki_blobs', expected_count: expected_count) do |user|
described_class.new(project, user, search: 'term').execute
before do
project.wiki.create_page('test.md', "# term")
project.wiki.index_wiki_blobs
end
where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
permission_table_for_guest_feature_access
end
with_them do
it_behaves_like 'search respects visibility'
end
end
......@@ -180,10 +286,9 @@ RSpec.describe Search::ProjectService do
end
end
context 'sorting', :elastic, :sidekiq_inline do
context 'issues' do
let(:scope) { 'issues' }
context 'sorting', :elastic do
let_it_be(:project) { create(:project, :public) }
let!(:old_result) { create(:issue, project: project, title: 'sorted old', created_at: 1.month.ago) }
......@@ -203,12 +308,9 @@ RSpec.describe Search::ProjectService do
let(:results_updated) { described_class.new(project, nil, search: 'updated', sort: sort).execute }
end
end
end
context 'merge requests' do
let(:scope) { 'merge_requests' }
context 'sorting', :elastic do
let(:project) { create(:project, :public) }
let!(:old_result) { create(:merge_request, :opened, source_project: project, source_branch: 'old-1', title: 'sorted old', created_at: 1.month.ago) }
......
ee/spec/support/shared_examples/services/search_notes_shared_examples.rb
View file @ def42f65
......@@ -32,7 +32,7 @@ RSpec.shared_examples 'search notes shared examples' do
end
context 'when user can read confidential notes' do
it 'does not filter confidental notes' do
it 'does not filter confidential notes' do
noteable.project.add_reporter(user)
expect_search_results(user, 'notes', expected_objects: [not_confidential_note, nil_confidential_note, confidential_note]) do |user|
......
ee/spec/support/shared_examples/services/search_query_applies_joins_based_on_migrations_shared_examples.rb 0 → 100644
View file @ def42f65
# frozen_string_literal: true
RSpec.shared_examples 'search query applies joins based on migrations shared examples' do |migration_name|
context 'using joins for global permission checks', :elastic do
let(:es_host) { Gitlab::CurrentSettings.elasticsearch_url[0] }
let(:search_url) { Addressable::Template.new("#{es_host}/{index}/doc/_search{?params*}") }
context "when #{migration_name} migration is finished" do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?)
.with(migration_name)
.and_return(true)
end
it 'does not use joins to apply permissions' do
request = a_request(:get, search_url).with do |req|
expect(req.body).not_to include("has_parent")
end
results
expect(request).to have_been_made
end
end
context "when #{migration_name} migration is not finished" do
before do
allow(Elastic::DataMigrationService).to receive(:migration_has_finished?)
.with(migration_name)
.and_return(false)
end
it 'uses joins to apply permissions' do
request = a_request(:get, search_url).with do |req|
expect(req.body).to include("has_parent")
end
results
expect(request).to have_been_made
end
end
end
end
spec/support/shared_contexts/policies/project_policy_table_shared_context.rb
View file @ def42f65
......@@ -356,7 +356,7 @@ RSpec.shared_context 'ProjectPolicyTable context' do
:private | :anonymous | 0
end
# :snippet_level, :project_level, :feature_access_level, :membership, :expected_count
# :snippet_level, :project_level, :feature_access_level, :membership, :admin_mode, :expected_count
def permission_table_for_project_snippet_access
:public | :public | :enabled | :admin | true | 1
:public | :public | :enabled | :admin | false | 1
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7