Skip to content

G
gitlab-ce
Commit 1f2d7bd3 authored by Douglas Barbosa Alexandre's avatar Douglas Barbosa Alexandre
Browse files
Options

Rename the finder methods to make clear its real intent

parent 5da58325
Hide whitespace changes
Inline Side-by-side
Showing with 65 additions and 107 deletions
ee/app/finders/geo/repository_verification_finder.rb
View file @ 1f2d7bd3
......@@ -29,9 +29,9 @@ module Geo
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def find_outdated_projects(batch_size:)
query = build_query_to_find_outdated_projects(batch_size: batch_size)
cte = Gitlab::SQL::CTE.new(:outdated_projects, query)
def find_recently_updated_projects(batch_size:)
query = build_query_to_find_recently_updated_projects(batch_size: batch_size)
cte = Gitlab::SQL::CTE.new(:recently_updated_projects, query)
Project.with(cte.to_arel)
.from(cte.alias_to(projects_table))
......@@ -40,12 +40,12 @@ module Geo
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def find_unverified_projects(batch_size:)
def find_never_verified_projects(batch_size:)
relation =
Project.select(:id)
.with_route
.joins(left_join_repository_state)
.where(repository_never_verified)
.where(repository_state_table[:project_id].eq(nil))
.limit(batch_size)
apply_shard_restriction(relation)
......@@ -96,12 +96,20 @@ module Geo
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def build_query_to_find_outdated_projects(batch_size:)
def build_query_to_find_recently_updated_projects(batch_size:)
repository_recently_updated =
repository_state_table[:repository_verification_checksum].eq(nil)
.and(repository_state_table[:last_repository_verification_failure].eq(nil))
wiki_recently_updated =
repository_state_table[:wiki_verification_checksum].eq(nil)
.and(repository_state_table[:last_wiki_verification_failure].eq(nil))
query =
projects_table
.join(repository_state_table).on(project_id_matcher)
.project(projects_table[:id], projects_table[:last_repository_updated_at])
.where(repository_outdated.or(wiki_outdated))
.where(repository_recently_updated.or(wiki_recently_updated))
.take(batch_size)
apply_shard_restriction(query)
......@@ -149,20 +157,6 @@ module Geo
.join_sources
end
def repository_outdated
repository_state_table[:repository_verification_checksum].eq(nil)
.and(repository_state_table[:last_repository_verification_failure].eq(nil))
end
def wiki_outdated
repository_state_table[:wiki_verification_checksum].eq(nil)
.and(repository_state_table[:last_wiki_verification_failure].eq(nil))
end
def repository_never_verified
repository_state_table[:project_id].eq(nil)
end
def last_repository_updated_at_asc
Gitlab::Database.nulls_last_order('projects.last_repository_updated_at', 'ASC')
end
......
ee/app/workers/geo/repository_verification/primary/shard_worker.rb
View file @ 1f2d7bd3
......@@ -44,11 +44,11 @@ module Geo
end
def load_pending_resources
resources = find_unverified_project_ids(batch_size: db_retrieve_batch_size)
resources = find_never_verified_project_ids(batch_size: db_retrieve_batch_size)
remaining_capacity = db_retrieve_batch_size - resources.size
return resources if remaining_capacity.zero?
resources += find_outdated_project_ids(batch_size: remaining_capacity)
resources += find_recently_updated_project_ids(batch_size: remaining_capacity)
remaining_capacity = db_retrieve_batch_size - resources.size
return resources if remaining_capacity.zero?
......@@ -56,14 +56,14 @@ module Geo
end
# rubocop: disable CodeReuse/ActiveRecord
def find_unverified_project_ids(batch_size:)
finder.find_unverified_projects(batch_size: batch_size).pluck(:id)
def find_never_verified_project_ids(batch_size:)
finder.find_never_verified_projects(batch_size: batch_size).pluck(:id)
end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def find_outdated_project_ids(batch_size:)
finder.find_outdated_projects(batch_size: batch_size).pluck(:id)
def find_recently_updated_project_ids(batch_size:)
finder.find_recently_updated_projects(batch_size: batch_size).pluck(:id)
end
# rubocop: enable CodeReuse/ActiveRecord
......
ee/spec/finders/geo/repository_verification_finder_spec.rb
View file @ 1f2d7bd3
......@@ -17,7 +17,7 @@ describe Geo::RepositoryVerificationFinder, :postgresql do
expect(subject.find_failed_repositories(batch_size: 10)).to be_empty
end
it 'does not return projects where repository verification is outdated' do
it 'does not return projects where repository was recently updated' do
create(:repository_state, :repository_outdated, project: project)
expect(subject.find_failed_repositories(batch_size: 10)).to be_empty
......@@ -66,7 +66,7 @@ describe Geo::RepositoryVerificationFinder, :postgresql do
expect(subject.find_failed_wikis(batch_size: 10)).to be_empty
end
it 'does not return projects where wiki verification is outdated' do
it 'does not return projects where wiki was recently updated' do
create(:repository_state, :wiki_outdated, project: project)
expect(subject.find_failed_wikis(batch_size: 10)).to be_empty
......@@ -101,45 +101,45 @@ describe Geo::RepositoryVerificationFinder, :postgresql do
end
end
describe '#find_outdated_projects' do
it 'returns projects where repository verification is outdated' do
describe '#find_recently_updated_projects' do
it 'returns projects where repository was recently updated' do
create(:repository_state, :repository_outdated, project: project)
expect(subject.find_outdated_projects(batch_size: 10))
expect(subject.find_recently_updated_projects(batch_size: 10))
.to match_array(project)
end
it 'returns projects where repository verification is pending' do
create(:repository_state, :wiki_verified, project: project)
expect(subject.find_outdated_projects(batch_size: 10))
expect(subject.find_recently_updated_projects(batch_size: 10))
.to match_array(project)
end
it 'does not return projects where repository verification failed' do
create(:repository_state, :repository_failed, :wiki_verified, project: project)
expect(subject.find_outdated_projects(batch_size: 10)).to be_empty
expect(subject.find_recently_updated_projects(batch_size: 10)).to be_empty
end
it 'returns projects where wiki verification is outdated' do
it 'returns projects where wiki was recently updated' do
create(:repository_state, :wiki_outdated, project: project)
expect(subject.find_outdated_projects(batch_size: 10))
expect(subject.find_recently_updated_projects(batch_size: 10))
.to match_array(project)
end
it 'returns projects where wiki verification is pending' do
create(:repository_state, :repository_verified, project: project)
expect(subject.find_outdated_projects(batch_size: 10))
expect(subject.find_recently_updated_projects(batch_size: 10))
.to match_array(project)
end
it 'does not return projects where wiki verification failed' do
create(:repository_state, :repository_verified, :wiki_failed, project: project)
expect(subject.find_outdated_projects(batch_size: 10)).to be_empty
expect(subject.find_recently_updated_projects(batch_size: 10)).to be_empty
end
it 'returns less active projects first' do
......@@ -149,7 +149,7 @@ describe Geo::RepositoryVerificationFinder, :postgresql do
project.update_column(:last_repository_updated_at, 30.minutes.ago)
less_active_project.update_column(:last_repository_updated_at, 2.days.ago)
expect(subject.find_outdated_projects(batch_size: 10)).to eq [less_active_project, project]
expect(subject.find_recently_updated_projects(batch_size: 10)).to eq [less_active_project, project]
end
context 'with shard restriction' do
......@@ -161,18 +161,18 @@ describe Geo::RepositoryVerificationFinder, :postgresql do
create(:repository_state, :repository_outdated, project: project)
create(:repository_state, :repository_outdated, project: project_other_shard)
expect(subject.find_outdated_projects(batch_size: 10))
expect(subject.find_recently_updated_projects(batch_size: 10))
.to match_array(project)
end
end
end
describe '#find_unverified_projects' do
describe '#find_never_verified_projects' do
it 'returns projects that never have been verified' do
create(:repository_state, :repository_outdated)
create(:repository_state, :wiki_outdated)
expect(subject.find_unverified_projects(batch_size: 10))
expect(subject.find_never_verified_projects(batch_size: 10))
.to match_array(project)
end
......@@ -183,44 +183,50 @@ describe Geo::RepositoryVerificationFinder, :postgresql do
project_other_shard = create(:project)
project_other_shard.update_column(:repository_storage, 'other')
expect(subject.find_unverified_projects(batch_size: 10))
expect(subject.find_never_verified_projects(batch_size: 10))
.to match_array(project)
end
end
end
describe '#find_reverifiable_repositories' do
it 'returns projects where repository was verified before the minimum re-verification interval' do
shared_examples 'find reverifiable projects' do |type|
let(:failed) { "#{type}_failed".to_sym }
let(:verified) { "#{type}_verified".to_sym }
let(:outdated) { "#{type}_outdated".to_sym }
let(:last_verification_ran_at) { "last_#{type}_verification_ran_at" }
let(:finder_method) { "find_reverifiable_#{type.to_s.pluralize}" }
it "returns projects where #{type} was verified before the minimum re-verification interval" do
project_recently_verified = create(:project)
create(:repository_state, :repository_verified, project: project, last_repository_verification_ran_at: 2.days.ago)
create(:repository_state, :repository_verified, project: project_recently_verified, last_repository_verification_ran_at: Time.now)
create(:repository_state, verified, project: project, last_verification_ran_at => 2.days.ago)
create(:repository_state, verified, project: project_recently_verified, last_verification_ran_at => Time.now)
expect(subject.find_reverifiable_repositories(interval: 1.day.ago, batch_size: 10))
expect(subject.public_send(finder_method, interval: 1.day.ago, batch_size: 10))
.to match_array(project)
end
it 'does not return projects where repository verification is outdated' do
create(:repository_state, :repository_outdated, project: project, last_repository_verification_ran_at: 2.days.ago)
it "does not return projects where #{type} was recently updated" do
create(:repository_state, outdated, project: project, last_verification_ran_at => 2.days.ago)
expect(subject.find_reverifiable_repositories(interval: 1.day.ago, batch_size: 10))
expect(subject.public_send(finder_method, interval: 1.day.ago, batch_size: 10))
.to be_empty
end
it 'does not return projects where repository verification failed' do
create(:repository_state, :repository_failed, project: project, last_repository_verification_ran_at: 2.days.ago)
it "does not return projects where #{type} verification failed" do
create(:repository_state, failed, project: project, last_verification_ran_at => 2.days.ago)
expect(subject.find_reverifiable_repositories(interval: 1.day.ago, batch_size: 10))
expect(subject.public_send(finder_method, interval: 1.day.ago, batch_size: 10))
.to be_empty
end
it 'returns less active projects first' do
less_active_project = create(:project)
create(:repository_state, :repository_verified, project: project, last_repository_verification_ran_at: 2.days.ago)
create(:repository_state, :repository_verified, project: less_active_project, last_repository_verification_ran_at: 2.days.ago)
create(:repository_state, verified, project: project, last_verification_ran_at => 2.days.ago)
create(:repository_state, verified, project: less_active_project, last_verification_ran_at => 2.days.ago)
project.update_column(:last_repository_updated_at, 30.minutes.ago)
less_active_project.update_column(:last_repository_updated_at, 2.days.ago)
expect(subject.find_reverifiable_repositories(interval: 1.day.ago, batch_size: 10))
expect(subject.public_send(finder_method, interval: 1.day.ago, batch_size: 10))
.to eq [less_active_project, project]
end
......@@ -230,62 +236,20 @@ describe Geo::RepositoryVerificationFinder, :postgresql do
it 'does not return projects on other shards' do
project_other_shard = create(:project)
project_other_shard.update_column(:repository_storage, 'other')
create(:repository_state, :repository_verified, project: project, last_repository_verification_ran_at: 2.days.ago)
create(:repository_state, :repository_verified, project: project_other_shard, last_repository_verification_ran_at: 2.days.ago)
create(:repository_state, verified, project: project, last_verification_ran_at => 2.days.ago)
create(:repository_state, verified, project: project_other_shard, last_verification_ran_at => 2.days.ago)
expect(subject.find_reverifiable_repositories(interval: 1.day.ago, batch_size: 10))
expect(subject.public_send(finder_method, interval: 1.day.ago, batch_size: 10))
.to match_array(project)
end
end
end
describe '#find_reverifiable_wikis' do
it 'returns projects where wiki was verified before the minimum re-verification interval' do
project_recently_verified = create(:project)
create(:repository_state, :wiki_verified, project: project, last_wiki_verification_ran_at: 2.days.ago)
create(:repository_state, :wiki_verified, project: project_recently_verified, last_wiki_verification_ran_at: Time.now)
expect(subject.find_reverifiable_wikis(interval: 1.day.ago, batch_size: 10))
.to match_array(project)
end
it 'does not return projects where wiki verification is outdated' do
create(:repository_state, :wiki_outdated, project: project, last_wiki_verification_ran_at: 2.days.ago)
expect(subject.find_reverifiable_wikis(interval: 1.day.ago, batch_size: 10))
.to be_empty
end
it 'does not return projects where wiki verification failed' do
create(:repository_state, :wiki_failed, project: project, last_wiki_verification_ran_at: 2.days.ago)
expect(subject.find_reverifiable_wikis(interval: 1.day.ago, batch_size: 10))
.to be_empty
end
it 'returns less active projects first' do
less_active_project = create(:project)
create(:repository_state, :wiki_verified, project: project, last_wiki_verification_ran_at: 2.days.ago)
create(:repository_state, :wiki_verified, project: less_active_project, last_wiki_verification_ran_at: 2.days.ago)
project.update_column(:last_repository_updated_at, 30.minutes.ago)
less_active_project.update_column(:last_repository_updated_at, 2.days.ago)
expect(subject.find_reverifiable_wikis(interval: 1.day.ago, batch_size: 10))
.to eq [less_active_project, project]
end
context 'with shard restriction' do
subject { described_class.new(shard_name: project.repository_storage) }
it 'does not return projects on other shards' do
project_other_shard = create(:project)
project_other_shard.update_column(:repository_storage, 'other')
create(:repository_state, :wiki_verified, project: project, last_wiki_verification_ran_at: 2.days.ago)
create(:repository_state, :wiki_verified, project: project_other_shard, last_wiki_verification_ran_at: 2.days.ago)
describe '#find_reverifiable_repositories' do
it_behaves_like 'find reverifiable projects', :repository
end
expect(subject.find_reverifiable_wikis(interval: 1.day.ago, batch_size: 10))
.to match_array(project)
end
end
describe '#find_reverifiable_wikis' do
it_behaves_like 'find reverifiable projects', :wiki
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7