Commit 25810d03 authored by Shinya Maeda's avatar Shinya Maeda

Added flush-to-db process for the cron worker. Rename to RescueStaleLiveTraceWorker.

parent 93103bb0
...@@ -17,7 +17,7 @@ ...@@ -17,7 +17,7 @@
- cronjob:stuck_ci_jobs - cronjob:stuck_ci_jobs
- cronjob:stuck_import_jobs - cronjob:stuck_import_jobs
- cronjob:stuck_merge_jobs - cronjob:stuck_merge_jobs
- cronjob:fource_archive_stale_live_trace - cronjob:rescue_stale_live_trace
- cronjob:trending_projects - cronjob:trending_projects
- cronjob:issue_due_scheduler - cronjob:issue_due_scheduler
......
class FourceArchiveStaleLiveTraceWorker
include ApplicationWorker
include CronjobQueue
def perform
# Find jobs with the following condition
# - Finished 4 hours ago (Jobs finished 4 hours ago should have an archived trace)
# - Not archived yet (Because ArchiveTraceWorker failed by some reason)
Ci::Build.finished
.where('finished_at < ?', 4.hours.ago)
.where('NOT EXISTS (?)', Ci::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id'))
.find_each(batch_size: 1000) do |job|
begin
job.trace.archive!
rescue => e
Rails.logger.error "#{job.id}: Failed to archive stable live trace: #{e.message}"
end
Rails.logger.warning "#{job.id}: Live trace was force archived because it was considered as stale"
end
end
end
end
class RescueStaleLiveTraceWorker
include ApplicationWorker
include CronjobQueue
def perform
# Reschedule to archive live traces
#
# The target jobs are with the following conditions
# - Finished 4 hours ago, but it's not archived yet
# Jobs finished 4 hours ago should have an archived trace. Probably ArchiveTraceWorker failed by Sidekiq's inconsistancy
Ci::Build.finished
.where('finished_at BETWEEN ? AND ?', 1.week.ago, 4.hours.ago)
.where('NOT EXISTS (?)', Ci::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id'))
.find_in_batch(batch_size: 1000) do |jobs|
job_ids = jobs.map { |job| [job.id] }
ArchiveTraceWorker.bulk_perform_async(job_ids)
Rails.logger.warning "Scheduled to archive stale live traces from #{job_ids.min} to #{job_ids.max}"
end
# Schedule to flush redis-chunk to database
#
# The target build_trace_chunks are with the following conditions
# - The last patching of the trace was 1 hour ago
# - The job is still running
Ci::BuildTraceChunk.redis
.joins(:build)
.where('ci_builds.update_at < ?', 1.hour.ago)
.where('ci_builds.status = ?', 'running')
.find_in_batch(batch_size: 1000) do |build_trace_chunks|
build_trace_chunk_ids = build_trace_chunks.map { |build_trace_chunk| [build_trace_chunk.id] }
BuildTraceChunkFlushToDBWorker.bulk_perform_async(build_trace_chunk_ids)
Rails.logger.warning "Scheduled to flush stale live traces to database from #{build_trace_chunk_ids.min} to #{build_trace_chunk_ids.max}"
end
end
end
...@@ -289,9 +289,9 @@ Settings.cron_jobs['repository_archive_cache_worker']['job_class'] = 'Repository ...@@ -289,9 +289,9 @@ Settings.cron_jobs['repository_archive_cache_worker']['job_class'] = 'Repository
Settings.cron_jobs['import_export_project_cleanup_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['import_export_project_cleanup_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['import_export_project_cleanup_worker']['cron'] ||= '0 * * * *' Settings.cron_jobs['import_export_project_cleanup_worker']['cron'] ||= '0 * * * *'
Settings.cron_jobs['import_export_project_cleanup_worker']['job_class'] = 'ImportExportProjectCleanupWorker' Settings.cron_jobs['import_export_project_cleanup_worker']['job_class'] = 'ImportExportProjectCleanupWorker'
Settings.cron_jobs['fource_archive_stale_live_trace_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['rescue_stale_live_trace_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['fource_archive_stale_live_trace_worker']['cron'] ||= '*/4 * * * *' Settings.cron_jobs['rescue_stale_live_trace_worker']['cron'] ||= '*/1 * * * *'
Settings.cron_jobs['fource_archive_stale_live_trace_worker']['job_class'] = 'BuildTraceChunkArchiveStaleObjectsWorker' Settings.cron_jobs['rescue_stale_live_trace_worker']['job_class'] = 'BuildTraceChunkArchiveStaleObjectsWorker'
Settings.cron_jobs['requests_profiles_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['requests_profiles_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['requests_profiles_worker']['cron'] ||= '0 0 * * *' Settings.cron_jobs['requests_profiles_worker']['cron'] ||= '0 0 * * *'
Settings.cron_jobs['requests_profiles_worker']['job_class'] = 'RequestsProfilesWorker' Settings.cron_jobs['requests_profiles_worker']['job_class'] = 'RequestsProfilesWorker'
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment