Commit a9c868d1 authored by Michael Kozono's avatar Michael Kozono

Bulk insert uploads

parent 908aacdd
...@@ -56,7 +56,8 @@ module Gitlab ...@@ -56,7 +56,8 @@ module Gitlab
uploader: uploader, uploader: uploader,
model_type: model_type, model_type: model_type,
model_id: model_id, model_id: model_id,
size: file_size size: file_size,
checksum: checksum
} }
end end
...@@ -90,10 +91,13 @@ module Gitlab ...@@ -90,10 +91,13 @@ module Gitlab
end end
def file_size def file_size
absolute_path = File.join(CarrierWave.root, path)
File.size(absolute_path) File.size(absolute_path)
end end
def checksum
Digest::SHA256.file(absolute_path).hexdigest
end
# Not including a leading slash # Not including a leading slash
def path_relative_to_upload_dir def path_relative_to_upload_dir
base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR)}/} base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR)}/}
...@@ -120,55 +124,14 @@ module Gitlab ...@@ -120,55 +124,14 @@ module Gitlab
project = Project.find_by_full_path(full_path) project = Project.find_by_full_path(full_path)
project.id.to_s project.id.to_s
end end
end
# Copy-pasted class for less fragile migration
class Upload < ActiveRecord::Base
self.table_name = 'uploads' # This is the only line different from copy-paste
# Upper limit for foreground checksum processing
CHECKSUM_THRESHOLD = 100.megabytes
belongs_to :model, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations
before_save :calculate_checksum, if: :foreground_checksum?
after_commit :schedule_checksum, unless: :foreground_checksum?
def absolute_path def absolute_path
return path unless relative_path? File.join(CarrierWave.root, path)
uploader_class.absolute_path(self)
end
def calculate_checksum
return unless exist?
self.checksum = Digest::SHA256.file(absolute_path).hexdigest
rescue StandardError
schedule_checksum
end end
def exist?
File.exist?(absolute_path)
end
private
def foreground_checksum?
size <= CHECKSUM_THRESHOLD
end
def schedule_checksum
UploadChecksumWorker.perform_async(id)
end
def relative_path?
!path.start_with?('/')
end end
def uploader_class class Upload < ActiveRecord::Base
Object.const_get(uploader) self.table_name = 'uploads'
end
end end
def perform(start_id, end_id) def perform(start_id, end_id)
...@@ -207,9 +170,11 @@ module Gitlab ...@@ -207,9 +170,11 @@ module Gitlab
end end
def insert(files) def insert(files)
files.each do |file| rows = files.map do |file|
Upload.create!(file.to_h) file.to_h.merge(created_at: 'NOW()')
end end
Gitlab::Database.bulk_insert('uploads', rows)
end end
def drop_temp_table_if_finished def drop_temp_table_if_finished
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment