Commit a7dae52e authored by Sean McGivern's avatar Sean McGivern Committed by Kamil Trzciński

Merge branch '4163-move-uploads-to-object-storage' into 'master'

Move uploads to object storage

Closes #4163

See merge request gitlab-org/gitlab-ee!3867
parent 45d2c316
module UploadsActions module UploadsActions
include Gitlab::Utils::StrongMemoize include Gitlab::Utils::StrongMemoize
UPLOAD_MOUNTS = %w(avatar attachment file logo header_logo).freeze
def create def create
link_to_file = UploadService.new(model, params[:file], uploader_class).execute link_to_file = UploadService.new(model, params[:file], uploader_class).execute
...@@ -17,34 +19,71 @@ module UploadsActions ...@@ -17,34 +19,71 @@ module UploadsActions
end end
end end
# This should either
# - send the file directly
# - or redirect to its URL
#
def show def show
return render_404 unless uploader.exists? return render_404 unless uploader.exists?
disposition = uploader.image_or_video? ? 'inline' : 'attachment' if uploader.file_storage?
disposition = uploader.image_or_video? ? 'inline' : 'attachment'
expires_in 0.seconds, must_revalidate: true, private: true expires_in 0.seconds, must_revalidate: true, private: true
send_file uploader.file.path, disposition: disposition send_file uploader.file.path, disposition: disposition
else
redirect_to uploader.url
end
end end
private private
def uploader_class
raise NotImplementedError
end
def upload_mount
mounted_as = params[:mounted_as]
mounted_as if UPLOAD_MOUNTS.include?(mounted_as)
end
def uploader_mounted?
upload_model_class < CarrierWave::Mount::Extension && !upload_mount.nil?
end
def uploader def uploader
strong_memoize(:uploader) do strong_memoize(:uploader) do
return if show_model.nil? if uploader_mounted?
model.public_send(upload_mount) # rubocop:disable GitlabSecurity/PublicSend
else
build_uploader_from_upload || build_uploader_from_params
end
end
end
file_uploader = FileUploader.new(show_model, params[:secret]) def build_uploader_from_upload
file_uploader.retrieve_from_store!(params[:filename]) return nil unless params[:secret] && params[:filename]
file_uploader upload_path = uploader_class.upload_path(params[:secret], params[:filename])
end upload = Upload.find_by(uploader: uploader_class.to_s, path: upload_path)
upload&.build_uploader
end
def build_uploader_from_params
uploader = uploader_class.new(model, params[:secret])
uploader.retrieve_from_store!(params[:filename])
uploader
end end
def image_or_video? def image_or_video?
uploader && uploader.exists? && uploader.image_or_video? uploader && uploader.exists? && uploader.image_or_video?
end end
def uploader_class def find_model
FileUploader nil
end
def model
strong_memoize(:model) { find_model }
end end
end end
...@@ -7,29 +7,23 @@ class Groups::UploadsController < Groups::ApplicationController ...@@ -7,29 +7,23 @@ class Groups::UploadsController < Groups::ApplicationController
private private
def show_model def upload_model_class
strong_memoize(:show_model) do Group
group_id = params[:group_id]
Group.find_by_full_path(group_id)
end
end end
def authorize_upload_file! def uploader_class
render_404 unless can?(current_user, :upload_file, group) NamespaceFileUploader
end end
def uploader def find_model
strong_memoize(:uploader) do return @group if @group
file_uploader = uploader_class.new(show_model, params[:secret])
file_uploader.retrieve_from_store!(params[:filename])
file_uploader
end
end
def uploader_class group_id = params[:group_id]
NamespaceFileUploader
Group.find_by_full_path(group_id)
end end
alias_method :model, :group def authorize_upload_file!
render_404 unless can?(current_user, :upload_file, group)
end
end end
...@@ -61,7 +61,7 @@ class Projects::LfsStorageController < Projects::GitHttpClientController ...@@ -61,7 +61,7 @@ class Projects::LfsStorageController < Projects::GitHttpClientController
def store_file(oid, size, tmp_file) def store_file(oid, size, tmp_file)
# Define tmp_file_path early because we use it in "ensure" # Define tmp_file_path early because we use it in "ensure"
tmp_file_path = File.join("#{Gitlab.config.lfs.storage_path}/tmp/upload", tmp_file) tmp_file_path = File.join(LfsObjectUploader.workhorse_upload_path, tmp_file)
object = LfsObject.find_or_create_by(oid: oid, size: size) object = LfsObject.find_or_create_by(oid: oid, size: size)
file_exists = object.file.exists? || move_tmp_file_to_storage(object, tmp_file_path) file_exists = object.file.exists? || move_tmp_file_to_storage(object, tmp_file_path)
......
class Projects::UploadsController < Projects::ApplicationController class Projects::UploadsController < Projects::ApplicationController
include UploadsActions include UploadsActions
# These will kick you out if you don't have access.
skip_before_action :project, :repository, skip_before_action :project, :repository,
if: -> { action_name == 'show' && image_or_video? } if: -> { action_name == 'show' && image_or_video? }
...@@ -8,14 +9,20 @@ class Projects::UploadsController < Projects::ApplicationController ...@@ -8,14 +9,20 @@ class Projects::UploadsController < Projects::ApplicationController
private private
def show_model def upload_model_class
strong_memoize(:show_model) do Project
namespace = params[:namespace_id] end
id = params[:project_id]
Project.find_by_full_path("#{namespace}/#{id}") def uploader_class
end FileUploader
end end
alias_method :model, :project def find_model
return @project if @project
namespace = params[:namespace_id]
id = params[:project_id]
Project.find_by_full_path("#{namespace}/#{id}")
end
end end
class UploadsController < ApplicationController class UploadsController < ApplicationController
include UploadsActions include UploadsActions
UnknownUploadModelError = Class.new(StandardError)
MODEL_CLASSES = {
"user" => User,
"project" => Project,
"note" => Note,
"group" => Group,
"appearance" => Appearance,
"personal_snippet" => PersonalSnippet,
nil => PersonalSnippet
}.freeze
rescue_from UnknownUploadModelError, with: :render_404
skip_before_action :authenticate_user! skip_before_action :authenticate_user!
before_action :upload_mount_satisfied?
before_action :find_model before_action :find_model
before_action :authorize_access!, only: [:show] before_action :authorize_access!, only: [:show]
before_action :authorize_create_access!, only: [:create] before_action :authorize_create_access!, only: [:create]
private def uploader_class
PersonalFileUploader
end
def find_model def find_model
return nil unless params[:id] return nil unless params[:id]
return render_404 unless upload_model && upload_mount upload_model_class.find(params[:id])
@model = upload_model.find(params[:id])
end end
def authorize_access! def authorize_access!
...@@ -53,55 +68,17 @@ class UploadsController < ApplicationController ...@@ -53,55 +68,17 @@ class UploadsController < ApplicationController
end end
end end
def upload_model def upload_model_class
upload_models = { MODEL_CLASSES[params[:model]] || raise(UnknownUploadModelError)
"user" => User,
"project" => Project,
"note" => Note,
"group" => Group,
"appearance" => Appearance,
"personal_snippet" => PersonalSnippet
}
upload_models[params[:model]]
end
def upload_mount
return true unless params[:mounted_as]
upload_mounts = %w(avatar attachment file logo header_logo)
if upload_mounts.include?(params[:mounted_as])
params[:mounted_as]
end
end end
def uploader def upload_model_class_has_mounts?
return @uploader if defined?(@uploader) upload_model_class < CarrierWave::Mount::Extension
case model
when nil
@uploader = PersonalFileUploader.new(nil, params[:secret])
@uploader.retrieve_from_store!(params[:filename])
when PersonalSnippet
@uploader = PersonalFileUploader.new(model, params[:secret])
@uploader.retrieve_from_store!(params[:filename])
else
@uploader = @model.public_send(upload_mount) # rubocop:disable GitlabSecurity/PublicSend
redirect_to @uploader.url unless @uploader.file_storage?
end
@uploader
end end
def uploader_class def upload_mount_satisfied?
PersonalFileUploader return true unless upload_model_class_has_mounts?
end
def model upload_model_class.uploader_options.has_key?(upload_mount)
@model ||= find_model
end end
end end
...@@ -11,6 +11,7 @@ class Appearance < ActiveRecord::Base ...@@ -11,6 +11,7 @@ class Appearance < ActiveRecord::Base
mount_uploader :logo, AttachmentUploader mount_uploader :logo, AttachmentUploader
mount_uploader :header_logo, AttachmentUploader mount_uploader :header_logo, AttachmentUploader
has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
CACHE_KEY = 'current_appearance'.freeze CACHE_KEY = 'current_appearance'.freeze
......
...@@ -45,7 +45,7 @@ module Ci ...@@ -45,7 +45,7 @@ module Ci
end end
scope :with_artifacts_not_expired, ->() { with_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } scope :with_artifacts_not_expired, ->() { with_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) }
scope :with_expired_artifacts, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) } scope :with_expired_artifacts, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) }
scope :with_artifacts_stored_locally, ->() { with_artifacts.where(artifacts_file_store: [nil, LegacyArtifactUploader::LOCAL_STORE]) } scope :with_artifacts_stored_locally, ->() { with_artifacts.where(artifacts_file_store: [nil, LegacyArtifactUploader::Store::LOCAL]) }
scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) } scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) }
scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + [:manual]) } scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + [:manual]) }
scope :ref_protected, -> { where(protected: true) } scope :ref_protected, -> { where(protected: true) }
......
module Avatarable module Avatarable
extend ActiveSupport::Concern extend ActiveSupport::Concern
included do
prepend ShadowMethods
validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? }
validates :avatar, file_size: { maximum: 200.kilobytes.to_i }
mount_uploader :avatar, AvatarUploader
end
module ShadowMethods
def avatar_url(**args)
# We use avatar_path instead of overriding avatar_url because of carrierwave.
# See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11001/diffs#note_28659864
avatar_path(only_path: args.fetch(:only_path, true)) || super
end
end
def avatar_type
unless self.avatar.image?
self.errors.add :avatar, "only images allowed"
end
end
def avatar_path(only_path: true) def avatar_path(only_path: true)
return unless self[:avatar].present? return unless self[:avatar].present?
......
...@@ -29,18 +29,14 @@ class Group < Namespace ...@@ -29,18 +29,14 @@ class Group < Namespace
has_many :variables, class_name: 'Ci::GroupVariable' has_many :variables, class_name: 'Ci::GroupVariable'
has_many :custom_attributes, class_name: 'GroupCustomAttribute' has_many :custom_attributes, class_name: 'GroupCustomAttribute'
validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? } has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
validate :visibility_level_allowed_by_projects validate :visibility_level_allowed_by_projects
validate :visibility_level_allowed_by_sub_groups validate :visibility_level_allowed_by_sub_groups
validate :visibility_level_allowed_by_parent validate :visibility_level_allowed_by_parent
validates :avatar, file_size: { maximum: 200.kilobytes.to_i }
validates :two_factor_grace_period, presence: true, numericality: { greater_than_or_equal_to: 0 } validates :two_factor_grace_period, presence: true, numericality: { greater_than_or_equal_to: 0 }
mount_uploader :avatar, AvatarUploader
has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
after_create :post_create_hook after_create :post_create_hook
after_destroy :post_destroy_hook after_destroy :post_destroy_hook
after_save :update_two_factor_requirement after_save :update_two_factor_requirement
...@@ -116,12 +112,6 @@ class Group < Namespace ...@@ -116,12 +112,6 @@ class Group < Namespace
visibility_level_allowed_by_sub_groups?(level) visibility_level_allowed_by_sub_groups?(level)
end end
def avatar_url(**args)
# We use avatar_path instead of overriding avatar_url because of carrierwave.
# See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11001/diffs#note_28659864
avatar_path(args)
end
def lfs_enabled? def lfs_enabled?
return false unless Gitlab.config.lfs.enabled return false unless Gitlab.config.lfs.enabled
return Gitlab.config.lfs.enabled if self[:lfs_enabled].nil? return Gitlab.config.lfs.enabled if self[:lfs_enabled].nil?
...@@ -193,12 +183,6 @@ class Group < Namespace ...@@ -193,12 +183,6 @@ class Group < Namespace
owners.include?(user) && owners.size == 1 owners.include?(user) && owners.size == 1
end end
def avatar_type
unless self.avatar.image?
self.errors.add :avatar, "only images allowed"
end
end
def post_create_hook def post_create_hook
Gitlab::AppLogger.info("Group \"#{name}\" was created") Gitlab::AppLogger.info("Group \"#{name}\" was created")
......
...@@ -7,7 +7,7 @@ class LfsObject < ActiveRecord::Base ...@@ -7,7 +7,7 @@ class LfsObject < ActiveRecord::Base
validates :oid, presence: true, uniqueness: true validates :oid, presence: true, uniqueness: true
scope :with_files_stored_locally, ->() { where(file_store: [nil, LfsObjectUploader::LOCAL_STORE]) } scope :with_files_stored_locally, -> { where(file_store: [nil, LfsObjectUploader::Store::LOCAL]) }
mount_uploader :file, LfsObjectUploader mount_uploader :file, LfsObjectUploader
......
...@@ -88,6 +88,7 @@ class Note < ActiveRecord::Base ...@@ -88,6 +88,7 @@ class Note < ActiveRecord::Base
end end
end end
# @deprecated attachments are handler by the MarkdownUploader
mount_uploader :attachment, AttachmentUploader mount_uploader :attachment, AttachmentUploader
# Scopes # Scopes
......
...@@ -255,9 +255,6 @@ class Project < ActiveRecord::Base ...@@ -255,9 +255,6 @@ class Project < ActiveRecord::Base
validates :star_count, numericality: { greater_than_or_equal_to: 0 } validates :star_count, numericality: { greater_than_or_equal_to: 0 }
validate :check_limit, on: :create validate :check_limit, on: :create
validate :check_repository_path_availability, on: :update, if: ->(project) { project.renamed? } validate :check_repository_path_availability, on: :update, if: ->(project) { project.renamed? }
validate :avatar_type,
if: ->(project) { project.avatar.present? && project.avatar_changed? }
validates :avatar, file_size: { maximum: 200.kilobytes.to_i }
validate :visibility_level_allowed_by_group validate :visibility_level_allowed_by_group
validate :visibility_level_allowed_as_fork validate :visibility_level_allowed_as_fork
validate :check_wiki_path_conflict validate :check_wiki_path_conflict
...@@ -265,7 +262,6 @@ class Project < ActiveRecord::Base ...@@ -265,7 +262,6 @@ class Project < ActiveRecord::Base
presence: true, presence: true,
inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } } inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } }
mount_uploader :avatar, AvatarUploader
has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
# Scopes # Scopes
...@@ -917,20 +913,12 @@ class Project < ActiveRecord::Base ...@@ -917,20 +913,12 @@ class Project < ActiveRecord::Base
issues_tracker.to_param == 'jira' issues_tracker.to_param == 'jira'
end end
def avatar_type
unless self.avatar.image?
self.errors.add :avatar, 'only images allowed'
end
end
def avatar_in_git def avatar_in_git
repository.avatar repository.avatar
end end
def avatar_url(**args) def avatar_url(**args)
# We use avatar_path instead of overriding avatar_url because of carrierwave. Gitlab::Routing.url_helpers.project_avatar_url(self) if avatar_in_git
# See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11001/diffs#note_28659864
avatar_path(args) || (Gitlab::Routing.url_helpers.project_avatar_url(self) if avatar_in_git)
end end
# For compatibility with old code # For compatibility with old code
......
...@@ -9,44 +9,52 @@ class Upload < ActiveRecord::Base ...@@ -9,44 +9,52 @@ class Upload < ActiveRecord::Base
validates :model, presence: true validates :model, presence: true
validates :uploader, presence: true validates :uploader, presence: true
before_save :calculate_checksum, if: :foreground_checksum? before_save :calculate_checksum!, if: :foreground_checksummable?
after_commit :schedule_checksum, unless: :foreground_checksum? after_commit :schedule_checksum, if: :checksummable?
def self.remove_path(path) def self.hexdigest(path)
where(path: path).destroy_all Digest::SHA256.file(path).hexdigest
end
def self.record(uploader)
remove_path(uploader.relative_path)
create(
size: uploader.file.size,
path: uploader.relative_path,
model: uploader.model,
uploader: uploader.class.to_s
)
end end
def absolute_path def absolute_path
raise ObjectStorage::RemoteStoreError, "Remote object has no absolute path." unless local?
return path unless relative_path? return path unless relative_path?
uploader_class.absolute_path(self) uploader_class.absolute_path(self)
end end
def calculate_checksum def calculate_checksum!
return unless exist? self.checksum = nil
return unless checksummable?
self.checksum = Digest::SHA256.file(absolute_path).hexdigest self.checksum = Digest::SHA256.file(absolute_path).hexdigest
end end
def build_uploader
uploader_class.new(model).tap do |uploader|
uploader.upload = self
uploader.retrieve_from_store!(identifier)
end
end
def exist? def exist?
File.exist?(absolute_path) File.exist?(absolute_path)
end end
private private
def foreground_checksum? def checksummable?
size <= CHECKSUM_THRESHOLD checksum.nil? && local? && exist?
end
def local?
return true if store.nil?
store == ObjectStorage::Store::LOCAL
end
def foreground_checksummable?
checksummable? && size <= CHECKSUM_THRESHOLD
end end
def schedule_checksum def schedule_checksum
...@@ -57,6 +65,10 @@ class Upload < ActiveRecord::Base ...@@ -57,6 +65,10 @@ class Upload < ActiveRecord::Base
!path.start_with?('/') !path.start_with?('/')
end end
def identifier
File.basename(path)
end
def uploader_class def uploader_class
Object.const_get(uploader) Object.const_get(uploader)
end end
......
...@@ -134,6 +134,7 @@ class User < ActiveRecord::Base ...@@ -134,6 +134,7 @@ class User < ActiveRecord::Base
has_many :assigned_merge_requests, dependent: :nullify, foreign_key: :assignee_id, class_name: "MergeRequest" # rubocop:disable Cop/ActiveRecordDependent has_many :assigned_merge_requests, dependent: :nullify, foreign_key: :assignee_id, class_name: "MergeRequest" # rubocop:disable Cop/ActiveRecordDependent
has_many :custom_attributes, class_name: 'UserCustomAttribute' has_many :custom_attributes, class_name: 'UserCustomAttribute'
has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
# #
# Validations # Validations
...@@ -156,12 +157,10 @@ class User < ActiveRecord::Base ...@@ -156,12 +157,10 @@ class User < ActiveRecord::Base
validate :namespace_uniq, if: :username_changed? validate :namespace_uniq, if: :username_changed?
validate :namespace_move_dir_allowed, if: :username_changed? validate :namespace_move_dir_allowed, if: :username_changed?
validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? }
validate :unique_email, if: :email_changed? validate :unique_email, if: :email_changed?
validate :owns_notification_email, if: :notification_email_changed? validate :owns_notification_email, if: :notification_email_changed?
validate :owns_public_email, if: :public_email_changed? validate :owns_public_email, if: :public_email_changed?
validate :signup_domain_valid?, on: :create, if: ->(user) { !user.created_by_id } validate :signup_domain_valid?, on: :create, if: ->(user) { !user.created_by_id }
validates :avatar, file_size: { maximum: 200.kilobytes.to_i }
before_validation :sanitize_attrs before_validation :sanitize_attrs
before_validation :set_notification_email, if: :email_changed? before_validation :set_notification_email, if: :email_changed?
...@@ -223,9 +222,6 @@ class User < ActiveRecord::Base ...@@ -223,9 +222,6 @@ class User < ActiveRecord::Base
end end
end end
mount_uploader :avatar, AvatarUploader
has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
# Scopes # Scopes
scope :admins, -> { where(admin: true) } scope :admins, -> { where(admin: true) }
scope :blocked, -> { with_states(:blocked, :ldap_blocked) } scope :blocked, -> { with_states(:blocked, :ldap_blocked) }
...@@ -521,12 +517,6 @@ class User < ActiveRecord::Base ...@@ -521,12 +517,6 @@ class User < ActiveRecord::Base
end end
end end
def avatar_type
unless avatar.image?
errors.add :avatar, "only images allowed"
end
end
def unique_email def unique_email
if !emails.exists?(email: email) && Email.exists?(email: email) if !emails.exists?(email: email) && Email.exists?(email: email)
errors.add(:email, 'has already been taken') errors.add(:email, 'has already been taken')
...@@ -854,9 +844,7 @@ class User < ActiveRecord::Base ...@@ -854,9 +844,7 @@ class User < ActiveRecord::Base
end end
def avatar_url(size: nil, scale: 2, **args) def avatar_url(size: nil, scale: 2, **args)
# We use avatar_path instead of overriding avatar_url because of carrierwave. GravatarService.new.execute(email, size, scale, username: username)
# See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11001/diffs#note_28659864
avatar_path(args) || GravatarService.new.execute(email, size, scale, username: username)
end end
def primary_email_verified? def primary_email_verified?
......
...@@ -14,9 +14,9 @@ module Projects ...@@ -14,9 +14,9 @@ module Projects
@old_path = project.full_path @old_path = project.full_path
@new_path = project.disk_path @new_path = project.disk_path
origin = FileUploader.dynamic_path_segment(project) origin = FileUploader.absolute_base_dir(project)
project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:attachments] project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:attachments]
target = FileUploader.dynamic_path_segment(project) target = FileUploader.absolute_base_dir(project)
result = move_folder!(origin, target) result = move_folder!(origin, target)
project.save! project.save!
......
class AttachmentUploader < GitlabUploader class AttachmentUploader < GitlabUploader
include RecordsUploads include RecordsUploads::Concern
include ObjectStorage::Concern
prepend ObjectStorage::Extension::RecordsUploads
include UploaderHelper include UploaderHelper
storage :file private
def store_dir def dynamic_segment
"#{base_dir}/#{model.class.to_s.underscore}/#{mounted_as}/#{model.id}" File.join(model.class.to_s.underscore, mounted_as.to_s, model.id.to_s)
end end
end end
class AvatarUploader < GitlabUploader class AvatarUploader < GitlabUploader
include RecordsUploads
include UploaderHelper include UploaderHelper
include RecordsUploads::Concern
storage :file include ObjectStorage::Concern
prepend ObjectStorage::Extension::RecordsUploads
def store_dir
"#{base_dir}/#{model.class.to_s.underscore}/#{mounted_as}/#{model.id}"
end
def exists? def exists?
model.avatar.file && model.avatar.file.present? model.avatar.file && model.avatar.file.present?
end end
# We set move_to_store and move_to_cache to 'false' to prevent stealing
# the avatar file from a project when forking it.
# https://gitlab.com/gitlab-org/gitlab-ce/issues/26158
def move_to_store def move_to_store
false false
end end
...@@ -22,4 +15,10 @@ class AvatarUploader < GitlabUploader ...@@ -22,4 +15,10 @@ class AvatarUploader < GitlabUploader
def move_to_cache def move_to_cache
false false
end end
private
def dynamic_segment
File.join(model.class.to_s.underscore, mounted_as.to_s, model.id.to_s)
end
end end
...@@ -21,13 +21,11 @@ class FileMover ...@@ -21,13 +21,11 @@ class FileMover
end end
def update_markdown def update_markdown
updated_text = model.read_attribute(update_field).gsub(temp_file_uploader.to_markdown, uploader.to_markdown) updated_text = model.read_attribute(update_field)
.gsub(temp_file_uploader.markdown_link, uploader.markdown_link)
model.update_attribute(update_field, updated_text) model.update_attribute(update_field, updated_text)
true
rescue rescue
revert revert
false false
end end
......
# This class breaks the actual CarrierWave concept.
# Every uploader should use a base_dir that is model agnostic so we can build
# back URLs from base_dir-relative paths saved in the `Upload` model.
#
# As the `.base_dir` is model dependent and **not** saved in the upload model (see #upload_path)
# there is no way to build back the correct file path without the model, which defies
# CarrierWave way of storing files.
#
class FileUploader < GitlabUploader class FileUploader < GitlabUploader
include RecordsUploads
include UploaderHelper include UploaderHelper
include RecordsUploads::Concern
include ObjectStorage::Concern
prepend ObjectStorage::Extension::RecordsUploads
MARKDOWN_PATTERN = %r{\!?\[.*?\]\(/uploads/(?<secret>[0-9a-f]{32})/(?<file>.*?)\)} MARKDOWN_PATTERN = %r{\!?\[.*?\]\(/uploads/(?<secret>[0-9a-f]{32})/(?<file>.*?)\)}
DYNAMIC_PATH_PATTERN = %r{(?<secret>\h{32})/(?<identifier>.*)}
storage :file attr_accessor :model
def self.root
File.join(options.storage_path, 'uploads')
end
def self.absolute_path(upload_record) def self.absolute_path(upload)
File.join( File.join(
self.dynamic_path_segment(upload_record.model), absolute_base_dir(upload.model),
upload_record.path upload.path # already contain the dynamic_segment, see #upload_path
) )
end end
# Not using `GitlabUploader.base_dir` because all project namespaces are in def self.base_dir(model)
# the `public/uploads` dir. model_path_segment(model)
# end
def self.base_dir
root_dir # used in migrations and import/exports
def self.absolute_base_dir(model)
File.join(root, base_dir(model))
end end
# Returns the part of `store_dir` that can change based on the model's current # Returns the part of `store_dir` that can change based on the model's current
...@@ -29,63 +46,94 @@ class FileUploader < GitlabUploader ...@@ -29,63 +46,94 @@ class FileUploader < GitlabUploader
# model - Object that responds to `full_path` and `disk_path` # model - Object that responds to `full_path` and `disk_path`
# #
# Returns a String without a trailing slash # Returns a String without a trailing slash
def self.dynamic_path_segment(model) def self.model_path_segment(model)
if model.hashed_storage?(:attachments) if model.hashed_storage?(:attachments)
dynamic_path_builder(model.disk_path) model.disk_path
else else
dynamic_path_builder(model.full_path) model.full_path
end end
end end
# Auxiliary method to build dynamic path segment when not using a project model def self.upload_path(secret, identifier)
# File.join(secret, identifier)
# Prefer to use the `.dynamic_path_segment` as it includes Hashed Storage specific logic
def self.dynamic_path_builder(path)
File.join(CarrierWave.root, base_dir, path)
end end
attr_accessor :model def self.generate_secret
attr_reader :secret SecureRandom.hex
end
def initialize(model, secret = nil) def initialize(model, secret = nil)
@model = model @model = model
@secret = secret || generate_secret @secret = secret
end end
def store_dir def base_dir
File.join(dynamic_path_segment, @secret) self.class.base_dir(@model)
end end
def relative_path # we don't need to know the actual path, an uploader instance should be
self.file.path.sub("#{dynamic_path_segment}/", '') # able to yield the file content on demand, so we should build the digest
def absolute_path
self.class.absolute_path(@upload)
end end
def to_markdown def upload_path
to_h[:markdown] self.class.upload_path(dynamic_segment, identifier)
end end
def to_h def model_path_segment
filename = image_or_video? ? self.file.basename : self.file.filename self.class.model_path_segment(@model)
escaped_filename = filename.gsub("]", "\\]") end
markdown = "[#{escaped_filename}](#{secure_url})" def store_dir
File.join(base_dir, dynamic_segment)
end
def markdown_link
markdown = "[#{markdown_name}](#{secure_url})"
markdown.prepend("!") if image_or_video? || dangerous? markdown.prepend("!") if image_or_video? || dangerous?
markdown
end
def to_h
{ {
alt: filename, alt: markdown_name,
url: secure_url, url: secure_url,
markdown: markdown markdown: markdown_link
} }
end end
def filename
self.file.filename
end
# the upload does not hold the secret, but holds the path
# which contains the secret: extract it
def upload=(value)
if matches = DYNAMIC_PATH_PATTERN.match(value.path)
@secret = matches[:secret]
@identifier = matches[:identifier]
end
super
end
def secret
@secret ||= self.class.generate_secret
end
private private
def dynamic_path_segment def markdown_name
self.class.dynamic_path_segment(model) (image_or_video? ? File.basename(filename, File.extname(filename)) : filename).gsub("]", "\\]")
end end
def generate_secret def identifier
SecureRandom.hex @identifier ||= filename
end
def dynamic_segment
secret
end end
def secure_url def secure_url
......
class GitlabUploader < CarrierWave::Uploader::Base class GitlabUploader < CarrierWave::Uploader::Base
def self.absolute_path(upload_record) class_attribute :options
File.join(CarrierWave.root, upload_record.path)
end
def self.root_dir class << self
'uploads' # DSL setter
end def storage_options(options)
self.options = options
end
# When object storage is used, keep the `root_dir` as `base_dir`. def root
# The files aren't really in folders there, they just have a name. options.storage_path
# The files that contain user input in their name, also contain a hash, so end
# the names are still unique
#
# This method is overridden in the `FileUploader`
def self.base_dir
return root_dir unless file_storage?
File.join(root_dir, '-', 'system') # represent the directory namespacing at the class level
end def base_dir
options.fetch('base_dir', '')
end
def self.file_storage? def file_storage?
self.storage == CarrierWave::Storage::File storage == CarrierWave::Storage::File
end
def absolute_path(upload_record)
File.join(root, upload_record.path)
end
end end
storage_options Gitlab.config.uploads
delegate :base_dir, :file_storage?, to: :class delegate :base_dir, :file_storage?, to: :class
def file_cache_storage? def file_cache_storage?
cache_storage.is_a?(CarrierWave::Storage::File) cache_storage.is_a?(CarrierWave::Storage::File)
end end
# Reduce disk IO
def move_to_cache def move_to_cache
true file_storage?
end end
# Reduce disk IO
def move_to_store def move_to_store
true file_storage?
end
# Designed to be overridden by child uploaders that have a dynamic path
# segment -- that is, a path that changes based on mutable attributes of its
# associated model
#
# For example, `FileUploader` builds the storage path based on the associated
# project model's `path_with_namespace` value, which can change when the
# project or its containing namespace is moved or renamed.
def relative_path
self.file.path.sub("#{root}/", '')
end end
def exists? def exists?
file.present? file.present?
end end
# Override this if you don't want to save files by default to the Rails.root directory def cache_dir
File.join(root, base_dir, 'tmp/cache')
end
def work_dir def work_dir
# Default path set by CarrierWave: File.join(root, base_dir, 'tmp/work')
# https://github.com/carrierwaveuploader/carrierwave/blob/v1.0.0/lib/carrierwave/uploader/cache.rb#L182
CarrierWave.tmp_path
end end
def filename def filename
...@@ -67,6 +59,17 @@ class GitlabUploader < CarrierWave::Uploader::Base ...@@ -67,6 +59,17 @@ class GitlabUploader < CarrierWave::Uploader::Base
private private
# Designed to be overridden by child uploaders that have a dynamic path
# segment -- that is, a path that changes based on mutable attributes of its
# associated model
#
# For example, `FileUploader` builds the storage path based on the associated
# project model's `path_with_namespace` value, which can change when the
# project or its containing namespace is moved or renamed.
def dynamic_segment
raise(NotImplementedError)
end
# To prevent files from moving across filesystems, override the default # To prevent files from moving across filesystems, override the default
# implementation: # implementation:
# http://github.com/carrierwaveuploader/carrierwave/blob/v1.0.0/lib/carrierwave/uploader/cache.rb#L181-L183 # http://github.com/carrierwaveuploader/carrierwave/blob/v1.0.0/lib/carrierwave/uploader/cache.rb#L181-L183
...@@ -74,6 +77,6 @@ class GitlabUploader < CarrierWave::Uploader::Base ...@@ -74,6 +77,6 @@ class GitlabUploader < CarrierWave::Uploader::Base
# To be safe, keep this directory outside of the the cache directory # To be safe, keep this directory outside of the the cache directory
# because calling CarrierWave.clean_cache_files! will remove any files in # because calling CarrierWave.clean_cache_files! will remove any files in
# the cache directory. # the cache directory.
File.join(work_dir, @cache_id, version_name.to_s, for_file) File.join(work_dir, cache_id, version_name.to_s, for_file)
end end
end end
class JobArtifactUploader < ObjectStoreUploader class JobArtifactUploader < GitlabUploader
storage_options Gitlab.config.artifacts extend Workhorse::UploadPath
include ObjectStorage::Concern
def self.local_store_path
Gitlab.config.artifacts.path
end
def self.artifacts_upload_path storage_options Gitlab.config.artifacts
File.join(self.local_store_path, 'tmp/uploads/')
end
def size def size
return super if model.size.nil? return super if model.size.nil?
...@@ -15,9 +10,13 @@ class JobArtifactUploader < ObjectStoreUploader ...@@ -15,9 +10,13 @@ class JobArtifactUploader < ObjectStoreUploader
model.size model.size
end end
def store_dir
dynamic_segment
end
private private
def default_path def dynamic_segment
creation_date = model.created_at.utc.strftime('%Y_%m_%d') creation_date = model.created_at.utc.strftime('%Y_%m_%d')
File.join(disk_hash[0..1], disk_hash[2..3], disk_hash, File.join(disk_hash[0..1], disk_hash[2..3], disk_hash,
......
class LegacyArtifactUploader < ObjectStoreUploader class LegacyArtifactUploader < GitlabUploader
storage_options Gitlab.config.artifacts extend Workhorse::UploadPath
include ObjectStorage::Concern
def self.local_store_path storage_options Gitlab.config.artifacts
Gitlab.config.artifacts.path
end
def self.artifacts_upload_path def store_dir
File.join(self.local_store_path, 'tmp/uploads/') dynamic_segment
end end
private private
def default_path def dynamic_segment
File.join(model.created_at.utc.strftime('%Y_%m'), model.project_id.to_s, model.id.to_s) File.join(model.created_at.utc.strftime('%Y_%m'), model.project_id.to_s, model.id.to_s)
end end
end end
class LfsObjectUploader < ObjectStoreUploader class LfsObjectUploader < GitlabUploader
storage_options Gitlab.config.lfs extend Workhorse::UploadPath
include ObjectStorage::Concern
def self.local_store_path # LfsObject are in `tmp/upload` instead of `tmp/uploads`
Gitlab.config.lfs.storage_path def self.workhorse_upload_path
File.join(root, 'tmp/upload')
end end
storage_options Gitlab.config.lfs
def filename def filename
model.oid[4..-1] model.oid[4..-1]
end end
def store_dir
dynamic_segment
end
private private
def default_path def dynamic_segment
"#{model.oid[0, 2]}/#{model.oid[2, 2]}" File.join(model.oid[0, 2], model.oid[2, 2])
end end
end end
class NamespaceFileUploader < FileUploader class NamespaceFileUploader < FileUploader
def self.base_dir # Re-Override
File.join(root_dir, '-', 'system', 'namespace') def self.root
options.storage_path
end end
def self.dynamic_path_segment(model) def self.base_dir(model)
dynamic_path_builder(model.id.to_s) File.join(options.base_dir, 'namespace', model_path_segment(model))
end end
private def self.model_path_segment(model)
File.join(model.id.to_s)
end
# Re-Override
def store_dir
store_dirs[object_store]
end
def secure_url def store_dirs
File.join('/uploads', @secret, file.filename) {
Store::LOCAL => File.join(base_dir, dynamic_segment),
Store::REMOTE => File.join('namespace', model_path_segment, dynamic_segment)
}
end end
end end
require 'fog/aws'
require 'carrierwave/storage/fog'
class ObjectStoreUploader < GitlabUploader
before :store, :set_default_local_store
before :store, :verify_license!
LOCAL_STORE = 1
REMOTE_STORE = 2
class << self
def storage_options(options)
@storage_options = options
end
def object_store_options
@storage_options&.object_store
end
def object_store_enabled?
object_store_options&.enabled
end
def background_upload_enabled?
object_store_options&.background_upload
end
def object_store_credentials
@object_store_credentials ||= object_store_options&.connection&.to_hash&.deep_symbolize_keys
end
def object_store_directory
object_store_options&.remote_directory
end
def local_store_path
raise NotImplementedError
end
end
def file_storage?
storage.is_a?(CarrierWave::Storage::File)
end
def file_cache_storage?
cache_storage.is_a?(CarrierWave::Storage::File)
end
def real_object_store
model.public_send(store_serialization_column) # rubocop:disable GitlabSecurity/PublicSend
end
def object_store
subject.public_send(:"#{field}_store")
end
def object_store=(value)
@storage = nil
model.public_send(:"#{store_serialization_column}=", value) # rubocop:disable GitlabSecurity/PublicSend
end
def store_dir
if file_storage?
default_local_path
else
default_path
end
end
def use_file
if file_storage?
return yield path
end
begin
cache_stored_file!
yield cache_path
ensure
cache_storage.delete_dir!(cache_path(nil))
end
end
def filename
super || file&.filename
end
def migrate!(new_store)
raise 'Undefined new store' unless new_store
return unless object_store != new_store
return unless file
old_file = file
old_store = object_store
# for moving remote file we need to first store it locally
cache_stored_file! unless file_storage?
# change storage
self.object_store = new_store
with_callbacks(:store, file) do
storage.store!(file).tap do |new_file|
# since we change storage store the new storage
# in case of failure delete new file
begin
model.save!
rescue => e
new_file.delete
self.object_store = old_store
raise e
end
old_file.delete
end
end
end
def schedule_migration_to_object_storage(*args)
return unless self.class.object_store_enabled?
return unless self.class.background_upload_enabled?
return unless self.licensed?
return unless self.file_storage?
ObjectStorageUploadWorker.perform_async(self.class.name, model.class.name, mounted_as, model.id)
end
def fog_directory
self.class.object_store_options.remote_directory
end
def fog_credentials
self.class.object_store_options.connection
end
def fog_public
false
end
def move_to_store
file.try(:storage) == storage
end
def move_to_cache
file.try(:storage) == cache_storage
end
# We block storing artifacts on Object Storage, not receiving
def verify_license!(new_file)
return if file_storage?
raise 'Object Storage feature is missing' unless licensed?
end
def exists?
file.try(:exists?)
end
def cache_dir
File.join(self.class.local_store_path, 'tmp/cache')
end
# Override this if you don't want to save local files by default to the Rails.root directory
def work_dir
# Default path set by CarrierWave:
# https://github.com/carrierwaveuploader/carrierwave/blob/v1.1.0/lib/carrierwave/uploader/cache.rb#L182
# CarrierWave.tmp_path
File.join(self.class.local_store_path, 'tmp/work')
end
def licensed?
License.feature_available?(:object_storage)
end
private
def set_default_local_store(new_file)
self.object_store = LOCAL_STORE unless self.object_store
end
def default_local_path
File.join(self.class.local_store_path, default_path)
end
def default_path
raise NotImplementedError
end
def serialization_column
model.class.uploader_option(mounted_as, :mount_on) || mounted_as
end
def store_serialization_column
:"#{serialization_column}_store"
end
def storage
@storage ||=
if object_store == REMOTE_STORE
remote_storage
else
local_storage
end
end
def remote_storage
raise 'Object Storage is not enabled' unless self.class.object_store_enabled?
CarrierWave::Storage::Fog.new(self)
end
def local_storage
CarrierWave::Storage::File.new(self)
end
end
class PersonalFileUploader < FileUploader class PersonalFileUploader < FileUploader
def self.dynamic_path_segment(model) # Re-Override
File.join(CarrierWave.root, model_path(model)) def self.root
options.storage_path
end end
def self.base_dir def self.base_dir(model)
File.join(root_dir, '-', 'system') File.join(options.base_dir, model_path_segment(model))
end end
private def self.model_path_segment(model)
return 'temp/' unless model
def secure_url File.join(model.class.to_s.underscore, model.id.to_s)
File.join(self.class.model_path(model), secret, file.filename) end
def object_store
return Store::LOCAL unless model
super
end
# Revert-Override
def store_dir
store_dirs[object_store]
end
def store_dirs
{
Store::LOCAL => File.join(base_dir, dynamic_segment),
Store::REMOTE => File.join(model_path_segment, dynamic_segment)
}
end end
def self.model_path(model) private
if model
File.join("/#{base_dir}", model.class.to_s.underscore, model.id.to_s) def secure_url
else File.join('/', base_dir, secret, file.filename)
File.join("/#{base_dir}", 'temp')
end
end end
end end
module RecordsUploads module RecordsUploads
extend ActiveSupport::Concern module Concern
extend ActiveSupport::Concern
included do attr_accessor :upload
after :store, :record_upload
before :remove, :destroy_upload
end
# After storing an attachment, create a corresponding Upload record included do
# after :store, :record_upload
# NOTE: We're ignoring the argument passed to this callback because we want before :remove, :destroy_upload
# the `SanitizedFile` object from `CarrierWave::Uploader::Base#file`, not the end
# `Tempfile` object the callback gets.
# # After storing an attachment, create a corresponding Upload record
# Called `after :store` #
def record_upload(_tempfile = nil) # NOTE: We're ignoring the argument passed to this callback because we want
return unless model # the `SanitizedFile` object from `CarrierWave::Uploader::Base#file`, not the
return unless file_storage? # `Tempfile` object the callback gets.
return unless file.exists? #
# Called `after :store`
Upload.record(self) def record_upload(_tempfile = nil)
end return unless model
return unless file && file.exists?
Upload.transaction do
uploads.where(path: upload_path).delete_all
upload.destroy! if upload
self.upload = build_upload_from_uploader(self)
upload.save!
end
end
def upload_path
File.join(store_dir, filename.to_s)
end
private
def uploads
Upload.order(id: :desc).where(uploader: self.class.to_s)
end
private def build_upload_from_uploader(uploader)
Upload.new(
size: uploader.file.size,
path: uploader.upload_path,
model: uploader.model,
uploader: uploader.class.to_s
)
end
# Before removing an attachment, destroy any Upload records at the same path # Before removing an attachment, destroy any Upload records at the same path
# #
# Called `before :remove` # Called `before :remove`
def destroy_upload(*args) def destroy_upload(*args)
return unless file_storage? return unless file && file.exists?
return unless file
Upload.remove_path(relative_path) self.upload = nil
uploads.where(path: upload_path).delete_all
end
end end
end end
...@@ -32,14 +32,7 @@ module UploaderHelper ...@@ -32,14 +32,7 @@ module UploaderHelper
def extension_match?(extensions) def extension_match?(extensions)
return false unless file return false unless file
extension = extension = file.try(:extension) || File.extname(file.path).delete('.')
if file.respond_to?(:extension)
file.extension
else
# Not all CarrierWave storages respond to :extension
File.extname(file.path).delete('.')
end
extensions.include?(extension.downcase) extensions.include?(extension.downcase)
end end
end end
module Workhorse
module UploadPath
def workhorse_upload_path
File.join(root, base_dir, 'tmp/uploads')
end
end
end
...@@ -8,16 +8,16 @@ class ObjectStorageUploadWorker ...@@ -8,16 +8,16 @@ class ObjectStorageUploadWorker
uploader_class = uploader_class_name.constantize uploader_class = uploader_class_name.constantize
subject_class = subject_class_name.constantize subject_class = subject_class_name.constantize
return unless uploader_class < ObjectStorage::Concern
return unless uploader_class.object_store_enabled? return unless uploader_class.object_store_enabled?
return unless uploader_class.licensed?
return unless uploader_class.background_upload_enabled? return unless uploader_class.background_upload_enabled?
subject = subject_class.find_by(id: subject_id) subject = subject_class.find(subject_id)
return unless subject uploader = subject.public_send(file_field) # rubocop:disable GitlabSecurity/PublicSend
uploader.migrate!(ObjectStorage::Store::REMOTE)
file = subject.public_send(file_field) # rubocop:disable GitlabSecurity/PublicSend rescue RecordNotFound
# does not retry when the record do not exists
return unless file.licensed? Rails.logger.warn("Cannot find subject #{subject_class} with id=#{subject_id}.")
file.migrate!(uploader_class::REMOTE_STORE)
end end
end end
...@@ -3,7 +3,7 @@ class UploadChecksumWorker ...@@ -3,7 +3,7 @@ class UploadChecksumWorker
def perform(upload_id) def perform(upload_id)
upload = Upload.find(upload_id) upload = Upload.find(upload_id)
upload.calculate_checksum upload.calculate_checksum!
upload.save! upload.save!
rescue ActiveRecord::RecordNotFound rescue ActiveRecord::RecordNotFound
Rails.logger.error("UploadChecksumWorker: couldn't find upload #{upload_id}, skipping") Rails.logger.error("UploadChecksumWorker: couldn't find upload #{upload_id}, skipping")
......
---
title: Add object storage support for uploads.
merge_request: 3867
author:
type: added
...@@ -174,6 +174,25 @@ production: &base ...@@ -174,6 +174,25 @@ production: &base
# endpoint: 'http://127.0.0.1:9000' # default: nil # endpoint: 'http://127.0.0.1:9000' # default: nil
# path_style: true # Use 'host/bucket_name/object' instead of 'bucket_name.host/object' # path_style: true # Use 'host/bucket_name/object' instead of 'bucket_name.host/object'
## Uploads (attachments, avatars, etc...)
uploads:
# The location where uploads objects are stored (default: public/).
# storage_path: public/
# base_dir: uploads/-/system
object_store:
enabled: true
remote_directory: uploads # Bucket name
# background_upload: false # Temporary option to limit automatic upload (Default: true)
connection:
provider: AWS
aws_access_key_id: AWS_ACCESS_KEY_ID
aws_secret_access_key: AWS_SECRET_ACCESS_KEY
region: eu-central-1
# Use the following options to configure an AWS compatible host
# host: 'localhost' # default: s3.amazonaws.com
# endpoint: 'http://127.0.0.1:9000' # default: nil
# path_style: true # Use 'host/bucket_name/object' instead of 'bucket_name.host/object'
## GitLab Pages ## GitLab Pages
pages: pages:
enabled: false enabled: false
...@@ -686,6 +705,16 @@ test: ...@@ -686,6 +705,16 @@ test:
aws_access_key_id: AWS_ACCESS_KEY_ID aws_access_key_id: AWS_ACCESS_KEY_ID
aws_secret_access_key: AWS_SECRET_ACCESS_KEY aws_secret_access_key: AWS_SECRET_ACCESS_KEY
region: eu-central-1 region: eu-central-1
uploads:
storage_path: tmp/tests/public
enabled: true
object_store:
enabled: false
connection:
provider: AWS # Only AWS supported at the moment
aws_access_key_id: AWS_ACCESS_KEY_ID
aws_secret_access_key: AWS_SECRET_ACCESS_KEY
region: eu-central-1
gitlab: gitlab:
host: localhost host: localhost
port: 80 port: 80
......
...@@ -298,13 +298,15 @@ Settings.incoming_email['enabled'] = false if Settings.incoming_email['enabled'] ...@@ -298,13 +298,15 @@ Settings.incoming_email['enabled'] = false if Settings.incoming_email['enabled']
# #
Settings['artifacts'] ||= Settingslogic.new({}) Settings['artifacts'] ||= Settingslogic.new({})
Settings.artifacts['enabled'] = true if Settings.artifacts['enabled'].nil? Settings.artifacts['enabled'] = true if Settings.artifacts['enabled'].nil?
Settings.artifacts['path'] = Settings.absolute(Settings.artifacts['path'] || File.join(Settings.shared['path'], "artifacts")) Settings.artifacts['storage_path'] = Settings.absolute(Settings.artifacts.values_at('path', 'storage_path').compact.first || File.join(Settings.shared['path'], "artifacts"))
Settings.artifacts['max_size'] ||= 100 # in megabytes # Settings.artifact['path'] is deprecated, use `storage_path` instead
Settings.artifacts['path'] = Settings.artifacts['storage_path']
Settings.artifacts['max_size'] ||= 100 # in megabytes
Settings.artifacts['object_store'] ||= Settingslogic.new({}) Settings.artifacts['object_store'] ||= Settingslogic.new({})
Settings.artifacts['object_store']['enabled'] = false if Settings.artifacts['object_store']['enabled'].nil? Settings.artifacts['object_store']['enabled'] ||= false
Settings.artifacts['object_store']['remote_directory'] ||= nil Settings.artifacts['object_store']['remote_directory'] ||= nil
Settings.artifacts['object_store']['background_upload'] = true if Settings.artifacts['object_store']['background_upload'].nil? Settings.artifacts['object_store']['background_upload'] ||= true
# Convert upload connection settings to use string keys, to make Fog happy # Convert upload connection settings to use string keys, to make Fog happy
Settings.artifacts['object_store']['connection']&.deep_stringify_keys! Settings.artifacts['object_store']['connection']&.deep_stringify_keys!
...@@ -342,14 +344,26 @@ Settings.pages['artifacts_server'] ||= Settings.pages['enabled'] if Settings.pa ...@@ -342,14 +344,26 @@ Settings.pages['artifacts_server'] ||= Settings.pages['enabled'] if Settings.pa
Settings['lfs'] ||= Settingslogic.new({}) Settings['lfs'] ||= Settingslogic.new({})
Settings.lfs['enabled'] = true if Settings.lfs['enabled'].nil? Settings.lfs['enabled'] = true if Settings.lfs['enabled'].nil?
Settings.lfs['storage_path'] = Settings.absolute(Settings.lfs['storage_path'] || File.join(Settings.shared['path'], "lfs-objects")) Settings.lfs['storage_path'] = Settings.absolute(Settings.lfs['storage_path'] || File.join(Settings.shared['path'], "lfs-objects"))
Settings.lfs['object_store'] ||= Settingslogic.new({}) Settings.lfs['object_store'] ||= Settingslogic.new({})
Settings.lfs['object_store']['enabled'] = false if Settings.lfs['object_store']['enabled'].nil? Settings.lfs['object_store']['enabled'] ||= false
Settings.lfs['object_store']['remote_directory'] ||= nil Settings.lfs['object_store']['remote_directory'] ||= nil
Settings.lfs['object_store']['background_upload'] = true if Settings.lfs['object_store']['background_upload'].nil? Settings.lfs['object_store']['background_upload'] ||= true
# Convert upload connection settings to use string keys, to make Fog happy # Convert upload connection settings to use string keys, to make Fog happy
Settings.lfs['object_store']['connection']&.deep_stringify_keys! Settings.lfs['object_store']['connection']&.deep_stringify_keys!
#
# Uploads
#
Settings['uploads'] ||= Settingslogic.new({})
Settings.uploads['storage_path'] = Settings.absolute(Settings.uploads['storage_path'] || 'public')
Settings.uploads['base_dir'] = Settings.uploads['base_dir'] || 'uploads/-/system'
Settings.uploads['object_store'] ||= Settingslogic.new({})
Settings.uploads['object_store']['enabled'] ||= false
Settings.uploads['object_store']['remote_directory'] ||= 'uploads'
Settings.uploads['object_store']['background_upload'] ||= true
# Convert upload connection settings to use string keys, to make Fog happy
Settings.uploads['object_store']['connection']&.deep_stringify_keys!
# #
# Mattermost # Mattermost
# #
......
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class AddStoreColumnToUploads < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
add_column :uploads, :store, :integer
end
end
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class AddUploaderIndexToUploads < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
remove_concurrent_index :uploads, :path
add_concurrent_index :uploads, [:uploader, :path], using: :btree
end
def down
remove_concurrent_index :uploads, [:uploader, :path]
add_concurrent_index :uploads, :path, using: :btree
end
end
...@@ -1760,11 +1760,12 @@ ActiveRecord::Schema.define(version: 20171230123729) do ...@@ -1760,11 +1760,12 @@ ActiveRecord::Schema.define(version: 20171230123729) do
t.string "model_type" t.string "model_type"
t.string "uploader", null: false t.string "uploader", null: false
t.datetime "created_at", null: false t.datetime "created_at", null: false
t.integer "store"
end end
add_index "uploads", ["checksum"], name: "index_uploads_on_checksum", using: :btree add_index "uploads", ["checksum"], name: "index_uploads_on_checksum", using: :btree
add_index "uploads", ["model_id", "model_type"], name: "index_uploads_on_model_id_and_model_type", using: :btree add_index "uploads", ["model_id", "model_type"], name: "index_uploads_on_model_id_and_model_type", using: :btree
add_index "uploads", ["path"], name: "index_uploads_on_path", using: :btree add_index "uploads", ["uploader", "path"], name: "index_uploads_on_uploader_and_path", using: :btree
create_table "user_agent_details", force: :cascade do |t| create_table "user_agent_details", force: :cascade do |t|
t.string "user_agent", null: false t.string "user_agent", null: false
......
...@@ -14,8 +14,8 @@ There are many places where file uploading is used, according to contexts: ...@@ -14,8 +14,8 @@ There are many places where file uploading is used, according to contexts:
- User snippet attachments - User snippet attachments
* Project * Project
- Project avatars - Project avatars
- Issues/MR Markdown attachments - Issues/MR/Notes Markdown attachments
- Issues/MR Legacy Markdown attachments - Issues/MR/Notes Legacy Markdown attachments
- CI Build Artifacts - CI Build Artifacts
- LFS Objects - LFS Objects
...@@ -25,7 +25,7 @@ There are many places where file uploading is used, according to contexts: ...@@ -25,7 +25,7 @@ There are many places where file uploading is used, according to contexts:
GitLab started saving everything on local disk. While directory location changed from previous versions, GitLab started saving everything on local disk. While directory location changed from previous versions,
they are still not 100% standardized. You can see them below: they are still not 100% standardized. You can see them below:
| Description | In DB? | Relative path | Uploader class | model_type | | Description | In DB? | Relative path (from CarrierWave.root) | Uploader class | model_type |
| ------------------------------------- | ------ | ----------------------------------------------------------- | ---------------------- | ---------- | | ------------------------------------- | ------ | ----------------------------------------------------------- | ---------------------- | ---------- |
| Instance logo | yes | uploads/-/system/appearance/logo/:id/:filename | `AttachmentUploader` | Appearance | | Instance logo | yes | uploads/-/system/appearance/logo/:id/:filename | `AttachmentUploader` | Appearance |
| Header logo | yes | uploads/-/system/appearance/header_logo/:id/:filename | `AttachmentUploader` | Appearance | | Header logo | yes | uploads/-/system/appearance/header_logo/:id/:filename | `AttachmentUploader` | Appearance |
...@@ -33,17 +33,107 @@ they are still not 100% standardized. You can see them below: ...@@ -33,17 +33,107 @@ they are still not 100% standardized. You can see them below:
| User avatars | yes | uploads/-/system/user/avatar/:id/:filename | `AvatarUploader` | User | | User avatars | yes | uploads/-/system/user/avatar/:id/:filename | `AvatarUploader` | User |
| User snippet attachments | yes | uploads/-/system/personal_snippet/:id/:random_hex/:filename | `PersonalFileUploader` | Snippet | | User snippet attachments | yes | uploads/-/system/personal_snippet/:id/:random_hex/:filename | `PersonalFileUploader` | Snippet |
| Project avatars | yes | uploads/-/system/project/avatar/:id/:filename | `AvatarUploader` | Project | | Project avatars | yes | uploads/-/system/project/avatar/:id/:filename | `AvatarUploader` | Project |
| Issues/MR Markdown attachments | yes | uploads/:project_path_with_namespace/:random_hex/:filename | `FileUploader` | Project | | Issues/MR/Notes Markdown attachments | yes | uploads/:project_path_with_namespace/:random_hex/:filename | `FileUploader` | Project |
| Issues/MR Legacy Markdown attachments | no | uploads/-/system/note/attachment/:id/:filename | `AttachmentUploader` | Note | | Issues/MR/Notes Legacy Markdown attachments | no | uploads/-/system/note/attachment/:id/:filename | `AttachmentUploader` | Note |
| CI Artifacts (CE) | yes | shared/artifacts/:year_:month/:project_id/:id | `ArtifactUploader` | Ci::Build | | CI Artifacts (CE) | yes | shared/artifacts/:year_:month/:project_id/:id | `ArtifactUploader` | Ci::Build |
| LFS Objects (CE) | yes | shared/lfs-objects/:hex/:hex/:object_hash | `LfsObjectUploader` | LfsObject | | LFS Objects (CE) | yes | shared/lfs-objects/:hex/:hex/:object_hash | `LfsObjectUploader` | LfsObject |
CI Artifacts and LFS Objects behave differently in CE and EE. In CE they inherit the `GitlabUploader` CI Artifacts and LFS Objects behave differently in CE and EE. In CE they inherit the `GitlabUploader`
while in EE they inherit the `ObjectStoreUploader` and store files in and S3 API compatible object store. while in EE they inherit the `ObjectStorage` and store files in and S3 API compatible object store.
In the case of Issues/MR Markdown attachments, there is a different approach using the [Hashed Storage] layout, In the case of Issues/MR/Notes Markdown attachments, there is a different approach using the [Hashed Storage] layout,
instead of basing the path into a mutable variable `:project_path_with_namespace`, it's possible to use the instead of basing the path into a mutable variable `:project_path_with_namespace`, it's possible to use the
hash of the project ID instead, if project migrates to the new approach (introduced in 10.2). hash of the project ID instead, if project migrates to the new approach (introduced in 10.2).
### Path segments
Files are stored at multiple locations and use different path schemes.
All the `GitlabUploader` derived classes should comply with this path segment schema:
```
| GitlabUploader
| ----------------------- + ------------------------- + --------------------------------- + -------------------------------- |
| `<gitlab_root>/public/` | `uploads/-/system/` | `user/avatar/:id/` | `:filename` |
| ----------------------- + ------------------------- + --------------------------------- + -------------------------------- |
| `CarrierWave.root` | `GitlabUploader.base_dir` | `GitlabUploader#dynamic_segment` | `CarrierWave::Uploader#filename` |
| | `CarrierWave::Uploader#store_dir` | |
| FileUploader
| ----------------------- + ------------------------- + --------------------------------- + -------------------------------- |
| `<gitlab_root>/shared/` | `artifacts/` | `:year_:month/:id` | `:filename` |
| `<gitlab_root>/shared/` | `snippets/` | `:secret/` | `:filename` |
| ----------------------- + ------------------------- + --------------------------------- + -------------------------------- |
| `CarrierWave.root` | `GitlabUploader.base_dir` | `GitlabUploader#dynamic_segment` | `CarrierWave::Uploader#filename` |
| | `CarrierWave::Uploader#store_dir` | |
| | | `FileUploader#upload_path |
| ObjectStore::Concern (store = remote)
| ----------------------- + ------------------------- + ----------------------------------- + -------------------------------- |
| `<bucket_name>` | <ignored> | `user/avatar/:id/` | `:filename` |
| ----------------------- + ------------------------- + ----------------------------------- + -------------------------------- |
| `#fog_dir` | `GitlabUploader.base_dir` | `GitlabUploader#dynamic_segment` | `CarrierWave::Uploader#filename` |
| | | `ObjectStorage::Concern#store_dir` | |
| | | `ObjectStorage::Concern#upload_path |
```
The `RecordsUploads::Concern` concern will create an `Upload` entry for every file stored by a `GitlabUploader` persisting the dynamic parts of the path using
`GitlabUploader#dynamic_path`. You may then use the `Upload#build_uploader` method to manipulate the file.
## Object Storage
By including the `ObjectStorage::Concern` in the `GitlabUploader` derived class, you may enable the object storage for this uploader. To enable the object storage
in your uploader, you need to either 1) include `RecordsUpload::Concern` and prepend `ObjectStorage::Extension::RecordsUploads` or 2) mount the uploader and create a new field named `<mount>_store`.
The `CarrierWave::Uploader#store_dir` is overriden to
- `GitlabUploader.base_dir` + `GitlabUploader.dynamic_segment` when the store is LOCAL
- `GitlabUploader.dynamic_segment` when the store is REMOTE (the bucket name is used to namespace)
### Using `ObjectStorage::Extension::RecordsUploads`
> Note: this concern will automatically include `RecordsUploads::Concern` if not already included.
The `ObjectStorage::Concern` uploader will search for the matching `Upload` to select the correct object store. The `Upload` is mapped using `#store_dirs + identifier` for each store (LOCAL/REMOTE).
```ruby
class SongUploader < GitlabUploader
include RecordsUploads::Concern
include ObjectStorage::Concern
prepend ObjectStorage::Extension::RecordsUploads
...
end
class Thing < ActiveRecord::Base
mount :theme, SongUploader # we have a great theme song!
...
end
```
### Using a mounted uploader
The `ObjectStorage::Concern` will query the `model.<mount>_store` attribute to select the correct object store.
This column must be present in the model schema.
```ruby
class SongUploader < GitlabUploader
include ObjectStorage::Concern
...
end
class Thing < ActiveRecord::Base
attr_reader :theme_store # this is an ActiveRecord attribute
mount :theme, SongUploader # we have a great theme song!
def theme_store
super || ObjectStorage::Store::LOCAL
end
...
end
```
[CarrierWave]: https://github.com/carrierwaveuploader/carrierwave [CarrierWave]: https://github.com/carrierwaveuploader/carrierwave
[Hashed Storage]: ../administration/repository_storage_types.md [Hashed Storage]: ../administration/repository_storage_types.md
module EE
# CI::JobArtifact EE mixin
#
# This module is intended to encapsulate EE-specific model logic
# and be prepended in the `Ci::JobArtifact` model
module Ci::JobArtifact
extend ActiveSupport::Concern
prepended do
after_destroy :log_geo_event
scope :with_files_stored_locally, -> { where(file_store: [nil, JobArtifactUploader::Store::LOCAL]) }
end
def local_store?
[nil, JobArtifactUploader::Store::LOCAL].include?(self.file_store)
end
private
def log_geo_event
::Geo::JobArtifactDeletedEventStore.new(self).create
end
end
end
module EE
# LFS Object EE mixin
#
# This module is intended to encapsulate EE-specific model logic
# and be prepended in the `LfsObject` model
module LfsObject
extend ActiveSupport::Concern
prepended do
after_destroy :log_geo_event
end
def local_store?
[nil, LfsObjectUploader::Store::LOCAL].include?(self.file_store)
end
private
def log_geo_event
::Geo::LfsObjectDeletedEventStore.new(self).create
end
end
end
module Geo
module Fdw
module Ci
class JobArtifact < ::Geo::BaseFdw
self.table_name = Gitlab::Geo.fdw_table('ci_job_artifacts')
scope :with_files_stored_locally, -> { where(file_store: [nil, JobArtifactUploader::Store::LOCAL]) }
end
end
end
end
module Geo
module Fdw
class LfsObject < ::Geo::BaseFdw
self.table_name = Gitlab::Geo.fdw_table('lfs_objects')
scope :with_files_stored_locally, -> { where(file_store: [nil, LfsObjectUploader::Store::LOCAL]) }
end
end
end
module Geo
class FilesExpireService
include ::Gitlab::Geo::LogHelpers
BATCH_SIZE = 500
attr_reader :project, :old_full_path
def initialize(project, old_full_path)
@project = project
@old_full_path = old_full_path
end
# Expire already replicated uploads
#
# This is a fallback solution to support projects that haven't rolled out to hashed-storage yet.
#
# Note: Unless we add some locking mechanism, this will be best effort only
# as if there are files that are being replicated during this execution, they will not
# be expired.
#
# The long-term solution is to use hashed storage.
def execute
return unless Gitlab::Geo.secondary?
uploads = finder.find_project_uploads(project)
log_info("Expiring replicated attachments after project rename", count: uploads.count)
schedule_file_removal(uploads)
mark_for_resync!
end
# Project's base directory for attachments storage
#
# @return base directory where all uploads for the project are stored
def base_dir
@base_dir ||= File.join(FileUploader.root, old_full_path)
end
private
def schedule_file_removal(uploads)
paths_to_remove = uploads.find_each(batch_size: BATCH_SIZE).each_with_object([]) do |upload, to_remove|
file_path = File.join(base_dir, upload.path)
if File.exist?(file_path)
to_remove << [file_path]
log_info("Scheduled to remove file", file_path: file_path)
end
end
Geo::FileRemovalWorker.bulk_perform_async(paths_to_remove)
end
def mark_for_resync!
finder.find_file_registries_uploads(project).delete_all
end
def finder
@finder ||= ::Geo::ExpireUploadsFinder.new
end
# This is called by LogHelpers to build json log with context info
#
# @see ::Gitlab::Geo::LogHelpers
def base_log_data(message)
{
class: self.class.name,
project_id: project.id,
project_path: project.full_path,
project_old_path: old_full_path,
message: message
}
end
end
end
module Geo
AttachmentMigrationError = Class.new(StandardError)
class HashedStorageAttachmentsMigrationService
include ::Gitlab::Geo::LogHelpers
attr_reader :project_id, :old_attachments_path, :new_attachments_path
def initialize(project_id, old_attachments_path:, new_attachments_path:)
@project_id = project_id
@old_attachments_path = old_attachments_path
@new_attachments_path = new_attachments_path
end
def async_execute
Geo::HashedStorageAttachmentsMigrationWorker.perform_async(
project_id,
old_attachments_path,
new_attachments_path
)
end
def execute
origin = File.join(FileUploader.root, old_attachments_path)
target = File.join(FileUploader.root, new_attachments_path)
move_folder!(origin, target)
end
private
def project
@project ||= Project.find(project_id)
end
def move_folder!(old_path, new_path)
unless File.directory?(old_path)
log_info("Skipped attachments migration to Hashed Storage, source path doesn't exist or is not a directory", project_id: project.id, source: old_path, target: new_path)
return
end
if File.exist?(new_path)
log_error("Cannot migrate attachments to Hashed Storage, target path already exist", project_id: project.id, source: old_path, target: new_path)
raise AttachmentMigrationError, "Target path '#{new_path}' already exist"
end
# Create hashed storage base path folder
FileUtils.mkdir_p(File.dirname(new_path))
FileUtils.mv(old_path, new_path)
log_info("Migrated project attachments to Hashed Storage", project_id: project.id, source: old_path, target: new_path)
true
end
end
end
module Geo
class JobArtifactDeletedEventStore < EventStore
self.event_type = :job_artifact_deleted_event
attr_reader :job_artifact
def initialize(job_artifact)
@job_artifact = job_artifact
end
def create
return unless job_artifact.local_store?
super
end
private
def build_event
Geo::JobArtifactDeletedEvent.new(
job_artifact: job_artifact,
file_path: relative_file_path
)
end
def local_store_path
Pathname.new(JobArtifactUploader.root)
end
def relative_file_path
return unless job_artifact.file.present?
Pathname.new(job_artifact.file.path).relative_path_from(local_store_path)
end
# This is called by ProjectLogHelpers to build json log with context info
#
# @see ::Gitlab::Geo::ProjectLogHelpers
def base_log_data(message)
{
class: self.class.name,
job_artifact_id: job_artifact.id,
file_path: job_artifact.file.path,
message: message
}
end
end
end
module Geo
class LfsObjectDeletedEventStore < EventStore
self.event_type = :lfs_object_deleted_event
attr_reader :lfs_object
def initialize(lfs_object)
@lfs_object = lfs_object
end
def create
return unless lfs_object.local_store?
super
end
private
def build_event
Geo::LfsObjectDeletedEvent.new(
lfs_object: lfs_object,
oid: lfs_object.oid,
file_path: relative_file_path
)
end
def local_store_path
Pathname.new(LfsObjectUploader.root)
end
def relative_file_path
return unless lfs_object.file.present?
Pathname.new(lfs_object.file.path).relative_path_from(local_store_path)
end
# This is called by ProjectLogHelpers to build json log with context info
#
# @see ::Gitlab::Geo::ProjectLogHelpers
def base_log_data(message)
{
class: self.class.name,
lfs_object_id: lfs_object.id,
file_path: lfs_object.file.path,
message: message
}
end
end
end
require 'fog/aws'
require 'carrierwave/storage/fog'
#
# This concern should add object storage support
# to the GitlabUploader class
#
module ObjectStorage
RemoteStoreError = Class.new(StandardError)
UnknownStoreError = Class.new(StandardError)
ObjectStoreUnavailable = Class.new(StandardError)
module Store
LOCAL = 1
REMOTE = 2
end
module Extension
# this extension is the glue between the ObjectStorage::Concern and RecordsUploads::Concern
module RecordsUploads
extend ActiveSupport::Concern
prepended do |base|
raise ObjectStoreUnavailable, "#{base} must include ObjectStorage::Concern to use extensions." unless base < Concern
base.include(::RecordsUploads::Concern)
end
def retrieve_from_store!(identifier)
paths = store_dirs.map { |store, path| File.join(path, identifier) }
unless current_upload_satisfies?(paths, model)
# the upload we already have isn't right, find the correct one
self.upload = uploads.find_by(model: model, path: paths)
end
super
end
def build_upload_from_uploader(uploader)
super.tap { |upload| upload.store = object_store }
end
def upload=(upload)
return unless upload
self.object_store = upload.store
super
end
private
def current_upload_satisfies?(paths, model)
return false unless upload
return false unless model
paths.include?(upload.path) &&
upload.model_id == model.id &&
upload.model_type == model.class.base_class.sti_name
end
end
end
module Concern
extend ActiveSupport::Concern
included do |base|
base.include(ObjectStorage)
before :store, :verify_license!
after :migrate, :delete_migrated_file
end
class_methods do
def object_store_options
options.object_store
end
def object_store_enabled?
object_store_options.enabled
end
def background_upload_enabled?
object_store_options.background_upload
end
def object_store_credentials
object_store_options.connection.to_hash.deep_symbolize_keys
end
def remote_store_path
object_store_options.remote_directory
end
def licensed?
License.feature_available?(:object_storage)
end
end
def file_storage?
storage.is_a?(CarrierWave::Storage::File)
end
def file_cache_storage?
cache_storage.is_a?(CarrierWave::Storage::File)
end
def object_store
@object_store ||= model.try(store_serialization_column) || Store::LOCAL
end
# rubocop:disable Gitlab/ModuleWithInstanceVariables
def object_store=(value)
@object_store = value || Store::LOCAL
@storage = storage_for(object_store)
end
# rubocop:enable Gitlab/ModuleWithInstanceVariables
# Return true if the current file is part or the model (i.e. is mounted in the model)
#
def persist_object_store?
model.respond_to?(:"#{store_serialization_column}=")
end
# Save the current @object_store to the model <mounted_as>_store column
def persist_object_store!
return unless persist_object_store?
updated = model.update_column(store_serialization_column, object_store)
raise ActiveRecordError unless updated
end
def use_file
if file_storage?
return yield path
end
begin
cache_stored_file!
yield cache_path
ensure
cache_storage.delete_dir!(cache_path(nil))
end
end
def filename
super || file&.filename
end
#
# Move the file to another store
#
# new_store: Enum (Store::LOCAL, Store::REMOTE)
#
def migrate!(new_store)
return unless object_store != new_store
return unless file
new_file = nil
file_to_delete = file
from_object_store = object_store
self.object_store = new_store # changes the storage and file
cache_stored_file! if file_storage?
with_callbacks(:migrate, file_to_delete) do
with_callbacks(:store, file_to_delete) do # for #store_versions!
new_file = storage.store!(file)
persist_object_store!
self.file = new_file
end
end
file
rescue => e
# in case of failure delete new file
new_file.delete unless new_file.nil?
# revert back to the old file
self.object_store = from_object_store
self.file = file_to_delete
raise e
end
def schedule_migration_to_object_storage(*args)
return unless self.class.object_store_enabled?
return unless self.class.background_upload_enabled?
return unless self.class.licensed?
return unless self.file_storage?
ObjectStorageUploadWorker.perform_async(self.class.name, model.class.name, mounted_as, model.id)
end
def fog_directory
self.class.remote_store_path
end
def fog_credentials
self.class.object_store_credentials
end
def fog_public
false
end
def delete_migrated_file(migrated_file)
migrated_file.delete if exists?
end
def verify_license!(_file)
return if file_storage?
raise 'Object Storage feature is missing' unless self.class.licensed?
end
def exists?
file.present?
end
def store_dir(store = nil)
store_dirs[store || object_store]
end
def store_dirs
{
Store::LOCAL => File.join(base_dir, dynamic_segment),
Store::REMOTE => File.join(dynamic_segment)
}
end
private
# this is a hack around CarrierWave. The #migrate method needs to be
# able to force the current file to the migrated file upon success.
def file=(file)
@file = file # rubocop:disable Gitlab/ModuleWithInstanceVariables
end
def serialization_column
model.class.uploader_options.dig(mounted_as, :mount_on) || mounted_as
end
# Returns the column where the 'store' is saved
# defaults to 'store'
def store_serialization_column
[serialization_column, 'store'].compact.join('_').to_sym
end
def storage
@storage ||= storage_for(object_store)
end
def storage_for(store)
case store
when Store::REMOTE
raise 'Object Storage is not enabled' unless self.class.object_store_enabled?
CarrierWave::Storage::Fog.new(self)
when Store::LOCAL
CarrierWave::Storage::File.new(self)
else
raise UnknownStoreError
end
end
end
end
module Gitlab
module Geo
class FileTransfer < Transfer
def initialize(file_type, upload)
@file_type = file_type
@file_id = upload.id
@filename = upload.absolute_path
@request_data = build_request_data(upload)
rescue ObjectStorage::RemoteStoreError
Rails.logger.warn "Cannot transfer a remote object."
end
private
def build_request_data(upload)
{
id: upload.model_id,
type: upload.model_type,
checksum: upload.checksum
}
end
end
end
end
module Gitlab
module Geo
module LogCursor
class Daemon
VERSION = '0.2.0'.freeze
BATCH_SIZE = 250
SECONDARY_CHECK_INTERVAL = 1.minute
attr_reader :options
def initialize(options = {})
@options = options
@exit = false
logger.geo_logger.build.level = options[:debug] ? :debug : Rails.logger.level
end
def run!
trap_signals
until exit?
# Prevent the node from processing events unless it's a secondary
unless Geo.secondary?
sleep(SECONDARY_CHECK_INTERVAL)
next
end
lease = Lease.try_obtain_with_ttl { run_once! }
return if exit?
# When no new event is found sleep for a few moments
arbitrary_sleep(lease[:ttl])
end
end
def run_once!
LogCursor::Events.fetch_in_batches { |batch| handle_events(batch) }
end
def handle_events(batch)
batch.each do |event_log|
next unless can_replay?(event_log)
begin
event = event_log.event
handler = "handle_#{event.class.name.demodulize.underscore}"
__send__(handler, event, event_log.created_at) # rubocop:disable GitlabSecurity/PublicSend
rescue NoMethodError => e
logger.error(e.message)
raise e
end
end
end
private
def trap_signals
trap(:TERM) do
quit!
end
trap(:INT) do
quit!
end
end
# Safe shutdown
def quit!
$stdout.puts 'Exiting...'
@exit = true
end
def exit?
@exit
end
def can_replay?(event_log)
return true if event_log.project_id.nil?
Gitlab::Geo.current_node&.projects_include?(event_log.project_id)
end
def handle_repository_created_event(event, created_at)
registry = find_or_initialize_registry(event.project_id, resync_repository: true, resync_wiki: event.wiki_path.present?)
logger.event_info(
created_at,
message: 'Repository created',
project_id: event.project_id,
repo_path: event.repo_path,
wiki_path: event.wiki_path,
resync_repository: registry.resync_repository,
resync_wiki: registry.resync_wiki)
registry.save!
::Geo::ProjectSyncWorker.perform_async(event.project_id, Time.now)
end
def handle_repository_updated_event(event, created_at)
registry = find_or_initialize_registry(event.project_id, "resync_#{event.source}" => true)
logger.event_info(
created_at,
message: 'Repository update',
project_id: event.project_id,
source: event.source,
resync_repository: registry.resync_repository,
resync_wiki: registry.resync_wiki)
registry.save!
::Geo::ProjectSyncWorker.perform_async(event.project_id, Time.now)
end
def handle_repository_deleted_event(event, created_at)
job_id = ::Geo::RepositoryDestroyService
.new(event.project_id, event.deleted_project_name, event.deleted_path, event.repository_storage_name)
.async_execute
logger.event_info(
created_at,
message: 'Deleted project',
project_id: event.project_id,
repository_storage_name: event.repository_storage_name,
disk_path: event.deleted_path,
job_id: job_id)
# No need to create a project entry if it doesn't exist
::Geo::ProjectRegistry.where(project_id: event.project_id).delete_all
end
def handle_repositories_changed_event(event, created_at)
return unless Gitlab::Geo.current_node.id == event.geo_node_id
job_id = ::Geo::RepositoriesCleanUpWorker.perform_in(1.hour, event.geo_node_id)
if job_id
logger.info('Scheduled repositories clean up for Geo node', geo_node_id: event.geo_node_id, job_id: job_id)
else
logger.error('Could not schedule repositories clean up for Geo node', geo_node_id: event.geo_node_id)
end
end
def handle_repository_renamed_event(event, created_at)
return unless event.project_id
old_path = event.old_path_with_namespace
new_path = event.new_path_with_namespace
job_id = ::Geo::RenameRepositoryService
.new(event.project_id, old_path, new_path)
.async_execute
logger.event_info(
created_at,
message: 'Renaming project',
project_id: event.project_id,
old_path: old_path,
new_path: new_path,
job_id: job_id)
end
def handle_hashed_storage_migrated_event(event, created_at)
return unless event.project_id
job_id = ::Geo::HashedStorageMigrationService.new(
event.project_id,
old_disk_path: event.old_disk_path,
new_disk_path: event.new_disk_path,
old_storage_version: event.old_storage_version
).async_execute
logger.event_info(
created_at,
message: 'Migrating project to hashed storage',
project_id: event.project_id,
old_storage_version: event.old_storage_version,
new_storage_version: event.new_storage_version,
old_disk_path: event.old_disk_path,
new_disk_path: event.new_disk_path,
job_id: job_id)
end
def handle_hashed_storage_attachments_event(event, created_at)
job_id = ::Geo::HashedStorageAttachmentsMigrationService.new(
event.project_id,
old_attachments_path: event.old_attachments_path,
new_attachments_path: event.new_attachments_path
).async_execute
logger.event_info(
created_at,
message: 'Migrating attachments to hashed storage',
project_id: event.project_id,
old_attachments_path: event.old_attachments_path,
new_attachments_path: event.new_attachments_path,
job_id: job_id
)
end
def handle_lfs_object_deleted_event(event, created_at)
file_path = File.join(LfsObjectUploader.root, event.file_path)
job_id = ::Geo::FileRemovalWorker.perform_async(file_path)
logger.event_info(
created_at,
message: 'Deleted LFS object',
oid: event.oid,
file_id: event.lfs_object_id,
file_path: file_path,
job_id: job_id)
::Geo::FileRegistry.lfs_objects.where(file_id: event.lfs_object_id).delete_all
end
def handle_job_artifact_deleted_event(event, created_at)
file_registry_job_artifacts = ::Geo::FileRegistry.job_artifacts.where(file_id: event.job_artifact_id)
return unless file_registry_job_artifacts.any? # avoid race condition
file_path = File.join(::JobArtifactUploader.root, event.file_path)
if File.file?(file_path)
deleted = delete_file(file_path) # delete synchronously to ensure consistency
return unless deleted # do not delete file from registry if deletion failed
end
logger.event_info(
created_at,
message: 'Deleted job artifact',
file_id: event.job_artifact_id,
file_path: file_path)
file_registry_job_artifacts.delete_all
end
def find_or_initialize_registry(project_id, attrs)
registry = ::Geo::ProjectRegistry.find_or_initialize_by(project_id: project_id)
registry.assign_attributes(attrs)
registry
end
def delete_file(path)
File.delete(path)
rescue => ex
logger.error("Failed to remove file", exception: ex.class.name, details: ex.message, filename: path)
false
end
# Sleeps for the expired TTL that remains on the lease plus some random seconds.
#
# This allows multiple GeoLogCursors to randomly process a batch of events,
# without favouring the shortest path (or latency).
def arbitrary_sleep(delay)
sleep(delay + rand(1..20) * 0.1)
end
def logger
Gitlab::Geo::LogCursor::Logger
end
end
end
end
end
...@@ -215,9 +215,9 @@ module API ...@@ -215,9 +215,9 @@ module API
job = authenticate_job! job = authenticate_job!
forbidden!('Job is not running!') unless job.running? forbidden!('Job is not running!') unless job.running?
artifacts_upload_path = JobArtifactUploader.artifacts_upload_path workhorse_upload_path = JobArtifactUploader.workhorse_upload_path
artifacts = uploaded_file(:file, artifacts_upload_path) artifacts = uploaded_file(:file, workhorse_upload_path)
metadata = uploaded_file(:metadata, artifacts_upload_path) metadata = uploaded_file(:metadata, workhorse_upload_path)
bad_request!('Missing artifacts file!') unless artifacts bad_request!('Missing artifacts file!') unless artifacts
file_to_large! unless artifacts.size < max_artifacts_size file_to_large! unless artifacts.size < max_artifacts_size
......
...@@ -3,7 +3,7 @@ require 'backup/files' ...@@ -3,7 +3,7 @@ require 'backup/files'
module Backup module Backup
class Artifacts < Files class Artifacts < Files
def initialize def initialize
super('artifacts', LegacyArtifactUploader.local_store_path) super('artifacts', JobArtifactUploader.root)
end end
def create_files_dir def create_files_dir
......
...@@ -143,7 +143,7 @@ module Gitlab ...@@ -143,7 +143,7 @@ module Gitlab
end end
def absolute_path def absolute_path
File.join(CarrierWave.root, path) File.join(Gitlab.config.uploads.storage_path, path)
end end
end end
......
...@@ -10,9 +10,12 @@ module Gitlab ...@@ -10,9 +10,12 @@ module Gitlab
FIND_BATCH_SIZE = 500 FIND_BATCH_SIZE = 500
RELATIVE_UPLOAD_DIR = "uploads".freeze RELATIVE_UPLOAD_DIR = "uploads".freeze
ABSOLUTE_UPLOAD_DIR = "#{CarrierWave.root}/#{RELATIVE_UPLOAD_DIR}".freeze ABSOLUTE_UPLOAD_DIR = File.join(
Gitlab.config.uploads.storage_path,
RELATIVE_UPLOAD_DIR
)
FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze
START_WITH_CARRIERWAVE_ROOT_REGEX = %r{\A#{CarrierWave.root}/} START_WITH_ROOT_REGEX = %r{\A#{Gitlab.config.uploads.storage_path}/}
EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze
EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze
...@@ -80,7 +83,7 @@ module Gitlab ...@@ -80,7 +83,7 @@ module Gitlab
paths = [] paths = []
stdout.each_line("\0") do |line| stdout.each_line("\0") do |line|
paths << line.chomp("\0").sub(START_WITH_CARRIERWAVE_ROOT_REGEX, '') paths << line.chomp("\0").sub(START_WITH_ROOT_REGEX, '')
if paths.size >= batch_size if paths.size >= batch_size
yield(paths) yield(paths)
......
...@@ -27,7 +27,7 @@ module Gitlab ...@@ -27,7 +27,7 @@ module Gitlab
with_link_in_tmp_dir(file.file) do |open_tmp_file| with_link_in_tmp_dir(file.file) do |open_tmp_file|
new_uploader.store!(open_tmp_file) new_uploader.store!(open_tmp_file)
end end
new_uploader.to_markdown new_uploader.markdown_link
end end
end end
......
...@@ -17,15 +17,13 @@ module Gitlab ...@@ -17,15 +17,13 @@ module Gitlab
false false
end end
private def uploads_path
FileUploader.absolute_base_dir(@project)
end
def uploads_export_path def uploads_export_path
File.join(@shared.export_path, 'uploads') File.join(@shared.export_path, 'uploads')
end end
def uploads_path
FileUploader.dynamic_path_segment(@project)
end
end end
end end
end end
module Gitlab module Gitlab
class UploadsTransfer < ProjectTransfer class UploadsTransfer < ProjectTransfer
def root_dir def root_dir
File.join(CarrierWave.root, FileUploader.base_dir) FileUploader.root
end end
end end
end end
...@@ -51,14 +51,14 @@ module Gitlab ...@@ -51,14 +51,14 @@ module Gitlab
def lfs_upload_ok(oid, size) def lfs_upload_ok(oid, size)
{ {
StoreLFSPath: "#{Gitlab.config.lfs.storage_path}/tmp/upload", StoreLFSPath: LfsObjectUploader.workhorse_upload_path,
LfsOid: oid, LfsOid: oid,
LfsSize: size LfsSize: size
} }
end end
def artifact_upload_ok def artifact_upload_ok
{ TempPath: JobArtifactUploader.artifacts_upload_path } { TempPath: JobArtifactUploader.workhorse_upload_path }
end end
def send_git_blob(repository, blob) def send_git_blob(repository, blob)
......
...@@ -12,8 +12,8 @@ namespace :gitlab do ...@@ -12,8 +12,8 @@ namespace :gitlab do
.with_artifacts_stored_locally .with_artifacts_stored_locally
.find_each(batch_size: 10) do |build| .find_each(batch_size: 10) do |build|
begin begin
build.artifacts_file.migrate!(ObjectStoreUploader::REMOTE_STORE) build.artifacts_file.migrate!(ObjectStorage::Store::REMOTE)
build.artifacts_metadata.migrate!(ObjectStoreUploader::REMOTE_STORE) build.artifacts_metadata.migrate!(ObjectStorage::Store::REMOTE)
logger.info("Transferred artifacts of #{build.id} of #{build.artifacts_size} to object storage") logger.info("Transferred artifacts of #{build.id} of #{build.artifacts_size} to object storage")
rescue => e rescue => e
......
...@@ -10,7 +10,7 @@ namespace :gitlab do ...@@ -10,7 +10,7 @@ namespace :gitlab do
LfsObject.with_files_stored_locally LfsObject.with_files_stored_locally
.find_each(batch_size: 10) do |lfs_object| .find_each(batch_size: 10) do |lfs_object|
begin begin
lfs_object.file.migrate!(LfsObjectUploader::REMOTE_STORE) lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE)
logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to object storage") logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to object storage")
rescue => e rescue => e
......
...@@ -6,5 +6,7 @@ describe Groups::UploadsController do ...@@ -6,5 +6,7 @@ describe Groups::UploadsController do
{ group_id: model } { group_id: model }
end end
it_behaves_like 'handle uploads' it_behaves_like 'handle uploads' do
let(:uploader_class) { NamespaceFileUploader }
end
end end
...@@ -145,8 +145,8 @@ describe Projects::ArtifactsController do ...@@ -145,8 +145,8 @@ describe Projects::ArtifactsController do
context 'when using local file storage' do context 'when using local file storage' do
it_behaves_like 'a valid file' do it_behaves_like 'a valid file' do
let(:job) { create(:ci_build, :success, :artifacts, pipeline: pipeline) } let(:job) { create(:ci_build, :success, :artifacts, pipeline: pipeline) }
let(:store) { ObjectStoreUploader::LOCAL_STORE } let(:store) { ObjectStorage::Store::LOCAL }
let(:archive_path) { JobArtifactUploader.local_store_path } let(:archive_path) { JobArtifactUploader.root }
end end
end end
...@@ -158,7 +158,7 @@ describe Projects::ArtifactsController do ...@@ -158,7 +158,7 @@ describe Projects::ArtifactsController do
it_behaves_like 'a valid file' do it_behaves_like 'a valid file' do
let!(:artifact) { create(:ci_job_artifact, :archive, :remote_store, job: job) } let!(:artifact) { create(:ci_job_artifact, :archive, :remote_store, job: job) }
let!(:job) { create(:ci_build, :success, pipeline: pipeline) } let!(:job) { create(:ci_build, :success, pipeline: pipeline) }
let(:store) { ObjectStoreUploader::REMOTE_STORE } let(:store) { ObjectStorage::Store::REMOTE }
let(:archive_path) { 'https://' } let(:archive_path) { 'https://' }
end end
end end
......
...@@ -47,7 +47,7 @@ describe Projects::RawController do ...@@ -47,7 +47,7 @@ describe Projects::RawController do
end end
it 'serves the file' do it 'serves the file' do
expect(controller).to receive(:send_file).with("#{Gitlab.config.shared.path}/lfs-objects/91/ef/f75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897", filename: 'lfs_object.iso', disposition: 'attachment') expect(controller).to receive(:send_file).with("#{LfsObjectUploader.root}/91/ef/f75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897", filename: 'lfs_object.iso', disposition: 'attachment')
get_show(public_project, id) get_show(public_project, id)
expect(response).to have_gitlab_http_status(200) expect(response).to have_gitlab_http_status(200)
...@@ -58,7 +58,7 @@ describe Projects::RawController do ...@@ -58,7 +58,7 @@ describe Projects::RawController do
lfs_object.file = fixture_file_upload(Rails.root + "spec/fixtures/dk.png", "`/png") lfs_object.file = fixture_file_upload(Rails.root + "spec/fixtures/dk.png", "`/png")
lfs_object.save! lfs_object.save!
stub_lfs_object_storage stub_lfs_object_storage
lfs_object.file.migrate!(LfsObjectUploader::REMOTE_STORE) lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE)
end end
it 'responds with redirect to file' do it 'responds with redirect to file' do
......
...@@ -180,6 +180,7 @@ describe UploadsController do ...@@ -180,6 +180,7 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'user', mounted_as: 'avatar', id: user.id, filename: 'image.png' get :show, model: 'user', mounted_as: 'avatar', id: user.id, filename: 'image.png'
response response
end end
end end
...@@ -196,6 +197,7 @@ describe UploadsController do ...@@ -196,6 +197,7 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'user', mounted_as: 'avatar', id: user.id, filename: 'image.png' get :show, model: 'user', mounted_as: 'avatar', id: user.id, filename: 'image.png'
response response
end end
end end
...@@ -220,6 +222,7 @@ describe UploadsController do ...@@ -220,6 +222,7 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'project', mounted_as: 'avatar', id: project.id, filename: 'image.png' get :show, model: 'project', mounted_as: 'avatar', id: project.id, filename: 'image.png'
response response
end end
end end
...@@ -239,6 +242,7 @@ describe UploadsController do ...@@ -239,6 +242,7 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'project', mounted_as: 'avatar', id: project.id, filename: 'image.png' get :show, model: 'project', mounted_as: 'avatar', id: project.id, filename: 'image.png'
response response
end end
end end
...@@ -291,6 +295,7 @@ describe UploadsController do ...@@ -291,6 +295,7 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'project', mounted_as: 'avatar', id: project.id, filename: 'image.png' get :show, model: 'project', mounted_as: 'avatar', id: project.id, filename: 'image.png'
response response
end end
end end
...@@ -322,6 +327,7 @@ describe UploadsController do ...@@ -322,6 +327,7 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'group', mounted_as: 'avatar', id: group.id, filename: 'image.png' get :show, model: 'group', mounted_as: 'avatar', id: group.id, filename: 'image.png'
response response
end end
end end
...@@ -341,6 +347,7 @@ describe UploadsController do ...@@ -341,6 +347,7 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'group', mounted_as: 'avatar', id: group.id, filename: 'image.png' get :show, model: 'group', mounted_as: 'avatar', id: group.id, filename: 'image.png'
response response
end end
end end
...@@ -384,6 +391,7 @@ describe UploadsController do ...@@ -384,6 +391,7 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'group', mounted_as: 'avatar', id: group.id, filename: 'image.png' get :show, model: 'group', mounted_as: 'avatar', id: group.id, filename: 'image.png'
response response
end end
end end
...@@ -420,6 +428,7 @@ describe UploadsController do ...@@ -420,6 +428,7 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'note', mounted_as: 'attachment', id: note.id, filename: 'image.png' get :show, model: 'note', mounted_as: 'attachment', id: note.id, filename: 'image.png'
response response
end end
end end
...@@ -439,6 +448,7 @@ describe UploadsController do ...@@ -439,6 +448,7 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'note', mounted_as: 'attachment', id: note.id, filename: 'image.png' get :show, model: 'note', mounted_as: 'attachment', id: note.id, filename: 'image.png'
response response
end end
end end
...@@ -491,6 +501,7 @@ describe UploadsController do ...@@ -491,6 +501,7 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'note', mounted_as: 'attachment', id: note.id, filename: 'image.png' get :show, model: 'note', mounted_as: 'attachment', id: note.id, filename: 'image.png'
response response
end end
end end
...@@ -522,6 +533,7 @@ describe UploadsController do ...@@ -522,6 +533,7 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'appearance', mounted_as: 'header_logo', id: appearance.id, filename: 'dk.png' get :show, model: 'appearance', mounted_as: 'header_logo', id: appearance.id, filename: 'dk.png'
response response
end end
end end
...@@ -541,6 +553,7 @@ describe UploadsController do ...@@ -541,6 +553,7 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'appearance', mounted_as: 'logo', id: appearance.id, filename: 'dk.png' get :show, model: 'appearance', mounted_as: 'logo', id: appearance.id, filename: 'dk.png'
response response
end end
end end
......
This diff is collapsed.
require 'spec_helper'
describe Gitlab::Geo::FileTransfer do
let(:user) { create(:user, avatar: fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'image/png')) }
let(:upload) { Upload.find_by(model: user, uploader: 'AvatarUploader') }
subject { described_class.new(:file, upload) }
describe '#execute' do
context 'user avatar' do
it 'sets an absolute path' do
expect(subject.file_type).to eq(:file)
expect(subject.file_id).to eq(upload.id)
expect(subject.filename).to eq(upload.absolute_path)
expect(Pathname.new(subject.filename).absolute?).to be_truthy
expect(subject.request_data).to eq({ id: upload.model_id,
type: 'User',
checksum: upload.checksum })
end
end
end
end
This diff is collapsed.
...@@ -8,14 +8,14 @@ describe LfsObject do ...@@ -8,14 +8,14 @@ describe LfsObject do
expect(subject.local_store?).to eq true expect(subject.local_store?).to eq true
end end
it 'returns true when file_store is equal to LfsObjectUploader::LOCAL_STORE' do it 'returns true when file_store is equal to LfsObjectUploader::Store::LOCAL' do
subject.file_store = LfsObjectUploader::LOCAL_STORE subject.file_store = LfsObjectUploader::Store::LOCAL
expect(subject.local_store?).to eq true expect(subject.local_store?).to eq true
end end
it 'returns false whe file_store is equal to LfsObjectUploader::REMOTE_STORE' do it 'returns false whe file_store is equal to LfsObjectUploader::Store::REMOTE' do
subject.file_store = LfsObjectUploader::REMOTE_STORE subject.file_store = LfsObjectUploader::Store::REMOTE
expect(subject.local_store?).to eq false expect(subject.local_store?).to eq false
end end
......
require 'spec_helper'
describe Projects::HashedStorage::MigrateAttachmentsService do
let(:project) { create(:project, storage_version: 1) }
let(:service) { described_class.new(project) }
let(:legacy_storage) { Storage::LegacyProject.new(project) }
let(:hashed_storage) { Storage::HashedProject.new(project) }
let(:old_attachments_path) { legacy_storage.disk_path }
let(:new_attachments_path) { hashed_storage.disk_path }
describe '#execute' do
set(:primary) { create(:geo_node, :primary) }
set(:secondary) { create(:geo_node) }
context 'on success' do
before do
TestEnv.clean_test_path
FileUtils.mkdir_p(File.join(FileUploader.root, old_attachments_path))
end
it 'returns true' do
expect(service.execute).to be_truthy
end
it 'creates a Geo::HashedStorageAttachmentsEvent' do
expect { service.execute }.to change(Geo::EventLog, :count).by(1)
event = Geo::EventLog.first.event
expect(event).to be_a(Geo::HashedStorageAttachmentsEvent)
expect(event).to have_attributes(
old_attachments_path: old_attachments_path,
new_attachments_path: new_attachments_path
)
end
end
context 'on failure' do
it 'does not create a Geo event when skipped' do
expect { service.execute }.not_to change { Geo::EventLog.count }
end
it 'does not create a Geo event on failure' do
expect(service).to receive(:move_folder!).and_raise(::Projects::HashedStorage::AttachmentMigrationError)
expect { service.execute }.to raise_error(::Projects::HashedStorage::AttachmentMigrationError)
expect(Geo::EventLog.count).to eq(0)
end
end
end
end
require 'spec_helper'
describe Geo::FileDownloadService do
include ::EE::GeoHelpers
set(:primary) { create(:geo_node, :primary) }
set(:secondary) { create(:geo_node) }
before do
stub_current_geo_node(secondary)
allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(true)
end
describe '#execute' do
context 'user avatar' do
let(:user) { create(:user, avatar: fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'image/png')) }
let(:upload) { Upload.find_by(model: user, uploader: 'AvatarUploader') }
subject(:execute!) { described_class.new(:avatar, upload.id).execute }
it 'downloads a user avatar' do
stub_transfer(Gitlab::Geo::FileTransfer, 100)
expect { execute! }.to change { Geo::FileRegistry.synced.count }.by(1)
end
it 'registers when the download fails' do
stub_transfer(Gitlab::Geo::FileTransfer, -1)
expect { execute! }.to change { Geo::FileRegistry.failed.count }.by(1)
expect(Geo::FileRegistry.last.retry_count).to eq(1)
expect(Geo::FileRegistry.last.retry_at).to be_present
end
it 'registers when the download fails with some other error' do
stub_transfer(Gitlab::Geo::FileTransfer, nil)
expect { execute! }.to change { Geo::FileRegistry.failed.count }.by(1)
end
end
context 'group avatar' do
let(:group) { create(:group, avatar: fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'image/png')) }
let(:upload) { Upload.find_by(model: group, uploader: 'AvatarUploader') }
subject(:execute!) { described_class.new(:avatar, upload.id).execute }
it 'downloads a group avatar' do
stub_transfer(Gitlab::Geo::FileTransfer, 100)
expect { execute! }.to change { Geo::FileRegistry.synced.count }.by(1)
end
it 'registers when the download fails' do
stub_transfer(Gitlab::Geo::FileTransfer, -1)
expect { execute! }.to change { Geo::FileRegistry.failed.count }.by(1)
end
end
context 'project avatar' do
let(:project) { create(:project, avatar: fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'image/png')) }
let(:upload) { Upload.find_by(model: project, uploader: 'AvatarUploader') }
subject(:execute!) { described_class.new(:avatar, upload.id).execute }
it 'downloads a project avatar' do
stub_transfer(Gitlab::Geo::FileTransfer, 100)
expect { execute! }.to change { Geo::FileRegistry.synced.count }.by(1)
end
it 'registers when the download fails' do
stub_transfer(Gitlab::Geo::FileTransfer, -1)
expect { execute! }.to change { Geo::FileRegistry.failed.count }.by(1)
end
end
context 'with an attachment' do
let(:note) { create(:note, :with_attachment) }
let(:upload) { Upload.find_by(model: note, uploader: 'AttachmentUploader') }
subject(:execute!) { described_class.new(:attachment, upload.id).execute }
it 'downloads the attachment' do
stub_transfer(Gitlab::Geo::FileTransfer, 100)
expect { execute! }.to change { Geo::FileRegistry.synced.count }.by(1)
end
it 'registers when the download fails' do
stub_transfer(Gitlab::Geo::FileTransfer, -1)
expect { execute! }.to change { Geo::FileRegistry.failed.count }.by(1)
end
end
context 'with a snippet' do
let(:upload) { create(:upload, :personal_snippet_upload) }
subject(:execute!) { described_class.new(:personal_file, upload.id).execute }
it 'downloads the file' do
stub_transfer(Gitlab::Geo::FileTransfer, 100)
expect { execute! }.to change { Geo::FileRegistry.synced.count }.by(1)
end
it 'registers when the download fails' do
stub_transfer(Gitlab::Geo::FileTransfer, -1)
expect { execute! }.to change { Geo::FileRegistry.failed.count }.by(1)
end
end
context 'with file upload' do
let(:project) { create(:project) }
let(:upload) { Upload.find_by(model: project, uploader: 'FileUploader') }
subject { described_class.new(:file, upload.id) }
before do
FileUploader.new(project).store!(fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'image/png'))
end
it 'downloads the file' do
stub_transfer(Gitlab::Geo::FileTransfer, 100)
expect { subject.execute }.to change { Geo::FileRegistry.synced.count }.by(1)
end
it 'registers when the download fails' do
stub_transfer(Gitlab::Geo::FileTransfer, -1)
expect { subject.execute }.to change { Geo::FileRegistry.failed.count }.by(1)
end
end
context 'with namespace file upload' do
let(:group) { create(:group) }
let(:upload) { Upload.find_by(model: group, uploader: 'NamespaceFileUploader') }
subject { described_class.new(:file, upload.id) }
before do
NamespaceFileUploader.new(group).store!(fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'image/png'))
end
it 'downloads the file' do
stub_transfer(Gitlab::Geo::FileTransfer, 100)
expect { subject.execute }.to change { Geo::FileRegistry.synced.count }.by(1)
end
it 'registers when the download fails' do
stub_transfer(Gitlab::Geo::FileTransfer, -1)
expect { subject.execute }.to change { Geo::FileRegistry.failed.count }.by(1)
end
end
context 'LFS object' do
let(:lfs_object) { create(:lfs_object) }
subject { described_class.new(:lfs, lfs_object.id) }
it 'downloads an LFS object' do
stub_transfer(Gitlab::Geo::LfsTransfer, 100)
expect { subject.execute }.to change { Geo::FileRegistry.synced.count }.by(1)
end
it 'registers when the download fails' do
stub_transfer(Gitlab::Geo::LfsTransfer, -1)
expect { subject.execute }.to change { Geo::FileRegistry.failed.count }.by(1)
end
it 'logs a message' do
stub_transfer(Gitlab::Geo::LfsTransfer, 100)
expect(Gitlab::Geo::Logger).to receive(:info).with(hash_including(:message, :download_time_s, success: true, bytes_downloaded: 100)).and_call_original
subject.execute
end
end
context 'job artifacts' do
let(:job_artifact) { create(:ci_job_artifact) }
subject { described_class.new(:job_artifact, job_artifact.id) }
it 'downloads a job artifact' do
stub_transfer(Gitlab::Geo::JobArtifactTransfer, 100)
expect { subject.execute }.to change { Geo::FileRegistry.synced.count }.by(1)
end
it 'registers when the download fails' do
stub_transfer(Gitlab::Geo::JobArtifactTransfer, -1)
expect { subject.execute }.to change { Geo::FileRegistry.failed.count }.by(1)
end
it 'logs a message' do
stub_transfer(Gitlab::Geo::JobArtifactTransfer, 100)
expect(Gitlab::Geo::Logger).to receive(:info).with(hash_including(:message, :download_time_s, success: true, bytes_downloaded: 100)).and_call_original
subject.execute
end
end
context 'bad object type' do
it 'raises an error' do
expect { described_class.new(:bad, 1).execute }.to raise_error(NameError)
end
end
def stub_transfer(kls, result)
instance = double("(instance of #{kls})", download_from_primary: result)
allow(kls).to receive(:new).and_return(instance)
end
end
end
require 'spec_helper'
# Disable transactions via :delete method because a foreign table
# can't see changes inside a transaction of a different connection.
describe Geo::FilesExpireService, :geo, :delete do
let(:project) { create(:project) }
let!(:old_full_path) { project.full_path }
subject { described_class.new(project, old_full_path) }
describe '#execute' do
let(:file_uploader) { build(:file_uploader, project: project) }
let!(:upload) { Upload.find_by(path: file_uploader.upload_path) }
let!(:file_registry) { create(:geo_file_registry, file_id: upload.id) }
before do
project.update(path: "#{project.path}_renamed")
end
context 'when in Geo secondary node' do
before do
allow(Gitlab::Geo).to receive(:secondary?) { true }
end
it 'remove file from disk' do
file_path = File.join(subject.base_dir, upload.path)
expect(File.exist?(file_path)).to be_truthy
Sidekiq::Testing.inline! { subject.execute }
expect(File.exist?(file_path)).to be_falsey
end
it 'removes file_registry associates with upload' do
expect(file_registry.success).to be_truthy
subject.execute
expect { file_registry.reload }.to raise_error(ActiveRecord::RecordNotFound)
end
end
context 'when not in Geo secondary node' do
it 'no-op execute action' do
expect(subject).not_to receive(:schedule_file_removal)
expect(subject).not_to receive(:mark_for_resync!)
subject.execute
end
end
end
end
require 'spec_helper'
def base_path(storage)
File.join(FileUploader.root, storage.disk_path)
end
describe Geo::HashedStorageAttachmentsMigrationService do
let!(:project) { create(:project) }
let(:legacy_storage) { Storage::LegacyProject.new(project) }
let(:hashed_storage) { Storage::HashedProject.new(project) }
let!(:upload) { Upload.find_by(path: file_uploader.upload_path) }
let(:file_uploader) { build(:file_uploader, project: project) }
let(:old_path) { File.join(base_path(legacy_storage), upload.path) }
let(:new_path) { File.join(base_path(hashed_storage), upload.path) }
subject(:service) do
described_class.new(project.id,
old_attachments_path: legacy_storage.disk_path,
new_attachments_path: hashed_storage.disk_path)
end
describe '#execute' do
context 'when succeeds' do
it 'moves attachments to hashed storage layout' do
expect(File.file?(old_path)).to be_truthy
expect(File.file?(new_path)).to be_falsey
expect(File.exist?(base_path(legacy_storage))).to be_truthy
expect(File.exist?(base_path(hashed_storage))).to be_falsey
expect(FileUtils).to receive(:mv).with(base_path(legacy_storage), base_path(hashed_storage)).and_call_original
service.execute
expect(File.exist?(base_path(hashed_storage))).to be_truthy
expect(File.exist?(base_path(legacy_storage))).to be_falsey
expect(File.file?(old_path)).to be_falsey
expect(File.file?(new_path)).to be_truthy
end
end
context 'when original folder does not exist anymore' do
before do
FileUtils.rm_rf(base_path(legacy_storage))
end
it 'skips moving folders and go to next' do
expect(FileUtils).not_to receive(:mv).with(base_path(legacy_storage), base_path(hashed_storage))
service.execute
expect(File.exist?(base_path(hashed_storage))).to be_falsey
expect(File.file?(new_path)).to be_falsey
end
end
context 'when target folder already exists' do
before do
FileUtils.mkdir_p(base_path(hashed_storage))
end
it 'raises AttachmentMigrationError' do
expect(FileUtils).not_to receive(:mv).with(base_path(legacy_storage), base_path(hashed_storage))
expect { service.execute }.to raise_error(::Geo::AttachmentMigrationError)
end
end
end
describe '#async_execute' do
it 'starts the worker' do
expect(Geo::HashedStorageAttachmentsMigrationWorker).to receive(:perform_async)
service.async_execute
end
it 'returns job id' do
allow(Geo::HashedStorageAttachmentsMigrationWorker).to receive(:perform_async).and_return('foo')
expect(service.async_execute).to eq('foo')
end
end
end
This diff is collapsed.
require 'spec_helper' require 'spec_helper'
describe ObjectStorageUploadWorker do describe ObjectStorageUploadWorker do
let(:local) { ObjectStoreUploader::LOCAL_STORE } let(:local) { ObjectStorage::Store::LOCAL }
let(:remote) { ObjectStoreUploader::REMOTE_STORE } let(:remote) { ObjectStorage::Store::REMOTE }
def perform def perform
described_class.perform_async(uploader_class.name, subject_class, file_field, subject_id) described_class.perform_async(uploader_class.name, subject_class, file_field, subject_id)
......
...@@ -6,7 +6,7 @@ FactoryBot.define do ...@@ -6,7 +6,7 @@ FactoryBot.define do
file_type :archive file_type :archive
trait :remote_store do trait :remote_store do
file_store JobArtifactUploader::REMOTE_STORE file_store JobArtifactUploader::Store::REMOTE
end end
after :build do |artifact| after :build do |artifact|
......
FactoryBot.define do
factory :geo_event_log, class: Geo::EventLog do
trait :created_event do
repository_created_event factory: :geo_repository_created_event
end
trait :updated_event do
repository_updated_event factory: :geo_repository_updated_event
end
trait :deleted_event do
repository_deleted_event factory: :geo_repository_deleted_event
end
trait :renamed_event do
repository_renamed_event factory: :geo_repository_renamed_event
end
trait :hashed_storage_migration_event do
hashed_storage_migrated_event factory: :geo_hashed_storage_migrated_event
end
trait :hashed_storage_attachments_event do
hashed_storage_attachments_event factory: :geo_hashed_storage_attachments_event
end
trait :lfs_object_deleted_event do
lfs_object_deleted_event factory: :geo_lfs_object_deleted_event
end
trait :job_artifact_deleted_event do
job_artifact_deleted_event factory: :geo_job_artifact_deleted_event
end
end
factory :geo_repository_created_event, class: Geo::RepositoryCreatedEvent do
project
repository_storage_name { project.repository_storage }
repository_storage_path { project.repository_storage_path }
add_attribute(:repo_path) { project.disk_path }
project_name { project.name }
wiki_path { project.wiki.disk_path }
end
factory :geo_repository_updated_event, class: Geo::RepositoryUpdatedEvent do
project
source 0
branches_affected 0
tags_affected 0
end
factory :geo_repository_deleted_event, class: Geo::RepositoryDeletedEvent do
project
repository_storage_name { project.repository_storage }
repository_storage_path { project.repository_storage_path }
deleted_path { project.path_with_namespace }
deleted_project_name { project.name }
end
factory :geo_repositories_changed_event, class: Geo::RepositoriesChangedEvent do
geo_node
end
factory :geo_repository_renamed_event, class: Geo::RepositoryRenamedEvent do
project { create(:project, :repository) }
repository_storage_name { project.repository_storage }
repository_storage_path { project.repository_storage_path }
old_path_with_namespace { project.path_with_namespace }
new_path_with_namespace { project.path_with_namespace + '_new' }
old_wiki_path_with_namespace { project.wiki.path_with_namespace }
new_wiki_path_with_namespace { project.wiki.path_with_namespace + '_new' }
old_path { project.path }
new_path { project.path + '_new' }
end
factory :geo_hashed_storage_migrated_event, class: Geo::HashedStorageMigratedEvent do
project { create(:project, :repository) }
repository_storage_name { project.repository_storage }
repository_storage_path { project.repository_storage_path }
old_disk_path { project.path_with_namespace }
new_disk_path { project.path_with_namespace + '_new' }
old_wiki_disk_path { project.wiki.path_with_namespace }
new_wiki_disk_path { project.wiki.path_with_namespace + '_new' }
new_storage_version { Project::HASHED_STORAGE_FEATURES[:repository] }
end
factory :geo_hashed_storage_attachments_event, class: Geo::HashedStorageAttachmentsEvent do
project { create(:project, :repository) }
old_attachments_path { Storage::LegacyProject.new(project).disk_path }
new_attachments_path { Storage::HashedProject.new(project).disk_path }
end
factory :geo_lfs_object_deleted_event, class: Geo::LfsObjectDeletedEvent do
lfs_object { create(:lfs_object, :with_file) }
after(:build, :stub) do |event, _|
local_store_path = Pathname.new(LfsObjectUploader.root)
relative_path = Pathname.new(event.lfs_object.file.path).relative_path_from(local_store_path)
event.oid = event.lfs_object.oid
event.file_path = relative_path
end
end
factory :geo_job_artifact_deleted_event, class: Geo::JobArtifactDeletedEvent do
job_artifact { create(:ci_job_artifact, :archive) }
after(:build, :stub) do |event, _|
local_store_path = Pathname.new(JobArtifactUploader.root)
relative_path = Pathname.new(event.job_artifact.file.path).relative_path_from(local_store_path)
event.file_path = relative_path
end
end
end
...@@ -18,7 +18,7 @@ FactoryBot.define do ...@@ -18,7 +18,7 @@ FactoryBot.define do
end end
trait :with_avatar do trait :with_avatar do
avatar { File.open(Rails.root.join('spec/fixtures/dk.png')) } avatar { fixture_file_upload('spec/fixtures/dk.png') }
end end
trait :access_requestable do trait :access_requestable do
......
...@@ -122,11 +122,11 @@ FactoryBot.define do ...@@ -122,11 +122,11 @@ FactoryBot.define do
end end
trait :with_attachment do trait :with_attachment do
attachment { fixture_file_upload(Rails.root + "spec/fixtures/dk.png", "image/png") } attachment { fixture_file_upload(Rails.root.join( "spec/fixtures/dk.png"), "image/png") }
end end
trait :with_svg_attachment do trait :with_svg_attachment do
attachment { fixture_file_upload(Rails.root + "spec/fixtures/unsanitized.svg", "image/svg+xml") } attachment { fixture_file_upload(Rails.root.join("spec/fixtures/unsanitized.svg"), "image/svg+xml") }
end end
transient do transient do
......
...@@ -90,7 +90,7 @@ FactoryBot.define do ...@@ -90,7 +90,7 @@ FactoryBot.define do
end end
trait :with_avatar do trait :with_avatar do
avatar { File.open(Rails.root.join('spec/fixtures/dk.png')) } avatar { fixture_file_upload('spec/fixtures/dk.png') }
end end
trait :broken_storage do trait :broken_storage do
......
FactoryBot.define do FactoryBot.define do
factory :upload do factory :upload do
model { build(:project) } model { build(:project) }
path { "uploads/-/system/project/avatar/avatar.jpg" }
size 100.kilobytes size 100.kilobytes
uploader "AvatarUploader" uploader "AvatarUploader"
store ObjectStorage::Store::LOCAL
trait :personal_snippet do # we should build a mount agnostic upload by default
transient do
mounted_as :avatar
secret SecureRandom.hex
end
# this needs to comply with RecordsUpload::Concern#upload_path
path { File.join("uploads/-/system", model.class.to_s.underscore, mounted_as.to_s, 'avatar.jpg') }
trait :personal_snippet_upload do
model { build(:personal_snippet) } model { build(:personal_snippet) }
path { File.join(secret, 'myfile.jpg') }
uploader "PersonalFileUploader" uploader "PersonalFileUploader"
end end
trait :issuable_upload do trait :issuable_upload do
path { "#{SecureRandom.hex}/myfile.jpg" } path { File.join(secret, 'myfile.jpg') }
uploader "FileUploader" uploader "FileUploader"
end end
trait :namespace_upload do trait :namespace_upload do
path { "#{SecureRandom.hex}/myfile.jpg" }
model { build(:group) } model { build(:group) }
path { File.join(secret, 'myfile.jpg') }
uploader "NamespaceFileUploader" uploader "NamespaceFileUploader"
end end
trait :attachment_upload do
transient do
mounted_as :attachment
end
model { build(:note) }
uploader "AttachmentUploader"
end
end end
end end
...@@ -38,7 +38,7 @@ FactoryBot.define do ...@@ -38,7 +38,7 @@ FactoryBot.define do
end end
trait :with_avatar do trait :with_avatar do
avatar { File.open(Rails.root.join('spec/fixtures/dk.png')) } avatar { fixture_file_upload('spec/fixtures/dk.png') }
end end
trait :two_factor_via_otp do trait :two_factor_via_otp do
......
...@@ -23,6 +23,27 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do ...@@ -23,6 +23,27 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do
end end
end end
# E.g. The installation is in use at the time of migration, and someone has
# just uploaded a file
shared_examples 'does not add files in /uploads/tmp' do
let(:tmp_file) { Rails.root.join(described_class::ABSOLUTE_UPLOAD_DIR, 'tmp', 'some_file.jpg') }
before do
FileUtils.mkdir(File.dirname(tmp_file))
FileUtils.touch(tmp_file)
end
after do
FileUtils.rm(tmp_file)
end
it 'does not add files from /uploads/tmp' do
described_class.new.perform
expect(untracked_files_for_uploads.count).to eq(5)
end
end
it 'ensures the untracked_files_for_uploads table exists' do it 'ensures the untracked_files_for_uploads table exists' do
expect do expect do
described_class.new.perform described_class.new.perform
...@@ -109,24 +130,8 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do ...@@ -109,24 +130,8 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do
end end
end end
# E.g. The installation is in use at the time of migration, and someone has
# just uploaded a file
context 'when there are files in /uploads/tmp' do context 'when there are files in /uploads/tmp' do
let(:tmp_file) { Rails.root.join(described_class::ABSOLUTE_UPLOAD_DIR, 'tmp', 'some_file.jpg') } it_behaves_like 'does not add files in /uploads/tmp'
before do
FileUtils.touch(tmp_file)
end
after do
FileUtils.rm(tmp_file)
end
it 'does not add files from /uploads/tmp' do
described_class.new.perform
expect(untracked_files_for_uploads.count).to eq(5)
end
end end
end end
end end
...@@ -197,24 +202,8 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do ...@@ -197,24 +202,8 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do
end end
end end
# E.g. The installation is in use at the time of migration, and someone has
# just uploaded a file
context 'when there are files in /uploads/tmp' do context 'when there are files in /uploads/tmp' do
let(:tmp_file) { Rails.root.join(described_class::ABSOLUTE_UPLOAD_DIR, 'tmp', 'some_file.jpg') } it_behaves_like 'does not add files in /uploads/tmp'
before do
FileUtils.touch(tmp_file)
end
after do
FileUtils.rm(tmp_file)
end
it 'does not add files from /uploads/tmp' do
described_class.new.perform
expect(untracked_files_for_uploads.count).to eq(5)
end
end end
end end
end end
......
...@@ -17,7 +17,7 @@ describe Gitlab::Gfm::UploadsRewriter do ...@@ -17,7 +17,7 @@ describe Gitlab::Gfm::UploadsRewriter do
end end
let(:text) do let(:text) do
"Text and #{image_uploader.to_markdown} and #{zip_uploader.to_markdown}" "Text and #{image_uploader.markdown_link} and #{zip_uploader.markdown_link}"
end end
describe '#rewrite' do describe '#rewrite' do
......
...@@ -4,7 +4,6 @@ describe Gitlab::ImportExport::UploadsRestorer do ...@@ -4,7 +4,6 @@ describe Gitlab::ImportExport::UploadsRestorer do
describe 'bundle a project Git repo' do describe 'bundle a project Git repo' do
let(:export_path) { "#{Dir.tmpdir}/uploads_saver_spec" } let(:export_path) { "#{Dir.tmpdir}/uploads_saver_spec" }
let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.full_path) } let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.full_path) }
let(:uploads_path) { FileUploader.dynamic_path_segment(project) }
before do before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
...@@ -26,9 +25,9 @@ describe Gitlab::ImportExport::UploadsRestorer do ...@@ -26,9 +25,9 @@ describe Gitlab::ImportExport::UploadsRestorer do
end end
it 'copies the uploads to the project path' do it 'copies the uploads to the project path' do
restorer.restore subject.restore
uploads = Dir.glob(File.join(uploads_path, '**/*')).map { |file| File.basename(file) } uploads = Dir.glob(File.join(subject.uploads_path, '**/*')).map { |file| File.basename(file) }
expect(uploads).to include('dummy.txt') expect(uploads).to include('dummy.txt')
end end
...@@ -44,9 +43,9 @@ describe Gitlab::ImportExport::UploadsRestorer do ...@@ -44,9 +43,9 @@ describe Gitlab::ImportExport::UploadsRestorer do
end end
it 'copies the uploads to the project path' do it 'copies the uploads to the project path' do
restorer.restore subject.restore
uploads = Dir.glob(File.join(uploads_path, '**/*')).map { |file| File.basename(file) } uploads = Dir.glob(File.join(subject.uploads_path, '**/*')).map { |file| File.basename(file) }
expect(uploads).to include('dummy.txt') expect(uploads).to include('dummy.txt')
end end
......
...@@ -30,7 +30,7 @@ describe Gitlab::ImportExport::UploadsSaver do ...@@ -30,7 +30,7 @@ describe Gitlab::ImportExport::UploadsSaver do
it 'copies the uploads to the export path' do it 'copies the uploads to the export path' do
saver.save saver.save
uploads = Dir.glob(File.join(shared.export_path, 'uploads', '**/*')).map { |file| File.basename(file) } uploads = Dir.glob(File.join(saver.uploads_export_path, '**/*')).map { |file| File.basename(file) }
expect(uploads).to include('banana_sample.gif') expect(uploads).to include('banana_sample.gif')
end end
...@@ -52,7 +52,7 @@ describe Gitlab::ImportExport::UploadsSaver do ...@@ -52,7 +52,7 @@ describe Gitlab::ImportExport::UploadsSaver do
it 'copies the uploads to the export path' do it 'copies the uploads to the export path' do
saver.save saver.save
uploads = Dir.glob(File.join(shared.export_path, 'uploads', '**/*')).map { |file| File.basename(file) } uploads = Dir.glob(File.join(saver.uploads_export_path, '**/*')).map { |file| File.basename(file) }
expect(uploads).to include('banana_sample.gif') expect(uploads).to include('banana_sample.gif')
end end
......
...@@ -12,6 +12,10 @@ describe RemoveEmptyForkNetworks, :migration do ...@@ -12,6 +12,10 @@ describe RemoveEmptyForkNetworks, :migration do
deleted_project.destroy! deleted_project.destroy!
end end
after do
Upload.reset_column_information
end
it 'deletes only the fork network without members' do it 'deletes only the fork network without members' do
expect(fork_networks.count).to eq(2) expect(fork_networks.count).to eq(2)
......
...@@ -204,7 +204,7 @@ describe Namespace do ...@@ -204,7 +204,7 @@ describe Namespace do
let(:parent) { create(:group, name: 'parent', path: 'parent') } let(:parent) { create(:group, name: 'parent', path: 'parent') }
let(:child) { create(:group, name: 'child', path: 'child', parent: parent) } let(:child) { create(:group, name: 'child', path: 'child', parent: parent) }
let!(:project) { create(:project_empty_repo, path: 'the-project', namespace: child, skip_disk_validation: true) } let!(:project) { create(:project_empty_repo, path: 'the-project', namespace: child, skip_disk_validation: true) }
let(:uploads_dir) { File.join(CarrierWave.root, FileUploader.base_dir) } let(:uploads_dir) { FileUploader.root }
let(:pages_dir) { File.join(TestEnv.pages_path) } let(:pages_dir) { File.join(TestEnv.pages_path) }
before do before do
......
...@@ -45,51 +45,6 @@ describe Upload do ...@@ -45,51 +45,6 @@ describe Upload do
end end
end end
describe '.remove_path' do
it 'removes all records at the given path' do
described_class.create!(
size: File.size(__FILE__),
path: __FILE__,
model: build_stubbed(:user),
uploader: 'AvatarUploader'
)
expect { described_class.remove_path(__FILE__) }
.to change { described_class.count }.from(1).to(0)
end
end
describe '.record' do
let(:fake_uploader) do
double(
file: double(size: 12_345),
relative_path: 'foo/bar.jpg',
model: build_stubbed(:user),
class: 'AvatarUploader'
)
end
it 'removes existing paths before creation' do
expect(described_class).to receive(:remove_path)
.with(fake_uploader.relative_path)
described_class.record(fake_uploader)
end
it 'creates a new record and assigns size, path, model, and uploader' do
upload = described_class.record(fake_uploader)
aggregate_failures do
expect(upload).to be_persisted
expect(upload.size).to eq fake_uploader.file.size
expect(upload.path).to eq fake_uploader.relative_path
expect(upload.model_id).to eq fake_uploader.model.id
expect(upload.model_type).to eq fake_uploader.model.class.to_s
expect(upload.uploader).to eq fake_uploader.class
end
end
end
describe '#absolute_path' do describe '#absolute_path' do
it 'returns the path directly when already absolute' do it 'returns the path directly when already absolute' do
path = '/path/to/namespace/project/secret/file.jpg' path = '/path/to/namespace/project/secret/file.jpg'
...@@ -111,27 +66,27 @@ describe Upload do ...@@ -111,27 +66,27 @@ describe Upload do
end end
end end
describe '#calculate_checksum' do describe '#calculate_checksum!' do
it 'calculates the SHA256 sum' do let(:upload) do
upload = described_class.new( described_class.new(path: __FILE__,
path: __FILE__, size: described_class::CHECKSUM_THRESHOLD - 1.megabyte)
size: described_class::CHECKSUM_THRESHOLD - 1.megabyte end
)
it 'sets `checksum` to SHA256 sum of the file' do
expected = Digest::SHA256.file(__FILE__).hexdigest expected = Digest::SHA256.file(__FILE__).hexdigest
expect { upload.calculate_checksum } expect { upload.calculate_checksum! }
.to change { upload.checksum }.from(nil).to(expected) .to change { upload.checksum }.from(nil).to(expected)
end end
it 'returns nil for a non-existant file' do it 'sets `checksum` to nil for a non-existant file' do
upload = described_class.new(
path: __FILE__,
size: described_class::CHECKSUM_THRESHOLD - 1.megabyte
)
expect(upload).to receive(:exist?).and_return(false) expect(upload).to receive(:exist?).and_return(false)
expect(upload.calculate_checksum).to be_nil checksum = Digest::SHA256.file(__FILE__).hexdigest
upload.checksum = checksum
expect { upload.calculate_checksum! }
.to change { upload.checksum }.from(checksum).to(nil)
end end
end end
......
...@@ -947,7 +947,7 @@ describe API::Runner do ...@@ -947,7 +947,7 @@ describe API::Runner do
context 'when artifacts are being stored inside of tmp path' do context 'when artifacts are being stored inside of tmp path' do
before do before do
# by configuring this path we allow to pass temp file from any path # by configuring this path we allow to pass temp file from any path
allow(JobArtifactUploader).to receive(:artifacts_upload_path).and_return('/') allow(JobArtifactUploader).to receive(:workhorse_upload_path).and_return('/')
end end
context 'when job has been erased' do context 'when job has been erased' do
...@@ -1124,7 +1124,7 @@ describe API::Runner do ...@@ -1124,7 +1124,7 @@ describe API::Runner do
# by configuring this path we allow to pass file from @tmpdir only # by configuring this path we allow to pass file from @tmpdir only
# but all temporary files are stored in system tmp directory # but all temporary files are stored in system tmp directory
@tmpdir = Dir.mktmpdir @tmpdir = Dir.mktmpdir
allow(JobArtifactUploader).to receive(:artifacts_upload_path).and_return(@tmpdir) allow(JobArtifactUploader).to receive(:workhorse_upload_path).and_return(@tmpdir)
end end
after do after do
...@@ -1153,7 +1153,7 @@ describe API::Runner do ...@@ -1153,7 +1153,7 @@ describe API::Runner do
context 'when job has artifacts' do context 'when job has artifacts' do
let(:job) { create(:ci_build) } let(:job) { create(:ci_build) }
let(:store) { JobArtifactUploader::LOCAL_STORE } let(:store) { JobArtifactUploader::Store::LOCAL }
before do before do
create(:ci_job_artifact, :archive, file_store: store, job: job) create(:ci_job_artifact, :archive, file_store: store, job: job)
...@@ -1175,7 +1175,7 @@ describe API::Runner do ...@@ -1175,7 +1175,7 @@ describe API::Runner do
end end
context 'when artifacts are stored remotely' do context 'when artifacts are stored remotely' do
let(:store) { JobArtifactUploader::REMOTE_STORE } let(:store) { JobArtifactUploader::Store::REMOTE }
let!(:job) { create(:ci_build) } let!(:job) { create(:ci_build) }
it 'download artifacts' do it 'download artifacts' do
......
...@@ -245,7 +245,7 @@ describe 'Git LFS API and storage' do ...@@ -245,7 +245,7 @@ describe 'Git LFS API and storage' do
context 'when LFS uses object storage' do context 'when LFS uses object storage' do
let(:before_get) do let(:before_get) do
stub_lfs_object_storage stub_lfs_object_storage
lfs_object.file.migrate!(LfsObjectUploader::REMOTE_STORE) lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE)
end end
it 'responds with redirect' do it 'responds with redirect' do
...@@ -975,7 +975,7 @@ describe 'Git LFS API and storage' do ...@@ -975,7 +975,7 @@ describe 'Git LFS API and storage' do
end end
it 'responds with status 200, location of lfs store and object details' do it 'responds with status 200, location of lfs store and object details' do
expect(json_response['StoreLFSPath']).to eq("#{Gitlab.config.shared.path}/lfs-objects/tmp/upload") expect(json_response['StoreLFSPath']).to eq(LfsObjectUploader.workhorse_upload_path)
expect(json_response['LfsOid']).to eq(sample_oid) expect(json_response['LfsOid']).to eq(sample_oid)
expect(json_response['LfsSize']).to eq(sample_size) expect(json_response['LfsSize']).to eq(sample_size)
end end
...@@ -1132,7 +1132,7 @@ describe 'Git LFS API and storage' do ...@@ -1132,7 +1132,7 @@ describe 'Git LFS API and storage' do
end end
it 'with location of lfs store and object details' do it 'with location of lfs store and object details' do
expect(json_response['StoreLFSPath']).to eq("#{Gitlab.config.shared.path}/lfs-objects/tmp/upload") expect(json_response['StoreLFSPath']).to eq(LfsObjectUploader.workhorse_upload_path)
expect(json_response['LfsOid']).to eq(sample_oid) expect(json_response['LfsOid']).to eq(sample_oid)
expect(json_response['LfsSize']).to eq(sample_size) expect(json_response['LfsSize']).to eq(sample_size)
end end
...@@ -1246,7 +1246,7 @@ describe 'Git LFS API and storage' do ...@@ -1246,7 +1246,7 @@ describe 'Git LFS API and storage' do
end end
def setup_tempfile(lfs_tmp) def setup_tempfile(lfs_tmp)
upload_path = "#{Gitlab.config.lfs.storage_path}/tmp/upload" upload_path = LfsObjectUploader.workhorse_upload_path
FileUtils.mkdir_p(upload_path) FileUtils.mkdir_p(upload_path)
FileUtils.touch(File.join(upload_path, lfs_tmp)) FileUtils.touch(File.join(upload_path, lfs_tmp))
......
...@@ -244,7 +244,7 @@ describe Issues::MoveService do ...@@ -244,7 +244,7 @@ describe Issues::MoveService do
context 'issue description with uploads' do context 'issue description with uploads' do
let(:uploader) { build(:file_uploader, project: old_project) } let(:uploader) { build(:file_uploader, project: old_project) }
let(:description) { "Text and #{uploader.to_markdown}" } let(:description) { "Text and #{uploader.markdown_link}" }
include_context 'issue move executed' include_context 'issue move executed'
......
...@@ -6,7 +6,7 @@ describe Projects::HashedStorage::MigrateAttachmentsService do ...@@ -6,7 +6,7 @@ describe Projects::HashedStorage::MigrateAttachmentsService do
let(:legacy_storage) { Storage::LegacyProject.new(project) } let(:legacy_storage) { Storage::LegacyProject.new(project) }
let(:hashed_storage) { Storage::HashedProject.new(project) } let(:hashed_storage) { Storage::HashedProject.new(project) }
let!(:upload) { Upload.find_by(path: file_uploader.relative_path) } let!(:upload) { Upload.find_by(path: file_uploader.upload_path) }
let(:file_uploader) { build(:file_uploader, project: project) } let(:file_uploader) { build(:file_uploader, project: project) }
let(:old_path) { File.join(base_path(legacy_storage), upload.path) } let(:old_path) { File.join(base_path(legacy_storage), upload.path) }
let(:new_path) { File.join(base_path(hashed_storage), upload.path) } let(:new_path) { File.join(base_path(hashed_storage), upload.path) }
...@@ -58,6 +58,6 @@ describe Projects::HashedStorage::MigrateAttachmentsService do ...@@ -58,6 +58,6 @@ describe Projects::HashedStorage::MigrateAttachmentsService do
end end
def base_path(storage) def base_path(storage)
FileUploader.dynamic_path_builder(storage.disk_path) File.join(FileUploader.root, storage.disk_path)
end end
end end
...@@ -2,6 +2,8 @@ shared_examples 'handle uploads' do ...@@ -2,6 +2,8 @@ shared_examples 'handle uploads' do
let(:user) { create(:user) } let(:user) { create(:user) }
let(:jpg) { fixture_file_upload(Rails.root + 'spec/fixtures/rails_sample.jpg', 'image/jpg') } let(:jpg) { fixture_file_upload(Rails.root + 'spec/fixtures/rails_sample.jpg', 'image/jpg') }
let(:txt) { fixture_file_upload(Rails.root + 'spec/fixtures/doc_sample.txt', 'text/plain') } let(:txt) { fixture_file_upload(Rails.root + 'spec/fixtures/doc_sample.txt', 'text/plain') }
let(:secret) { FileUploader.generate_secret }
let(:uploader_class) { FileUploader }
describe "POST #create" do describe "POST #create" do
context 'when a user is not authorized to upload a file' do context 'when a user is not authorized to upload a file' do
...@@ -65,7 +67,12 @@ shared_examples 'handle uploads' do ...@@ -65,7 +67,12 @@ shared_examples 'handle uploads' do
describe "GET #show" do describe "GET #show" do
let(:show_upload) do let(:show_upload) do
get :show, params.merge(secret: "123456", filename: "image.jpg") get :show, params.merge(secret: secret, filename: "rails_sample.jpg")
end
before do
expect(FileUploader).to receive(:generate_secret).and_return(secret)
UploadService.new(model, jpg, uploader_class).execute
end end
context "when the model is public" do context "when the model is public" do
...@@ -75,11 +82,6 @@ shared_examples 'handle uploads' do ...@@ -75,11 +82,6 @@ shared_examples 'handle uploads' do
context "when not signed in" do context "when not signed in" do
context "when the file exists" do context "when the file exists" do
before do
allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg)
allow(jpg).to receive(:exists?).and_return(true)
end
it "responds with status 200" do it "responds with status 200" do
show_upload show_upload
...@@ -88,6 +90,10 @@ shared_examples 'handle uploads' do ...@@ -88,6 +90,10 @@ shared_examples 'handle uploads' do
end end
context "when the file doesn't exist" do context "when the file doesn't exist" do
before do
allow_any_instance_of(FileUploader).to receive(:exists?).and_return(false)
end
it "responds with status 404" do it "responds with status 404" do
show_upload show_upload
...@@ -102,11 +108,6 @@ shared_examples 'handle uploads' do ...@@ -102,11 +108,6 @@ shared_examples 'handle uploads' do
end end
context "when the file exists" do context "when the file exists" do
before do
allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg)
allow(jpg).to receive(:exists?).and_return(true)
end
it "responds with status 200" do it "responds with status 200" do
show_upload show_upload
...@@ -115,6 +116,10 @@ shared_examples 'handle uploads' do ...@@ -115,6 +116,10 @@ shared_examples 'handle uploads' do
end end
context "when the file doesn't exist" do context "when the file doesn't exist" do
before do
allow_any_instance_of(FileUploader).to receive(:exists?).and_return(false)
end
it "responds with status 404" do it "responds with status 404" do
show_upload show_upload
...@@ -131,11 +136,6 @@ shared_examples 'handle uploads' do ...@@ -131,11 +136,6 @@ shared_examples 'handle uploads' do
context "when not signed in" do context "when not signed in" do
context "when the file exists" do context "when the file exists" do
before do
allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg)
allow(jpg).to receive(:exists?).and_return(true)
end
context "when the file is an image" do context "when the file is an image" do
before do before do
allow_any_instance_of(FileUploader).to receive(:image?).and_return(true) allow_any_instance_of(FileUploader).to receive(:image?).and_return(true)
...@@ -149,6 +149,10 @@ shared_examples 'handle uploads' do ...@@ -149,6 +149,10 @@ shared_examples 'handle uploads' do
end end
context "when the file is not an image" do context "when the file is not an image" do
before do
allow_any_instance_of(FileUploader).to receive(:image?).and_return(false)
end
it "redirects to the sign in page" do it "redirects to the sign in page" do
show_upload show_upload
...@@ -158,6 +162,10 @@ shared_examples 'handle uploads' do ...@@ -158,6 +162,10 @@ shared_examples 'handle uploads' do
end end
context "when the file doesn't exist" do context "when the file doesn't exist" do
before do
allow_any_instance_of(FileUploader).to receive(:exists?).and_return(false)
end
it "redirects to the sign in page" do it "redirects to the sign in page" do
show_upload show_upload
...@@ -177,11 +185,6 @@ shared_examples 'handle uploads' do ...@@ -177,11 +185,6 @@ shared_examples 'handle uploads' do
end end
context "when the file exists" do context "when the file exists" do
before do
allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg)
allow(jpg).to receive(:exists?).and_return(true)
end
it "responds with status 200" do it "responds with status 200" do
show_upload show_upload
...@@ -190,6 +193,10 @@ shared_examples 'handle uploads' do ...@@ -190,6 +193,10 @@ shared_examples 'handle uploads' do
end end
context "when the file doesn't exist" do context "when the file doesn't exist" do
before do
allow_any_instance_of(FileUploader).to receive(:exists?).and_return(false)
end
it "responds with status 404" do it "responds with status 404" do
show_upload show_upload
...@@ -200,11 +207,6 @@ shared_examples 'handle uploads' do ...@@ -200,11 +207,6 @@ shared_examples 'handle uploads' do
context "when the user doesn't have access to the model" do context "when the user doesn't have access to the model" do
context "when the file exists" do context "when the file exists" do
before do
allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg)
allow(jpg).to receive(:exists?).and_return(true)
end
context "when the file is an image" do context "when the file is an image" do
before do before do
allow_any_instance_of(FileUploader).to receive(:image?).and_return(true) allow_any_instance_of(FileUploader).to receive(:image?).and_return(true)
...@@ -218,6 +220,10 @@ shared_examples 'handle uploads' do ...@@ -218,6 +220,10 @@ shared_examples 'handle uploads' do
end end
context "when the file is not an image" do context "when the file is not an image" do
before do
allow_any_instance_of(FileUploader).to receive(:image?).and_return(false)
end
it "responds with status 404" do it "responds with status 404" do
show_upload show_upload
...@@ -227,6 +233,10 @@ shared_examples 'handle uploads' do ...@@ -227,6 +233,10 @@ shared_examples 'handle uploads' do
end end
context "when the file doesn't exist" do context "when the file doesn't exist" do
before do
allow_any_instance_of(FileUploader).to receive(:exists?).and_return(false)
end
it "responds with status 404" do it "responds with status 404" do
show_upload show_upload
......
...@@ -30,4 +30,11 @@ module StubConfiguration ...@@ -30,4 +30,11 @@ module StubConfiguration
remote_directory: 'lfs-objects', remote_directory: 'lfs-objects',
**params) **params)
end end
def stub_uploads_object_storage(uploader = described_class, **params)
stub_object_storage_uploader(config: Gitlab.config.uploads.object_store,
uploader: uploader,
remote_directory: 'uploads',
**params)
end
end end
...@@ -239,7 +239,7 @@ module TestEnv ...@@ -239,7 +239,7 @@ module TestEnv
end end
def artifacts_path def artifacts_path
Gitlab.config.artifacts.path Gitlab.config.artifacts.storage_path
end end
# When no cached assets exist, manually hit the root path to create them # When no cached assets exist, manually hit the root path to create them
......
module TrackUntrackedUploadsHelpers module TrackUntrackedUploadsHelpers
def uploaded_file def uploaded_file
fixture_path = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') fixture_path = Rails.root.join('spec/fixtures/rails_sample.jpg')
fixture_file_upload(fixture_path) fixture_file_upload(fixture_path)
end end
......
This diff is collapsed.
...@@ -6,8 +6,8 @@ describe 'gitlab:lfs namespace rake task' do ...@@ -6,8 +6,8 @@ describe 'gitlab:lfs namespace rake task' do
end end
describe 'migrate' do describe 'migrate' do
let(:local) { ObjectStoreUploader::LOCAL_STORE } let(:local) { ObjectStorage::Store::LOCAL }
let(:remote) { ObjectStoreUploader::REMOTE_STORE } let(:remote) { ObjectStorage::Store::REMOTE }
let!(:lfs_object) { create(:lfs_object, :with_file, file_store: local) } let!(:lfs_object) { create(:lfs_object, :with_file, file_store: local) }
def lfs_migrate def lfs_migrate
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment