Commit 8af23def authored by Kamil Trzciński's avatar Kamil Trzciński

Revert "Merge branch '3867-port-to-ce' into 'master'"

This reverts commit 54a575f1, reversing
changes made to c63af942.
parent 54a575f1
module UploadsActions module UploadsActions
include Gitlab::Utils::StrongMemoize include Gitlab::Utils::StrongMemoize
UPLOAD_MOUNTS = %w(avatar attachment file logo header_logo).freeze
def create def create
link_to_file = UploadService.new(model, params[:file], uploader_class).execute link_to_file = UploadService.new(model, params[:file], uploader_class).execute
...@@ -19,71 +17,34 @@ module UploadsActions ...@@ -19,71 +17,34 @@ module UploadsActions
end end
end end
# This should either
# - send the file directly
# - or redirect to its URL
#
def show def show
return render_404 unless uploader.exists? return render_404 unless uploader.exists?
if uploader.file_storage? disposition = uploader.image_or_video? ? 'inline' : 'attachment'
disposition = uploader.image_or_video? ? 'inline' : 'attachment'
expires_in 0.seconds, must_revalidate: true, private: true
send_file uploader.file.path, disposition: disposition
else
redirect_to uploader.url
end
end
private
def uploader_class expires_in 0.seconds, must_revalidate: true, private: true
raise NotImplementedError
end
def upload_mount send_file uploader.file.path, disposition: disposition
mounted_as = params[:mounted_as]
mounted_as if UPLOAD_MOUNTS.include?(mounted_as)
end end
def uploader_mounted? private
upload_model_class < CarrierWave::Mount::Extension && !upload_mount.nil?
end
def uploader def uploader
strong_memoize(:uploader) do strong_memoize(:uploader) do
if uploader_mounted? return if show_model.nil?
model.public_send(upload_mount) # rubocop:disable GitlabSecurity/PublicSend
else
build_uploader_from_upload || build_uploader_from_params
end
end
end
def build_uploader_from_upload
return nil unless params[:secret] && params[:filename]
upload_path = uploader_class.upload_path(params[:secret], params[:filename]) file_uploader = FileUploader.new(show_model, params[:secret])
upload = Upload.find_by(uploader: uploader_class.to_s, path: upload_path) file_uploader.retrieve_from_store!(params[:filename])
upload&.build_uploader
end
def build_uploader_from_params file_uploader
uploader = uploader_class.new(model, params[:secret]) end
uploader.retrieve_from_store!(params[:filename])
uploader
end end
def image_or_video? def image_or_video?
uploader && uploader.exists? && uploader.image_or_video? uploader && uploader.exists? && uploader.image_or_video?
end end
def find_model def uploader_class
nil FileUploader
end
def model
strong_memoize(:model) { find_model }
end end
end end
...@@ -7,23 +7,29 @@ class Groups::UploadsController < Groups::ApplicationController ...@@ -7,23 +7,29 @@ class Groups::UploadsController < Groups::ApplicationController
private private
def upload_model_class def show_model
Group strong_memoize(:show_model) do
end group_id = params[:group_id]
def uploader_class Group.find_by_full_path(group_id)
NamespaceFileUploader end
end end
def find_model def authorize_upload_file!
return @group if @group render_404 unless can?(current_user, :upload_file, group)
end
group_id = params[:group_id]
Group.find_by_full_path(group_id) def uploader
strong_memoize(:uploader) do
file_uploader = uploader_class.new(show_model, params[:secret])
file_uploader.retrieve_from_store!(params[:filename])
file_uploader
end
end end
def authorize_upload_file! def uploader_class
render_404 unless can?(current_user, :upload_file, group) NamespaceFileUploader
end end
alias_method :model, :group
end end
...@@ -60,7 +60,7 @@ class Projects::LfsStorageController < Projects::GitHttpClientController ...@@ -60,7 +60,7 @@ class Projects::LfsStorageController < Projects::GitHttpClientController
def store_file(oid, size, tmp_file) def store_file(oid, size, tmp_file)
# Define tmp_file_path early because we use it in "ensure" # Define tmp_file_path early because we use it in "ensure"
tmp_file_path = File.join(LfsObjectUploader.workhorse_upload_path, tmp_file) tmp_file_path = File.join("#{Gitlab.config.lfs.storage_path}/tmp/upload", tmp_file)
object = LfsObject.find_or_create_by(oid: oid, size: size) object = LfsObject.find_or_create_by(oid: oid, size: size)
file_exists = object.file.exists? || move_tmp_file_to_storage(object, tmp_file_path) file_exists = object.file.exists? || move_tmp_file_to_storage(object, tmp_file_path)
......
class Projects::UploadsController < Projects::ApplicationController class Projects::UploadsController < Projects::ApplicationController
include UploadsActions include UploadsActions
# These will kick you out if you don't have access.
skip_before_action :project, :repository, skip_before_action :project, :repository,
if: -> { action_name == 'show' && image_or_video? } if: -> { action_name == 'show' && image_or_video? }
...@@ -9,20 +8,14 @@ class Projects::UploadsController < Projects::ApplicationController ...@@ -9,20 +8,14 @@ class Projects::UploadsController < Projects::ApplicationController
private private
def upload_model_class def show_model
Project strong_memoize(:show_model) do
end namespace = params[:namespace_id]
id = params[:project_id]
def uploader_class Project.find_by_full_path("#{namespace}/#{id}")
FileUploader end
end end
def find_model alias_method :model, :project
return @project if @project
namespace = params[:namespace_id]
id = params[:project_id]
Project.find_by_full_path("#{namespace}/#{id}")
end
end end
class UploadsController < ApplicationController class UploadsController < ApplicationController
include UploadsActions include UploadsActions
UnknownUploadModelError = Class.new(StandardError)
MODEL_CLASSES = {
"user" => User,
"project" => Project,
"note" => Note,
"group" => Group,
"appearance" => Appearance,
"personal_snippet" => PersonalSnippet,
nil => PersonalSnippet
}.freeze
rescue_from UnknownUploadModelError, with: :render_404
skip_before_action :authenticate_user! skip_before_action :authenticate_user!
before_action :upload_mount_satisfied?
before_action :find_model before_action :find_model
before_action :authorize_access!, only: [:show] before_action :authorize_access!, only: [:show]
before_action :authorize_create_access!, only: [:create] before_action :authorize_create_access!, only: [:create]
def uploader_class private
PersonalFileUploader
end
def find_model def find_model
return nil unless params[:id] return nil unless params[:id]
upload_model_class.find(params[:id]) return render_404 unless upload_model && upload_mount
@model = upload_model.find(params[:id])
end end
def authorize_access! def authorize_access!
...@@ -68,17 +53,55 @@ class UploadsController < ApplicationController ...@@ -68,17 +53,55 @@ class UploadsController < ApplicationController
end end
end end
def upload_model_class def upload_model
MODEL_CLASSES[params[:model]] || raise(UnknownUploadModelError) upload_models = {
"user" => User,
"project" => Project,
"note" => Note,
"group" => Group,
"appearance" => Appearance,
"personal_snippet" => PersonalSnippet
}
upload_models[params[:model]]
end
def upload_mount
return true unless params[:mounted_as]
upload_mounts = %w(avatar attachment file logo header_logo)
if upload_mounts.include?(params[:mounted_as])
params[:mounted_as]
end
end end
def upload_model_class_has_mounts? def uploader
upload_model_class < CarrierWave::Mount::Extension return @uploader if defined?(@uploader)
case model
when nil
@uploader = PersonalFileUploader.new(nil, params[:secret])
@uploader.retrieve_from_store!(params[:filename])
when PersonalSnippet
@uploader = PersonalFileUploader.new(model, params[:secret])
@uploader.retrieve_from_store!(params[:filename])
else
@uploader = @model.public_send(upload_mount) # rubocop:disable GitlabSecurity/PublicSend
redirect_to @uploader.url unless @uploader.file_storage?
end
@uploader
end end
def upload_mount_satisfied? def uploader_class
return true unless upload_model_class_has_mounts? PersonalFileUploader
end
upload_model_class.uploader_options.has_key?(upload_mount) def model
@model ||= find_model
end end
end end
...@@ -11,7 +11,6 @@ class Appearance < ActiveRecord::Base ...@@ -11,7 +11,6 @@ class Appearance < ActiveRecord::Base
mount_uploader :logo, AttachmentUploader mount_uploader :logo, AttachmentUploader
mount_uploader :header_logo, AttachmentUploader mount_uploader :header_logo, AttachmentUploader
has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
CACHE_KEY = 'current_appearance'.freeze CACHE_KEY = 'current_appearance'.freeze
......
module Avatarable module Avatarable
extend ActiveSupport::Concern extend ActiveSupport::Concern
included do
prepend ShadowMethods
validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? }
validates :avatar, file_size: { maximum: 200.kilobytes.to_i }
mount_uploader :avatar, AvatarUploader
end
module ShadowMethods
def avatar_url(**args)
# We use avatar_path instead of overriding avatar_url because of carrierwave.
# See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11001/diffs#note_28659864
avatar_path(only_path: args.fetch(:only_path, true)) || super
end
end
def avatar_type
unless self.avatar.image?
self.errors.add :avatar, "only images allowed"
end
end
def avatar_path(only_path: true) def avatar_path(only_path: true)
return unless self[:avatar].present? return unless self[:avatar].present?
......
...@@ -29,14 +29,18 @@ class Group < Namespace ...@@ -29,14 +29,18 @@ class Group < Namespace
has_many :variables, class_name: 'Ci::GroupVariable' has_many :variables, class_name: 'Ci::GroupVariable'
has_many :custom_attributes, class_name: 'GroupCustomAttribute' has_many :custom_attributes, class_name: 'GroupCustomAttribute'
has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? }
validate :visibility_level_allowed_by_projects validate :visibility_level_allowed_by_projects
validate :visibility_level_allowed_by_sub_groups validate :visibility_level_allowed_by_sub_groups
validate :visibility_level_allowed_by_parent validate :visibility_level_allowed_by_parent
validates :avatar, file_size: { maximum: 200.kilobytes.to_i }
validates :two_factor_grace_period, presence: true, numericality: { greater_than_or_equal_to: 0 } validates :two_factor_grace_period, presence: true, numericality: { greater_than_or_equal_to: 0 }
mount_uploader :avatar, AvatarUploader
has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
after_create :post_create_hook after_create :post_create_hook
after_destroy :post_destroy_hook after_destroy :post_destroy_hook
after_save :update_two_factor_requirement after_save :update_two_factor_requirement
...@@ -112,6 +116,12 @@ class Group < Namespace ...@@ -112,6 +116,12 @@ class Group < Namespace
visibility_level_allowed_by_sub_groups?(level) visibility_level_allowed_by_sub_groups?(level)
end end
def avatar_url(**args)
# We use avatar_path instead of overriding avatar_url because of carrierwave.
# See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11001/diffs#note_28659864
avatar_path(args)
end
def lfs_enabled? def lfs_enabled?
return false unless Gitlab.config.lfs.enabled return false unless Gitlab.config.lfs.enabled
return Gitlab.config.lfs.enabled if self[:lfs_enabled].nil? return Gitlab.config.lfs.enabled if self[:lfs_enabled].nil?
......
...@@ -88,7 +88,6 @@ class Note < ActiveRecord::Base ...@@ -88,7 +88,6 @@ class Note < ActiveRecord::Base
end end
end end
# @deprecated attachments are handler by the MarkdownUploader
mount_uploader :attachment, AttachmentUploader mount_uploader :attachment, AttachmentUploader
# Scopes # Scopes
......
...@@ -256,6 +256,9 @@ class Project < ActiveRecord::Base ...@@ -256,6 +256,9 @@ class Project < ActiveRecord::Base
validates :star_count, numericality: { greater_than_or_equal_to: 0 } validates :star_count, numericality: { greater_than_or_equal_to: 0 }
validate :check_limit, on: :create validate :check_limit, on: :create
validate :check_repository_path_availability, on: :update, if: ->(project) { project.renamed? } validate :check_repository_path_availability, on: :update, if: ->(project) { project.renamed? }
validate :avatar_type,
if: ->(project) { project.avatar.present? && project.avatar_changed? }
validates :avatar, file_size: { maximum: 200.kilobytes.to_i }
validate :visibility_level_allowed_by_group validate :visibility_level_allowed_by_group
validate :visibility_level_allowed_as_fork validate :visibility_level_allowed_as_fork
validate :check_wiki_path_conflict validate :check_wiki_path_conflict
...@@ -263,6 +266,7 @@ class Project < ActiveRecord::Base ...@@ -263,6 +266,7 @@ class Project < ActiveRecord::Base
presence: true, presence: true,
inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } } inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } }
mount_uploader :avatar, AvatarUploader
has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
# Scopes # Scopes
...@@ -285,6 +289,7 @@ class Project < ActiveRecord::Base ...@@ -285,6 +289,7 @@ class Project < ActiveRecord::Base
scope :non_archived, -> { where(archived: false) } scope :non_archived, -> { where(archived: false) }
scope :for_milestones, ->(ids) { joins(:milestones).where('milestones.id' => ids).distinct } scope :for_milestones, ->(ids) { joins(:milestones).where('milestones.id' => ids).distinct }
scope :with_push, -> { joins(:events).where('events.action = ?', Event::PUSHED) } scope :with_push, -> { joins(:events).where('events.action = ?', Event::PUSHED) }
scope :with_project_feature, -> { joins('LEFT JOIN project_features ON projects.id = project_features.project_id') } scope :with_project_feature, -> { joins('LEFT JOIN project_features ON projects.id = project_features.project_id') }
scope :with_statistics, -> { includes(:statistics) } scope :with_statistics, -> { includes(:statistics) }
scope :with_shared_runners, -> { where(shared_runners_enabled: true) } scope :with_shared_runners, -> { where(shared_runners_enabled: true) }
...@@ -918,12 +923,20 @@ class Project < ActiveRecord::Base ...@@ -918,12 +923,20 @@ class Project < ActiveRecord::Base
issues_tracker.to_param == 'jira' issues_tracker.to_param == 'jira'
end end
def avatar_type
unless self.avatar.image?
self.errors.add :avatar, 'only images allowed'
end
end
def avatar_in_git def avatar_in_git
repository.avatar repository.avatar
end end
def avatar_url(**args) def avatar_url(**args)
Gitlab::Routing.url_helpers.project_avatar_url(self) if avatar_in_git # We use avatar_path instead of overriding avatar_url because of carrierwave.
# See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11001/diffs#note_28659864
avatar_path(args) || (Gitlab::Routing.url_helpers.project_avatar_url(self) if avatar_in_git)
end end
# For compatibility with old code # For compatibility with old code
......
...@@ -9,11 +9,22 @@ class Upload < ActiveRecord::Base ...@@ -9,11 +9,22 @@ class Upload < ActiveRecord::Base
validates :model, presence: true validates :model, presence: true
validates :uploader, presence: true validates :uploader, presence: true
before_save :calculate_checksum!, if: :foreground_checksummable? before_save :calculate_checksum, if: :foreground_checksum?
after_commit :schedule_checksum, if: :checksummable? after_commit :schedule_checksum, unless: :foreground_checksum?
def self.hexdigest(path) def self.remove_path(path)
Digest::SHA256.file(path).hexdigest where(path: path).destroy_all
end
def self.record(uploader)
remove_path(uploader.relative_path)
create(
size: uploader.file.size,
path: uploader.relative_path,
model: uploader.model,
uploader: uploader.class.to_s
)
end end
def absolute_path def absolute_path
...@@ -22,18 +33,10 @@ class Upload < ActiveRecord::Base ...@@ -22,18 +33,10 @@ class Upload < ActiveRecord::Base
uploader_class.absolute_path(self) uploader_class.absolute_path(self)
end end
def calculate_checksum! def calculate_checksum
self.checksum = nil return unless exist?
return unless checksummable?
self.checksum = self.class.hexdigest(absolute_path) self.checksum = Digest::SHA256.file(absolute_path).hexdigest
end
def build_uploader
uploader_class.new(model).tap do |uploader|
uploader.upload = self
uploader.retrieve_from_store!(identifier)
end
end end
def exist? def exist?
...@@ -42,16 +45,8 @@ class Upload < ActiveRecord::Base ...@@ -42,16 +45,8 @@ class Upload < ActiveRecord::Base
private private
def checksummable? def foreground_checksum?
checksum.nil? && local? && exist? size <= CHECKSUM_THRESHOLD
end
def local?
true
end
def foreground_checksummable?
checksummable? && size <= CHECKSUM_THRESHOLD
end end
def schedule_checksum def schedule_checksum
...@@ -62,10 +57,6 @@ class Upload < ActiveRecord::Base ...@@ -62,10 +57,6 @@ class Upload < ActiveRecord::Base
!path.start_with?('/') !path.start_with?('/')
end end
def identifier
File.basename(path)
end
def uploader_class def uploader_class
Object.const_get(uploader) Object.const_get(uploader)
end end
......
...@@ -137,7 +137,6 @@ class User < ActiveRecord::Base ...@@ -137,7 +137,6 @@ class User < ActiveRecord::Base
has_many :assigned_merge_requests, dependent: :nullify, foreign_key: :assignee_id, class_name: "MergeRequest" # rubocop:disable Cop/ActiveRecordDependent has_many :assigned_merge_requests, dependent: :nullify, foreign_key: :assignee_id, class_name: "MergeRequest" # rubocop:disable Cop/ActiveRecordDependent
has_many :custom_attributes, class_name: 'UserCustomAttribute' has_many :custom_attributes, class_name: 'UserCustomAttribute'
has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
# #
# Validations # Validations
...@@ -160,10 +159,12 @@ class User < ActiveRecord::Base ...@@ -160,10 +159,12 @@ class User < ActiveRecord::Base
validate :namespace_uniq, if: :username_changed? validate :namespace_uniq, if: :username_changed?
validate :namespace_move_dir_allowed, if: :username_changed? validate :namespace_move_dir_allowed, if: :username_changed?
validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? }
validate :unique_email, if: :email_changed? validate :unique_email, if: :email_changed?
validate :owns_notification_email, if: :notification_email_changed? validate :owns_notification_email, if: :notification_email_changed?
validate :owns_public_email, if: :public_email_changed? validate :owns_public_email, if: :public_email_changed?
validate :signup_domain_valid?, on: :create, if: ->(user) { !user.created_by_id } validate :signup_domain_valid?, on: :create, if: ->(user) { !user.created_by_id }
validates :avatar, file_size: { maximum: 200.kilobytes.to_i }
before_validation :sanitize_attrs before_validation :sanitize_attrs
before_validation :set_notification_email, if: :email_changed? before_validation :set_notification_email, if: :email_changed?
...@@ -224,6 +225,9 @@ class User < ActiveRecord::Base ...@@ -224,6 +225,9 @@ class User < ActiveRecord::Base
end end
end end
mount_uploader :avatar, AvatarUploader
has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
# Scopes # Scopes
scope :admins, -> { where(admin: true) } scope :admins, -> { where(admin: true) }
scope :blocked, -> { with_states(:blocked, :ldap_blocked) } scope :blocked, -> { with_states(:blocked, :ldap_blocked) }
...@@ -523,6 +527,12 @@ class User < ActiveRecord::Base ...@@ -523,6 +527,12 @@ class User < ActiveRecord::Base
end end
end end
def avatar_type
unless avatar.image?
errors.add :avatar, "only images allowed"
end
end
def unique_email def unique_email
if !emails.exists?(email: email) && Email.exists?(email: email) if !emails.exists?(email: email) && Email.exists?(email: email)
errors.add(:email, 'has already been taken') errors.add(:email, 'has already been taken')
...@@ -850,7 +860,9 @@ class User < ActiveRecord::Base ...@@ -850,7 +860,9 @@ class User < ActiveRecord::Base
end end
def avatar_url(size: nil, scale: 2, **args) def avatar_url(size: nil, scale: 2, **args)
GravatarService.new.execute(email, size, scale, username: username) # We use avatar_path instead of overriding avatar_url because of carrierwave.
# See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11001/diffs#note_28659864
avatar_path(args) || GravatarService.new.execute(email, size, scale, username: username)
end end
def primary_email_verified? def primary_email_verified?
......
...@@ -14,9 +14,9 @@ module Projects ...@@ -14,9 +14,9 @@ module Projects
@old_path = project.full_path @old_path = project.full_path
@new_path = project.disk_path @new_path = project.disk_path
origin = FileUploader.absolute_base_dir(project) origin = FileUploader.dynamic_path_segment(project)
project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:attachments] project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:attachments]
target = FileUploader.absolute_base_dir(project) target = FileUploader.dynamic_path_segment(project)
result = move_folder!(origin, target) result = move_folder!(origin, target)
project.save! project.save!
......
class AttachmentUploader < GitlabUploader class AttachmentUploader < GitlabUploader
include RecordsUploads
include UploaderHelper include UploaderHelper
include RecordsUploads::Concern
storage :file storage :file
private def store_dir
"#{base_dir}/#{model.class.to_s.underscore}/#{mounted_as}/#{model.id}"
def dynamic_segment
File.join(model.class.to_s.underscore, mounted_as.to_s, model.id.to_s)
end end
end end
class AvatarUploader < GitlabUploader class AvatarUploader < GitlabUploader
include RecordsUploads
include UploaderHelper include UploaderHelper
include RecordsUploads::Concern
storage :file storage :file
def exists? def store_dir
model.avatar.file && model.avatar.file.present? "#{base_dir}/#{model.class.to_s.underscore}/#{mounted_as}/#{model.id}"
end end
def move_to_cache def exists?
false model.avatar.file && model.avatar.file.present?
end end
# We set move_to_store and move_to_cache to 'false' to prevent stealing
# the avatar file from a project when forking it.
# https://gitlab.com/gitlab-org/gitlab-ce/issues/26158
def move_to_store def move_to_store
false false
end end
private def move_to_cache
false
def dynamic_segment
File.join(model.class.to_s.underscore, mounted_as.to_s, model.id.to_s)
end end
end end
...@@ -21,8 +21,7 @@ class FileMover ...@@ -21,8 +21,7 @@ class FileMover
end end
def update_markdown def update_markdown
updated_text = model.read_attribute(update_field) updated_text = model.read_attribute(update_field).gsub(temp_file_uploader.to_markdown, uploader.to_markdown)
.gsub(temp_file_uploader.markdown_link, uploader.markdown_link)
model.update_attribute(update_field, updated_text) model.update_attribute(update_field, updated_text)
true true
......
# This class breaks the actual CarrierWave concept.
# Every uploader should use a base_dir that is model agnostic so we can build
# back URLs from base_dir-relative paths saved in the `Upload` model.
#
# As the `.base_dir` is model dependent and **not** saved in the upload model (see #upload_path)
# there is no way to build back the correct file path without the model, which defies
# CarrierWave way of storing files.
#
class FileUploader < GitlabUploader class FileUploader < GitlabUploader
include RecordsUploads
include UploaderHelper include UploaderHelper
include RecordsUploads::Concern
MARKDOWN_PATTERN = %r{\!?\[.*?\]\(/uploads/(?<secret>[0-9a-f]{32})/(?<file>.*?)\)} MARKDOWN_PATTERN = %r{\!?\[.*?\]\(/uploads/(?<secret>[0-9a-f]{32})/(?<file>.*?)\)}
DYNAMIC_PATH_PATTERN = %r{(?<secret>\h{32})/(?<identifier>.*)}
storage :file storage :file
def self.root def self.absolute_path(upload_record)
File.join(options.storage_path, 'uploads')
end
def self.absolute_path(upload)
File.join( File.join(
absolute_base_dir(upload.model), self.dynamic_path_segment(upload_record.model),
upload.path # already contain the dynamic_segment, see #upload_path upload_record.path
) )
end end
def self.base_dir(model) # Not using `GitlabUploader.base_dir` because all project namespaces are in
model_path_segment(model) # the `public/uploads` dir.
end #
def self.base_dir
# used in migrations and import/exports root_dir
def self.absolute_base_dir(model)
File.join(root, base_dir(model))
end end
# Returns the part of `store_dir` that can change based on the model's current # Returns the part of `store_dir` that can change based on the model's current
...@@ -44,96 +29,63 @@ class FileUploader < GitlabUploader ...@@ -44,96 +29,63 @@ class FileUploader < GitlabUploader
# model - Object that responds to `full_path` and `disk_path` # model - Object that responds to `full_path` and `disk_path`
# #
# Returns a String without a trailing slash # Returns a String without a trailing slash
def self.model_path_segment(model) def self.dynamic_path_segment(model)
if model.hashed_storage?(:attachments) if model.hashed_storage?(:attachments)
model.disk_path dynamic_path_builder(model.disk_path)
else else
model.full_path dynamic_path_builder(model.full_path)
end end
end end
def self.upload_path(secret, identifier) # Auxiliary method to build dynamic path segment when not using a project model
File.join(secret, identifier) #
end # Prefer to use the `.dynamic_path_segment` as it includes Hashed Storage specific logic
def self.dynamic_path_builder(path)
def self.generate_secret File.join(CarrierWave.root, base_dir, path)
SecureRandom.hex
end end
attr_accessor :model attr_accessor :model
attr_reader :secret
def initialize(model, secret = nil) def initialize(model, secret = nil)
@model = model @model = model
@secret = secret @secret = secret || generate_secret
end
def base_dir
self.class.base_dir(@model)
end end
# we don't need to know the actual path, an uploader instance should be def store_dir
# able to yield the file content on demand, so we should build the digest File.join(dynamic_path_segment, @secret)
def absolute_path
self.class.absolute_path(@upload)
end end
def upload_path def relative_path
self.class.upload_path(dynamic_segment, identifier) self.file.path.sub("#{dynamic_path_segment}/", '')
end end
def model_path_segment def to_markdown
self.class.model_path_segment(@model) to_h[:markdown]
end end
def store_dir def to_h
File.join(base_dir, dynamic_segment) filename = image_or_video? ? self.file.basename : self.file.filename
end escaped_filename = filename.gsub("]", "\\]")
def markdown_link markdown = "[#{escaped_filename}](#{secure_url})"
markdown = "[#{markdown_name}](#{secure_url})"
markdown.prepend("!") if image_or_video? || dangerous? markdown.prepend("!") if image_or_video? || dangerous?
markdown
end
def to_h
{ {
alt: markdown_name, alt: filename,
url: secure_url, url: secure_url,
markdown: markdown_link markdown: markdown
} }
end end
def filename
self.file.filename
end
# the upload does not hold the secret, but holds the path
# which contains the secret: extract it
def upload=(value)
if matches = DYNAMIC_PATH_PATTERN.match(value.path)
@secret = matches[:secret]
@identifier = matches[:identifier]
end
super
end
def secret
@secret ||= self.class.generate_secret
end
private private
def markdown_name def dynamic_path_segment
(image_or_video? ? File.basename(filename, File.extname(filename)) : filename).gsub("]", "\\]") self.class.dynamic_path_segment(model)
end end
def identifier def generate_secret
@identifier ||= filename SecureRandom.hex
end
def dynamic_segment
secret
end end
def secure_url def secure_url
......
class GitlabUploader < CarrierWave::Uploader::Base class GitlabUploader < CarrierWave::Uploader::Base
class_attribute :options def self.absolute_path(upload_record)
File.join(CarrierWave.root, upload_record.path)
class << self end
# DSL setter
def storage_options(options)
self.options = options
end
def root
options.storage_path
end
# represent the directory namespacing at the class level def self.root_dir
def base_dir 'uploads'
options.fetch('base_dir', '') end
end
def file_storage? # When object storage is used, keep the `root_dir` as `base_dir`.
storage == CarrierWave::Storage::File # The files aren't really in folders there, they just have a name.
end # The files that contain user input in their name, also contain a hash, so
# the names are still unique
#
# This method is overridden in the `FileUploader`
def self.base_dir
return root_dir unless file_storage?
def absolute_path(upload_record) File.join(root_dir, '-', 'system')
File.join(root, upload_record.path)
end
end end
storage_options Gitlab.config.uploads def self.file_storage?
self.storage == CarrierWave::Storage::File
end
delegate :base_dir, :file_storage?, to: :class delegate :base_dir, :file_storage?, to: :class
...@@ -35,28 +31,34 @@ class GitlabUploader < CarrierWave::Uploader::Base ...@@ -35,28 +31,34 @@ class GitlabUploader < CarrierWave::Uploader::Base
# Reduce disk IO # Reduce disk IO
def move_to_cache def move_to_cache
file_storage? true
end end
# Reduce disk IO # Reduce disk IO
def move_to_store def move_to_store
file_storage? true
end end
def exists? # Designed to be overridden by child uploaders that have a dynamic path
file.present? # segment -- that is, a path that changes based on mutable attributes of its
end # associated model
#
def store_dir # For example, `FileUploader` builds the storage path based on the associated
File.join(base_dir, dynamic_segment) # project model's `path_with_namespace` value, which can change when the
# project or its containing namespace is moved or renamed.
def relative_path
self.file.path.sub("#{root}/", '')
end end
def cache_dir def exists?
File.join(root, base_dir, 'tmp/cache') file.present?
end end
# Override this if you don't want to save files by default to the Rails.root directory
def work_dir def work_dir
File.join(root, base_dir, 'tmp/work') # Default path set by CarrierWave:
# https://github.com/carrierwaveuploader/carrierwave/blob/v1.0.0/lib/carrierwave/uploader/cache.rb#L182
CarrierWave.tmp_path
end end
def filename def filename
...@@ -65,13 +67,6 @@ class GitlabUploader < CarrierWave::Uploader::Base ...@@ -65,13 +67,6 @@ class GitlabUploader < CarrierWave::Uploader::Base
private private
# Designed to be overridden by child uploaders that have a dynamic path
# segment -- that is, a path that changes based on mutable attributes of its
# associated model
def dynamic_segment
raise(NotImplementedError)
end
# To prevent files from moving across filesystems, override the default # To prevent files from moving across filesystems, override the default
# implementation: # implementation:
# http://github.com/carrierwaveuploader/carrierwave/blob/v1.0.0/lib/carrierwave/uploader/cache.rb#L181-L183 # http://github.com/carrierwaveuploader/carrierwave/blob/v1.0.0/lib/carrierwave/uploader/cache.rb#L181-L183
...@@ -79,6 +74,6 @@ class GitlabUploader < CarrierWave::Uploader::Base ...@@ -79,6 +74,6 @@ class GitlabUploader < CarrierWave::Uploader::Base
# To be safe, keep this directory outside of the the cache directory # To be safe, keep this directory outside of the the cache directory
# because calling CarrierWave.clean_cache_files! will remove any files in # because calling CarrierWave.clean_cache_files! will remove any files in
# the cache directory. # the cache directory.
File.join(work_dir, cache_id, version_name.to_s, for_file) File.join(work_dir, @cache_id, version_name.to_s, for_file)
end end
end end
class JobArtifactUploader < GitlabUploader class JobArtifactUploader < GitlabUploader
extend Workhorse::UploadPath storage :file
storage_options Gitlab.config.artifacts def self.local_store_path
Gitlab.config.artifacts.path
end
def self.artifacts_upload_path
File.join(self.local_store_path, 'tmp/uploads/')
end
def size def size
return super if model.size.nil? return super if model.size.nil?
...@@ -10,12 +16,24 @@ class JobArtifactUploader < GitlabUploader ...@@ -10,12 +16,24 @@ class JobArtifactUploader < GitlabUploader
end end
def store_dir def store_dir
dynamic_segment default_local_path
end
def cache_dir
File.join(self.class.local_store_path, 'tmp/cache')
end
def work_dir
File.join(self.class.local_store_path, 'tmp/work')
end end
private private
def dynamic_segment def default_local_path
File.join(self.class.local_store_path, default_path)
end
def default_path
creation_date = model.created_at.utc.strftime('%Y_%m_%d') creation_date = model.created_at.utc.strftime('%Y_%m_%d')
File.join(disk_hash[0..1], disk_hash[2..3], disk_hash, File.join(disk_hash[0..1], disk_hash[2..3], disk_hash,
......
class LegacyArtifactUploader < GitlabUploader class LegacyArtifactUploader < GitlabUploader
extend Workhorse::UploadPath storage :file
storage_options Gitlab.config.artifacts def self.local_store_path
Gitlab.config.artifacts.path
end
def self.artifacts_upload_path
File.join(self.local_store_path, 'tmp/uploads/')
end
def store_dir def store_dir
dynamic_segment default_local_path
end
def cache_dir
File.join(self.class.local_store_path, 'tmp/cache')
end
def work_dir
File.join(self.class.local_store_path, 'tmp/work')
end end
private private
def dynamic_segment def default_local_path
File.join(self.class.local_store_path, default_path)
end
def default_path
File.join(model.created_at.utc.strftime('%Y_%m'), model.project_id.to_s, model.id.to_s) File.join(model.created_at.utc.strftime('%Y_%m'), model.project_id.to_s, model.id.to_s)
end end
end end
class LfsObjectUploader < GitlabUploader class LfsObjectUploader < GitlabUploader
extend Workhorse::UploadPath storage :file
# LfsObject are in `tmp/upload` instead of `tmp/uploads` def store_dir
def self.workhorse_upload_path "#{Gitlab.config.lfs.storage_path}/#{model.oid[0, 2]}/#{model.oid[2, 2]}"
File.join(root, 'tmp/upload')
end end
storage_options Gitlab.config.lfs def cache_dir
"#{Gitlab.config.lfs.storage_path}/tmp/cache"
end
def filename def filename
model.oid[4..-1] model.oid[4..-1]
end end
def store_dir def work_dir
dynamic_segment File.join(Gitlab.config.lfs.storage_path, 'tmp', 'work')
end
private
def dynamic_segment
File.join(model.oid[0, 2], model.oid[2, 2])
end end
end end
class NamespaceFileUploader < FileUploader class NamespaceFileUploader < FileUploader
# Re-Override def self.base_dir
def self.root File.join(root_dir, '-', 'system', 'namespace')
options.storage_path
end end
def self.base_dir(model) def self.dynamic_path_segment(model)
File.join(options.base_dir, 'namespace', model_path_segment(model)) dynamic_path_builder(model.id.to_s)
end end
def self.model_path_segment(model) private
File.join(model.id.to_s)
end
# Re-Override def secure_url
def store_dir File.join('/uploads', @secret, file.filename)
File.join(base_dir, dynamic_segment)
end end
end end
class PersonalFileUploader < FileUploader class PersonalFileUploader < FileUploader
# Re-Override def self.dynamic_path_segment(model)
def self.root File.join(CarrierWave.root, model_path(model))
options.storage_path
end end
def self.base_dir(model) def self.base_dir
File.join(options.base_dir, model_path_segment(model)) File.join(root_dir, '-', 'system')
end
def self.model_path_segment(model)
return 'temp/' unless model
File.join(model.class.to_s.underscore, model.id.to_s)
end
# Revert-Override
def store_dir
File.join(base_dir, dynamic_segment)
end end
private private
def secure_url def secure_url
File.join('/', base_dir, secret, file.filename) File.join(self.class.model_path(model), secret, file.filename)
end
def self.model_path(model)
if model
File.join("/#{base_dir}", model.class.to_s.underscore, model.id.to_s)
else
File.join("/#{base_dir}", 'temp')
end
end end
end end
module RecordsUploads module RecordsUploads
module Concern extend ActiveSupport::Concern
extend ActiveSupport::Concern
attr_accessor :upload included do
after :store, :record_upload
included do before :remove, :destroy_upload
after :store, :record_upload end
before :remove, :destroy_upload
end
# After storing an attachment, create a corresponding Upload record
#
# NOTE: We're ignoring the argument passed to this callback because we want
# the `SanitizedFile` object from `CarrierWave::Uploader::Base#file`, not the
# `Tempfile` object the callback gets.
#
# Called `after :store`
def record_upload(_tempfile = nil)
return unless model
return unless file && file.exists?
Upload.transaction do
uploads.where(path: upload_path).delete_all
upload.destroy! if upload
self.upload = build_upload_from_uploader(self)
upload.save!
end
end
def upload_path
File.join(store_dir, filename.to_s)
end
private
def uploads # After storing an attachment, create a corresponding Upload record
Upload.order(id: :desc).where(uploader: self.class.to_s) #
end # NOTE: We're ignoring the argument passed to this callback because we want
# the `SanitizedFile` object from `CarrierWave::Uploader::Base#file`, not the
# `Tempfile` object the callback gets.
#
# Called `after :store`
def record_upload(_tempfile = nil)
return unless model
return unless file_storage?
return unless file.exists?
Upload.record(self)
end
def build_upload_from_uploader(uploader) private
Upload.new(
size: uploader.file.size,
path: uploader.upload_path,
model: uploader.model,
uploader: uploader.class.to_s
)
end
# Before removing an attachment, destroy any Upload records at the same path # Before removing an attachment, destroy any Upload records at the same path
# #
# Called `before :remove` # Called `before :remove`
def destroy_upload(*args) def destroy_upload(*args)
return unless file && file.exists? return unless file_storage?
return unless file
self.upload = nil Upload.remove_path(relative_path)
uploads.where(path: upload_path).delete_all
end
end end
end end
...@@ -32,7 +32,14 @@ module UploaderHelper ...@@ -32,7 +32,14 @@ module UploaderHelper
def extension_match?(extensions) def extension_match?(extensions)
return false unless file return false unless file
extension = file.try(:extension) || File.extname(file.path).delete('.') extension =
if file.respond_to?(:extension)
file.extension
else
# Not all CarrierWave storages respond to :extension
File.extname(file.path).delete('.')
end
extensions.include?(extension.downcase) extensions.include?(extension.downcase)
end end
end end
module Workhorse
module UploadPath
def workhorse_upload_path
File.join(root, base_dir, 'tmp/uploads')
end
end
end
...@@ -3,7 +3,7 @@ class UploadChecksumWorker ...@@ -3,7 +3,7 @@ class UploadChecksumWorker
def perform(upload_id) def perform(upload_id)
upload = Upload.find(upload_id) upload = Upload.find(upload_id)
upload.calculate_checksum! upload.calculate_checksum
upload.save! upload.save!
rescue ActiveRecord::RecordNotFound rescue ActiveRecord::RecordNotFound
Rails.logger.error("UploadChecksumWorker: couldn't find upload #{upload_id}, skipping") Rails.logger.error("UploadChecksumWorker: couldn't find upload #{upload_id}, skipping")
......
...@@ -152,12 +152,6 @@ production: &base ...@@ -152,12 +152,6 @@ production: &base
# The location where LFS objects are stored (default: shared/lfs-objects). # The location where LFS objects are stored (default: shared/lfs-objects).
# storage_path: shared/lfs-objects # storage_path: shared/lfs-objects
## Uploads (attachments, avatars, etc...)
uploads:
# The location where uploads objects are stored (default: public/).
# storage_path: public/
# base_dir: uploads/-/system
## GitLab Pages ## GitLab Pages
pages: pages:
enabled: false enabled: false
...@@ -650,8 +644,6 @@ test: ...@@ -650,8 +644,6 @@ test:
enabled: false enabled: false
artifacts: artifacts:
path: tmp/tests/artifacts path: tmp/tests/artifacts
uploads:
storage_path: tmp/tests/public
gitlab: gitlab:
host: localhost host: localhost
port: 80 port: 80
......
...@@ -300,10 +300,8 @@ Settings.incoming_email['enabled'] = false if Settings.incoming_email['enabled'] ...@@ -300,10 +300,8 @@ Settings.incoming_email['enabled'] = false if Settings.incoming_email['enabled']
# #
Settings['artifacts'] ||= Settingslogic.new({}) Settings['artifacts'] ||= Settingslogic.new({})
Settings.artifacts['enabled'] = true if Settings.artifacts['enabled'].nil? Settings.artifacts['enabled'] = true if Settings.artifacts['enabled'].nil?
Settings.artifacts['storage_path'] = Settings.absolute(Settings.artifacts.values_at('path', 'storage_path').compact.first || File.join(Settings.shared['path'], "artifacts")) Settings.artifacts['path'] = Settings.absolute(Settings.artifacts['path'] || File.join(Settings.shared['path'], "artifacts"))
# Settings.artifact['path'] is deprecated, use `storage_path` instead Settings.artifacts['max_size'] ||= 100 # in megabytes
Settings.artifacts['path'] = Settings.artifacts['storage_path']
Settings.artifacts['max_size'] ||= 100 # in megabytes
# #
# Registry # Registry
...@@ -340,13 +338,6 @@ Settings['lfs'] ||= Settingslogic.new({}) ...@@ -340,13 +338,6 @@ Settings['lfs'] ||= Settingslogic.new({})
Settings.lfs['enabled'] = true if Settings.lfs['enabled'].nil? Settings.lfs['enabled'] = true if Settings.lfs['enabled'].nil?
Settings.lfs['storage_path'] = Settings.absolute(Settings.lfs['storage_path'] || File.join(Settings.shared['path'], "lfs-objects")) Settings.lfs['storage_path'] = Settings.absolute(Settings.lfs['storage_path'] || File.join(Settings.shared['path'], "lfs-objects"))
#
# Uploads
#
Settings['uploads'] ||= Settingslogic.new({})
Settings.uploads['storage_path'] = Settings.absolute(Settings.uploads['storage_path'] || 'public')
Settings.uploads['base_dir'] = Settings.uploads['base_dir'] || 'uploads/-/system'
# #
# Mattermost # Mattermost
# #
......
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class AddUploaderIndexToUploads < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
remove_concurrent_index :uploads, :path
add_concurrent_index :uploads, [:uploader, :path], using: :btree
end
def down
remove_concurrent_index :uploads, [:uploader, :path]
add_concurrent_index :uploads, :path, using: :btree
end
end
...@@ -1755,7 +1755,7 @@ ActiveRecord::Schema.define(version: 20180201145907) do ...@@ -1755,7 +1755,7 @@ ActiveRecord::Schema.define(version: 20180201145907) do
add_index "uploads", ["checksum"], name: "index_uploads_on_checksum", using: :btree add_index "uploads", ["checksum"], name: "index_uploads_on_checksum", using: :btree
add_index "uploads", ["model_id", "model_type"], name: "index_uploads_on_model_id_and_model_type", using: :btree add_index "uploads", ["model_id", "model_type"], name: "index_uploads_on_model_id_and_model_type", using: :btree
add_index "uploads", ["uploader", "path"], name: "index_uploads_on_uploader_and_path", using: :btree add_index "uploads", ["path"], name: "index_uploads_on_path", using: :btree
create_table "user_agent_details", force: :cascade do |t| create_table "user_agent_details", force: :cascade do |t|
t.string "user_agent", null: false t.string "user_agent", null: false
......
...@@ -14,8 +14,8 @@ There are many places where file uploading is used, according to contexts: ...@@ -14,8 +14,8 @@ There are many places where file uploading is used, according to contexts:
- User snippet attachments - User snippet attachments
* Project * Project
- Project avatars - Project avatars
- Issues/MR/Notes Markdown attachments - Issues/MR Markdown attachments
- Issues/MR/Notes Legacy Markdown attachments - Issues/MR Legacy Markdown attachments
- CI Build Artifacts - CI Build Artifacts
- LFS Objects - LFS Objects
...@@ -25,7 +25,7 @@ There are many places where file uploading is used, according to contexts: ...@@ -25,7 +25,7 @@ There are many places where file uploading is used, according to contexts:
GitLab started saving everything on local disk. While directory location changed from previous versions, GitLab started saving everything on local disk. While directory location changed from previous versions,
they are still not 100% standardized. You can see them below: they are still not 100% standardized. You can see them below:
| Description | In DB? | Relative path (from CarrierWave.root) | Uploader class | model_type | | Description | In DB? | Relative path | Uploader class | model_type |
| ------------------------------------- | ------ | ----------------------------------------------------------- | ---------------------- | ---------- | | ------------------------------------- | ------ | ----------------------------------------------------------- | ---------------------- | ---------- |
| Instance logo | yes | uploads/-/system/appearance/logo/:id/:filename | `AttachmentUploader` | Appearance | | Instance logo | yes | uploads/-/system/appearance/logo/:id/:filename | `AttachmentUploader` | Appearance |
| Header logo | yes | uploads/-/system/appearance/header_logo/:id/:filename | `AttachmentUploader` | Appearance | | Header logo | yes | uploads/-/system/appearance/header_logo/:id/:filename | `AttachmentUploader` | Appearance |
...@@ -33,107 +33,17 @@ they are still not 100% standardized. You can see them below: ...@@ -33,107 +33,17 @@ they are still not 100% standardized. You can see them below:
| User avatars | yes | uploads/-/system/user/avatar/:id/:filename | `AvatarUploader` | User | | User avatars | yes | uploads/-/system/user/avatar/:id/:filename | `AvatarUploader` | User |
| User snippet attachments | yes | uploads/-/system/personal_snippet/:id/:random_hex/:filename | `PersonalFileUploader` | Snippet | | User snippet attachments | yes | uploads/-/system/personal_snippet/:id/:random_hex/:filename | `PersonalFileUploader` | Snippet |
| Project avatars | yes | uploads/-/system/project/avatar/:id/:filename | `AvatarUploader` | Project | | Project avatars | yes | uploads/-/system/project/avatar/:id/:filename | `AvatarUploader` | Project |
| Issues/MR/Notes Markdown attachments | yes | uploads/:project_path_with_namespace/:random_hex/:filename | `FileUploader` | Project | | Issues/MR Markdown attachments | yes | uploads/:project_path_with_namespace/:random_hex/:filename | `FileUploader` | Project |
| Issues/MR/Notes Legacy Markdown attachments | no | uploads/-/system/note/attachment/:id/:filename | `AttachmentUploader` | Note | | Issues/MR Legacy Markdown attachments | no | uploads/-/system/note/attachment/:id/:filename | `AttachmentUploader` | Note |
| CI Artifacts (CE) | yes | shared/artifacts/:year_:month/:project_id/:id | `ArtifactUploader` | Ci::Build | | CI Artifacts (CE) | yes | shared/artifacts/:year_:month/:project_id/:id | `ArtifactUploader` | Ci::Build |
| LFS Objects (CE) | yes | shared/lfs-objects/:hex/:hex/:object_hash | `LfsObjectUploader` | LfsObject | | LFS Objects (CE) | yes | shared/lfs-objects/:hex/:hex/:object_hash | `LfsObjectUploader` | LfsObject |
CI Artifacts and LFS Objects behave differently in CE and EE. In CE they inherit the `GitlabUploader` CI Artifacts and LFS Objects behave differently in CE and EE. In CE they inherit the `GitlabUploader`
while in EE they inherit the `ObjectStorage` and store files in and S3 API compatible object store. while in EE they inherit the `ObjectStoreUploader` and store files in and S3 API compatible object store.
In the case of Issues/MR/Notes Markdown attachments, there is a different approach using the [Hashed Storage] layout, In the case of Issues/MR Markdown attachments, there is a different approach using the [Hashed Storage] layout,
instead of basing the path into a mutable variable `:project_path_with_namespace`, it's possible to use the instead of basing the path into a mutable variable `:project_path_with_namespace`, it's possible to use the
hash of the project ID instead, if project migrates to the new approach (introduced in 10.2). hash of the project ID instead, if project migrates to the new approach (introduced in 10.2).
### Path segments
Files are stored at multiple locations and use different path schemes.
All the `GitlabUploader` derived classes should comply with this path segment schema:
```
| GitlabUploader
| ----------------------- + ------------------------- + --------------------------------- + -------------------------------- |
| `<gitlab_root>/public/` | `uploads/-/system/` | `user/avatar/:id/` | `:filename` |
| ----------------------- + ------------------------- + --------------------------------- + -------------------------------- |
| `CarrierWave.root` | `GitlabUploader.base_dir` | `GitlabUploader#dynamic_segment` | `CarrierWave::Uploader#filename` |
| | `CarrierWave::Uploader#store_dir` | |
| FileUploader
| ----------------------- + ------------------------- + --------------------------------- + -------------------------------- |
| `<gitlab_root>/shared/` | `artifacts/` | `:year_:month/:id` | `:filename` |
| `<gitlab_root>/shared/` | `snippets/` | `:secret/` | `:filename` |
| ----------------------- + ------------------------- + --------------------------------- + -------------------------------- |
| `CarrierWave.root` | `GitlabUploader.base_dir` | `GitlabUploader#dynamic_segment` | `CarrierWave::Uploader#filename` |
| | `CarrierWave::Uploader#store_dir` | |
| | | `FileUploader#upload_path |
| ObjectStore::Concern (store = remote)
| ----------------------- + ------------------------- + ----------------------------------- + -------------------------------- |
| `<bucket_name>` | <ignored> | `user/avatar/:id/` | `:filename` |
| ----------------------- + ------------------------- + ----------------------------------- + -------------------------------- |
| `#fog_dir` | `GitlabUploader.base_dir` | `GitlabUploader#dynamic_segment` | `CarrierWave::Uploader#filename` |
| | | `ObjectStorage::Concern#store_dir` | |
| | | `ObjectStorage::Concern#upload_path |
```
The `RecordsUploads::Concern` concern will create an `Upload` entry for every file stored by a `GitlabUploader` persisting the dynamic parts of the path using
`GitlabUploader#dynamic_path`. You may then use the `Upload#build_uploader` method to manipulate the file.
## Object Storage
By including the `ObjectStorage::Concern` in the `GitlabUploader` derived class, you may enable the object storage for this uploader. To enable the object storage
in your uploader, you need to either 1) include `RecordsUpload::Concern` and prepend `ObjectStorage::Extension::RecordsUploads` or 2) mount the uploader and create a new field named `<mount>_store`.
The `CarrierWave::Uploader#store_dir` is overriden to
- `GitlabUploader.base_dir` + `GitlabUploader.dynamic_segment` when the store is LOCAL
- `GitlabUploader.dynamic_segment` when the store is REMOTE (the bucket name is used to namespace)
### Using `ObjectStorage::Extension::RecordsUploads`
> Note: this concern will automatically include `RecordsUploads::Concern` if not already included.
The `ObjectStorage::Concern` uploader will search for the matching `Upload` to select the correct object store. The `Upload` is mapped using `#store_dirs + identifier` for each store (LOCAL/REMOTE).
```ruby
class SongUploader < GitlabUploader
include RecordsUploads::Concern
include ObjectStorage::Concern
prepend ObjectStorage::Extension::RecordsUploads
...
end
class Thing < ActiveRecord::Base
mount :theme, SongUploader # we have a great theme song!
...
end
```
### Using a mounted uploader
The `ObjectStorage::Concern` will query the `model.<mount>_store` attribute to select the correct object store.
This column must be present in the model schema.
```ruby
class SongUploader < GitlabUploader
include ObjectStorage::Concern
...
end
class Thing < ActiveRecord::Base
attr_reader :theme_store # this is an ActiveRecord attribute
mount :theme, SongUploader # we have a great theme song!
def theme_store
super || ObjectStorage::Store::LOCAL
end
...
end
```
[CarrierWave]: https://github.com/carrierwaveuploader/carrierwave [CarrierWave]: https://github.com/carrierwaveuploader/carrierwave
[Hashed Storage]: ../administration/repository_storage_types.md [Hashed Storage]: ../administration/repository_storage_types.md
...@@ -215,9 +215,9 @@ module API ...@@ -215,9 +215,9 @@ module API
job = authenticate_job! job = authenticate_job!
forbidden!('Job is not running!') unless job.running? forbidden!('Job is not running!') unless job.running?
workhorse_upload_path = JobArtifactUploader.workhorse_upload_path artifacts_upload_path = JobArtifactUploader.artifacts_upload_path
artifacts = uploaded_file(:file, workhorse_upload_path) artifacts = uploaded_file(:file, artifacts_upload_path)
metadata = uploaded_file(:metadata, workhorse_upload_path) metadata = uploaded_file(:metadata, artifacts_upload_path)
bad_request!('Missing artifacts file!') unless artifacts bad_request!('Missing artifacts file!') unless artifacts
file_to_large! unless artifacts.size < max_artifacts_size file_to_large! unless artifacts.size < max_artifacts_size
......
...@@ -3,7 +3,7 @@ require 'backup/files' ...@@ -3,7 +3,7 @@ require 'backup/files'
module Backup module Backup
class Artifacts < Files class Artifacts < Files
def initialize def initialize
super('artifacts', JobArtifactUploader.root) super('artifacts', LegacyArtifactUploader.local_store_path)
end end
def create_files_dir def create_files_dir
......
...@@ -143,7 +143,7 @@ module Gitlab ...@@ -143,7 +143,7 @@ module Gitlab
end end
def absolute_path def absolute_path
File.join(Gitlab.config.uploads.storage_path, path) File.join(CarrierWave.root, path)
end end
end end
......
...@@ -11,12 +11,9 @@ module Gitlab ...@@ -11,12 +11,9 @@ module Gitlab
FIND_BATCH_SIZE = 500 FIND_BATCH_SIZE = 500
RELATIVE_UPLOAD_DIR = "uploads".freeze RELATIVE_UPLOAD_DIR = "uploads".freeze
ABSOLUTE_UPLOAD_DIR = File.join( ABSOLUTE_UPLOAD_DIR = "#{CarrierWave.root}/#{RELATIVE_UPLOAD_DIR}".freeze
Gitlab.config.uploads.storage_path,
RELATIVE_UPLOAD_DIR
)
FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze
START_WITH_ROOT_REGEX = %r{\A#{Gitlab.config.uploads.storage_path}/} START_WITH_CARRIERWAVE_ROOT_REGEX = %r{\A#{CarrierWave.root}/}
EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze
EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze
...@@ -84,7 +81,7 @@ module Gitlab ...@@ -84,7 +81,7 @@ module Gitlab
paths = [] paths = []
stdout.each_line("\0") do |line| stdout.each_line("\0") do |line|
paths << line.chomp("\0").sub(START_WITH_ROOT_REGEX, '') paths << line.chomp("\0").sub(START_WITH_CARRIERWAVE_ROOT_REGEX, '')
if paths.size >= batch_size if paths.size >= batch_size
yield(paths) yield(paths)
......
...@@ -27,7 +27,7 @@ module Gitlab ...@@ -27,7 +27,7 @@ module Gitlab
with_link_in_tmp_dir(file.file) do |open_tmp_file| with_link_in_tmp_dir(file.file) do |open_tmp_file|
new_uploader.store!(open_tmp_file) new_uploader.store!(open_tmp_file)
end end
new_uploader.markdown_link new_uploader.to_markdown
end end
end end
......
...@@ -17,13 +17,15 @@ module Gitlab ...@@ -17,13 +17,15 @@ module Gitlab
false false
end end
def uploads_path private
FileUploader.absolute_base_dir(@project)
end
def uploads_export_path def uploads_export_path
File.join(@shared.export_path, 'uploads') File.join(@shared.export_path, 'uploads')
end end
def uploads_path
FileUploader.dynamic_path_segment(@project)
end
end end
end end
end end
module Gitlab module Gitlab
class UploadsTransfer < ProjectTransfer class UploadsTransfer < ProjectTransfer
def root_dir def root_dir
FileUploader.root File.join(CarrierWave.root, FileUploader.base_dir)
end end
end end
end end
...@@ -55,14 +55,14 @@ module Gitlab ...@@ -55,14 +55,14 @@ module Gitlab
def lfs_upload_ok(oid, size) def lfs_upload_ok(oid, size)
{ {
StoreLFSPath: LfsObjectUploader.workhorse_upload_path, StoreLFSPath: "#{Gitlab.config.lfs.storage_path}/tmp/upload",
LfsOid: oid, LfsOid: oid,
LfsSize: size LfsSize: size
} }
end end
def artifact_upload_ok def artifact_upload_ok
{ TempPath: JobArtifactUploader.workhorse_upload_path } { TempPath: JobArtifactUploader.artifacts_upload_path }
end end
def send_git_blob(repository, blob) def send_git_blob(repository, blob)
...@@ -147,11 +147,8 @@ module Gitlab ...@@ -147,11 +147,8 @@ module Gitlab
end end
def send_artifacts_entry(build, entry) def send_artifacts_entry(build, entry)
file = build.artifacts_file
archive = file.file_storage? ? file.path : file.url
params = { params = {
'Archive' => archive, 'Archive' => build.artifacts_file.path,
'Entry' => Base64.encode64(entry.to_s) 'Entry' => Base64.encode64(entry.to_s)
} }
......
...@@ -6,7 +6,5 @@ describe Groups::UploadsController do ...@@ -6,7 +6,5 @@ describe Groups::UploadsController do
{ group_id: model } { group_id: model }
end end
it_behaves_like 'handle uploads' do it_behaves_like 'handle uploads'
let(:uploader_class) { NamespaceFileUploader }
end
end end
...@@ -145,7 +145,8 @@ describe Projects::ArtifactsController do ...@@ -145,7 +145,8 @@ describe Projects::ArtifactsController do
context 'when using local file storage' do context 'when using local file storage' do
it_behaves_like 'a valid file' do it_behaves_like 'a valid file' do
let(:job) { create(:ci_build, :success, :artifacts, pipeline: pipeline) } let(:job) { create(:ci_build, :success, :artifacts, pipeline: pipeline) }
let(:archive_path) { JobArtifactUploader.root } let(:store) { ObjectStoreUploader::LOCAL_STORE }
let(:archive_path) { JobArtifactUploader.local_store_path }
end end
end end
end end
......
...@@ -53,7 +53,7 @@ describe Projects::RawController do ...@@ -53,7 +53,7 @@ describe Projects::RawController do
end end
it 'serves the file' do it 'serves the file' do
expect(controller).to receive(:send_file).with("#{LfsObjectUploader.root}/91/ef/f75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897", filename: 'lfs_object.iso', disposition: 'attachment') expect(controller).to receive(:send_file).with("#{Gitlab.config.shared.path}/lfs-objects/91/ef/f75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897", filename: 'lfs_object.iso', disposition: 'attachment')
get(:show, get(:show,
namespace_id: public_project.namespace.to_param, namespace_id: public_project.namespace.to_param,
project_id: public_project, project_id: public_project,
......
...@@ -180,7 +180,6 @@ describe UploadsController do ...@@ -180,7 +180,6 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'user', mounted_as: 'avatar', id: user.id, filename: 'image.png' get :show, model: 'user', mounted_as: 'avatar', id: user.id, filename: 'image.png'
response response
end end
end end
...@@ -197,7 +196,6 @@ describe UploadsController do ...@@ -197,7 +196,6 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'user', mounted_as: 'avatar', id: user.id, filename: 'image.png' get :show, model: 'user', mounted_as: 'avatar', id: user.id, filename: 'image.png'
response response
end end
end end
...@@ -222,7 +220,6 @@ describe UploadsController do ...@@ -222,7 +220,6 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'project', mounted_as: 'avatar', id: project.id, filename: 'image.png' get :show, model: 'project', mounted_as: 'avatar', id: project.id, filename: 'image.png'
response response
end end
end end
...@@ -242,7 +239,6 @@ describe UploadsController do ...@@ -242,7 +239,6 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'project', mounted_as: 'avatar', id: project.id, filename: 'image.png' get :show, model: 'project', mounted_as: 'avatar', id: project.id, filename: 'image.png'
response response
end end
end end
...@@ -295,7 +291,6 @@ describe UploadsController do ...@@ -295,7 +291,6 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'project', mounted_as: 'avatar', id: project.id, filename: 'image.png' get :show, model: 'project', mounted_as: 'avatar', id: project.id, filename: 'image.png'
response response
end end
end end
...@@ -327,7 +322,6 @@ describe UploadsController do ...@@ -327,7 +322,6 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'group', mounted_as: 'avatar', id: group.id, filename: 'image.png' get :show, model: 'group', mounted_as: 'avatar', id: group.id, filename: 'image.png'
response response
end end
end end
...@@ -347,7 +341,6 @@ describe UploadsController do ...@@ -347,7 +341,6 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'group', mounted_as: 'avatar', id: group.id, filename: 'image.png' get :show, model: 'group', mounted_as: 'avatar', id: group.id, filename: 'image.png'
response response
end end
end end
...@@ -391,7 +384,6 @@ describe UploadsController do ...@@ -391,7 +384,6 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'group', mounted_as: 'avatar', id: group.id, filename: 'image.png' get :show, model: 'group', mounted_as: 'avatar', id: group.id, filename: 'image.png'
response response
end end
end end
...@@ -428,7 +420,6 @@ describe UploadsController do ...@@ -428,7 +420,6 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'note', mounted_as: 'attachment', id: note.id, filename: 'image.png' get :show, model: 'note', mounted_as: 'attachment', id: note.id, filename: 'image.png'
response response
end end
end end
...@@ -448,7 +439,6 @@ describe UploadsController do ...@@ -448,7 +439,6 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'note', mounted_as: 'attachment', id: note.id, filename: 'image.png' get :show, model: 'note', mounted_as: 'attachment', id: note.id, filename: 'image.png'
response response
end end
end end
...@@ -501,7 +491,6 @@ describe UploadsController do ...@@ -501,7 +491,6 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'note', mounted_as: 'attachment', id: note.id, filename: 'image.png' get :show, model: 'note', mounted_as: 'attachment', id: note.id, filename: 'image.png'
response response
end end
end end
...@@ -533,7 +522,6 @@ describe UploadsController do ...@@ -533,7 +522,6 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'appearance', mounted_as: 'header_logo', id: appearance.id, filename: 'dk.png' get :show, model: 'appearance', mounted_as: 'header_logo', id: appearance.id, filename: 'dk.png'
response response
end end
end end
...@@ -553,7 +541,6 @@ describe UploadsController do ...@@ -553,7 +541,6 @@ describe UploadsController do
it_behaves_like 'content not cached without revalidation' do it_behaves_like 'content not cached without revalidation' do
subject do subject do
get :show, model: 'appearance', mounted_as: 'logo', id: appearance.id, filename: 'dk.png' get :show, model: 'appearance', mounted_as: 'logo', id: appearance.id, filename: 'dk.png'
response response
end end
end end
......
...@@ -18,7 +18,7 @@ FactoryBot.define do ...@@ -18,7 +18,7 @@ FactoryBot.define do
end end
trait :with_avatar do trait :with_avatar do
avatar { fixture_file_upload('spec/fixtures/dk.png') } avatar { File.open(Rails.root.join('spec/fixtures/dk.png')) }
end end
trait :access_requestable do trait :access_requestable do
......
...@@ -122,11 +122,11 @@ FactoryBot.define do ...@@ -122,11 +122,11 @@ FactoryBot.define do
end end
trait :with_attachment do trait :with_attachment do
attachment { fixture_file_upload(Rails.root.join( "spec/fixtures/dk.png"), "image/png") } attachment { fixture_file_upload(Rails.root + "spec/fixtures/dk.png", "image/png") }
end end
trait :with_svg_attachment do trait :with_svg_attachment do
attachment { fixture_file_upload(Rails.root.join("spec/fixtures/unsanitized.svg"), "image/svg+xml") } attachment { fixture_file_upload(Rails.root + "spec/fixtures/unsanitized.svg", "image/svg+xml") }
end end
transient do transient do
......
...@@ -90,7 +90,7 @@ FactoryBot.define do ...@@ -90,7 +90,7 @@ FactoryBot.define do
end end
trait :with_avatar do trait :with_avatar do
avatar { fixture_file_upload('spec/fixtures/dk.png') } avatar { File.open(Rails.root.join('spec/fixtures/dk.png')) }
end end
trait :broken_storage do trait :broken_storage do
......
FactoryBot.define do FactoryBot.define do
factory :upload do factory :upload do
model { build(:project) } model { build(:project) }
path { "uploads/-/system/project/avatar/avatar.jpg" }
size 100.kilobytes size 100.kilobytes
uploader "AvatarUploader" uploader "AvatarUploader"
# we should build a mount agnostic upload by default trait :personal_snippet do
transient do
mounted_as :avatar
secret SecureRandom.hex
end
# this needs to comply with RecordsUpload::Concern#upload_path
path { File.join("uploads/-/system", model.class.to_s.underscore, mounted_as.to_s, 'avatar.jpg') }
trait :personal_snippet_upload do
model { build(:personal_snippet) } model { build(:personal_snippet) }
path { File.join(secret, 'myfile.jpg') }
uploader "PersonalFileUploader" uploader "PersonalFileUploader"
end end
trait :issuable_upload do trait :issuable_upload do
path { File.join(secret, 'myfile.jpg') } path { "#{SecureRandom.hex}/myfile.jpg" }
uploader "FileUploader" uploader "FileUploader"
end end
trait :namespace_upload do trait :namespace_upload do
path { "#{SecureRandom.hex}/myfile.jpg" }
model { build(:group) } model { build(:group) }
path { File.join(secret, 'myfile.jpg') }
uploader "NamespaceFileUploader" uploader "NamespaceFileUploader"
end end
trait :attachment_upload do
transient do
mounted_as :attachment
end
model { build(:note) }
uploader "AttachmentUploader"
end
end end
end end
...@@ -38,7 +38,7 @@ FactoryBot.define do ...@@ -38,7 +38,7 @@ FactoryBot.define do
end end
trait :with_avatar do trait :with_avatar do
avatar { fixture_file_upload('spec/fixtures/dk.png') } avatar { File.open(Rails.root.join('spec/fixtures/dk.png')) }
end end
trait :two_factor_via_otp do trait :two_factor_via_otp do
......
...@@ -23,27 +23,6 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do ...@@ -23,27 +23,6 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do
end end
end end
# E.g. The installation is in use at the time of migration, and someone has
# just uploaded a file
shared_examples 'does not add files in /uploads/tmp' do
let(:tmp_file) { Rails.root.join(described_class::ABSOLUTE_UPLOAD_DIR, 'tmp', 'some_file.jpg') }
before do
FileUtils.mkdir(File.dirname(tmp_file))
FileUtils.touch(tmp_file)
end
after do
FileUtils.rm(tmp_file)
end
it 'does not add files from /uploads/tmp' do
described_class.new.perform
expect(untracked_files_for_uploads.count).to eq(5)
end
end
it 'ensures the untracked_files_for_uploads table exists' do it 'ensures the untracked_files_for_uploads table exists' do
expect do expect do
described_class.new.perform described_class.new.perform
...@@ -130,8 +109,24 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do ...@@ -130,8 +109,24 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do
end end
end end
# E.g. The installation is in use at the time of migration, and someone has
# just uploaded a file
context 'when there are files in /uploads/tmp' do context 'when there are files in /uploads/tmp' do
it_behaves_like 'does not add files in /uploads/tmp' let(:tmp_file) { Rails.root.join(described_class::ABSOLUTE_UPLOAD_DIR, 'tmp', 'some_file.jpg') }
before do
FileUtils.touch(tmp_file)
end
after do
FileUtils.rm(tmp_file)
end
it 'does not add files from /uploads/tmp' do
described_class.new.perform
expect(untracked_files_for_uploads.count).to eq(5)
end
end end
end end
end end
...@@ -202,8 +197,24 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do ...@@ -202,8 +197,24 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do
end end
end end
# E.g. The installation is in use at the time of migration, and someone has
# just uploaded a file
context 'when there are files in /uploads/tmp' do context 'when there are files in /uploads/tmp' do
it_behaves_like 'does not add files in /uploads/tmp' let(:tmp_file) { Rails.root.join(described_class::ABSOLUTE_UPLOAD_DIR, 'tmp', 'some_file.jpg') }
before do
FileUtils.touch(tmp_file)
end
after do
FileUtils.rm(tmp_file)
end
it 'does not add files from /uploads/tmp' do
described_class.new.perform
expect(untracked_files_for_uploads.count).to eq(5)
end
end end
end end
end end
......
...@@ -17,7 +17,7 @@ describe Gitlab::Gfm::UploadsRewriter do ...@@ -17,7 +17,7 @@ describe Gitlab::Gfm::UploadsRewriter do
end end
let(:text) do let(:text) do
"Text and #{image_uploader.markdown_link} and #{zip_uploader.markdown_link}" "Text and #{image_uploader.to_markdown} and #{zip_uploader.to_markdown}"
end end
describe '#rewrite' do describe '#rewrite' do
......
...@@ -4,6 +4,7 @@ describe Gitlab::ImportExport::UploadsRestorer do ...@@ -4,6 +4,7 @@ describe Gitlab::ImportExport::UploadsRestorer do
describe 'bundle a project Git repo' do describe 'bundle a project Git repo' do
let(:export_path) { "#{Dir.tmpdir}/uploads_saver_spec" } let(:export_path) { "#{Dir.tmpdir}/uploads_saver_spec" }
let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.full_path) } let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.full_path) }
let(:uploads_path) { FileUploader.dynamic_path_segment(project) }
before do before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
...@@ -25,9 +26,9 @@ describe Gitlab::ImportExport::UploadsRestorer do ...@@ -25,9 +26,9 @@ describe Gitlab::ImportExport::UploadsRestorer do
end end
it 'copies the uploads to the project path' do it 'copies the uploads to the project path' do
subject.restore restorer.restore
uploads = Dir.glob(File.join(subject.uploads_path, '**/*')).map { |file| File.basename(file) } uploads = Dir.glob(File.join(uploads_path, '**/*')).map { |file| File.basename(file) }
expect(uploads).to include('dummy.txt') expect(uploads).to include('dummy.txt')
end end
...@@ -43,9 +44,9 @@ describe Gitlab::ImportExport::UploadsRestorer do ...@@ -43,9 +44,9 @@ describe Gitlab::ImportExport::UploadsRestorer do
end end
it 'copies the uploads to the project path' do it 'copies the uploads to the project path' do
subject.restore restorer.restore
uploads = Dir.glob(File.join(subject.uploads_path, '**/*')).map { |file| File.basename(file) } uploads = Dir.glob(File.join(uploads_path, '**/*')).map { |file| File.basename(file) }
expect(uploads).to include('dummy.txt') expect(uploads).to include('dummy.txt')
end end
......
...@@ -30,7 +30,7 @@ describe Gitlab::ImportExport::UploadsSaver do ...@@ -30,7 +30,7 @@ describe Gitlab::ImportExport::UploadsSaver do
it 'copies the uploads to the export path' do it 'copies the uploads to the export path' do
saver.save saver.save
uploads = Dir.glob(File.join(saver.uploads_export_path, '**/*')).map { |file| File.basename(file) } uploads = Dir.glob(File.join(shared.export_path, 'uploads', '**/*')).map { |file| File.basename(file) }
expect(uploads).to include('banana_sample.gif') expect(uploads).to include('banana_sample.gif')
end end
...@@ -52,7 +52,7 @@ describe Gitlab::ImportExport::UploadsSaver do ...@@ -52,7 +52,7 @@ describe Gitlab::ImportExport::UploadsSaver do
it 'copies the uploads to the export path' do it 'copies the uploads to the export path' do
saver.save saver.save
uploads = Dir.glob(File.join(saver.uploads_export_path, '**/*')).map { |file| File.basename(file) } uploads = Dir.glob(File.join(shared.export_path, 'uploads', '**/*')).map { |file| File.basename(file) }
expect(uploads).to include('banana_sample.gif') expect(uploads).to include('banana_sample.gif')
end end
......
...@@ -204,7 +204,7 @@ describe Namespace do ...@@ -204,7 +204,7 @@ describe Namespace do
let(:parent) { create(:group, name: 'parent', path: 'parent') } let(:parent) { create(:group, name: 'parent', path: 'parent') }
let(:child) { create(:group, name: 'child', path: 'child', parent: parent) } let(:child) { create(:group, name: 'child', path: 'child', parent: parent) }
let!(:project) { create(:project_empty_repo, path: 'the-project', namespace: child, skip_disk_validation: true) } let!(:project) { create(:project_empty_repo, path: 'the-project', namespace: child, skip_disk_validation: true) }
let(:uploads_dir) { FileUploader.root } let(:uploads_dir) { File.join(CarrierWave.root, FileUploader.base_dir) }
let(:pages_dir) { File.join(TestEnv.pages_path) } let(:pages_dir) { File.join(TestEnv.pages_path) }
before do before do
......
...@@ -45,6 +45,51 @@ describe Upload do ...@@ -45,6 +45,51 @@ describe Upload do
end end
end end
describe '.remove_path' do
it 'removes all records at the given path' do
described_class.create!(
size: File.size(__FILE__),
path: __FILE__,
model: build_stubbed(:user),
uploader: 'AvatarUploader'
)
expect { described_class.remove_path(__FILE__) }
.to change { described_class.count }.from(1).to(0)
end
end
describe '.record' do
let(:fake_uploader) do
double(
file: double(size: 12_345),
relative_path: 'foo/bar.jpg',
model: build_stubbed(:user),
class: 'AvatarUploader'
)
end
it 'removes existing paths before creation' do
expect(described_class).to receive(:remove_path)
.with(fake_uploader.relative_path)
described_class.record(fake_uploader)
end
it 'creates a new record and assigns size, path, model, and uploader' do
upload = described_class.record(fake_uploader)
aggregate_failures do
expect(upload).to be_persisted
expect(upload.size).to eq fake_uploader.file.size
expect(upload.path).to eq fake_uploader.relative_path
expect(upload.model_id).to eq fake_uploader.model.id
expect(upload.model_type).to eq fake_uploader.model.class.to_s
expect(upload.uploader).to eq fake_uploader.class
end
end
end
describe '#absolute_path' do describe '#absolute_path' do
it 'returns the path directly when already absolute' do it 'returns the path directly when already absolute' do
path = '/path/to/namespace/project/secret/file.jpg' path = '/path/to/namespace/project/secret/file.jpg'
...@@ -66,27 +111,27 @@ describe Upload do ...@@ -66,27 +111,27 @@ describe Upload do
end end
end end
describe '#calculate_checksum!' do describe '#calculate_checksum' do
let(:upload) do it 'calculates the SHA256 sum' do
described_class.new(path: __FILE__, upload = described_class.new(
size: described_class::CHECKSUM_THRESHOLD - 1.megabyte) path: __FILE__,
end size: described_class::CHECKSUM_THRESHOLD - 1.megabyte
)
it 'sets `checksum` to SHA256 sum of the file' do
expected = Digest::SHA256.file(__FILE__).hexdigest expected = Digest::SHA256.file(__FILE__).hexdigest
expect { upload.calculate_checksum! } expect { upload.calculate_checksum }
.to change { upload.checksum }.from(nil).to(expected) .to change { upload.checksum }.from(nil).to(expected)
end end
it 'sets `checksum` to nil for a non-existant file' do it 'returns nil for a non-existant file' do
expect(upload).to receive(:exist?).and_return(false) upload = described_class.new(
path: __FILE__,
size: described_class::CHECKSUM_THRESHOLD - 1.megabyte
)
checksum = Digest::SHA256.file(__FILE__).hexdigest expect(upload).to receive(:exist?).and_return(false)
upload.checksum = checksum
expect { upload.calculate_checksum! } expect(upload.calculate_checksum).to be_nil
.to change { upload.checksum }.from(checksum).to(nil)
end end
end end
......
...@@ -945,7 +945,7 @@ describe API::Runner do ...@@ -945,7 +945,7 @@ describe API::Runner do
context 'when artifacts are being stored inside of tmp path' do context 'when artifacts are being stored inside of tmp path' do
before do before do
# by configuring this path we allow to pass temp file from any path # by configuring this path we allow to pass temp file from any path
allow(JobArtifactUploader).to receive(:workhorse_upload_path).and_return('/') allow(JobArtifactUploader).to receive(:artifacts_upload_path).and_return('/')
end end
context 'when job has been erased' do context 'when job has been erased' do
...@@ -1122,7 +1122,7 @@ describe API::Runner do ...@@ -1122,7 +1122,7 @@ describe API::Runner do
# by configuring this path we allow to pass file from @tmpdir only # by configuring this path we allow to pass file from @tmpdir only
# but all temporary files are stored in system tmp directory # but all temporary files are stored in system tmp directory
@tmpdir = Dir.mktmpdir @tmpdir = Dir.mktmpdir
allow(JobArtifactUploader).to receive(:workhorse_upload_path).and_return(@tmpdir) allow(JobArtifactUploader).to receive(:artifacts_upload_path).and_return(@tmpdir)
end end
after do after do
......
...@@ -958,7 +958,7 @@ describe 'Git LFS API and storage' do ...@@ -958,7 +958,7 @@ describe 'Git LFS API and storage' do
end end
it 'responds with status 200, location of lfs store and object details' do it 'responds with status 200, location of lfs store and object details' do
expect(json_response['StoreLFSPath']).to eq(LfsObjectUploader.workhorse_upload_path) expect(json_response['StoreLFSPath']).to eq("#{Gitlab.config.shared.path}/lfs-objects/tmp/upload")
expect(json_response['LfsOid']).to eq(sample_oid) expect(json_response['LfsOid']).to eq(sample_oid)
expect(json_response['LfsSize']).to eq(sample_size) expect(json_response['LfsSize']).to eq(sample_size)
end end
...@@ -1075,7 +1075,7 @@ describe 'Git LFS API and storage' do ...@@ -1075,7 +1075,7 @@ describe 'Git LFS API and storage' do
end end
it 'with location of lfs store and object details' do it 'with location of lfs store and object details' do
expect(json_response['StoreLFSPath']).to eq(LfsObjectUploader.workhorse_upload_path) expect(json_response['StoreLFSPath']).to eq("#{Gitlab.config.shared.path}/lfs-objects/tmp/upload")
expect(json_response['LfsOid']).to eq(sample_oid) expect(json_response['LfsOid']).to eq(sample_oid)
expect(json_response['LfsSize']).to eq(sample_size) expect(json_response['LfsSize']).to eq(sample_size)
end end
......
...@@ -6,7 +6,7 @@ describe Issues::MoveService do ...@@ -6,7 +6,7 @@ describe Issues::MoveService do
let(:title) { 'Some issue' } let(:title) { 'Some issue' }
let(:description) { 'Some issue description' } let(:description) { 'Some issue description' }
let(:old_project) { create(:project) } let(:old_project) { create(:project) }
let(:new_project) { create(:project, group: create(:group)) } let(:new_project) { create(:project) }
let(:milestone1) { create(:milestone, project_id: old_project.id, title: 'v9.0') } let(:milestone1) { create(:milestone, project_id: old_project.id, title: 'v9.0') }
let(:old_issue) do let(:old_issue) do
...@@ -250,7 +250,7 @@ describe Issues::MoveService do ...@@ -250,7 +250,7 @@ describe Issues::MoveService do
context 'issue description with uploads' do context 'issue description with uploads' do
let(:uploader) { build(:file_uploader, project: old_project) } let(:uploader) { build(:file_uploader, project: old_project) }
let(:description) { "Text and #{uploader.markdown_link}" } let(:description) { "Text and #{uploader.to_markdown}" }
include_context 'issue move executed' include_context 'issue move executed'
......
...@@ -6,7 +6,7 @@ describe Projects::HashedStorage::MigrateAttachmentsService do ...@@ -6,7 +6,7 @@ describe Projects::HashedStorage::MigrateAttachmentsService do
let(:legacy_storage) { Storage::LegacyProject.new(project) } let(:legacy_storage) { Storage::LegacyProject.new(project) }
let(:hashed_storage) { Storage::HashedProject.new(project) } let(:hashed_storage) { Storage::HashedProject.new(project) }
let!(:upload) { Upload.find_by(path: file_uploader.upload_path) } let!(:upload) { Upload.find_by(path: file_uploader.relative_path) }
let(:file_uploader) { build(:file_uploader, project: project) } let(:file_uploader) { build(:file_uploader, project: project) }
let(:old_path) { File.join(base_path(legacy_storage), upload.path) } let(:old_path) { File.join(base_path(legacy_storage), upload.path) }
let(:new_path) { File.join(base_path(hashed_storage), upload.path) } let(:new_path) { File.join(base_path(hashed_storage), upload.path) }
...@@ -58,6 +58,6 @@ describe Projects::HashedStorage::MigrateAttachmentsService do ...@@ -58,6 +58,6 @@ describe Projects::HashedStorage::MigrateAttachmentsService do
end end
def base_path(storage) def base_path(storage)
File.join(FileUploader.root, storage.disk_path) FileUploader.dynamic_path_builder(storage.disk_path)
end end
end end
...@@ -2,8 +2,6 @@ shared_examples 'handle uploads' do ...@@ -2,8 +2,6 @@ shared_examples 'handle uploads' do
let(:user) { create(:user) } let(:user) { create(:user) }
let(:jpg) { fixture_file_upload(Rails.root + 'spec/fixtures/rails_sample.jpg', 'image/jpg') } let(:jpg) { fixture_file_upload(Rails.root + 'spec/fixtures/rails_sample.jpg', 'image/jpg') }
let(:txt) { fixture_file_upload(Rails.root + 'spec/fixtures/doc_sample.txt', 'text/plain') } let(:txt) { fixture_file_upload(Rails.root + 'spec/fixtures/doc_sample.txt', 'text/plain') }
let(:secret) { FileUploader.generate_secret }
let(:uploader_class) { FileUploader }
describe "POST #create" do describe "POST #create" do
context 'when a user is not authorized to upload a file' do context 'when a user is not authorized to upload a file' do
...@@ -67,12 +65,7 @@ shared_examples 'handle uploads' do ...@@ -67,12 +65,7 @@ shared_examples 'handle uploads' do
describe "GET #show" do describe "GET #show" do
let(:show_upload) do let(:show_upload) do
get :show, params.merge(secret: secret, filename: "rails_sample.jpg") get :show, params.merge(secret: "123456", filename: "image.jpg")
end
before do
expect(FileUploader).to receive(:generate_secret).and_return(secret)
UploadService.new(model, jpg, uploader_class).execute
end end
context "when the model is public" do context "when the model is public" do
...@@ -82,6 +75,11 @@ shared_examples 'handle uploads' do ...@@ -82,6 +75,11 @@ shared_examples 'handle uploads' do
context "when not signed in" do context "when not signed in" do
context "when the file exists" do context "when the file exists" do
before do
allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg)
allow(jpg).to receive(:exists?).and_return(true)
end
it "responds with status 200" do it "responds with status 200" do
show_upload show_upload
...@@ -90,10 +88,6 @@ shared_examples 'handle uploads' do ...@@ -90,10 +88,6 @@ shared_examples 'handle uploads' do
end end
context "when the file doesn't exist" do context "when the file doesn't exist" do
before do
allow_any_instance_of(FileUploader).to receive(:exists?).and_return(false)
end
it "responds with status 404" do it "responds with status 404" do
show_upload show_upload
...@@ -108,6 +102,11 @@ shared_examples 'handle uploads' do ...@@ -108,6 +102,11 @@ shared_examples 'handle uploads' do
end end
context "when the file exists" do context "when the file exists" do
before do
allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg)
allow(jpg).to receive(:exists?).and_return(true)
end
it "responds with status 200" do it "responds with status 200" do
show_upload show_upload
...@@ -116,10 +115,6 @@ shared_examples 'handle uploads' do ...@@ -116,10 +115,6 @@ shared_examples 'handle uploads' do
end end
context "when the file doesn't exist" do context "when the file doesn't exist" do
before do
allow_any_instance_of(FileUploader).to receive(:exists?).and_return(false)
end
it "responds with status 404" do it "responds with status 404" do
show_upload show_upload
...@@ -136,6 +131,11 @@ shared_examples 'handle uploads' do ...@@ -136,6 +131,11 @@ shared_examples 'handle uploads' do
context "when not signed in" do context "when not signed in" do
context "when the file exists" do context "when the file exists" do
before do
allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg)
allow(jpg).to receive(:exists?).and_return(true)
end
context "when the file is an image" do context "when the file is an image" do
before do before do
allow_any_instance_of(FileUploader).to receive(:image?).and_return(true) allow_any_instance_of(FileUploader).to receive(:image?).and_return(true)
...@@ -149,10 +149,6 @@ shared_examples 'handle uploads' do ...@@ -149,10 +149,6 @@ shared_examples 'handle uploads' do
end end
context "when the file is not an image" do context "when the file is not an image" do
before do
allow_any_instance_of(FileUploader).to receive(:image?).and_return(false)
end
it "redirects to the sign in page" do it "redirects to the sign in page" do
show_upload show_upload
...@@ -162,10 +158,6 @@ shared_examples 'handle uploads' do ...@@ -162,10 +158,6 @@ shared_examples 'handle uploads' do
end end
context "when the file doesn't exist" do context "when the file doesn't exist" do
before do
allow_any_instance_of(FileUploader).to receive(:exists?).and_return(false)
end
it "redirects to the sign in page" do it "redirects to the sign in page" do
show_upload show_upload
...@@ -185,6 +177,11 @@ shared_examples 'handle uploads' do ...@@ -185,6 +177,11 @@ shared_examples 'handle uploads' do
end end
context "when the file exists" do context "when the file exists" do
before do
allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg)
allow(jpg).to receive(:exists?).and_return(true)
end
it "responds with status 200" do it "responds with status 200" do
show_upload show_upload
...@@ -193,10 +190,6 @@ shared_examples 'handle uploads' do ...@@ -193,10 +190,6 @@ shared_examples 'handle uploads' do
end end
context "when the file doesn't exist" do context "when the file doesn't exist" do
before do
allow_any_instance_of(FileUploader).to receive(:exists?).and_return(false)
end
it "responds with status 404" do it "responds with status 404" do
show_upload show_upload
...@@ -207,6 +200,11 @@ shared_examples 'handle uploads' do ...@@ -207,6 +200,11 @@ shared_examples 'handle uploads' do
context "when the user doesn't have access to the model" do context "when the user doesn't have access to the model" do
context "when the file exists" do context "when the file exists" do
before do
allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg)
allow(jpg).to receive(:exists?).and_return(true)
end
context "when the file is an image" do context "when the file is an image" do
before do before do
allow_any_instance_of(FileUploader).to receive(:image?).and_return(true) allow_any_instance_of(FileUploader).to receive(:image?).and_return(true)
...@@ -220,10 +218,6 @@ shared_examples 'handle uploads' do ...@@ -220,10 +218,6 @@ shared_examples 'handle uploads' do
end end
context "when the file is not an image" do context "when the file is not an image" do
before do
allow_any_instance_of(FileUploader).to receive(:image?).and_return(false)
end
it "responds with status 404" do it "responds with status 404" do
show_upload show_upload
...@@ -233,10 +227,6 @@ shared_examples 'handle uploads' do ...@@ -233,10 +227,6 @@ shared_examples 'handle uploads' do
end end
context "when the file doesn't exist" do context "when the file doesn't exist" do
before do
allow_any_instance_of(FileUploader).to receive(:exists?).and_return(false)
end
it "responds with status 404" do it "responds with status 404" do
show_upload show_upload
......
shared_examples "matches the method pattern" do |method|
let(:target) { subject }
let(:args) { nil }
let(:pattern) { patterns[method] }
it do
return skip "No pattern provided, skipping." unless pattern
expect(target.method(method).call(*args)).to match(pattern)
end
end
shared_examples "builds correct paths" do |**patterns|
let(:patterns) { patterns }
before do
allow(subject).to receive(:filename).and_return('<filename>')
end
describe "#store_dir" do
it_behaves_like "matches the method pattern", :store_dir
end
describe "#cache_dir" do
it_behaves_like "matches the method pattern", :cache_dir
end
describe "#work_dir" do
it_behaves_like "matches the method pattern", :work_dir
end
describe "#upload_path" do
it_behaves_like "matches the method pattern", :upload_path
end
describe ".absolute_path" do
it_behaves_like "matches the method pattern", :absolute_path do
let(:target) { subject.class }
let(:args) { [upload] }
end
end
describe ".base_dir" do
it_behaves_like "matches the method pattern", :base_dir do
let(:target) { subject.class }
end
end
end
...@@ -237,7 +237,7 @@ module TestEnv ...@@ -237,7 +237,7 @@ module TestEnv
end end
def artifacts_path def artifacts_path
Gitlab.config.artifacts.storage_path Gitlab.config.artifacts.path
end end
# When no cached assets exist, manually hit the root path to create them # When no cached assets exist, manually hit the root path to create them
......
module TrackUntrackedUploadsHelpers module TrackUntrackedUploadsHelpers
def uploaded_file def uploaded_file
fixture_path = Rails.root.join('spec/fixtures/rails_sample.jpg') fixture_path = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg')
fixture_file_upload(fixture_path) fixture_file_upload(fixture_path)
end end
......
require 'spec_helper' require 'spec_helper'
describe AttachmentUploader do describe AttachmentUploader do
let(:note) { create(:note, :with_attachment) } let(:uploader) { described_class.new(build_stubbed(:user)) }
let(:uploader) { note.attachment }
let(:upload) { create(:upload, :attachment_upload, model: uploader.model) }
subject { uploader } describe "#store_dir" do
it "stores in the system dir" do
expect(uploader.store_dir).to start_with("uploads/-/system/user")
end
it_behaves_like 'builds correct paths', it "uses the old path when using object storage" do
store_dir: %r[uploads/-/system/note/attachment/], expect(described_class).to receive(:file_storage?).and_return(false)
upload_path: %r[uploads/-/system/note/attachment/], expect(uploader.store_dir).to start_with("uploads/user")
absolute_path: %r[#{CarrierWave.root}/uploads/-/system/note/attachment/] end
end
describe '#move_to_cache' do
it 'is true' do
expect(uploader.move_to_cache).to eq(true)
end
end
describe '#move_to_store' do
it 'is true' do
expect(uploader.move_to_store).to eq(true)
end
end
end end
require 'spec_helper' require 'spec_helper'
describe AvatarUploader do describe AvatarUploader do
let(:model) { create(:user, :with_avatar) } let(:uploader) { described_class.new(build_stubbed(:user)) }
let(:uploader) { described_class.new(model, :avatar) }
let(:upload) { create(:upload, model: model) }
subject { uploader } describe "#store_dir" do
it "stores in the system dir" do
expect(uploader.store_dir).to start_with("uploads/-/system/user")
end
it_behaves_like 'builds correct paths', it "uses the old path when using object storage" do
store_dir: %r[uploads/-/system/user/avatar/], expect(described_class).to receive(:file_storage?).and_return(false)
upload_path: %r[uploads/-/system/user/avatar/], expect(uploader.store_dir).to start_with("uploads/user")
absolute_path: %r[#{CarrierWave.root}/uploads/-/system/user/avatar/] end
end
describe '#move_to_cache' do describe '#move_to_cache' do
it 'is false' do it 'is false' do
......
...@@ -3,13 +3,13 @@ require 'spec_helper' ...@@ -3,13 +3,13 @@ require 'spec_helper'
describe FileMover do describe FileMover do
let(:filename) { 'banana_sample.gif' } let(:filename) { 'banana_sample.gif' }
let(:file) { fixture_file_upload(Rails.root.join('spec', 'fixtures', filename)) } let(:file) { fixture_file_upload(Rails.root.join('spec', 'fixtures', filename)) }
let(:temp_file_path) { File.join('uploads/-/system/temp', 'secret55', filename) }
let(:temp_description) do let(:temp_description) do
"test ![banana_sample](/#{temp_file_path}) "\ 'test ![banana_sample](/uploads/-/system/temp/secret55/banana_sample.gif) same ![banana_sample]'\
"same ![banana_sample](/#{temp_file_path}) " '(/uploads/-/system/temp/secret55/banana_sample.gif)'
end end
let(:file_path) { File.join('uploads/-/system/personal_snippet', snippet.id.to_s, 'secret55', filename) } let(:temp_file_path) { File.join('secret55', filename).to_s }
let(:file_path) { File.join('uploads', '-', 'system', 'personal_snippet', snippet.id.to_s, 'secret55', filename).to_s }
let(:snippet) { create(:personal_snippet, description: temp_description) } let(:snippet) { create(:personal_snippet, description: temp_description) }
subject { described_class.new(file_path, snippet).execute } subject { described_class.new(file_path, snippet).execute }
...@@ -28,8 +28,8 @@ describe FileMover do ...@@ -28,8 +28,8 @@ describe FileMover do
expect(snippet.reload.description) expect(snippet.reload.description)
.to eq( .to eq(
"test ![banana_sample](/uploads/-/system/personal_snippet/#{snippet.id}/secret55/banana_sample.gif) "\ "test ![banana_sample](/uploads/-/system/personal_snippet/#{snippet.id}/secret55/banana_sample.gif)"\
"same ![banana_sample](/uploads/-/system/personal_snippet/#{snippet.id}/secret55/banana_sample.gif) " " same ![banana_sample](/uploads/-/system/personal_snippet/#{snippet.id}/secret55/banana_sample.gif)"
) )
end end
...@@ -50,8 +50,8 @@ describe FileMover do ...@@ -50,8 +50,8 @@ describe FileMover do
expect(snippet.reload.description) expect(snippet.reload.description)
.to eq( .to eq(
"test ![banana_sample](/uploads/-/system/temp/secret55/banana_sample.gif) "\ "test ![banana_sample](/uploads/-/system/temp/secret55/banana_sample.gif)"\
"same ![banana_sample](/uploads/-/system/temp/secret55/banana_sample.gif) " " same ![banana_sample](/uploads/-/system/temp/secret55/banana_sample.gif)"
) )
end end
......
require 'spec_helper' require 'spec_helper'
describe FileUploader do describe FileUploader do
let(:group) { create(:group, name: 'awesome') } let(:uploader) { described_class.new(build_stubbed(:project)) }
let(:project) { create(:project, namespace: group, name: 'project') }
let(:uploader) { described_class.new(project) }
let(:upload) { double(model: project, path: 'secret/foo.jpg') }
subject { uploader } context 'legacy storage' do
let(:project) { build_stubbed(:project) }
describe '.absolute_path' do
it 'returns the correct absolute path by building it dynamically' do
upload = double(model: project, path: 'secret/foo.jpg')
dynamic_segment = project.full_path
shared_examples 'builds correct legacy storage paths' do expect(described_class.absolute_path(upload))
include_examples 'builds correct paths', .to end_with("#{dynamic_segment}/secret/foo.jpg")
store_dir: %r{awesome/project/\h+}, end
absolute_path: %r{#{described_class.root}/awesome/project/secret/foo.jpg} end
describe "#store_dir" do
it "stores in the namespace path" do
uploader = described_class.new(project)
expect(uploader.store_dir).to include(project.full_path)
expect(uploader.store_dir).not_to include("system")
end
end
end end
shared_examples 'uses hashed storage' do context 'hashed storage' do
context 'when rolled out attachments' do context 'when rolled out attachments' do
before do let(:project) { build_stubbed(:project, :hashed) }
allow(project).to receive(:disk_path).and_return('ca/fe/fe/ed')
describe '.absolute_path' do
it 'returns the correct absolute path by building it dynamically' do
upload = double(model: project, path: 'secret/foo.jpg')
dynamic_segment = project.disk_path
expect(described_class.absolute_path(upload))
.to end_with("#{dynamic_segment}/secret/foo.jpg")
end
end end
let(:project) { build_stubbed(:project, :hashed, namespace: group, name: 'project') } describe "#store_dir" do
it "stores in the namespace path" do
uploader = described_class.new(project)
it_behaves_like 'builds correct paths', expect(uploader.store_dir).to include(project.disk_path)
store_dir: %r{ca/fe/fe/ed/\h+}, expect(uploader.store_dir).not_to include("system")
absolute_path: %r{#{described_class.root}/ca/fe/fe/ed/secret/foo.jpg} end
end
end end
context 'when only repositories are rolled out' do context 'when only repositories are rolled out' do
let(:project) { build_stubbed(:project, namespace: group, name: 'project', storage_version: Project::HASHED_STORAGE_FEATURES[:repository]) } let(:project) { build_stubbed(:project, storage_version: Project::HASHED_STORAGE_FEATURES[:repository]) }
it_behaves_like 'builds correct legacy storage paths' describe '.absolute_path' do
end it 'returns the correct absolute path by building it dynamically' do
end upload = double(model: project, path: 'secret/foo.jpg')
context 'legacy storage' do dynamic_segment = project.full_path
it_behaves_like 'builds correct legacy storage paths'
include_examples 'uses hashed storage' expect(described_class.absolute_path(upload))
.to end_with("#{dynamic_segment}/secret/foo.jpg")
end
end
describe "#store_dir" do
it "stores in the namespace path" do
uploader = described_class.new(project)
expect(uploader.store_dir).to include(project.full_path)
expect(uploader.store_dir).not_to include("system")
end
end
end
end end
describe 'initialize' do describe 'initialize' do
let(:uploader) { described_class.new(double, 'secret') } it 'generates a secret if none is provided' do
expect(SecureRandom).to receive(:hex).and_return('secret')
uploader = described_class.new(double)
expect(uploader.secret).to eq 'secret'
end
it 'accepts a secret parameter' do it 'accepts a secret parameter' do
expect(described_class).not_to receive(:generate_secret) expect(SecureRandom).not_to receive(:hex)
expect(uploader.secret).to eq('secret')
uploader = described_class.new(double, 'secret')
expect(uploader.secret).to eq 'secret'
end end
end end
describe '#secret' do describe '#move_to_cache' do
it 'generates a secret if none is provided' do it 'is true' do
expect(described_class).to receive(:generate_secret).and_return('secret') expect(uploader.move_to_cache).to eq(true)
expect(uploader.secret).to eq('secret') end
end
describe '#move_to_store' do
it 'is true' do
expect(uploader.move_to_store).to eq(true)
end
end
describe '#relative_path' do
it 'removes the leading dynamic path segment' do
fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg')
uploader.store!(fixture_file_upload(fixture))
expect(uploader.relative_path).to match(%r{\A\h{32}/rails_sample.jpg\z})
end end
end end
end end
...@@ -3,13 +3,33 @@ require 'spec_helper' ...@@ -3,13 +3,33 @@ require 'spec_helper'
describe JobArtifactUploader do describe JobArtifactUploader do
let(:job_artifact) { create(:ci_job_artifact) } let(:job_artifact) { create(:ci_job_artifact) }
let(:uploader) { described_class.new(job_artifact, :file) } let(:uploader) { described_class.new(job_artifact, :file) }
let(:local_path) { Gitlab.config.artifacts.path }
subject { uploader } describe '#store_dir' do
subject { uploader.store_dir }
it_behaves_like "builds correct paths", let(:path) { "#{job_artifact.created_at.utc.strftime('%Y_%m_%d')}/#{job_artifact.job_id}/#{job_artifact.id}" }
store_dir: %r[\h{2}/\h{2}/\h{64}/\d{4}_\d{1,2}_\d{1,2}/\d+/\d+\z],
cache_dir: %r[artifacts/tmp/cache], context 'when using local storage' do
work_dir: %r[artifacts/tmp/work] it { is_expected.to start_with(local_path) }
it { is_expected.to match(%r{\h{2}/\h{2}/\h{64}/\d{4}_\d{1,2}_\d{1,2}/\d+/\d+\z}) }
it { is_expected.to end_with(path) }
end
end
describe '#cache_dir' do
subject { uploader.cache_dir }
it { is_expected.to start_with(local_path) }
it { is_expected.to end_with('/tmp/cache') }
end
describe '#work_dir' do
subject { uploader.work_dir }
it { is_expected.to start_with(local_path) }
it { is_expected.to end_with('/tmp/work') }
end
context 'file is stored in valid local_path' do context 'file is stored in valid local_path' do
let(:file) do let(:file) do
...@@ -23,7 +43,7 @@ describe JobArtifactUploader do ...@@ -23,7 +43,7 @@ describe JobArtifactUploader do
subject { uploader.file.path } subject { uploader.file.path }
it { is_expected.to start_with("#{uploader.root}/#{uploader.class.base_dir}") } it { is_expected.to start_with(local_path) }
it { is_expected.to include("/#{job_artifact.created_at.utc.strftime('%Y_%m_%d')}/") } it { is_expected.to include("/#{job_artifact.created_at.utc.strftime('%Y_%m_%d')}/") }
it { is_expected.to include("/#{job_artifact.job_id}/#{job_artifact.id}/") } it { is_expected.to include("/#{job_artifact.job_id}/#{job_artifact.id}/") }
it { is_expected.to end_with("ci_build_artifacts.zip") } it { is_expected.to end_with("ci_build_artifacts.zip") }
......
...@@ -3,22 +3,49 @@ require 'rails_helper' ...@@ -3,22 +3,49 @@ require 'rails_helper'
describe LegacyArtifactUploader do describe LegacyArtifactUploader do
let(:job) { create(:ci_build) } let(:job) { create(:ci_build) }
let(:uploader) { described_class.new(job, :legacy_artifacts_file) } let(:uploader) { described_class.new(job, :legacy_artifacts_file) }
let(:local_path) { described_class.root } let(:local_path) { Gitlab.config.artifacts.path }
subject { uploader } describe '.local_store_path' do
subject { described_class.local_store_path }
# TODO: move to Workhorse::UploadPath it "delegate to artifacts path" do
describe '.workhorse_upload_path' do expect(Gitlab.config.artifacts).to receive(:path)
subject { described_class.workhorse_upload_path }
subject
end
end
describe '.artifacts_upload_path' do
subject { described_class.artifacts_upload_path }
it { is_expected.to start_with(local_path) } it { is_expected.to start_with(local_path) }
it { is_expected.to end_with('tmp/uploads') } it { is_expected.to end_with('tmp/uploads/') }
end
describe '#store_dir' do
subject { uploader.store_dir }
let(:path) { "#{job.created_at.utc.strftime('%Y_%m')}/#{job.project_id}/#{job.id}" }
context 'when using local storage' do
it { is_expected.to start_with(local_path) }
it { is_expected.to end_with(path) }
end
end end
it_behaves_like "builds correct paths", describe '#cache_dir' do
store_dir: %r[\d{4}_\d{1,2}/\d+/\d+\z], subject { uploader.cache_dir }
cache_dir: %r[artifacts/tmp/cache],
work_dir: %r[artifacts/tmp/work] it { is_expected.to start_with(local_path) }
it { is_expected.to end_with('/tmp/cache') }
end
describe '#work_dir' do
subject { uploader.work_dir }
it { is_expected.to start_with(local_path) }
it { is_expected.to end_with('/tmp/work') }
end
describe '#filename' do describe '#filename' do
# we need to use uploader, as this makes to use mounter # we need to use uploader, as this makes to use mounter
...@@ -42,7 +69,7 @@ describe LegacyArtifactUploader do ...@@ -42,7 +69,7 @@ describe LegacyArtifactUploader do
subject { uploader.file.path } subject { uploader.file.path }
it { is_expected.to start_with("#{uploader.root}") } it { is_expected.to start_with(local_path) }
it { is_expected.to include("/#{job.created_at.utc.strftime('%Y_%m')}/") } it { is_expected.to include("/#{job.created_at.utc.strftime('%Y_%m')}/") }
it { is_expected.to include("/#{job.project_id}/") } it { is_expected.to include("/#{job.project_id}/") }
it { is_expected.to end_with("ci_build_artifacts.zip") } it { is_expected.to end_with("ci_build_artifacts.zip") }
......
...@@ -2,13 +2,39 @@ require 'spec_helper' ...@@ -2,13 +2,39 @@ require 'spec_helper'
describe LfsObjectUploader do describe LfsObjectUploader do
let(:lfs_object) { create(:lfs_object, :with_file) } let(:lfs_object) { create(:lfs_object, :with_file) }
let(:uploader) { described_class.new(lfs_object, :file) } let(:uploader) { described_class.new(lfs_object) }
let(:path) { Gitlab.config.lfs.storage_path } let(:path) { Gitlab.config.lfs.storage_path }
subject { uploader } describe '#move_to_cache' do
it 'is true' do
expect(uploader.move_to_cache).to eq(true)
end
end
it_behaves_like "builds correct paths", describe '#move_to_store' do
store_dir: %r[\h{2}/\h{2}], it 'is true' do
cache_dir: %r[/lfs-objects/tmp/cache], expect(uploader.move_to_store).to eq(true)
work_dir: %r[/lfs-objects/tmp/work] end
end
describe '#store_dir' do
subject { uploader.store_dir }
it { is_expected.to start_with(path) }
it { is_expected.to end_with("#{lfs_object.oid[0, 2]}/#{lfs_object.oid[2, 2]}") }
end
describe '#cache_dir' do
subject { uploader.cache_dir }
it { is_expected.to start_with(path) }
it { is_expected.to end_with('/tmp/cache') }
end
describe '#work_dir' do
subject { uploader.work_dir }
it { is_expected.to start_with(path) }
it { is_expected.to end_with('/tmp/work') }
end
end end
require 'spec_helper' require 'spec_helper'
IDENTIFIER = %r{\h+/\S+}
describe NamespaceFileUploader do describe NamespaceFileUploader do
let(:group) { build_stubbed(:group) } let(:group) { build_stubbed(:group) }
let(:uploader) { described_class.new(group) } let(:uploader) { described_class.new(group) }
let(:upload) { create(:upload, :namespace_upload, model: group) }
subject { uploader } describe "#store_dir" do
it "stores in the namespace id directory" do
expect(uploader.store_dir).to include(group.id.to_s)
end
end
describe ".absolute_path" do
it "stores in thecorrect directory" do
upload_record = create(:upload, :namespace_upload, model: group)
it_behaves_like 'builds correct paths', expect(described_class.absolute_path(upload_record))
store_dir: %r[uploads/-/system/namespace/\d+], .to include("-/system/namespace/#{group.id}")
upload_path: IDENTIFIER, end
absolute_path: %r[#{CarrierWave.root}/uploads/-/system/namespace/\d+/#{IDENTIFIER}] end
end end
require 'spec_helper' require 'spec_helper'
IDENTIFIER = %r{\h+/\S+}
describe PersonalFileUploader do describe PersonalFileUploader do
let(:model) { create(:personal_snippet) } let(:uploader) { described_class.new(build_stubbed(:project)) }
let(:uploader) { described_class.new(model) } let(:snippet) { create(:personal_snippet) }
let(:upload) { create(:upload, :personal_snippet_upload) }
subject { uploader } describe '.absolute_path' do
it 'returns the correct absolute path by building it dynamically' do
upload = double(model: snippet, path: 'secret/foo.jpg')
it_behaves_like 'builds correct paths', dynamic_segment = "personal_snippet/#{snippet.id}"
store_dir: %r[uploads/-/system/personal_snippet/\d+],
upload_path: IDENTIFIER,
absolute_path: %r[#{CarrierWave.root}/uploads/-/system/personal_snippet/\d+/#{IDENTIFIER}]
describe '#to_h' do expect(described_class.absolute_path(upload)).to end_with("/-/system/#{dynamic_segment}/secret/foo.jpg")
before do
subject.instance_variable_set(:@secret, 'secret')
end end
end
describe '#to_h' do
it 'returns the hass' do
uploader = described_class.new(snippet, 'secret')
it 'is correct' do
allow(uploader).to receive(:file).and_return(double(extension: 'txt', filename: 'file_name')) allow(uploader).to receive(:file).and_return(double(extension: 'txt', filename: 'file_name'))
expected_url = "/uploads/-/system/personal_snippet/#{model.id}/secret/file_name" expected_url = "/uploads/-/system/personal_snippet/#{snippet.id}/secret/file_name"
expect(uploader.to_h).to eq( expect(uploader.to_h).to eq(
alt: 'file_name', alt: 'file_name',
......
...@@ -3,16 +3,16 @@ require 'rails_helper' ...@@ -3,16 +3,16 @@ require 'rails_helper'
describe RecordsUploads do describe RecordsUploads do
let!(:uploader) do let!(:uploader) do
class RecordsUploadsExampleUploader < GitlabUploader class RecordsUploadsExampleUploader < GitlabUploader
include RecordsUploads::Concern include RecordsUploads
storage :file storage :file
def dynamic_segment def model
'co/fe/ee' FactoryBot.build_stubbed(:user)
end end
end end
RecordsUploadsExampleUploader.new(build_stubbed(:user)) RecordsUploadsExampleUploader.new
end end
def upload_fixture(filename) def upload_fixture(filename)
...@@ -20,55 +20,48 @@ describe RecordsUploads do ...@@ -20,55 +20,48 @@ describe RecordsUploads do
end end
describe 'callbacks' do describe 'callbacks' do
let(:upload) { create(:upload) } it 'calls `record_upload` after `store`' do
before do
uploader.upload = upload
end
it '#record_upload after `store`' do
expect(uploader).to receive(:record_upload).once expect(uploader).to receive(:record_upload).once
uploader.store!(upload_fixture('doc_sample.txt')) uploader.store!(upload_fixture('doc_sample.txt'))
end end
it '#destroy_upload after `remove`' do it 'calls `destroy_upload` after `remove`' do
expect(uploader).to receive(:destroy_upload).once
uploader.store!(upload_fixture('doc_sample.txt')) uploader.store!(upload_fixture('doc_sample.txt'))
expect(uploader).to receive(:destroy_upload).once
uploader.remove! uploader.remove!
end end
end end
describe '#record_upload callback' do describe '#record_upload callback' do
it 'creates an Upload record after store' do it 'returns early when not using file storage' do
expect { uploader.store!(upload_fixture('rails_sample.jpg')) }.to change { Upload.count }.by(1) allow(uploader).to receive(:file_storage?).and_return(false)
end expect(Upload).not_to receive(:record)
it 'creates a new record and assigns size, path, model, and uploader' do
uploader.store!(upload_fixture('rails_sample.jpg')) uploader.store!(upload_fixture('rails_sample.jpg'))
upload = uploader.upload
aggregate_failures do
expect(upload).to be_persisted
expect(upload.size).to eq uploader.file.size
expect(upload.path).to eq uploader.upload_path
expect(upload.model_id).to eq uploader.model.id
expect(upload.model_type).to eq uploader.model.class.to_s
expect(upload.uploader).to eq uploader.class.to_s
end
end end
it "does not create an Upload record when the file doesn't exist" do it "returns early when the file doesn't exist" do
allow(uploader).to receive(:file).and_return(double(exists?: false)) allow(uploader).to receive(:file).and_return(double(exists?: false))
expect(Upload).not_to receive(:record)
expect { uploader.store!(upload_fixture('rails_sample.jpg')) }.not_to change { Upload.count } uploader.store!(upload_fixture('rails_sample.jpg'))
end
it 'creates an Upload record after store' do
expect(Upload).to receive(:record)
.with(uploader)
uploader.store!(upload_fixture('rails_sample.jpg'))
end end
it 'does not create an Upload record if model is missing' do it 'does not create an Upload record if model is missing' do
allow_any_instance_of(RecordsUploadsExampleUploader).to receive(:model).and_return(nil) expect_any_instance_of(RecordsUploadsExampleUploader).to receive(:model).and_return(nil)
expect(Upload).not_to receive(:record).with(uploader)
expect { uploader.store!(upload_fixture('rails_sample.jpg')) }.not_to change { Upload.count } uploader.store!(upload_fixture('rails_sample.jpg'))
end end
it 'it destroys Upload records at the same path before recording' do it 'it destroys Upload records at the same path before recording' do
...@@ -79,15 +72,29 @@ describe RecordsUploads do ...@@ -79,15 +72,29 @@ describe RecordsUploads do
uploader: uploader.class.to_s uploader: uploader.class.to_s
) )
uploader.upload = existing
uploader.store!(upload_fixture('rails_sample.jpg')) uploader.store!(upload_fixture('rails_sample.jpg'))
expect { existing.reload }.to raise_error(ActiveRecord::RecordNotFound) expect { existing.reload }.to raise_error(ActiveRecord::RecordNotFound)
expect(Upload.count).to eq(1) expect(Upload.count).to eq 1
end end
end end
describe '#destroy_upload callback' do describe '#destroy_upload callback' do
it 'returns early when not using file storage' do
uploader.store!(upload_fixture('rails_sample.jpg'))
allow(uploader).to receive(:file_storage?).and_return(false)
expect(Upload).not_to receive(:remove_path)
uploader.remove!
end
it 'returns early when file is nil' do
expect(Upload).not_to receive(:remove_path)
uploader.remove!
end
it 'it destroys Upload records at the same path after removal' do it 'it destroys Upload records at the same path after removal' do
uploader.store!(upload_fixture('rails_sample.jpg')) uploader.store!(upload_fixture('rails_sample.jpg'))
......
...@@ -2,31 +2,18 @@ require 'rails_helper' ...@@ -2,31 +2,18 @@ require 'rails_helper'
describe UploadChecksumWorker do describe UploadChecksumWorker do
describe '#perform' do describe '#perform' do
subject { described_class.new } it 'rescues ActiveRecord::RecordNotFound' do
expect { described_class.new.perform(999_999) }.not_to raise_error
context 'without a valid record' do
it 'rescues ActiveRecord::RecordNotFound' do
expect { subject.perform(999_999) }.not_to raise_error
end
end end
context 'with a valid record' do it 'calls calculate_checksum_without_delay and save!' do
let(:upload) { create(:user, :with_avatar).avatar.upload } upload = spy
expect(Upload).to receive(:find).with(999_999).and_return(upload)
before do
expect(Upload).to receive(:find).and_return(upload)
allow(upload).to receive(:foreground_checksumable?).and_return(false)
end
it 'calls calculate_checksum!' do described_class.new.perform(999_999)
expect(upload).to receive(:calculate_checksum!)
subject.perform(upload.id)
end
it 'calls save!' do expect(upload).to have_received(:calculate_checksum)
expect(upload).to receive(:save!) expect(upload).to have_received(:save!)
subject.perform(upload.id)
end
end end
end end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment