Commit 6df47023 authored by Stan Hu's avatar Stan Hu

Adapt RepositoryArchiveCleanUpService to @v2 prefix

Since we added a `@v2` prefix for files that may have LFS blobs, we need
to make sure they are cleaned up periodically.
parent 98714d0e
# frozen_string_literal: true
# RepositoryArchiveCleanUpService removes cached repository archives
# that are generated on-the-fly by Gitaly. These files are stored in the
# following form (as defined in lib/gitlab/git/repository.rb) and served
# by GitLab Workhorse:
#
# /path/to/repository/downloads/project-N/sha/@v2/archive.format
#
# Legacy paths omit the @v2 prefix.
#
# For example:
#
# /var/opt/gitlab/gitlab-rails/shared/cache/archive/project-1/master/@v2/archive.zip
class RepositoryArchiveCleanUpService
LAST_MODIFIED_TIME_IN_MINUTES = 120
# For `/path/project-N/sha/@v2/archive.zip`, `find /path -maxdepth 4` will find this file
MAX_ARCHIVE_DEPTH = 4
attr_reader :mmin, :path
def initialize(mmin = LAST_MODIFIED_TIME_IN_MINUTES)
......@@ -22,12 +37,15 @@ class RepositoryArchiveCleanUpService
private
def clean_up_old_archives
run(%W(find #{path} -mindepth 1 -maxdepth 3 -type f \( -name \*.tar -o -name \*.bz2 -o -name \*.tar.gz -o -name \*.zip \) -mmin +#{mmin} -delete))
run(%W(find #{path} -mindepth 1 -maxdepth #{MAX_ARCHIVE_DEPTH} -type f \( -name \*.tar -o -name \*.bz2 -o -name \*.tar.gz -o -name \*.zip \) -mmin +#{mmin} -delete))
end
def clean_up_empty_directories
run(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -empty -delete))
run(%W(find #{path} -mindepth 1 -maxdepth 1 -type d -empty -delete))
(1...MAX_ARCHIVE_DEPTH).reverse_each { |depth| clean_up_empty_directories_with_depth(depth) }
end
def clean_up_empty_directories_with_depth(depth)
run(%W(find #{path} -mindepth #{depth} -maxdepth #{depth} -type d -empty -delete))
end
def run(cmd)
......
......@@ -18,6 +18,16 @@ RSpec.describe RepositoryArchiveCleanUpService do
end
end
it 'removes outdated archives and directories in a versioned path' do
in_directory_with_files("project-#{non_existing_record_id}/#{sha}/@v2", %w[tar tar.bz2 tar.gz zip], 3.hours) do |dirname, files|
service.execute
files.each { |filename| expect(File.exist?(filename)).to be_falsy }
expect(File.directory?(dirname)).to be_falsy
expect(File.directory?(File.dirname(dirname))).to be_falsy
end
end
it 'does not remove directories when they contain outdated non-archives' do
in_directory_with_files("project-#{non_existing_record_id}/#{sha}", %w[tar conf rb], 2.hours) do |dirname, files|
service.execute
......@@ -64,7 +74,9 @@ RSpec.describe RepositoryArchiveCleanUpService do
end
it 'removes files older than 2 hours that matches valid archive extensions' do
in_directory_with_files('sample.git', %w[tar tar.bz2 tar.gz zip], 2.hours) do |dir, files|
# In macOS, the the `mmin` parameter for `find` rounds up, so add a full
# minute to ensure these files are deemed old.
in_directory_with_files('sample.git', %w[tar tar.bz2 tar.gz zip], 121.minutes) do |dir, files|
service.execute
files.each { |file| expect(File.exist?(file)).to eq false }
......@@ -73,11 +85,11 @@ RSpec.describe RepositoryArchiveCleanUpService do
end
context 'with files older than 2 hours that does not matches valid archive extensions' do
it_behaves_like 'invalid archive files', 'sample.git', %w[conf rb], 2.hours
it_behaves_like 'invalid archive files', 'sample.git', %w[conf rb], 121.minutes
end
context 'with files older than 2 hours inside invalid directories' do
it_behaves_like 'invalid archive files', 'john/doe/sample.git', %w[conf rb tar tar.gz], 2.hours
it_behaves_like 'invalid archive files', 'john/t/doe/sample.git', %w[conf rb tar tar.gz], 121.minutes
end
context 'with files newer than 2 hours that matches valid archive extensions' do
......@@ -110,8 +122,6 @@ RSpec.describe RepositoryArchiveCleanUpService do
def create_temporary_files(dir, extensions, mtime)
FileUtils.mkdir_p(dir)
# rubocop: disable Rails/TimeZone
FileUtils.touch(extensions.map { |ext| File.join(dir, "sample.#{ext}") }, mtime: Time.now - mtime)
# rubocop: enable Rails/TimeZone
FileUtils.touch(extensions.map { |ext| File.join(dir, "sample.#{ext}") }, mtime: Time.now.utc - mtime)
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment