Skip to content

G
gitlab-ce
Commit 4a55b9ff authored by Mathieu Parent's avatar Mathieu Parent Committed by David Fernandez
Browse files
Options

Generate Debian component files for udeb and source files 

Changelog: added
parent d63213fb
Hide whitespace changes
Inline Side-by-side
Showing with 162 additions and 24 deletions
app/models/packages/package_file.rb
View file @ 4a55b9ff
......@@ -5,7 +5,7 @@ class Packages::PackageFile < ApplicationRecord
delegate :project, :project_id, to: :package
delegate :conan_file_type, to: :conan_file_metadatum
delegate :file_type, :component, :architecture, :fields, to: :debian_file_metadatum, prefix: :debian
delegate :file_type, :dsc?, :component, :architecture, :fields, to: :debian_file_metadatum, prefix: :debian
delegate :channel, :metadata, to: :helm_file_metadatum, prefix: :helm
belongs_to :package
......@@ -33,6 +33,8 @@ class Packages::PackageFile < ApplicationRecord
scope :with_file_name_like, ->(file_name) { where(arel_table[:file_name].matches(file_name)) }
scope :with_files_stored_locally, -> { where(file_store: ::Packages::PackageFileUploader::Store::LOCAL) }
scope :with_format, ->(format) { where(::Packages::PackageFile.arel_table[:file_name].matches("%.#{format}")) }
scope :preload_package, -> { preload(:package) }
scope :preload_conan_file_metadata, -> { preload(:conan_file_metadatum) }
scope :preload_debian_file_metadata, -> { preload(:debian_file_metadatum) }
scope :preload_helm_file_metadata, -> { preload(:helm_file_metadatum) }
......
app/services/packages/debian/generate_distribution_service.rb
View file @ 4a55b9ff
......@@ -12,7 +12,7 @@ module Packages
DEFAULT_LEASE_TIMEOUT = 1.hour.to_i.freeze
# From https://salsa.debian.org/ftp-team/dak/-/blob/991aaa27a7f7aa773bb9c0cf2d516e383d9cffa0/setup/core-init.d/080_metadatakeys#L9
BINARIES_METADATA = %w(
METADATA_KEYS = %w(
Package
Source
Binary
......@@ -89,15 +89,18 @@ module Packages
@distribution.components.ordered_by_name.each do |component|
@distribution.architectures.ordered_by_name.each do |architecture|
generate_component_file(component, :packages, architecture, :deb)
generate_component_file(component, :di_packages, architecture, :udeb)
end
generate_component_file(component, :source, nil, :dsc)
end
end
def generate_component_file(component, component_file_type, architecture, package_file_type)
paragraphs = @distribution.package_files
.preload_package
.preload_debian_file_metadata
.with_debian_component_name(component.name)
.with_debian_architecture_name(architecture.name)
.with_debian_architecture_name(architecture&.name)
.with_debian_file_type(package_file_type)
.find_each
.map(&method(:package_stanza_from_fields))
......@@ -106,21 +109,49 @@ module Packages
def package_stanza_from_fields(package_file)
[
BINARIES_METADATA.map do |metadata_key|
rfc822_field(metadata_key, package_file.debian_fields[metadata_key])
METADATA_KEYS.map do |metadata_key|
metadata_name = metadata_key
metadata_value = package_file.debian_fields[metadata_key]
if package_file.debian_dsc?
metadata_name = 'Package' if metadata_key == 'Source'
checksum = case metadata_key
when 'Files' then package_file.file_md5
when 'Checksums-Sha256' then package_file.file_sha256
when 'Checksums-Sha1' then package_file.file_sha1
end
if checksum
metadata_value = "\n#{checksum} #{package_file.size} #{package_file.file_name}#{metadata_value}"
end
end
rfc822_field(metadata_name, metadata_value)
end,
rfc822_field('Section', package_file.debian_fields['Section'] || 'misc'),
rfc822_field('Priority', package_file.debian_fields['Priority'] || 'extra'),
rfc822_field('Filename', package_filename(package_file)),
rfc822_field('Size', package_file.size),
rfc822_field('MD5sum', package_file.file_md5),
rfc822_field('SHA256', package_file.file_sha256)
package_file_extra_fields(package_file)
].flatten.compact.join('')
end
def package_filename(package_file)
def package_file_extra_fields(package_file)
if package_file.debian_dsc?
[
rfc822_field('Directory', package_dirname(package_file))
]
else
[
rfc822_field('Filename', "#{package_dirname(package_file)}/#{package_file.file_name}"),
rfc822_field('Size', package_file.size),
rfc822_field('MD5sum', package_file.file_md5),
rfc822_field('SHA256', package_file.file_sha256)
]
end
end
def package_dirname(package_file)
letter = package_file.package.name.start_with?('lib') ? package_file.package.name[0..3] : package_file.package.name[0]
"#{pool_prefix(package_file)}/#{letter}/#{package_file.package.name}/#{package_file.package.version}/#{package_file.file_name}"
"#{pool_prefix(package_file)}/#{letter}/#{package_file.package.name}/#{package_file.package.version}"
end
def pool_prefix(package_file)
......@@ -206,7 +237,8 @@ module Packages
return unless condition
return if value.blank?
"#{name}: #{value.to_s.gsub("\n\n", "\n.\n").gsub("\n", "\n ")}\n"
value = " #{value}" unless value[0] == "\n"
"#{name}:#{value.to_s.gsub("\n\n", "\n.\n").gsub("\n", "\n ")}\n"
end
def valid_until_field
......
spec/factories/packages/debian/file_metadatum.rb
View file @ 4a55b9ff
......@@ -26,7 +26,27 @@ FactoryBot.define do
file_type { 'dsc' }
component { 'main' }
architecture { nil }
fields { { 'a': 'b' } }
fields do
{
'Format' => '3.0 (native)',
'Source' => package_file.package.name,
'Binary' => 'sample-dev, libsample0, sample-udeb',
'Architecture' => 'any',
'Version': package_file.package.version,
'Maintainer' => "#{FFaker::Name.name} <#{FFaker::Internet.email}>",
'Homepage' => FFaker::Internet.http_url,
'Standards-Version' => '4.5.0',
'Build-Depends' => 'debhelper-compat (= 13)',
'Package-List' => <<~EOF.rstrip,
libsample0 deb libs optional arch=any',
sample-dev deb libdevel optional arch=any',
sample-udeb udeb libs optional arch=any',
EOF
'Checksums-Sha1' => "\nc5cfc111ea924842a89a06d5673f07dfd07de8ca 864 sample_1.2.3~alpha2.tar.xz",
'Checksums-Sha256' => "\n40e4682bb24a73251ccd7c7798c0094a649091e5625d6a14bcec9b4e7174f3da 864 sample_1.2.3~alpha2.tar.xz",
'Files' => "\nd5ca476e4229d135a88f9c729c7606c9 864 sample_1.2.3~alpha2.tar.xz"
}
end
end
trait(:deb) do
......
spec/services/packages/debian/generate_distribution_service_spec.rb
View file @ 4a55b9ff
......@@ -6,6 +6,9 @@ RSpec.describe Packages::Debian::GenerateDistributionService do
describe '#execute' do
subject { described_class.new(distribution).execute }
let(:subject2) { described_class.new(distribution).execute }
let(:subject3) { described_class.new(distribution).execute }
include_context 'with published Debian package'
[:project, :group].each do |container_type|
......
spec/support/shared_examples/services/packages/debian/generate_distribution_shared_examples.rb
View file @ 4a55b9ff
......@@ -57,10 +57,13 @@ RSpec.shared_examples 'Generate Debian Distribution and component files' do
travel_to(current_time) do
expect(Gitlab::ErrorTracking).not_to receive(:log_exception)
components_count = 2
architectures_count = 3
initial_count = 6
destroyed_count = 2
# updated_count = 1
created_count = 5
updated_count = 1
created_count = components_count * (architectures_count * 2 + 1) - updated_count
expect { subject }
.to not_change { Packages::Package.count }
......@@ -69,7 +72,7 @@ RSpec.shared_examples 'Generate Debian Distribution and component files' do
.and change { distribution.component_files.reset.count }.from(initial_count).to(initial_count - destroyed_count + created_count)
.and change { component_file1.reload.updated_at }.to(current_time.round)
debs = package.package_files.with_debian_file_type(:deb).preload_debian_file_metadata.to_a
package_files = package.package_files.order(id: :asc).preload_debian_file_metadata.to_a
pool_prefix = 'pool/unstable'
pool_prefix += "/#{project.id}" if container_type == :group
pool_prefix += "/p/#{package.name}/#{package.version}"
......@@ -78,26 +81,26 @@ RSpec.shared_examples 'Generate Debian Distribution and component files' do
Source: #{package.name}
Version: #{package.version}
Installed-Size: 7
Maintainer: #{debs[0].debian_fields['Maintainer']}
Maintainer: #{package_files[2].debian_fields['Maintainer']}
Architecture: amd64
Description: Some mostly empty lib
Used in GitLab tests.
.
Testing another paragraph.
Multi-Arch: same
Homepage: #{debs[0].debian_fields['Homepage']}
Homepage: #{package_files[2].debian_fields['Homepage']}
Section: libs
Priority: optional
Filename: #{pool_prefix}/libsample0_1.2.3~alpha2_amd64.deb
Size: 409600
MD5sum: #{debs[0].file_md5}
SHA256: #{debs[0].file_sha256}
MD5sum: #{package_files[2].file_md5}
SHA256: #{package_files[2].file_sha256}
Package: sample-dev
Source: #{package.name} (#{package.version})
Version: 1.2.3~binary
Installed-Size: 7
Maintainer: #{debs[1].debian_fields['Maintainer']}
Maintainer: #{package_files[3].debian_fields['Maintainer']}
Architecture: amd64
Depends: libsample0 (= 1.2.3~binary)
Description: Some mostly empty development files
......@@ -105,23 +108,68 @@ RSpec.shared_examples 'Generate Debian Distribution and component files' do
.
Testing another paragraph.
Multi-Arch: same
Homepage: #{debs[1].debian_fields['Homepage']}
Homepage: #{package_files[3].debian_fields['Homepage']}
Section: libdevel
Priority: optional
Filename: #{pool_prefix}/sample-dev_1.2.3~binary_amd64.deb
Size: 409600
MD5sum: #{debs[1].file_md5}
SHA256: #{debs[1].file_sha256}
MD5sum: #{package_files[3].file_md5}
SHA256: #{package_files[3].file_sha256}
EOF
expected_main_amd64_di_content = <<~EOF
Section: misc
Priority: extra
Filename: #{pool_prefix}/sample-udeb_1.2.3~alpha2_amd64.udeb
Size: 409600
MD5sum: #{package_files[4].file_md5}
SHA256: #{package_files[4].file_sha256}
EOF
expected_main_source_content = <<~EOF
Package: #{package.name}
Binary: sample-dev, libsample0, sample-udeb
Version: #{package.version}
Maintainer: #{package_files[1].debian_fields['Maintainer']}
Build-Depends: debhelper-compat (= 13)
Architecture: any
Standards-Version: 4.5.0
Format: 3.0 (native)
Files:
#{package_files[1].file_md5} #{package_files[1].size} #{package_files[1].file_name}
d5ca476e4229d135a88f9c729c7606c9 864 sample_1.2.3~alpha2.tar.xz
Checksums-Sha256:
#{package_files[1].file_sha256} #{package_files[1].size} #{package_files[1].file_name}
40e4682bb24a73251ccd7c7798c0094a649091e5625d6a14bcec9b4e7174f3da 864 sample_1.2.3~alpha2.tar.xz
Checksums-Sha1:
#{package_files[1].file_sha1} #{package_files[1].size} #{package_files[1].file_name}
c5cfc111ea924842a89a06d5673f07dfd07de8ca 864 sample_1.2.3~alpha2.tar.xz
Homepage: #{package_files[1].debian_fields['Homepage']}
Section: misc
Priority: extra
Directory: #{pool_prefix}
EOF
check_component_file(current_time.round, 'main', :packages, 'all', nil)
check_component_file(current_time.round, 'main', :packages, 'amd64', expected_main_amd64_content)
check_component_file(current_time.round, 'main', :packages, 'arm64', nil)
check_component_file(current_time.round, 'main', :di_packages, 'all', nil)
check_component_file(current_time.round, 'main', :di_packages, 'amd64', expected_main_amd64_di_content)
check_component_file(current_time.round, 'main', :di_packages, 'arm64', nil)
check_component_file(current_time.round, 'main', :source, nil, expected_main_source_content)
check_component_file(current_time.round, 'contrib', :packages, 'all', nil)
check_component_file(current_time.round, 'contrib', :packages, 'amd64', nil)
check_component_file(current_time.round, 'contrib', :packages, 'arm64', nil)
check_component_file(current_time.round, 'contrib', :di_packages, 'all', nil)
check_component_file(current_time.round, 'contrib', :di_packages, 'amd64', nil)
check_component_file(current_time.round, 'contrib', :di_packages, 'arm64', nil)
check_component_file(current_time.round, 'contrib', :source, nil, nil)
main_amd64_size = expected_main_amd64_content.length
main_amd64_md5sum = Digest::MD5.hexdigest(expected_main_amd64_content)
main_amd64_sha256 = Digest::SHA256.hexdigest(expected_main_amd64_content)
......@@ -130,6 +178,14 @@ RSpec.shared_examples 'Generate Debian Distribution and component files' do
contrib_all_md5sum = component_file1.file_md5
contrib_all_sha256 = component_file1.file_sha256
main_amd64_di_size = expected_main_amd64_di_content.length
main_amd64_di_md5sum = Digest::MD5.hexdigest(expected_main_amd64_di_content)
main_amd64_di_sha256 = Digest::SHA256.hexdigest(expected_main_amd64_di_content)
main_source_size = expected_main_source_content.length
main_source_md5sum = Digest::MD5.hexdigest(expected_main_source_content)
main_source_sha256 = Digest::SHA256.hexdigest(expected_main_source_content)
expected_release_content = <<~EOF
Codename: unstable
Date: Sat, 25 Jan 2020 15:17:18 +0000
......@@ -138,22 +194,44 @@ RSpec.shared_examples 'Generate Debian Distribution and component files' do
Components: contrib main
MD5Sum:
#{contrib_all_md5sum} #{contrib_all_size} contrib/binary-all/Packages
d41d8cd98f00b204e9800998ecf8427e 0 contrib/debian-installer/binary-all/Packages
d41d8cd98f00b204e9800998ecf8427e 0 contrib/binary-amd64/Packages
d41d8cd98f00b204e9800998ecf8427e 0 contrib/debian-installer/binary-amd64/Packages
d41d8cd98f00b204e9800998ecf8427e 0 contrib/binary-arm64/Packages
d41d8cd98f00b204e9800998ecf8427e 0 contrib/debian-installer/binary-arm64/Packages
d41d8cd98f00b204e9800998ecf8427e 0 contrib/source/Source
d41d8cd98f00b204e9800998ecf8427e 0 main/binary-all/Packages
d41d8cd98f00b204e9800998ecf8427e 0 main/debian-installer/binary-all/Packages
#{main_amd64_md5sum} #{main_amd64_size} main/binary-amd64/Packages
#{main_amd64_di_md5sum} #{main_amd64_di_size} main/debian-installer/binary-amd64/Packages
d41d8cd98f00b204e9800998ecf8427e 0 main/binary-arm64/Packages
d41d8cd98f00b204e9800998ecf8427e 0 main/debian-installer/binary-arm64/Packages
#{main_source_md5sum} #{main_source_size} main/source/Source
SHA256:
#{contrib_all_sha256} #{contrib_all_size} contrib/binary-all/Packages
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 contrib/debian-installer/binary-all/Packages
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 contrib/binary-amd64/Packages
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 contrib/debian-installer/binary-amd64/Packages
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 contrib/binary-arm64/Packages
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 contrib/debian-installer/binary-arm64/Packages
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 contrib/source/Source
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 main/binary-all/Packages
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 main/debian-installer/binary-all/Packages
#{main_amd64_sha256} #{main_amd64_size} main/binary-amd64/Packages
#{main_amd64_di_sha256} #{main_amd64_di_size} main/debian-installer/binary-amd64/Packages
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 main/binary-arm64/Packages
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 main/debian-installer/binary-arm64/Packages
#{main_source_sha256} #{main_source_size} main/source/Source
EOF
check_release_files(expected_release_content)
end
create_list(:debian_package, 10, project: project, published_in: project_distribution)
control_count = ActiveRecord::QueryRecorder.new { subject2 }.count
create_list(:debian_package, 10, project: project, published_in: project_distribution)
expect { subject3 }.not_to exceed_query_limit(control_count)
end
end
......
spec/workers/packages/debian/generate_distribution_worker_spec.rb
View file @ 4a55b9ff
......@@ -9,6 +9,9 @@ RSpec.describe Packages::Debian::GenerateDistributionWorker, type: :worker do
subject { described_class.new.perform(container_type, distribution_id) }
let(:subject2) { described_class.new.perform(container_type, distribution_id) }
let(:subject3) { described_class.new.perform(container_type, distribution_id) }
include_context 'with published Debian package'
[:project, :group].each do |container_type|
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7