Commit e5b99b51 authored by David Fernandez's avatar David Fernandez

Merge branch 'debian_extract_changes_metadata_service' into 'master'

Add Packages::Debian:ExtractChangesMetadataService

See merge request gitlab-org/gitlab!55896
parents 22af78bb ea576590
# frozen_string_literal: true
module Packages
module Debian
class FileEntry
include ActiveModel::Model
DIGESTS = %i[md5 sha1 sha256].freeze
FILENAME_REGEX = %r{\A[a-zA-Z0-9][a-zA-Z0-9_.~+-]*\z}.freeze
attr_accessor :filename,
:size,
:md5sum,
:section,
:priority,
:sha1sum,
:sha256sum,
:package_file
validates :filename, :size, :md5sum, :section, :priority, :sha1sum, :sha256sum, :package_file, presence: true
validates :filename, format: { with: FILENAME_REGEX }
validate :valid_package_file_digests, if: -> { md5sum.present? && sha1sum.present? && sha256sum.present? && package_file.present? }
def component
return 'main' if section.blank?
return 'main' unless section.include?('/')
section.split('/')[0]
end
private
def valid_package_file_digests
DIGESTS.each do |digest|
package_file_digest = package_file["file_#{digest}"]
sum = public_send("#{digest}sum") # rubocop:disable GitlabSecurity/PublicSend
next if package_file_digest == sum
errors.add("#{digest}sum".to_sym, "mismatch for #{filename}: #{package_file_digest} != #{sum}")
end
end
end
end
end
# frozen_string_literal: true
module Packages
module Debian
class ExtractChangesMetadataService
include Gitlab::Utils::StrongMemoize
ExtractionError = Class.new(StandardError)
def initialize(package_file)
@package_file = package_file
@entries = {}
end
def execute
{
file_type: file_type,
architecture: metadata[:architecture],
fields: fields,
files: files
}
rescue ActiveModel::ValidationError => e
raise ExtractionError.new(e.message)
end
private
def metadata
strong_memoize(:metadata) do
::Packages::Debian::ExtractMetadataService.new(@package_file).execute
end
end
def file_type
metadata[:file_type]
end
def fields
metadata[:fields]
end
def files
strong_memoize(:files) do
raise ExtractionError.new("is not a changes file") unless file_type == :changes
raise ExtractionError.new("Files field is missing") if fields[:Files].blank?
raise ExtractionError.new("Checksums-Sha1 field is missing") if fields[:'Checksums-Sha1'].blank?
raise ExtractionError.new("Checksums-Sha256 field is missing") if fields[:'Checksums-Sha256'].blank?
init_entries_from_files
entries_from_checksums_sha1
entries_from_checksums_sha256
entries_from_package_files
@entries
end
end
def init_entries_from_files
each_lines_for(:Files) do |line|
md5sum, size, section, priority, filename = line.split
entry = FileEntry.new(
filename: filename,
size: size.to_i,
md5sum: md5sum,
section: section,
priority: priority)
@entries[filename] = entry
end
end
def entries_from_checksums_sha1
each_lines_for(:'Checksums-Sha1') do |line|
sha1sum, size, filename = line.split
entry = @entries[filename]
raise ExtractionError.new("#{filename} is listed in Checksums-Sha1 but not in Files") unless entry
raise ExtractionError.new("Size for #{filename} in Files and Checksums-Sha1 differ") unless entry.size == size.to_i
entry.sha1sum = sha1sum
end
end
def entries_from_checksums_sha256
each_lines_for(:'Checksums-Sha256') do |line|
sha256sum, size, filename = line.split
entry = @entries[filename]
raise ExtractionError.new("#{filename} is listed in Checksums-Sha256 but not in Files") unless entry
raise ExtractionError.new("Size for #{filename} in Files and Checksums-Sha256 differ") unless entry.size == size.to_i
entry.sha256sum = sha256sum
end
end
def each_lines_for(field)
fields[field].split("\n").each do |line|
next if line.blank?
yield(line)
end
end
def entries_from_package_files
@entries.each do |filename, entry|
entry.package_file = ::Packages::PackageFileFinder.new(@package_file.package, filename).execute!
entry.validate!
rescue ActiveRecord::RecordNotFound
raise ExtractionError.new("#{filename} is listed in Files but was not uploaded")
end
end
end
end
end
......@@ -125,6 +125,9 @@ FactoryBot.define do
trait(:source) do
file_name { 'sample_1.2.3~alpha2.tar.xz' }
file_md5 { 'd79b34f58f61ff4ad696d9bd0b8daa68' }
file_sha1 { '5f8bba5574eb01ac3b1f5e2988e8c29307788236' }
file_sha256 { 'b5a599e88e7cbdda3bde808160a21ba1dd1ec76b2ec8d4912aae769648d68362' }
transient do
file_metadatum_trait { :source }
......@@ -133,6 +136,9 @@ FactoryBot.define do
trait(:dsc) do
file_name { 'sample_1.2.3~alpha2.dsc' }
file_md5 { '3b0817804f669e16cdefac583ad88f0e' }
file_sha1 { '32ecbd674f0bfd310df68484d87752490685a8d6' }
file_sha256 { '844f79825b7e8aaa191e514b58a81f9ac1e58e2180134b0c9512fa66d896d7ba' }
transient do
file_metadatum_trait { :dsc }
......@@ -141,6 +147,9 @@ FactoryBot.define do
trait(:deb) do
file_name { 'libsample0_1.2.3~alpha2_amd64.deb' }
file_md5 { 'fb0842b21adc44207996296fe14439dd' }
file_sha1 { '5248b95600e85bfe7f63c0dfce330a75f5777366' }
file_sha256 { '1c383a525bfcba619c7305ccd106d61db501a6bbaf0003bf8d0c429fbdb7fcc1' }
transient do
file_metadatum_trait { :deb }
......@@ -149,6 +158,9 @@ FactoryBot.define do
trait(:deb2) do
file_name { 'sample-dev_1.2.3~binary_amd64.deb' }
file_md5 { 'd2afbd28e4d74430d22f9504e18bfdf5' }
file_sha1 { 'f81e4f66c8c6bb899653a3340c157965ee69634f' }
file_sha256 { '9fbeee2191ce4dab5288fad5ecac1bd369f58fef9a992a880eadf0caf25f086d' }
transient do
file_metadatum_trait { :deb }
......@@ -157,6 +169,9 @@ FactoryBot.define do
trait(:udeb) do
file_name { 'sample-udeb_1.2.3~alpha2_amd64.udeb' }
file_md5 { '72b1dd7d98229e2fb0355feda1d3a165' }
file_sha1 { 'e42e8f2fe04ed1bb73b44a187674480d0e49dcba' }
file_sha256 { '2b0c152b3ab4cc07663350424de972c2b7621d69fe6df2e0b94308a191e4632f' }
transient do
file_metadatum_trait { :udeb }
......@@ -165,6 +180,9 @@ FactoryBot.define do
trait(:buildinfo) do
file_name { 'sample_1.2.3~alpha2_amd64.buildinfo' }
file_md5 { '4e085dd67c120ca967ec314f65770a42' }
file_sha1 { '0d47e899f3cc67a2253a4629456ff927e0db5c60' }
file_sha256 { 'f9900d3c94e94b329232668dcbef3dba2d96c07147b15b6dc0533452e4dd8a43' }
transient do
file_metadatum_trait { :buildinfo }
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Packages::Debian::FileEntry, type: :model do
let_it_be(:package_file) { create(:debian_package_file, :dsc) }
let(:filename) { 'sample_1.2.3~alpha2.dsc' }
let(:size) { 671 }
let(:md5sum) { '3b0817804f669e16cdefac583ad88f0e' }
let(:section) { 'libs' }
let(:priority) { 'optional' }
let(:sha1sum) { '32ecbd674f0bfd310df68484d87752490685a8d6' }
let(:sha256sum) { '844f79825b7e8aaa191e514b58a81f9ac1e58e2180134b0c9512fa66d896d7ba' }
let(:file_entry) do
described_class.new(
filename: filename,
size: size,
md5sum: md5sum,
section: section,
priority: priority,
sha1sum: sha1sum,
sha256sum: sha256sum,
package_file: package_file
)
end
subject { file_entry }
describe 'validations' do
it { is_expected.to be_valid }
describe '#filename' do
it { is_expected.to validate_presence_of(:filename) }
it { is_expected.not_to allow_value('Hé').for(:filename) }
end
describe '#size' do
it { is_expected.to validate_presence_of(:size) }
end
describe '#md5sum' do
it { is_expected.to validate_presence_of(:md5sum) }
it { is_expected.not_to allow_value('12345678901234567890123456789012').for(:md5sum).with_message('mismatch for sample_1.2.3~alpha2.dsc: 3b0817804f669e16cdefac583ad88f0e != 12345678901234567890123456789012') }
end
describe '#section' do
it { is_expected.to validate_presence_of(:section) }
end
describe '#priority' do
it { is_expected.to validate_presence_of(:priority) }
end
describe '#sha1sum' do
it { is_expected.to validate_presence_of(:sha1sum) }
it { is_expected.not_to allow_value('1234567890123456789012345678901234567890').for(:sha1sum).with_message('mismatch for sample_1.2.3~alpha2.dsc: 32ecbd674f0bfd310df68484d87752490685a8d6 != 1234567890123456789012345678901234567890') }
end
describe '#sha256sum' do
it { is_expected.to validate_presence_of(:sha256sum) }
it { is_expected.not_to allow_value('1234567890123456789012345678901234567890123456789012345678901234').for(:sha256sum).with_message('mismatch for sample_1.2.3~alpha2.dsc: 844f79825b7e8aaa191e514b58a81f9ac1e58e2180134b0c9512fa66d896d7ba != 1234567890123456789012345678901234567890123456789012345678901234') }
end
describe '#package_file' do
it { is_expected.to validate_presence_of(:package_file) }
end
end
describe '#component' do
subject { file_entry.component }
context 'without section' do
let(:section) { nil }
it { is_expected.to eq 'main' }
end
context 'with empty section' do
let(:section) { '' }
it { is_expected.to eq 'main' }
end
context 'with ruby section' do
let(:section) { 'ruby' }
it { is_expected.to eq 'main' }
end
context 'with contrib/ruby section' do
let(:section) { 'contrib/ruby' }
it { is_expected.to eq 'contrib' }
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Packages::Debian::ExtractChangesMetadataService do
describe '#execute' do
let_it_be(:distribution) { create(:debian_project_distribution, codename: 'unstable') }
let_it_be(:incoming) { create(:debian_incoming, project: distribution.project) }
let(:package_file) { incoming.package_files.last }
let(:service) { described_class.new(package_file) }
subject { service.execute }
context 'with valid package file' do
it 'extract metadata', :aggregate_failures do
expected_fields = { 'Architecture': 'source amd64', 'Binary': 'libsample0 sample-dev sample-udeb' }
expect(subject[:file_type]).to eq(:changes)
expect(subject[:architecture]).to be_nil
expect(subject[:fields]).to include(expected_fields)
expect(subject[:files].count).to eq(6)
end
end
context 'with invalid package file' do
let(:package_file) { incoming.package_files.first }
it 'raise ArgumentError', :aggregate_failures do
expect { subject }.to raise_error(described_class::ExtractionError, "is not a changes file")
end
end
context 'with invalid metadata' do
let(:md5_dsc) { '3b0817804f669e16cdefac583ad88f0e 671 libs optional sample_1.2.3~alpha2.dsc' }
let(:md5_source) { 'd79b34f58f61ff4ad696d9bd0b8daa68 864 libs optional sample_1.2.3~alpha2.tar.xz' }
let(:md5s) { "#{md5_dsc}\n#{md5_source}" }
let(:sha1_dsc) { '32ecbd674f0bfd310df68484d87752490685a8d6 671 sample_1.2.3~alpha2.dsc' }
let(:sha1_source) { '5f8bba5574eb01ac3b1f5e2988e8c29307788236 864 sample_1.2.3~alpha2.tar.xz' }
let(:sha1s) { "#{sha1_dsc}\n#{sha1_source}" }
let(:sha256_dsc) { '844f79825b7e8aaa191e514b58a81f9ac1e58e2180134b0c9512fa66d896d7ba 671 sample_1.2.3~alpha2.dsc' }
let(:sha256_source) { 'b5a599e88e7cbdda3bde808160a21ba1dd1ec76b2ec8d4912aae769648d68362 864 sample_1.2.3~alpha2.tar.xz' }
let(:sha256s) { "#{sha256_dsc}\n#{sha256_source}" }
let(:fields) { { Files: md5s, 'Checksums-Sha1': sha1s, 'Checksums-Sha256': sha256s } }
let(:metadata) { { file_type: :changes, architecture: 'amd64', fields: fields } }
before do
allow_next_instance_of(::Packages::Debian::ExtractMetadataService) do |extract_metadata_service|
allow(extract_metadata_service).to receive(:execute).and_return(metadata)
end
end
context 'without Files field' do
let(:md5s) { nil }
it 'raise ArgumentError', :aggregate_failures do
expect { subject }.to raise_error(described_class::ExtractionError, "Files field is missing")
end
end
context 'without Checksums-Sha1 field' do
let(:sha1s) { nil }
it 'raise ArgumentError', :aggregate_failures do
expect { subject }.to raise_error(described_class::ExtractionError, "Checksums-Sha1 field is missing")
end
end
context 'without Checksums-Sha256 field' do
let(:sha256s) { nil }
it 'raise ArgumentError', :aggregate_failures do
expect { subject }.to raise_error(described_class::ExtractionError, "Checksums-Sha256 field is missing")
end
end
context 'with file in Checksums-Sha1 but not in Files' do
let(:md5_dsc) { '' }
it 'raise ArgumentError', :aggregate_failures do
expect { subject }.to raise_error(described_class::ExtractionError, "sample_1.2.3~alpha2.dsc is listed in Checksums-Sha1 but not in Files")
end
end
context 'with different size in Checksums-Sha1' do
let(:sha1_dsc) { '32ecbd674f0bfd310df68484d87752490685a8d6 42 sample_1.2.3~alpha2.dsc' }
it 'raise ArgumentError', :aggregate_failures do
expect { subject }.to raise_error(described_class::ExtractionError, "Size for sample_1.2.3~alpha2.dsc in Files and Checksums-Sha1 differ")
end
end
context 'with file in Checksums-Sha256 but not in Files' do
let(:md5_dsc) { '' }
let(:sha1_dsc) { '' }
it 'raise ArgumentError', :aggregate_failures do
expect { subject }.to raise_error(described_class::ExtractionError, "sample_1.2.3~alpha2.dsc is listed in Checksums-Sha256 but not in Files")
end
end
context 'with different size in Checksums-Sha256' do
let(:sha256_dsc) { '844f79825b7e8aaa191e514b58a81f9ac1e58e2180134b0c9512fa66d896d7ba 42 sample_1.2.3~alpha2.dsc' }
it 'raise ArgumentError', :aggregate_failures do
expect { subject }.to raise_error(described_class::ExtractionError, "Size for sample_1.2.3~alpha2.dsc in Files and Checksums-Sha256 differ")
end
end
context 'with file in Files but not in Checksums-Sha1' do
let(:sha1_dsc) { '' }
it 'raise ArgumentError', :aggregate_failures do
expect { subject }.to raise_error(described_class::ExtractionError, "Validation failed: Sha1sum can't be blank")
end
end
context 'with file in Files but not in Checksums-Sha256' do
let(:sha256_dsc) { '' }
it 'raise ArgumentError', :aggregate_failures do
expect { subject }.to raise_error(described_class::ExtractionError, "Validation failed: Sha256sum can't be blank")
end
end
context 'with invalid MD5' do
let(:md5_dsc) { '1234567890123456789012345678012 671 libs optional sample_1.2.3~alpha2.dsc' }
it 'raise ArgumentError', :aggregate_failures do
expect { subject }.to raise_error(described_class::ExtractionError, "Validation failed: Md5sum mismatch for sample_1.2.3~alpha2.dsc: 3b0817804f669e16cdefac583ad88f0e != 1234567890123456789012345678012")
end
end
context 'with invalid SHA1' do
let(:sha1_dsc) { '1234567890123456789012345678901234567890 671 sample_1.2.3~alpha2.dsc' }
it 'raise ArgumentError', :aggregate_failures do
expect { subject }.to raise_error(described_class::ExtractionError, "Validation failed: Sha1sum mismatch for sample_1.2.3~alpha2.dsc: 32ecbd674f0bfd310df68484d87752490685a8d6 != 1234567890123456789012345678901234567890")
end
end
context 'with invalid SHA256' do
let(:sha256_dsc) { '1234567890123456789012345678901234567890123456789012345678901234 671 sample_1.2.3~alpha2.dsc' }
it 'raise ArgumentError', :aggregate_failures do
expect { subject }.to raise_error(described_class::ExtractionError, "Validation failed: Sha256sum mismatch for sample_1.2.3~alpha2.dsc: 844f79825b7e8aaa191e514b58a81f9ac1e58e2180134b0c9512fa66d896d7ba != 1234567890123456789012345678901234567890123456789012345678901234")
end
end
end
context 'with missing package file' do
before do
incoming.package_files.first.destroy!
end
it 'raise ArgumentError' do
expect { subject }.to raise_error(described_class::ExtractionError, "sample_1.2.3~alpha2.tar.xz is listed in Files but was not uploaded")
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment