Commit 6eb8c57c authored by Ash McKenzie's avatar Ash McKenzie

Merge branch 'debian_parsers' into 'master'

Debian parsers

See merge request gitlab-org/gitlab!44029
parents b3eccfe6 a023336e
# frozen_string_literal: true
module Packages
module Debian
# Returns .deb file metadata
class ExtractDebMetadataService
CommandFailedError = Class.new(StandardError)
def initialize(file_path)
@file_path = file_path
end
def execute
unless success?
raise CommandFailedError, "The `#{cmd}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}"
end
sections = ParseDebian822Service.new(result.stdout).execute
sections.each_value.first
end
private
def cmd
@cmd ||= begin
dpkg_deb_path = Gitlab.config.packages.dpkg_deb_path
[dpkg_deb_path, '--field', @file_path]
end
end
def result
@result ||= Gitlab::Popen.popen_with_detail(cmd)
end
def success?
result.status&.exitstatus == 0
end
end
end
end
# frozen_string_literal: true
module Packages
module Debian
# Parse String as Debian RFC822 control data format
# https://manpages.debian.org/unstable/dpkg-dev/deb822.5
class ParseDebian822Service
InvalidDebian822Error = Class.new(StandardError)
def initialize(input)
@input = input
end
def execute
output = {}
@input.each_line('', chomp: true) do |block|
section = {}
section_name, field = nil
block.each_line(chomp: true) do |line|
next if comment_line?(line)
if continuation_line?(line)
raise InvalidDebian822Error, "Parse error. Unexpected continuation line" if field.nil?
section[field] += "\n"
section[field] += line[1..] unless paragraph_separator?(line)
elsif match = match_section_line(line)
section_name = match[:name] if section_name.nil?
field = match[:field].to_sym
raise InvalidDebian822Error, "Duplicate field '#{field}' in section '#{section_name}'" if section.include?(field)
section[field] = match[:value]
else
raise InvalidDebian822Error, "Parse error on line #{line}"
end
end
raise InvalidDebian822Error, "Duplicate section '#{section_name}'" if output[section_name]
output[section_name] = section
end
output
end
private
def comment_line?(line)
line.match?(/^#/)
end
def continuation_line?(line)
line.match?(/^ /)
end
def paragraph_separator?(line)
line == ' .'
end
def match_section_line(line)
line.match(/(?<name>(?<field>^\S+):\s*(?<value>.*))/)
end
end
end
end
---
title: Debian RFC822 and .deb metadata extractor
merge_request: 44029
author: Mathieu Parent
type: added
......@@ -317,6 +317,7 @@ production: &base
## Packages (maven repository, npm registry, etc...)
packages:
enabled: true
dpkg_deb_path: /usr/bin/dpkg-deb
# The location where build packages are stored (default: shared/packages).
# storage_path: shared/packages
object_store:
......
......@@ -355,6 +355,7 @@ Settings.uploads['object_store']['remote_directory'] ||= 'uploads'
#
Settings['packages'] ||= Settingslogic.new({})
Settings.packages['enabled'] = true if Settings.packages['enabled'].nil?
Settings.packages['dpkg_deb_path'] = '/usr/bin/dpkg-deb' if Settings.packages['dpkg_deb_path'].nil?
Settings.packages['storage_path'] = Settings.absolute(Settings.packages['storage_path'] || File.join(Settings.shared['path'], "packages"))
Settings.packages['object_store'] = ObjectStoreSettings.legacy_parse(Settings.packages['object_store'])
......
# Build a Debian package
Install the build dependencies:
```shell
sudo apt install dpkg-dev
```
Go to the `spec/fixtures/packages/debian` directory and clean up old files:
```shell
cd spec/fixtures/packages/debian
rm -v *.tar.* *.dsc *.deb *.udeb *.buildinfo *.changes
```
Go to the package source directory and build:
```shell
cd sample
dpkg-buildpackage --no-sign
```
.debhelper
debhelper-build-stamp
files
*.substvars
libsample0
sample-dev
sample-udeb
sample (1.2.3~alpha2) unstable; urgency=medium
* Initial release
-- John Doe <john.doe@example.com> Thu, 01 Oct 2020 09:35:15 +0200
Source: sample
Priority: optional
Maintainer: John Doe <john.doe@example.com>
Build-Depends: debhelper-compat (= 13)
Standards-Version: 4.5.0
Section: libs
Homepage: https://gitlab.com/
#Vcs-Browser: https://salsa.debian.org/debian/sample-1.2.3
#Vcs-Git: https://salsa.debian.org/debian/sample-1.2.3.git
Rules-Requires-Root: no
Package: sample-dev
Section: libdevel
Architecture: any
Multi-Arch: same
Depends: libsample0 (= ${binary:Version}), ${misc:Depends}
Description: Some mostly empty developpement files
Used in GitLab tests.
.
Testing another paragraph.
Package: libsample0
Architecture: any
Multi-Arch: same
Depends: ${shlibs:Depends}, ${misc:Depends}
Description: Some mostly empty lib
Used in GitLab tests.
.
Testing another paragraph.
Package: sample-udeb
Package-Type: udeb
Architecture: any
Depends: installed-base
Description: Some mostly empty udeb
#!/usr/bin/make -f
%:
dh $@
override_dh_gencontrol:
dh_gencontrol -psample-dev -- -v'1.2.3~binary'
dh_gencontrol --remaining-packages
Format: 3.0 (native)
Source: sample
Binary: sample-dev, libsample0, sample-udeb
Architecture: any
Version: 1.2.3~alpha2
Maintainer: John Doe <john.doe@example.com>
Homepage: https://gitlab.com/
Standards-Version: 4.5.0
Build-Depends: debhelper-compat (= 13)
Package-List:
libsample0 deb libs optional arch=any
sample-dev deb libdevel optional arch=any
sample-udeb udeb libs optional arch=any
Checksums-Sha1:
5f8bba5574eb01ac3b1f5e2988e8c29307788236 864 sample_1.2.3~alpha2.tar.xz
Checksums-Sha256:
b5a599e88e7cbdda3bde808160a21ba1dd1ec76b2ec8d4912aae769648d68362 864 sample_1.2.3~alpha2.tar.xz
Files:
d79b34f58f61ff4ad696d9bd0b8daa68 864 sample_1.2.3~alpha2.tar.xz
Format: 1.0
Source: sample
Binary: libsample0 sample-dev sample-udeb
Architecture: amd64 source
Version: 1.2.3~alpha2
Checksums-Md5:
3b0817804f669e16cdefac583ad88f0e 671 sample_1.2.3~alpha2.dsc
fb0842b21adc44207996296fe14439dd 1124 libsample0_1.2.3~alpha2_amd64.deb
d2afbd28e4d74430d22f9504e18bfdf5 1164 sample-dev_1.2.3~binary_amd64.deb
72b1dd7d98229e2fb0355feda1d3a165 736 sample-udeb_1.2.3~alpha2_amd64.udeb
Checksums-Sha1:
32ecbd674f0bfd310df68484d87752490685a8d6 671 sample_1.2.3~alpha2.dsc
5248b95600e85bfe7f63c0dfce330a75f5777366 1124 libsample0_1.2.3~alpha2_amd64.deb
f81e4f66c8c6bb899653a3340c157965ee69634f 1164 sample-dev_1.2.3~binary_amd64.deb
e42e8f2fe04ed1bb73b44a187674480d0e49dcba 736 sample-udeb_1.2.3~alpha2_amd64.udeb
Checksums-Sha256:
844f79825b7e8aaa191e514b58a81f9ac1e58e2180134b0c9512fa66d896d7ba 671 sample_1.2.3~alpha2.dsc
1c383a525bfcba619c7305ccd106d61db501a6bbaf0003bf8d0c429fbdb7fcc1 1124 libsample0_1.2.3~alpha2_amd64.deb
9fbeee2191ce4dab5288fad5ecac1bd369f58fef9a992a880eadf0caf25f086d 1164 sample-dev_1.2.3~binary_amd64.deb
2b0c152b3ab4cc07663350424de972c2b7621d69fe6df2e0b94308a191e4632f 736 sample-udeb_1.2.3~alpha2_amd64.udeb
Build-Origin: Debian
Build-Architecture: amd64
Build-Date: Thu, 08 Oct 2020 15:15:24 +0200
Build-Tainted-By:
merged-usr-via-symlinks
usr-local-has-includes
usr-local-has-libraries
usr-local-has-programs
Installed-Build-Depends:
autoconf (= 2.69-11.1),
automake (= 1:1.16.2-4),
autopoint (= 0.19.8.1-10),
autotools-dev (= 20180224.1),
base-files (= 11),
base-passwd (= 3.5.47),
bash (= 5.0-7),
binutils (= 2.35.1-1),
binutils-common (= 2.35.1-1),
binutils-x86-64-linux-gnu (= 2.35.1-1),
bsdextrautils (= 2.36-3+b1),
bsdmainutils (= 12.1.7),
bsdutils (= 1:2.36-3+b1),
build-essential (= 12.8),
bzip2 (= 1.0.8-4),
calendar (= 12.1.7),
coreutils (= 8.32-4+b1),
cpp (= 4:10.2.0-1),
cpp-10 (= 10.2.0-9),
cpp-9 (= 9.3.0-18),
dash (= 0.5.10.2-7),
debconf (= 1.5.74),
debhelper (= 13.2.1),
debianutils (= 4.11.2),
dh-autoreconf (= 19),
dh-strip-nondeterminism (= 1.9.0-1),
diffutils (= 1:3.7-3),
dpkg (= 1.20.5),
dpkg-dev (= 1.20.5),
dwz (= 0.13-5),
file (= 1:5.38-5),
findutils (= 4.7.0-1),
g++ (= 4:10.2.0-1),
g++-10 (= 10.2.0-9),
gcc (= 4:10.2.0-1),
gcc-10 (= 10.2.0-9),
gcc-10-base (= 10.2.0-9),
gcc-9 (= 9.3.0-18),
gcc-9-base (= 9.3.0-18),
gettext (= 0.19.8.1-10),
gettext-base (= 0.19.8.1-10),
grep (= 3.4-1),
groff-base (= 1.22.4-5),
gzip (= 1.10-2),
hostname (= 3.23),
init-system-helpers (= 1.58),
intltool-debian (= 0.35.0+20060710.5),
libacl1 (= 2.2.53-8),
libarchive-zip-perl (= 1.68-1),
libasan5 (= 9.3.0-18),
libasan6 (= 10.2.0-9),
libatomic1 (= 10.2.0-9),
libattr1 (= 1:2.4.48-5),
libaudit-common (= 1:2.8.5-3),
libaudit1 (= 1:2.8.5-3+b1),
libbinutils (= 2.35.1-1),
libblkid1 (= 2.36-3+b1),
libbsd0 (= 0.10.0-1),
libbz2-1.0 (= 1.0.8-4),
libc-bin (= 2.31-3),
libc-dev-bin (= 2.31-3),
libc6 (= 2.31-3),
libc6-dev (= 2.31-3),
libcap-ng0 (= 0.7.9-2.2),
libcc1-0 (= 10.2.0-9),
libcroco3 (= 0.6.13-1),
libcrypt-dev (= 1:4.4.17-1),
libcrypt1 (= 1:4.4.17-1),
libctf-nobfd0 (= 2.35.1-1),
libctf0 (= 2.35.1-1),
libdb5.3 (= 5.3.28+dfsg1-0.6),
libdebconfclient0 (= 0.254),
libdebhelper-perl (= 13.2.1),
libdpkg-perl (= 1.20.5),
libelf1 (= 0.181-1),
libffi7 (= 3.3-4),
libfile-stripnondeterminism-perl (= 1.9.0-1),
libgcc-10-dev (= 10.2.0-9),
libgcc-9-dev (= 9.3.0-18),
libgcc-s1 (= 10.2.0-9),
libgcrypt20 (= 1.8.6-2),
libgdbm-compat4 (= 1.18.1-5.1),
libgdbm6 (= 1.18.1-5.1),
libglib2.0-0 (= 2.66.0-2),
libgmp10 (= 2:6.2.0+dfsg-6),
libgomp1 (= 10.2.0-9),
libgpg-error0 (= 1.38-2),
libicu67 (= 67.1-4),
libisl22 (= 0.22.1-1),
libitm1 (= 10.2.0-9),
liblsan0 (= 10.2.0-9),
liblz4-1 (= 1.9.2-2),
liblzma5 (= 5.2.4-1+b1),
libmagic-mgc (= 1:5.38-5),
libmagic1 (= 1:5.38-5),
libmount1 (= 2.36-3+b1),
libmpc3 (= 1.2.0-1),
libmpfr6 (= 4.1.0-3),
libpam-modules (= 1.3.1-5),
libpam-modules-bin (= 1.3.1-5),
libpam-runtime (= 1.3.1-5),
libpam0g (= 1.3.1-5),
libpcre2-8-0 (= 10.34-7),
libpcre3 (= 2:8.39-13),
libperl5.30 (= 5.30.3-4),
libpipeline1 (= 1.5.3-1),
libquadmath0 (= 10.2.0-9),
libseccomp2 (= 2.4.4-1),
libselinux1 (= 3.1-2),
libsigsegv2 (= 2.12-2),
libsmartcols1 (= 2.36-3+b1),
libstdc++-10-dev (= 10.2.0-9),
libstdc++6 (= 10.2.0-9),
libsub-override-perl (= 0.09-2),
libsystemd0 (= 246.6-1),
libtinfo6 (= 6.2+20200918-1),
libtool (= 2.4.6-14),
libtsan0 (= 10.2.0-9),
libubsan1 (= 10.2.0-9),
libuchardet0 (= 0.0.7-1),
libudev1 (= 246.6-1),
libunistring2 (= 0.9.10-4),
libuuid1 (= 2.36-3+b1),
libxml2 (= 2.9.10+dfsg-6),
libzstd1 (= 1.4.5+dfsg-4),
linux-libc-dev (= 5.8.10-1),
login (= 1:4.8.1-1),
lsb-base (= 11.1.0),
m4 (= 1.4.18-4),
make (= 4.3-4),
man-db (= 2.9.3-2),
mawk (= 1.3.4.20200120-2),
ncal (= 12.1.7),
ncurses-base (= 6.2+20200918-1),
ncurses-bin (= 6.2+20200918-1),
patch (= 2.7.6-6),
perl (= 5.30.3-4),
perl-base (= 5.30.3-4),
perl-modules-5.30 (= 5.30.3-4),
po-debconf (= 1.0.21),
sed (= 4.7-1),
sensible-utils (= 0.0.12+nmu1),
sysvinit-utils (= 2.96-5),
tar (= 1.30+dfsg-7),
util-linux (= 2.36-3+b1),
xz-utils (= 5.2.4-1+b1),
zlib1g (= 1:1.2.11.dfsg-2)
Environment:
DEB_BUILD_OPTIONS="parallel=8"
LANG="fr_FR.UTF-8"
SOURCE_DATE_EPOCH="1601537715"
Format: 1.8
Date: Thu, 01 Oct 2020 09:35:15 +0200
Source: sample
Binary: libsample0 sample-dev sample-udeb
Architecture: source amd64
Version: 1.2.3~alpha2
Distribution: unstable
Urgency: medium
Maintainer: John Doe <john.doe@example.com>
Changed-By: John Doe <john.doe@example.com>
Description:
libsample0 - Some mostly empty lib
sample-dev - Some mostly empty developpement files
sample-udeb - Some mostly empty udeb (udeb)
Changes:
sample (1.2.3~alpha2) unstable; urgency=medium
.
* Initial release
Checksums-Sha1:
32ecbd674f0bfd310df68484d87752490685a8d6 671 sample_1.2.3~alpha2.dsc
5f8bba5574eb01ac3b1f5e2988e8c29307788236 864 sample_1.2.3~alpha2.tar.xz
5248b95600e85bfe7f63c0dfce330a75f5777366 1124 libsample0_1.2.3~alpha2_amd64.deb
f81e4f66c8c6bb899653a3340c157965ee69634f 1164 sample-dev_1.2.3~binary_amd64.deb
e42e8f2fe04ed1bb73b44a187674480d0e49dcba 736 sample-udeb_1.2.3~alpha2_amd64.udeb
0d47e899f3cc67a2253a4629456ff927e0db5c60 5280 sample_1.2.3~alpha2_amd64.buildinfo
Checksums-Sha256:
844f79825b7e8aaa191e514b58a81f9ac1e58e2180134b0c9512fa66d896d7ba 671 sample_1.2.3~alpha2.dsc
b5a599e88e7cbdda3bde808160a21ba1dd1ec76b2ec8d4912aae769648d68362 864 sample_1.2.3~alpha2.tar.xz
1c383a525bfcba619c7305ccd106d61db501a6bbaf0003bf8d0c429fbdb7fcc1 1124 libsample0_1.2.3~alpha2_amd64.deb
9fbeee2191ce4dab5288fad5ecac1bd369f58fef9a992a880eadf0caf25f086d 1164 sample-dev_1.2.3~binary_amd64.deb
2b0c152b3ab4cc07663350424de972c2b7621d69fe6df2e0b94308a191e4632f 736 sample-udeb_1.2.3~alpha2_amd64.udeb
f9900d3c94e94b329232668dcbef3dba2d96c07147b15b6dc0533452e4dd8a43 5280 sample_1.2.3~alpha2_amd64.buildinfo
Files:
3b0817804f669e16cdefac583ad88f0e 671 libs optional sample_1.2.3~alpha2.dsc
d79b34f58f61ff4ad696d9bd0b8daa68 864 libs optional sample_1.2.3~alpha2.tar.xz
fb0842b21adc44207996296fe14439dd 1124 libs optional libsample0_1.2.3~alpha2_amd64.deb
d2afbd28e4d74430d22f9504e18bfdf5 1164 libdevel optional sample-dev_1.2.3~binary_amd64.deb
72b1dd7d98229e2fb0355feda1d3a165 736 libs optional sample-udeb_1.2.3~alpha2_amd64.udeb
4e085dd67c120ca967ec314f65770a42 5280 libs optional sample_1.2.3~alpha2_amd64.buildinfo
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Packages::Debian::ExtractDebMetadataService do
subject { described_class.new(file_path) }
let(:file_name) { 'libsample0_1.2.3~alpha2_amd64.deb' }
let(:file_path) { "spec/fixtures/packages/debian/#{file_name}" }
context 'with correct file' do
it 'return as expected' do
expected = {
'Package': 'libsample0',
'Source': 'sample',
'Version': '1.2.3~alpha2',
'Architecture': 'amd64',
'Maintainer': 'John Doe <john.doe@example.com>',
'Installed-Size': '7',
'Section': 'libs',
'Priority': 'optional',
'Multi-Arch': 'same',
'Homepage': 'https://gitlab.com/',
'Description': "Some mostly empty lib\nUsed in GitLab tests.\n\nTesting another paragraph."
}
expect(subject.execute).to eq expected
end
end
context 'with incorrect file' do
let(:file_name) { 'README.md' }
it 'raise error' do
expect {subject.execute}.to raise_error(described_class::CommandFailedError, /is not a Debian format archive/i)
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Packages::Debian::ParseDebian822Service do
subject { described_class.new(input) }
context 'with dpkg-deb --field output' do
let(:input) do
<<~HEREDOC
Package: libsample0
Source: sample
Version: 1.2.3~alpha2
Architecture: amd64
Maintainer: John Doe <john.doe@example.com>
Installed-Size: 9
Section: libs
Priority: optional
Multi-Arch: same
Homepage: https://gitlab.com/
Description: Some mostly empty lib
Used in GitLab tests.
.
Testing another paragraph.
HEREDOC
end
it 'return as expected, preserving order' do
expected = {
'Package: libsample0' => {
'Package': 'libsample0',
'Source': 'sample',
'Version': '1.2.3~alpha2',
'Architecture': 'amd64',
'Maintainer': 'John Doe <john.doe@example.com>',
'Installed-Size': '9',
'Section': 'libs',
'Priority': 'optional',
'Multi-Arch': 'same',
'Homepage': 'https://gitlab.com/',
'Description': "Some mostly empty lib\nUsed in GitLab tests.\n\nTesting another paragraph."
}
}
expect(subject.execute.to_s).to eq(expected.to_s)
end
end
context 'with control file' do
let(:input) { fixture_file('packages/debian/sample/debian/control') }
it 'return as expected, preserving order' do
expected = {
'Source: sample' => {
'Source': 'sample',
'Priority': 'optional',
'Maintainer': 'John Doe <john.doe@example.com>',
'Build-Depends': 'debhelper-compat (= 13)',
'Standards-Version': '4.5.0',
'Section': 'libs',
'Homepage': 'https://gitlab.com/',
# 'Vcs-Browser': 'https://salsa.debian.org/debian/sample-1.2.3',
# '#Vcs-Git': 'https://salsa.debian.org/debian/sample-1.2.3.git',
'Rules-Requires-Root': 'no'
},
'Package: sample-dev' => {
'Package': 'sample-dev',
'Section': 'libdevel',
'Architecture': 'any',
'Multi-Arch': 'same',
'Depends': 'libsample0 (= ${binary:Version}), ${misc:Depends}',
'Description': "Some mostly empty developpement files\nUsed in GitLab tests.\n\nTesting another paragraph."
},
'Package: libsample0' => {
'Package': 'libsample0',
'Architecture': 'any',
'Multi-Arch': 'same',
'Depends': '${shlibs:Depends}, ${misc:Depends}',
'Description': "Some mostly empty lib\nUsed in GitLab tests.\n\nTesting another paragraph."
},
'Package: sample-udeb' => {
'Package': 'sample-udeb',
'Package-Type': 'udeb',
'Architecture': 'any',
'Depends': 'installed-base',
'Description': 'Some mostly empty udeb'
}
}
expect(subject.execute.to_s).to eq(expected.to_s)
end
end
context 'with empty input' do
let(:input) { '' }
it 'return a empty hash' do
expect(subject.execute).to eq({})
end
end
context 'with unexpected continuation line' do
let(:input) { ' continuation' }
it 'raise error' do
expect {subject.execute}.to raise_error(described_class::InvalidDebian822Error, 'Parse error. Unexpected continuation line')
end
end
context 'with duplicate field' do
let(:input) do
<<~HEREDOC
Package: libsample0
Source: sample
Source: sample
HEREDOC
end
it 'raise error' do
expect {subject.execute}.to raise_error(described_class::InvalidDebian822Error, "Duplicate field 'Source' in section 'Package: libsample0'")
end
end
context 'with incorrect input' do
let(:input) do
<<~HEREDOC
Hello
HEREDOC
end
it 'raise error' do
expect {subject.execute}.to raise_error(described_class::InvalidDebian822Error, 'Parse error on line Hello')
end
end
context 'with duplicate section' do
let(:input) do
<<~HEREDOC
Package: libsample0
Package: libsample0
HEREDOC
end
it 'raise error' do
expect {subject.execute}.to raise_error(described_class::InvalidDebian822Error, "Duplicate section 'Package: libsample0'")
end
end
end
......@@ -20,7 +20,7 @@ RSpec.shared_context 'Debian repository shared context' do |object_type|
let(:source_package) { 'sample' }
let(:letter) { source_package[0..2] == 'lib' ? source_package[0..3] : source_package[0] }
let(:package_name) { 'libsample0' }
let(:package_version) { '1.2.3~alpha2-1' }
let(:package_version) { '1.2.3~alpha2' }
let(:file_name) { "#{package_name}_#{package_version}_#{architecture}.deb" }
let(:method) { :get }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment