Merge branch 'debian_file_metadatum' into 'master'

Debian file metadatum

See merge request gitlab-org/gitlab!49692

app/models/packages/debian.rb

+# frozen_string_literal: true
+
+module Packages
+  module Debian
+    def self.table_name_prefix
+      'packages_debian_'
+    end
+  end
+end

app/models/packages/debian/file_metadatum.rb

+# frozen_string_literal: true
+
+class Packages::Debian::FileMetadatum < ApplicationRecord
+  belongs_to :package_file, inverse_of: :debian_file_metadatum
+
+  validates :package_file, presence: true
+  validate :valid_debian_package_type
+
+  enum file_type: {
+    unknown: 1, source: 2, dsc: 3, deb: 4, udeb: 5, buildinfo: 6, changes: 7
+  }
+
+  validates :file_type, presence: true
+  validates :file_type, inclusion: { in: %w[unknown] }, if: -> { package_file&.package&.debian_incoming? }
+  validates :file_type,
+    inclusion: { in: %w[source dsc deb udeb buildinfo changes] },
+    if: -> { package_file&.package&.debian_package? }
+
+  validates :component,
+    presence: true,
+    format: { with: Gitlab::Regex.debian_component_regex },
+    if: :requires_component?
+  validates :component, absence: true, unless: :requires_component?
+
+  validates :architecture,
+    presence: true,
+    format: { with: Gitlab::Regex.debian_architecture_regex },
+    if: :requires_architecture?
+  validates :architecture, absence: true, unless: :requires_architecture?
+
+  validates :fields,
+    presence: true,
+    json_schema: { filename: "debian_fields" },
+    if: :requires_fields?
+  validates :fields, absence: true, unless: :requires_fields?
+
+  private
+
+  def valid_debian_package_type
+    return if package_file&.package&.debian?
+
+    errors.add(:package_file, _('Package type must be Debian'))
+  end
+
+  def requires_architecture?
+    deb? || udeb?
+  end
+
+  def requires_component?
+    source? || dsc? || requires_architecture? || buildinfo?
+  end
+
+  def requires_fields?
+    dsc? || requires_architecture? || buildinfo? || changes?
+  end
+end

app/models/packages/package_file.rb

@@ -5,14 +5,17 @@ class Packages::PackageFile < ApplicationRecord
 
   delegate :project, :project_id, to: :package
   delegate :conan_file_type, to: :conan_file_metadatum
+  delegate :file_type, :architecture, :fields, to: :debian_file_metadatum, prefix: :debian
 
   belongs_to :package
 
   has_one :conan_file_metadatum, inverse_of: :package_file, class_name: 'Packages::Conan::FileMetadatum'
   has_many :package_file_build_infos, inverse_of: :package_file, class_name: 'Packages::PackageFileBuildInfo'
   has_many :pipelines, through: :package_file_build_infos
+  has_one :debian_file_metadatum, inverse_of: :package_file, class_name: 'Packages::Debian::FileMetadatum'
 
   accepts_nested_attributes_for :conan_file_metadatum
+  accepts_nested_attributes_for :debian_file_metadatum
 
   validates :package, presence: true
   validates :file, presence: true
@@ -25,12 +28,18 @@ class Packages::PackageFile < ApplicationRecord
   scope :with_file_name_like, ->(file_name) { where(arel_table[:file_name].matches(file_name)) }
   scope :with_files_stored_locally, -> { where(file_store: ::Packages::PackageFileUploader::Store::LOCAL) }
   scope :preload_conan_file_metadata, -> { preload(:conan_file_metadatum) }
+  scope :preload_debian_file_metadata, -> { preload(:debian_file_metadatum) }
 
   scope :with_conan_file_type, ->(file_type) do
     joins(:conan_file_metadatum)
       .where(packages_conan_file_metadata: { conan_file_type: ::Packages::Conan::FileMetadatum.conan_file_types[file_type] })
   end
 
+  scope :with_debian_file_type, ->(file_type) do
+    joins(:debian_file_metadatum)
+      .where(packages_debian_file_metadata: { debian_file_type: ::Packages::Debian::FileMetadatum.debian_file_types[file_type] })
+  end
+
   scope :with_conan_package_reference, ->(conan_package_reference) do
     joins(:conan_file_metadatum)
       .where(packages_conan_file_metadata: { conan_package_reference: conan_package_reference })

app/validators/json_schemas/debian_fields.json

+{
+  "description": "Debian fields",
+  "type": "object",
+  "patternProperties": {
+    ".*": {
+      "type": "string"
+    }
+  }
+}

changelogs/unreleased/debian_file_metadatum.yml

+---
+title: Debian File Metadata
+merge_request: 49692
+author: Mathieu Parent
+type: added

db/migrate/20201204105300_create_packages_debian_file_metadata.rb

+# frozen_string_literal: true
+
+class CreatePackagesDebianFileMetadata < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    unless table_exists?(:packages_debian_file_metadata)
+      create_table :packages_debian_file_metadata, id: false do |t|
+        t.timestamps_with_timezone
+        t.references :package_file, primary_key: true, index: false, default: nil, null: false, foreign_key: { to_table: :packages_package_files, on_delete: :cascade }, type: :bigint
+        t.integer :file_type, limit: 2, null: false
+        t.text :component
+        t.text :architecture
+        t.jsonb :fields
+      end
+    end
+
+    add_text_limit :packages_debian_file_metadata, :component, 255
+    add_text_limit :packages_debian_file_metadata, :architecture, 255
+  end
+
+  def down
+    drop_table :packages_debian_file_metadata
+  end
+end

db/schema_migrations/20201204105300

+15f48c654c08b58c90e46ce6e6413efa14d5a6e8299f100fc65f09f38190132a
\ No newline at end of file

db/structure.sql

@@ -14646,6 +14646,18 @@ CREATE SEQUENCE packages_conan_metadata_id_seq
 
 ALTER SEQUENCE packages_conan_metadata_id_seq OWNED BY packages_conan_metadata.id;
 
+CREATE TABLE packages_debian_file_metadata (
+    created_at timestamp with time zone NOT NULL,
+    updated_at timestamp with time zone NOT NULL,
+    package_file_id bigint NOT NULL,
+    file_type smallint NOT NULL,
+    component text,
+    architecture text,
+    fields jsonb,
+    CONSTRAINT check_2ebedda4b6 CHECK ((char_length(component) <= 255)),
+    CONSTRAINT check_e6e1fffcca CHECK ((char_length(architecture) <= 255))
+);
+
 CREATE TABLE packages_dependencies (
     id bigint NOT NULL,
     name character varying(255) NOT NULL,
@@ -19909,6 +19921,9 @@ ALTER TABLE ONLY packages_conan_file_metadata
 ALTER TABLE ONLY packages_conan_metadata
     ADD CONSTRAINT packages_conan_metadata_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY packages_debian_file_metadata
+    ADD CONSTRAINT packages_debian_file_metadata_pkey PRIMARY KEY (package_file_id);
+
 ALTER TABLE ONLY packages_dependencies
     ADD CONSTRAINT packages_dependencies_pkey PRIMARY KEY (id);
 
@@ -24147,6 +24162,9 @@ ALTER TABLE ONLY incident_management_oncall_schedules
 ALTER TABLE ONLY vulnerability_user_mentions
     ADD CONSTRAINT fk_rails_1a41c485cd FOREIGN KEY (vulnerability_id) REFERENCES vulnerabilities(id) ON DELETE CASCADE;
 
+ALTER TABLE ONLY packages_debian_file_metadata
+    ADD CONSTRAINT fk_rails_1ae85be112 FOREIGN KEY (package_file_id) REFERENCES packages_package_files(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY issuable_slas
     ADD CONSTRAINT fk_rails_1b8768cd63 FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE;
 

locale/gitlab.pot

@@ -19777,6 +19777,9 @@ msgstr ""
 msgid "Package type must be Conan"
 msgstr ""
 
+msgid "Package type must be Debian"
+msgstr ""
+
 msgid "Package type must be Maven"
 msgstr ""
 

spec/factories/packages.rb

@@ -26,9 +26,31 @@ FactoryBot.define do
       sequence(:version) { |n| "1.0-#{n}" }
       package_type { :debian }
 
+      transient do
+        without_package_files { false }
+        file_metadatum_trait { :keep }
+      end
+
+      after :create do |package, evaluator|
+        unless evaluator.without_package_files
+          create :debian_package_file, :source, evaluator.file_metadatum_trait, package: package
+          create :debian_package_file, :dsc, evaluator.file_metadatum_trait, package: package
+          create :debian_package_file, :deb, evaluator.file_metadatum_trait, package: package
+          create :debian_package_file, :deb2, evaluator.file_metadatum_trait, package: package
+          create :debian_package_file, :udeb, evaluator.file_metadatum_trait, package: package
+          create :debian_package_file, :buildinfo, evaluator.file_metadatum_trait, package: package
+          create :debian_package_file, :changes, evaluator.file_metadatum_trait, package: package
+        end
+      end
+
       factory :debian_incoming do
         name { 'incoming' }
         version { nil }
+
+        transient do
+          without_package_files { false }
+          file_metadatum_trait { :unknown }
+        end
       end
     end
 

spec/factories/packages/debian/file_metadatum.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :debian_file_metadatum, class: 'Packages::Debian::FileMetadatum' do
+    package_file { association(:debian_package_file, without_loaded_metadatum: true) }
+    file_type { 'deb' }
+    component { 'main' }
+    architecture { 'amd64' }
+    fields { { 'a': 'b' } }
+
+    trait(:unknown) do
+      file_type { 'unknown' }
+      component { nil }
+      architecture { nil }
+      fields { nil }
+    end
+
+    trait(:source) do
+      file_type { 'source' }
+      component { 'main' }
+      architecture { nil }
+      fields { nil }
+    end
+
+    trait(:dsc) do
+      file_type { 'dsc' }
+      component { 'main' }
+      architecture { nil }
+      fields { { 'a': 'b' } }
+    end
+
+    trait(:deb) do
+      file_type { 'deb' }
+      component { 'main' }
+      architecture { 'amd64' }
+      fields { { 'a': 'b' } }
+    end
+
+    trait(:udeb) do
+      file_type { 'udeb' }
+      component { 'main' }
+      architecture { 'amd64' }
+      fields { { 'a': 'b' } }
+    end
+
+    trait(:buildinfo) do
+      file_type { 'buildinfo' }
+      component { 'main' }
+      architecture { nil }
+      fields { { 'Architecture': 'amd64 source' } }
+    end
+
+    trait(:changes) do
+      file_type { 'changes' }
+      component { nil }
+      architecture { nil }
+      fields { { 'Architecture': 'source amd64' } }
+    end
+  end
+end

spec/factories/packages/package_file.rb

@@ -92,6 +92,97 @@ FactoryBot.define do
       end
     end
 
+    factory :debian_package_file do
+      package { association(:debian_package, without_package_files: true) }
+      file_name { 'libsample0_1.2.3~alpha2_amd64.deb' }
+      file_fixture { "spec/fixtures/packages/debian/#{file_name}" }
+      file_sha1 { 'be93151dc23ac34a82752444556fe79b32c7a1ad' }
+      file_md5 { '12345abcde' }
+      size { 400.kilobytes }
+
+      transient do
+        without_loaded_metadatum { false }
+        file_metadatum_trait { :deb }
+      end
+
+      after :create do |package_file, evaluator|
+        unless evaluator.without_loaded_metadatum
+          create :debian_file_metadatum, evaluator.file_metadatum_trait, package_file: package_file
+        end
+      end
+
+      trait(:unknown) do
+        package { association(:debian_incoming, without_package_files: true) }
+
+        transient do
+          file_metadatum_trait { :unknown }
+        end
+      end
+
+      trait(:invalid) do
+        file_name { 'README.md' }
+      end
+
+      trait(:source) do
+        file_name { 'sample_1.2.3~alpha2.tar.xz' }
+
+        transient do
+          file_metadatum_trait { :source }
+        end
+      end
+
+      trait(:dsc) do
+        file_name { 'sample_1.2.3~alpha2.dsc' }
+
+        transient do
+          file_metadatum_trait { :dsc }
+        end
+      end
+
+      trait(:deb) do
+        file_name { 'libsample0_1.2.3~alpha2_amd64.deb' }
+
+        transient do
+          file_metadatum_trait { :deb }
+        end
+      end
+
+      trait(:deb2) do
+        file_name { 'sample-dev_1.2.3~binary_amd64.deb' }
+
+        transient do
+          file_metadatum_trait { :deb }
+        end
+      end
+
+      trait(:udeb) do
+        file_name { 'sample-udeb_1.2.3~alpha2_amd64.udeb' }
+
+        transient do
+          file_metadatum_trait { :udeb }
+        end
+      end
+
+      trait(:buildinfo) do
+        file_name { 'sample_1.2.3~alpha2_amd64.buildinfo' }
+
+        transient do
+          file_metadatum_trait { :buildinfo }
+        end
+      end
+
+      trait(:changes) do
+        file_name { 'sample_1.2.3~alpha2_amd64.changes' }
+
+        transient do
+          file_metadatum_trait { :changes }
+        end
+      end
+
+      trait(:keep) do
+      end
+    end
+
     trait(:jar) do
       file_fixture { 'spec/fixtures/packages/maven/my-app-1.0-20180724.124855-1.jar' }
       file_name { 'my-app-1.0-20180724.124855-1.jar' }

spec/models/packages/debian/file_metadatum_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Packages::Debian::FileMetadatum, type: :model do
+  RSpec.shared_context 'Debian file metadatum' do |factory, trait|
+    let_it_be_with_reload(:debian_package_file) { create(factory, trait) }
+    let(:debian_file_metadatum) { debian_package_file.debian_file_metadatum }
+
+    subject { debian_file_metadatum }
+  end
+
+  RSpec.shared_examples 'Test Debian file metadatum' do |has_component, has_architecture, has_fields, has_outdated|
+    describe 'relationships' do
+      it { is_expected.to belong_to(:package_file) }
+    end
+
+    describe 'validations' do
+      describe '#package_file' do
+        it { is_expected.to validate_presence_of(:package_file) }
+      end
+
+      describe '#file_type' do
+        it { is_expected.to validate_presence_of(:file_type) }
+      end
+
+      describe '#component' do
+        it "has_component=#{has_component}" do
+          if has_component
+            is_expected.to validate_presence_of(:component)
+            is_expected.to allow_value('main').for(:component)
+            is_expected.not_to allow_value('hé').for(:component)
+          else
+            is_expected.to validate_absence_of(:component)
+          end
+        end
+      end
+
+      describe '#architecture' do
+        it "has_architecture=#{has_architecture}" do
+          if has_architecture
+            is_expected.to validate_presence_of(:architecture)
+            is_expected.to allow_value('amd64').for(:architecture)
+            is_expected.not_to allow_value('-a').for(:architecture)
+          else
+            is_expected.to validate_absence_of(:architecture)
+          end
+        end
+      end
+
+      describe '#fields' do
+        if has_fields
+          it { is_expected.to validate_presence_of(:fields) }
+          it { is_expected.to allow_value({ 'a': 'b' }).for(:fields) }
+          it { is_expected.not_to allow_value({ 'a': { 'b': 'c' } }).for(:fields) }
+        else
+          it { is_expected.to validate_absence_of(:fields) }
+        end
+      end
+
+      describe '#debian_package_type' do
+        before do
+          debian_package_file.package.package_type = :pypi
+        end
+
+        it 'validates package of type debian' do
+          expect(debian_file_metadatum).not_to be_valid
+          expect(debian_file_metadatum.errors.to_a).to contain_exactly('Package file Package type must be Debian')
+        end
+      end
+    end
+  end
+
+  using RSpec::Parameterized::TableSyntax
+
+  where(:factory, :trait, :has_component, :has_architecture, :has_fields) do
+    :debian_package_file      | :unknown   | false | false | false
+    :debian_package_file      | :source    | true  | false | false
+    :debian_package_file      | :dsc       | true  | false | true
+    :debian_package_file      | :deb       | true  | true  | true
+    :debian_package_file      | :udeb      | true  | true  | true
+    :debian_package_file      | :buildinfo | true  | false | true
+    :debian_package_file      | :changes   | false | false | true
+  end
+
+  with_them do
+    include_context 'Debian file metadatum', params[:factory], params[:trait] do
+      it_behaves_like 'Test Debian file metadatum', params[:has_component], params[:has_architecture], params[:has_fields], params[:has_outdated]
+    end
+  end
+end

spec/models/packages/package_file_spec.rb

@@ -7,6 +7,7 @@ RSpec.describe Packages::PackageFile, type: :model do
     it { is_expected.to have_one(:conan_file_metadatum) }
     it { is_expected.to have_many(:package_file_build_infos).inverse_of(:package_file) }
     it { is_expected.to have_many(:pipelines).through(:package_file_build_infos) }
+    it { is_expected.to have_one(:debian_file_metadatum).inverse_of(:package_file).class_name('Packages::Debian::FileMetadatum') }
   end
 
   describe 'validations' do