Merge branch 'debian_generate_distribution_service' into 'master'

Add Packages::Debian::GenerateDistributionService

See merge request gitlab-org/gitlab!53818

app/models/concerns/packages/debian/architecture.rb

@@ -23,6 +23,7 @@ module Packages
           uniqueness: { scope: %i[distribution_id] },
           format: { with: Gitlab::Regex.debian_architecture_regex }
 
+        scope :ordered_by_name, -> { order(:name) }
         scope :with_distribution, ->(distribution) { where(distribution: distribution) }
         scope :with_name, ->(name) { where(name: name) }
       end

app/models/concerns/packages/debian/component.rb

@@ -23,6 +23,7 @@ module Packages
           uniqueness: { scope: %i[distribution_id] },
           format: { with: Gitlab::Regex.debian_component_regex }
 
+        scope :ordered_by_name, -> { order(:name) }
         scope :with_distribution, ->(distribution) { where(distribution: distribution) }
         scope :with_name, ->(name) { where(name: name) }
       end

app/models/concerns/packages/debian/component_file.rb

@@ -60,6 +60,8 @@ module Packages
 
         scope :preload_distribution, -> { includes(component: :distribution) }
 
+        scope :created_before, ->(reference) { where("#{table_name}.created_at < ?", reference) }
+
         mount_file_store_uploader Packages::Debian::ComponentFileUploader
 
         before_validation :update_size_from_file

app/models/packages/debian/group_distribution.rb

@@ -6,4 +6,14 @@ class Packages::Debian::GroupDistribution < ApplicationRecord
   end
 
   include Packages::Debian::Distribution
+
+  def packages
+    Packages::Package
+      .for_projects(group.all_projects.public_only)
+      .with_debian_codename(codename)
+  end
+
+  def package_files
+    ::Packages::PackageFile.for_package_ids(packages.select(:id))
+  end
 end

app/models/packages/debian/project_distribution.rb

@@ -5,8 +5,9 @@ class Packages::Debian::ProjectDistribution < ApplicationRecord
     :project
   end
 
+  include Packages::Debian::Distribution
+
   has_many :publications, class_name: 'Packages::Debian::Publication', inverse_of: :distribution, foreign_key: :distribution_id
   has_many :packages, class_name: 'Packages::Package', through: :publications
-
-  include Packages::Debian::Distribution
+  has_many :package_files, class_name: 'Packages::PackageFile', through: :packages
 end

app/models/packages/package_file.rb

@@ -5,7 +5,7 @@ class Packages::PackageFile < ApplicationRecord
 
   delegate :project, :project_id, to: :package
   delegate :conan_file_type, to: :conan_file_metadatum
-  delegate :file_type, :architecture, :fields, to: :debian_file_metadatum, prefix: :debian
+  delegate :file_type, :component, :architecture, :fields, to: :debian_file_metadatum, prefix: :debian
   delegate :channel, :metadata, to: :helm_file_metadatum, prefix: :helm
 
   belongs_to :package
@@ -27,6 +27,7 @@ class Packages::PackageFile < ApplicationRecord
   validates :file_name, uniqueness: { scope: :package }, if: -> { package&.pypi? }
 
   scope :recent, -> { order(id: :desc) }
+  scope :for_package_ids, ->(ids) { where(package_id: ids) }
   scope :with_file_name, ->(file_name) { where(file_name: file_name) }
   scope :with_file_name_like, ->(file_name) { where(arel_table[:file_name].matches(file_name)) }
   scope :with_files_stored_locally, -> { where(file_store: ::Packages::PackageFileUploader::Store::LOCAL) }
@@ -44,7 +45,17 @@ class Packages::PackageFile < ApplicationRecord
 
   scope :with_debian_file_type, ->(file_type) do
     joins(:debian_file_metadatum)
-      .where(packages_debian_file_metadata: { debian_file_type: ::Packages::Debian::FileMetadatum.debian_file_types[file_type] })
+      .where(packages_debian_file_metadata: { file_type: ::Packages::Debian::FileMetadatum.file_types[file_type] })
+  end
+
+  scope :with_debian_component_name, ->(component_name) do
+    joins(:debian_file_metadatum)
+      .where(packages_debian_file_metadata: { component: component_name })
+  end
+
+  scope :with_debian_architecture_name, ->(architecture_name) do
+    joins(:debian_file_metadatum)
+      .where(packages_debian_file_metadata: { architecture: architecture_name })
   end
 
   scope :with_conan_package_reference, ->(conan_package_reference) do

app/services/packages/debian/generate_distribution_service.rb

+# frozen_string_literal: true
+
+module Packages
+  module Debian
+    class GenerateDistributionService
+      include Gitlab::Utils::StrongMemoize
+      include ExclusiveLeaseGuard
+
+      # used by ExclusiveLeaseGuard
+      DEFAULT_LEASE_TIMEOUT = 1.hour.to_i.freeze
+
+      # From https://salsa.debian.org/ftp-team/dak/-/blob/991aaa27a7f7aa773bb9c0cf2d516e383d9cffa0/setup/core-init.d/080_metadatakeys#L9
+      BINARIES_METADATA = %w(
+        Package
+        Source
+        Binary
+        Version
+        Essential
+        Installed-Size
+        Maintainer
+        Uploaders
+        Original-Maintainer
+        Build-Depends
+        Build-Depends-Indep
+        Build-Conflicts
+        Build-Conflicts-Indep
+        Architecture
+        Standards-Version
+        Format
+        Files
+        Dm-Upload-Allowed
+        Vcs-Browse
+        Vcs-Hg
+        Vcs-Darcs
+        Vcs-Svn
+        Vcs-Git
+        Vcs-Browser
+        Vcs-Arch
+        Vcs-Bzr
+        Vcs-Mtn
+        Vcs-Cvs
+        Checksums-Sha256
+        Checksums-Sha1
+        Replaces
+        Provides
+        Depends
+        Pre-Depends
+        Recommends
+        Suggests
+        Enhances
+        Conflicts
+        Breaks
+        Description
+        Origin
+        Bugs
+        Multi-Arch
+        Homepage
+        Tag
+        Package-Type
+        Installer-Menu-Item
+      ).freeze
+
+      def initialize(distribution)
+        @distribution = distribution
+        @last_generated_at = nil
+        @md5sum = []
+        @sha256 = []
+      end
+
+      def execute
+        try_obtain_lease do
+          @distribution.transaction do
+            @last_generated_at = @distribution.component_files.maximum(:created_at)
+            generate_component_files
+            generate_release
+            destroy_old_component_files
+          end
+        end
+      end
+
+      private
+
+      def generate_component_files
+        @distribution.components.ordered_by_name.each do |component|
+          @distribution.architectures.ordered_by_name.each do |architecture|
+            generate_component_file(component, :packages, architecture, :deb)
+          end
+        end
+      end
+
+      def generate_component_file(component, component_file_type, architecture, package_file_type)
+        paragraphs = @distribution.package_files
+                                  .preload_debian_file_metadata
+                                  .with_debian_component_name(component.name)
+                                  .with_debian_architecture_name(architecture.name)
+                                  .with_debian_file_type(package_file_type)
+                                  .find_each
+                                  .map(&method(:package_stanza_from_fields))
+        create_component_file(component, component_file_type, architecture, package_file_type, paragraphs.join("\n"))
+      end
+
+      def package_stanza_from_fields(package_file)
+        [
+          BINARIES_METADATA.map do |metadata_key|
+            rfc822_field(metadata_key, package_file.debian_fields[metadata_key])
+          end,
+          rfc822_field('Section', package_file.debian_fields['Section'] || 'misc'),
+          rfc822_field('Priority', package_file.debian_fields['Priority'] || 'extra'),
+          rfc822_field('Filename', package_filename(package_file)),
+          rfc822_field('Size', package_file.size),
+          rfc822_field('MD5sum', package_file.file_md5),
+          rfc822_field('SHA256', package_file.file_sha256)
+        ].flatten.compact.join('')
+      end
+
+      def package_filename(package_file)
+        letter = package_file.package.name.start_with?('lib') ? package_file.package.name[0..3] : package_file.package.name[0]
+        "#{pool_prefix(package_file)}/#{letter}/#{package_file.package.name}/#{package_file.file_name}"
+      end
+
+      def pool_prefix(package_file)
+        case @distribution
+        when ::Packages::Debian::GroupDistribution
+          "pool/#{@distribution.codename}/#{package_file.package.project_id}"
+        else
+          "pool/#{@distribution.codename}/#{@distribution.container_id}"
+        end
+      end
+
+      def create_component_file(component, component_file_type, architecture, package_file_type, content)
+        component_file = component.files.create!(
+          file_type: component_file_type,
+          architecture: architecture,
+          compression_type: nil,
+          file: CarrierWaveStringFile.new(content),
+          file_md5: Digest::MD5.hexdigest(content),
+          file_sha256: Digest::SHA256.hexdigest(content)
+        )
+        @md5sum.append(" #{component_file.file_md5} #{component_file.size.to_s.rjust(8)} #{component_file.relative_path}")
+        @sha256.append(" #{component_file.file_sha256} #{component_file.size.to_s.rjust(8)} #{component_file.relative_path}")
+      end
+
+      def generate_release
+        @distribution.file = CarrierWaveStringFile.new(release_header + release_sums)
+        @distribution.updated_at = release_date
+        @distribution.save!
+      end
+
+      def release_header
+        strong_memoize(:release_header) do
+          [
+            %w[origin label suite version codename].map do |attribute|
+              rfc822_field(attribute.capitalize, @distribution.attributes[attribute])
+            end,
+            rfc822_field('Date', release_date.to_formatted_s(:rfc822)),
+            valid_until_field,
+            rfc822_field('NotAutomatic', !@distribution.automatic, !@distribution.automatic),
+            rfc822_field('ButAutomaticUpgrades', @distribution.automatic_upgrades, !@distribution.automatic && @distribution.automatic_upgrades),
+            rfc822_field('Architectures', @distribution.architectures.map { |architecture| architecture.name }.sort.join(' ')),
+            rfc822_field('Components', @distribution.components.map { |component| component.name }.sort.join(' ')),
+            rfc822_field('Description', @distribution.description)
+          ].flatten.compact.join('')
+        end
+      end
+
+      def release_date
+        strong_memoize(:release_date) do
+          Time.now.utc
+        end
+      end
+
+      def release_sums
+        ["MD5Sum:", @md5sum, "SHA256:", @sha256].flatten.compact.join("\n") + "\n"
+      end
+
+      def rfc822_field(name, value, condition = true)
+        return unless condition
+        return if value.blank?
+
+        "#{name}: #{value.to_s.gsub("\n\n", "\n.\n").gsub("\n", "\n ")}\n"
+      end
+
+      def valid_until_field
+        return unless @distribution.valid_time_duration_seconds
+
+        rfc822_field('Valid-Until', release_date.since(@distribution.valid_time_duration_seconds).to_formatted_s(:rfc822))
+      end
+
+      def destroy_old_component_files
+        # Only keep the last generation and one hour before
+        return if @last_generated_at.nil?
+
+        @distribution.component_files.created_before(@last_generated_at - 1.hour).destroy_all # rubocop:disable Cop/DestroyAll
+      end
+
+      # used by ExclusiveLeaseGuard
+      def lease_key
+        "packages:debian:generate_distribution_service:distribution:#{@distribution.id}"
+      end
+
+      # used by ExclusiveLeaseGuard
+      def lease_timeout
+        DEFAULT_LEASE_TIMEOUT
+      end
+    end
+  end
+end

spec/models/packages/package_file_spec.rb

@@ -2,6 +2,11 @@
 require 'spec_helper'
 
 RSpec.describe Packages::PackageFile, type: :model do
+  let_it_be(:project) { create(:project) }
+  let_it_be(:package_file1) { create(:package_file, :xml, file_name: 'FooBar') }
+  let_it_be(:package_file2) { create(:package_file, :xml, file_name: 'ThisIsATest') }
+  let_it_be(:debian_package) { create(:debian_package, project: project) }
+
   describe 'relationships' do
     it { is_expected.to belong_to(:package) }
     it { is_expected.to have_one(:conan_file_metadatum) }
@@ -16,9 +21,6 @@ RSpec.describe Packages::PackageFile, type: :model do
   end
 
   context 'with package filenames' do
-    let_it_be(:package_file1) { create(:package_file, :xml, file_name: 'FooBar') }
-    let_it_be(:package_file2) { create(:package_file, :xml, file_name: 'ThisIsATest') }
-
     describe '.with_file_name' do
       let(:filename) { 'FooBar' }
 
@@ -52,6 +54,13 @@ RSpec.describe Packages::PackageFile, type: :model do
     end
   end
 
+  describe '.for_package_ids' do
+    it 'returns matching packages' do
+      expect(described_class.for_package_ids([package_file1.package.id, package_file2.package.id]))
+        .to contain_exactly(package_file1, package_file2)
+    end
+  end
+
   describe '.with_conan_package_reference' do
     let_it_be(:non_matching_package_file) { create(:package_file, :nuget) }
     let_it_be(:metadatum) { create(:conan_file_metadatum, :package_file) }
@@ -64,7 +73,6 @@ RSpec.describe Packages::PackageFile, type: :model do
   end
 
   describe '.for_rubygem_with_file_name' do
-    let_it_be(:project) { create(:project) }
     let_it_be(:non_ruby_package) { create(:nuget_package, project: project, package_type: :nuget) }
     let_it_be(:ruby_package) { create(:rubygems_package, project: project, package_type: :rubygems) }
     let_it_be(:file_name) { 'other.gem' }
@@ -78,6 +86,36 @@ RSpec.describe Packages::PackageFile, type: :model do
     end
   end
 
+  context 'Debian scopes' do
+    let_it_be(:debian_changes) { debian_package.package_files.last }
+    let_it_be(:debian_deb) { create(:debian_package_file, package: debian_package)}
+    let_it_be(:debian_udeb) { create(:debian_package_file, :udeb, package: debian_package)}
+
+    let_it_be(:debian_contrib) do
+      create(:debian_package_file, package: debian_package).tap do |pf|
+        pf.debian_file_metadatum.update!(component: 'contrib')
+      end
+    end
+
+    let_it_be(:debian_mipsel) do
+      create(:debian_package_file, package: debian_package).tap do |pf|
+        pf.debian_file_metadatum.update!(architecture: 'mipsel')
+      end
+    end
+
+    describe '#with_debian_file_type' do
+      it { expect(described_class.with_debian_file_type(:changes)).to contain_exactly(debian_changes) }
+    end
+
+    describe '#with_debian_component_name' do
+      it { expect(described_class.with_debian_component_name('contrib')).to contain_exactly(debian_contrib) }
+    end
+
+    describe '#with_debian_architecture_name' do
+      it { expect(described_class.with_debian_architecture_name('mipsel')).to contain_exactly(debian_mipsel) }
+    end
+  end
+
   describe '#update_file_store callback' do
     let_it_be(:package_file) { build(:package_file, :nuget, size: nil) }
 

spec/services/packages/debian/generate_distribution_service_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+RSpec.describe Packages::Debian::GenerateDistributionService do
+  let_it_be(:group) { create(:group, :public) }
+  let_it_be(:project) { create(:project, :public, group: group) }
+  let_it_be(:project_distribution) { create("debian_project_distribution", container: project, codename: 'unstable', valid_time_duration_seconds: 48.hours.to_i) }
+
+  let_it_be(:incoming) { create(:debian_incoming, project: project) }
+
+  before_all do
+    ::Packages::Debian::ProcessChangesService.new(incoming.package_files.last, nil).execute
+  end
+
+  let(:service) { described_class.new(distribution) }
+
+  describe '#execute' do
+    subject { service.execute }
+
+    shared_examples 'Generate Distribution' do |container_type|
+      context "for #{container_type}" do
+        if container_type == :group
+          let_it_be(:container) { group }
+          let_it_be(:distribution, reload: true) { create('debian_group_distribution', container: group, codename: 'unstable', valid_time_duration_seconds: 48.hours.to_i) }
+        else
+          let_it_be(:container) { project }
+          let_it_be(:distribution, reload: true) { project_distribution }
+        end
+
+        context 'with components and architectures' do
+          let_it_be(:component_main   ) { create("debian_#{container_type}_component", distribution: distribution, name: 'main') }
+          let_it_be(:component_contrib) { create("debian_#{container_type}_component", distribution: distribution, name: 'contrib') }
+
+          let_it_be(:architecture_all  ) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'all') }
+          let_it_be(:architecture_amd64) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'amd64') }
+          let_it_be(:architecture_arm64) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'arm64') }
+
+          let_it_be(:component_file1) { create("debian_#{container_type}_component_file", component: component_main,    architecture: architecture_all,   created_at: '2020-01-24T09:00:00.000Z') } # destroyed
+          let_it_be(:component_file2) { create("debian_#{container_type}_component_file", component: component_main,    architecture: architecture_amd64, created_at: '2020-01-24T10:29:59.000Z') } # destroyed
+          let_it_be(:component_file3) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_all,   created_at: '2020-01-24T10:30:00.000Z') } # kept
+          let_it_be(:component_file4) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_amd64, created_at: '2020-01-24T11:30:00.000Z') } # kept
+
+          def check_component_file(component_name, component_file_type, architecture_name, expected_content)
+            component_file = distribution
+              .component_files
+              .with_component_name(component_name)
+              .with_file_type(component_file_type)
+              .with_architecture_name(architecture_name)
+              .last
+
+            expect(component_file).not_to be_nil
+            expect(component_file.file.exists?).to eq(!expected_content.nil?)
+
+            unless expected_content.nil?
+              component_file.file.use_file do |file_path|
+                expect(File.read(file_path)).to eq(expected_content)
+              end
+            end
+          end
+
+          it 'updates distribution and component files', :aggregate_failures do
+            travel_to(Time.utc(2020, 01, 25, 15, 17, 18, 123456)) do
+              expect(Gitlab::ErrorTracking).not_to receive(:log_exception)
+
+              expect { subject }
+                .to not_change { Packages::Package.count }
+                .and not_change { Packages::PackageFile.count }
+                .and change { distribution.component_files.count }.from(4).to(2 + 6)
+
+              expected_main_amd64_content = <<~EOF
+              Package: libsample0
+              Source: sample
+              Version: 1.2.3~alpha2
+              Installed-Size: 7
+              Maintainer: John Doe <john.doe@example.com>
+              Architecture: amd64
+              Description: Some mostly empty lib
+               Used in GitLab tests.
+               .
+               Testing another paragraph.
+              Multi-Arch: same
+              Homepage: https://gitlab.com/
+              Section: libs
+              Priority: optional
+              Filename: pool/unstable/#{project.id}/s/sample/libsample0_1.2.3~alpha2_amd64.deb
+              Size: 409600
+              MD5sum: fb0842b21adc44207996296fe14439dd
+              SHA256: 1c383a525bfcba619c7305ccd106d61db501a6bbaf0003bf8d0c429fbdb7fcc1
+
+              Package: sample-dev
+              Source: sample (1.2.3~alpha2)
+              Version: 1.2.3~binary
+              Installed-Size: 7
+              Maintainer: John Doe <john.doe@example.com>
+              Architecture: amd64
+              Depends: libsample0 (= 1.2.3~binary)
+              Description: Some mostly empty developpement files
+               Used in GitLab tests.
+               .
+               Testing another paragraph.
+              Multi-Arch: same
+              Homepage: https://gitlab.com/
+              Section: libdevel
+              Priority: optional
+              Filename: pool/unstable/#{project.id}/s/sample/sample-dev_1.2.3~binary_amd64.deb
+              Size: 409600
+              MD5sum: d2afbd28e4d74430d22f9504e18bfdf5
+              SHA256: 9fbeee2191ce4dab5288fad5ecac1bd369f58fef9a992a880eadf0caf25f086d
+              EOF
+
+              check_component_file('main', :packages, 'all', nil)
+              check_component_file('main', :packages, 'amd64', expected_main_amd64_content)
+              check_component_file('main', :packages, 'arm64', nil)
+
+              check_component_file('contrib', :packages, 'all', nil)
+              check_component_file('contrib', :packages, 'amd64', nil)
+              check_component_file('contrib', :packages, 'arm64', nil)
+
+              size = expected_main_amd64_content.length
+              md5sum = Digest::MD5.hexdigest(expected_main_amd64_content)
+              sha256 = Digest::SHA256.hexdigest(expected_main_amd64_content)
+
+              expected_release_content = <<~EOF
+              Codename: unstable
+              Date: Sat, 25 Jan 2020 15:17:18 +0000
+              Valid-Until: Mon, 27 Jan 2020 15:17:18 +0000
+              Architectures: all amd64 arm64
+              Components: contrib main
+              MD5Sum:
+               d41d8cd98f00b204e9800998ecf8427e        0 contrib/binary-all/Packages
+               d41d8cd98f00b204e9800998ecf8427e        0 contrib/binary-amd64/Packages
+               d41d8cd98f00b204e9800998ecf8427e        0 contrib/binary-arm64/Packages
+               d41d8cd98f00b204e9800998ecf8427e        0 main/binary-all/Packages
+               #{md5sum}     #{size} main/binary-amd64/Packages
+               d41d8cd98f00b204e9800998ecf8427e        0 main/binary-arm64/Packages
+              SHA256:
+               e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/binary-all/Packages
+               e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/binary-amd64/Packages
+               e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/binary-arm64/Packages
+               e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 main/binary-all/Packages
+               #{sha256}     #{size} main/binary-amd64/Packages
+               e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 main/binary-arm64/Packages
+              EOF
+
+              distribution.file.use_file do |file_path|
+                expect(File.read(file_path)).to eq(expected_release_content)
+              end
+            end
+          end
+        end
+
+        context 'without components and architectures' do
+          it 'updates distribution and component files', :aggregate_failures do
+            travel_to(Time.utc(2020, 01, 25, 15, 17, 18, 123456)) do
+              expect(Gitlab::ErrorTracking).not_to receive(:log_exception)
+
+              expect { subject }
+                .to not_change { Packages::Package.count }
+                .and not_change { Packages::PackageFile.count }
+                .and not_change { distribution.component_files.count }
+
+              expected_release_content = <<~EOF
+              Codename: unstable
+              Date: Sat, 25 Jan 2020 15:17:18 +0000
+              Valid-Until: Mon, 27 Jan 2020 15:17:18 +0000
+              MD5Sum:
+              SHA256:
+              EOF
+
+              distribution.file.use_file do |file_path|
+                expect(File.read(file_path)).to eq(expected_release_content)
+              end
+            end
+          end
+        end
+      end
+    end
+
+    it_behaves_like 'Generate Distribution', :project
+    it_behaves_like 'Generate Distribution', :group
+  end
+end

spec/support/shared_examples/models/packages/debian/architecture_shared_examples.rb

@@ -3,8 +3,8 @@
 require 'spec_helper'
 
 RSpec.shared_examples 'Debian Distribution Architecture' do |factory, container, can_freeze|
-  let_it_be_with_refind(:architecture) { create(factory) } # rubocop:disable Rails/SaveBang
-  let_it_be(:architecture_same_distribution, freeze: can_freeze) { create(factory, distribution: architecture.distribution) }
+  let_it_be_with_refind(:architecture) { create(factory, name: 'name1') }
+  let_it_be(:architecture_same_distribution, freeze: can_freeze) { create(factory, distribution: architecture.distribution, name: 'name2') }
   let_it_be(:architecture_same_name, freeze: can_freeze) { create(factory, name: architecture.name) }
 
   subject { architecture }
@@ -30,20 +30,22 @@ RSpec.shared_examples 'Debian Distribution Architecture' do |factory, container,
   end
 
   describe 'scopes' do
+    describe '.ordered_by_name' do
+      subject { described_class.with_distribution(architecture.distribution).ordered_by_name }
+
+      it { expect(subject).to match_array([architecture, architecture_same_distribution]) }
+    end
+
     describe '.with_distribution' do
       subject { described_class.with_distribution(architecture.distribution) }
 
-      it 'does not return other distributions' do
-        expect(subject.to_a).to match_array([architecture, architecture_same_distribution])
-      end
+      it { expect(subject).to match_array([architecture, architecture_same_distribution]) }
     end
 
     describe '.with_name' do
       subject { described_class.with_name(architecture.name) }
 
-      it 'does not return other distributions' do
-        expect(subject.to_a).to match_array([architecture, architecture_same_name])
-      end
+      it { expect(subject).to match_array([architecture, architecture_same_name]) }
     end
   end
 end

spec/support/shared_examples/models/packages/debian/component_file_shared_example.rb

@@ -114,11 +114,7 @@ RSpec.shared_examples 'Debian Component File' do |container_type, can_freeze|
       subject { described_class.with_container(container2) }
 
       it do
-        queries = ActiveRecord::QueryRecorder.new do
-          expect(subject.to_a).to contain_exactly(component_file_other_container)
-        end
-
-        expect(queries.count).to eq(1)
+        expect(subject.to_a).to contain_exactly(component_file_other_container)
       end
     end
 
@@ -126,11 +122,7 @@ RSpec.shared_examples 'Debian Component File' do |container_type, can_freeze|
       subject { described_class.with_codename_or_suite(distribution2.codename) }
 
       it do
-        queries = ActiveRecord::QueryRecorder.new do
-          expect(subject.to_a).to contain_exactly(component_file_other_container)
-        end
-
-        expect(queries.count).to eq(1)
+        expect(subject.to_a).to contain_exactly(component_file_other_container)
       end
     end
 
@@ -138,11 +130,7 @@ RSpec.shared_examples 'Debian Component File' do |container_type, can_freeze|
       subject { described_class.with_component_name(component1_2.name) }
 
       it do
-        queries = ActiveRecord::QueryRecorder.new do
-          expect(subject.to_a).to contain_exactly(component_file_other_component)
-        end
-
-        expect(queries.count).to eq(1)
+        expect(subject.to_a).to contain_exactly(component_file_other_component)
       end
     end
 
@@ -150,14 +138,7 @@ RSpec.shared_examples 'Debian Component File' do |container_type, can_freeze|
       subject { described_class.with_file_type(:source) }
 
       it do
-        # let_it_be_with_refind triggers a query
-        component_file_with_file_type_source
-
-        queries = ActiveRecord::QueryRecorder.new do
-          expect(subject.to_a).to contain_exactly(component_file_with_file_type_source)
-        end
-
-        expect(queries.count).to eq(1)
+        expect(subject.to_a).to contain_exactly(component_file_with_file_type_source)
       end
     end
 
@@ -165,11 +146,7 @@ RSpec.shared_examples 'Debian Component File' do |container_type, can_freeze|
       subject { described_class.with_architecture_name(architecture1_2.name) }
 
       it do
-        queries = ActiveRecord::QueryRecorder.new do
-          expect(subject.to_a).to contain_exactly(component_file_other_architecture)
-        end
-
-        expect(queries.count).to eq(1)
+        expect(subject.to_a).to contain_exactly(component_file_other_architecture)
       end
     end
 
@@ -177,11 +154,7 @@ RSpec.shared_examples 'Debian Component File' do |container_type, can_freeze|
       subject { described_class.with_compression_type(:xz) }
 
       it do
-        queries = ActiveRecord::QueryRecorder.new do
-          expect(subject.to_a).to contain_exactly(component_file_other_compression_type)
-        end
-
-        expect(queries.count).to eq(1)
+        expect(subject.to_a).to contain_exactly(component_file_other_compression_type)
       end
     end
 
@@ -189,11 +162,19 @@ RSpec.shared_examples 'Debian Component File' do |container_type, can_freeze|
       subject { described_class.with_file_sha256('other_sha256') }
 
       it do
-        queries = ActiveRecord::QueryRecorder.new do
-          expect(subject.to_a).to contain_exactly(component_file_other_file_sha256)
-        end
+        expect(subject.to_a).to contain_exactly(component_file_other_file_sha256)
+      end
+    end
+
+    describe '.created_before' do
+      let_it_be(:component_file1) { create("debian_#{container_type}_component_file", component: component1_1, architecture: architecture1_1, created_at: 4.hours.ago) }
+      let_it_be(:component_file2) { create("debian_#{container_type}_component_file", component: component1_1, architecture: architecture1_1, created_at: 3.hours.ago) }
+      let_it_be(:component_file3) { create("debian_#{container_type}_component_file", component: component1_1, architecture: architecture1_1, created_at: 1.hour.ago) }
 
-        expect(queries.count).to eq(1)
+      subject { described_class.created_before(2.hours.ago) }
+
+      it do
+        expect(subject.to_a).to contain_exactly(component_file1, component_file2)
       end
     end
   end

spec/support/shared_examples/models/packages/debian/component_shared_examples.rb

@@ -3,8 +3,8 @@
 require 'spec_helper'
 
 RSpec.shared_examples 'Debian Distribution Component' do |factory, container, can_freeze|
-  let_it_be_with_refind(:component) { create(factory) } # rubocop:disable Rails/SaveBang
-  let_it_be(:component_same_distribution, freeze: can_freeze) { create(factory, distribution: component.distribution) }
+  let_it_be_with_refind(:component) { create(factory, name: 'name1') }
+  let_it_be(:component_same_distribution, freeze: can_freeze) { create(factory, distribution: component.distribution, name: 'name2') }
   let_it_be(:component_same_name, freeze: can_freeze) { create(factory, name: component.name) }
 
   subject { component }
@@ -32,6 +32,14 @@ RSpec.shared_examples 'Debian Distribution Component' do |factory, container, ca
   end
 
   describe 'scopes' do
+    describe '.ordered_by_name' do
+      subject { described_class.with_distribution(component.distribution).ordered_by_name }
+
+      it 'sorts by name' do
+        expect(subject.to_a).to eq([component, component_same_distribution])
+      end
+    end
+
     describe '.with_distribution' do
       subject { described_class.with_distribution(component.distribution) }
 

spec/support/shared_examples/models/packages/debian/distribution_shared_examples.rb

@@ -19,11 +19,6 @@ RSpec.shared_examples 'Debian Distribution' do |factory, container, can_freeze|
 
     it { is_expected.to have_many(:components).class_name("Packages::Debian::#{container.capitalize}Component").inverse_of(:distribution) }
     it { is_expected.to have_many(:architectures).class_name("Packages::Debian::#{container.capitalize}Architecture").inverse_of(:distribution) }
-
-    if container != :group
-      it { is_expected.to have_many(:publications).class_name('Packages::Debian::Publication').inverse_of(:distribution).with_foreign_key(:distribution_id) }
-      it { is_expected.to have_many(:packages).class_name('Packages::Package').through(:publications) }
-    end
   end
 
   describe 'validations' do
@@ -228,4 +223,44 @@ RSpec.shared_examples 'Debian Distribution' do |factory, container, can_freeze|
       end
     end
   end
+
+  if container == :project
+    describe 'project distribution specifics' do
+      describe 'relationships' do
+        it { is_expected.to have_many(:publications).class_name('Packages::Debian::Publication').inverse_of(:distribution).with_foreign_key(:distribution_id) }
+        it { is_expected.to have_many(:packages).class_name('Packages::Package').through(:publications) }
+        it { is_expected.to have_many(:package_files).class_name('Packages::PackageFile').through(:packages) }
+      end
+    end
+  else
+    describe 'group distribution specifics' do
+      let_it_be(:public_project) { create(:project, :public, group: distribution_with_suite.container)}
+      let_it_be(:public_distribution_with_same_codename) { create(:debian_project_distribution, container: public_project, codename: distribution_with_suite.codename) }
+      let_it_be(:public_package_with_same_codename) { create(:debian_package, project: public_project, published_in: public_distribution_with_same_codename)}
+      let_it_be(:public_distribution_with_same_suite) { create(:debian_project_distribution, container: public_project, suite: distribution_with_suite.suite) }
+      let_it_be(:public_package_with_same_suite) { create(:debian_package, project: public_project, published_in: public_distribution_with_same_suite)}
+
+      let_it_be(:private_project) { create(:project, :private, group: distribution_with_suite.container)}
+      let_it_be(:private_distribution_with_same_codename) { create(:debian_project_distribution, container: private_project, codename: distribution_with_suite.codename) }
+      let_it_be(:private_package_with_same_codename) { create(:debian_package, project: private_project, published_in: private_distribution_with_same_codename)}
+      let_it_be(:private_distribution_with_same_suite) { create(:debian_project_distribution, container: private_project, suite: distribution_with_suite.suite) }
+      let_it_be(:private_package_with_same_suite) { create(:debian_package, project: private_project, published_in: private_distribution_with_same_codename)}
+
+      describe '#packages' do
+        subject { distribution_with_suite.packages }
+
+        it 'returns only public packages with same codename' do
+          expect(subject.to_a).to contain_exactly(public_package_with_same_codename)
+        end
+      end
+
+      describe '#package_files' do
+        subject { distribution_with_suite.package_files }
+
+        it 'returns only files from public packages with same codename' do
+          expect(subject.to_a).to contain_exactly(*public_package_with_same_codename.package_files)
+        end
+      end
+    end
+  end
 end