Merge branch 'debian_download_file' into 'master'

Add Debian API endpoint for deb, udeb, ... files

See merge request gitlab-org/gitlab!64923

app/models/packages/debian.rb

@@ -6,6 +6,8 @@ module Packages
     COMPONENT_REGEX = DISTRIBUTION_REGEX.freeze
     ARCHITECTURE_REGEX = %r{[a-z0-9][-a-z0-9]*}.freeze
 
+    LETTER_REGEX = %r{(lib)?[a-z0-9]}.freeze
+
     def self.table_name_prefix
       'packages_debian_'
     end

app/services/packages/debian/generate_distribution_service.rb

@@ -120,7 +120,7 @@ module Packages
 
       def package_filename(package_file)
         letter = package_file.package.name.start_with?('lib') ? package_file.package.name[0..3] : package_file.package.name[0]
-        "#{pool_prefix(package_file)}/#{letter}/#{package_file.package.name}/#{package_file.file_name}"
+        "#{pool_prefix(package_file)}/#{letter}/#{package_file.package.name}/#{package_file.package.version}/#{package_file.file_name}"
       end
 
       def pool_prefix(package_file)
@@ -128,7 +128,7 @@ module Packages
         when ::Packages::Debian::GroupDistribution
           "pool/#{@distribution.codename}/#{package_file.package.project_id}"
         else
-          "pool/#{@distribution.codename}/#{@distribution.container_id}"
+          "pool/#{@distribution.codename}"
         end
       end
 

lib/api/concerns/packages/debian_package_endpoints.rb

@@ -6,8 +6,6 @@ module API
       module DebianPackageEndpoints
         extend ActiveSupport::Concern
 
-        LETTER_REGEX = %r{(lib)?[a-z0-9]}.freeze
-        PACKAGE_REGEX = API::NO_SLASH_URL_PART_REGEX
         DISTRIBUTION_REQUIREMENTS = {
           distribution: ::Packages::Debian::DISTRIBUTION_REGEX
         }.freeze
@@ -15,14 +13,6 @@ module API
           component: ::Packages::Debian::COMPONENT_REGEX,
           architecture: ::Packages::Debian::ARCHITECTURE_REGEX
         }.freeze
-        COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS = {
-          component: ::Packages::Debian::COMPONENT_REGEX,
-          letter: LETTER_REGEX,
-          source_package: PACKAGE_REGEX
-        }.freeze
-        FILE_NAME_REQUIREMENTS = {
-          file_name: API::NO_SLASH_URL_PART_REGEX
-        }.freeze
 
         included do
           feature_category :package_registry
@@ -31,109 +21,107 @@ module API
           helpers ::API::Helpers::Packages::BasicAuthHelpers
           include ::API::Helpers::Authentication
 
-          namespace 'packages/debian' do
-            authenticate_with do |accept|
-              accept.token_types(:personal_access_token, :deploy_token, :job_token)
-                    .sent_through(:http_basic_auth)
+          helpers do
+            params :shared_package_file_params do
+              requires :distribution, type: String, desc: 'The Debian Codename or Suite', regexp: Gitlab::Regex.debian_distribution_regex
+              requires :letter, type: String, desc: 'The Debian Classification (first-letter or lib-first-letter)'
+              requires :package_name, type: String, desc: 'The Debian Source Package Name', regexp: Gitlab::Regex.debian_package_name_regex
+              requires :package_version, type: String, desc: 'The Debian Source Package Version', regexp: Gitlab::Regex.debian_version_regex
+              requires :file_name, type: String, desc: 'The Debian File Name'
             end
 
-            helpers do
-              def present_release_file
-                distribution = ::Packages::Debian::DistributionsFinder.new(project_or_group, codename_or_suite: params[:distribution]).execute.last!
-
-                present_carrierwave_file!(distribution.file)
-              end
+            def distribution_from!(container)
+              ::Packages::Debian::DistributionsFinder.new(container, codename_or_suite: params[:distribution]).execute.last!
             end
 
-            format :txt
-            content_type :txt, 'text/plain'
+            def present_package_file!
+              not_found! unless params[:package_name].start_with?(params[:letter])
 
-            params do
-              requires :distribution, type: String, desc: 'The Debian Codename', regexp: Gitlab::Regex.debian_distribution_regex
+              package_file = distribution_from!(user_project).package_files.with_file_name(params[:file_name]).last!
+
+              present_carrierwave_file!(package_file.file)
             end
+          end
 
-            namespace 'dists/*distribution', requirements: DISTRIBUTION_REQUIREMENTS do
-              # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release.gpg
-              desc 'The Release file signature' do
-                detail 'This feature was introduced in GitLab 13.5'
-              end
+          authenticate_with do |accept|
+            accept.token_types(:personal_access_token, :deploy_token, :job_token)
+                  .sent_through(:http_basic_auth)
+          end
 
-              route_setting :authentication, authenticate_non_public: true
-              get 'Release.gpg' do
-                not_found!
-              end
+          rescue_from ArgumentError do |e|
+            render_api_error!(e.message, 400)
+          end
 
-              # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release
-              desc 'The unsigned Release file' do
-                detail 'This feature was introduced in GitLab 13.5'
-              end
+          rescue_from ActiveRecord::RecordInvalid do |e|
+            render_api_error!(e.message, 400)
+          end
 
-              route_setting :authentication, authenticate_non_public: true
-              get 'Release' do
-                present_release_file
-              end
+          format :txt
+          content_type :txt, 'text/plain'
 
-              # GET {projects|groups}/:id/packages/debian/dists/*distribution/InRelease
-              desc 'The signed Release file' do
-                detail 'This feature was introduced in GitLab 13.5'
-              end
+          params do
+            requires :distribution, type: String, desc: 'The Debian Codename or Suite', regexp: Gitlab::Regex.debian_distribution_regex
+          end
 
-              route_setting :authentication, authenticate_non_public: true
-              get 'InRelease' do
-                # Signature to be added in 7.3 of https://gitlab.com/groups/gitlab-org/-/epics/6057#note_582697034
-                present_release_file
-              end
+          namespace 'dists/*distribution', requirements: DISTRIBUTION_REQUIREMENTS do
+            # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release.gpg
+            desc 'The Release file signature' do
+              detail 'This feature was introduced in GitLab 13.5'
+            end
 
-              params do
-                requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex
-                requires :architecture, type: String, desc: 'The Debian Architecture', regexp: Gitlab::Regex.debian_architecture_regex
-              end
+            route_setting :authentication, authenticate_non_public: true
+            get 'Release.gpg' do
+              not_found!
+            end
 
-              namespace ':component/binary-:architecture', requirements: COMPONENT_ARCHITECTURE_REQUIREMENTS do
-                # GET {projects|groups}/:id/packages/debian/dists/*distribution/:component/binary-:architecture/Packages
-                desc 'The binary files index' do
-                  detail 'This feature was introduced in GitLab 13.5'
-                end
-
-                route_setting :authentication, authenticate_non_public: true
-                get 'Packages' do
-                  relation = "::Packages::Debian::#{project_or_group.class.name}ComponentFile".constantize
-
-                  component_file = relation
-                    .preload_distribution
-                    .with_container(project_or_group)
-                    .with_codename_or_suite(params[:distribution])
-                    .with_component_name(params[:component])
-                    .with_file_type(:packages)
-                    .with_architecture_name(params[:architecture])
-                    .with_compression_type(nil)
-                    .order_created_asc
-                    .last!
-
-                  present_carrierwave_file!(component_file.file)
-                end
-              end
+            # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release
+            desc 'The unsigned Release file' do
+              detail 'This feature was introduced in GitLab 13.5'
+            end
+
+            route_setting :authentication, authenticate_non_public: true
+            get 'Release' do
+              present_carrierwave_file!(distribution_from!(project_or_group).file)
+            end
+
+            # GET {projects|groups}/:id/packages/debian/dists/*distribution/InRelease
+            desc 'The signed Release file' do
+              detail 'This feature was introduced in GitLab 13.5'
+            end
+
+            route_setting :authentication, authenticate_non_public: true
+            get 'InRelease' do
+              # Signature to be added in 7.3 of https://gitlab.com/groups/gitlab-org/-/epics/6057#note_582697034
+              present_carrierwave_file!(distribution_from!(project_or_group).file)
             end
 
             params do
               requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex
-              requires :letter, type: String, desc: 'The Debian Classification (first-letter or lib-first-letter)'
-              requires :source_package, type: String, desc: 'The Debian Source Package Name', regexp: Gitlab::Regex.debian_package_name_regex
+              requires :architecture, type: String, desc: 'The Debian Architecture', regexp: Gitlab::Regex.debian_architecture_regex
             end
 
-            namespace 'pool/:component/:letter/:source_package', requirements: COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS do
-              # GET {projects|groups}/:id/packages/debian/pool/:component/:letter/:source_package/:file_name
-              params do
-                requires :file_name, type: String, desc: 'The Debian File Name'
-              end
-              desc 'The package' do
+            namespace ':component/binary-:architecture', requirements: COMPONENT_ARCHITECTURE_REQUIREMENTS do
+              # GET {projects|groups}/:id/packages/debian/dists/*distribution/:component/binary-:architecture/Packages
+              desc 'The binary files index' do
                 detail 'This feature was introduced in GitLab 13.5'
               end
 
               route_setting :authentication, authenticate_non_public: true
-              get ':file_name', requirements: FILE_NAME_REQUIREMENTS do
-                # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286
-                'TODO File'
+              get 'Packages' do
+                relation = "::Packages::Debian::#{project_or_group.class.name}ComponentFile".constantize
+
+                component_file = relation
+                  .preload_distribution
+                  .with_container(project_or_group)
+                  .with_codename_or_suite(params[:distribution])
+                  .with_component_name(params[:component])
+                  .with_file_type(:packages)
+                  .with_architecture_name(params[:architecture])
+                  .with_compression_type(nil)
+                  .order_created_asc
+                  .last!
+
+                present_carrierwave_file!(component_file.file)
               end
             end
           end

lib/api/debian_group_packages.rb

@@ -2,20 +2,22 @@
 
 module API
   class DebianGroupPackages < ::API::Base
-    params do
-      requires :id, type: String, desc: 'The ID of a group'
-    end
+    PACKAGE_FILE_REQUIREMENTS = ::API::DebianProjectPackages::PACKAGE_FILE_REQUIREMENTS.merge(
+      project_id: %r{[0-9]+}.freeze
+    ).freeze
 
     resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
-      rescue_from ArgumentError do |e|
-        render_api_error!(e.message, 400)
-      end
+      helpers do
+        def user_project
+          @project ||= find_project!(params[:project_id])
+        end
 
-      rescue_from ActiveRecord::RecordInvalid do |e|
-        render_api_error!(e.message, 400)
+        def project_or_group
+          user_group
+        end
       end
 
-      before do
+      after_validation do
         require_packages_enabled!
 
         not_found! unless ::Feature.enabled?(:debian_group_packages, user_group)
@@ -23,14 +25,27 @@ module API
         authorize_read_package!(user_group)
       end
 
-      namespace ':id/-' do
-        helpers do
-          def project_or_group
-            user_group
-          end
-        end
+      params do
+        requires :id, type: String, desc: 'The ID of a group'
+      end
 
+      namespace ':id/-/packages/debian' do
         include ::API::Concerns::Packages::DebianPackageEndpoints
+
+        # GET groups/:id/packages/debian/pool/:distribution/:project_id/:letter/:package_name/:package_version/:file_name
+        params do
+          requires :project_id, type: Integer, desc: 'The Project Id'
+          use :shared_package_file_params
+        end
+
+        desc 'The package' do
+          detail 'This feature was introduced in GitLab 14.2'
+        end
+
+        route_setting :authentication, authenticate_non_public: true
+        get 'pool/:distribution/:project_id/:letter/:package_name/:package_version/:file_name', requirements: PACKAGE_FILE_REQUIREMENTS do
+          present_package_file!
+        end
       end
     end
   end

lib/api/debian_project_packages.rb

@@ -2,17 +2,23 @@
 
 module API
   class DebianProjectPackages < ::API::Base
-    params do
-      requires :id, type: String, desc: 'The ID of a project'
-    end
+    PACKAGE_FILE_REQUIREMENTS = {
+      id: API::NO_SLASH_URL_PART_REGEX,
+      distribution: ::Packages::Debian::DISTRIBUTION_REGEX,
+      letter: ::Packages::Debian::LETTER_REGEX,
+      package_name: API::NO_SLASH_URL_PART_REGEX,
+      package_version: API::NO_SLASH_URL_PART_REGEX,
+      file_name: API::NO_SLASH_URL_PART_REGEX
+    }.freeze
+    FILE_NAME_REQUIREMENTS = {
+      file_name: API::NO_SLASH_URL_PART_REGEX
+    }.freeze
 
     resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
-      rescue_from ArgumentError do |e|
-        render_api_error!(e.message, 400)
-      end
-
-      rescue_from ActiveRecord::RecordInvalid do |e|
-        render_api_error!(e.message, 400)
+      helpers do
+        def project_or_group
+          user_project
+        end
       end
 
       after_validation do
@@ -23,20 +29,32 @@ module API
         authorize_read_package!
       end
 
-      namespace ':id' do
-        helpers do
-          def project_or_group
-            user_project
-          end
-        end
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
 
+      namespace ':id/packages/debian' do
         include ::API::Concerns::Packages::DebianPackageEndpoints
 
+        # GET projects/:id/packages/debian/pool/:distribution/:letter/:package_name/:package_version/:file_name
+        params do
+          use :shared_package_file_params
+        end
+
+        desc 'The package' do
+          detail 'This feature was introduced in GitLab 14.2'
+        end
+
+        route_setting :authentication, authenticate_non_public: true
+        get 'pool/:distribution/:letter/:package_name/:package_version/:file_name', requirements: PACKAGE_FILE_REQUIREMENTS do
+          present_package_file!
+        end
+
         params do
           requires :file_name, type: String, desc: 'The file name'
         end
 
-        namespace 'packages/debian/:file_name', requirements: FILE_NAME_REQUIREMENTS do
+        namespace ':file_name', requirements: FILE_NAME_REQUIREMENTS do
           format :txt
           content_type :json, Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
 

spec/requests/api/debian_group_packages_spec.rb

@@ -6,6 +6,12 @@ RSpec.describe API::DebianGroupPackages do
   include WorkhorseHelpers
 
   include_context 'Debian repository shared context', :group, false do
+    context 'with invalid parameter' do
+      let(:url) { "/groups/1/-/packages/debian/dists/with+space/InRelease" }
+
+      it_behaves_like 'Debian repository GET request', :bad_request, /^distribution is invalid$/
+    end
+
     describe 'GET groups/:id/-/packages/debian/dists/*distribution/Release.gpg' do
       let(:url) { "/groups/#{container.id}/-/packages/debian/dists/#{distribution.codename}/Release.gpg" }
 
@@ -30,10 +36,25 @@ RSpec.describe API::DebianGroupPackages do
       it_behaves_like 'Debian repository read endpoint', 'GET request', :success, /Description: This is an incomplete Packages file/
     end
 
-    describe 'GET groups/:id/-/packages/debian/pool/:component/:letter/:source_package/:file_name' do
-      let(:url) { "/groups/#{container.id}/-/packages/debian/pool/#{component.name}/#{letter}/#{source_package}/#{package_name}_#{package_version}_#{architecture.name}.deb" }
+    describe 'GET groups/:id/-/packages/debian/pool/:codename/:project_id/:letter/:package_name/:package_version/:file_name' do
+      let(:url) { "/groups/#{container.id}/-/packages/debian/pool/#{package.debian_distribution.codename}/#{project.id}/#{letter}/#{package.name}/#{package.version}/#{file_name}" }
+
+      using RSpec::Parameterized::TableSyntax
+
+      where(:file_name, :success_body) do
+        'sample_1.2.3~alpha2.tar.xz'          | /^.7zXZ/
+        'sample_1.2.3~alpha2.dsc'             | /^Format: 3.0 \(native\)/
+        'libsample0_1.2.3~alpha2_amd64.deb'   | /^!<arch>/
+        'sample-udeb_1.2.3~alpha2_amd64.udeb' | /^!<arch>/
+        'sample_1.2.3~alpha2_amd64.buildinfo' | /Build-Tainted-By/
+        'sample_1.2.3~alpha2_amd64.changes'   | /urgency=medium/
+      end
+
+      with_them do
+        include_context 'with file_name', params[:file_name]
 
-      it_behaves_like 'Debian repository read endpoint', 'GET request', :success, /^TODO File$/
+        it_behaves_like 'Debian repository read endpoint', 'GET request', :success, params[:success_body]
+      end
     end
   end
 end

spec/requests/api/debian_project_packages_spec.rb

@@ -6,6 +6,12 @@ RSpec.describe API::DebianProjectPackages do
   include WorkhorseHelpers
 
   include_context 'Debian repository shared context', :project, true do
+    context 'with invalid parameter' do
+      let(:url) { "/projects/1/packages/debian/dists/with+space/InRelease" }
+
+      it_behaves_like 'Debian repository GET request', :bad_request, /^distribution is invalid$/
+    end
+
     describe 'GET projects/:id/packages/debian/dists/*distribution/Release.gpg' do
       let(:url) { "/projects/#{container.id}/packages/debian/dists/#{distribution.codename}/Release.gpg" }
 
@@ -30,10 +36,25 @@ RSpec.describe API::DebianProjectPackages do
       it_behaves_like 'Debian repository read endpoint', 'GET request', :success, /Description: This is an incomplete Packages file/
     end
 
-    describe 'GET projects/:id/packages/debian/pool/:component/:letter/:source_package/:file_name' do
-      let(:url) { "/projects/#{container.id}/packages/debian/pool/#{component.name}/#{letter}/#{source_package}/#{package_name}_#{package_version}_#{architecture.name}.deb" }
+    describe 'GET projects/:id/packages/debian/pool/:codename/:letter/:package_name/:package_version/:file_name' do
+      let(:url) { "/projects/#{container.id}/packages/debian/pool/#{package.debian_distribution.codename}/#{letter}/#{package.name}/#{package.version}/#{file_name}" }
+
+      using RSpec::Parameterized::TableSyntax
 
-      it_behaves_like 'Debian repository read endpoint', 'GET request', :success, /^TODO File$/
+      where(:file_name, :success_body) do
+        'sample_1.2.3~alpha2.tar.xz'          | /^.7zXZ/
+        'sample_1.2.3~alpha2.dsc'             | /^Format: 3.0 \(native\)/
+        'libsample0_1.2.3~alpha2_amd64.deb'   | /^!<arch>/
+        'sample-udeb_1.2.3~alpha2_amd64.udeb' | /^!<arch>/
+        'sample_1.2.3~alpha2_amd64.buildinfo' | /Build-Tainted-By/
+        'sample_1.2.3~alpha2_amd64.changes'   | /urgency=medium/
+      end
+
+      with_them do
+        include_context 'with file_name', params[:file_name]
+
+        it_behaves_like 'Debian repository read endpoint', 'GET request', :success, params[:success_body]
+      end
     end
 
     describe 'PUT projects/:id/packages/debian/:file_name' do

spec/support/shared_examples/requests/api/debian_packages_shared_examples.rb

@@ -29,6 +29,8 @@ RSpec.shared_context 'Debian repository shared context' do |container_type, can_
     let_it_be(:public_project) { create(:project, :public, group: public_container) }
     let_it_be(:private_project_distribution) { create(:debian_project_distribution, container: private_project, codename: 'existing-codename') }
     let_it_be(:public_project_distribution) { create(:debian_project_distribution, container: public_project, codename: 'existing-codename') }
+
+    let(:project) { { private: private_project, public: public_project }[visibility_level] }
   else
     let_it_be(:private_project) { private_container }
     let_it_be(:public_project) { public_container }
@@ -45,12 +47,8 @@ RSpec.shared_context 'Debian repository shared context' do |container_type, can_
   let(:architecture) { { private: private_architecture, public: public_architecture }[visibility_level] }
   let(:component) { { private: private_component, public: public_component }[visibility_level] }
   let(:component_file) { { private: private_component_file, public: public_component_file }[visibility_level] }
-
-  let(:source_package) { 'sample' }
-  let(:letter) { source_package[0..2] == 'lib' ? source_package[0..3] : source_package[0] }
-  let(:package_name) { 'libsample0' }
-  let(:package_version) { '1.2.3~alpha2' }
-  let(:file_name) { "#{package_name}_#{package_version}_#{architecture.name}.deb" }
+  let(:package) { { private: private_package, public: public_package }[visibility_level] }
+  let(:letter) { package.name[0..2] == 'lib' ? package.name[0..3] : package.name[0] }
 
   let(:method) { :get }
 
@@ -94,6 +92,10 @@ RSpec.shared_context 'Debian repository shared context' do |container_type, can_
   end
 end
 
+RSpec.shared_context 'with file_name' do |file_name|
+  let(:file_name) { file_name }
+end
+
 RSpec.shared_context 'Debian repository auth headers' do |user_role, user_token, auth_method = :token|
   let(:token) { user_token ? personal_access_token.token : 'wrong' }
 

spec/support/shared_examples/services/packages/debian/generate_distribution_shared_examples.rb

@@ -53,7 +53,9 @@ RSpec.shared_examples 'Generate Debian Distribution and component files' do
         .and change { component_file1.reload.updated_at }.to(current_time.round)
 
       debs = package.package_files.with_debian_file_type(:deb).preload_debian_file_metadata.to_a
-      pool_prefix = "pool/unstable/#{project.id}/p/#{package.name}"
+      pool_prefix = 'pool/unstable'
+      pool_prefix += "/#{project.id}" if container_type == :group
+      pool_prefix += "/p/#{package.name}/#{package.version}"
       expected_main_amd64_content = <<~EOF
       Package: libsample0
       Source: #{package.name}