Allow dynamic files for Conan package uploads

Updates the upload_urls endpoints to accept dynamic
lists of package files to allow conan_export.tgz
and conan_sources.tgz

changelogs/unreleased/210550-conan-missing-files.yml

+---
+title: Conan packages allow for conan_sources.tgz and conan_export.tgz files
+merge_request: 39559
+author:
+type: fixed

lib/api/conan_packages.rb

@@ -189,9 +189,7 @@ module API
           authorize!(:read_package, project)
 
           status 200
-          upload_urls = package_upload_urls(::Packages::Conan::FileMetadatum::PACKAGE_FILES)
-
-          present upload_urls, with: ::API::Entities::ConanPackage::ConanUploadUrls
+          present package_upload_urls, with: ::API::Entities::ConanPackage::ConanUploadUrls
         end
 
         desc 'Recipe Upload Urls' do
@@ -202,9 +200,7 @@ module API
           authorize!(:read_package, project)
 
           status 200
-          upload_urls = recipe_upload_urls(::Packages::Conan::FileMetadatum::RECIPE_FILES)
-
-          present upload_urls, with: ::API::Entities::ConanPackage::ConanUploadUrls
+          present recipe_upload_urls, with: ::API::Entities::ConanPackage::ConanUploadUrls
         end
 
         desc 'Delete Package' do

lib/api/helpers/packages/conan/api_helpers.rb

@@ -28,22 +28,30 @@ module API
             present_download_urls(::API::Entities::ConanPackage::ConanRecipeManifest, &:recipe_urls)
           end
 
-          def recipe_upload_urls(file_names)
+          def recipe_upload_urls
             { upload_urls: Hash[
-              file_names.collect do |file_name|
+              file_names.select(&method(:recipe_file?)).map do |file_name|
                 [file_name, recipe_file_upload_url(file_name)]
               end
             ] }
           end
 
-          def package_upload_urls(file_names)
+          def package_upload_urls
             { upload_urls: Hash[
-              file_names.collect do |file_name|
+              file_names.select(&method(:package_file?)).map do |file_name|
                 [file_name, package_file_upload_url(file_name)]
               end
             ] }
           end
 
+          def recipe_file?(file_name)
+            file_name.in?(::Packages::Conan::FileMetadatum::RECIPE_FILES)
+          end
+
+          def package_file?(file_name)
+            file_name.in?(::Packages::Conan::FileMetadatum::PACKAGE_FILES)
+          end
+
           def package_file_upload_url(file_name)
             expose_url(
               api_v4_packages_conan_v1_files_package_path(
@@ -130,6 +138,14 @@ module API
             end
           end
 
+          def file_names
+            json_payload = Gitlab::Json.parse(request.body.string)
+
+            bad_request!(nil) unless json_payload.is_a?(Hash)
+
+            json_payload.keys
+          end
+
           def create_package_file_with_type(file_type, current_package)
             unless params['file.size'] == 0
               # conan sends two upload requests, the first has no file, so we skip record creation if file.size == 0

lib/gitlab/regex.rb

@@ -3,7 +3,7 @@
 module Gitlab
   module Regex
     module Packages
-      CONAN_RECIPE_FILES = %w[conanfile.py conanmanifest.txt].freeze
+      CONAN_RECIPE_FILES = %w[conanfile.py conanmanifest.txt conan_sources.tgz conan_export.tgz].freeze
       CONAN_PACKAGE_FILES = %w[conaninfo.txt conanmanifest.txt conan_package.tgz].freeze
 
       def conan_file_name_regex

spec/requests/api/conan_packages_spec.rb

@@ -331,6 +331,18 @@ RSpec.describe API::ConanPackages do
         .and_return(presenter)
     end
 
+    shared_examples 'rejects invalid upload_url params' do
+      context 'with unaccepted json format' do
+        let(:params) { %w[foo bar] }
+
+        it 'returns 400' do
+          subject
+
+          expect(response).to have_gitlab_http_status(:bad_request)
+        end
+      end
+    end
+
     describe 'GET /api/v4/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel' do
       let(:recipe_path) { package.conan_recipe_path }
 
@@ -418,13 +430,14 @@ RSpec.describe API::ConanPackages do
       let(:recipe_path) { package.conan_recipe_path }
 
       let(:params) do
-        { "conanfile.py": 24,
-          "conanmanifext.txt": 123 }
+        { 'conanfile.py': 24,
+          'conanmanifest.txt': 123 }
       end
 
-      subject { post api("/packages/conan/v1/conans/#{recipe_path}/upload_urls"), params: params, headers: headers }
+      subject { post api("/packages/conan/v1/conans/#{recipe_path}/upload_urls"), params: params.to_json, headers: headers }
 
       it_behaves_like 'rejects invalid recipe'
+      it_behaves_like 'rejects invalid upload_url params'
 
       it 'returns a set of upload urls for the files requested' do
         subject
@@ -436,20 +449,58 @@ RSpec.describe API::ConanPackages do
 
         expect(response.body).to eq(expected_response.to_json)
       end
+
+      context 'with conan_sources and conan_export files' do
+        let(:params) do
+          { 'conan_sources.tgz': 345,
+            'conan_export.tgz': 234,
+            'conanmanifest.txt': 123 }
+        end
+
+        it 'returns upload urls for the additional files' do
+          subject
+
+          expected_response = {
+            'conan_sources.tgz': "#{Settings.gitlab.base_url}/api/v4/packages/conan/v1/files/#{package.conan_recipe_path}/0/export/conan_sources.tgz",
+            'conan_export.tgz':  "#{Settings.gitlab.base_url}/api/v4/packages/conan/v1/files/#{package.conan_recipe_path}/0/export/conan_export.tgz",
+            'conanmanifest.txt': "#{Settings.gitlab.base_url}/api/v4/packages/conan/v1/files/#{package.conan_recipe_path}/0/export/conanmanifest.txt"
+          }
+
+          expect(response.body).to eq(expected_response.to_json)
+        end
+      end
+
+      context 'with an invalid file' do
+        let(:params) do
+          { 'invalid_file.txt': 10,
+            'conanmanifest.txt': 123 }
+        end
+
+        it 'does not return the invalid file as an upload_url' do
+          subject
+
+          expected_response = {
+            'conanmanifest.txt': "#{Settings.gitlab.base_url}/api/v4/packages/conan/v1/files/#{package.conan_recipe_path}/0/export/conanmanifest.txt"
+          }
+
+          expect(response.body).to eq(expected_response.to_json)
+        end
+      end
     end
 
     describe 'POST /api/v4/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel/packages/:conan_package_reference/upload_urls' do
       let(:recipe_path) { package.conan_recipe_path }
 
       let(:params) do
-        { "conaninfo.txt": 24,
-          "conanmanifext.txt": 123,
-          "conan_package.tgz": 523 }
+        { 'conaninfo.txt': 24,
+          'conanmanifest.txt': 123,
+          'conan_package.tgz': 523 }
       end
 
-      subject { post api("/packages/conan/v1/conans/#{recipe_path}/packages/123456789/upload_urls"), params: params, headers: headers }
+      subject { post api("/packages/conan/v1/conans/#{recipe_path}/packages/123456789/upload_urls"), params: params.to_json, headers: headers }
 
       it_behaves_like 'rejects invalid recipe'
+      it_behaves_like 'rejects invalid upload_url params'
 
       it 'returns a set of upload urls for the files requested' do
         expected_response = {
@@ -462,6 +513,23 @@ RSpec.describe API::ConanPackages do
 
         expect(response.body).to eq(expected_response.to_json)
       end
+
+      context 'with invalid files' do
+        let(:params) do
+          { 'conaninfo.txt': 24,
+            'invalid_file.txt': 10 }
+        end
+
+        it 'returns upload urls only for the valid requested files' do
+          expected_response = {
+            'conaninfo.txt': "#{Settings.gitlab.base_url}/api/v4/packages/conan/v1/files/#{package.conan_recipe_path}/0/package/123456789/0/conaninfo.txt"
+          }
+
+          subject
+
+          expect(response.body).to eq(expected_response.to_json)
+        end
+      end
     end
 
     describe 'DELETE /api/v4/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel' do