Merge branch '13347-conan-search-api-endpoints' into 'master'

Conan Search API Endpoints

See merge request gitlab-org/gitlab!16213

db/migrate/20190919183411_add_index_packages_on_name_trigram_to_packages_packages.rb

+# frozen_string_literal: true
+
+class AddIndexPackagesOnNameTrigramToPackagesPackages < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INDEX_NAME = 'index_packages_packages_on_name_trigram'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :packages_packages, :name, name: INDEX_NAME, using: :gin, opclass: { name: :gin_trgm_ops }
+  end
+
+  def down
+    remove_concurrent_index_by_name(:packages_packages, INDEX_NAME)
+  end
+end

db/schema.rb

@@ -2556,6 +2556,7 @@ ActiveRecord::Schema.define(version: 2019_09_27_074328) do
     t.string "name", null: false
     t.string "version"
     t.integer "package_type", limit: 2, null: false
+    t.index ["name"], name: "index_packages_packages_on_name_trigram", opclass: :gin_trgm_ops, using: :gin
     t.index ["project_id"], name: "index_packages_packages_on_project_id"
   end
 

ee/app/finders/packages/conan_package_finder.rb

+# frozen_string_literal: true
+
+module Packages
+  class ConanPackageFinder
+    attr_reader :query, :current_user
+
+    def initialize(query, current_user)
+      @query = query
+      @current_user = current_user
+    end
+
+    def execute
+      packages_for_current_user.with_name_like(query).order_name_asc
+    end
+
+    private
+
+    def packages
+      Packages::Package.conan
+    end
+
+    def packages_for_current_user
+      packages.for_projects(projects_visible_to_current_user)
+    end
+
+    def projects_visible_to_current_user
+      ::Project.public_or_visible_to_user(current_user)
+    end
+  end
+end

ee/app/models/packages/package.rb

 # frozen_string_literal: true
 class Packages::Package < ApplicationRecord
+  include Sortable
+
   belongs_to :project
   # package_files must be destroyed by ruby code in order to properly remove carrierwave uploads and update project statistics
   has_many :package_files, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
@@ -16,9 +18,10 @@ class Packages::Package < ApplicationRecord
   validate :valid_npm_package_name, if: :npm?
   validate :package_already_taken, if: :npm?
 
-  enum package_type: { maven: 1, npm: 2 }
+  enum package_type: { maven: 1, npm: 2, conan: 3 }
 
   scope :with_name, ->(name) { where(name: name) }
+  scope :with_name_like, ->(name) { where(arel_table[:name].matches(name)) }
   scope :with_version, ->(version) { where(version: version) }
   scope :has_version, -> { where.not(version: nil) }
   scope :preload_files, -> { preload(:package_files) }
@@ -50,6 +53,10 @@ class Packages::Package < ApplicationRecord
       .where(packages_package_files: { file_name: file_name }).last!
   end
 
+  def self.pluck_names
+    pluck(:name)
+  end
+
   def self.sort_by_attribute(method)
     case method.to_s
     when 'created_asc' then order_created

ee/app/services/packages/conan/search_service.rb

+# frozen_string_literal: true
+
+module Packages
+  module Conan
+    class SearchService < BaseService
+      include ActiveRecord::Sanitization::ClassMethods
+
+      WILDCARD = '*'
+      RECIPE_SEPARATOR = '@'
+
+      def initialize(user, params)
+        super(nil, user, params)
+      end
+
+      def execute
+        return ServiceResponse.error(message: 'not found', http_status: :not_found) unless feature_available?
+
+        ServiceResponse.success(payload: { results: search_results })
+      end
+
+      private
+
+      def search_results
+        return [] if wildcard_query?
+
+        search_packages(build_query)
+      end
+
+      def wildcard_query?
+        params[:query] == WILDCARD
+      end
+
+      def build_query
+        sanitized_query = sanitize_sql_like(params[:query].delete(WILDCARD))
+        return "#{sanitized_query}%" if params[:query].end_with?(WILDCARD)
+        return sanitized_query if sanitized_query.include?(RECIPE_SEPARATOR)
+
+        "#{sanitized_query}/%"
+      end
+
+      def feature_available?
+        Feature.enabled?(:conan_package_registry)
+      end
+
+      def search_packages(query)
+        Packages::ConanPackageFinder.new(query, current_user).execute.pluck_names
+      end
+    end
+  end
+end

ee/lib/api/conan_packages.rb

@@ -40,6 +40,17 @@ module API
       get 'ping' do
         header 'X-Conan-Server-Capabilities', [].join(',')
       end
+
+      desc 'Search for packages' do
+        detail 'This feature was introduced in GitLab 12.3'
+      end
+      params do
+        requires :q, type: String, desc: 'Search query'
+      end
+      get 'conans/search' do
+        service = ::Packages::Conan::SearchService.new(current_user, query: params[:q]).execute
+        service.payload
+      end
     end
 
     namespace 'packages/conan/v1/conans/*url_recipe' do

ee/lib/ee/gitlab/regex.rb

@@ -7,7 +7,7 @@ module EE
 
       class_methods do
         def package_name_regex
-          @package_name_regex ||= %r{\A\@?(([\w\-\.]*)/)*([\w\-\.]*)\z}.freeze
+          @package_name_regex ||= %r{\A\@?(([\w\-\.\+]*)\/)*([\w\-\.]+)@?(([\w\-\.\+]*)\/)*([\w\-\.]*)\z}.freeze
         end
 
         def maven_path_regex

ee/spec/factories/packages.rb

@@ -29,6 +29,12 @@ FactoryBot.define do
         create :package_file, :npm, package: package
       end
     end
+
+    factory :conan_package do
+      sequence(:name) { |n| "package-#{n}/1.0.0@#{project.full_path.tr('/', '+')}/stable"}
+      version { '1.0.0' }
+      package_type { 'conan' }
+    end
   end
 
   factory :package_file, class: Packages::PackageFile do

ee/spec/finders/packages/conan_package_finder_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+describe Packages::ConanPackageFinder do
+  describe '#execute' do
+    let_it_be(:user) { create(:user) }
+    let_it_be(:project) { create(:project, :public) }
+    let!(:conan_package) { create(:conan_package, project: project) }
+    let!(:conan_package2) { create(:conan_package, project: project) }
+
+    subject { described_class.new(query, user).execute }
+
+    context 'packages that are not visible to user' do
+      let!(:non_visible_project) { create(:project, :private) }
+      let!(:non_visible_conan_package) { create(:conan_package, project: non_visible_project) }
+      let(:query) { "#{conan_package.name.split('/').first[0, 3]}%" }
+
+      it { is_expected.to eq [conan_package, conan_package2] }
+    end
+  end
+end

ee/spec/lib/gitlab/regex_spec.rb

@@ -21,8 +21,11 @@ describe Gitlab::Regex do
     it { is_expected.to match('foo/bar') }
     it { is_expected.to match('@foo/bar') }
     it { is_expected.to match('com/mycompany/app/my-app') }
+    it { is_expected.to match('my-package/1.0.0@my+project+path/beta') }
+    it { is_expected.not_to match('my-package/1.0.0@@@@@my+project+path/beta') }
     it { is_expected.not_to match('$foo/bar') }
     it { is_expected.not_to match('@foo/@/bar') }
+    it { is_expected.not_to match('@@foo/bar') }
     it { is_expected.not_to match('my package name') }
     it { is_expected.not_to match('!!()()') }
   end

ee/spec/requests/api/conan_packages_spec.rb

@@ -3,6 +3,10 @@ require 'spec_helper'
 
 describe API::ConanPackages do
   let(:base_secret) { SecureRandom.base64(64) }
+  let(:personal_access_token) { create(:personal_access_token) }
+  let(:headers) do
+    { 'HTTP_AUTHORIZATION' => ActionController::HttpAuthentication::Basic.encode_credentials('foo', personal_access_token.token) }
+  end
 
   let(:jwt_secret) do
     OpenSSL::HMAC.hexdigest(
@@ -12,11 +16,6 @@ describe API::ConanPackages do
     )
   end
 
-  let(:personal_access_token) { create(:personal_access_token) }
-  let(:headers) do
-    { 'HTTP_AUTHORIZATION' => ActionController::HttpAuthentication::Basic.encode_credentials('foo', personal_access_token.token) }
-  end
-
   before do
     stub_licensed_features(packages: true)
     allow(Settings).to receive(:attr_encrypted_db_key_base).and_return(base_secret)
@@ -99,6 +98,35 @@ describe API::ConanPackages do
     end
   end
 
+  describe 'GET /api/v4/packages/conan/v1/conans/search' do
+    let(:project) { create(:project, :public) }
+    let(:package) { create(:conan_package, project: project) }
+
+    before do
+      get api('/packages/conan/v1/conans/search'), headers: headers, params: params
+    end
+
+    subject { JSON.parse(response.body)['results'] }
+
+    context 'returns packages with a matching name' do
+      let(:params) { { q: package.name } }
+
+      it { is_expected.to contain_exactly(package.name) }
+    end
+
+    context 'returns packages using a * wildcard' do
+      let(:params) {{ q: "#{package.name[0, 3]}*" }}
+
+      it { is_expected.to contain_exactly(package.name) }
+    end
+
+    context 'does not return non-matching packages' do
+      let(:params) {{ q: "foo" }}
+
+      it { is_expected.to be_blank }
+    end
+  end
+
   describe 'GET /api/v4/packages/conan/v1/users/authenticate' do
     it 'responds with 401 Unauthorized when invalid token is provided' do
       headers = { 'HTTP_AUTHORIZATION' => ActionController::HttpAuthentication::Basic.encode_credentials('foo', 'wrong-token') }

ee/spec/requests/api/npm_packages_spec.rb

@@ -163,7 +163,7 @@ describe API::NpmPackages do
         end
 
         context 'invalid package name' do
-          let(:package_name) { "@#{group.path}/my_inv@lid_package_name" }
+          let(:package_name) { "@#{group.path}/my_inv@@lid_package_name" }
           let(:params) { upload_params(package_name) }
 
           it 'handles an ActiveRecord::RecordInvalid exception with 400 error' do

ee/spec/services/packages/conan/search_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Packages::Conan::SearchService do
+  let_it_be(:user) { create(:user) }
+  let_it_be(:project) { create(:project, :public) }
+  let!(:conan_package) { create(:conan_package, project: project) }
+  let!(:conan_package2) { create(:conan_package, project: project) }
+
+  subject { described_class.new(user, query: query) }
+
+  describe '#execute' do
+    context 'feature unavailable' do
+      let(:query) { '' }
+
+      before do
+        stub_feature_flags(conan_package_registry: false)
+      end
+
+      it 'responds with 404 not found' do
+        result = subject.execute
+
+        expect(result.http_status).to eq :not_found
+        expect(result.message).to eq('not found')
+      end
+    end
+
+    context 'with wildcard' do
+      let(:partial_name) { conan_package.name.first[0, 3] }
+      let(:query) { "#{partial_name}*" }
+
+      it 'makes a wildcard query' do
+        result = subject.execute
+
+        expect(result.status).to eq :success
+        expect(result.payload).to eq(results: [conan_package.name, conan_package2.name])
+      end
+    end
+
+    context 'with only wildcard' do
+      let(:query) { '*' }
+
+      it 'returns empty' do
+        result = subject.execute
+
+        expect(result.status).to eq :success
+        expect(result.payload).to eq(results: [])
+      end
+    end
+
+    context 'with no wildcard' do
+      let(:query) { conan_package.name.split('/').first }
+
+      it 'makes a search using the beginning of the recipe' do
+        result = subject.execute
+
+        expect(result.status).to eq :success
+        expect(result.payload).to eq(results: [conan_package.name])
+      end
+    end
+
+    context 'with full recipe match' do
+      let(:query) { conan_package.name }
+
+      it 'makes an exact search' do
+        result = subject.execute
+
+        expect(result.status).to eq :success
+        expect(result.payload).to eq(results: [conan_package.name])
+      end
+    end
+
+    context 'with malicious query' do
+      let(:query) { 'DROP TABLE foo;' }
+
+      it 'returns empty' do
+        result = subject.execute
+
+        expect(result.status).to eq :success
+        expect(result.payload).to eq(results: [])
+      end
+    end
+  end
+end