Add NPM registry support to GitLab packages

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>

config/initializers/seed_fu_fixtures.rb

+# frozen_string_literal: true
+# Include EE-only fixtures into seed_fu
+SeedFu.fixture_paths.push("#{Rails.root}/ee/db/fixtures/#{Rails.env}")

ee/app/models/packages/package.rb

@@ -12,7 +12,10 @@ class Packages::Package < ActiveRecord::Base
     presence: true,
     format: { with: Gitlab::Regex.package_name_regex }
 
-  enum package_type: { maven: 1 }
+  enum package_type: { maven: 1, npm: 2 }
+
+  scope :with_name, ->(name) { where(name: name) }
+  scope :preload_files, -> { preload(:package_files) }
 
   def self.for_projects(projects)
     return none unless projects.any?
@@ -23,4 +26,19 @@ class Packages::Package < ActiveRecord::Base
   def self.only_maven_packages_with_path(path)
     joins(:maven_metadatum).where(packages_maven_metadata: { path: path })
   end
+
+  def self.by_name_and_file_name(name, file_name)
+    where(name: name)
+      .joins(:package_files)
+      .where(packages_package_files: { file_name: file_name }).last!
+  end
+
+  def self.last_of_each_version
+    ids = self
+      .select('MAX(id) AS id')
+      .group(:version)
+      .map(&:id)
+
+    where(id: ids)
+  end
 end

ee/app/presenters/npm_package_presenter.rb

+# frozen_string_literal: true
+
+class NpmPackagePresenter
+  attr_reader :project, :name, :packages
+
+  def initialize(project, name, packages)
+    @project = project
+    @name = name
+    @packages = packages
+  end
+
+  def versions
+    package_versions = {}
+
+    packages.each do |package|
+      package_file = package.package_files.last
+
+      next unless package_file
+
+      package_versions[package.version] = build_package_version(package, package_file)
+    end
+
+    package_versions
+  end
+
+  private
+
+  def build_package_version(package, package_file)
+    {
+      "name": package.name,
+      "version": package.version,
+      "dist": {
+        "shasum": package_file.file_sha1,
+        "tarball": tarball_url(package, package_file)
+      }
+    }
+  end
+
+  def tarball_url(package, package_file)
+    "#{Gitlab.config.gitlab.url}/api/v4/projects/" \
+      "#{package.project_id}/packages/npm/#{package.name}" \
+      "/-/#{package_file.file_name}"
+  end
+end

ee/app/services/packages/create_npm_package_service.rb

+# frozen_string_literal: true
+module Packages
+  class CreateNpmPackageService < BaseService
+    def execute
+      name = params[:name]
+      version = params[:versions].keys.first
+      version_data = params[:versions][version]
+
+      package = project.packages.create!(
+        name: name,
+        version: version,
+        package_type: 'npm'
+      )
+
+      package_file_name = "#{name}-#{version}.tgz"
+      attachment = params['_attachments'][package_file_name]
+
+      file_params = {
+        file:      CarrierWaveStringFile.new(Base64.decode64(attachment['data'])),
+        size:      attachment['length'],
+        file_sha1: version_data[:dist][:shasum],
+        file_name: package_file_name
+      }
+
+      ::Packages::CreatePackageFileService.new(package, file_params).execute
+
+      package
+    end
+  end
+end

ee/app/views/projects/packages/packages/index.html.haml

@@ -26,7 +26,7 @@
         .table-section.section-20
           .table-mobile-header{ role: "rowheader" }= _("Type")
           .table-mobile-content
-            = _('Maven package')
+            = package.package_type
         .table-section.section-20
           .table-mobile-header{ role: "rowheader" }= _("Created")
           .table-mobile-content

ee/changelogs/unreleased/5934-npm-package-management-mvc.yml

+---
+title: Add NPM registry support to GitLab packages
+merge_request: 8673
+author:
+type: added

ee/db/fixtures/development/25_packages.rb

+# frozen_string_literal: true
+require './spec/support/sidekiq'
+
+Sidekiq::Testing.inline! do
+  Gitlab::Seeder.quiet do
+    user = User.first
+    group_path = 'foo'
+    project_path = 'bar'
+    full_path = "#{group_path}/#{project_path}"
+    package_name = "@#{full_path}"
+
+    group = Group.find_by(path: group_path)
+
+    unless group
+      group = Group.new(
+        name: group_path,
+        path: group_path
+      )
+      group.description = FFaker::Lorem.sentence
+      group.save
+
+      group.add_owner(user)
+    end
+
+    project = Project.find_by_full_path(full_path)
+
+    unless project
+      params = {
+        namespace_id: group.id,
+        name: project_path,
+        description: FFaker::Lorem.sentence,
+        visibility_level: Gitlab::VisibilityLevel::PRIVATE,
+        skip_disk_validation: true
+      }
+
+      project = Projects::CreateService.new(user, params).execute
+    end
+
+    (1..10).each do |patch|
+      version = "1.0.#{patch}"
+
+      params = {
+        name:  package_name,
+        versions: {
+          version => {
+            dist: {
+              shasum: 'f572d396fae9206628714fb2ce00f72e94f2258f'
+            }
+          }
+        },
+        '_attachments' => {
+          "#{package_name}-#{version}.tgz" => {
+            'data' => 'aGVsbG8K',
+            'length' => 8
+          }
+        }
+      }
+
+      ::Packages::CreateNpmPackageService
+        .new(project, user, params).execute
+
+      print '.'
+    end
+  end
+end

ee/lib/api/helpers/packages_helpers.rb

+# frozen_string_literal: true
+
+module API
+  module Helpers
+    module PackagesHelpers
+      def require_packages_enabled!
+        not_found! unless ::Gitlab.config.packages.enabled
+      end
+
+      def authorize_packages_feature!
+        forbidden! unless user_project.feature_available?(:packages)
+      end
+
+      def authorize_download_package!
+        authorize!(:read_package, user_project)
+      end
+
+      def authorize_create_package!
+        authorize!(:create_package, user_project)
+      end
+    end
+  end
+end

ee/lib/api/maven_packages.rb

@@ -14,23 +14,9 @@ module API
       authenticate_non_get!
     end
 
-    helpers do
-      def require_packages_enabled!
-        not_found! unless Gitlab.config.packages.enabled
-      end
-
-      def authorize_packages_feature!
-        forbidden! unless user_project.feature_available?(:packages)
-      end
-
-      def authorize_download_package!
-        authorize!(:read_package, user_project)
-      end
-
-      def authorize_create_package!
-        authorize!(:create_package, user_project)
-      end
+    helpers ::API::Helpers::PackagesHelpers
 
+    helpers do
       def extract_format(file_name)
         name, _, format = file_name.rpartition('.')
 

ee/lib/api/npm_packages.rb

+# frozen_string_literal: true
+module API
+  class NpmPackages < Grape::API
+    NPM_ENDPOINT_REQUIREMENTS = {
+      package_name: API::NO_SLASH_URL_PART_REGEX
+    }.freeze
+
+    before do
+      require_packages_enabled!
+      authenticate_non_get!
+    end
+
+    helpers ::API::Helpers::PackagesHelpers
+
+    helpers do
+      def find_project_by_package_name(name)
+        Project.find_by_full_path(name.sub('@', ''))
+      end
+    end
+
+    desc 'NPM registry endpoint at instance level' do
+      detail 'This feature was introduced in GitLab 11.8'
+    end
+    params do
+      requires :package_name, type: String, desc: 'Package name'
+    end
+    route_setting :authentication, job_token_allowed: true
+    get 'packages/npm/*package_name', requirements: NPM_ENDPOINT_REQUIREMENTS do
+      package_name = params[:package_name]
+
+      # To avoid name collision we require project path and project package be the same.
+      # For packages that have different name from the project we should use
+      # the endpoint that includes project id
+      project = find_project_by_package_name(package_name)
+
+      authorize!(:read_package, project)
+      forbidden! unless project.feature_available?(:packages)
+
+      packages = project.packages
+        .with_name(package_name)
+        .last_of_each_version
+        .preload_files
+
+      present NpmPackagePresenter.new(project, package_name, packages),
+        with: EE::API::Entities::NpmPackage
+    end
+
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+      before do
+        authorize_packages_feature!
+      end
+
+      desc 'Download the NPM tarball' do
+        detail 'This feature was introduced in GitLab 11.8'
+      end
+      params do
+        requires :package_name, type: String, desc: 'Package name'
+        requires :file_name, type: String, desc: 'Package file name'
+      end
+      route_setting :authentication, job_token_allowed: true
+      get ':id/packages/npm/*package_name/-/*file_name', format: false do
+        authorize_download_package!
+
+        package = user_project.packages
+          .by_name_and_file_name(params[:package_name], params[:file_name])
+
+        package_file = ::Packages::PackageFileFinder
+          .new(package, params[:file_name]).execute!
+
+        present_carrierwave_file!(package_file.file)
+      end
+
+      desc 'Create NPM package' do
+        detail 'This feature was introduced in GitLab 11.8'
+      end
+      params do
+        requires :package_name, type: String, desc: 'Package name'
+      end
+      route_setting :authentication, job_token_allowed: true
+      put ':id/packages/npm/:package_name', requirements: NPM_ENDPOINT_REQUIREMENTS do
+        authorize_create_package!
+
+        ::Packages::CreateNpmPackageService
+          .new(user_project, current_user, params).execute
+      end
+    end
+  end
+end

ee/lib/ee/api/endpoints.rb

@@ -21,6 +21,7 @@ module EE
         mount ::API::ProjectMirror
         mount ::API::ProjectPushRule
         mount ::API::MavenPackages
+        mount ::API::NpmPackages
       end
     end
   end

ee/lib/ee/api/entities.rb

@@ -492,6 +492,11 @@ module EE
           expose :trial_ends_on
         end
       end
+
+      class NpmPackage < Grape::Entity
+        expose :name
+        expose :versions
+      end
     end
   end
 end

ee/lib/ee/gitlab/regex.rb

@@ -14,7 +14,7 @@ module EE
       end
 
       def package_name_regex
-        @package_name_regex ||= %r{\A(([\w\-\.]*)/)*([\w\-\.]*)\z}.freeze
+        @package_name_regex ||= %r{\A(([\w\-\.\@]*)/)*([\w\-\.]*)\z}.freeze
       end
 
       def maven_path_regex

ee/spec/factories/packages.rb

@@ -19,6 +19,16 @@ FactoryBot.define do
         create :package_file, :pom, package: package
       end
     end
+
+    factory :npm_package do
+      name 'foo'
+      version '1.0.0'
+      package_type 'npm'
+
+      after :create do |package|
+        create :package_file, :npm, package: package
+      end
+    end
   end
 
   factory :package_file, class: Packages::PackageFile do
@@ -44,6 +54,13 @@ FactoryBot.define do
       file_sha1 '42b1bdc80de64953b6876f5a8c644f20204011b0'
       file_type 'xml'
     end
+
+    trait(:npm) do
+      file { fixture_file_upload('ee/spec/fixtures/npm/foo-1.0.1.tgz') }
+      file_name 'foo-1.0.1.tgz'
+      file_sha1 'f572d396fae9206628714fb2ce00f72e94f2258f'
+      file_type 'tgz'
+    end
   end
 
   factory :maven_metadatum, class: Packages::MavenMetadatum do

ee/spec/fixtures/api/schemas/public_api/v4/packages/npm_package.json

+{
+  "type": "object",
+  "required" : ["name", "versions"],
+  "properties" : {
+    "name": { "type": "string" },
+    "versions": { "type": "object" }
+  }
+}

ee/spec/fixtures/api/schemas/public_api/v4/packages/npm_package_version.json

+{
+  "type": "object",
+  "required": ["name", "version", "dist"],
+  "properties" : {
+    "name": { "type": "string" },
+    "version": { "type": "string" },
+    "dist": { 
+      "type": "object",
+      "required": ["shasum", "tarball"],
+      "properties" : {
+        "shasum": { "type": "string" },
+        "tarball": { "type": "string" }
+      }
+    }
+  }
+}

ee/spec/fixtures/npm/foo-1.0.1.tgz

+hello

ee/spec/models/packages/package_spec.rb

@@ -16,4 +16,32 @@ RSpec.describe Packages::Package, type: :model do
       it { is_expected.not_to allow_value("my(dom$$$ain)com.my-app").for(:name) }
     end
   end
+
+  describe '.by_name_and_file_name' do
+    let!(:package) { create(:npm_package) }
+    let!(:package_file) { package.package_files.first }
+
+    subject { described_class }
+
+    it 'finds a package with correct arguiments' do
+      expect(subject.by_name_and_file_name(package.name, package_file.file_name)).to eq(package)
+    end
+
+    it 'will raise error if not found' do
+      expect { subject.by_name_and_file_name('foo', 'foo-5.5.5.tgz') }.to raise_error(ActiveRecord::RecordNotFound)
+    end
+  end
+
+  describe '.last_of_each_version' do
+    let!(:package1) { create(:npm_package, version: '1.0.0') }
+    let!(:package2) { create(:npm_package, version: '1.0.1') }
+    let!(:package3) { create(:npm_package, version: '1.0.1') }
+
+    subject { described_class.last_of_each_version }
+
+    it 'includes only latest package per version' do
+      is_expected.to include(package1, package3)
+      is_expected.not_to include(package2)
+    end
+  end
 end

ee/spec/requests/api/npm_packages_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+describe API::NpmPackages do
+  let(:group)   { create(:group) }
+  let(:user)    { create(:user) }
+  let(:project) { create(:project, :public, namespace: group) }
+  let(:token)   { create(:oauth_access_token, scopes: 'api', resource_owner: user) }
+
+  before do
+    project.add_developer(user)
+    stub_licensed_features(packages: true)
+  end
+
+  shared_examples 'a package that requires auth' do
+    it 'returns the package info with oauth token' do
+      get_package_with_token(package)
+
+      expect_a_valid_package_response
+    end
+
+    it 'denies request without oauth token' do
+      get_package(package)
+
+      expect(response).to have_gitlab_http_status(403)
+    end
+  end
+
+  describe 'GET /api/v4/packages/npm/*package_name' do
+    let(:package) { create(:npm_package, project: project, name: "@#{project.full_path}") }
+
+    context 'a public project' do
+      it 'returns the package info without oauth token' do
+        get_package(package)
+
+        expect_a_valid_package_response
+      end
+    end
+
+    context 'internal project' do
+      before do
+        project.team.truncate
+        project.update!(visibility_level: Gitlab::VisibilityLevel::INTERNAL)
+      end
+
+      it_behaves_like 'a package that requires auth'
+    end
+
+    context 'private project' do
+      before do
+        project.update!(visibility_level: Gitlab::VisibilityLevel::PRIVATE)
+      end
+
+      it_behaves_like 'a package that requires auth'
+
+      it 'denies request when not enough permissions' do
+        project.add_guest(user)
+
+        get_package_with_token(package)
+
+        expect(response).to have_gitlab_http_status(403)
+      end
+    end
+
+    it 'rejects request if feature is not in the license' do
+      stub_licensed_features(packages: false)
+
+      get_package(package)
+
+      expect(response).to have_gitlab_http_status(403)
+    end
+
+    context 'project name is different from a package name' do
+      let(:package) { create(:npm_package, project: project) }
+
+      it 'rejects request' do
+        get_package(package)
+
+        expect(response).to have_gitlab_http_status(403)
+      end
+    end
+
+    def get_package(package, params = {})
+      get api("/packages/npm/#{package.name}"), params
+    end
+
+    def get_package_with_token(package, params = {})
+      get_package(package, params.merge(access_token: token.token))
+    end
+  end
+
+  describe 'GET /api/v4/projects/:id/packages/npm/*package_name/-/*file_name' do
+    let(:package) { create(:npm_package, project: project, name: "@#{project.full_path}") }
+    let(:package_file) { package.package_files.first }
+
+    context 'a public project' do
+      it 'returns the file' do
+        get_file_with_token(package_file)
+
+        expect(response).to have_gitlab_http_status(200)
+        expect(response.content_type.to_s).to eq('application/octet-stream')
+      end
+    end
+
+    context 'private project' do
+      before do
+        project.update!(visibility_level: Gitlab::VisibilityLevel::PRIVATE)
+      end
+
+      it 'returns the file' do
+        get_file_with_token(package_file)
+
+        expect(response).to have_gitlab_http_status(200)
+        expect(response.content_type.to_s).to eq('application/octet-stream')
+      end
+
+      it 'denies download when not enough permissions' do
+        project.add_guest(user)
+
+        get_file_with_token(package_file)
+
+        expect(response).to have_gitlab_http_status(403)
+      end
+
+      it 'denies download when no private token' do
+        get_file(package_file)
+
+        expect(response).to have_gitlab_http_status(404)
+      end
+    end
+
+    it 'rejects request if feature is not in the license' do
+      stub_licensed_features(packages: false)
+
+      get_file(package_file)
+
+      expect(response).to have_gitlab_http_status(403)
+    end
+
+    def get_file(package_file, params = {})
+      get api("/projects/#{project.id}/packages/npm/" \
+              "#{package_file.package.name}/-/#{package_file.file_name}"), params
+    end
+
+    def get_file_with_token(package_file, params = {})
+      get_file(package_file, params.merge(access_token: token.token))
+    end
+  end
+
+  describe 'PUT /api/v4/projects/:id/packages/npm/:package_name' do
+    context 'when params are correct' do
+      context 'unscoped package' do
+        let(:params) do
+          {
+            name: 'foo',
+            versions: {
+              '1.0.1' => {
+                dist: {
+                  shasum: 'f572d396fae9206628714fb2ce00f72e94f2258f'
+                }
+              }
+            },
+            '_attachments' => {
+              "foo-1.0.1.tgz" => {
+                'data' => 'aGVsbG8K',
+                'length' => 8
+              }
+            }
+          }
+        end
+
+        it 'creates npm package with file' do
+          expect { upload_package_with_token('foo', params) }
+            .to change { project.packages.count }.by(1)
+            .and change { Packages::PackageFile.count }.by(1)
+
+          expect(response).to have_gitlab_http_status(200)
+        end
+      end
+
+      context 'scoped package' do
+        let(:params) do
+          {
+            name: '@bar/foo',
+            versions: {
+              '1.0.1' => {
+                dist: {
+                  shasum: 'f572d396fae9206628714fb2ce00f72e94f2258f'
+                }
+              }
+            },
+            '_attachments' => {
+              "@bar/foo-1.0.1.tgz" => {
+                'data' => 'aGVsbG8K',
+                'length' => 8
+              }
+            }
+          }
+        end
+
+        it 'creates npm package with file' do
+          expect { upload_package_with_token('@bar%2Ffoo', params) }
+            .to change { project.packages.count }.by(1)
+            .and change { Packages::PackageFile.count }.by(1)
+
+          expect(response).to have_gitlab_http_status(200)
+        end
+      end
+    end
+
+    def upload_package(package_name, params = {})
+      put api("/projects/#{project.id}/packages/npm/#{package_name}"), params
+    end
+
+    def upload_package_with_token(package_name, params = {})
+      upload_package(package_name, params.merge(access_token: token.token))
+    end
+  end
+
+  def expect_a_valid_package_response
+    expect(response).to have_gitlab_http_status(200)
+    expect(response.content_type.to_s).to eq('application/json')
+    expect(response).to match_response_schema('public_api/v4/packages/npm_package', dir: 'ee')
+    expect(json_response['name']).to eq(package.name)
+    expect(json_response['versions'][package.version]).to match_schema('public_api/v4/packages/npm_package_version', dir: 'ee')
+  end
+end

ee/spec/services/packages/create_npm_package_service_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+describe Packages::CreateNpmPackageService do
+  let(:project) { create(:project) }
+  let(:user) { create(:user) }
+  let(:version) { '1.0.1'.freeze }
+
+  let(:params) do
+    {
+      name: package_name,
+      versions: {
+        version => {
+          dist: {
+            shasum: 'f572d396fae9206628714fb2ce00f72e94f2258f'
+          }
+        }
+      },
+      '_attachments' => {
+        "#{package_name}-#{version}.tgz" => {
+          'content_type' => 'application/octet-stream',
+          'data' => 'aGVsbG8K',
+          'length' => 8
+        }
+      }
+    }
+  end
+
+  shared_examples 'valid package' do
+    it 'creates a valid package' do
+      package = described_class.new(project, user, params).execute
+
+      expect(package).to be_valid
+      expect(package.name).to eq(package_name)
+      expect(package.version).to eq(version)
+    end
+  end
+
+  describe '#execute' do
+    context 'scoped package' do
+      let(:package_name) { '@gitlab/my-app'.freeze }
+
+      it_behaves_like 'valid package'
+    end
+
+    context 'normal package' do
+      let(:package_name) { 'my-app'.freeze }
+
+      it_behaves_like 'valid package'
+    end
+  end
+end

locale/gitlab.pot

@@ -5357,9 +5357,6 @@ msgstr ""
 msgid "Maven Metadata"
 msgstr ""
 
-msgid "Maven package"
-msgstr ""
-
 msgid "Max access level"
 msgstr ""