Merge branch '290288-composer-cache-pages' into 'master'

Add composer cache class and table

See merge request gitlab-org/gitlab!51509

app/models/packages/composer/cache_file.rb

+# frozen_string_literal: true
+
+module Packages
+  module Composer
+    class CacheFile < ApplicationRecord
+      include FileStoreMounter
+
+      self.table_name = 'packages_composer_cache_files'
+
+      mount_file_store_uploader Packages::Composer::CacheUploader
+
+      belongs_to :group, -> { where(type: 'Group') }, foreign_key: 'namespace_id'
+      belongs_to :namespace
+
+      validates :namespace, presence: true
+
+      scope :with_namespace, ->(namespace) { where(namespace: namespace) }
+      scope :with_sha, ->(sha) { where(file_sha256: sha) }
+    end
+  end
+end

app/models/packages/composer/metadatum.rb

@@ -9,6 +9,9 @@ module Packages
       belongs_to :package, -> { where(package_type: :composer) }, inverse_of: :composer_metadatum
 
       validates :package, :target_sha, :composer_json, presence: true
+
+      scope :for_package, ->(name, project_id) { joins(:package).where(packages_packages: { name: name, project_id: project_id, package_type: Packages::Package.package_types[:composer] }) }
+      scope :locked_for_update, -> { lock('FOR UPDATE') }
     end
   end
 end

app/uploaders/packages/composer/cache_uploader.rb

+# frozen_string_literal: true
+class Packages::Composer::CacheUploader < GitlabUploader
+  include ObjectStorage::Concern
+
+  storage_options Gitlab.config.packages
+
+  after :store, :schedule_background_upload
+
+  alias_method :upload, :model
+
+  def filename
+    "#{model.file_sha256}.json"
+  end
+
+  def store_dir
+    dynamic_segment
+  end
+
+  private
+
+  def dynamic_segment
+    raise ObjectNotReadyError, 'Package model not ready' unless model.id
+
+    Gitlab::HashedPath.new("packages", "composer_cache", model.namespace_id, root_hash: model.namespace_id)
+  end
+end

changelogs/unreleased/290288-composer-cache-pages.yml

+---
+title: Add Composer cache classes and table
+merge_request: 51509
+author:
+type: other

db/migrate/20210112202949_create_composer_cache_file.rb

+# frozen_string_literal: true
+
+class CreateComposerCacheFile < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    # rubocop:disable Migration/AddLimitToTextColumns
+    create_table_with_constraints :packages_composer_cache_files do |t|
+      t.timestamps_with_timezone
+
+      # record can be deleted after `delete_at`
+      t.datetime_with_timezone :delete_at
+
+      # which namespace it belongs to
+      t.integer :namespace_id, null: true
+
+      # file storage related fields
+      t.integer :file_store, limit: 2, null: false, default: 1
+      t.text :file, null: false
+      t.binary :file_sha256, null: false
+
+      t.index [:namespace_id, :file_sha256], name: "index_packages_composer_cache_namespace_and_sha", using: :btree, unique: true
+      t.foreign_key :namespaces, column: :namespace_id, on_delete: :nullify
+
+      t.text_limit :file, 255
+    end
+  end
+
+  def down
+    drop_table :packages_composer_cache_files
+  end
+end

db/schema_migrations/20210112202949

+56595e67e9e78a9558e6874d75bdcc295b89ab0096d1b37e4d9366e1574d241c
\ No newline at end of file

db/structure.sql

@@ -14786,6 +14786,27 @@ CREATE SEQUENCE packages_build_infos_id_seq
 
 ALTER SEQUENCE packages_build_infos_id_seq OWNED BY packages_build_infos.id;
 
+CREATE TABLE packages_composer_cache_files (
+    id bigint NOT NULL,
+    created_at timestamp with time zone NOT NULL,
+    updated_at timestamp with time zone NOT NULL,
+    delete_at timestamp with time zone,
+    namespace_id integer,
+    file_store smallint DEFAULT 1 NOT NULL,
+    file text NOT NULL,
+    file_sha256 bytea NOT NULL,
+    CONSTRAINT check_84f5ba81f5 CHECK ((char_length(file) <= 255))
+);
+
+CREATE SEQUENCE packages_composer_cache_files_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE packages_composer_cache_files_id_seq OWNED BY packages_composer_cache_files.id;
+
 CREATE TABLE packages_composer_metadata (
     package_id bigint NOT NULL,
     target_sha bytea NOT NULL,
@@ -18963,6 +18984,8 @@ ALTER TABLE ONLY operations_user_lists ALTER COLUMN id SET DEFAULT nextval('oper
 
 ALTER TABLE ONLY packages_build_infos ALTER COLUMN id SET DEFAULT nextval('packages_build_infos_id_seq'::regclass);
 
+ALTER TABLE ONLY packages_composer_cache_files ALTER COLUMN id SET DEFAULT nextval('packages_composer_cache_files_id_seq'::regclass);
+
 ALTER TABLE ONLY packages_conan_file_metadata ALTER COLUMN id SET DEFAULT nextval('packages_conan_file_metadata_id_seq'::regclass);
 
 ALTER TABLE ONLY packages_conan_metadata ALTER COLUMN id SET DEFAULT nextval('packages_conan_metadata_id_seq'::regclass);
@@ -20329,6 +20352,9 @@ ALTER TABLE ONLY operations_user_lists
 ALTER TABLE ONLY packages_build_infos
     ADD CONSTRAINT packages_build_infos_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY packages_composer_cache_files
+    ADD CONSTRAINT packages_composer_cache_files_pkey PRIMARY KEY (id);
+
 ALTER TABLE ONLY packages_composer_metadata
     ADD CONSTRAINT packages_composer_metadata_pkey PRIMARY KEY (package_id);
 
@@ -22570,6 +22596,8 @@ CREATE UNIQUE INDEX index_ops_strategies_user_lists_on_strategy_id_and_user_list
 
 CREATE INDEX index_packages_build_infos_on_pipeline_id ON packages_build_infos USING btree (pipeline_id);
 
+CREATE UNIQUE INDEX index_packages_composer_cache_namespace_and_sha ON packages_composer_cache_files USING btree (namespace_id, file_sha256);
+
 CREATE UNIQUE INDEX index_packages_composer_metadata_on_package_id_and_target_sha ON packages_composer_metadata USING btree (package_id, target_sha);
 
 CREATE UNIQUE INDEX index_packages_conan_file_metadata_on_package_file_id ON packages_conan_file_metadata USING btree (package_file_id);
@@ -25557,6 +25585,9 @@ ALTER TABLE ONLY namespace_aggregation_schedules
 ALTER TABLE ONLY approval_project_rules_protected_branches
     ADD CONSTRAINT fk_rails_b7567b031b FOREIGN KEY (protected_branch_id) REFERENCES protected_branches(id) ON DELETE CASCADE;
 
+ALTER TABLE ONLY packages_composer_cache_files
+    ADD CONSTRAINT fk_rails_b82cea43a0 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE SET NULL;
+
 ALTER TABLE ONLY alerts_service_data
     ADD CONSTRAINT fk_rails_b93215a42c FOREIGN KEY (service_id) REFERENCES services(id) ON DELETE CASCADE;
 

lib/gitlab/composer/cache.rb

+# frozen_string_literal: true
+
+require 'tempfile'
+
+module Gitlab
+  module Composer
+    class Cache
+      def initialize(project:, name:, last_page_sha: nil)
+        @project = project
+        @name = name
+        @last_page_sha = last_page_sha
+      end
+
+      def execute
+        Packages::Composer::Metadatum.transaction do # rubocop: disable CodeReuse/ActiveRecord
+          # make sure we lock these records at the start
+          locked_package_metadata
+
+          if locked_package_metadata.any?
+            mark_pages_for_delete(shas_to_delete)
+
+            create_cache_page!
+
+            # assign the newest page SHA to the packages
+            locked_package_metadata.update_all(version_cache_sha: version_index.sha)
+          elsif @last_page_sha
+            mark_pages_for_delete([@last_page_sha])
+          end
+        end
+      end
+
+      private
+
+      def mark_pages_for_delete(shas)
+        Packages::Composer::CacheFile
+          .with_namespace(@project.namespace)
+          .with_sha(shas)
+          .update_all(delete_at: 1.day.from_now)
+      end
+
+      def create_cache_page!
+        Packages::Composer::CacheFile
+          .safe_find_or_create_by!(namespace_id: @project.namespace_id, file_sha256: version_index.sha) do |cache_file|
+            cache_file.file = CarrierWaveStringFile.new(version_index.to_json)
+          end
+      end
+
+      def version_index
+        @version_index ||= ::Gitlab::Composer::VersionIndex.new(siblings)
+      end
+
+      def siblings
+        @siblings ||= locked_package_metadata.map(&:package)
+      end
+
+      # find all metadata of the package versions and lock it for update
+      def locked_package_metadata
+        @locked_package_metadata ||= Packages::Composer::Metadatum
+          .for_package(@name, @project.id)
+          .locked_for_update
+      end
+
+      def shas_to_delete
+        locked_package_metadata
+          .map(&:version_cache_sha)
+          .reject { |sha| sha == version_index.sha }
+          .compact
+      end
+    end
+  end
+end

lib/gitlab/composer/version_index.rb

@@ -20,7 +20,7 @@ module Gitlab
       private
 
       def package_versions_map
-        @packages.each_with_object({}) do |package, map|
+        @packages.sort_by(&:version).each_with_object({}) do |package, map|
           map[package.version] = package_metadata(package)
         end
       end

spec/factories/packages.rb

@@ -176,6 +176,24 @@ FactoryBot.define do
     composer_json { { name: 'foo' } }
   end
 
+  factory :composer_cache_file, class: 'Packages::Composer::CacheFile' do
+    group
+
+    file_sha256 { '1' * 64 }
+
+    transient do
+      file_fixture { 'spec/fixtures/packages/composer/package.json' }
+    end
+
+    after(:build) do |cache_file, evaluator|
+      cache_file.file = fixture_file_upload(evaluator.file_fixture)
+    end
+
+    trait(:object_storage) do
+      file_store { Packages::Composer::CacheUploader::Store::REMOTE }
+    end
+  end
+
   factory :maven_metadatum, class: 'Packages::Maven::Metadatum' do
     association :package, package_type: :maven
     path { 'my/company/app/my-app/1.0-SNAPSHOT' }

spec/fixtures/packages/composer/package.json

+{}

spec/lib/gitlab/composer/cache_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Composer::Cache do
+  let_it_be(:package_name) { 'sample-project' }
+  let_it_be(:json) { { 'name' => package_name } }
+  let_it_be(:group) { create(:group) }
+  let_it_be(:project) { create(:project, :custom_repo, files: { 'composer.json' => json.to_json }, group: group) }
+  let(:branch) { project.repository.find_branch('master') }
+  let(:sha_regex) { /^[A-Fa-f0-9]{64}$/ }
+
+  shared_examples 'Composer create cache page' do
+    let(:expected_json) { ::Gitlab::Composer::VersionIndex.new(packages).to_json }
+
+    before do
+      stub_composer_cache_object_storage
+    end
+
+    it 'creates the cached page' do
+      expect { subject }.to change { Packages::Composer::CacheFile.count }.by(1)
+      cache_file = Packages::Composer::CacheFile.last
+      expect(cache_file.file_sha256).to eq package.reload.composer_metadatum.version_cache_sha
+      expect(cache_file.file.read).to eq expected_json
+    end
+  end
+
+  shared_examples 'Composer marks cache page for deletion' do
+    it 'marks the page for deletion' do
+      cache_file = Packages::Composer::CacheFile.last
+
+      freeze_time do
+        expect { subject }.to change { cache_file.reload.delete_at}.from(nil).to(1.day.from_now)
+      end
+    end
+  end
+
+  describe '#execute' do
+    subject { described_class.new(project: project, name: package_name).execute }
+
+    context 'creating packages' do
+      context 'with a pre-existing package' do
+        let(:package) { create(:composer_package, :with_metadatum, project: project, name: package_name, version: '1.0.0', json: json) }
+        let(:package2) { create(:composer_package, :with_metadatum, project: project, name: package_name, version: '2.0.0', json: json) }
+        let(:packages) { [package, package2] }
+
+        before do
+          package
+          described_class.new(project: project, name: package_name).execute
+          package.reload
+          package2
+        end
+
+        it 'updates the sha and creates the cache page' do
+          expect { subject }.to change { package2.reload.composer_metadatum.version_cache_sha }.from(nil).to(sha_regex)
+            .and change { package.reload.composer_metadatum.version_cache_sha }.to(sha_regex)
+        end
+
+        it_behaves_like 'Composer create cache page'
+        it_behaves_like 'Composer marks cache page for deletion'
+      end
+
+      context 'first package' do
+        let!(:package) { create(:composer_package, :with_metadatum, project: project, name: package_name, version: '1.0.0', json: json) }
+        let(:packages) { [package] }
+
+        it 'updates the sha and creates the cache page' do
+          expect { subject }.to change { package.reload.composer_metadatum.version_cache_sha }.from(nil).to(sha_regex)
+        end
+
+        it_behaves_like 'Composer create cache page'
+      end
+    end
+
+    context 'updating packages' do
+      let(:package) { create(:composer_package, :with_metadatum, project: project, name: package_name, version: '1.0.0', json: json) }
+      let(:package2) { create(:composer_package, :with_metadatum, project: project, name: package_name, version: '2.0.0', json: json) }
+      let(:packages) { [package, package2] }
+
+      before do
+        packages
+
+        described_class.new(project: project, name: package_name).execute
+
+        package.update!(version: '1.2.0')
+        package.reload
+      end
+
+      it_behaves_like 'Composer create cache page'
+      it_behaves_like 'Composer marks cache page for deletion'
+    end
+
+    context 'deleting packages' do
+      context 'when it is not the last package' do
+        let(:package) { create(:composer_package, :with_metadatum, project: project, name: package_name, version: '1.0.0', json: json) }
+        let(:package2) { create(:composer_package, :with_metadatum, project: project, name: package_name, version: '2.0.0', json: json) }
+        let(:packages) { [package] }
+
+        before do
+          package
+          package2
+
+          described_class.new(project: project, name: package_name).execute
+
+          package2.destroy!
+        end
+
+        it_behaves_like 'Composer create cache page'
+        it_behaves_like 'Composer marks cache page for deletion'
+      end
+
+      context 'when it is the last package' do
+        let!(:package) { create(:composer_package, :with_metadatum, project: project, name: package_name, version: '1.0.0', json: json) }
+        let!(:last_sha) do
+          described_class.new(project: project, name: package_name).execute
+          package.reload.composer_metadatum.version_cache_sha
+        end
+
+        before do
+          package.destroy!
+        end
+
+        subject { described_class.new(project: project, name: package_name, last_page_sha: last_sha).execute }
+
+        it_behaves_like 'Composer marks cache page for deletion'
+
+        it 'does not create a new page' do
+          expect { subject }.not_to change { Packages::Composer::CacheFile.count }
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/composer/version_index_spec.rb

@@ -15,7 +15,9 @@ RSpec.describe Gitlab::Composer::VersionIndex do
   let(:packages) { [package1, package2] }
 
   describe '#as_json' do
-    subject(:index) { described_class.new(packages).as_json }
+    subject(:package_index) { index['packages'][package_name] }
+
+    let(:index) { described_class.new(packages).as_json }
 
     def expected_json(package)
       {
@@ -32,10 +34,16 @@ RSpec.describe Gitlab::Composer::VersionIndex do
     end
 
     it 'returns the packages json' do
-      packages = index['packages'][package_name]
+      expect(package_index['1.0.0']).to eq(expected_json(package1))
+      expect(package_index['2.0.0']).to eq(expected_json(package2))
+    end
 
-      expect(packages['1.0.0']).to eq(expected_json(package1))
-      expect(packages['2.0.0']).to eq(expected_json(package2))
+    context 'with an unordered list of packages' do
+      let(:packages) { [package2, package1] }
+
+      it 'returns the packages sorted by version' do
+        expect(package_index.keys).to eq ['1.0.0', '2.0.0']
+      end
     end
   end
 

spec/models/packages/composer/cache_file_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+RSpec.describe Packages::Composer::CacheFile, type: :model do
+  describe 'relationships' do
+    it { is_expected.to belong_to(:group) }
+    it { is_expected.to belong_to(:namespace) }
+  end
+
+  describe 'validations' do
+    it { is_expected.to validate_presence_of(:namespace) }
+  end
+
+  describe 'scopes' do
+    let_it_be(:group1) { create(:group) }
+    let_it_be(:group2) { create(:group) }
+    let_it_be(:cache_file1) { create(:composer_cache_file, file_sha256: '123456', group: group1) }
+    let_it_be(:cache_file2) { create(:composer_cache_file, file_sha256: '456778', group: group2) }
+
+    describe '.with_namespace' do
+      subject { described_class.with_namespace(group1) }
+
+      it { is_expected.to eq [cache_file1] }
+    end
+
+    describe '.with_sha' do
+      subject { described_class.with_sha('123456') }
+
+      it { is_expected.to eq [cache_file1] }
+    end
+  end
+end

spec/models/packages/composer/metadatum_spec.rb

@@ -11,4 +11,20 @@ RSpec.describe Packages::Composer::Metadatum, type: :model do
     it { is_expected.to validate_presence_of(:target_sha) }
     it { is_expected.to validate_presence_of(:composer_json) }
   end
+
+  describe 'scopes' do
+    let_it_be(:package_name) { 'sample-project' }
+    let_it_be(:json) { { 'name' => package_name } }
+    let_it_be(:group) { create(:group) }
+    let_it_be(:project) { create(:project, :custom_repo, files: { 'composer.json' => json.to_json }, group: group) }
+    let_it_be(:package1) { create(:composer_package, :with_metadatum, project: project, name: package_name, version: '1.0.0', json: json) }
+    let_it_be(:package2) { create(:composer_package, :with_metadatum, project: project, name: 'other-name', version: '1.0.0', json: json) }
+    let_it_be(:package3) { create(:pypi_package, name: package_name, project: project) }
+
+    describe '.for_package' do
+      subject { described_class.for_package(package_name, project.id) }
+
+      it { is_expected.to eq [package1.composer_metadatum] }
+    end
+  end
 end

spec/support/helpers/stub_object_storage.rb

@@ -85,6 +85,13 @@ module StubObjectStorage
                                  **params)
   end
 
+  def stub_composer_cache_object_storage(**params)
+    stub_object_storage_uploader(config: Gitlab.config.packages.object_store,
+                                 uploader: ::Packages::Composer::CacheUploader,
+                                 remote_directory: 'packages',
+                                 **params)
+  end
+
   def stub_uploads_object_storage(uploader = described_class, **params)
     stub_object_storage_uploader(config: Gitlab.config.uploads.object_store,
                                  uploader: uploader,

spec/uploaders/packages/composer/cache_uploader_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+RSpec.describe Packages::Composer::CacheUploader do
+  let(:cache_file) { create(:composer_cache_file) } # rubocop:disable Rails/SaveBang
+  let(:uploader) { described_class.new(cache_file, :file) }
+  let(:path) { Gitlab.config.packages.storage_path }
+
+  subject { uploader }
+
+  it_behaves_like "builds correct paths",
+                  store_dir: %r[^\h{2}/\h{2}/\h{64}/packages/composer_cache/\d+$],
+                  cache_dir: %r[/packages/tmp/cache],
+                  work_dir: %r[/packages/tmp/work]
+
+  context 'object store is remote' do
+    before do
+      stub_composer_cache_object_storage
+    end
+
+    include_context 'with storage', described_class::Store::REMOTE
+
+    it_behaves_like "builds correct paths",
+                  store_dir: %r[^\h{2}/\h{2}/\h{64}/packages/composer_cache/\d+$]
+  end
+
+  describe 'remote file' do
+    let(:cache_file) { create(:composer_cache_file, :object_storage) }
+
+    context 'with object storage enabled' do
+      before do
+        stub_composer_cache_object_storage
+      end
+
+      it 'can store file remotely' do
+        allow(ObjectStorage::BackgroundMoveWorker).to receive(:perform_async)
+
+        cache_file
+
+        expect(cache_file.file_store).to eq(described_class::Store::REMOTE)
+        expect(cache_file.file.path).not_to be_blank
+      end
+    end
+  end
+end