Merge branch 'tc-repo-full-path-in-db-ee' into 'master'

Store hashed storage paths in the database

See merge request gitlab-org/gitlab-ee!8614

app/models/concerns/shardable.rb

+# frozen_string_literal: true
+
+module Shardable
+  extend ActiveSupport::Concern
+
+  included do
+    belongs_to :shard
+    validates :shard, presence: true
+  end
+
+  def shard_name
+    shard&.name
+  end
+
+  def shard_name=(name)
+    self.shard = Shard.by_name(name)
+  end
+end

app/models/pool_repository.rb

 # frozen_string_literal: true
 
 class PoolRepository < ActiveRecord::Base
-  belongs_to :shard
-  validates :shard, presence: true
+  include Shardable
 
   has_many :member_projects, class_name: 'Project'
 
   after_create :correct_disk_path
 
-  def shard_name
-    shard&.name
-  end
-
-  def shard_name=(name)
-    self.shard = Shard.by_name(name)
-  end
-
   private
 
   def correct_disk_path

app/models/project.rb

@@ -187,6 +187,7 @@ class Project < ActiveRecord::Base
 
   has_one :import_state, autosave: true, class_name: 'ProjectImportState', inverse_of: :project
   has_one :import_export_upload, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
+  has_one :project_repository, inverse_of: :project
 
   # Merge Requests for target project should be removed with it
   has_many :merge_requests, foreign_key: 'target_project_id', inverse_of: :target_project
@@ -1207,6 +1208,13 @@ class Project < ActiveRecord::Base
     false
   end
 
+  def track_project_repository
+    return unless hashed_storage?(:repository)
+
+    project_repo = project_repository || build_project_repository
+    project_repo.update!(shard_name: repository_storage, disk_path: disk_path)
+  end
+
   def create_repository(force: false)
     # Forked import is handled asynchronously
     return if forked? && !force

app/models/project_repository.rb

+# frozen_string_literal: true
+
+class ProjectRepository < ActiveRecord::Base
+  include Shardable
+
+  belongs_to :project, inverse_of: :project_repository
+
+  class << self
+    def find_project(disk_path)
+      find_by(disk_path: disk_path)&.project
+    end
+  end
+end

app/services/projects/create_service.rb

@@ -88,6 +88,8 @@ module Projects
         @project.create_wiki unless skip_wiki?
       end
 
+      @project.track_project_repository
+
       event_service.create_project(@project, current_user)
       system_hook_service.execute_hooks_for(@project, :create)
 

app/services/projects/hashed_storage/migrate_repository_service.rb

@@ -32,6 +32,7 @@ module Projects
 
         if result
           project.write_repository_config
+          project.track_project_repository
         else
           rollback_folder_move
           project.storage_version = nil

changelogs/unreleased/tc-repo-full-path-in-db.yml

+---
+title: Add model and relation to store repo full path in database
+merge_request: 23143
+author:
+type: added

db/migrate/20181122160027_create_project_repositories.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CreateProjectRepositories < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :project_repositories, id: :bigserial do |t|
+      t.references :shard, null: false, index: true, foreign_key: { on_delete: :restrict }
+      t.string :disk_path, null: false, index: { unique: true }
+      t.references :project, null: false, index: { unique: true }, foreign_key: { on_delete: :cascade }
+    end
+  end
+end

db/schema.rb

@@ -2149,6 +2149,15 @@ ActiveRecord::Schema.define(version: 20181126153547) do
     t.index ["status"], name: "index_project_mirror_data_on_status", using: :btree
   end
 
+  create_table "project_repositories", id: :bigserial, force: :cascade do |t|
+    t.integer "shard_id", null: false
+    t.string "disk_path", null: false
+    t.integer "project_id", null: false
+    t.index ["disk_path"], name: "index_project_repositories_on_disk_path", unique: true, using: :btree
+    t.index ["project_id"], name: "index_project_repositories_on_project_id", unique: true, using: :btree
+    t.index ["shard_id"], name: "index_project_repositories_on_shard_id", using: :btree
+  end
+
   create_table "project_repository_states", force: :cascade do |t|
     t.integer "project_id", null: false
     t.binary "repository_verification_checksum"
@@ -3269,6 +3278,8 @@ ActiveRecord::Schema.define(version: 20181126153547) do
   add_foreign_key "project_group_links", "projects", name: "fk_daa8cee94c", on_delete: :cascade
   add_foreign_key "project_import_data", "projects", name: "fk_ffb9ee3a10", on_delete: :cascade
   add_foreign_key "project_mirror_data", "projects", name: "fk_d1aad367d7", on_delete: :cascade
+  add_foreign_key "project_repositories", "projects", on_delete: :cascade
+  add_foreign_key "project_repositories", "shards", on_delete: :restrict
   add_foreign_key "project_repository_states", "projects", on_delete: :cascade
   add_foreign_key "project_statistics", "projects", on_delete: :cascade
   add_foreign_key "project_tracing_settings", "projects", on_delete: :cascade

ee/app/services/projects/update_repository_storage_service.rb

@@ -17,6 +17,7 @@ module Projects
         mark_old_paths_for_archive
 
         project.update(repository_storage: new_repository_storage_key, repository_read_only: false)
+        project.track_project_repository
       else
         project.update(repository_read_only: false)
       end

ee/spec/services/projects/update_repository_storage_service_spec.rb

@@ -28,6 +28,7 @@ describe Projects::UpdateRepositoryStorageService do
           expect(project).not_to be_repository_read_only
           expect(project.repository_storage).to eq('test_second_storage')
           expect(gitlab_shell.exists?('default', old_path)).to be(false)
+          expect(project.project_repository.shard_name).to eq('test_second_storage')
         end
       end
 

spec/lib/gitlab/import_export/all_models.yml

@@ -245,6 +245,7 @@ project:
 - protected_branches
 - protected_tags
 - project_members
+- project_repository
 - users
 - requesters
 - deploy_keys_projects

spec/models/pool_repository_spec.rb

+# frozen_string_literal: true
+
 require 'spec_helper'
 
 describe PoolRepository do

spec/models/project_repository_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe ProjectRepository do
+  describe 'associations' do
+    it { is_expected.to belong_to(:shard) }
+    it { is_expected.to belong_to(:project) }
+  end
+
+  describe '.find_project' do
+    it 'finds project by disk path' do
+      project = create(:project)
+      project.track_project_repository
+
+      expect(described_class.find_project(project.disk_path)).to eq(project)
+    end
+
+    it 'returns nil when it does not find the project' do
+      expect(described_class.find_project('@@unexisting/path/to/project')).to be_nil
+    end
+  end
+end

spec/models/project_spec.rb

@@ -55,6 +55,7 @@ describe Project do
     it { is_expected.to have_one(:gitlab_issue_tracker_service) }
     it { is_expected.to have_one(:external_wiki_service) }
     it { is_expected.to have_one(:project_feature) }
+    it { is_expected.to have_one(:project_repository) }
     it { is_expected.to have_one(:statistics).class_name('ProjectStatistics') }
     it { is_expected.to have_one(:import_data).class_name('ProjectImportData') }
     it { is_expected.to have_one(:last_event).class_name('Event') }
@@ -1729,6 +1730,30 @@ describe Project do
     end
   end
 
+  describe '#track_project_repository' do
+    let(:project) { create(:project, :repository) }
+
+    it 'creates a project_repository' do
+      project.track_project_repository
+
+      expect(project.reload.project_repository).to be_present
+      expect(project.project_repository.disk_path).to eq(project.disk_path)
+      expect(project.project_repository.shard_name).to eq(project.repository_storage)
+    end
+
+    it 'updates the project_repository' do
+      project.track_project_repository
+
+      allow(project).to receive(:disk_path).and_return('@fancy/new/path')
+
+      expect do
+        project.track_project_repository
+      end.not_to change(ProjectRepository, :count)
+
+      expect(project.reload.project_repository.disk_path).to eq(project.disk_path)
+    end
+  end
+
   describe '#create_repository' do
     let(:project) { create(:project, :repository) }
     let(:shell) { Gitlab::Shell.new }