Commit 256e7a63 authored by Sean McGivern's avatar Sean McGivern

Merge branch 'zj-gitaly-migrate-old-migrations' into 'master'

Gitaly migrate old DB migrations

Closes gitaly#1106

See merge request gitlab-org/gitlab-ce!20708
parents 44dbeccb a53a4a43
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class MigrateProcessCommitWorkerJobs < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
class Repository
attr_reader :storage
def initialize(storage, relative_path)
@storage = storage
@relative_path = relative_path
end
def gitaly_repository
Gitaly::Repository.new(storage_name: @storage, relative_path: @relative_path)
end
end
class Project < ActiveRecord::Base
def self.find_including_path(id)
select("projects.*, CONCAT(namespaces.path, '/', projects.path) AS path_with_namespace")
......@@ -11,19 +21,12 @@ class MigrateProcessCommitWorkerJobs < ActiveRecord::Migration
.find_by(id: id)
end
def repository_storage_path
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
Gitlab.config.repositories.storages[repository_storage].legacy_disk_path
end
end
def repository_path
# TODO: review if the change from Legacy storage needs to reflect here as well.
File.join(repository_storage_path, read_attribute(:path_with_namespace) + '.git')
def commit(rev)
Gitlab::GitalyClient::CommitService.new(repository).find_commit(rev)
end
def repository
@repository ||= Rugged::Repository.new(repository_path)
@repository ||= Repository.new(repository_storage, read_attribute(:path_with_namespace) + '.git')
end
end
......@@ -42,22 +45,19 @@ class MigrateProcessCommitWorkerJobs < ActiveRecord::Migration
next unless project
begin
commit = project.repository.lookup(payload['args'][2])
rescue Rugged::OdbError
next
end
commit = project.commit(payload['args'][2])
next unless commit
hash = {
id: commit.oid,
message: encode(commit.message),
parent_ids: commit.parent_ids,
authored_date: commit.author[:time],
author_name: encode(commit.author[:name]),
author_email: encode(commit.author[:email]),
committed_date: commit.committer[:time],
committer_email: encode(commit.committer[:email]),
committer_name: encode(commit.committer[:name])
id: commit.id,
message: encode(commit.body),
parent_ids: commit.parent_ids.to_a,
authored_date: Time.at(commit.author.date.seconds).utc,
author_name: encode(commit.author.name),
author_email: encode(commit.author.email),
committed_date: Time.at(commit.committer.date.seconds).utc,
committer_email: encode(commit.committer.email),
committer_name: encode(commit.committer.name)
}
payload['args'][2] = hash
......
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class RemoveDotGitFromUsernames < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
include Gitlab::ShellAdapter
# Set this constant to true if this migration requires downtime.
DOWNTIME = false
def up
......@@ -64,16 +60,14 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration
# we rename suffix instead of removing it
path = path.sub(/\.git\z/, '_git')
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
check_routes(path.dup, 0, path)
end
check_routes(path.dup, 0, path)
end
def check_routes(base, counter, path)
route_exists = route_exists?(path)
Gitlab.config.repositories.storages.each do |shard, storage|
if route_exists || path_exists?(shard, storage.legacy_disk_path)
Gitlab.config.repositories.storages.each do |shard, _storage|
if route_exists || path_exists?(shard, path)
counter += 1
path = "#{base}#{counter}"
......
# Gitlab::Git::Repository is a wrapper around native Rugged::Repository object
require 'tempfile'
require 'forwardable'
require "rubygems/package"
......
......@@ -4,14 +4,11 @@ require 'spec_helper'
require Rails.root.join('db', 'migrate', '20161124141322_migrate_process_commit_worker_jobs.rb')
describe MigrateProcessCommitWorkerJobs do
let(:project) { create(:project, :legacy_storage, :repository) } # rubocop:disable RSpec/FactoriesInMigrationSpecs
let(:user) { create(:user) } # rubocop:disable RSpec/FactoriesInMigrationSpecs
let(:rugged) do
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
project.repository.rugged
end
set(:project) { create(:project, :legacy_storage, :repository) } # rubocop:disable RSpec/FactoriesInMigrationSpecs
set(:user) { create(:user) } # rubocop:disable RSpec/FactoriesInMigrationSpecs
let(:commit) do
Gitlab::Git::Commit.last(project.repository.raw)
end
let(:commit) { rugged.rev_parse(project.commit.id) }
describe 'Project' do
describe 'find_including_path' do
......@@ -29,32 +26,13 @@ describe MigrateProcessCommitWorkerJobs do
end
end
describe '#repository_storage_path' do
it 'returns the storage path for the repository' do
migration_project = described_class::Project
.find_including_path(project.id)
expect(File.directory?(migration_project.repository_storage_path))
.to eq(true)
end
end
describe '#repository_path' do
it 'returns the path to the repository' do
migration_project = described_class::Project
.find_including_path(project.id)
expect(File.directory?(migration_project.repository_path)).to eq(true)
end
end
describe '#repository' do
it 'returns a Rugged::Repository' do
it 'returns a mock implemention of ::Repository' do
migration_project = described_class::Project
.find_including_path(project.id)
expect(migration_project.repository)
.to be_an_instance_of(Rugged::Repository)
expect(migration_project.repository).to respond_to(:storage)
expect(migration_project.repository).to respond_to(:gitaly_repository)
end
end
end
......@@ -72,7 +50,7 @@ describe MigrateProcessCommitWorkerJobs do
before do
Sidekiq.redis do |redis|
job = JSON.dump(args: [project.id, user.id, commit.oid])
job = JSON.dump(args: [project.id, user.id, commit.id])
redis.lpush('queue:process_commit', job)
end
end
......@@ -88,9 +66,10 @@ describe MigrateProcessCommitWorkerJobs do
end
it 'skips jobs using commits that no longer exist' do
allow_any_instance_of(Rugged::Repository).to receive(:lookup)
.with(commit.oid)
.and_raise(Rugged::OdbError)
allow_any_instance_of(Gitlab::GitalyClient::CommitService)
.to receive(:find_commit)
.with(commit.id)
.and_return(nil)
migration.up
......@@ -104,11 +83,7 @@ describe MigrateProcessCommitWorkerJobs do
end
it 'encodes data to UTF-8' do
allow_any_instance_of(Rugged::Repository).to receive(:lookup)
.with(commit.oid)
.and_return(commit)
allow(commit).to receive(:message)
allow(commit).to receive(:body)
.and_return('김치'.force_encoding('BINARY'))
migration.up
......@@ -140,7 +115,7 @@ describe MigrateProcessCommitWorkerJobs do
end
it 'includes the commit ID' do
expect(commit_hash['id']).to eq(commit.oid)
expect(commit_hash['id']).to eq(commit.id)
end
it 'includes the commit message' do
......@@ -152,27 +127,27 @@ describe MigrateProcessCommitWorkerJobs do
end
it 'includes the author date' do
expect(commit_hash['authored_date']).to eq(commit.author[:time].to_s)
expect(commit_hash['authored_date']).to eq(commit.authored_date.to_s)
end
it 'includes the author name' do
expect(commit_hash['author_name']).to eq(commit.author[:name])
expect(commit_hash['author_name']).to eq(commit.author_name)
end
it 'includes the author Email' do
expect(commit_hash['author_email']).to eq(commit.author[:email])
expect(commit_hash['author_email']).to eq(commit.author_email)
end
it 'includes the commit date' do
expect(commit_hash['committed_date']).to eq(commit.committer[:time].to_s)
expect(commit_hash['committed_date']).to eq(commit.committed_date.to_s)
end
it 'includes the committer name' do
expect(commit_hash['committer_name']).to eq(commit.committer[:name])
expect(commit_hash['committer_name']).to eq(commit.committer_name)
end
it 'includes the committer Email' do
expect(commit_hash['committer_email']).to eq(commit.committer[:email])
expect(commit_hash['committer_email']).to eq(commit.committer_email)
end
end
end
......@@ -186,7 +161,7 @@ describe MigrateProcessCommitWorkerJobs do
before do
Sidekiq.redis do |redis|
job = JSON.dump(args: [project.id, user.id, commit.oid])
job = JSON.dump(args: [project.id, user.id, commit.id])
redis.lpush('queue:process_commit', job)
migration.up
......@@ -215,7 +190,7 @@ describe MigrateProcessCommitWorkerJobs do
end
it 'includes the commit SHA' do
expect(job['args'][2]).to eq(commit.oid)
expect(job['args'][2]).to eq(commit.id)
end
end
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment