Commit df6db81e authored by Dmitriy Zaporozhets's avatar Dmitriy Zaporozhets

Merge branch 'features/async_gitolite' of dev.gitlabhq.com:gitlab/gitlabhq

parents b4f16faa f7ade3b6
web: bundle exec unicorn_rails -p $PORT web: bundle exec unicorn_rails -p $PORT
worker: bundle exec sidekiq -q post_receive,mailer,system_hook,project_web_hook,common,default worker: bundle exec sidekiq -q post_receive,mailer,system_hook,project_web_hook,common,default,gitolite
...@@ -32,16 +32,10 @@ module Projects ...@@ -32,16 +32,10 @@ module Projects
@project.namespace_id = current_user.namespace_id @project.namespace_id = current_user.namespace_id
end end
Project.transaction do @project.creator = current_user
@project.creator = current_user
@project.save!
# Add user as project master if @project.save
@project.users_projects.create!(project_access: UsersProject::MASTER, user: current_user) @project.users_projects.create(project_access: UsersProject::MASTER, user: current_user)
# when project saved no team member exist so
# project repository should be updated after first user add
@project.update_repository
end end
@project @project
......
...@@ -299,6 +299,9 @@ class Project < ActiveRecord::Base ...@@ -299,6 +299,9 @@ class Project < ActiveRecord::Base
def trigger_post_receive(oldrev, newrev, ref, user) def trigger_post_receive(oldrev, newrev, ref, user)
data = post_receive_data(oldrev, newrev, ref, user) data = post_receive_data(oldrev, newrev, ref, user)
# Create satellite
self.satellite.create unless self.satellite.exists?
# Create push event # Create push event
self.observe_push(data) self.observe_push(data)
...@@ -313,9 +316,6 @@ class Project < ActiveRecord::Base ...@@ -313,9 +316,6 @@ class Project < ActiveRecord::Base
self.execute_services(data.dup) self.execute_services(data.dup)
end end
# Create satellite
self.satellite.create unless self.satellite.exists?
# Discover the default branch, but only if it hasn't already been set to # Discover the default branch, but only if it hasn't already been set to
# something else # something else
if repository && default_branch.nil? if repository && default_branch.nil?
...@@ -460,11 +460,17 @@ class Project < ActiveRecord::Base ...@@ -460,11 +460,17 @@ class Project < ActiveRecord::Base
end end
def update_repository def update_repository
gitolite.update_repository(self) GitoliteWorker.perform_async(
:update_repository,
self.id
)
end end
def destroy_repository def destroy_repository
gitolite.remove_repository(self) GitoliteWorker.perform_async(
:remove_repository,
self.path_with_namespace
)
end end
def repo_exists? def repo_exists?
......
...@@ -22,7 +22,7 @@ class ProtectedBranch < ActiveRecord::Base ...@@ -22,7 +22,7 @@ class ProtectedBranch < ActiveRecord::Base
after_destroy :update_repository after_destroy :update_repository
def update_repository def update_repository
gitolite.update_repository(project) project.update_repository
end end
def commit def commit
......
...@@ -82,9 +82,13 @@ class UsersProject < ActiveRecord::Base ...@@ -82,9 +82,13 @@ class UsersProject < ActiveRecord::Base
users_project.save users_project.save
end end
end end
Gitlab::Gitolite.new.update_repositories(Project.where(id: project_ids))
end end
GitoliteWorker.perform_async(
:update_repositories,
project_ids
)
true true
rescue rescue
false false
...@@ -97,9 +101,13 @@ class UsersProject < ActiveRecord::Base ...@@ -97,9 +101,13 @@ class UsersProject < ActiveRecord::Base
users_project.skip_git = true users_project.skip_git = true
users_project.destroy users_project.destroy
end end
Gitlab::Gitolite.new.update_repositories(Project.where(id: project_ids))
end end
GitoliteWorker.perform_async(
:update_repositories,
project_ids
)
true true
rescue rescue
false false
...@@ -129,7 +137,7 @@ class UsersProject < ActiveRecord::Base ...@@ -129,7 +137,7 @@ class UsersProject < ActiveRecord::Base
end end
def update_repository def update_repository
gitolite.update_repository(project) project.update_repository
end end
def project_access_human def project_access_human
......
...@@ -10,6 +10,7 @@ class ProjectObserver < ActiveRecord::Observer ...@@ -10,6 +10,7 @@ class ProjectObserver < ActiveRecord::Observer
def after_destroy(project) def after_destroy(project)
log_info("Project \"#{project.name}\" was removed") log_info("Project \"#{project.name}\" was removed")
project.satellite.destroy
project.destroy_repository project.destroy_repository
end end
......
class GitoliteWorker
include Sidekiq::Worker
include Gitolited
sidekiq_options queue: :gitolite
def perform(action, arg)
gitolite.send(action, arg)
end
end
...@@ -13,13 +13,14 @@ class PostReceive ...@@ -13,13 +13,14 @@ class PostReceive
# Ignore push from non-gitlab users # Ignore push from non-gitlab users
user = if identifier.eql? Gitlab.config.gitolite.admin_key user = if identifier.eql? Gitlab.config.gitolite.admin_key
email = project.repository.commit(newrev).author.email rescue nil email = project.repository.commit(newrev).author.email rescue nil
User.find_by_email(email) if email User.find_by_email(email) if email
elsif /^[A-Z0-9._%a-z\-]+@(?:[A-Z0-9a-z\-]+\.)+[A-Za-z]{2,4}$/.match(identifier) elsif /^[A-Z0-9._%a-z\-]+@(?:[A-Z0-9a-z\-]+\.)+[A-Za-z]{2,4}$/.match(identifier)
User.find_by_email(identifier) User.find_by_email(identifier)
else else
Key.find_by_identifier(identifier).try(:user) Key.find_by_identifier(identifier).try(:user)
end end
return false unless user return false unless user
project.trigger_post_receive(oldrev, newrev, ref, user) project.trigger_post_receive(oldrev, newrev, ref, user)
......
...@@ -22,7 +22,13 @@ module Gitlab ...@@ -22,7 +22,13 @@ module Gitlab
end end
end end
def update_repository project # Update project config in gitolite by project id
#
# Ex.
# update_repository(23)
#
def update_repository(project_id)
project = Project.find(project_id)
config.update_project!(project) config.update_project!(project)
end end
...@@ -33,8 +39,28 @@ module Gitlab ...@@ -33,8 +39,28 @@ module Gitlab
end end
end end
def remove_repository project # Remove repository from gitolite
config.destroy_project!(project) #
# name - project path with namespace
#
# Ex.
# remove_repository("gitlab/gitlab-ci")
#
def remove_repository(name)
config.destroy_project!(name)
end
# Update projects configs in gitolite by project ids
#
# Ex.
# update_repositories([1, 4, 6])
#
def update_repositories(project_ids)
projects = Project.where(id: project_ids)
config.apply do |config|
config.update_projects(projects)
end
end end
def url_to_repo path def url_to_repo path
...@@ -45,12 +71,6 @@ module Gitlab ...@@ -45,12 +71,6 @@ module Gitlab
config.admin_all_repo! config.admin_all_repo!
end end
def update_repositories projects
config.apply do |config|
config.update_projects(projects)
end
end
alias_method :create_repository, :update_repository alias_method :create_repository, :update_repository
end end
end end
...@@ -4,6 +4,8 @@ require 'fileutils' ...@@ -4,6 +4,8 @@ require 'fileutils'
module Gitlab module Gitlab
class GitoliteConfig class GitoliteConfig
include Gitlab::Popen
class PullError < StandardError; end class PullError < StandardError; end
class PushError < StandardError; end class PushError < StandardError; end
class BrokenGitolite < StandardError; end class BrokenGitolite < StandardError; end
...@@ -87,12 +89,14 @@ module Gitlab ...@@ -87,12 +89,14 @@ module Gitlab
Gitlab::GitLogger.error(message) Gitlab::GitLogger.error(message)
end end
def destroy_project(project) def path_to_repo(name)
# do rm-rf only if repository exists File.join(Gitlab.config.gitolite.repos_path, "#{name}.git")
if project.repository end
FileUtils.rm_rf(project.repository.path_to_repo)
end def destroy_project(name)
conf.rm_repo(project.path_with_namespace) full_path = path_to_repo(name)
FileUtils.rm_rf(full_path) if File.exists?(full_path)
conf.rm_repo(name)
end end
def clean_repo repo_name def clean_repo repo_name
...@@ -210,14 +214,14 @@ module Gitlab ...@@ -210,14 +214,14 @@ module Gitlab
end end
def push def push
output, status = popen('git add -A') output, status = popen('git add -A', tmp_conf_path)
raise "Git add failed." unless status.zero? raise "Git add failed." unless status.zero?
# git commit returns 0 on success, and 1 if there is nothing to commit # git commit returns 0 on success, and 1 if there is nothing to commit
output, status = popen('git commit -m "GitLab"') output, status = popen('git commit -m "GitLab"', tmp_conf_path)
raise "Git add failed." unless [0,1].include?(status) raise "Git add failed." unless [0,1].include?(status)
output, status = popen('git push') output, status = popen('git push', tmp_conf_path)
if output =~ /remote\: FATAL/ if output =~ /remote\: FATAL/
raise BrokenGitolite, output raise BrokenGitolite, output
...@@ -230,20 +234,8 @@ module Gitlab ...@@ -230,20 +234,8 @@ module Gitlab
end end
end end
def popen(cmd, path = nil) def tmp_conf_path
path ||= File.join(config_tmp_dir,'gitolite') File.join(config_tmp_dir,'gitolite')
vars = { "PWD" => path }
options = { :chdir => path }
@cmd_output = ""
@cmd_status = 0
Open3.popen3(vars, cmd, options) do |stdin, stdout, stderr, wait_thr|
@cmd_status = wait_thr.value.exitstatus
@cmd_output << stdout.read
@cmd_output << stderr.read
end
return @cmd_output, @cmd_status
end end
end end
end end
module Gitlab
module Popen
def popen(cmd, path)
vars = { "PWD" => path }
options = { :chdir => path }
@cmd_output = ""
@cmd_status = 0
Open3.popen3(vars, cmd, options) do |stdin, stdout, stderr, wait_thr|
@cmd_status = wait_thr.value.exitstatus
@cmd_output << stdout.read
@cmd_output << stderr.read
end
return @cmd_output, @cmd_status
end
end
end
...@@ -3,6 +3,8 @@ module Gitlab ...@@ -3,6 +3,8 @@ module Gitlab
module Satellite module Satellite
class Satellite class Satellite
include Gitlab::Popen
PARKING_BRANCH = "__parking_branch" PARKING_BRANCH = "__parking_branch"
attr_accessor :project attr_accessor :project
...@@ -24,8 +26,10 @@ module Gitlab ...@@ -24,8 +26,10 @@ module Gitlab
end end
def create def create
create_cmd = "git clone #{project.url_to_repo} #{path}" output, status = popen("git clone #{project.url_to_repo} #{path}",
if system(create_cmd) Gitlab.config.satellites.path)
if status.zero?
true true
else else
Gitlab::GitLogger.error("Failed to create satellite for #{project.name_with_namespace}") Gitlab::GitLogger.error("Failed to create satellite for #{project.name_with_namespace}")
...@@ -66,6 +70,10 @@ module Gitlab ...@@ -66,6 +70,10 @@ module Gitlab
@repo ||= Grit::Repo.new(path) @repo ||= Grit::Repo.new(path)
end end
def destroy
FileUtils.rm_rf(path)
end
private private
# Clear the working directory # Clear the working directory
......
...@@ -6,7 +6,7 @@ namespace :sidekiq do ...@@ -6,7 +6,7 @@ namespace :sidekiq do
desc "GITLAB | Start sidekiq" desc "GITLAB | Start sidekiq"
task :start do task :start do
run "nohup bundle exec sidekiq -q post_receive,mailer,system_hook,project_web_hook,common,default -e #{Rails.env} -P #{pidfile} >> #{Rails.root.join("log", "sidekiq.log")} 2>&1 &" run "nohup bundle exec sidekiq -q post_receive,mailer,system_hook,project_web_hook,gitolite,common,default -e #{Rails.env} -P #{pidfile} >> #{Rails.root.join("log", "sidekiq.log")} 2>&1 &"
end end
def pidfile def pidfile
......
...@@ -12,7 +12,7 @@ FactoryGirl.define do ...@@ -12,7 +12,7 @@ FactoryGirl.define do
factory :user, aliases: [:author, :assignee, :owner, :creator] do factory :user, aliases: [:author, :assignee, :owner, :creator] do
email { Faker::Internet.email } email { Faker::Internet.email }
name name
username { Faker::Internet.user_name } sequence(:username) { |n| "#{Faker::Internet.user_name}#{n}" }
password "123456" password "123456"
password_confirmation { password } password_confirmation { password }
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Gitolite do describe Gitlab::Gitolite do
let(:project) { double('Project', path: 'diaspora') } let(:project) { double('Project', id: 7, path: 'diaspora') }
let(:gitolite_config) { double('Gitlab::GitoliteConfig') } let(:gitolite_config) { double('Gitlab::GitoliteConfig') }
let(:gitolite) { Gitlab::Gitolite.new } let(:gitolite) { Gitlab::Gitolite.new }
before do before do
gitolite.stub(config: gitolite_config) gitolite.stub(config: gitolite_config)
Project.stub(find: project)
end end
it { should respond_to :set_key } it { should respond_to :set_key }
...@@ -20,6 +21,6 @@ describe Gitlab::Gitolite do ...@@ -20,6 +21,6 @@ describe Gitlab::Gitolite do
it "should call config update" do it "should call config update" do
gitolite_config.should_receive(:update_project!) gitolite_config.should_receive(:update_project!)
gitolite.update_repository project gitolite.update_repository(project.id)
end end
end end
...@@ -21,6 +21,10 @@ class Project ...@@ -21,6 +21,10 @@ class Project
true true
end end
def destroy
true
end
def create def create
true true
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment