Implement updating of mirrors.

app/controllers/projects_controller.rb

@@ -89,7 +89,8 @@ class ProjectsController < ApplicationController
   end
 
   def show
-    if @project.import_in_progress?
+    # If we're importing while we do have a repository, we're simply updating the mirror.
+    if @project.import_in_progress? && !@project.updating_mirror?
       redirect_to namespace_project_import_path(@project.namespace, @project)
       return
     end

app/models/project.rb

@@ -328,6 +328,14 @@ class Project < ActiveRecord::Base
   end
 
   def add_import_job
+    if repository_exists?
+      if mirror?
+        RepositoryUpdateMirrorWorker.perform_async(self.id)
+      end
+
+      return
+    end
+
     if forked?
       RepositoryForkWorker.perform_async(self.id, forked_from_project.path_with_namespace, self.namespace.path)
     else
@@ -417,6 +425,12 @@ class Project < ActiveRecord::Base
     end
   end
 
+  def fetch_mirror
+    return unless mirror?
+
+    repository.fetch_upstream(self.import_url)
+  end
+
   def check_limit
     unless creator.can_create_project? or namespace.kind == 'group'
       errors[:limit_reached] << ("Your project limit is #{creator.projects_limit} projects! Please contact your administrator to increase it")

app/models/repository.rb

@@ -4,6 +4,8 @@ class Repository
   class PreReceiveError < StandardError; end
   class CommitError < StandardError; end
 
+  MIRROR_REMOTE = "upstream"
+
   include Gitlab::ShellAdapter
 
   attr_accessor :raw_repository, :path_with_namespace, :project
@@ -128,10 +130,24 @@ class Repository
     gitlab_shell.rm_tag(path_with_namespace, tag_name)
   end
 
+  def add_remote(name, url)
+    raw_repository.remote_add(name, url)
+  rescue Rugged::ConfigError
+    raw_repository.remote_update(name, url: url)
+  end
+
+  def fetch_remote(remote)
+    gitlab_shell.fetch_remote(path_with_namespace, remote)
+  end
+
   def branch_names
     cache.fetch(:branch_names) { raw_repository.branch_names }
   end
 
+  def branch_exists?(name)
+    branch_names.include?(name)
+  end
+
   def tag_names
     cache.fetch(:tag_names) { raw_repository.tag_names }
   end
@@ -517,6 +533,35 @@ class Repository
     end
   end
 
+  def fetch_upstream(url)
+    add_remote(Repository::MIRROR_REMOTE, url)
+    fetch_remote(Repository::MIRROR_REMOTE)
+  end
+
+  def upstream_branches
+    rugged.references.each("refs/remotes/#{Repository::MIRROR_REMOTE}/*").map do |ref|
+      name = ref.name.sub(/\Arefs\/remotes\/#{Repository::MIRROR_REMOTE}\//, "")
+      source_sha = ref.target.oid
+
+      begin
+        Gitlab::Git::Branch.new(name, ref.target)
+      rescue Rugged::ReferenceError
+        # Omit invalid branch
+      end
+    end.compact
+  end
+
+  def diverged_from_upstream?(branch_name)
+    branch_commit = commit(branch_name)
+    upstream_commit = commit("refs/remotes/#{MIRROR_REMOTE}/#{branch_name}")
+
+    if upstream_commit
+      !is_ancestor?(branch_commit.id, upstream_commit.id)
+    else
+      false
+    end
+  end
+
   def merge_base(first_commit_id, second_commit_id)
     rugged.merge_base(first_commit_id, second_commit_id)
   end

app/services/projects/update_mirror_service.rb

+module Projects
+  class UpdateMirrorService < BaseService
+    class FetchError < StandardError; end
+
+    def execute
+      return false unless project.mirror?
+
+      begin
+        update_tags do
+          project.fetch_mirror
+        end
+      rescue Gitlab::Shell::Error => e
+        raise FetchError, e.message
+      end
+
+      update_branches
+
+      true
+    end
+
+    private
+
+    def update_branches
+      local_branches = repository.branches.each_with_object({}) { |branch, branches| branches[branch.name] = branch }
+
+      repository.upstream_branches.each do |upstream_branch|
+        name = upstream_branch.name
+
+        local_branch = local_branches[name]
+
+        if local_branch.nil?
+          CreateBranchService.new(project, current_user).execute(name, upstream_branch.target)
+        elsif local_branch.target == upstream_branch.target
+          # Already up to date
+        elsif repository.diverged_from_upstream?(name)
+          # Cannot be updated
+        else
+          repository.ff_merge(current_user, upstream_branch.target, name)
+        end
+      end
+    end
+
+    def update_tags(&block)
+      old_tags = repository.tags.each_with_object({}) { |tag, tags| tags[tag.name] = tag }
+
+      fetch_result = block.call
+      return fetch_result unless fetch_result
+
+      repository.expire_tags_cache
+
+      tags = repository.tags
+
+      tags.each do |tag|
+        old_tag = old_tags[tag.name]
+        old_tag_target = old_tag ? old_tag.target : Gitlab::Git::BLANK_SHA
+
+        next if old_tag_target == tag.target
+
+        GitTagPushService.new.execute(project, current_user, old_tag_target, tag.target, "#{Gitlab::Git::TAG_REF_PREFIX}#{tag.name}")
+      end
+
+      fetch_result
+    end
+  end
+end

app/workers/repository_update_mirror_worker.rb

+class RepositoryUpdateMirrorWorker
+  include Sidekiq::Worker
+  include Gitlab::ShellAdapter
+
+  sidekiq_options queue: :gitlab_shell
+
+  attr_accessor :project, :repository, :current_user
+
+  def perform(project_id)
+    @project = Project.find(project_id)
+    # TODO: Use actual user
+    @current_user = User.last
+
+    begin
+      Projects::UpdateMirrorService.new(@project, @current_user).execute
+    rescue Projects::UpdateMirrorService::FetchError => e
+      project.update(import_error: e.message)
+      project.import_fail
+      return
+    end
+
+    project.import_finish
+  end
+end

app/workers/update_all_mirrors_worker.rb

+class UpdateAllMirrorsWorker
+  include Sidekiq::Worker
+  include Sidetiq::Schedulable
+
+  recurrence { hourly }
+
+  def perform
+    Project.mirror.each(&:update_mirror)
+  end
+end

lib/gitlab/backend/shell.rb

@@ -41,6 +41,20 @@ module Gitlab
       true
     end
 
+    # Fetch remote for repository
+    #
+    # name - project path with namespace
+    # remote - remote name
+    #
+    # Ex.
+    #   fetch_remote("gitlab/gitlab-ci", "upstream")
+    #
+    def fetch_remote(name, remote)
+      output, status = Popen::popen([gitlab_shell_projects_path, 'fetch-remote', "#{name}.git", remote, '600'])
+      raise Error, output unless status.zero?
+      true
+    end
+
     # Move repository
     #
     # path - project path with namespace