Add the ability to sync remote mirrors.

CHANGELOG

@@ -172,6 +172,7 @@ v 8.5.8
 
 v 8.5.7
   - Bump Git version requirement to 2.7.3
+  - Add ability to sync to remote mirrors.
 
 v 8.5.6
   - Obtain a lease before querying LDAP

app/assets/stylesheets/pages/projects.scss

@@ -461,7 +461,7 @@ pre.light-well {
 
 .cannot-be-merged,
 .cannot-be-merged:hover {
-  color: #e62958;
+  color: $error-exclamation-point;
   margin-top: 2px;
 }
 
@@ -474,3 +474,7 @@ pre.light-well {
     color: #fff;
   }
 }
+
+.disabled-item {
+  @extend .btn.disabled;
+}

app/controllers/projects/mirrors_controller.rb

@@ -2,6 +2,7 @@ class Projects::MirrorsController < Projects::ApplicationController
   # Authorize
   before_action :authorize_admin_project!, except: [:update_now]
   before_action :authorize_push_code!, only: [:update_now]
+  before_action :remote_mirror, only: [:show, :update]
 
   layout "project_settings"
 
@@ -34,9 +35,21 @@ class Projects::MirrorsController < Projects::ApplicationController
     redirect_back_or_default(default: namespace_project_path(@project.namespace, @project))
   end
 
+  def update_remote_now
+    @project.update_remote_mirrors
+
+    flash[:notice] = "The remote repository is being updated..."
+    redirect_back_or_default(default: namespace_project_path(@project.namespace, @project))
+  end
+
   private
 
+  def remote_mirror
+    @remote_mirror = @project.remote_mirrors.first_or_initialize
+  end
+
   def mirror_params
-    params.require(:project).permit(:mirror, :import_url, :mirror_user_id, :mirror_trigger_builds)
+    params.require(:project).permit(:mirror, :import_url, :mirror_user_id, :mirror_trigger_builds,
+      remote_mirrors_attributes: [:url, :id, :enabled])
   end
 end

app/models/project.rb

@@ -2,41 +2,54 @@
 #
 # Table name: projects
 #
-#  id                     :integer          not null, primary key
-#  name                   :string(255)
-#  path                   :string(255)
-#  description            :text
-#  created_at             :datetime
-#  updated_at             :datetime
-#  creator_id             :integer
-#  issues_enabled         :boolean          default(TRUE), not null
-#  wall_enabled           :boolean          default(TRUE), not null
-#  merge_requests_enabled :boolean          default(TRUE), not null
-#  wiki_enabled           :boolean          default(TRUE), not null
-#  namespace_id           :integer
-#  issues_tracker         :string(255)      default("gitlab"), not null
-#  issues_tracker_id      :string(255)
-#  snippets_enabled       :boolean          default(TRUE), not null
-#  last_activity_at       :datetime
-#  import_url             :string(255)
-#  visibility_level       :integer          default(0), not null
-#  archived               :boolean          default(FALSE), not null
-#  avatar                 :string(255)
-#  import_status          :string(255)
-#  repository_size        :float            default(0.0)
-#  star_count             :integer          default(0), not null
-#  import_type            :string(255)
-#  import_source          :string(255)
-#  commit_count           :integer          default(0)
-#  import_error           :text
-#  ci_id                  :integer
-#  builds_enabled         :boolean          default(TRUE), not null
-#  shared_runners_enabled :boolean          default(TRUE), not null
-#  runners_token          :string
-#  build_coverage_regex   :string
-#  build_allow_git_fetch  :boolean          default(TRUE), not null
-#  build_timeout          :integer          default(3600), not null
-#  pending_delete         :boolean
+#  id                               :integer          not null, primary key
+#  name                             :string(255)
+#  path                             :string(255)
+#  description                      :text
+#  created_at                       :datetime
+#  updated_at                       :datetime
+#  creator_id                       :integer
+#  issues_enabled                   :boolean          default(TRUE), not null
+#  wall_enabled                     :boolean          default(TRUE), not null
+#  merge_requests_enabled           :boolean          default(TRUE), not null
+#  wiki_enabled                     :boolean          default(TRUE), not null
+#  namespace_id                     :integer
+#  issues_tracker                   :string(255)      default("gitlab"), not null
+#  issues_tracker_id                :string(255)
+#  snippets_enabled                 :boolean          default(TRUE), not null
+#  last_activity_at                 :datetime
+#  import_url                       :string(255)
+#  visibility_level                 :integer          default(0), not null
+#  archived                         :boolean          default(FALSE), not null
+#  avatar                           :string(255)
+#  import_status                    :string(255)
+#  repository_size                  :float            default(0.0)
+#  star_count                       :integer          default(0), not null
+#  import_type                      :string(255)
+#  import_source                    :string(255)
+#  commit_count                     :integer          default(0)
+#  import_error                     :text
+#  ci_id                            :integer
+#  builds_enabled                   :boolean          default(TRUE), not null
+#  shared_runners_enabled           :boolean          default(TRUE), not null
+#  runners_token                    :string
+#  build_coverage_regex             :string
+#  build_allow_git_fetch            :boolean          default(TRUE), not null
+#  build_timeout                    :integer          default(3600), not null
+#  pending_delete                   :boolean          default(FALSE)
+#  public_builds                    :boolean          default(TRUE), not null
+#  merge_requests_template          :text
+#  merge_requests_rebase_enabled    :boolean          default(FALSE)
+#  approvals_before_merge           :integer          default(0), not null
+#  reset_approvals_on_push          :boolean          default(TRUE)
+#  merge_requests_ff_only_enabled   :boolean          default(FALSE)
+#  issues_template                  :text
+#  mirror                           :boolean          default(FALSE), not null
+#  mirror_last_update_at            :datetime
+#  mirror_last_successful_update_at :datetime
+#  mirror_user_id                   :integer
+#  mirror_trigger_builds            :boolean          default(FALSE), not null
+#  main_language                    :string
 #
 
 require 'carrierwave/orm/activerecord'
@@ -76,18 +89,8 @@ class Project < ActiveRecord::Base
 
   after_destroy :remove_pages
 
-  # update visibility_level of forks
   after_update :update_forks_visibility_level
-  def update_forks_visibility_level
-    return unless visibility_level < visibility_level_was
-
-    forks.each do |forked_project|
-      if forked_project.visibility_level > visibility_level
-        forked_project.visibility_level = visibility_level
-        forked_project.save!
-      end
-    end
-  end
+  after_update :remove_mirror_repository_reference, if: :import_url_changed?
 
   ActsAsTaggableOn.strict_case_match = true
   acts_as_taggable_on :tags
@@ -175,8 +178,11 @@ class Project < ActiveRecord::Base
   has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner'
   has_many :variables, dependent: :destroy, class_name: 'Ci::Variable', foreign_key: :gl_project_id
   has_many :triggers, dependent: :destroy, class_name: 'Ci::Trigger', foreign_key: :gl_project_id
+  has_many :remote_mirrors, dependent: :destroy
 
   accepts_nested_attributes_for :variables, allow_destroy: true
+  accepts_nested_attributes_for :remote_mirrors,
+    allow_destroy: true, reject_if: proc { |attrs| attrs[:id].blank? && attrs[:url].blank? }
 
   delegate :name, to: :owner, allow_nil: true, prefix: true
   delegate :members, to: :team, prefix: true
@@ -204,6 +210,7 @@ class Project < ActiveRecord::Base
     url: { protocols: %w(ssh git http https) },
     if: :external_import?
   validates :import_url, presence: true, if: :mirror?
+  validate  :import_url_availability, if: :import_url_changed?
   validates :mirror_user, presence: true, if: :mirror?
   validates :star_count, numericality: { greater_than_or_equal_to: 0 }
   validate :check_limit, on: :create
@@ -216,6 +223,7 @@ class Project < ActiveRecord::Base
 
   add_authentication_token_field :runners_token
   before_save :ensure_runners_token
+  before_validation :mark_remote_mirrors_for_removal
 
   mount_uploader :avatar, AvatarUploader
 
@@ -236,6 +244,7 @@ class Project < ActiveRecord::Base
   scope :non_archived, -> { where(archived: false) }
   scope :mirror, -> { where(mirror: true) }
   scope :for_milestones, ->(ids) { joins(:milestones).where('milestones.id' => ids).distinct }
+  scope :with_remote_mirrors, -> { joins(:remote_mirrors).where(remote_mirrors: { enabled: true }).distinct }
 
   state_machine :import_status, initial: :none do
     event :import_start do
@@ -524,6 +533,23 @@ class Project < ActiveRecord::Base
     end
   end
 
+  def mark_import_as_failed(error_message)
+    import_fail
+    update_column(:import_error, error_message)
+  end
+
+  def remote_mirror?
+    remote_mirrors.enabled.exists?
+  end
+
+  def updating_remote_mirror?
+    remote_mirrors.enabled.started.exists?
+  end
+
+  def update_remote_mirrors
+    remote_mirrors.each(&:sync)
+  end
+
   def fetch_mirror
     return unless mirror?
 
@@ -1184,4 +1210,31 @@ class Project < ActiveRecord::Base
   def ff_merge_must_be_possible?
     self.merge_requests_ff_only_enabled || self.merge_requests_rebase_enabled
   end
+
+  private
+
+  def remove_mirror_repository_reference
+    repository.remove_remote(Repository::MIRROR_REMOTE)
+  end
+
+  def update_forks_visibility_level
+    return unless visibility_level < visibility_level_was
+
+    forks.each do |forked_project|
+      if forked_project.visibility_level > visibility_level
+        forked_project.visibility_level = visibility_level
+        forked_project.save!
+      end
+    end
+  end
+
+  def import_url_availability
+    if remote_mirrors.find_by(url: import_url)
+      errors.add(:import_url, 'is already in use by the remote mirror')
+    end
+  end
+
+  def mark_remote_mirrors_for_removal
+    remote_mirrors.each(&:mark_for_delete_if_blank_url)
+  end
 end

app/models/remote_mirror.rb

+# == Schema Information
+#
+# Table name: remote_mirrors
+#
+#  id                        :integer          not null, primary key
+#  project_id                :integer
+#  url                       :string
+#  last_update_at            :datetime
+#  last_error                :string
+#  created_at                :datetime         not null
+#  updated_at                :datetime         not null
+#  last_successful_update_at :datetime
+#  update_status             :string
+#  enabled                   :boolean          default(TRUE)
+#
+
+class RemoteMirror < ActiveRecord::Base
+  include AfterCommitQueue
+
+  attr_encrypted :credentials, key: Gitlab::Application.secrets.db_key_base, marshal: true, encode: true, mode: :per_attribute_iv_and_salt
+
+  belongs_to :project
+
+  validates :url, presence: true, url: { protocols: %w(ssh git http https), allow_blank: true }, on: :create
+  validate  :url_availability, if: :url_changed?
+
+  after_save :refresh_remote, if: :url_changed?
+  after_update :reset_fields, if: :url_changed?
+  after_destroy :remove_remote
+
+  scope :enabled, -> { where(enabled: true) }
+  scope :started, -> { with_update_status(:started) }
+
+  state_machine :update_status, initial: :none do
+    event :update_start do
+      transition [:none, :finished] => :started
+    end
+
+    event :update_finish do
+      transition started: :finished
+    end
+
+    event :update_fail do
+      transition started: :failed
+    end
+
+    event :update_retry do
+      transition failed: :started
+    end
+
+    state :started
+    state :finished
+    state :failed
+
+    after_transition any => :started, do: :schedule_update_job
+
+    after_transition started: :finished do |remote_mirror, transaction|
+      timestamp = DateTime.now
+      remote_mirror.update_attributes!(
+        last_update_at: timestamp, last_successful_update_at: timestamp, last_error: nil
+      )
+    end
+
+    after_transition started: :failed do |remote_mirror, transaction|
+      remote_mirror.update(last_update_at: DateTime.now)
+    end
+  end
+
+  def ref_name
+    "remote_mirror_#{id}"
+  end
+
+  def update_failed?
+    update_status == 'failed'
+  end
+
+  def update_in_progress?
+    update_status == 'started'
+  end
+
+  def sync
+    return if !enabled || update_in_progress?
+
+    update_failed? ?  update_retry : update_start
+  end
+
+  def mark_for_delete_if_blank_url
+    mark_for_destruction if url.blank?
+  end
+
+  def mark_as_failed(error_message)
+    update_fail
+    update_column(:last_error, error_message)
+  end
+
+  def url=(value)
+    mirror_url = Gitlab::ImportUrl.new(value)
+    self.credentials = mirror_url.credentials if mirror_url.credentials.values.any?
+
+    super(mirror_url.sanitized_url)
+  end
+
+  def full_url
+    mirror_url = Gitlab::ImportUrl.new(super, credentials: credentials)
+    mirror_url.full_url
+  end
+
+  private
+
+  def url_availability
+    if project.import_url == url
+      errors.add(:url, 'is already in use')
+    end
+  end
+
+  def reset_fields
+    update_columns(
+      last_error: nil,
+      last_update_at: nil,
+      last_successful_update_at: nil,
+      update_status: 'finished'
+    )
+  end
+
+  def schedule_update_job
+    run_after_commit(:add_update_job)
+  end
+
+  def add_update_job
+    if project.repository_exists?
+      RepositoryUpdateRemoteMirrorWorker.perform_async(self.id)
+    end
+  end
+
+  def refresh_remote
+    project.repository.remove_remote(ref_name)
+    project.repository.add_remote(ref_name, full_url)
+  end
+
+  def remove_remote
+    project.repository.remove_remote(ref_name)
+  end
+end

app/models/repository.rb

@@ -183,12 +183,27 @@ class Repository
     gitlab_shell.rm_tag(path_with_namespace, tag_name)
   end
 
+  def push_branches(project_name, remote, branch_names)
+    gitlab_shell.push_branches(project_name, remote, branch_names)
+  end
+
+  def delete_remote_branches(project_name, remote, branch_names)
+    gitlab_shell.delete_remote_branches(project_name, remote, branch_names)
+  end
+
   def add_remote(name, url)
     raw_repository.remote_add(name, url)
   rescue Rugged::ConfigError
     raw_repository.remote_update(name, url: url)
   end
 
+  def remove_remote(name)
+    raw_repository.remote_delete(name)
+    true
+  rescue Rugged::ConfigError
+    false
+  end
+
   def set_remote_as_mirror(name)
     remote_config = raw_repository.rugged.config
 
@@ -667,6 +682,10 @@ class Repository
 
   alias_method :branches, :local_branches
 
+  def remote_branches(remote_name)
+    branches_from_ref("remotes/#{remote_name}")
+  end
+
   def tags
     @tags ||= raw_repository.tags
   end
@@ -841,9 +860,9 @@ class Repository
     fetch_remote_forced!(Repository::MIRROR_GEO)
   end
 
-  def upstream_branches
-    rugged.references.each("refs/remotes/#{Repository::MIRROR_REMOTE}/*").map do |ref|
-      name = ref.name.sub(/\Arefs\/remotes\/#{Repository::MIRROR_REMOTE}\//, "")
+  def branches_from_ref(ref_name)
+    rugged.references.each("refs/#{ref_name}/*").map do |ref|
+      name = ref.name.sub(/\Arefs\/#{ref_name}\//, "")
 
       begin
         Gitlab::Git::Branch.new(name, ref.target)
@@ -853,6 +872,10 @@ class Repository
     end.compact
   end
 
+  def upstream_branches
+    branches_from_ref("remotes/#{Repository::MIRROR_REMOTE}")
+  end
+
   def diverged_from_upstream?(branch_name)
     branch_commit = commit(branch_name)
     upstream_commit = commit("refs/remotes/#{MIRROR_REMOTE}/#{branch_name}")
@@ -864,6 +887,17 @@ class Repository
     end
   end
 
+  def upstream_has_diverged?(branch_name, remote_ref)
+    branch_commit = commit(branch_name)
+    upstream_commit = commit("refs/remotes/#{remote_ref}/#{branch_name}")
+
+    if upstream_commit
+      !is_ancestor?(upstream_commit.id, branch_commit.id)
+    else
+      false
+    end
+  end
+
   def up_to_date_with_upstream?(branch_name)
     branch_commit = commit(branch_name)
     upstream_commit = commit("refs/remotes/#{MIRROR_REMOTE}/#{branch_name}")

app/services/projects/update_remote_mirror_service.rb

+module Projects
+  class UpdateRemoteMirrorService < BaseService
+    attr_reader :mirror, :errors
+
+    def execute(remote_mirror)
+      @mirror = remote_mirror
+      @errors = []
+
+      begin
+        repository.fetch_remote(mirror.ref_name)
+
+        if divergent_branches.present?
+          errors << "The following branches have diverged from their local counterparts: #{divergent_branches.to_sentence}"
+        end
+
+        push_to_mirror if changed_branches.present?
+        delete_from_mirror if deleted_branches.present?
+      rescue Gitlab::Shell::Error => e
+        errors << e.message.strip
+      end
+
+      if errors.present?
+        error(errors.join("\n\n"))
+      else
+        success
+      end
+    end
+
+    private
+
+    def changed_branches
+      @changed_branches ||= local_branches.each_with_object([]) do |(name, branch), branches|
+        remote_branch = remote_branches[name]
+
+        if remote_branch.nil?
+          branches << name
+        elsif branch.target == remote_branch.target
+          # Already up to date
+        elsif !repository.upstream_has_diverged?(name, mirror.ref_name)
+          branches << name
+        end
+      end
+    end
+
+    def deleted_branches
+      @deleted_branches ||= remote_branches.each_with_object([]) do |(name, branch), branches|
+        local_branch = local_branches[name]
+
+        if local_branch.nil? && project.commit(branch.target)
+          branches << name
+        end
+      end
+    end
+
+    def push_to_mirror
+      default_branch, branches = changed_branches.partition { |name| project.default_branch == name }
+
+      # Push the default branch first so it works fine when remote mirror is empty.
+      branches.unshift(*default_branch)
+
+      repository.push_branches(project.path_with_namespace, mirror.ref_name, branches)
+    end
+
+    def delete_from_mirror
+      repository.delete_remote_branches(project.path_with_namespace, mirror.ref_name, deleted_branches)
+    end
+
+    def local_branches
+      @local_branches ||= repository.local_branches.each_with_object({}) do |branch, branches|
+        branches[branch.name] = branch
+      end
+    end
+
+    def remote_branches
+      @remote_branches ||= repository.remote_branches(mirror.ref_name).each_with_object({}) do |branch, branches|
+        branches[branch.name] = branch
+      end
+    end
+
+    def divergent_branches
+      remote_branches.each_with_object([]) do |(name, branch), branches|
+        if local_branches[name] && repository.upstream_has_diverged?(name, mirror.ref_name)
+          branches << name
+        end
+      end
+    end
+
+  end
+end

app/views/projects/buttons/_update_mirror.html.haml

-- if @project.mirror? && can?(current_user, :push_code, @project)
-  - size = nil unless defined?(size) && size
-  - if @project.updating_mirror?
-    %span.btn.disabled.update-mirror-button.has-tooltip{title: "Updating from upstream..."}
-      = icon('refresh')
-  - else
-    = link_to update_now_namespace_project_mirror_path(@project.namespace, @project), method: :post, class: "btn update-mirror-button has-tooltip", title: "Update from upstream" do
-      = icon('refresh')
+- if can?(current_user, :push_code, @project)
+  - if !@project.remote_mirror? && @project.mirror?
+    - size = nil unless defined?(size) && size
+    - if @project.updating_mirror?
+      %span.btn.disabled.update-mirror-button.has_tooltip{title: "Updating from upstream..."}
+        = icon('refresh')
+    - else
+      = link_to update_now_namespace_project_mirror_path(@project.namespace, @project), method: :post, class: "btn update-mirror-button has_tooltip", title: "Update from upstream" do
+        = icon('refresh')
+  - elsif @project.remote_mirror? && !@project.mirror?
+    - if @project.updating_remote_mirror?
+      %span.btn.disabled.update-mirror-button.has_tooltip{title: "Updating remote repository..."}
+        = icon('refresh')
+    - else
+      = link_to update_remote_now_namespace_project_mirror_path(@project.namespace, @project), method: :post, class: "btn update-mirror-button has_tooltip", title: "Update remote repository" do
+        = icon('refresh')
+  - elsif @project.remote_mirror? && @project.mirror?
+    .btn-group
+      %a.btn.dropdown-toggle{href: '#', 'data-toggle' => 'dropdown'}
+        = icon('refresh')
+      %ul.dropdown-menu.dropdown-menu-right
+        %li
+          - if @project.updating_mirror?
+            %a.disabled-item Updating from upstream...
+          - else
+            = link_to "Update this repository", update_now_namespace_project_mirror_path(@project.namespace, @project), method: :post
+        %li
+          - if @project.updating_remote_mirror?
+            %a.disabled-item Updating remote repository...
+          - else
+            = link_to "Update remote repository", update_remote_now_namespace_project_mirror_path(@project.namespace, @project), method: :post

app/views/projects/mirrors/show.html.haml

 - page_title "Mirror Repository"
-.pull-right
-  - if @project.mirror_last_update_success?
-    Successfully updated #{time_ago_with_tooltip(@project.mirror_last_successful_update_at)}.
-  %span.prepend-left-10
-    = render "shared/mirror_update_button"
-%h3.page-title
+%h3.page_title
   Mirror Repository
 
 %p.light
-  Set up your project to automatically have its branches, tags, and commits updated from an upstream repository every hour.
+  A repository can be setup as a mirror of another repository, and can also have a remote mirror associated.
 
-%hr.clearfix
+%p.light
+  When the repository is configured as a mirror, all of its content will automatically be updated from the repository configured in the <strong>Pull from a remote repository</strong> section.
 
-- if @project.mirror_last_update_failed?
-  .panel.panel-danger
-    .panel-heading
-      The repository failed to update #{time_ago_with_tooltip(@project.mirror_last_update_at)}.
-      - if @project.mirror_ever_updated_successfully?
-        Last successful update #{time_ago_with_tooltip(@project.mirror_last_successful_update_at)}.
-    .panel-body
-      %pre
-        :preserve
-          #{@project.import_error.try(:strip)}
+%p.light
+  When the repository has a remote mirror associated, it means that the remote repository configured in the <strong>Push to a remote repository</strong> section will automatically receive updates from the current repository.
+
+%hr.clearfix
 
 = form_for @project, url: namespace_project_mirror_path(@project.namespace, @project), html: { class: 'form-horizontal' } do |f|
   - if @project.errors.any?
@@ -29,6 +19,25 @@
       - @project.errors.full_messages.each do |msg|
         %p= msg
 
+  %h4 Pull from a remote repository
+  %p.light
+    Set up your project to automatically have its branches, tags, and commits updated from an upstream repository every hour.
+  = render "shared/mirror_update_button"
+  - if @project.mirror_last_update_success?
+    %span &nbsp;Successfully updated #{time_ago_with_tooltip(@project.mirror_last_successful_update_at)}.
+  %hr.clearfix
+
+  - if @project.mirror_last_update_failed?
+    .panel.panel-danger
+      .panel-heading
+        The repository failed to update #{time_ago_with_tooltip(@project.mirror_last_update_at)}.
+        - if @project.mirror_ever_updated_successfully?
+          Last successful update #{time_ago_with_tooltip(@project.mirror_last_successful_update_at)}.
+      .panel-body
+        %pre
+          :preserve
+            #{@project.import_error.try(:strip)}
+
   .form-group
     .col-sm-offset-2.col-sm-10
       .checkbox
@@ -69,5 +78,44 @@
   - if @project.builds_enabled?
     = render 'shared/mirror_trigger_builds_setting', f: f
 
+  %h4 Push to a remote repository
+  %p.light
+    Set up the remote repository that you want to update with the content of the current repository every hour.
+  = render "shared/remote_mirror_update_button", remote_mirror: @remote_mirror
+  - if @remote_mirror.last_successful_update_at
+    %span &nbsp;Successfully updated #{time_ago_with_tooltip(@remote_mirror.last_successful_update_at)}.
+
+  %hr.clearfix
+
+  - if @remote_mirror.last_error.present?
+    .panel.panel-danger
+      .panel-heading
+        The remote repository failed to update #{time_ago_with_tooltip(@remote_mirror.last_update_at)}.
+        - if @remote_mirror.last_successful_update_at
+          Last successful update #{time_ago_with_tooltip(@remote_mirror.last_successful_update_at)}.
+      .panel-body
+        %pre
+          :preserve
+            #{@remote_mirror.last_error.strip}
+
+  = f.fields_for :remote_mirrors, @remote_mirror do |rm_form|
+    .form-group
+      .col-sm-offset-2.col-sm-10
+        .checkbox
+          = rm_form.label :enabled do
+            = rm_form.check_box :enabled
+            %strong
+              Remote Mirror Repository
+          .help-block
+            Automatically update the remote mirror's branches, tags, and commits from this repository every hour.
+
+    .form-group.has-feedback
+      = rm_form.label :url, class: 'control-label' do
+        %span Git repository URL
+      .col-sm-10
+        = rm_form.text_field :url, class: 'form-control', placeholder: 'https://username:password@gitlab.company.com/group/project.git'
+        .well.prepend-top-20
+          The requirements for the URL format are the same mentioned in the <strong>Pull from a remote repository</strong> section.
+
   .form-actions
     = f.submit "Save Changes", class: "btn btn-create"

app/views/shared/_remote_mirror_update_button.html.haml

+- if @project.remote_mirror?
+  - if remote_mirror.update_in_progress?
+    %span.btn.disabled
+      = icon('refresh')
+      Updating…
+  - else
+    = link_to update_remote_now_namespace_project_mirror_path(@project.namespace, @project), method: :post, class: "btn" do
+      = icon('refresh')
+      Update Now
+

app/workers/repository_update_mirror_worker.rb

@@ -12,8 +12,7 @@ class RepositoryUpdateMirrorWorker
 
     result = Projects::UpdateMirrorService.new(@project, @current_user).execute
     if result[:status] == :error
-      project.update(import_error: result[:message])
-      project.import_fail
+      project.mark_import_as_failed(result[:message])
       return
     end
 

app/workers/repository_update_remote_mirror_worker.rb

+class RepositoryUpdateRemoteMirrorWorker
+  include Sidekiq::Worker
+  include Gitlab::ShellAdapter
+
+  sidekiq_options queue: :gitlab_shell
+
+  def perform(remote_mirror_id)
+    remote_mirror = RemoteMirror.find(remote_mirror_id)
+    project       = remote_mirror.project
+    current_user  = project.creator
+    result        = Projects::UpdateRemoteMirrorService.new(project, current_user).execute(remote_mirror)
+
+    if result[:status] == :error
+      remote_mirror.mark_as_failed(result[:message])
+    else
+      remote_mirror.update_finish
+    end
+  end
+end

app/workers/update_all_mirrors_worker.rb

@@ -13,8 +13,7 @@ class UpdateAllMirrorsWorker
       where('mirror_last_update_at < ?', 1.day.ago)
 
     stuck.find_each(batch_size: 50) do |project|
-      project.import_fail
-      project.update_attribute(:import_error, 'The mirror update took too long to complete.')
+      project.mark_import_as_failed('The mirror update took too long to complete.')
     end
   end
 end

app/workers/update_all_remote_mirrors_worker.rb

+class UpdateAllRemoteMirrorsWorker
+  include Sidekiq::Worker
+
+  def perform
+    fail_stuck_mirrors!
+
+    RemoteMirror.find_each(batch_size: 50).each(&:sync)
+  end
+
+  def fail_stuck_mirrors!
+    stuck = RemoteMirror.started.
+      where("last_update_at < ?", 1.day.ago)
+
+    stuck.find_each(batch_size: 50) do |remote_mirror|
+      remote_mirror.mark_as_failed('The mirror update took too long to complete.')
+    end
+  end
+end

config/initializers/1_settings.rb

@@ -326,6 +326,9 @@ Settings.cron_jobs['historical_data_worker']['job_class'] = 'HistoricalDataWorke
 Settings.cron_jobs['update_all_mirrors_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['update_all_mirrors_worker']['cron'] ||= '0 * * * *'
 Settings.cron_jobs['update_all_mirrors_worker']['job_class'] = 'UpdateAllMirrorsWorker'
+Settings.cron_jobs['update_all_remote_mirrors_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['update_all_remote_mirrors_worker']['cron'] ||= '30 * * * *'
+Settings.cron_jobs['update_all_remote_mirrors_worker']['job_class'] = 'UpdateAllRemoteMirrorsWorker'
 Settings.cron_jobs['ldap_sync_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['ldap_sync_worker']['cron'] ||= '30 1 * * *'
 Settings.cron_jobs['ldap_sync_worker']['job_class'] = 'LdapSyncWorker'

config/routes.rb

@@ -699,6 +699,7 @@ Rails.application.routes.draw do
         resource :mirror, only: [:show, :update] do
           member do
             post :update_now
+            post :update_remote_now
           end
         end
         resources :git_hooks, constraints: { id: /\d+/ }

db/migrate/20160321161032_create_remote_mirrors.rb

+class CreateRemoteMirrors < ActiveRecord::Migration
+  def change
+    create_table :remote_mirrors do |t|
+      t.references :project, index: true, foreign_key: true
+      t.string :url
+      t.boolean :enabled, default: true
+      t.string :update_status
+      t.datetime :last_update_at
+      t.datetime :last_successful_update_at
+      t.string :last_error
+
+      t.timestamps null: false
+    end
+  end
+end

db/migrate/20160330031024_add_mirror_credentials_to_remote_mirrors.rb

+class AddMirrorCredentialsToRemoteMirrors < ActiveRecord::Migration
+  def change
+    add_column :remote_mirrors, :encrypted_credentials, :text
+    add_column :remote_mirrors, :encrypted_credentials_iv, :text
+    add_column :remote_mirrors, :encrypted_credentials_salt, :text
+  end
+end

db/schema.rb

@@ -883,6 +883,23 @@ ActiveRecord::Schema.define(version: 20160331223143) do
   add_index "releases", ["project_id", "tag"], name: "index_releases_on_project_id_and_tag", using: :btree
   add_index "releases", ["project_id"], name: "index_releases_on_project_id", using: :btree
 
+  create_table "remote_mirrors", force: :cascade do |t|
+    t.integer  "project_id"
+    t.string   "url"
+    t.boolean  "enabled",                   default: true
+    t.string   "update_status"
+    t.datetime "last_update_at"
+    t.datetime "last_successful_update_at"
+    t.string   "last_error"
+    t.datetime "created_at",                               null: false
+    t.datetime "updated_at",                               null: false
+    t.text     "encrypted_credentials"
+    t.text     "encrypted_credentials_iv"
+    t.text     "encrypted_credentials_salt"
+  end
+
+  add_index "remote_mirrors", ["project_id"], name: "index_remote_mirrors_on_project_id", using: :btree
+
   create_table "sent_notifications", force: :cascade do |t|
     t.integer "project_id"
     t.integer "noteable_id"

lib/gitlab/backend/shell.rb

@@ -271,6 +271,38 @@ module Gitlab
       File.exists?(full_path(dir_name))
     end
 
+    # Push branch to remote repository
+    #
+    # project_name - project's name with namespace
+    # remote_name - remote name
+    # branch_name - remote branch name
+    #
+    # Ex.
+    #   push_branches('upstream', 'feature')
+    #
+    def push_branches(project_name, remote_name, branch_names)
+      args = [gitlab_shell_projects_path, 'push-branches', "#{project_name}.git", remote_name, *branch_names]
+      output, status = Popen::popen(args)
+      raise Error, output unless status.zero?
+      true
+    end
+
+    # Delete branch from remote repository
+    #
+    # project_name - project's name with namespace
+    # remote_name - remote name
+    # branch_name - remote branch name
+    #
+    # Ex.
+    #   delete_remote_branches('upstream', 'feature')
+    #
+    def delete_remote_branches(project_name, remote_name, branch_names)
+      args = [gitlab_shell_projects_path, 'delete-remote-branches', "#{project_name}.git", remote_name, *branch_names]
+      output, status = Popen::popen(args)
+      raise Error, output unless status.zero?
+      true
+    end
+
     protected
 
     def gitlab_shell_path

lib/gitlab/import_url.rb

+# I'm borrowing this class from: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3066
+# So we should fix the conflict once the CE -> EE merge process starts.
+module Gitlab
+  class ImportUrl
+    def initialize(url, credentials: nil)
+      @url = URI.parse(url)
+      @credentials = credentials
+    end
+
+    def sanitized_url
+      @sanitized_url ||= safe_url.to_s
+    end
+
+    def credentials
+      @credentials ||= { user: @url.user, password: @url.password }
+    end
+
+    def full_url
+      @full_url ||= generate_full_url.to_s
+    end
+
+    private
+
+    def generate_full_url
+      return @url unless valid_credentials?
+      @full_url = @url.dup
+      @full_url.user = credentials[:user]
+      @full_url.password = credentials[:password]
+      @full_url
+    end
+
+    def safe_url
+      safe_url = @url.dup
+      safe_url.password = nil
+      safe_url.user = nil
+      safe_url
+    end
+
+    def valid_credentials?
+      credentials && credentials.is_a?(Hash) && credentials.any?
+    end
+  end
+end

spec/factories/projects.rb

@@ -2,40 +2,54 @@
 #
 # Table name: projects
 #
-#  id                     :integer          not null, primary key
-#  name                   :string(255)
-#  path                   :string(255)
-#  description            :text
-#  created_at             :datetime
-#  updated_at             :datetime
-#  creator_id             :integer
-#  issues_enabled         :boolean          default(TRUE), not null
-#  wall_enabled           :boolean          default(TRUE), not null
-#  merge_requests_enabled :boolean          default(TRUE), not null
-#  wiki_enabled           :boolean          default(TRUE), not null
-#  namespace_id           :integer
-#  issues_tracker         :string(255)      default("gitlab"), not null
-#  issues_tracker_id      :string(255)
-#  snippets_enabled       :boolean          default(TRUE), not null
-#  last_activity_at       :datetime
-#  import_url             :string(255)
-#  visibility_level       :integer          default(0), not null
-#  archived               :boolean          default(FALSE), not null
-#  avatar                 :string(255)
-#  import_status          :string(255)
-#  repository_size        :float            default(0.0)
-#  star_count             :integer          default(0), not null
-#  import_type            :string(255)
-#  import_source          :string(255)
-#  commit_count           :integer          default(0)
-#  import_error           :text
-#  ci_id                  :integer
-#  builds_enabled         :boolean          default(TRUE), not null
-#  shared_runners_enabled :boolean          default(TRUE), not null
-#  runners_token          :string
-#  build_coverage_regex   :string
-#  build_allow_git_fetch  :boolean          default(TRUE), not null
-#  build_timeout          :integer          default(3600), not null
+#  id                               :integer          not null, primary key
+#  name                             :string(255)
+#  path                             :string(255)
+#  description                      :text
+#  created_at                       :datetime
+#  updated_at                       :datetime
+#  creator_id                       :integer
+#  issues_enabled                   :boolean          default(TRUE), not null
+#  wall_enabled                     :boolean          default(TRUE), not null
+#  merge_requests_enabled           :boolean          default(TRUE), not null
+#  wiki_enabled                     :boolean          default(TRUE), not null
+#  namespace_id                     :integer
+#  issues_tracker                   :string(255)      default("gitlab"), not null
+#  issues_tracker_id                :string(255)
+#  snippets_enabled                 :boolean          default(TRUE), not null
+#  last_activity_at                 :datetime
+#  import_url                       :string(255)
+#  visibility_level                 :integer          default(0), not null
+#  archived                         :boolean          default(FALSE), not null
+#  avatar                           :string(255)
+#  import_status                    :string(255)
+#  repository_size                  :float            default(0.0)
+#  star_count                       :integer          default(0), not null
+#  import_type                      :string(255)
+#  import_source                    :string(255)
+#  commit_count                     :integer          default(0)
+#  import_error                     :text
+#  ci_id                            :integer
+#  builds_enabled                   :boolean          default(TRUE), not null
+#  shared_runners_enabled           :boolean          default(TRUE), not null
+#  runners_token                    :string
+#  build_coverage_regex             :string
+#  build_allow_git_fetch            :boolean          default(TRUE), not null
+#  build_timeout                    :integer          default(3600), not null
+#  pending_delete                   :boolean          default(FALSE)
+#  public_builds                    :boolean          default(TRUE), not null
+#  merge_requests_template          :text
+#  merge_requests_rebase_enabled    :boolean          default(FALSE)
+#  approvals_before_merge           :integer          default(0), not null
+#  reset_approvals_on_push          :boolean          default(TRUE)
+#  merge_requests_ff_only_enabled   :boolean          default(FALSE)
+#  issues_template                  :text
+#  mirror                           :boolean          default(FALSE), not null
+#  mirror_last_update_at            :datetime
+#  mirror_last_successful_update_at :datetime
+#  mirror_user_id                   :integer
+#  mirror_trigger_builds            :boolean          default(FALSE), not null
+#  main_language                    :string
 #
 
 FactoryGirl.define do

spec/models/project_spec.rb

@@ -2,40 +2,54 @@
 #
 # Table name: projects
 #
-#  id                     :integer          not null, primary key
-#  name                   :string(255)
-#  path                   :string(255)
-#  description            :text
-#  created_at             :datetime
-#  updated_at             :datetime
-#  creator_id             :integer
-#  issues_enabled         :boolean          default(TRUE), not null
-#  wall_enabled           :boolean          default(TRUE), not null
-#  merge_requests_enabled :boolean          default(TRUE), not null
-#  wiki_enabled           :boolean          default(TRUE), not null
-#  namespace_id           :integer
-#  issues_tracker         :string(255)      default("gitlab"), not null
-#  issues_tracker_id      :string(255)
-#  snippets_enabled       :boolean          default(TRUE), not null
-#  last_activity_at       :datetime
-#  import_url             :string(255)
-#  visibility_level       :integer          default(0), not null
-#  archived               :boolean          default(FALSE), not null
-#  avatar                 :string(255)
-#  import_status          :string(255)
-#  repository_size        :float            default(0.0)
-#  star_count             :integer          default(0), not null
-#  import_type            :string(255)
-#  import_source          :string(255)
-#  commit_count           :integer          default(0)
-#  import_error           :text
-#  ci_id                  :integer
-#  builds_enabled         :boolean          default(TRUE), not null
-#  shared_runners_enabled :boolean          default(TRUE), not null
-#  runners_token          :string
-#  build_coverage_regex   :string
-#  build_allow_git_fetch  :boolean          default(TRUE), not null
-#  build_timeout          :integer          default(3600), not null
+#  id                               :integer          not null, primary key
+#  name                             :string(255)
+#  path                             :string(255)
+#  description                      :text
+#  created_at                       :datetime
+#  updated_at                       :datetime
+#  creator_id                       :integer
+#  issues_enabled                   :boolean          default(TRUE), not null
+#  wall_enabled                     :boolean          default(TRUE), not null
+#  merge_requests_enabled           :boolean          default(TRUE), not null
+#  wiki_enabled                     :boolean          default(TRUE), not null
+#  namespace_id                     :integer
+#  issues_tracker                   :string(255)      default("gitlab"), not null
+#  issues_tracker_id                :string(255)
+#  snippets_enabled                 :boolean          default(TRUE), not null
+#  last_activity_at                 :datetime
+#  import_url                       :string(255)
+#  visibility_level                 :integer          default(0), not null
+#  archived                         :boolean          default(FALSE), not null
+#  avatar                           :string(255)
+#  import_status                    :string(255)
+#  repository_size                  :float            default(0.0)
+#  star_count                       :integer          default(0), not null
+#  import_type                      :string(255)
+#  import_source                    :string(255)
+#  commit_count                     :integer          default(0)
+#  import_error                     :text
+#  ci_id                            :integer
+#  builds_enabled                   :boolean          default(TRUE), not null
+#  shared_runners_enabled           :boolean          default(TRUE), not null
+#  runners_token                    :string
+#  build_coverage_regex             :string
+#  build_allow_git_fetch            :boolean          default(TRUE), not null
+#  build_timeout                    :integer          default(3600), not null
+#  pending_delete                   :boolean          default(FALSE)
+#  public_builds                    :boolean          default(TRUE), not null
+#  merge_requests_template          :text
+#  merge_requests_rebase_enabled    :boolean          default(FALSE)
+#  approvals_before_merge           :integer          default(0), not null
+#  reset_approvals_on_push          :boolean          default(TRUE)
+#  merge_requests_ff_only_enabled   :boolean          default(FALSE)
+#  issues_template                  :text
+#  mirror                           :boolean          default(FALSE), not null
+#  mirror_last_update_at            :datetime
+#  mirror_last_successful_update_at :datetime
+#  mirror_user_id                   :integer
+#  mirror_trigger_builds            :boolean          default(FALSE), not null
+#  main_language                    :string
 #
 
 require 'spec_helper'

spec/services/projects/update_remote_mirror_service_spec.rb

+require 'spec_helper'
+
+describe Projects::UpdateRemoteMirrorService do
+  let(:project) { create(:project) }
+  let(:remote_project) { create(:forked_project_with_submodules) }
+  let(:repository) { project.repository }
+  let(:remote_repository) { remote_project.repository }
+  let(:remote_mirror) { project.remote_mirrors.create!(url: remote_project.http_url_to_repo) }
+  let(:all_branches) { ["master", 'existing-branch', "'test'", "empty-branch", "feature", "feature_conflict", "fix", "flatten-dir", "improve/awesome", "lfs", "markdown"] }
+
+  subject { described_class.new(project, project.creator) }
+
+  describe "#execute" do
+    before do
+      create_branch(repository, 'existing-branch')
+    end
+
+    it "fetches the remote repository" do
+      expect(repository).to receive(:fetch_remote).with(remote_mirror.ref_name)
+
+      subject.execute(remote_mirror)
+    end
+
+    it "succeeds" do
+      allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.ref_name, all_branches) }
+
+      result = subject.execute(remote_mirror)
+
+      expect(result[:status]).to eq(:success)
+    end
+
+    describe 'Updating branches' do
+      it "push all the branches the first time" do
+        allow(repository).to receive(:fetch_remote)
+
+        expect(repository).to receive(:push_branches).with(project.path_with_namespace, remote_mirror.ref_name, all_branches)
+
+        subject.execute(remote_mirror)
+      end
+
+      it "does not push anything is remote is up to date" do
+        allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.ref_name, all_branches) }
+
+        expect(repository).not_to receive(:push_branches)
+
+        subject.execute(remote_mirror)
+      end
+
+      it "sync new branches" do
+        allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.ref_name, all_branches) }
+        create_branch(repository, 'my-new-branch')
+
+        expect(repository).to receive(:push_branches).with(project.path_with_namespace, remote_mirror.ref_name, ['my-new-branch'])
+
+        subject.execute(remote_mirror)
+      end
+
+      it "sync updated branches" do
+        allow(repository).to receive(:fetch_remote) do
+          sync_remote(repository, remote_mirror.ref_name, all_branches)
+          update_branch(repository, 'existing-branch')
+        end
+
+        expect(repository).to receive(:push_branches).with(project.path_with_namespace, remote_mirror.ref_name, ['existing-branch'])
+
+        subject.execute(remote_mirror)
+      end
+
+      it "sync deleted branches" do
+        allow(repository).to receive(:fetch_remote) do
+          sync_remote(repository, remote_mirror.ref_name, all_branches)
+          delete_branch(repository, 'existing-branch')
+        end
+
+        expect(repository).to receive(:delete_remote_branches).with(project.path_with_namespace, remote_mirror.ref_name, ['existing-branch'])
+
+        subject.execute(remote_mirror)
+      end
+    end
+
+  end
+
+  def create_branch(repository, branch_name)
+    rugged = repository.rugged
+    masterrev = repository.find_branch('master').target
+    parentrev = repository.commit(masterrev).parent_id
+
+    rugged.references.create("refs/heads/#{branch_name}", parentrev)
+
+    repository.expire_branches_cache
+  end
+
+  def sync_remote(repository, remote_name, all_branches)
+    rugged = repository.rugged
+
+    all_branches.each do |branch|
+      target = repository.find_branch(branch).try(:target)
+      rugged.references.create("refs/remotes/#{remote_name}/#{branch}", target) if target
+    end
+  end
+
+  def update_branch(repository, branch)
+    rugged = repository.rugged
+    masterrev = repository.find_branch('master').target
+
+    # # Updated existing branch
+    rugged.references.create("refs/heads/#{branch}", masterrev, force: true)
+    repository.expire_branches_cache
+  end
+
+  def delete_branch(repository, branch)
+    rugged = repository.rugged
+
+    rugged.references.delete("refs/heads/#{branch}")
+    repository.expire_branches_cache
+  end
+end