Merge branch '3844-add-option-to-projects-to-only-import-protected-branches' into 'master'

Adds option for new project to only import protected branches

Closes #3844

See merge request gitlab-org/gitlab-ee!3326

app/models/project.rb

@@ -60,6 +60,7 @@ class Project < ActiveRecord::Base
   default_value_for :wiki_enabled, gitlab_config_features.wiki
   default_value_for :snippets_enabled, gitlab_config_features.snippets
   default_value_for :only_allow_merge_if_all_discussions_are_resolved, false
+  default_value_for :only_mirror_protected_branches, true
 
   add_authentication_token_field :runners_token
   before_save :ensure_runners_token

changelogs/unreleased-ee/3844-add-option-to-projects-to-only-import-protected-branches.yml

+---
+title: Add option for projects to only mirror protected branches.
+merge_request: 3326
+author:
+type: added

db/migrate/20171109115718_add_only_mirror_protected_branches_to_projects.rb

+class AddOnlyMirrorProtectedBranchesToProjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def change
+    add_column :projects, :only_mirror_protected_branches, :boolean
+  end
+end

db/schema.rb

@@ -1840,6 +1840,7 @@ ActiveRecord::Schema.define(version: 20171124165823) do
     t.integer "storage_version", limit: 2
     t.boolean "resolve_outdated_diff_discussions"
     t.boolean "remote_mirror_available_overridden"
+    t.boolean "only_mirror_protected_branches"
   end
 
   add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree

doc/workflow/repository_mirroring.md

@@ -84,6 +84,14 @@ this branch to prevent any changes from being lost.
 
 ![Diverged branch](repository_mirroring/repository_mirroring_diverged_branch.png)
 
+### Pull only protected branches
+
+>[Introduced][ee-3326] in Gitlab Enterprise Edition 10.3.
+
+You can choose to only pull the protected branches from your remote repository to GitLab.
+
+To use this option go to your project's repository settings page under pull mirror.
+
 ### Hard failure
 
 >[Introduced][ee-3117] in GitLab Enterprise Edition 10.2.

ee/app/controllers/projects/mirrors_controller.rb

@@ -84,6 +84,7 @@ class Projects::MirrorsController < Projects::ApplicationController
       :username_only_import_url,
       :mirror_user_id,
       :mirror_trigger_builds,
+      :only_mirror_protected_branches,
 
       import_data_attributes: %i[
         id

ee/app/services/projects/update_mirror_service.rb

@@ -33,6 +33,8 @@ module Projects
       repository.upstream_branches.each do |upstream_branch|
         name = upstream_branch.name
 
+        next if skip_branch?(name)
+
         local_branch = local_branches[name]
 
         if local_branch.nil?
@@ -98,5 +100,9 @@ module Projects
     def repository_tags_with_target
       repository.tags.select(&:dereferenced_target)
     end
+
+    def skip_branch?(name)
+      project.only_mirror_protected_branches && !ProtectedBranch.protected?(project, name)
+    end
   end
 end

ee/app/services/projects/update_remote_mirror_service.rb

@@ -31,9 +31,7 @@ module Projects
 
     def local_branches
       @local_branches ||= repository.local_branches.each_with_object({}) do |branch, branches|
-        next if mirror.only_protected_branches? && !protected_branch?(branch.name)
-
-        branches[branch.name] = branch
+        branches[branch.name] = branch unless skip_branch?(branch.name)
       end
     end
 
@@ -115,7 +113,7 @@ module Projects
 
       remote_refs.each_with_object([]) do |(name, remote_ref), refs_to_delete|
         next if local_refs[name] # skip if branch or tag exist in local repo
-        next if type == :branches && mirror.only_protected_branches? && !protected_branch?(name)
+        next if type == :branches && skip_branch?(name)
 
         remote_ref_id = remote_ref.dereferenced_target.try(:id)
 
@@ -125,8 +123,8 @@ module Projects
       end
     end
 
-    def protected_branch?(name)
-      ProtectedBranch.protected?(project, name)
+    def skip_branch?(name)
+      mirror.only_protected_branches? && !ProtectedBranch.protected?(project, name)
     end
   end
 end

ee/app/views/projects/mirrors/_pull.html.haml

@@ -40,6 +40,14 @@
           This user will be the author of all events in the activity feed that are the result of an update,
           like new branches being created or new commits being pushed to existing branches.
           You can only assign yourself to be the mirror user.
+
+      .form-group
+        = f.check_box :only_mirror_protected_branches, class: 'pull-left'
+        .prepend-left-20
+          = f.label :only_mirror_protected_branches, class: 'label-light'
+          = link_to icon('question-circle'), help_page_path('user/project/protected_branches')
+
       - if @project.builds_enabled?
         = render "shared/mirror_trigger_builds_setting", f: f
+
       = f.submit 'Save changes', class: 'btn btn-create', name: 'update_remote_mirror'

ee/app/views/shared/ee/_import_form.html.haml

@@ -11,5 +11,4 @@
       .help-block
         Mirroring will only be available if the feature is included in the plan of the selected group or user.
 
-
   = f.hidden_field :mirror_user_id, value: current_user.id

lib/gitlab/import_export/import_export.yml

@@ -107,6 +107,7 @@ excluded_attributes:
     - :mirror_trigger_builds
     - :storage_version
     - :remote_mirror_available_overridden
+    - :only_mirror_protected_branches
   snippets:
     - :expired_at
   merge_request_diff:

spec/lib/gitlab/background_migration/create_fork_network_memberships_range_spec.rb

@@ -2,19 +2,20 @@ require 'spec_helper'
 
 describe Gitlab::BackgroundMigration::CreateForkNetworkMembershipsRange, :migration, schema: 20170929131201 do
   let(:migration) { described_class.new }
+  let(:projects) { table(:projects) }
 
-  let(:base1) { create(:project) }
-  let(:base1_fork1) { create(:project) }
-  let(:base1_fork2) { create(:project) }
+  let(:base1) { projects.create }
+  let(:base1_fork1) { projects.create }
+  let(:base1_fork2) { projects.create }
 
-  let(:base2) { create(:project) }
-  let(:base2_fork1) { create(:project) }
-  let(:base2_fork2) { create(:project) }
+  let(:base2) { projects.create }
+  let(:base2_fork1) { projects.create }
+  let(:base2_fork2) { projects.create }
 
-  let(:fork_of_fork) { create(:project) }
-  let(:fork_of_fork2) { create(:project) }
-  let(:second_level_fork) { create(:project) }
-  let(:third_level_fork) { create(:project) }
+  let(:fork_of_fork) { projects.create }
+  let(:fork_of_fork2) { projects.create }
+  let(:second_level_fork) { projects.create }
+  let(:third_level_fork) { projects.create }
 
   let(:fork_network1) { fork_networks.find_by(root_project_id: base1.id) }
   let(:fork_network2) { fork_networks.find_by(root_project_id: base2.id) }
@@ -97,7 +98,7 @@ describe Gitlab::BackgroundMigration::CreateForkNetworkMembershipsRange, :migrat
   end
 
   it 'does not miss members for forks of forks for which the root was deleted' do
-    forked_project_links.create(id: 9, forked_from_project_id: base1_fork1.id, forked_to_project_id: create(:project).id)
+    forked_project_links.create(id: 9, forked_from_project_id: base1_fork1.id, forked_to_project_id: projects.create.id)
     base1.destroy
 
     expect(migration.missing_members?(7, 10)).to be_falsy
@@ -105,8 +106,8 @@ describe Gitlab::BackgroundMigration::CreateForkNetworkMembershipsRange, :migrat
 
   context 'with more forks' do
     before do
-      forked_project_links.create(id: 9, forked_from_project_id: fork_of_fork.id, forked_to_project_id: create(:project).id)
-      forked_project_links.create(id: 10, forked_from_project_id: fork_of_fork.id, forked_to_project_id: create(:project).id)
+      forked_project_links.create(id: 9, forked_from_project_id: fork_of_fork.id, forked_to_project_id: projects.create.id)
+      forked_project_links.create(id: 10, forked_from_project_id: fork_of_fork.id, forked_to_project_id: projects.create.id)
     end
 
     it 'only processes a single batch of links at a time' do

spec/lib/gitlab/background_migration/migrate_events_to_push_event_payloads_spec.rb

@@ -225,7 +225,8 @@ describe Gitlab::BackgroundMigration::MigrateEventsToPushEventPayloads, :migrati
   let(:user_class) { table(:users) }
   let(:author) { build(:user).becomes(user_class).tap(&:save!).becomes(User) }
   let(:namespace) { create(:namespace, owner: author) }
-  let(:project) { create(:project_empty_repo, namespace: namespace, creator: author) }
+  let(:projects) { table(:projects) }
+  let(:project) { projects.create(namespace_id: namespace.id, creator_id: author.id) }
 
   # We can not rely on FactoryGirl as the state of Event may change in ways that
   # the background migration does not expect, hence we use the Event class of

spec/lib/gitlab/background_migration/populate_fork_networks_range_spec.rb

@@ -2,10 +2,11 @@ require 'spec_helper'
 
 describe Gitlab::BackgroundMigration::PopulateForkNetworksRange, :migration, schema: 20170929131201 do
   let(:migration) { described_class.new }
-  let(:base1) { create(:project) }
+  let(:projects) { table(:projects) }
+  let(:base1) { projects.create }
 
-  let(:base2) { create(:project) }
-  let(:base2_fork1) { create(:project) }
+  let(:base2) { projects.create }
+  let(:base2_fork1) { projects.create }
 
   let!(:forked_project_links) { table(:forked_project_links) }
   let!(:fork_networks) { table(:fork_networks) }
@@ -18,10 +19,10 @@ describe Gitlab::BackgroundMigration::PopulateForkNetworksRange, :migration, sch
     # A normal fork link
     forked_project_links.create(id: 1,
                                 forked_from_project_id: base1.id,
-                                forked_to_project_id: create(:project).id)
+                                forked_to_project_id: projects.create.id)
     forked_project_links.create(id: 2,
                                 forked_from_project_id: base1.id,
-                                forked_to_project_id: create(:project).id)
+                                forked_to_project_id: projects.create.id)
     forked_project_links.create(id: 3,
                                 forked_from_project_id: base2.id,
                                 forked_to_project_id: base2_fork1.id)
@@ -29,10 +30,10 @@ describe Gitlab::BackgroundMigration::PopulateForkNetworksRange, :migration, sch
     # create a fork of a fork
     forked_project_links.create(id: 4,
                                 forked_from_project_id: base2_fork1.id,
-                                forked_to_project_id: create(:project).id)
+                                forked_to_project_id: projects.create.id)
     forked_project_links.create(id: 5,
-                                forked_from_project_id: create(:project).id,
-                                forked_to_project_id: create(:project).id)
+                                forked_from_project_id: projects.create.id,
+                                forked_to_project_id: projects.create.id)
 
     # Stub out the calls to the other migrations
     allow(BackgroundMigrationWorker).to receive(:perform_in)
@@ -63,7 +64,7 @@ describe Gitlab::BackgroundMigration::PopulateForkNetworksRange, :migration, sch
   end
 
   it 'creates a fork network for the fork of which the source was deleted' do
-    fork = create(:project)
+    fork = projects.create
     forked_project_links.create(id: 6, forked_from_project_id: 99999, forked_to_project_id: fork.id)
 
     migration.perform(5, 8)

spec/migrations/migrate_gcp_clusters_to_new_clusters_architectures_spec.rb

@@ -2,9 +2,10 @@ require 'spec_helper'
 require Rails.root.join('db', 'post_migrate', '20171013104327_migrate_gcp_clusters_to_new_clusters_architectures.rb')
 
 describe MigrateGcpClustersToNewClustersArchitectures, :migration do
-  let(:project) { create(:project) }
+  let(:projects) { table(:projects) }
+  let(:project) { projects.create }
   let(:user) { create(:user) }
-  let(:service) { create(:kubernetes_service, project: project) }
+  let(:service) { create(:kubernetes_service, project_id: project.id) }
 
   context 'when cluster is being created' do
     let(:project_id) { project.id }
@@ -56,8 +57,7 @@ describe MigrateGcpClustersToNewClustersArchitectures, :migration do
       expect(cluster.provider_type).to eq('gcp')
       expect(cluster.platform_type).to eq('kubernetes')
 
-      expect(cluster.project).to eq(project)
-      expect(project.cluster).to eq(cluster)
+      expect(cluster.project_ids).to include(project.id)
 
       expect(cluster.provider_gcp.cluster).to eq(cluster)
       expect(cluster.provider_gcp.status).to eq(status)
@@ -133,8 +133,7 @@ describe MigrateGcpClustersToNewClustersArchitectures, :migration do
       expect(cluster.provider_type).to eq('gcp')
       expect(cluster.platform_type).to eq('kubernetes')
 
-      expect(cluster.project).to eq(project)
-      expect(project.cluster).to eq(cluster)
+      expect(cluster.project_ids).to include(project.id)
 
       expect(cluster.provider_gcp.cluster).to eq(cluster)
       expect(cluster.provider_gcp.status).to eq(status)

spec/services/projects/update_mirror_service_spec.rb

 require 'spec_helper'
 
 describe Projects::UpdateMirrorService do
-  let(:project) { create(:project, :repository, :mirror, import_url: Project::UNKNOWN_IMPORT_URL) }
+  let(:project) do
+    create(:project, :repository, :mirror, import_url: Project::UNKNOWN_IMPORT_URL, only_mirror_protected_branches: false)
+  end
 
   describe "#execute" do
     context 'unlicensed' do
@@ -53,6 +55,55 @@ describe Projects::UpdateMirrorService do
     end
 
     describe "updating branches" do
+      context 'when mirror only protected branches option is set' do
+        let(:new_protected_branch_name) { 'new-branch' }
+        let(:protected_branch_name) { 'existing-branch' }
+
+        before do
+          project.update_attributes(only_mirror_protected_branches: true)
+        end
+
+        it 'creates a new protected branch' do
+          create(:protected_branch, project: project, name: new_protected_branch_name)
+          project.reload
+
+          stub_fetch_mirror(project)
+
+          described_class.new(project, project.owner).execute
+
+          expect(project.repository.branch_names).to include(new_protected_branch_name)
+        end
+
+        it 'does not create an unprotected branch' do
+          stub_fetch_mirror(project)
+
+          described_class.new(project, project.owner).execute
+
+          expect(project.repository.branch_names).not_to include(new_protected_branch_name)
+        end
+
+        it 'updates existing protected branches' do
+          create(:protected_branch, project: project, name: protected_branch_name)
+          project.reload
+
+          stub_fetch_mirror(project)
+
+          described_class.new(project, project.owner).execute
+
+          expect(project.repository.find_branch(protected_branch_name).dereferenced_target)
+            .to eq(project.repository.find_branch('master').dereferenced_target)
+        end
+
+        it "does not update unprotected branches" do
+          stub_fetch_mirror(project)
+
+          described_class.new(project, project.owner).execute
+
+          expect(project.repository.find_branch(protected_branch_name).dereferenced_target)
+            .not_to eq(project.repository.find_branch('master').dereferenced_target)
+        end
+      end
+
       it "creates new branches" do
         stub_fetch_mirror(project)