Commit 8ba4e8b3 authored by 🌴 Toon Claes 🌴's avatar 🌴 Toon Claes 🌴 Committed by Douwe Maan

Namespace license checks for Repository Mirrors

parent cc5a5196
class Admin::ApplicationSettingsController < Admin::ApplicationController class Admin::ApplicationSettingsController < Admin::ApplicationController
prepend EE::Admin::ApplicationSettingsController
before_action :set_application_setting before_action :set_application_setting
def show def show
...@@ -58,7 +60,6 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController ...@@ -58,7 +60,6 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
def application_setting_params def application_setting_params
import_sources = params[:application_setting][:import_sources] import_sources = params[:application_setting][:import_sources]
if import_sources.nil? if import_sources.nil?
params[:application_setting][:import_sources] = [] params[:application_setting][:import_sources] = []
else else
...@@ -77,11 +78,11 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController ...@@ -77,11 +78,11 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
params.delete(:domain_blacklist_raw) if params[:domain_blacklist_file] params.delete(:domain_blacklist_raw) if params[:domain_blacklist_file]
params.require(:application_setting).permit( params.require(:application_setting).permit(
application_setting_params_ce << application_setting_params_ee application_setting_params_attributes
) )
end end
def application_setting_params_ce def application_setting_params_attributes
[ [
:admin_notification_email, :admin_notification_email,
:after_sign_out_path, :after_sign_out_path,
...@@ -166,30 +167,4 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController ...@@ -166,30 +167,4 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
sidekiq_throttling_queues: [] sidekiq_throttling_queues: []
] ]
end end
def application_setting_params_ee
[
:help_text,
:elasticsearch_url,
:elasticsearch_indexing,
:elasticsearch_aws,
:elasticsearch_aws_access_key,
:elasticsearch_aws_secret_access_key,
:elasticsearch_aws_region,
:elasticsearch_search,
:repository_size_limit,
:shared_runners_minutes,
:geo_status_timeout,
:elasticsearch_experimental_indexer,
:check_namespace_plan,
:mirror_max_delay,
:mirror_max_capacity,
:mirror_capacity_threshold,
:authorized_keys_enabled,
:slack_app_enabled,
:slack_app_id,
:slack_app_secret,
:slack_app_verification_token
]
end
end end
module EE
module Admin
module ApplicationSettingsController
def application_setting_params_attributes
attrs = super + application_setting_params_attributes_ee
attrs += repository_mirrors_params_attributes if License.feature_available?(:repository_mirrors)
attrs
end
private
def application_setting_params_attributes_ee
[
:help_text,
:elasticsearch_url,
:elasticsearch_indexing,
:elasticsearch_aws,
:elasticsearch_aws_access_key,
:elasticsearch_aws_secret_access_key,
:elasticsearch_aws_region,
:elasticsearch_search,
:repository_size_limit,
:shared_runners_minutes,
:geo_status_timeout,
:elasticsearch_experimental_indexer,
:check_namespace_plan,
:authorized_keys_enabled,
:slack_app_enabled,
:slack_app_id,
:slack_app_secret,
:slack_app_verification_token
]
end
def repository_mirrors_params_attributes
[
:mirror_max_delay,
:mirror_max_capacity,
:mirror_capacity_threshold
]
end
end
end
end
...@@ -19,6 +19,8 @@ module EE ...@@ -19,6 +19,8 @@ module EE
end end
def remote_mirror def remote_mirror
return unless project.feature_available?(:repository_mirrors)
@remote_mirror = @project.remote_mirrors.first_or_initialize @remote_mirror = @project.remote_mirrors.first_or_initialize
end end
......
module EE
module ProjectsController
def project_params_attributes
attrs = super + project_params_ee
attrs += repository_mirrors_params if project&.feature_available?(:repository_mirrors)
attrs
end
private
def project_params_ee
%i[
approvals_before_merge
approver_group_ids
approver_ids
issues_template
merge_method
merge_requests_template
disable_overriding_approvers_per_merge_request
repository_size_limit
reset_approvals_on_push
service_desk_enabled
]
end
def repository_mirrors_params
%i[
mirror
mirror_trigger_builds
mirror_user_id
]
end
end
end
...@@ -22,6 +22,7 @@ class Projects::ApplicationController < ApplicationController ...@@ -22,6 +22,7 @@ class Projects::ApplicationController < ApplicationController
def project def project
return @project if @project return @project if @project
return nil unless params[:project_id] || params[:id]
path = File.join(params[:namespace_id], params[:project_id] || params[:id]) path = File.join(params[:namespace_id], params[:project_id] || params[:id])
auth_proc = ->(project) { !project.pending_delete? } auth_proc = ->(project) { !project.pending_delete? }
......
...@@ -5,6 +5,7 @@ class Projects::MirrorsController < Projects::ApplicationController ...@@ -5,6 +5,7 @@ class Projects::MirrorsController < Projects::ApplicationController
before_action :authorize_admin_project!, except: [:update_now] before_action :authorize_admin_project!, except: [:update_now]
before_action :authorize_push_code!, only: [:update_now] before_action :authorize_push_code!, only: [:update_now]
before_action :remote_mirror, only: [:update] before_action :remote_mirror, only: [:update]
before_action :check_repository_mirrors_available!
layout "project_settings" layout "project_settings"
......
class ProjectsController < Projects::ApplicationController class ProjectsController < Projects::ApplicationController
include IssuableCollections include IssuableCollections
include ExtractsPath include ExtractsPath
prepend EE::ProjectsController
before_action :authenticate_user!, except: [:index, :show, :activity, :refs] before_action :authenticate_user!, except: [:index, :show, :activity, :refs]
before_action :project, except: [:index, :new, :create] before_action :project, except: [:index, :new, :create]
...@@ -297,10 +298,10 @@ class ProjectsController < Projects::ApplicationController ...@@ -297,10 +298,10 @@ class ProjectsController < Projects::ApplicationController
def project_params def project_params
params.require(:project) params.require(:project)
.permit(project_params_ce << project_params_ee) .permit(project_params_attributes)
end end
def project_params_ce def project_params_attributes
[ [
:avatar, :avatar,
:build_allow_git_fetch, :build_allow_git_fetch,
...@@ -337,25 +338,6 @@ class ProjectsController < Projects::ApplicationController ...@@ -337,25 +338,6 @@ class ProjectsController < Projects::ApplicationController
] ]
end end
def project_params_ee
%i[
approvals_before_merge
approvals
approver_group_ids
approver_ids
issues_template
merge_method
merge_requests_template
mirror
mirror_trigger_builds
mirror_user_id
disable_overriding_approvers_per_merge_request
repository_size_limit
reset_approvals_on_push
service_desk_enabled
]
end
def repo_exists? def repo_exists?
project.repository_exists? && !project.empty_repo? && project.repo project.repository_exists? && !project.empty_repo? && project.repo
......
...@@ -42,11 +42,6 @@ module EE ...@@ -42,11 +42,6 @@ module EE
scope :with_shared_runners_limit_enabled, -> { with_shared_runners.non_public_only } scope :with_shared_runners_limit_enabled, -> { with_shared_runners.non_public_only }
scope :mirrors_to_sync, -> do
mirror.joins(:mirror_data).where("next_execution_timestamp <= ? AND import_status NOT IN ('scheduled', 'started')", Time.now)
.order_by(:next_execution_timestamp).limit(::Gitlab::Mirror.available_capacity)
end
scope :stuck_mirrors, -> do scope :stuck_mirrors, -> do
mirror.joins(:mirror_data) mirror.joins(:mirror_data)
.where("(import_status = 'started' AND project_mirror_data.last_update_started_at < :limit) OR (import_status = 'scheduled' AND project_mirror_data.last_update_scheduled_at < :limit)", .where("(import_status = 'started' AND project_mirror_data.last_update_started_at < :limit) OR (import_status = 'scheduled' AND project_mirror_data.last_update_scheduled_at < :limit)",
...@@ -84,6 +79,11 @@ module EE ...@@ -84,6 +79,11 @@ module EE
end end
end end
def mirror
super && feature_available?(:repository_mirrors)
end
alias_method :mirror?, :mirror
def mirror_updated? def mirror_updated?
mirror? && self.mirror_last_update_at mirror? && self.mirror_last_update_at
end end
...@@ -140,7 +140,7 @@ module EE ...@@ -140,7 +140,7 @@ module EE
end end
def has_remote_mirror? def has_remote_mirror?
remote_mirrors.enabled.exists? feature_available?(:repository_mirrors) && remote_mirrors.enabled.exists?
end end
def updating_remote_mirror? def updating_remote_mirror?
...@@ -148,7 +148,9 @@ module EE ...@@ -148,7 +148,9 @@ module EE
end end
def update_remote_mirrors def update_remote_mirrors
remote_mirrors.each(&:sync) return unless feature_available?(:repository_mirrors)
remote_mirrors.enabled.each(&:sync)
end end
def mark_stuck_remote_mirrors_as_failed! def mark_stuck_remote_mirrors_as_failed!
......
...@@ -28,6 +28,7 @@ class License < ActiveRecord::Base ...@@ -28,6 +28,7 @@ class License < ActiveRecord::Base
PROTECTED_REFS_FOR_USERS_FEATURE = 'GitLab_RefPermissionsForUsers'.freeze PROTECTED_REFS_FOR_USERS_FEATURE = 'GitLab_RefPermissionsForUsers'.freeze
PUSH_RULES_FEATURE = 'GitLab_PushRules'.freeze PUSH_RULES_FEATURE = 'GitLab_PushRules'.freeze
RELATED_ISSUES_FEATURE = 'GitLab_RelatedIssues'.freeze RELATED_ISSUES_FEATURE = 'GitLab_RelatedIssues'.freeze
REPOSITORY_MIRRORS_FEATURE = 'GitLab_RepositoryMirrors'.freeze
REPOSITORY_SIZE_LIMIT_FEATURE = 'GitLab_RepositorySizeLimit'.freeze REPOSITORY_SIZE_LIMIT_FEATURE = 'GitLab_RepositorySizeLimit'.freeze
SERVICE_DESK_FEATURE = 'GitLab_ServiceDesk'.freeze SERVICE_DESK_FEATURE = 'GitLab_ServiceDesk'.freeze
VARIABLE_ENVIRONMENT_SCOPE_FEATURE = 'GitLab_VariableEnvironmentScope'.freeze VARIABLE_ENVIRONMENT_SCOPE_FEATURE = 'GitLab_VariableEnvironmentScope'.freeze
...@@ -64,7 +65,8 @@ class License < ActiveRecord::Base ...@@ -64,7 +65,8 @@ class License < ActiveRecord::Base
multiple_issue_assignees: MULTIPLE_ISSUE_ASSIGNEES_FEATURE, multiple_issue_assignees: MULTIPLE_ISSUE_ASSIGNEES_FEATURE,
multiple_issue_boards: MULTIPLE_ISSUE_BOARDS_FEATURE, multiple_issue_boards: MULTIPLE_ISSUE_BOARDS_FEATURE,
protected_refs_for_users: PROTECTED_REFS_FOR_USERS_FEATURE, protected_refs_for_users: PROTECTED_REFS_FOR_USERS_FEATURE,
push_rules: PUSH_RULES_FEATURE push_rules: PUSH_RULES_FEATURE,
repository_mirrors: REPOSITORY_MIRRORS_FEATURE
}.freeze }.freeze
STARTER_PLAN = 'starter'.freeze STARTER_PLAN = 'starter'.freeze
...@@ -93,6 +95,7 @@ class License < ActiveRecord::Base ...@@ -93,6 +95,7 @@ class License < ActiveRecord::Base
{ PUSH_RULES_FEATURE => 1 }, { PUSH_RULES_FEATURE => 1 },
{ PROTECTED_REFS_FOR_USERS_FEATURE => 1 }, { PROTECTED_REFS_FOR_USERS_FEATURE => 1 },
{ RELATED_ISSUES_FEATURE => 1 }, { RELATED_ISSUES_FEATURE => 1 },
{ REPOSITORY_MIRRORS_FEATURE => 1 },
{ REPOSITORY_SIZE_LIMIT_FEATURE => 1 } { REPOSITORY_SIZE_LIMIT_FEATURE => 1 }
].freeze ].freeze
...@@ -143,6 +146,7 @@ class License < ActiveRecord::Base ...@@ -143,6 +146,7 @@ class License < ActiveRecord::Base
{ OBJECT_STORAGE_FEATURE => 1 }, { OBJECT_STORAGE_FEATURE => 1 },
{ PROTECTED_REFS_FOR_USERS_FEATURE => 1 }, { PROTECTED_REFS_FOR_USERS_FEATURE => 1 },
{ PUSH_RULES_FEATURE => 1 }, { PUSH_RULES_FEATURE => 1 },
{ REPOSITORY_MIRRORS_FEATURE => 1 },
{ SERVICE_DESK_FEATURE => 1 } { SERVICE_DESK_FEATURE => 1 }
].freeze ].freeze
......
...@@ -77,13 +77,22 @@ class RemoteMirror < ActiveRecord::Base ...@@ -77,13 +77,22 @@ class RemoteMirror < ActiveRecord::Base
end end
def sync def sync
return unless project && enabled return unless enabled?
return if project.pending_delete?
return if Gitlab::Geo.secondary? return if Gitlab::Geo.secondary?
RepositoryUpdateRemoteMirrorWorker.perform_in(BACKOFF_DELAY, self.id, Time.now) if project&.repository_exists? RepositoryUpdateRemoteMirrorWorker.perform_in(BACKOFF_DELAY, self.id, Time.now)
end end
def enabled
return false unless project && super
return false unless project.repository_exists?
return false if project.pending_delete?
# Sync is only enabled when the license permits it
project.feature_available?(:repository_mirrors)
end
alias_method :enabled?, :enabled
def updated_since?(timestamp) def updated_since?(timestamp)
last_update_started_at && last_update_started_at > timestamp && !update_failed? last_update_started_at && last_update_started_at > timestamp && !update_failed?
end end
......
...@@ -5,7 +5,7 @@ module Projects ...@@ -5,7 +5,7 @@ module Projects
def execute def execute
unless project.mirror? unless project.mirror?
return error("The project has no mirror to update") return success
end end
unless can?(current_user, :push_code_to_protected_branches, project) unless can?(current_user, :push_code_to_protected_branches, project)
......
...@@ -6,6 +6,8 @@ module Projects ...@@ -6,6 +6,8 @@ module Projects
@mirror = remote_mirror @mirror = remote_mirror
@errors = [] @errors = []
return success unless remote_mirror.enabled?
begin begin
repository.fetch_remote(mirror.ref_name, no_tags: true) repository.fetch_remote(mirror.ref_name, no_tags: true)
......
- if Gitlab.com? - if Gitlab.com? && License.feature_available?(:repository_mirrors)
%fieldset %fieldset
%legend Repository mirror settings %legend Repository mirror settings
.form-group .form-group
......
- expanded = Rails.env.test?
%section.settings.project-mirror-settings
.settings-header
%h4
Pull from a remote repository
%button.btn.js-settings-toggle
= expanded ? 'Collapse' : 'Expand'
%p
Set up your project to automatically have its branches, tags, and commits
updated from an upstream repository.
= link_to 'Read more', help_page_path('workflow/repository_mirroring', anchor: 'pulling-from-a-remote-repository'), target: '_blank'
.settings-content.no-animate{ class: ('expanded' if expanded) }
= form_for @project, url: project_mirror_path(@project) do |f|
%div
= form_errors(@project)
%h5
Set up mirror repository
= render "shared/mirror_update_button"
- if @project.mirror_last_update_failed?
.panel.panel-danger
.panel-heading
The repository failed to update #{time_ago_with_tooltip(@project.mirror_last_update_at)}.
- if @project.mirror_ever_updated_successfully?
Last successful update #{time_ago_with_tooltip(@project.mirror_last_successful_update_at)}.
.panel-body
%pre
:preserve
#{h(@project.import_error.try(:strip))}
.form-group
= f.check_box :mirror, class: "pull-left"
.prepend-left-20
= f.label :mirror, "Mirror repository", class: "label-light append-bottom-0"
.form-group
= f.label :import_url, "Git repository URL", class: "label-light"
= f.text_field :import_url, class: 'form-control', placeholder: 'https://username:password@gitlab.company.com/group/project.git'
= render "projects/mirrors/instructions"
.form-group
= f.label :mirror_user_id, "Mirror user", class: "label-light"
= select_tag('project[mirror_user_id]', options_for_mirror_user, class: "select2 lg", required: true)
.help-block
This user will be the author of all events in the activity feed that are the result of an update,
like new branches being created or new commits being pushed to existing branches.
You can only assign yourself to be the mirror user.
- if @project.builds_enabled?
= render "shared/mirror_trigger_builds_setting", f: f
= f.submit 'Save changes', class: 'btn btn-create', name: 'update_remote_mirror'
- expanded = Rails.env.test?
%section.settings
.settings-header
%h4
Push to a remote repository
%button.btn.js-settings-toggle
= expanded ? 'Collapse' : 'Expand'
%p
Set up the remote repository that you want to update with the content of the current repository
every time someone pushes to it.
= link_to 'Read more', help_page_path('workflow/repository_mirroring', anchor: 'pushing-to-a-remote-repository'), target: '_blank'
.settings-content.no-animate{ class: ('expanded' if expanded) }
= form_for @project, url: project_mirror_path(@project) do |f|
%div
= form_errors(@project)
= render "shared/remote_mirror_update_button", remote_mirror: @remote_mirror
- if @remote_mirror.last_error.present?
.panel.panel-danger
.panel-heading
The remote repository failed to update #{time_ago_with_tooltip(@remote_mirror.last_update_at)}.
- if @remote_mirror.last_successful_update_at
Last successful update #{time_ago_with_tooltip(@remote_mirror.last_successful_update_at)}.
.panel-body
%pre
:preserve
#{h(@remote_mirror.last_error.strip)}
= f.fields_for :remote_mirrors, @remote_mirror do |rm_form|
.form-group
= rm_form.check_box :enabled, class: "pull-left"
.prepend-left-20
= rm_form.label :enabled, "Remote mirror repository", class: "label-light append-bottom-0"
%p.light.append-bottom-0
Automatically update the remote mirror's branches, tags, and commits from this repository every time someone pushes to it.
.form-group.has-feedback
= rm_form.label :url, "Git repository URL", class: "label-light"
= rm_form.text_field :url, class: "form-control", placeholder: 'https://username:password@gitlab.company.com/group/project.git'
= render "projects/mirrors/instructions"
= f.submit 'Save changes', class: 'btn btn-create', name: 'update_remote_mirror'
- expanded = Rails.env.test? - if @project.feature_available?(:repository_mirrors)
%section.settings.project-mirror-settings = render 'projects/mirrors/pull'
.settings-header = render 'projects/mirrors/push'
%h4
Pull from a remote repository
%button.btn.js-settings-toggle
= expanded ? 'Collapse' : 'Expand'
%p
Set up your project to automatically have its branches, tags, and commits
updated from an upstream repository.
= link_to 'Read more', help_page_path('workflow/repository_mirroring', anchor: 'pulling-from-a-remote-repository'), target: '_blank'
.settings-content.no-animate{ class: ('expanded' if expanded) }
= form_for @project, url: project_mirror_path(@project) do |f|
%div
= form_errors(@project)
%h5
Set up mirror repository
= render "shared/mirror_update_button"
- if @project.mirror_last_update_failed?
.panel.panel-danger
.panel-heading
The repository failed to update #{time_ago_with_tooltip(@project.mirror_last_update_at)}.
- if @project.mirror_ever_updated_successfully?
Last successful update #{time_ago_with_tooltip(@project.mirror_last_successful_update_at)}.
.panel-body
%pre
:preserve
#{h(@project.import_error.try(:strip))}
.form-group
= f.check_box :mirror, class: "pull-left"
.prepend-left-20
= f.label :mirror, "Mirror repository", class: "label-light append-bottom-0"
.form-group
= f.label :import_url, "Git repository URL", class: "label-light"
= f.text_field :import_url, class: 'form-control', placeholder: 'https://username:password@gitlab.company.com/group/project.git'
= render "projects/mirrors/instructions"
.form-group
= f.label :mirror_user_id, "Mirror user", class: "label-light"
= select_tag('project[mirror_user_id]', options_for_mirror_user, class: "select2 lg", required: true)
.help-block
This user will be the author of all events in the activity feed that are the result of an update,
like new branches being created or new commits being pushed to existing branches.
You can only assign yourself to be the mirror user.
- if @project.builds_enabled?
= render "shared/mirror_trigger_builds_setting", f: f
= f.submit 'Save changes', class: 'btn btn-create', name: 'update_remote_mirror'
%section.settings
.settings-header
%h4
Push to a remote repository
%button.btn.js-settings-toggle
= expanded ? 'Collapse' : 'Expand'
%p
Set up the remote repository that you want to update with the content of the current repository
every time someone pushes to it.
= link_to 'Read more', help_page_path('workflow/repository_mirroring', anchor: 'pushing-to-a-remote-repository'), target: '_blank'
.settings-content.no-animate{ class: ('expanded' if expanded) }
= form_for @project, url: project_mirror_path(@project) do |f|
%div
= form_errors(@project)
= render "shared/remote_mirror_update_button", remote_mirror: @remote_mirror
- if @remote_mirror.last_error.present?
.panel.panel-danger
.panel-heading
The remote repository failed to update #{time_ago_with_tooltip(@remote_mirror.last_update_at)}.
- if @remote_mirror.last_successful_update_at
Last successful update #{time_ago_with_tooltip(@remote_mirror.last_successful_update_at)}.
.panel-body
%pre
:preserve
#{h(@remote_mirror.last_error.strip)}
= f.fields_for :remote_mirrors, @remote_mirror do |rm_form|
.form-group
= rm_form.check_box :enabled, class: "pull-left"
.prepend-left-20
= rm_form.label :enabled, "Remote mirror repository", class: "label-light append-bottom-0"
%p.light.append-bottom-0
Automatically update the remote mirror's branches, tags, and commits from this repository every time someone pushes to it.
.form-group.has-feedback
= rm_form.label :url, "Git repository URL", class: "label-light"
= rm_form.text_field :url, class: "form-control", placeholder: 'https://username:password@gitlab.company.com/group/project.git'
= render "projects/mirrors/instructions"
= f.submit 'Save changes', class: 'btn btn-create', name: 'update_remote_mirror'
...@@ -11,7 +11,7 @@ class UpdateAllMirrorsWorker ...@@ -11,7 +11,7 @@ class UpdateAllMirrorsWorker
fail_stuck_mirrors! fail_stuck_mirrors!
Project.mirrors_to_sync.each(&:import_schedule) unless Gitlab::Mirror.max_mirror_capacity_reached? schedule_mirrors!
cancel_lease(lease_uuid) cancel_lease(lease_uuid)
end end
...@@ -22,6 +22,32 @@ class UpdateAllMirrorsWorker ...@@ -22,6 +22,32 @@ class UpdateAllMirrorsWorker
end end
end end
def schedule_mirrors!
capacity = batch_size = Gitlab::Mirror.available_capacity
# Ignore mirrors that become due for scheduling once work begins, so we
# can't end up in an infinite loop
now = Time.now
last = nil
# Normally, this will complete in 1-2 batches. One batch will be added per
# `batch_size` unlicensed projects in the database.
while capacity > 0
projects = pull_mirrors_batch(freeze_at: now, batch_size: batch_size, offset_at: last)
break if projects.empty?
last = projects.last.mirror_data.next_execution_timestamp
projects.each do |project|
next unless project.feature_available?(:repository_mirrors)
capacity -= 1
project.import_schedule
break unless capacity > 0
end
end
end
private private
def try_obtain_lease def try_obtain_lease
...@@ -31,4 +57,17 @@ class UpdateAllMirrorsWorker ...@@ -31,4 +57,17 @@ class UpdateAllMirrorsWorker
def cancel_lease(uuid) def cancel_lease(uuid)
::Gitlab::ExclusiveLease.cancel(LEASE_KEY, uuid) ::Gitlab::ExclusiveLease.cancel(LEASE_KEY, uuid)
end end
def pull_mirrors_batch(freeze_at:, batch_size:, offset_at: nil)
relation = Project
.mirror
.joins(:mirror_data)
.where("next_execution_timestamp <= ? AND import_status NOT IN ('scheduled', 'started')", freeze_at)
.reorder('project_mirror_data.next_execution_timestamp')
.limit(batch_size)
relation = relation.where('next_execution_timestamp > ?', offset_at) if offset_at
relation
end
end end
---
title: Namespace license checks for Repository Mirrors
merge_request: 2328
author:
require 'spec_helper'
describe Admin::ApplicationSettingsController do # rubocop:disable RSpec/FilePath
include StubENV
let(:admin) { create(:admin) }
before do
stub_env('IN_MEMORY_APPLICATION_SETTINGS', 'false')
end
describe 'PUT #update' do
before do
sign_in(admin)
end
it 'updates the EE specific application settings' do
settings = {
help_text: 'help_text',
elasticsearch_url: 'http://my-elastic.search:9200',
elasticsearch_indexing: true,
elasticsearch_aws: true,
elasticsearch_aws_access_key: 'elasticsearch_aws_access_key',
elasticsearch_aws_secret_access_key: 'elasticsearch_aws_secret_access_key',
elasticsearch_aws_region: 'elasticsearch_aws_region',
elasticsearch_search: true,
repository_size_limit: 1024,
shared_runners_minutes: 60,
geo_status_timeout: 30,
elasticsearch_experimental_indexer: true,
check_namespace_plan: true,
authorized_keys_enabled: true,
slack_app_enabled: true,
slack_app_id: 'slack_app_id',
slack_app_secret: 'slack_app_secret',
slack_app_verification_token: 'slack_app_verification_token'
}
put :update, application_setting: settings
expect(response).to redirect_to(admin_application_settings_path)
settings.except(:elasticsearch_url, :repository_size_limit).each do |setting, value|
expect(ApplicationSetting.current.public_send(setting)).to eq(value)
end
expect(ApplicationSetting.current.repository_size_limit).to eq(settings[:repository_size_limit].megabytes)
expect(ApplicationSetting.current.elasticsearch_url).to contain_exactly(settings[:elasticsearch_url])
end
it 'does not update mirror settings when repository mirrors unlicensed' do
stub_licensed_features(repository_mirrors: false)
settings = {
mirror_max_delay: 12,
mirror_max_capacity: 2,
mirror_capacity_threshold: 2
}
settings.each do |setting, _value|
expect do
put :update, application_setting: settings
end.not_to change(ApplicationSetting.current.reload, setting)
end
end
it 'updates mirror settings when repository mirrors is licensed' do
stub_licensed_features(repository_mirrors: true)
settings = {
mirror_max_delay: 12,
mirror_max_capacity: 2,
mirror_capacity_threshold: 2
}
put :update, application_setting: settings
settings.each do |setting, value|
expect(ApplicationSetting.current.public_send(setting)).to eq(value)
end
end
end
end
require('spec_helper')
describe ProjectsController do # rubocop:disable RSpec/FilePath
let(:project) { create(:empty_project) }
let(:user) { create(:user) }
before do
project.add_master(user)
sign_in(user)
end
describe 'PUT #update' do
before do
controller.instance_variable_set(:@project, project)
end
it 'updates EE attributes' do
params = {
repository_size_limit: 1024
}
put :update,
namespace_id: project.namespace,
id: project.id,
project: params
expect(response).to have_http_status(302)
params.except(:repository_size_limit).each do |param, value|
expect(project.public_send(param)).to eq(value)
end
expect(project.repository_size_limit).to eq(params[:repository_size_limit].megabytes)
end
it 'updates Merge Request Approvers attributes' do
params = {
approvals_before_merge: 50,
approver_group_ids: create(:group).id,
approver_ids: create(:user).id,
reset_approvals_on_push: false
}
put :update,
namespace_id: project.namespace,
id: project.id,
project: params
expect(response).to have_http_status(302)
expect(project.approver_groups.pluck(:group_id)).to contain_exactly(params[:approver_group_ids])
expect(project.approvers.pluck(:user_id)).to contain_exactly(params[:approver_ids])
end
it 'updates Issuable Default Templates attributes' do
params = {
issues_template: 'You got issues?',
merge_requests_template: 'I got tissues'
}
put :update,
namespace_id: project.namespace,
id: project.id,
project: params
expect(response).to have_http_status(302)
params.each do |param, value|
expect(project.public_send(param)).to eq(value)
end
end
it 'updates Fast Forward Merge attributes' do
params = {
merge_method: :ff
}
put :update,
namespace_id: project.namespace,
id: project.id,
project: params
expect(response).to have_http_status(302)
params.each do |param, value|
expect(project.public_send(param)).to eq(value)
end
end
it 'updates Fast Forward Merge attributes' do
params = {
merge_method: :ff
}
put :update,
namespace_id: project.namespace,
id: project.id,
project: params
expect(response).to have_http_status(302)
params.each do |param, value|
expect(project.public_send(param)).to eq(value)
end
end
it 'updates Service Desk attributes' do
allow(Gitlab::IncomingEmail).to receive(:enabled?) { true }
allow(Gitlab::IncomingEmail).to receive(:supports_wildcard?) { true }
stub_licensed_features(service_desk: true)
params = {
service_desk_enabled: true
}
put :update,
namespace_id: project.namespace,
id: project.id,
project: params
expect(response).to have_http_status(302)
expect(project.service_desk_enabled).to eq(true)
end
context 'repository mirrors licensed' do
before do
stub_licensed_features(repository_mirrors: true)
end
it 'updates repository mirror attributes' do
params = {
mirror: true,
mirror_trigger_builds: true,
mirror_user_id: user.id
}
put :update,
namespace_id: project.namespace,
id: project.id,
project: params
params.each do |param, value|
expect(project.public_send(param)).to eq(value)
end
end
end
context 'repository mirrors unlicensed' do
before do
stub_licensed_features(repository_mirrors: false)
end
it 'does not update repository mirror attributes' do
params = {
mirror: true,
mirror_trigger_builds: true,
mirror_user_id: user.id
}
params.each do |param, _value|
expect do
put :update,
namespace_id: project.namespace,
id: project.id,
project: params
end.not_to change(project, param)
end
end
end
end
end
...@@ -10,6 +10,18 @@ feature 'Project mirror', feature: true do ...@@ -10,6 +10,18 @@ feature 'Project mirror', feature: true do
sign_in user sign_in user
end end
context 'unlicensed' do
before do
stub_licensed_features(repository_mirrors: false)
end
it 'returns 404' do
visit project_mirror_path(project)
expect(page.status_code).to eq(404)
end
end
context 'with Update now button' do context 'with Update now button' do
let(:timestamp) { Time.now } let(:timestamp) { Time.now }
......
require 'spec_helper' require 'spec_helper'
describe 'Project settings > [EE] repository', feature: true do describe 'Project settings > [EE] repository', feature: true do
include Select2Helper
let(:user) { create(:user) } let(:user) { create(:user) }
let(:project) { create(:project_empty_repo) } let(:project) { create(:project_empty_repo) }
...@@ -44,24 +42,4 @@ describe 'Project settings > [EE] repository', feature: true do ...@@ -44,24 +42,4 @@ describe 'Project settings > [EE] repository', feature: true do
end end
end end
end end
describe 'mirror settings', :js do
let(:user2) { create(:user) }
before do
project.team << [user2, :master]
visit project_settings_repository_path(project)
end
it 'sets mirror user' do
page.within('.project-mirror-settings') do
select2(user2.id, from: '#project_mirror_user_id')
click_button('Save changes')
expect(find('.select2-chosen')).to have_content(user.name)
end
end
end
end end
require 'spec_helper'
describe 'Project settings > [EE] repository', feature: true do
include Select2Helper
let(:user) { create(:user) }
let(:project) { create(:project_empty_repo) }
before do
project.add_master(user)
gitlab_sign_in(user)
end
context 'unlicensed' do
before do
stub_licensed_features(repository_mirrors: false)
visit project_settings_repository_path(project)
end
it 'does not show pull mirror settings' do
expect(page).to have_no_selector('#project_mirror')
expect(page).to have_no_selector('#project_import_url')
expect(page).to have_no_selector('#project_mirror_user_id', visible: false)
expect(page).to have_no_selector('#project_mirror_trigger_builds')
end
it 'does not show push mirror settings' do
expect(page).to have_no_selector('#project_remote_mirrors_attributes_0_enabled')
expect(page).to have_no_selector('#project_remote_mirrors_attributes_0_url')
end
end
describe 'mirror settings', :js do
let(:user2) { create(:user) }
before do
project.team << [user2, :master]
visit project_settings_repository_path(project)
end
it 'shows pull mirror settings' do
expect(page).to have_selector('#project_mirror')
expect(page).to have_selector('#project_import_url')
expect(page).to have_selector('#project_mirror_user_id', visible: false)
expect(page).to have_selector('#project_mirror_trigger_builds')
end
it 'shows push mirror settings' do
expect(page).to have_selector('#project_remote_mirrors_attributes_0_enabled')
expect(page).to have_selector('#project_remote_mirrors_attributes_0_url')
end
it 'sets mirror user' do
page.within('.project-mirror-settings') do
select2(user2.id, from: '#project_mirror_user_id')
click_button('Save changes')
expect(find('.select2-chosen')).to have_content(user.name)
end
end
end
end
...@@ -217,6 +217,62 @@ describe Project, models: true do ...@@ -217,6 +217,62 @@ describe Project, models: true do
end end
end end
describe '#has_remote_mirror?' do
let(:project) { create(:empty_project, :remote_mirror, :import_started) }
subject { project.has_remote_mirror? }
before do
allow_any_instance_of(RemoteMirror).to receive(:refresh_remote)
end
it 'returns true when a remote mirror is enabled' do
is_expected.to be_truthy
end
it 'returns false when unlicensed' do
stub_licensed_features(repository_mirrors: false)
is_expected.to be_falsy
end
it 'returns false when remote mirror is disabled' do
project.remote_mirrors.first.update_attributes(enabled: false)
is_expected.to be_falsy
end
end
describe '#update_remote_mirrors' do
let(:project) { create(:empty_project, :remote_mirror, :import_started) }
delegate :update_remote_mirrors, to: :project
before do
allow_any_instance_of(RemoteMirror).to receive(:refresh_remote)
end
it 'syncs enabled remote mirror' do
expect_any_instance_of(RemoteMirror).to receive(:sync)
update_remote_mirrors
end
it 'does nothing when unlicensed' do
stub_licensed_features(repository_mirrors: false)
expect_any_instance_of(RemoteMirror).not_to receive(:sync)
update_remote_mirrors
end
it 'does not sync disabled remote mirrors' do
project.remote_mirrors.first.update_attributes(enabled: false)
expect_any_instance_of(RemoteMirror).not_to receive(:sync)
update_remote_mirrors
end
end
describe '#any_runners_limit' do describe '#any_runners_limit' do
let(:project) { create(:empty_project, shared_runners_enabled: shared_runners_enabled) } let(:project) { create(:empty_project, shared_runners_enabled: shared_runners_enabled) }
let(:specific_runner) { create(:ci_runner) } let(:specific_runner) { create(:ci_runner) }
......
...@@ -96,6 +96,18 @@ describe RemoteMirror do ...@@ -96,6 +96,18 @@ describe RemoteMirror do
Timecop.return Timecop.return
end end
context 'repository mirrors not licensed' do
before do
stub_licensed_features(repository_mirrors: false)
end
it 'does not schedule RepositoryUpdateRemoteMirrorWorker' do
expect(RepositoryUpdateRemoteMirrorWorker).not_to receive(:perform_in)
remote_mirror.sync
end
end
context 'with remote mirroring enabled' do context 'with remote mirroring enabled' do
it 'schedules a RepositoryUpdateRemoteMirrorWorker to run within a certain backoff delay' do it 'schedules a RepositoryUpdateRemoteMirrorWorker to run within a certain backoff delay' do
expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_in).with(RemoteMirror::BACKOFF_DELAY, remote_mirror.id, Time.now) expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_in).with(RemoteMirror::BACKOFF_DELAY, remote_mirror.id, Time.now)
......
...@@ -4,6 +4,21 @@ describe Projects::UpdateMirrorService do ...@@ -4,6 +4,21 @@ describe Projects::UpdateMirrorService do
let(:project) { create(:project, :mirror, import_url: Project::UNKNOWN_IMPORT_URL) } let(:project) { create(:project, :mirror, import_url: Project::UNKNOWN_IMPORT_URL) }
describe "#execute" do describe "#execute" do
context 'unlicensed' do
before do
stub_licensed_features(repository_mirrors: false)
end
it 'does nothing' do
allow_any_instance_of(EE::Project).to receive(:destroy_mirror_data)
expect(project).not_to receive(:fetch_mirror)
result = described_class.new(project, project.owner).execute
expect(result[:status]).to eq(:success)
end
end
it "fetches the upstream repository" do it "fetches the upstream repository" do
expect(project).to receive(:fetch_mirror) expect(project).to receive(:fetch_mirror)
...@@ -111,12 +126,12 @@ describe Projects::UpdateMirrorService do ...@@ -111,12 +126,12 @@ describe Projects::UpdateMirrorService do
describe "when is no mirror" do describe "when is no mirror" do
let(:project) { build_stubbed(:project) } let(:project) { build_stubbed(:project) }
it "fails" do it "success" do
expect(project.mirror?).to eq(false) expect(project.mirror?).to eq(false)
result = described_class.new(project, build_stubbed(:user)).execute result = described_class.new(project, build_stubbed(:user)).execute
expect(result[:status]).to eq(:error) expect(result[:status]).to eq(:success)
end end
end end
end end
......
...@@ -4,7 +4,7 @@ describe Projects::UpdateRemoteMirrorService do ...@@ -4,7 +4,7 @@ describe Projects::UpdateRemoteMirrorService do
let(:project) { create(:project) } let(:project) { create(:project) }
let(:remote_project) { create(:forked_project_with_submodules) } let(:remote_project) { create(:forked_project_with_submodules) }
let(:repository) { project.repository } let(:repository) { project.repository }
let(:remote_mirror) { project.remote_mirrors.create!(url: remote_project.http_url_to_repo) } let(:remote_mirror) { project.remote_mirrors.create!(url: remote_project.http_url_to_repo, enabled: true) }
subject { described_class.new(project, project.creator) } subject { described_class.new(project, project.creator) }
...@@ -18,6 +18,14 @@ describe Projects::UpdateRemoteMirrorService do ...@@ -18,6 +18,14 @@ describe Projects::UpdateRemoteMirrorService do
allow(gitlab_shell).to receive(:push_remote_branches).and_return(true) allow(gitlab_shell).to receive(:push_remote_branches).and_return(true)
end end
it 'does nothing when unlicensed' do
stub_licensed_features(repository_mirrors: false)
expect(project.repository).not_to receive(:fetch_remote)
subject.execute(remote_mirror)
end
it "fetches the remote repository" do it "fetches the remote repository" do
expect(repository).to receive(:fetch_remote).with(remote_mirror.ref_name, no_tags: true) do expect(repository).to receive(:fetch_remote).with(remote_mirror.ref_name, no_tags: true) do
sync_remote(repository, remote_mirror.ref_name, local_branch_names) sync_remote(repository, remote_mirror.ref_name, local_branch_names)
......
require 'rails_helper' require 'spec_helper'
describe UpdateAllMirrorsWorker do describe UpdateAllMirrorsWorker do
subject(:worker) { described_class.new } subject(:worker) { described_class.new }
...@@ -23,6 +23,12 @@ describe UpdateAllMirrorsWorker do ...@@ -23,6 +23,12 @@ describe UpdateAllMirrorsWorker do
worker.perform worker.perform
end end
it 'schedules mirrors' do
expect(worker).to receive(:schedule_mirrors!)
worker.perform
end
end end
describe '#fail_stuck_mirrors!' do describe '#fail_stuck_mirrors!' do
...@@ -63,4 +69,80 @@ describe UpdateAllMirrorsWorker do ...@@ -63,4 +69,80 @@ describe UpdateAllMirrorsWorker do
expect(project.reload.import_error).to eq 'The mirror update took too long to complete.' expect(project.reload.import_error).to eq 'The mirror update took too long to complete.'
end end
end end
describe '#schedule_mirrors!' do
def schedule_mirrors!(capacity:)
allow(Gitlab::Mirror).to receive_messages(available_capacity: capacity)
Sidekiq::Testing.fake! do
worker.schedule_mirrors!
end
end
def expect_import_status(project, status)
expect(project.reload.import_status).to eq(status)
end
def expect_import_scheduled(*projects)
projects.each { |project| expect_import_status(project, 'scheduled') }
end
def expect_import_not_scheduled(*projects)
projects.each { |project| expect_import_status(project, 'none') }
end
context 'unlicensed' do
it 'does not schedule when project does not have repository mirrors available' do
project = create(:empty_project, :mirror)
stub_licensed_features(repository_mirrors: false)
schedule_mirrors!(capacity: 5)
expect_import_not_scheduled(project)
end
end
context 'licensed' do
def scheduled_mirror(at:, licensed:)
namespace = create(:group, :public, plan: (Namespace::BRONZE_PLAN if licensed))
project = create(:empty_project, :public, :mirror, namespace: namespace)
project.mirror_data.update!(next_execution_timestamp: at)
project.update!(visibility_level: Gitlab::VisibilityLevel::PRIVATE)
project
end
before do
stub_licensed_features(repository_mirrors: true)
stub_application_setting(check_namespace_plan: true)
allow(Gitlab).to receive_messages(com?: true)
end
let!(:unlicensed_project) { scheduled_mirror(at: 4.weeks.ago, licensed: false) }
let!(:earliest_project) { scheduled_mirror(at: 3.weeks.ago, licensed: true) }
let!(:latest_project) { scheduled_mirror(at: 2.weeks.ago, licensed: true) }
it "schedules all available mirrors when capacity is in excess" do
schedule_mirrors!(capacity: 3)
expect_import_scheduled(earliest_project, latest_project)
expect_import_not_scheduled(unlicensed_project)
end
it "schedules all available mirrors when capacity is sufficient" do
schedule_mirrors!(capacity: 2)
expect_import_scheduled(earliest_project, latest_project)
expect_import_not_scheduled(unlicensed_project)
end
it 'schedules mirrors by next_execution_timestamp when capacity is insufficient' do
schedule_mirrors!(capacity: 1)
expect_import_scheduled(earliest_project)
expect_import_not_scheduled(unlicensed_project, latest_project)
end
end
end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment