Commit 123e2c9d authored by Lin Jen-Shin's avatar Lin Jen-Shin

Merge remote-tracking branch 'ee/master' into ce-to-ee-2017-11-03

* ee/master: (28 commits)
  Add system hooks user_rename and group_rename
  Fix EE-only exception to the git default user SSH config check
  Allow Geo repository sync over HTTP/HTTPS
  Correctly handle a failure to create a new Geo node
  Refactor most EE-specific code out of Gitlab::Git::GitAccess
  More the specs more robust
  Make Geo::PruneEventLogWorker generate some logging
  Use Gitlab::Geo::LogHelpers where possible
  Improve mocking on Geo::NodeStatusService
  Do not generate Geo Log events when there are no secondaries
  Create Geo::PruneEventLogWorker
  Extract try_obtain_lease to a concern
  Update wording
  Spec the committer ChangeAccess check for merge requests
  Add a spec for a fast forward merge
  Validate that the committer email is verified on the user.
  Build messages used on the project settings page
  Rename commit_author to commit_committer
  Validate the commit using all known email addresses for the user
  Address some sections of first code review
  ...
parents 21788e48 54933f55
......@@ -5,19 +5,28 @@ import {
import '../flash';
import Api from '../api';
const onPrimaryCheckboxChange = function onPrimaryCheckboxChange(e, $namespaces) {
const onPrimaryCheckboxChange = function onPrimaryCheckboxChange(e, $namespaces, $key, $useSSH) {
const $namespacesSelect = $('.select2', $namespaces);
$namespacesSelect.select2('data', null);
$namespaces.toggleClass('hidden', e.currentTarget.checked);
$key.toggleClass('hidden', e.currentTarget.checked || !$useSSH.is(':checked'));
};
export default function geoNodeForm($container) {
const $namespaces = $('.js-hide-if-geo-primary', $container);
const $primaryCheckbox = $('input[type="checkbox"]', $container);
const $select2Dropdown = $('.js-geo-node-namespaces', $container);
const $useHTTP = $('.js-use-http', $container);
const $useSSH = $('.js-use-ssh', $container);
const $sshKey = $('.js-ssh-key', $container);
$primaryCheckbox.on('change', e =>
onPrimaryCheckboxChange(e, $namespaces, $sshKey, $useSSH));
$primaryCheckbox.on('change', e => onPrimaryCheckboxChange(e, $namespaces));
$useHTTP.on('click', () => $sshKey.toggleClass('hidden', true));
$useSSH.on('click', () => $sshKey.toggleClass('hidden', false));
$select2Dropdown.select2({
placeholder: s__('Geo|Select groups to replicate.'),
......
......@@ -24,9 +24,19 @@ class Admin::PushRulesController < Admin::ApplicationController
end
def push_rule_params
params.require(:push_rule).permit(:deny_delete_tag, :delete_branch_regex,
:commit_message_regex, :branch_name_regex, :force_push_regex, :author_email_regex, :member_check,
:file_name_regex, :max_file_size, :prevent_secrets, :reject_unsigned_commits)
allowed_fields = %i[deny_delete_tag delete_branch_regex commit_message_regex
branch_name_regex force_push_regex author_email_regex
member_check file_name_regex max_file_size prevent_secrets]
if @push_rule.available?(:reject_unsigned_commits)
allowed_fields << :reject_unsigned_commits
end
if @push_rule.available?(:commit_committer_check)
allowed_fields << :commit_committer_check
end
params.require(:push_rule).permit(allowed_fields)
end
def push_rule
......
class Projects::GitHttpController < Projects::GitHttpClientController
include WorkhorseRequest
prepend ::EE::Projects::GitHttpController
before_action :access_check
......
......@@ -33,6 +33,10 @@ class Projects::PushRulesController < Projects::ApplicationController
allowed_fields << :reject_unsigned_commits
end
if can?(current_user, :change_commit_committer_check, project)
allowed_fields << :commit_committer_check
end
params.require(:push_rule).permit(allowed_fields)
end
end
module PushRulesHelper
def reject_unsigned_commits_description(push_rule)
message = [s_("ProjectSettings|Only signed commits can be pushed to this repository.")]
message = s_("ProjectSettings|Only signed commits can be pushed to this repository.")
push_rule_update_description(message, push_rule, :reject_unsigned_commits)
end
def commit_committer_check_description(push_rule)
message = s_("ProjectSettings|Users can only push commits to this repository "\
"that were committed with one of their own verified emails.")
push_rule_update_description(message, push_rule, :commit_committer_check)
end
private
def push_rule_update_description(message, push_rule, rule)
messages = [message]
if push_rule.global?
message << s_("ProjectSettings|This setting will be applied to all projects unless overridden by an admin.")
messages << s_("ProjectSettings|This setting will be applied to all projects unless overridden by an admin.")
else
if PushRule.global&.reject_unsigned_commits
message << if push_rule.reject_unsigned_commits
s_("ProjectSettings|This setting is applied on the server level and can be overridden by an admin.")
else
s_("ProjectSettings|This setting is applied on the server level but has been overridden for this project.")
end
message << s_("ProjectSettings|Contact an admin to change this setting.") unless current_user.admin?
enabled_globally = PushRule.global&.public_send(rule) # rubocop:disable GitlabSecurity/PublicSend
enabled_in_project = push_rule.public_send(rule) # rubocop:disable GitlabSecurity/PublicSend
if enabled_globally
messages << if enabled_in_project
s_("ProjectSettings|This setting is applied on the server level and can be overridden by an admin.")
else
s_("ProjectSettings|This setting is applied on the server level but has been overridden for this project.")
end
messages << s_("ProjectSettings|Contact an admin to change this setting.") unless current_user.admin?
end
end
message.join(' ')
messages.join(' ')
end
end
......@@ -11,7 +11,8 @@ class GeoNode < ActiveRecord::Base
host: lambda { Gitlab.config.gitlab.host },
port: lambda { Gitlab.config.gitlab.port },
relative_url_root: lambda { Gitlab.config.gitlab.relative_url_root },
primary: false
primary: false,
clone_protocol: 'http'
accepts_nested_attributes_for :geo_node_key
......@@ -21,8 +22,9 @@ class GeoNode < ActiveRecord::Base
validates :relative_url_root, length: { minimum: 0, allow_nil: false }
validates :access_key, presence: true
validates :encrypted_secret_access_key, presence: true
validates :clone_protocol, presence: true, inclusion: %w(ssh http)
validates :geo_node_key, presence: true, if: :secondary?
validates :geo_node_key, presence: true, if: :uses_ssh_key?
validate :check_not_adding_primary_as_secondary, if: :secondary?
after_initialize :build_dependents
......@@ -46,6 +48,10 @@ class GeoNode < ActiveRecord::Base
!primary
end
def uses_ssh_key?
secondary? && clone_protocol == 'ssh'
end
def uri
if relative_url_root
relative_url = relative_url_root.starts_with?('/') ? relative_url_root : "/#{relative_url_root}"
......@@ -205,10 +211,12 @@ class GeoNode < ActiveRecord::Base
def update_dependents_attributes
if primary?
self.geo_node_key = nil
else
elsif uses_ssh_key?
self.geo_node_key&.title = "Geo node: #{self.url}"
end
self.geo_node_key = nil unless uses_ssh_key? || geo_node_key&.persisted?
if self.primary?
self.oauth_application = nil
update_clone_url
......
......@@ -22,6 +22,6 @@ class GeoNodeKey < Key
# but if it is made a primary and the keys are not removed, every user on the
# GitLab instance will be able to access every project using this key.
def active?
geo_node&.secondary?
geo_node&.uses_ssh_key?
end
end
......@@ -56,6 +56,7 @@ class Group < Namespace
after_create :post_create_hook
after_destroy :post_destroy_hook
after_save :update_two_factor_requirement
after_update :path_changed_hook, if: :path_changed?
scope :where_group_links_with_provider, ->(provider) do
joins(:ldap_group_links).where(ldap_group_links: { provider: provider })
......@@ -341,6 +342,12 @@ class Group < Namespace
list_of_ids.reverse.map { |group| variables[group.id] }.compact.flatten
end
def full_path_was
return path_was unless has_parent?
"#{parent.full_path}/#{path_was}"
end
private
def update_two_factor_requirement
......@@ -349,6 +356,10 @@ class Group < Namespace
users.find_each(&:update_two_factor_requirement)
end
def path_changed_hook
system_hook_service.execute_hooks_for(self, :rename)
end
def visibility_level_allowed_by_parent
return if visibility_level_allowed_by_parent?
......
......@@ -48,6 +48,7 @@ class License < ActiveRecord::Base
service_desk
variable_environment_scope
reject_unsigned_commits
commit_committer_check
].freeze
EEU_FEATURES = EEP_FEATURES
......
......@@ -5,6 +5,10 @@ class PushRule < ActiveRecord::Base
validates :max_file_size, numericality: { greater_than_or_equal_to: 0, only_integer: true }
FILES_BLACKLIST = YAML.load_file(Rails.root.join('lib/gitlab/checks/files_blacklist.yml'))
SETTINGS_WITH_GLOBAL_DEFAULT = %i[
reject_unsigned_commits
commit_committer_check
].freeze
def self.global
find_by(is_sample: true)
......@@ -15,6 +19,7 @@ class PushRule < ActiveRecord::Base
branch_name_regex.present? ||
author_email_regex.present? ||
reject_unsigned_commits ||
commit_committer_check ||
member_check ||
file_name_regex.present? ||
max_file_size > 0 ||
......@@ -28,6 +33,13 @@ class PushRule < ActiveRecord::Base
commit.has_signature?
end
def committer_allowed?(committer_email, current_user)
return true unless available?(:commit_committer_check)
return true unless commit_committer_check
current_user.verified_email?(committer_email)
end
def commit_message_allowed?(message)
data_match?(message, commit_message_regex)
end
......@@ -48,39 +60,34 @@ class PushRule < ActiveRecord::Base
regex_list.find { |regex| data_match?(file_path, regex) }
end
def reject_unsigned_commits=(value)
enabled_globally = PushRule.global&.reject_unsigned_commits
is_disabled = !Gitlab::Utils.to_boolean(value)
def global?
is_sample?
end
# If setting is globally disabled and user disable it at project level,
# reset the attr so we can use the default global if required later.
if !enabled_globally && is_disabled
super(nil)
def available?(feature_sym)
if global?
License.feature_available?(feature_sym)
else
super(value)
project&.feature_available?(feature_sym)
end
end
def reject_unsigned_commits
value = super
# return if value is true/false or if current object is the global setting
return value if global? || !value.nil?
PushRule.global&.reject_unsigned_commits
read_setting_with_global_default(:reject_unsigned_commits)
end
alias_method :reject_unsigned_commits?, :reject_unsigned_commits
def global?
is_sample?
def reject_unsigned_commits=(value)
write_setting_with_global_default(:reject_unsigned_commits, value)
end
def available?(feature_sym)
if global?
License.feature_available?(feature_sym)
else
project&.feature_available?(feature_sym)
end
def commit_committer_check
read_setting_with_global_default(:commit_committer_check)
end
alias_method :commit_committer_check?, :commit_committer_check
def commit_committer_check=(value)
write_setting_with_global_default(:commit_committer_check, value)
end
private
......@@ -92,4 +99,26 @@ class PushRule < ActiveRecord::Base
true
end
end
def read_setting_with_global_default(setting)
value = read_attribute(setting)
# return if value is true/false or if current object is the global setting
return value if global? || !value.nil?
PushRule.global&.public_send(setting)
end
def write_setting_with_global_default(setting, value)
enabled_globally = PushRule.global&.public_send(setting)
is_disabled = !Gitlab::Utils.to_boolean(value)
# If setting is globally disabled and user disable it at project level,
# reset the attr so we can use the default global if required later.
if !enabled_globally && is_disabled
write_attribute(setting, nil)
else
write_attribute(setting, value)
end
end
end
......@@ -171,6 +171,7 @@ class User < ActiveRecord::Base
before_save :skip_reconfirmation!, if: ->(user) { user.email_changed? && user.read_only_attribute?(:email) }
before_save :check_for_verified_email, if: ->(user) { user.email_changed? && !user.new_record? }
after_save :ensure_namespace_correct
after_update :username_changed_hook, if: :username_changed?
after_destroy :post_destroy_hook
after_commit :update_emails_with_primary_email, on: :update, if: -> { previous_changes.key?('email') }
after_commit :update_invalid_gpg_signatures, on: :update, if: -> { previous_changes.key?('email') }
......@@ -890,6 +891,10 @@ class User < ActiveRecord::Base
end
end
def username_changed_hook
system_hook_service.execute_hooks_for(self, :rename)
end
def post_destroy_hook
log_info("User \"#{name}\" (#{email}) was removed")
system_hook_service.execute_hooks_for(self, :destroy)
......
......@@ -37,12 +37,54 @@ module Geo
end
end
def primary_http_path_prefix
@primary_http_path_prefix ||= Gitlab::Geo.primary_node.url
end
private
def sync_repository
raise NotImplementedError, 'This class should implement sync_repository method'
end
def current_node
::Gitlab::Geo.current_node
end
def fetch_geo_mirror(repository)
case current_node&.clone_protocol
when 'http'
fetch_http_geo_mirror(repository)
when 'ssh'
fetch_ssh_geo_mirror(repository)
else
raise "Unknown clone protocol: #{current_node&.clone_protocol}"
end
end
def build_repository_url(prefix, repository)
url = prefix
url += '/' unless url.end_with?('/')
url + repository.full_path + '.git'
end
def fetch_http_geo_mirror(repository)
url = build_repository_url(primary_http_path_prefix, repository)
# Fetch the repository, using a JWT header for authentication
authorization = ::Gitlab::Geo::BaseRequest.new.authorization
header = { "http.#{url}.extraHeader" => "Authorization: #{authorization}" }
repository.with_config(header) { repository.fetch_geo_mirror(url) }
end
def fetch_ssh_geo_mirror(repository)
url = build_repository_url(primary_ssh_path_prefix, repository)
repository.fetch_geo_mirror(url)
end
def registry
@registry ||= Geo::ProjectRegistry.find_or_initialize_by(project_id: project.id)
end
......
......@@ -17,6 +17,8 @@ module Geo
# The `build_event` method is supposed to return an instance of the event
# that will be logged.
class EventStore
include ::Gitlab::Geo::ProjectLogHelpers
class << self
attr_accessor :event_type
end
......@@ -30,6 +32,7 @@ module Geo
def create
return unless Gitlab::Geo.primary?
return unless Gitlab::Geo.secondary_nodes.any? # no need to create an event if no one is listening
Geo::EventLog.create!("#{self.class.event_type}" => build_event)
rescue ActiveRecord::RecordInvalid, NoMethodError => e
......@@ -42,20 +45,5 @@ module Geo
raise NotImplementedError,
"#{self.class} does not implement #{__method__}"
end
def log_error(message, error)
Gitlab::Geo::Logger.error({
class: self.class.name,
message: message,
error: error
}.merge(log_params))
end
def log_params
{
project_id: project.id,
project_path: project.full_path
}
end
end
end
......@@ -15,9 +15,9 @@ module Geo
begin
project.ensure_repository
project.repository.fetch_geo_mirror(ssh_url_to_repo)
fetch_geo_mirror(project.repository)
update_registry(finished_at: DateTime.now)
log_info("Finished repository sync",
update_delay_s: update_delay_in_seconds,
download_time_s: download_time_in_seconds)
......
......@@ -14,9 +14,9 @@ module Geo
begin
project.wiki.ensure_repository
project.wiki.repository.fetch_geo_mirror(ssh_url_to_wiki)
fetch_geo_mirror(project.wiki.repository)
update_registry(finished_at: DateTime.now)
log_info("Finished wiki sync",
update_delay_s: update_delay_in_seconds,
download_time_s: download_time_in_seconds)
......
......@@ -38,18 +38,21 @@ class SystemHooksService
end
when User
data.merge!(user_data(model))
if event == :rename
data[:old_username] = model.username_was
end
when ProjectMember
data.merge!(project_member_data(model))
when Group
owner = model.owner
data.merge!(group_data(model))
data.merge!(
name: model.name,
path: model.path,
group_id: model.id,
owner_name: owner.respond_to?(:name) ? owner.name : nil,
owner_email: owner.respond_to?(:email) ? owner.email : nil
)
if event == :rename
data.merge!(
old_path: model.path_was,
old_full_path: model.full_path_was
)
end
when GroupMember
data.merge!(group_member_data(model))
end
......@@ -101,6 +104,19 @@ class SystemHooksService
}
end
def group_data(model)
owner = model.owner
{
name: model.name,
path: model.path,
full_path: model.full_path,
group_id: model.id,
owner_name: owner.try(:name),
owner_email: owner.try(:email)
}
end
def group_member_data(model)
{
group_name: model.group.name,
......
......@@ -12,15 +12,31 @@
.col-sm-10
= form.text_field :url, class: 'form-control'
.form-group.js-hide-if-geo-primary{ class: ('hidden' unless geo_node.secondary?) }
= form.label :clone_protocol, s_('Geo|Repository cloning'), class: 'control-label'
.col-sm-10
.radio
= form.label :clone_protocol_http do
= form.radio_button :clone_protocol, :http, class: 'js-use-http'
.option-title
HTTP/HTTPS
.option-description= _('Clone repositories and wikis from the primary using HTTP/HTTPS.')
.radio
= form.label :clone_protocol_ssh do
= form.radio_button :clone_protocol, :ssh, class: 'js-use-ssh'
.option-title
SSH (deprecated)
.option-description= _('Authentication must be manually configured. Deprecated since GitLab 10.2.')
= form.fields_for :geo_node_key, geo_node.geo_node_key, include_id: !disable_key_edit do |fg|
.form-group.js-hide-if-geo-primary{ class: ('hidden' unless geo_node.secondary?) }
.form-group.js-ssh-key{ class: ('hidden' unless geo_node.secondary? && geo_node.clone_protocol == 'ssh') }
= fg.label :key, 'Public Key', class: 'control-label'
.col-sm-10
= fg.text_area :key, class: 'form-control thin_area', rows: 5, disabled: disable_key_edit
- unless disable_key_edit
%p.help-block
Paste the ssh public key used by the node you are adding. Read more about it
For SSH authentication, paste the public key used by the node you are adding. Read more about it
= link_to 'here', help_page_path('gitlab-geo/configuration.html', anchor: 'step-5-enabling-the-secondary-gitlab-node')
.form-group.js-hide-if-geo-primary{ class: ('hidden' unless geo_node.secondary?) }
......
- return unless push_rule.available?(:commit_committer_check)
- form = local_assigns.fetch(:form)
- push_rule = local_assigns.fetch(:push_rule)
.form-group
= form.check_box :commit_committer_check, class: "pull-left", disabled: !can_change_push_rule?(form.object, :commit_committer_check)
.prepend-left-20
= form.label :commit_committer_check, class: "label-light append-bottom-0" do
= s_("PushRule|Committer restriction")
%p.light.append-bottom-0
= commit_committer_check_description(push_rule)
= render 'shared/push_rules/commit_committer_check_setting', form: f, push_rule: f.object
= render 'shared/push_rules/reject_unsigned_commits_setting', form: f, push_rule: f.object
.form-group
......
......@@ -4,7 +4,7 @@
- push_rule = local_assigns.fetch(:push_rule)
.form-group
= form.check_box :reject_unsigned_commits, class: "pull-left", disabled: !can_change_reject_unsigned_commits?(push_rule)
= form.check_box :reject_unsigned_commits, class: "pull-left", disabled: !can_change_push_rule?(form.object, :reject_unsigned_commits)
.prepend-left-20
= form.label :reject_unsigned_commits, class: "label-light append-bottom-0" do
Reject unsigned commits
......
#
# Concern that helps with getting an exclusive lease for running a worker
#
# `#try_obtain_lease` takes a block which will be run if it was able to obtain the lease.
# Implement `#lease_timeout` to configure the timeout for the exclusive lease.
# Optionally override `#lease_key` to set the lease key, it defaults to the class name with underscores.
#
module ExclusiveLeaseGuard
extend ActiveSupport::Concern
# override in subclass
def lease_timeout
raise NotImplementedError
end
def lease_key
@lease_key ||= self.class.name.underscore
end
def log_error(message, extra_args = {})
logger.error(messages)
end
def try_obtain_lease
lease = exclusive_lease.try_obtain
......@@ -27,4 +43,8 @@ module ExclusiveLeaseGuard
def release_lease(uuid)
Gitlab::ExclusiveLease.cancel(lease_key, uuid)
end
def renew_lease!
exclusive_lease.renew
end
end
......@@ -2,6 +2,7 @@ module Geo
class BaseSchedulerWorker
include Sidekiq::Worker
include CronjobQueue
include ExclusiveLeaseGuard
DB_RETRIEVE_BATCH_SIZE = 1000
LEASE_TIMEOUT = 60.minutes
......@@ -72,10 +73,6 @@ module Geo
DB_RETRIEVE_BATCH_SIZE
end
def lease_key
@lease_key ||= self.class.name.underscore
end
def lease_timeout
LEASE_TIMEOUT
end
......@@ -139,33 +136,6 @@ module Geo
scheduled_jobs.map { |data| data[:job_id] }
end
def try_obtain_lease
lease = exclusive_lease.try_obtain
unless lease
log_error('Cannot obtain an exclusive lease. There must be another worker already in execution.')
return
end
begin
yield lease
ensure
release_lease(lease)
end
end
def exclusive_lease
@lease ||= Gitlab::ExclusiveLease.new(lease_key, timeout: lease_timeout)
end
def renew_lease!
exclusive_lease.renew
end
def release_lease(uuid)
Gitlab::ExclusiveLease.cancel(lease_key, uuid)
end
def current_node
Gitlab::Geo.current_node
end
......
module Geo
class PruneEventLogWorker
include Sidekiq::Worker
include CronjobQueue
include ExclusiveLeaseGuard
include ::Gitlab::Geo::LogHelpers
LEASE_TIMEOUT = 60.minutes
def lease_timeout
LEASE_TIMEOUT
end
def perform
return unless Gitlab::Geo.primary?
try_obtain_lease do
if Gitlab::Geo.secondary_nodes.empty?
log_info('No secondary nodes, delete all Geo Event Log entries')
Geo::EventLog.delete_all
return
end
cursor_last_event_ids = Gitlab::Geo.secondary_nodes.map do |node|
Geo::NodeStatusService.new.call(node).cursor_last_event_id
end
if cursor_last_event_ids.include?(nil)
log_info('Could not get status of all nodes, not deleting any entries from Geo Event Log', unhealthy_node_count: cursor_last_event_ids.count(nil))
return
end
log_info('Delete Geo Event Log entries up to id', geo_event_log_id: cursor_last_event_ids.min)
Geo::EventLog.where('id < ?', cursor_last_event_ids.min).delete_all
end
end
end
end
......@@ -3,6 +3,7 @@ module Geo
include Sidekiq::Worker
include GeoQueue
include Gitlab::ShellAdapter
include ExclusiveLeaseGuard
BATCH_SIZE = 250
LEASE_TIMEOUT = 60.minutes
......@@ -40,31 +41,8 @@ module Geo
end
end
def try_obtain_lease
lease = exclusive_lease.try_obtain
unless lease
log_error('Cannot obtain an exclusive lease. There must be another worker already in execution.')
return
end
begin
yield lease
ensure
release_lease(lease)
end
end
def lease_key
@lease_key ||= self.class.name.underscore
end
def exclusive_lease
@lease ||= Gitlab::ExclusiveLease.new(lease_key, timeout: LEASE_TIMEOUT)
end
def release_lease(uuid)
Gitlab::ExclusiveLease.cancel(lease_key, uuid)
def lease_timeout
LEASE_TIMEOUT
end
def log_info(message, params = {})
......
---
title: Allow Geo repository sync over HTTPS
merge_request: 3341
author:
type: added
---
title: Add worker to prune the Geo Event Log
merge_request: 3172
author:
type: added
---
title: Add new push rule to enforce that only the author of a commit can push to the repository
merge_request: 3086
author:
type: added
---
title: Add system hooks user_rename and group_rename
merge_request: 15123
author:
type: changed
......@@ -246,6 +246,11 @@ production: &base
geo_metrics_update_worker:
cron: "*/1 * * * *"
# GitLab Geo prune event log worker
# NOTE: This will only take effect if Geo is enabled (primary node only)
geo_prune_event_log_worker:
cron: "0 */6 * * *"
# GitLab Geo repository sync worker
# NOTE: This will only take effect if Geo is enabled (secondary nodes only)
geo_repository_sync_worker:
......
......@@ -452,6 +452,9 @@ Settings.cron_jobs['geo_repository_sync_worker']['job_class'] ||= 'Geo::Reposito
Settings.cron_jobs['geo_file_download_dispatch_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['geo_file_download_dispatch_worker']['cron'] ||= '*/1 * * * *'
Settings.cron_jobs['geo_file_download_dispatch_worker']['job_class'] ||= 'Geo::FileDownloadDispatchWorker'
Settings.cron_jobs['geo_prune_event_log_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['geo_prune_event_log_worker']['cron'] ||= '0 */6 * * *'
Settings.cron_jobs['geo_prune_event_log_worker']['job_class'] ||= 'Geo::PruneEventLogWorker'
Settings.cron_jobs['import_export_project_cleanup_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['import_export_project_cleanup_worker']['cron'] ||= '0 * * * *'
Settings.cron_jobs['import_export_project_cleanup_worker']['job_class'] = 'ImportExportProjectCleanupWorker'
......
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class AddCommitCommitterCheckToPushRules < ActiveRecord::Migration
DOWNTIME = false
def change
add_column :push_rules, :commit_committer_check, :boolean
end
end
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class AddGeoNodeCloneProtocol < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_column_with_default :geo_nodes, :clone_protocol, :string, allow_null: false, default: 'ssh'
change_column_default :geo_nodes, :clone_protocol, 'http'
end
def down
remove_column :geo_nodes, :clone_protocol
end
end
......@@ -802,6 +802,7 @@ ActiveRecord::Schema.define(version: 20171026082505) do
t.string "clone_url_prefix"
t.integer "files_max_capacity", default: 10, null: false
t.integer "repos_max_capacity", default: 25, null: false
t.string "clone_protocol", default: "http", null: false
end
add_index "geo_nodes", ["access_key"], name: "index_geo_nodes_on_access_key", using: :btree
......@@ -1751,6 +1752,7 @@ ActiveRecord::Schema.define(version: 20171026082505) do
t.boolean "prevent_secrets", default: false, null: false
t.string "branch_name_regex"
t.boolean "reject_unsigned_commits"
t.boolean "commit_committer_check"
end
add_index "push_rules", ["is_sample"], name: "index_push_rules_on_is_sample", where: "is_sample", using: :btree
......
......@@ -12,6 +12,8 @@ and there is significant chance of data loss. For the latest updates, check the
We recommend you use it with at least GitLab Enterprise Edition 10.0 for
basic Geo features, or latest version for a better experience.
- You should make sure that all nodes run the same GitLab version.
- GitLab Geo requires PostgreSQL 9.6 and Git 2.9 in addition to GitLab's usual
[minimum requirements](../install/requirements.md)
GitLab Geo allows you to replicate your GitLab instance to other geographical
locations as a read-only fully operational version.
......@@ -54,7 +56,7 @@ The following diagram illustrates the underlying architecture of GitLab Geo:
![GitLab Geo architecture](img/geo-architecture.png)
[Source diagram](https://docs.google.com/drawings/d/1L44flo2Mxng928yAcHduaCJyGtKNEjk2WQkxaCU_cT8/edit)
[Source diagram](https://docs.google.com/drawings/d/1Abw0P_H0Ew1-2Lj_xPDRWP87clGIke-1fil7_KQqrtE/edit)
In this diagram, there is one Geo primary node and one secondary. The
secondary clones repositories via git over SSH. Attachments, LFS objects, and
......
......@@ -46,7 +46,7 @@ first two steps of the [Setup instructions](README.md#setup-instructions):
1. You have already installed on the secondary server the same version of
GitLab Enterprise Edition that is present on the primary server.
1. You have set up the database replication.
1. Your secondary node is allowed to communicate via HTTP/HTTPS and SSH with
1. Your secondary node is allowed to communicate via HTTP/HTTPS with
your primary node (make sure your firewall is not blocking that).
1. Your nodes must have an NTP service running to synchronize the clocks.
You can use different timezones, but the hour relative to UTC can't be more
......@@ -75,31 +75,7 @@ logins opened on all nodes as we will be moving back and forth.
This command will use your defined `external_url` in `gitlab.rb`
### Step 2. Updating the `known_hosts` file of the secondary nodes
1. SSH into the **secondary** node and login as root:
```
sudo -i
```
1. The secondary nodes need to know the SSH fingerprint of the primary node that
will be used for the Git clone/fetch operations. In order to add it to the
`known_hosts` file, run the following command and type `yes` when asked:
```
sudo -u git -H ssh git@<primary-node-url>
```
Replace `<primary-node-url>` with the FQDN of the primary node.
1. Verify that the fingerprint was added by checking `known_hosts`:
```
cat /var/opt/gitlab/.ssh/known_hosts
```
### Step 3. Copying the database encryption key
### Step 2. Copying the database encryption key
GitLab stores a unique encryption key in disk that we use to safely store
sensitive data in the database. Any secondary node must have the
......@@ -137,7 +113,7 @@ sensitive data in the database. Any secondary node must have the
gitlab-ctl reconfigure
```
### Step 4. Enabling hashed storage (from GitLab 10.0)
### Step 3. Enabling hashed storage (from GitLab 10.0)
1. Visit the **primary** node's **Admin Area ➔ Settings**
(`/admin/application_settings`) in your browser
......@@ -149,25 +125,12 @@ Using hashed storage significantly improves Geo replication - project and group
renames no longer require synchronization between nodes - so we recommend it is
used for all GitLab Geo installations.
### Step 5. Enabling the secondary GitLab node
1. SSH into the **secondary** node and login as root:
```
sudo -i
```
1. Get the contents of `id_rsa.pub` key that was pre-generated by Omnibus GitLab
and copy them:
```
sudo -u git cat /var/opt/gitlab/.ssh/id_rsa.pub
```
### Step 4. Enabling the secondary GitLab node
1. Visit the **primary** node's **Admin Area ➔ Geo Nodes** (`/admin/geo_nodes`)
in your browser.
1. Add the secondary node by providing its full URL and the public SSH key
you created previously. **Do NOT** check the box 'This is a primary node'.
1. Add the secondary node by providing its full URL. **Do NOT** check the box
'This is a primary node'.
1. Added in GitLab 9.5: Choose which namespaces should be replicated by the secondary node. Leave blank to replicate all. Read more in [selective replication](#selective-replication).
1. Click the **Add node** button.
1. Restart GitLab on the secondary:
......@@ -175,11 +138,14 @@ used for all GitLab Geo installations.
```
gitlab-ctl restart
```
---
After the **Add Node** button is pressed, the primary node will start to notify
changes to the secondary. Make sure the secondary instance is running and
accessible.
After the **Add Node** button is pressed, the secondary will start automatically
replicating missing data from the primary in a process known as backfill.
Meanwhile, the primary node will start to notify changes to the secondary, which
will act on those notifications immediately. Make sure the secondary instance is
running and accessible.
The two most obvious issues that replication can have here are:
......@@ -187,23 +153,15 @@ The two most obvious issues that replication can have here are:
1. Instance to instance notification not working. In that case, it can be
something of the following:
- You are using a custom certificate or custom CA (see the
[Troubleshooting](#troubleshooting) section)
[Troubleshooting](configuration.md#troubleshooting) section)
- Instance is firewalled (check your firewall rules)
### Step 6. Replicating the repositories data
Getting a new secondary Geo node up and running, will also require the
repositories data to be synced.
With GitLab 9.0 the syncing process starts automatically from the
secondary node after the **Add Node** button is pressed.
Currently, this is what is synced:
* Git repositories
* Wikis
* LFS objects
* Issue, merge request, and comment attachments
* Issue, merge request, snippet and comment attachments
* User, group, and project avatars
You can monitor the status of the syncing process on a secondary node
......@@ -217,39 +175,6 @@ repository shards you must duplicate the same configuration on the secondary.
Disabling a secondary node stops the syncing process.
With GitLab 8.14 this process is started manually from the primary node.
You can start the syncing process by clicking the "Backfill all repositories"
button on `Admin > Geo Nodes` screen.
On previous versions, you can use `rsync` for that:
Make sure `rsync` is installed in both primary and secondary servers and root
SSH access with a password is enabled. Otherwise, you can set up an SSH key-based
connection between the servers.
1. SSH into the **secondary** node and login as root:
```
sudo -i
```
1. Assuming `1.2.3.4` is the IP of the primary node, run the following command
to start the sync:
```bash
# For Omnibus installations
rsync -guavrP root@1.2.3.4:/var/opt/gitlab/git-data/repositories/ /var/opt/gitlab/git-data/repositories/
gitlab-ctl reconfigure # to fix directory permissions
```
If this step is not followed, the secondary node will eventually clone and
fetch every missing repository as they are updated with new commits on the
primary node, so syncing the repositories beforehand will buy you some time.
While active repositories will be eventually replicated, if you don't rsync,
the files, any archived/inactive repositories will not get in the secondary node
as Geo doesn't run any routine task to look for missing repositories.
## Next steps
Your nodes should now be ready to use. You can login to the secondary node
......@@ -280,19 +205,43 @@ groups to be replicated.
## Adding another secondary Geo node
To add another Geo node in an already Geo configured infrastructure, just follow
[the steps starting from step 2](#step-2-updating-the-known_hosts-file-of-the-secondary-nodes).
[the steps starting from step 2](#step-2-copying-the-database-encryption-key)
Just omit the first step that sets up the primary node.
## Additional information for the SSH key pairs
## Replicating wikis and repositories over SSH
In GitLab 10.2, replicating repositories and wikis over SSH was deprecated.
Support for this option will be removed within a few releases, but if you need
to add a new secondary in the short term, you can follow these instructions:
1. SSH into the **secondary** node and login as root:
```bash
sudo -i
```
1. Add the primary's SSH key fingerprint to the `known_hosts` file.
```bash
sudo -u git -H ssh git@<primary-node-url>
```
Replace `<primary-node-url>` with the FQDN of the primary node. You should
manually check the displayed fingerprint against a trusted record of the
expected value before accepting it!
When adding a new **secondary** Geo node, you must provide an SSH public key for
the system user that your GitLab instance runs as (unless changed, should be the
user `git`). This user will act as a "normal user" who fetches from the primary
Geo node.
1. Generate a *passphraseless* SSH keypair for the `git` user, and capture the
public component:
```bash
test -e ~git/.ssh/id_rsa || sudo -u git -H ssh-keygen -q -t rsa -b 4096 -f ~git/.ssh/id_rsa
cat ~git/.ssh/id_rsa.pub
```
Omnibus automatically generates `~git/.ssh/id_rsa` and `~git/.ssh/id_rsa.pub`
files on secondary Geo nodes. Primaries do not need these files, and you should
not create them manually.
Follow the steps above to set up the new Geo node. When you reach
[Step 4: Enabling the secondary GitLab node](#step-4-enabling-the-secondary-gitlab-node)
select "SSH (deprecated)" instead of "HTTP/HTTPS", and populate the "Public Key"
with the output of the previous command (beginning `ssh-rsa AAAA...`).
### Upgrading Geo
......
......@@ -46,7 +46,7 @@ first two steps of the [Setup instructions](README.md#setup-instructions):
1. You have already installed on the secondary server the same version of
GitLab Enterprise Edition that is present on the primary server.
1. You have set up the database replication.
1. Your secondary node is allowed to communicate via HTTP/HTTPS and SSH with
1. Your secondary node is allowed to communicate via HTTP/HTTPS with
your primary node (make sure your firewall is not blocking that).
1. Your nodes must have an NTP service running to synchronize the clocks.
You can use different timezones, but the hour relative to UTC can't be more
......@@ -73,31 +73,7 @@ logins opened on all nodes as we will be moving back and forth.
bundle exec rake geo:set_primary_node
```
### Step 2. Updating the `known_hosts` file of the secondary nodes
1. SSH into the **secondary** node and login as root:
```
sudo -i
```
1. The secondary nodes need to know the SSH fingerprint of the primary node that
will be used for the Git clone/fetch operations. In order to add it to the
`known_hosts` file, run the following command and type `yes` when asked:
```
sudo -u git -H ssh git@<primary-node-url>
```
Replace `<primary-node-url>` with the FQDN of the primary node.
1. Verify that the fingerprint was added by checking `known_hosts`:
```
cat /home/git/.ssh/known_hosts
```
### Step 3. Copying the database encryption key
### Step 2. Copying the database encryption key
GitLab stores a unique encryption key in disk that we use to safely store
sensitive data in the database. Any secondary node must have the
......@@ -130,7 +106,7 @@ sensitive data in the database. Any secondary node must have the
1. Save and close the file.
### Step 4. Enabling hashed storage (from GitLab 10.0)
### Step 3. Enabling hashed storage (from GitLab 10.0)
1. Visit the **primary** node's **Admin Area ➔ Settings**
(`/admin/application_settings`) in your browser
......@@ -143,41 +119,27 @@ renames no longer require synchronization between nodes - so we recommend it is
used for all GitLab Geo installations.
### Step 5. Enabling the secondary GitLab node
1. SSH into the **secondary** node and login as root:
```
sudo -i
```
1. Create a new SSH key pair for the secondary node. Choose the default location
and leave the password blank by hitting 'Enter' three times:
```bash
sudo -u git -H ssh-keygen -b 4096 -C 'Secondary GitLab Geo node'
```
Read more in [additional info for SSH key pairs](#additional-information-for-the-ssh-key-pairs).
1. Get the contents of `id_rsa.pub` the was just created:
```
sudo -u git cat /home/git/.ssh/id_rsa.pub
```
### Step 4. Enabling the secondary GitLab node
1. Visit the **primary** node's **Admin Area ➔ Geo Nodes** (`/admin/geo_nodes`)
in your browser.
1. Add the secondary node by providing its full URL and the public SSH key
you created previously. **Do NOT** check the box 'This is a primary node'.
1. Add the secondary node by providing its full URL. **Do NOT** check the box
'This is a primary node'.
1. Added in GitLab 9.5: Choose which namespaces should be replicated by the secondary node. Leave blank to replicate all. Read more in [selective replication](#selective-replication).
1. Click the **Add node** button.
1. Restart GitLab on the secondary:
```
gitlab-ctl restart
```
---
After the **Add Node** button is pressed, the primary node will start to notify
changes to the secondary. Make sure the secondary instance is running and
accessible.
After the **Add Node** button is pressed, the secondary will start automatically
replicating missing data from the primary in a process known as backfill.
Meanwhile, the primary node will start to notify changes to the secondary, which
will act on those notifications immediately. Make sure the secondary instance is
running and accessible.
The two most obvious issues that replication can have here are:
......@@ -188,20 +150,12 @@ The two most obvious issues that replication can have here are:
[Troubleshooting](configuration.md#troubleshooting) section)
- Instance is firewalled (check your firewall rules)
### Step 6. Replicating the repositories data
Getting a new secondary Geo node up and running, will also require the
repositories data to be synced.
With GitLab 9.0 the syncing process starts automatically from the
secondary node after the **Add Node** button is pressed.
Currently, this is what is synced:
* Git repositories
* Wikis
* LFS objects
* Issue, merge request, and comment attachments
* Issue, merge request, snippet and comment attachments
* User, group, and project avatars
You can monitor the status of the syncing process on a secondary node
......@@ -215,39 +169,6 @@ repository shards you must duplicate the same configuration on the secondary.
Disabling a secondary node stops the syncing process.
With GitLab 8.14 this process is started manually from the primary node.
You can start the syncing process by clicking the "Backfill all repositories"
button on `Admin > Geo Nodes` screen.
On previous versions, you can use `rsync` for that:
Make sure `rsync` is installed in both primary and secondary servers and root
SSH access with a password is enabled. Otherwise, you can set up an SSH key-based
connection between the servers.
1. SSH into the **secondary** node and login as root:
```
sudo -i
```
1. Assuming `1.2.3.4` is the IP of the primary node, run the following command
to start the sync:
```bash
# Installations from source
rsync -guavrP root@1.2.3.4:/home/git/repositories/ /home/git/repositories/
chmod ug+rwX,o-rwx /home/git/repositories
```
If this step is not followed, the secondary node will eventually clone and
fetch every missing repository as they are updated with new commits on the
primary node, so syncing the repositories beforehand will buy you some time.
While active repositories will be eventually replicated, if you don't rsync,
the files, any archived/inactive repositories will not get in the secondary node
as Geo doesn't run any routine task to look for missing repositories.
## Next steps
Your nodes should now be ready to use. You can login to the secondary node
......@@ -267,12 +188,12 @@ Read [Selective replication](configuration.md#selective-replication).
## Adding another secondary Geo node
To add another Geo node in an already Geo configured infrastructure, just follow
[the steps starting from step 2](#step-2-updating-the-known_hosts-file-of-the-secondary-nodes).
[the steps starting from step 2](#step-2-copying-the-database-encryption-key).
Just omit the first step that sets up the primary node.
## Additional information for the SSH key pairs
## Replicating wikis and repositories over SSH
Read [Additional information for the SSH key pairs](configuration.md#additional-information-for-the-ssh-key-pairs).
Read [Replicating wikis and repositories over SSH](configuration.md#replicating-wikis-and-repositories-over-ssh).
## Troubleshooting
......
doc/gitlab-geo/img/geo-architecture.png

59.3 KB | W: | H:

doc/gitlab-geo/img/geo-architecture.png

59.4 KB | W: | H:

doc/gitlab-geo/img/geo-architecture.png
doc/gitlab-geo/img/geo-architecture.png
doc/gitlab-geo/img/geo-architecture.png
doc/gitlab-geo/img/geo-architecture.png
  • 2-up
  • Swipe
  • Onion skin
......@@ -14,6 +14,31 @@ all you need to do is update GitLab itself:
the tracking database is enabled.
1. [Test](#check-status-after-updating) primary and secondary nodes, and check version in each.
## Upgrading to GitLab 10.2
Support for replicating repositories and wikis over HTTP/HTTPS has been added.
Replicating over SSH has been deprecated, and support for this option will be
removed in a future release.
To switch to HTTP/HTTPS replication, log into the primary node as an admin and visit
**Admin Area ➔ Geo Nodes** (`/admin/geo_nodes`). For each secondary listed,
press the "Edit" button, change the "Repository cloning" setting from
"SSH (deprecated)" to "HTTP/HTTPS", and press "Save changes". This should take
effect immediately.
Any new secondaries should be created using HTTP/HTTPS replication - this is the
default setting.
After you've verified that HTTP/HTTPS replication is working, you should remove
the now-unused SSH keys from your secondaries, as they may cause problems if the
secondary if ever promoted to a primary:
1. **[secondary]** Login to **all** your secondary nodes and run:
```ruby
sudo -u git -H rm ~git/.ssh/id_rsa ~git/.ssh/id_rsa.pub
```
## Upgrading to GitLab 10.1
[Hashed storage](../administration/repository_storage_types.md) was introduced
......
......@@ -63,6 +63,7 @@ The following options are available.
| --------- | :------------: | ----------- |
| Removal of tags with `git push` | 7.10 | Forbid users to remove git tags with `git push`. Tags will still be able to be deleted through the web UI. |
| Check whether author is a GitLab user | 7.10 | Restrict commits by author (email) to existing GitLab users. |
| Check whether committer is the current authenticated user | 10.2 | GitLab will reject any commit that was not committed by the current authenticated user |
| Check whether commit is signed through GPG | 10.1 | Reject commit when it is not signed through GPG. Read [signing commits with GPG][signing-commits]. |
| Prevent committing secrets to Git | 8.12 | GitLab will reject any files that are likely to contain secrets. Read [what files are forbidden](#prevent-pushing-secrets-to-the-repository). |
| Restrict by commit message | 7.10 | Only commit messages that match this Ruby regular expression are allowed to be pushed. Leave empty to allow any commit message. |
......
# System hooks
Your GitLab instance can perform HTTP POST requests on the following events: `project_create`, `project_destroy`, `project_rename`, `project_transfer`, `project_update`, `user_add_to_team`, `user_remove_from_team`, `user_create`, `user_destroy`, `key_create`, `key_destroy`, `group_create`, `group_destroy`, `user_add_to_group` and `user_remove_from_group`.
Your GitLab instance can perform HTTP POST requests on the following events:
- `project_create`
- `project_destroy`
- `project_rename`
- `project_transfer`
- `project_update`
- `user_add_to_team`
- `user_remove_from_team`
- `user_create`
- `user_destroy`
- `user_rename`
- `key_create`
- `key_destroy`
- `group_create`
- `group_destroy`
- `group_rename`
- `user_add_to_group`
- `user_remove_from_group`
The triggers for most of these are self-explanatory, but `project_update` and `project_rename` deserve some clarification: `project_update` is fired any time an attribute of a project is changed (name, description, tags, etc.) *unless* the `path` attribute is also changed. In that case, a `project_rename` is triggered instead (so that, for instance, if all you care about is the repo URL, you can just listen for `project_rename`).
......@@ -72,6 +90,9 @@ X-Gitlab-Event: System Hook
}
```
Note that `project_rename` is not triggered if the namespace changes.
Please refer to `group_rename` and `user_rename` for that case.
**Project transferred:**
```json
......@@ -175,6 +196,21 @@ X-Gitlab-Event: System Hook
}
```
**User renamed:**
```json
{
"event_name": "user_rename",
"created_at": "2017-11-01T11:21:04Z",
"updated_at": "2017-11-01T14:04:47Z",
"name": "new-name",
"email": "best-email@example.tld",
"user_id": 58,
"username": "new-exciting-name",
"old_username": "old-boring-name"
}
```
**Key added**
```json
......@@ -209,13 +245,15 @@ X-Gitlab-Event: System Hook
"updated_at": "2012-07-21T07:38:22Z",
"event_name": "group_create",
"name": "StoreCloud",
"owner_email": "johnsmith@gmail.com",
"owner_name": "John Smith",
"owner_email": null,
"owner_name": null,
"path": "storecloud",
"group_id": 78
}
```
`owner_name` and `owner_email` are always `null`. Please see https://gitlab.com/gitlab-org/gitlab-ce/issues/39675.
**Group removed:**
```json
......@@ -224,13 +262,35 @@ X-Gitlab-Event: System Hook
"updated_at": "2012-07-21T07:38:22Z",
"event_name": "group_destroy",
"name": "StoreCloud",
"owner_email": "johnsmith@gmail.com",
"owner_name": "John Smith",
"owner_email": null,
"owner_name": null,
"path": "storecloud",
"group_id": 78
}
```
`owner_name` and `owner_email` are always `null`. Please see https://gitlab.com/gitlab-org/gitlab-ce/issues/39675.
**Group renamed:**
```json
{
"event_name": "group_rename",
"created_at": "2017-10-30T15:09:00Z",
"updated_at": "2017-11-01T10:23:52Z",
"name": "Better Name",
"path": "better-name",
"full_path": "parent-group/better-name",
"group_id": 64,
"owner_name": null,
"owner_email": null,
"old_path": "old-name",
"old_full_path": "parent-group/old-name"
}
```
`owner_name` and `owner_email` are always `null`. Please see https://gitlab.com/gitlab-org/gitlab-ce/issues/39675.
**New Group Member:**
```json
......
......@@ -18,6 +18,9 @@ class Admin::GeoNodesController < Admin::ApplicationController
redirect_to admin_geo_nodes_path, notice: 'Node was successfully created.'
else
@nodes = GeoNode.all
@node = GeoNode.new(geo_node_params)
flash.now[:alert] = 'Failed to create new node'
render :index
end
end
......@@ -83,6 +86,7 @@ class Admin::GeoNodesController < Admin::ApplicationController
:namespace_ids,
:repos_max_capacity,
:files_max_capacity,
:clone_protocol,
geo_node_key_attributes: [:key]
)
end
......
module EE
module Projects
module GitHttpController
def render_ok
raise NotImplementedError.new unless defined?(super)
set_workhorse_internal_api_content_type
render json: ::Gitlab::Workhorse.git_http_ok(repository, wiki?, user, action_name, show_all_refs: geo_request?)
end
private
def geo_request?
::Gitlab::Geo::JwtRequestDecoder.geo_auth_attempt?(request.headers['Authorization'])
end
def geo?
authentication_result.geo?(project)
end
def access_actor
raise NotImplementedError.new unless defined?(super)
return :geo if geo?
super
end
def authenticate_user
raise NotImplementedError.new unless defined?(super)
return super unless geo_request?
payload = ::Gitlab::Geo::JwtRequestDecoder.new(request.headers['Authorization']).decode
if payload
@authentication_result = ::Gitlab::Auth::Result.new(nil, project, :geo, [:download_code])
return # grant access
end
render_bad_geo_auth('Bad token')
rescue ::Gitlab::Geo::InvalidDecryptionKeyError
render_bad_geo_auth("Invalid decryption key")
end
def render_bad_geo_auth(message)
render plain: "Geo JWT authentication failed: #{message}", status: 401
end
end
end
end
module EE
module ProjectsHelper
def can_change_reject_unsigned_commits?(push_rule)
def can_change_push_rule?(push_rule, rule)
return true if push_rule.global?
can?(current_user, :change_reject_unsigned_commits, @project)
can?(current_user, :"change_#{rule}", @project)
end
end
end
......@@ -6,6 +6,15 @@ module EE
module Repository
extend ActiveSupport::Concern
# Transiently sets a configuration variable
def with_config(values = {})
values.each { |k, v| rugged.config[k] = v }
yield
ensure
values.keys.each { |key| rugged.config.delete(key) }
end
# Runs code after a repository has been synced.
def after_sync
expire_all_method_caches
......
......@@ -20,6 +20,11 @@ module EE
!PushRule.global&.reject_unsigned_commits
end
with_scope :global
condition(:commit_committer_check_disabled_globally) do
!PushRule.global&.commit_committer_check
end
with_scope :global
condition(:remote_mirror_available) do
::Gitlab::CurrentSettings.current_application_settings.remote_mirror_available
......@@ -84,6 +89,8 @@ module EE
rule { ~can?(:push_code) }.prevent :push_code_to_protected_branches
rule { admin | (reject_unsigned_commits_disabled_globally & can?(:master_access)) }.enable :change_reject_unsigned_commits
rule { admin | (commit_committer_check_disabled_globally & can?(:master_access)) }.enable :change_commit_committer_check
end
end
end
module EE
module Gitlab
module Auth
module Result
def success?
raise NotImplementedError.new unless defined?(super)
type == :geo || super
end
def geo?(for_project)
type == :geo &&
project &&
project == for_project
end
end
end
end
end
module EE
module Gitlab
module GitAccess
def check(cmd, changes)
raise NotImplementedError.new unless defined?(super)
check_geo_license!
super
end
def can_read_project?
raise NotImplementedError.new unless defined?(super)
return geo_node_key.active? if geo_node_key?
return true if actor == :geo
super
end
protected
def user
raise NotImplementedError.new unless defined?(super)
return nil if geo?
super
end
private
def check_download_access!
raise NotImplementedError.new unless defined?(super)
return if geo?
super
end
def check_active_user!
raise NotImplementedError.new unless defined?(super)
return if geo?
super
end
def check_geo_license!
if ::Gitlab::Geo.secondary? && !::Gitlab::Geo.license_allows?
raise ::Gitlab::GitAccess::UnauthorizedError, 'Your current license does not have GitLab Geo add-on enabled.'
end
end
def geo_node_key
actor if geo_node_key?
end
def geo_node_key?
actor.is_a?(::GeoNodeKey)
end
def geo?
geo_node_key? || actor == :geo
end
end
end
end
module Gitlab
module Auth
Result = Struct.new(:actor, :project, :type, :authentication_abilities) do
prepend ::EE::Gitlab::Auth::Result
def ci?(for_project)
type == :ci &&
project &&
......
......@@ -15,7 +15,9 @@ module Gitlab
update_protected_tag: 'Protected tags cannot be updated.',
delete_protected_tag: 'Protected tags cannot be deleted.',
create_protected_tag: 'You are not allowed to create this tag as it is protected.',
push_rule_branch_name: "Branch name does not follow the pattern '%{branch_name_regex}'"
push_rule_branch_name: "Branch name does not follow the pattern '%{branch_name_regex}'",
push_rule_committer_not_verified: "Comitter email '%{commiter_email}' is not verified.",
push_rule_committer_not_allowed: "You cannot push commits for '%{committer_email}'. You can only push commits that were committed with one of your own verified emails."
}.freeze
# protocol is currently used only in EE
......@@ -213,6 +215,9 @@ module Gitlab
return "Author's email '#{commit.author_email}' does not follow the pattern '#{push_rule.author_email_regex}'"
end
committer_error_message = committer_check(commit, push_rule)
return committer_error_message if committer_error_message
if !updated_from_web? && !push_rule.commit_signature_allowed?(commit)
return "Commit must be signed with a GPG key"
end
......@@ -233,6 +238,18 @@ module Gitlab
nil
end
def committer_check(commit, push_rule)
unless push_rule.committer_allowed?(commit.committer_email, user_access.user)
committer_is_current_user = commit.committer == user_access.user
if committer_is_current_user && !commit.committer.verified_email?(commit.committer_email)
ERROR_MESSAGES[:push_rule_committer_not_verified] % { committer_email: commit.committer_email }
else
ERROR_MESSAGES[:push_rule_committer_not_allowed] % { committer_email: commit.committer_email }
end
end
end
def check_commit_diff(commit, push_rule)
validations = validations_for_commit(commit, push_rule)
......
......@@ -12,10 +12,14 @@ module Gitlab
# Raises GeoNodeNotFoundError if current node is not a Geo node
def headers
{
'Authorization' => geo_auth_token(request_data)
'Authorization' => authorization
}
end
def authorization
geo_auth_token(request_data)
end
private
def geo_auth_token(message)
......
......@@ -5,6 +5,11 @@ module Gitlab
class JwtRequestDecoder
IAT_LEEWAY = 60.seconds.to_i
def self.geo_auth_attempt?(header)
token_type, _ = header&.split(' ', 2)
token_type == ::Gitlab::Geo::BaseRequest::GITLAB_GEO_AUTH_TOKEN_TYPE
end
attr_reader :auth_header
def initialize(auth_header)
......
......@@ -7,9 +7,10 @@ module Gitlab
Gitlab::Geo::Logger.info(data)
end
def log_error(message, error)
def log_error(message, error = nil, details = {})
data = base_log_data(message)
data[:error] = error.to_s
data[:error] = error.to_s if error
data.merge!(details) if details
Gitlab::Geo::Logger.error(data)
end
......
......@@ -2,8 +2,10 @@
# class return an instance of `GitlabAccessStatus`
module Gitlab
class GitAccess
prepend ::EE::Gitlab::GitAccess
include ActionView::Helpers::SanitizeHelper
include PathLocksHelper
UnauthorizedError = Class.new(StandardError)
NotFoundError = Class.new(StandardError)
ProjectMovedError = Class.new(NotFoundError)
......@@ -47,8 +49,6 @@ module Gitlab
check_command_existence!(cmd)
check_repository_existence!
check_geo_license!
case cmd
when *DOWNLOAD_COMMANDS
check_download_access!
......@@ -92,7 +92,7 @@ module Gitlab
end
def check_active_user!
return if deploy_key? || geo_node_key?
return if deploy_key?
if user && !user_access.allowed?
raise UnauthorizedError, ERROR_MESSAGES[:account_blocked]
......@@ -146,12 +146,6 @@ module Gitlab
end
end
def check_geo_license!
if Gitlab::Geo.secondary? && !Gitlab::Geo.license_allows?
raise UnauthorizedError, 'Your current license does not have GitLab Geo add-on enabled.'
end
end
def check_repository_existence!
unless project.repository.exists?
raise UnauthorizedError, ERROR_MESSAGES[:no_repo]
......@@ -159,7 +153,7 @@ module Gitlab
end
def check_download_access!
return if deploy_key? || geo_node_key?
return if deploy_key?
passed = user_can_download_code? ||
build_can_download_code? ||
......@@ -253,14 +247,6 @@ module Gitlab
actor.is_a?(DeployKey)
end
def geo_node_key
actor if geo_node_key?
end
def geo_node_key?
actor.is_a?(GeoNodeKey)
end
def ci?
actor == :ci
end
......@@ -268,8 +254,6 @@ module Gitlab
def can_read_project?
if deploy_key?
deploy_key.has_access_to?(project)
elsif geo_node_key?
geo_node_key.active?
elsif user
user.can?(:read_project, project)
elsif ci?
......@@ -306,8 +290,6 @@ module Gitlab
case actor
when User
actor
when GeoNodeKey
nil
when Key
actor.user unless actor.is_a?(DeployKey)
when :ci
......
......@@ -11,10 +11,10 @@ module SystemCheck
].freeze
set_name 'Git user has default SSH configuration?'
set_skip_reason 'skipped (GitLab read-only, or git user is not present / configured)'
set_skip_reason 'skipped (Geo uses SSH key, or git user is not present / configured)'
def skip?
Gitlab::Database.read_only? || !home_dir || !File.directory?(home_dir)
::Gitlab::Geo.current_node&.uses_ssh_key? || !home_dir || !File.directory?(home_dir)
end
def check?
......
module SystemCheck
module Geo
class GitVersionCheck < ::SystemCheck::App::GitVersionCheck
set_name -> { "Git version >= #{self.required_version} ?" }
set_check_pass -> { "yes (#{self.current_version})" }
def self.required_version
@required_version ||= Gitlab::VersionInfo.new(2, 9, 5)
end
end
end
end
......@@ -458,6 +458,7 @@ namespace :gitlab do
checks = [
SystemCheck::Geo::LicenseCheck,
SystemCheck::Geo::EnabledCheck,
SystemCheck::Geo::GitVersionCheck,
SystemCheck::Geo::GeoDatabaseConfiguredCheck,
SystemCheck::Geo::DatabaseReplicationCheck,
SystemCheck::Geo::HttpConnectionCheck,
......
......@@ -8,8 +8,8 @@ msgid ""
msgstr ""
"Project-Id-Version: gitlab 1.0.0\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2017-10-22 16:40+0300\n"
"PO-Revision-Date: 2017-10-22 16:40+0300\n"
"POT-Creation-Date: 2017-11-02 14:42+0100\n"
"PO-Revision-Date: 2017-11-02 14:42+0100\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
"Language: \n"
......@@ -36,6 +36,11 @@ msgstr[1] ""
msgid "%{commit_author_link} committed %{commit_timeago}"
msgstr ""
msgid "%{count} participant"
msgid_plural "%{count} participants"
msgstr[0] ""
msgstr[1] ""
msgid "%{number_commits_behind} commits behind %{default_branch}, %{number_commits_ahead} commits ahead"
msgstr ""
......@@ -56,6 +61,12 @@ msgstr[1] ""
msgid "(checkout the %{link} for information on how to install it)."
msgstr ""
msgid "+ %{moreCount} more"
msgstr ""
msgid "- show less"
msgstr ""
msgid "1 pipeline"
msgid_plural "%d pipelines"
msgstr[0] ""
......@@ -480,9 +491,6 @@ msgstr ""
msgid "ClusterIntegration|A %{link_to_container_project} must have been created under this account"
msgstr ""
msgid "ClusterIntegration|Advanced settings"
msgstr ""
msgid "ClusterIntegration|Cluster details"
msgstr ""
......@@ -567,9 +575,6 @@ msgstr ""
msgid "ClusterIntegration|See and edit the details for your cluster"
msgstr ""
msgid "ClusterIntegration|See and edit the details for your cluster"
msgstr ""
msgid "ClusterIntegration|See machine types"
msgstr ""
......@@ -620,6 +625,11 @@ msgid_plural "Commits"
msgstr[0] ""
msgstr[1] ""
msgid "Commit %d file"
msgid_plural "Commit %d files"
msgstr[0] ""
msgstr[1] ""
msgid "Commit Message"
msgstr ""
......@@ -728,12 +738,21 @@ msgstr ""
msgid "Create empty bare repository"
msgstr ""
msgid "Create file"
msgstr ""
msgid "Create merge request"
msgstr ""
msgid "Create new branch"
msgstr ""
msgid "Create new directory"
msgstr ""
msgid "Create new file"
msgstr ""
msgid "Create new..."
msgstr ""
......@@ -904,6 +923,9 @@ msgstr ""
msgid "Failed to remove the pipeline schedule"
msgstr ""
msgid "File name"
msgstr ""
msgid "Files"
msgstr ""
......@@ -1343,6 +1365,12 @@ msgstr ""
msgid "Notifications"
msgstr ""
msgid "Number of access attempts"
msgstr ""
msgid "Number of failures before backing off"
msgstr ""
msgid "OfSearchInADropdown|Filter"
msgstr ""
......@@ -1598,6 +1626,9 @@ msgstr ""
msgid "ProjectSettings|This setting will be applied to all projects unless overridden by an admin."
msgstr ""
msgid "ProjectSettings|Users can only push commits to this repository that were committed with one of their own verified emails."
msgstr ""
msgid "Projects"
msgstr ""
......@@ -1634,6 +1665,9 @@ msgstr ""
msgid "Push events"
msgstr ""
msgid "PushRule|Committer restriction"
msgstr ""
msgid "Read more"
msgstr ""
......@@ -1906,6 +1940,9 @@ msgstr ""
msgid "Subgroups"
msgstr ""
msgid "Subscribe"
msgstr ""
msgid "Switch branch/tag"
msgstr ""
......@@ -1932,6 +1969,9 @@ msgstr ""
msgid "The Advanced Global Search in GitLab is a powerful search service that saves you time. Instead of creating duplicate code and wasting time, you can now search for code within other teams that can help your own project."
msgstr ""
msgid "The circuitbreaker backoff threshold should be lower than the failure count threshold"
msgstr ""
msgid "The coding stage shows the time from the first commit to creating the merge request. The data will automatically be added here once you create your first merge request."
msgstr ""
......@@ -1944,6 +1984,12 @@ msgstr ""
msgid "The issue stage shows the time it takes from creating an issue to assigning the issue to a milestone, or add the issue to a list on your Issue Board. Begin creating issues to see data for this stage."
msgstr ""
msgid "The number of attempts GitLab will make to access a storage."
msgstr ""
msgid "The number of failures after which GitLab will start temporarily disabling access to a storage shard on a host"
msgstr ""
msgid "The number of failures of after which GitLab will completely prevent access to the storage. The number of failures can be reset in the admin interface: %{link_to_health_page} or using the %{api_documentation_link}."
msgstr ""
......@@ -2179,6 +2225,9 @@ msgstr ""
msgid "Unstar"
msgstr ""
msgid "Unsubscribe"
msgstr ""
msgid "Upgrade your plan to activate Advanced Global Search."
msgstr ""
......
......@@ -114,8 +114,15 @@ describe Admin::GeoNodesController, :postgresql do
end
describe '#update' do
let(:geo_node_attributes) { { url: 'http://example.com', geo_node_key_attributes: attributes_for(:key) } }
let(:geo_node) { create(:geo_node) }
let(:geo_node_attributes) do
{
url: 'http://example.com',
clone_protocol: 'ssh',
geo_node_key_attributes: attributes_for(:key)
}
end
let(:geo_node) { create(:geo_node, :ssh) }
let!(:original_fingerprint) { geo_node.geo_node_key.fingerprint }
def go
......@@ -144,6 +151,19 @@ describe Admin::GeoNodesController, :postgresql do
expect(geo_node.geo_node_key.fingerprint).to eq(original_fingerprint)
end
context 'changing clone protocol' do
let(:geo_node_attributes) { { clone_protocol: 'http' } }
it 'changes the protocol without removing the key' do
go
geo_node.reload
expect(geo_node.clone_protocol).to eq('http')
expect(geo_node.geo_node_key.fingerprint).to eq(original_fingerprint)
end
end
it 'delegates the update of the Geo node to Geo::NodeUpdateService' do
expect_any_instance_of(Geo::NodeUpdateService).to receive(:execute).once
......@@ -259,25 +279,7 @@ describe Admin::GeoNodesController, :postgresql do
end
context 'with add-on license' do
let(:geo_node_status) do
GeoNodeStatus.new(
id: 1,
health: nil,
attachments_count: 329,
attachments_failed_count: 13,
attachments_synced_count: 141,
lfs_objects_count: 256,
lfs_objects_failed_count: 12,
lfs_objects_synced_count: 123,
repositories_count: 10,
repositories_synced_count: 5,
repositories_failed_count: 0,
last_event_id: 2,
last_event_timestamp: Time.now.to_i,
cursor_last_event_id: 1,
cursor_last_event_timestamp: Time.now.to_i
)
end
let(:geo_node_status) { build(:geo_node_status, :healthy) }
before do
allow(Gitlab::Geo).to receive(:license_allows?).and_return(true)
......
......@@ -34,52 +34,54 @@ describe Projects::PushRulesController do
end
end
context 'Updating reject unsigned commit rule' do
context 'as an admin' do
let(:user) { create(:admin) }
PushRule::SETTINGS_WITH_GLOBAL_DEFAULT.each do |rule_attr|
context "Updating #{rule_attr} rule" do
context 'as an admin' do
let(:user) { create(:admin) }
it 'updates the setting' do
patch :update, namespace_id: project.namespace, project_id: project, id: 1, push_rule: { reject_unsigned_commits: true }
expect(project.push_rule(true).reject_unsigned_commits).to be_truthy
end
end
context 'as a master user' do
before do
project.add_master(user)
end
context 'when global setting is disabled' do
it 'updates the setting' do
patch :update, namespace_id: project.namespace, project_id: project, id: 1, push_rule: { reject_unsigned_commits: true }
patch :update, namespace_id: project.namespace, project_id: project, id: 1, push_rule: { rule_attr => true }
expect(project.push_rule(true).reject_unsigned_commits).to be_truthy
expect(project.push_rule(true).public_send(rule_attr)).to be_truthy
end
end
context 'when global setting is enabled' do
context 'as a master user' do
before do
create(:push_rule_sample, reject_unsigned_commits: true)
project.add_master(user)
end
it 'does not update the setting' do
patch :update, namespace_id: project.namespace, project_id: project, id: 1, push_rule: { reject_unsigned_commits: false }
context 'when global setting is disabled' do
it 'updates the setting' do
patch :update, namespace_id: project.namespace, project_id: project, id: 1, push_rule: { rule_attr => true }
expect(project.push_rule(true).reject_unsigned_commits).to be_truthy
expect(project.push_rule(true).public_send(rule_attr)).to be_truthy
end
end
end
end
context 'as a developer user' do
before do
project.add_developer(user)
context 'when global setting is enabled' do
before do
create(:push_rule_sample, rule_attr => true)
end
it 'does not update the setting' do
patch :update, namespace_id: project.namespace, project_id: project, id: 1, push_rule: { rule_attr => false }
expect(project.push_rule(true).public_send(rule_attr)).to be_truthy
end
end
end
it 'does not update the setting' do
patch :update, namespace_id: project.namespace, project_id: project, id: 1, push_rule: { reject_unsigned_commits: true }
context 'as a developer user' do
before do
project.add_developer(user)
end
it 'does not update the setting' do
patch :update, namespace_id: project.namespace, project_id: project, id: 1, push_rule: { rule_attr => true }
expect(project.push_rule(true).reject_unsigned_commits).to be_falsy
expect(project.push_rule(true).public_send(rule_attr)).to be_falsy
end
end
end
end
......
......@@ -67,7 +67,8 @@ describe Namespace do
describe '#move_dir' do
context 'when running on a primary node' do
let!(:geo_node) { create(:geo_node, :primary) }
set(:primary) { create(:geo_node, :primary) }
set(:secondary) { create(:geo_node) }
let(:gitlab_shell) { Gitlab::Shell.new }
it 'logs the Geo::RepositoryRenamedEvent for each project inside namespace' do
......@@ -84,7 +85,6 @@ describe Namespace do
allow(parent).to receive(:full_path).and_return(new_path)
allow(gitlab_shell).to receive(:mv_namespace)
.ordered
.with(project_1.repository_storage_path, full_path_was, new_path)
.and_return(true)
......
......@@ -759,7 +759,8 @@ describe Project do
describe '#rename_repo' do
context 'when running on a primary node' do
let!(:geo_node) { create(:geo_node, :primary) }
set(:primary) { create(:geo_node, :primary) }
set(:secondary) { create(:geo_node) }
let(:project) { create(:project, :repository) }
let(:gitlab_shell) { Gitlab::Shell.new }
......
......@@ -6,7 +6,7 @@ describe API::Internal do
describe "POST /internal/allowed", :clean_gitlab_redis_shared_state do
context 'Geo Node' do
let(:geo_node) { create(:geo_node) }
let(:geo_node) { create(:geo_node, :ssh) }
it 'recognizes the Geo Node' do
post(
......
require 'spec_helper'
describe "Git HTTP requests (Geo)" do
include ::EE::GeoHelpers
include GitHttpHelpers
include WorkhorseHelpers
set(:project) { create(:project, :repository, :private) }
set(:primary) { create(:geo_node, :primary) }
set(:secondary) { create(:geo_node) }
# Ensure the token always comes from the real time of the request
let!(:auth_token) { Gitlab::Geo::BaseRequest.new.authorization }
before do
stub_licensed_features(geo: true)
stub_current_geo_node(secondary)
end
shared_examples_for 'Geo sync request' do
subject do
make_request
response
end
context 'valid Geo JWT token' do
let(:env) { valid_geo_env }
it 'returns an OK response' do
is_expected.to have_gitlab_http_status(:ok)
expect(response.content_type).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE)
expect(json_response).to include('ShowAllRefs' => true)
end
end
context 'post-dated Geo JWT token' do
let(:env) { valid_geo_env }
it { travel_to(2.minutes.ago) { is_expected.to have_gitlab_http_status(:unauthorized) } }
end
xcontext 'expired Geo JWT token' do
let(:env) { valid_geo_env }
it { travel_to(Time.now + 2.minutes) { is_expected.to have_gitlab_http_status(:unauthorized) } }
end
context 'invalid Geo JWT token' do
let(:env) { geo_env("GL-Geo xxyyzz:12345") }
it { is_expected.to have_gitlab_http_status(:unauthorized) }
end
context 'no Geo JWT token' do
let(:env) { workhorse_internal_api_request_header }
it { is_expected.to have_gitlab_http_status(:unauthorized) }
end
context 'Geo is unlicensed' do
let(:env) { valid_geo_env }
before do
stub_licensed_features(geo: false)
end
it { is_expected.to have_gitlab_http_status(:forbidden) }
end
end
describe 'GET info_refs' do
def make_request
get "/#{project.full_path}.git/info/refs", { service: 'git-upload-pack' }, env
end
it_behaves_like 'Geo sync request'
end
describe 'POST upload_pack' do
def make_request
post "/#{project.full_path}.git/git-upload-pack", {}, env
end
it_behaves_like 'Geo sync request'
end
def valid_geo_env
geo_env(auth_token)
end
def geo_env(authorization)
env = workhorse_internal_api_request_header
env['HTTP_AUTHORIZATION'] = authorization
env
end
end
......@@ -161,7 +161,8 @@ describe Projects::CreateService, '#execute' do
end
context 'when running on a primary node' do
let!(:geo_node) { create(:geo_node, :primary) }
set(:primary) { create(:geo_node, :primary) }
set(:secondary) { create(:geo_node) }
it 'logs an event to the Geo event log' do
expect { create_project(user, opts) }.to change(Geo::RepositoryCreatedEvent, :count).by(1)
......
......@@ -31,7 +31,8 @@ describe Projects::DestroyService do
end
context 'when running on a primary node' do
let!(:geo_node) { create(:geo_node, :primary) }
set(:primary) { create(:geo_node, :primary) }
set(:secondary) { create(:geo_node) }
it 'logs an event to the Geo event log' do
# Run Sidekiq immediately to check that renamed repository will be removed
......
......@@ -7,9 +7,10 @@ describe Projects::HashedStorageMigrationService do
let(:hashed_storage) { Storage::HashedProject.new(project) }
describe '#execute' do
it 'creates a Geo::RepositoryRenamedEvent on success' do
allow(Gitlab::Geo).to receive(:primary?).and_return(true)
set(:primary) { create(:geo_node, :primary) }
set(:secondary) { create(:geo_node) }
it 'creates a Geo::RepositoryRenamedEvent on success' do
expect { service.execute }.to change { Geo::EventLog.count }.by(1)
event = Geo::EventLog.first.event
......@@ -32,7 +33,6 @@ describe Projects::HashedStorageMigrationService do
allow(service).to receive(:move_repository).and_call_original
allow(service).to receive(:move_repository).with(from_name, to_name).once { false } # will disable first move only
allow(Gitlab::Geo).to receive(:primary?).and_return(true)
expect { service.execute }.not_to change { Geo::EventLog.count }
end
end
......
......@@ -12,7 +12,8 @@ describe Projects::TransferService do
end
context 'when running on a primary node' do
let!(:geo_node) { create(:geo_node, :primary) }
set(:primary) { create(:geo_node, :primary) }
set(:secondary) { create(:geo_node) }
it 'logs an event to the Geo event log' do
expect { subject.execute(group) }.to change(Geo::RepositoryRenamedEvent, :count).by(1)
......
FactoryGirl.define do
factory :geo_node_status do
skip_create
sequence(:id)
trait :healthy do
health nil
attachments_count 329
attachments_failed_count 13
attachments_synced_count 141
lfs_objects_count 256
lfs_objects_failed_count 12
lfs_objects_synced_count 123
repositories_count 10
repositories_synced_count 5
repositories_failed_count 0
last_event_id 2
last_event_timestamp Time.now.to_i
cursor_last_event_id 1
cursor_last_event_timestamp Time.now.to_i
end
trait :unhealthy do
health "Could not connect to Geo node - HTTP Status Code: 401 Unauthorized\nTest"
end
end
end
......@@ -2,12 +2,15 @@ FactoryGirl.define do
factory :geo_node do
host { Gitlab.config.gitlab.host }
sequence(:port) {|n| n}
association :geo_node_key
trait :ssh do
clone_protocol 'ssh'
association :geo_node_key
end
trait :primary do
primary true
port { Gitlab.config.gitlab.port }
geo_node_key nil
end
end
end
......@@ -7,27 +7,34 @@ describe "Admin::PushRules" do
sign_in(current_user)
end
context 'when reject_unsigned_commits is unlicensed' do
before do
stub_licensed_features(reject_unsigned_commits: false)
push_rules_with_titles = {
reject_unsigned_commits: 'Reject unsigned commits',
commit_committer_check: 'Committer restriction'
}
push_rules_with_titles.each do |rule_attr, title|
context "when #{rule_attr} is unlicensed" do
before do
stub_licensed_features(rule_attr => false)
end
it 'does not render the setting checkbox' do
visit admin_push_rule_path
expect(page).not_to have_content(title)
end
end
it 'does not render the setting checkbox' do
visit admin_push_rule_path
context "when #{rule_attr} is licensed" do
before do
stub_licensed_features(rule_attr => true)
end
expect(page).not_to have_content('Reject unsigned commits')
end
end
context 'when reject_unsigned_commits is licensed' do
before do
stub_licensed_features(reject_unsigned_commits: true)
end
it 'renders the setting checkbox' do
visit admin_push_rule_path
it 'renders the setting checkbox' do
visit admin_push_rule_path
expect(page).to have_content('Reject unsigned commits')
expect(page).to have_content(title)
end
end
end
end
......@@ -3,61 +3,69 @@ require 'spec_helper'
feature 'Projects > Push Rules', :js do
let(:user) { create(:user) }
let(:project) { create(:project, :repository, namespace: user.namespace) }
let(:foo) {{ reject_unsigned_commits: 'Reject unsigned commits' }}
before do
project.team << [user, :master]
sign_in(user)
end
describe 'Reject unsigned commits rule' do
context 'unlicensed' do
before do
stub_licensed_features(reject_unsigned_commits: false)
end
it 'does not render the setting checkbox' do
visit project_settings_repository_path(project)
push_rules_with_titles = {
reject_unsigned_commits: 'Reject unsigned commits',
commit_committer_check: 'Committer restriction'
}
expect(page).not_to have_content('Reject unsigned commits')
end
end
push_rules_with_titles.each do |rule_attr, title|
describe "#{rule_attr} rule" do
context 'unlicensed' do
before do
stub_licensed_features(rule_attr => false)
end
context 'licensed' do
let(:bronze_plan) { Plan.find_by!(name: 'bronze') }
let(:gold_plan) { Plan.find_by!(name: 'gold') }
it 'does not render the setting checkbox' do
visit project_settings_repository_path(project)
before do
stub_licensed_features(reject_unsigned_commits: true)
expect(page).not_to have_content(title)
end
end
it 'renders the setting checkbox' do
visit project_settings_repository_path(project)
context 'licensed' do
let(:bronze_plan) { Plan.find_by!(name: 'bronze') }
let(:gold_plan) { Plan.find_by!(name: 'gold') }
expect(page).to have_content('Reject unsigned commits')
end
describe 'with GL.com plans' do
before do
stub_application_setting(check_namespace_plan: true)
stub_licensed_features(rule_attr => true)
end
context 'when disabled' do
it 'does not render the setting checkbox' do
project.namespace.update!(plan_id: bronze_plan.id)
it 'renders the setting checkbox' do
visit project_settings_repository_path(project)
visit project_settings_repository_path(project)
expect(page).to have_content(title)
end
expect(page).not_to have_content('Reject unsigned commits')
describe 'with GL.com plans' do
before do
stub_application_setting(check_namespace_plan: true)
end
context 'when disabled' do
it 'does not render the setting checkbox' do
project.namespace.update!(plan_id: bronze_plan.id)
visit project_settings_repository_path(project)
expect(page).not_to have_content(title)
end
end
end
context 'when enabled' do
it 'renders the setting checkbox' do
project.namespace.update!(plan_id: gold_plan.id)
context 'when enabled' do
it 'renders the setting checkbox' do
project.namespace.update!(plan_id: gold_plan.id)
visit project_settings_repository_path(project)
visit project_settings_repository_path(project)
expect(page).to have_content('Reject unsigned commits')
expect(page).to have_content(title)
end
end
end
end
......
......@@ -7,7 +7,8 @@ describe PushRulesHelper do
let(:project_owner) { push_rule.project.owner }
let(:possible_help_texts) do
{
base_help: /Only signed commits can be pushed to this repository/,
commit_committer_check_base_help: /Users can only push commits to this repository that were committed with one of their own verified emails/,
reject_unsigned_commits_base_help: /Only signed commits can be pushed to this repository/,
default_admin_help: /This setting will be applied to all projects unless overridden by an admin/,
setting_can_be_overridden: /This setting is applied on the server level and can be overridden by an admin/,
setting_has_been_overridden: /This setting is applied on the server level but has been overridden for this project/,
......@@ -43,20 +44,23 @@ describe PushRulesHelper do
end
with_them do
before do
global_push_rule.update_column(:reject_unsigned_commits, enabled_globally)
push_rule.update_column(:reject_unsigned_commits, enabled_in_project)
PushRule::SETTINGS_WITH_GLOBAL_DEFAULT.each do |rule_attr|
before do
global_push_rule.update_column(rule_attr, enabled_globally)
push_rule.update_column(rule_attr, enabled_in_project)
allow(helper).to receive(:current_user).and_return(users[current_user])
end
allow(helper).to receive(:current_user).and_return(users[current_user])
end
it 'has the correct help text' do
rule = global_setting ? global_push_rule : push_rule
it 'has the correct help text' do
rule = global_setting ? global_push_rule : push_rule
message = possible_help_texts["#{rule_attr}_#{help_text}".to_sym].presence || possible_help_texts[help_text]
expect(helper.reject_unsigned_commits_description(rule)).to match(possible_help_texts[help_text])
expect(helper.public_send("#{rule_attr}_description", rule)).to match(message)
if invalid_text
expect(helper.reject_unsigned_commits_description(rule)).not_to match(possible_help_texts[invalid_text])
if invalid_text
expect(helper.public_send("#{rule_attr}_description", rule)).not_to match(possible_help_texts[invalid_text])
end
end
end
end
......
......@@ -10,16 +10,17 @@ describe Gitlab::Checks::ChangeAccess do
let(:ref) { 'refs/heads/master' }
let(:changes) { { oldrev: oldrev, newrev: newrev, ref: ref } }
let(:protocol) { 'ssh' }
subject do
let(:change_access) do
described_class.new(
changes,
project: project,
user_access: user_access,
protocol: protocol
).exec
)
end
subject { change_access.exec }
before do
project.add_developer(user)
end
......@@ -455,5 +456,89 @@ describe Gitlab::Checks::ChangeAccess do
expect { subject }.to raise_error(Gitlab::GitAccess::UnauthorizedError, "The path 'README' is locked by #{path_lock.user.name}")
end
end
context 'Check commit author rules' do
before do
stub_licensed_features(commit_committer_check: true)
end
let(:push_rule) { create(:push_rule, commit_committer_check: true) }
let(:project) { create(:project, :public, :repository, push_rule: push_rule) }
context 'with a commit from the authenticated user' do
before do
allow(project.repository).to receive(:new_commits).and_return(
project.repository.commits_between('be93687618e4b132087f430a4d8fc3a609c9b77c', '54fcc214b94e78d7a41a9a8fe6d87a5e59500e51')
)
allow_any_instance_of(Commit).to receive(:committer_email).and_return(user.email)
end
it 'does not return an error' do
expect { subject }.not_to raise_error
end
it 'allows the commit when they were done with another email that belongs to the current user' do
user.emails.create(email: 'secondary_email@user.com', confirmed_at: Time.now)
allow_any_instance_of(Commit).to receive(:committer_email).and_return('secondary_email@user.com')
expect { subject }.not_to raise_error
end
it 'raises an error when the commit was done with an unverified email' do
user.emails.create(email: 'secondary_email@user.com')
allow_any_instance_of(Commit).to receive(:committer_email).and_return('secondary_email@user.com')
expect { subject }
.to raise_error(Gitlab::GitAccess::UnauthorizedError,
"Comitter email '%{commiter_email}' is not verified.")
end
it 'raises an error when using an unknown email' do
allow_any_instance_of(Commit).to receive(:committer_email).and_return('some@mail.com')
expect { subject }
.to raise_error(Gitlab::GitAccess::UnauthorizedError,
"You cannot push commits for 'some@mail.com'. You can only push commits that were committed with one of your own verified emails.")
end
end
context 'for an ff merge request' do
# the signed-commits branch fast-forwards onto master
let(:newrev) { '2d1096e3' }
it 'does not raise errors for a fast forward' do
expect(change_access).not_to receive(:committer_check)
expect { subject }.not_to raise_error
end
end
context 'for a normal merge' do
# This creates a merge commit without adding it to a target branch
# that is what the repository would look like during the `pre-receive` hook.
#
# That means only the merge commit should be validated.
let(:newrev) do
rugged = project.repository.raw_repository.rugged
base = oldrev
to_merge = '2d1096e3a0ecf1d2baf6dee036cc80775d4940ba'
merge_index = rugged.merge_commits(base, to_merge)
options = {
parents: [base, to_merge],
tree: merge_index.write_tree(rugged),
message: 'The merge commit',
author: { name: user.name, email: user.email, time: Time.now },
committer: { name: user.name, email: user.email, time: Time.now }
}
Rugged::Commit.create(rugged, options)
end
it 'does not raise errors for a merge commit' do
expect(change_access).to receive(:committer_check).once
.and_call_original
expect { subject }.not_to raise_error
end
end
end
end
end
......@@ -355,19 +355,26 @@ describe Gitlab::GitAccess do
let(:key) { build(:geo_node_key, geo_node: geo_node) }
let(:actor) { key }
context 'assigned to primary geo node' do
let(:geo_node) { build(:geo_node, primary: true) }
context 'assigned to ssh primary geo node' do
let(:geo_node) { build(:geo_node, :ssh, primary: true) }
it { expect { pull_access_check }.to raise_not_found }
it { expect { push_access_check }.to raise_not_found }
end
context 'assigned to secondary geo node' do
let(:geo_node) { build(:geo_node, primary: false) }
context 'assigned to ssh secondary geo node' do
let(:geo_node) { build(:geo_node, :ssh, primary: false) }
it { expect { pull_access_check }.not_to raise_error }
it { expect { push_access_check }.to raise_unauthorized(described_class::ERROR_MESSAGES[:upload]) }
end
context 'assigned to http secondary geo node' do
let(:geo_node) { build(:geo_node, primary: false) }
it { expect { pull_access_check }.to raise_not_found }
it { expect { push_access_check }.to raise_not_found }
end
end
describe 'build authentication_abilities permissions' do
......
require 'spec_helper'
describe SystemCheck::App::GitUserDefaultSSHConfigCheck do
include ::EE::GeoHelpers
let(:username) { '_this_user_will_not_exist_unless_it_is_stubbed' }
let(:base_dir) { Dir.mktmpdir }
let(:home_dir) { File.join(base_dir, "/var/lib/#{username}") }
......@@ -40,10 +42,12 @@ describe SystemCheck::App::GitUserDefaultSSHConfigCheck do
it { is_expected.to eq(expected_result) }
end
it 'skips GitLab read-only instances' do
it 'skips Geo secondaries with SSH' do
stub_user
stub_home_dir
allow(Gitlab::Database).to receive(:read_only?).and_return(true)
node = create(:geo_node, :ssh)
stub_current_geo_node(node)
is_expected.to be_truthy
end
......@@ -78,6 +82,7 @@ describe SystemCheck::App::GitUserDefaultSSHConfigCheck do
end
def stub_user
allow(File).to receive(:expand_path).and_call_original
allow(File).to receive(:expand_path).with("~#{username}").and_return(home_dir)
end
......
require 'spec_helper'
describe SystemCheck::Geo::GitVersionCheck do
describe '#check?' do
subject { described_class.new.check? }
where(:git_version, :result) do
[
['2.8.99', false],
['2.9.0', false],
['2.9.4', false],
['2.9.5', true],
['2.9.55', true],
['10.0.0', true]
]
end
with_them do
before do
stub_git_version(git_version)
end
it { is_expected.to eq(result) }
end
end
def stub_git_version(version)
allow(described_class).to receive(:current_version) { Gitlab::VersionInfo.parse(version) }
end
end
......@@ -134,6 +134,7 @@ describe Group, 'Routable' do
context 'with RequestStore active', :request_store do
it 'does not load the route table more than once' do
group.expires_full_path_cache
expect(group).to receive(:uncached_full_path).once.and_call_original
3.times { group.full_path }
......
require 'spec_helper'
describe GeoNodeKey do
let(:geo_node) { create(:geo_node) }
let(:geo_node_key) { create(:geo_node_key, geo_nodes: [geo_node]) }
describe 'Associations' do
it { is_expected.to have_one(:geo_node) }
end
describe '#active?' do
let(:geo_node) { create(:geo_node, :ssh) }
let(:geo_node_key) { geo_node.geo_node_key }
subject { geo_node_key.active? }
it 'returns true for a secondary SSH Geo node' do
is_expected.to be_truthy
end
it 'returns false for a primary SSH Geo node' do
geo_node.primary = true
is_expected.to be_falsy
end
it 'returns false for a secondary HTTP Geo node' do
geo_node.clone_protocol = 'http'
is_expected.to be_falsy
end
end
end
......@@ -23,7 +23,10 @@ describe GeoNode, type: :model do
end
context 'validations' do
it { expect(new_node).to validate_presence_of(:geo_node_key) }
let(:ssh_node) { build(:geo_node, :ssh) }
it { expect(ssh_node).to validate_presence_of(:geo_node_key) }
it { expect(new_node).not_to validate_presence_of(:geo_node_key) }
it { expect(new_primary_node).not_to validate_presence_of(:geo_node_key) }
end
......@@ -38,6 +41,7 @@ describe GeoNode, type: :model do
:primary | false
:repos_max_capacity | 25
:files_max_capacity | 10
:clone_protocol | 'http'
end
with_them do
......@@ -69,18 +73,27 @@ describe GeoNode, type: :model do
context 'on initialize' do
it 'initializes a corresponding key' do
expect(new_node.geo_node_key).to be_present
expect(empty_node.geo_node_key).to be_present
end
it 'is valid when required attributes are present' do
new_node.clone_protocol = 'ssh'
new_node.geo_node_key_attributes = geo_node_key_attributes
expect(new_node).to be_valid
end
end
context 'on create' do
it 'saves a corresponding key' do
expect(node.geo_node_key).to be_persisted
context 'SSH node' do
let(:ssh_node) { create(:geo_node, :ssh) }
it 'saves a corresponding key' do
expect(ssh_node.geo_node_key).to be_persisted
end
end
it 'does not save a key' do
expect(node.geo_node_key).to be_nil
end
it 'saves a corresponding oauth application if it is a secondary node' do
......@@ -336,11 +349,20 @@ describe GeoNode, type: :model do
end
context 'secondary node' do
it 'is automatically set' do
it 'is not set for HTTP' do
node = build(:geo_node, url: 'http://example.com/')
expect(node.geo_node_key).to be_present
expect(node.geo_node_key.title).not_to include('example.com')
node.save!
expect(node.geo_node_key).to be_nil
end
it 'is automatically set for SSH' do
node = build(:geo_node, :ssh, url: 'http://example.com/')
expect(node.geo_node_key).to be_present
node.save!
......
......@@ -526,6 +526,47 @@ describe Group do
end
end
describe '#path_changed_hook' do
let(:system_hook_service) { SystemHooksService.new }
context 'for a new group' do
let(:group) { build(:group) }
before do
expect(group).to receive(:system_hook_service).and_return(system_hook_service)
end
it 'does not trigger system hook' do
expect(system_hook_service).to receive(:execute_hooks_for).with(group, :create)
group.save!
end
end
context 'for an existing group' do
let(:group) { create(:group, path: 'old-path') }
context 'when the path is changed' do
let(:new_path) { 'very-new-path' }
it 'triggers the rename system hook' do
expect(group).to receive(:system_hook_service).and_return(system_hook_service)
expect(system_hook_service).to receive(:execute_hooks_for).with(group, :rename)
group.update_attributes!(path: new_path)
end
end
context 'when the path is not changed' do
it 'does not trigger system hook' do
expect(group).not_to receive(:system_hook_service)
group.update_attributes!(name: 'new name')
end
end
end
end
describe '#secret_variables_for' do
let(:project) { create(:project, group: group) }
......
......@@ -24,6 +24,7 @@ describe PushRule do
author_email_regex: 'regex',
file_name_regex: 'regex',
reject_unsigned_commits: true,
commit_committer_check: true,
member_check: true,
prevent_secrets: true,
max_file_size: 1
......
......@@ -2339,6 +2339,42 @@ describe User do
end
end
describe '#username_changed_hook' do
context 'for a new user' do
let(:user) { build(:user) }
it 'does not trigger system hook' do
expect(user).not_to receive(:system_hook_service)
user.save!
end
end
context 'for an existing user' do
let(:user) { create(:user, username: 'old-username') }
context 'when the username is changed' do
let(:new_username) { 'very-new-name' }
it 'triggers the rename system hook' do
system_hook_service = SystemHooksService.new
expect(system_hook_service).to receive(:execute_hooks_for).with(user, :rename)
expect(user).to receive(:system_hook_service).and_return(system_hook_service)
user.update_attributes!(username: new_username)
end
end
context 'when the username is not changed' do
it 'does not trigger system hook' do
expect(user).not_to receive(:system_hook_service)
user.update_attributes!(email: 'asdf@asdf.com')
end
end
end
end
describe '#sync_attribute?' do
let(:user) { described_class.new }
......
......@@ -3,35 +3,32 @@ require 'spec_helper'
describe Geo::NodeCreateService do
describe '#execute' do
it 'creates a new node with valid params' do
params = { url: 'http://example.com', geo_node_key_attributes: attributes_for(:key) }
service = described_class.new(params)
service = described_class.new(url: 'http://example.com')
expect { service.execute }.to change(GeoNode, :count).by(1)
end
it 'does not create a node with invalid params' do
service = described_class.new({ url: 'http://example.com' })
service = described_class.new(url: 'ftp://example.com')
expect { service.execute }.not_to change(GeoNode, :count)
end
it 'returns true when creation succeeds' do
params = { url: 'http://example.com', geo_node_key_attributes: attributes_for(:key) }
service = described_class.new(params)
service = described_class.new(url: 'http://example.com')
expect(service.execute).to eq true
end
it 'returns false when creation fails' do
params = { url: 'http://example.com' }
service = described_class.new(params)
service = described_class.new(url: 'ftp://example.com')
expect(service.execute).to eq false
end
it 'parses the namespace_ids when node have namespace restrictions' do
groups = create_list(:group, 2)
params = { url: 'http://example.com', geo_node_key_attributes: attributes_for(:key), namespace_ids: groups.map(&:id).join(',') }
params = { url: 'http://example.com', namespace_ids: groups.map(&:id).join(',') }
service = described_class.new(params)
service.execute
......
......@@ -8,15 +8,16 @@ describe Geo::NodeUpdateService do
describe '#execute' do
it 'updates the node without changing the key' do
original_fingerprint = geo_node.geo_node_key.fingerprint
ssh_node = create(:geo_node, :ssh)
original_fingerprint = ssh_node.geo_node_key.fingerprint
params = { url: 'http://example.com', geo_node_key_attributes: attributes_for(:key) }
service = described_class.new(geo_node, params)
service = described_class.new(ssh_node, params)
service.execute
geo_node.reload
expect(geo_node.url.chomp('/')).to eq(params[:url])
expect(geo_node.geo_node_key.fingerprint).to eq(original_fingerprint)
ssh_node.reload
expect(ssh_node.url.chomp('/')).to eq(params[:url])
expect(ssh_node.geo_node_key.fingerprint).to eq(original_fingerprint)
end
it 'returns true when update succeeds' do
......
......@@ -2,6 +2,7 @@ require 'spec_helper'
describe Geo::RepositoryCreatedEventStore do
set(:project) { create(:project) }
set(:secondary_node) { create(:geo_node) }
subject(:create!) { described_class.new(project).create }
......@@ -17,6 +18,12 @@ describe Geo::RepositoryCreatedEventStore do
allow(Gitlab::Geo).to receive(:primary?) { true }
end
it 'does not create an event when there are no secondary nodes' do
allow(Gitlab::Geo).to receive(:secondary_nodes) { [] }
expect { create! }.not_to change(Geo::RepositoryCreatedEvent, :count)
end
it 'creates a created event' do
expect { create! }.to change(Geo::RepositoryCreatedEvent, :count).by(1)
end
......
require 'spec_helper'
describe Geo::RepositoryDeletedEventStore do
let(:project) { create(:project, path: 'bar') }
let!(:project_id) { project.id }
let!(:project_name) { project.name }
let!(:repo_path) { project.full_path }
let!(:wiki_path) { "#{project.full_path}.wiki" }
let!(:storage_name) { project.repository_storage }
let!(:storage_path) { project.repository_storage_path }
set(:project) { create(:project, path: 'bar') }
set(:secondary_node) { create(:geo_node) }
let(:project_id) { project.id }
let(:project_name) { project.name }
let(:repo_path) { project.full_path }
let(:wiki_path) { "#{project.full_path}.wiki" }
let(:storage_name) { project.repository_storage }
let(:storage_path) { project.repository_storage_path }
subject { described_class.new(project, repo_path: repo_path, wiki_path: wiki_path) }
......@@ -23,6 +24,12 @@ describe Geo::RepositoryDeletedEventStore do
allow(Gitlab::Geo).to receive(:primary?) { true }
end
it 'does not create an event when there are no secondary nodes' do
allow(Gitlab::Geo).to receive(:secondary_nodes) { [] }
expect { subject.create }.not_to change(Geo::RepositoryDeletedEvent, :count)
end
it 'creates a deleted event' do
expect { subject.create }.to change(Geo::RepositoryDeletedEvent, :count).by(1)
end
......
require 'spec_helper'
describe Geo::RepositoryRenamedEventStore do
let(:project) { create(:project, path: 'bar') }
set(:project) { create(:project, path: 'bar') }
set(:secondary_node) { create(:geo_node) }
let(:old_path) { 'foo' }
let(:old_path_with_namespace) { "#{project.namespace.full_path}/foo" }
......@@ -19,6 +20,12 @@ describe Geo::RepositoryRenamedEventStore do
allow(Gitlab::Geo).to receive(:primary?) { true }
end
it 'does not create an event when there are no secondary nodes' do
allow(Gitlab::Geo).to receive(:secondary_nodes) { [] }
expect { subject.create }.not_to change(Geo::RepositoryRenamedEvent, :count)
end
it 'creates a renamed event' do
expect { subject.create }.to change(Geo::RepositoryRenamedEvent, :count).by(1)
end
......
require 'spec_helper'
RSpec.describe Geo::RepositorySyncService do
let!(:primary) { create(:geo_node, :primary, host: 'primary-geo-node') }
include ::EE::GeoHelpers
set(:primary) { create(:geo_node, :primary, host: 'primary-geo-node', relative_url_root: '/gitlab') }
set(:secondary) { create(:geo_node) }
let(:lease) { double(try_obtain: true) }
subject { described_class.new(project) }
before do
stub_current_geo_node(secondary)
end
it_behaves_like 'geo base sync execution'
describe '#execute' do
let(:project) { create(:project_empty_repo) }
let(:repository) { project.repository }
let(:url_to_repo) { "#{primary.clone_url_prefix}#{project.full_path}.git" }
let(:url_to_repo) { "#{primary.url}/#{project.full_path}.git" }
before do
allow(Gitlab::ExclusiveLease).to receive(:new)
......@@ -22,7 +30,8 @@ RSpec.describe Geo::RepositorySyncService do
.and_return(true)
end
it 'fetches project repository' do
it 'fetches project repository with JWT credentials' do
expect(repository).to receive(:with_config).with("http.#{url_to_repo}.extraHeader" => anything).and_call_original
expect(repository).to receive(:fetch_geo_mirror).with(url_to_repo).once
subject.execute
......@@ -101,7 +110,6 @@ RSpec.describe Geo::RepositorySyncService do
context 'when repository sync fail' do
let(:registry) { Geo::ProjectRegistry.find_by(project_id: project.id) }
let(:url_to_repo) { "#{primary.clone_url_prefix}#{project.full_path}.git" }
before do
allow(repository).to receive(:fetch_geo_mirror).with(url_to_repo) { raise Gitlab::Shell::Error }
......@@ -118,5 +126,21 @@ RSpec.describe Geo::RepositorySyncService do
end
end
end
context 'secondary replicates over SSH' do
set(:ssh_secondary) { create(:geo_node, :ssh) }
let(:url_to_repo) { "#{primary.clone_url_prefix}/#{project.full_path}.git" }
before do
stub_current_geo_node(ssh_secondary)
end
it 'fetches wiki repository over SSH' do
expect(repository).to receive(:fetch_geo_mirror).with(url_to_repo).once
subject.execute
end
end
end
end
require 'spec_helper'
describe Geo::RepositoryUpdatedEventStore do
let(:project) { create(:project, :repository) }
set(:project) { create(:project, :repository) }
set(:secondary_node) { create(:geo_node) }
let(:blankrev) { Gitlab::Git::BLANK_SHA }
let(:refs) { ['refs/heads/tést', 'refs/tags/tag'] }
......@@ -26,6 +27,14 @@ describe Geo::RepositoryUpdatedEventStore do
allow(Gitlab::Geo).to receive(:primary?) { true }
end
it 'does not create an event when there are no secondary nodes' do
allow(Gitlab::Geo).to receive(:secondary_nodes) { [] }
subject = described_class.new(project, refs: refs, changes: changes)
expect { subject.create }.not_to change(Geo::RepositoryUpdatedEvent, :count)
end
it 'creates a push event' do
subject = described_class.new(project, refs: refs, changes: changes)
......
require 'spec_helper'
RSpec.describe Geo::WikiSyncService do
let!(:primary) { create(:geo_node, :primary, host: 'primary-geo-node') }
include ::EE::GeoHelpers
set(:primary) { create(:geo_node, :primary, host: 'primary-geo-node', relative_url_root: '/gitlab') }
set(:secondary) { create(:geo_node) }
let(:lease) { double(try_obtain: true) }
subject { described_class.new(project) }
before do
stub_current_geo_node(secondary)
end
it_behaves_like 'geo base sync execution'
describe '#execute' do
let(:project) { create(:project_empty_repo) }
let(:repository) { project.wiki.repository }
let(:url_to_repo) { "#{primary.clone_url_prefix}#{project.full_path}.wiki.git" }
let(:url_to_repo) { "#{primary.url}/#{project.full_path}.wiki.git" }
before do
allow(Gitlab::ExclusiveLease).to receive(:new)
......@@ -22,7 +30,8 @@ RSpec.describe Geo::WikiSyncService do
.and_return(true)
end
it 'fetches wiki repository' do
it 'fetches wiki repository with JWT credentials' do
expect(repository).to receive(:with_config).with("http.#{url_to_repo}.extraHeader" => anything).and_call_original
expect(repository).to receive(:fetch_geo_mirror).with(url_to_repo).once
subject.execute
......@@ -107,5 +116,21 @@ RSpec.describe Geo::WikiSyncService do
end
end
end
context 'secondary replicates over SSH' do
set(:ssh_secondary) { create(:geo_node, :ssh) }
let(:url_to_repo) { "#{primary.clone_url_prefix}/#{project.full_path}.wiki.git" }
before do
stub_current_geo_node(ssh_secondary)
end
it 'fetches wiki repository over SSH' do
expect(repository).to receive(:fetch_geo_mirror).with(url_to_repo).once
subject.execute
end
end
end
end
......@@ -69,11 +69,48 @@ describe SystemHooksService do
expect(data[:project_visibility]).to eq('private')
end
context 'group_rename' do
it 'contains old and new path' do
allow(group).to receive(:path_was).and_return('old-path')
data = event_data(group, :rename)
expect(data).to include(:event_name, :name, :created_at, :updated_at, :full_path, :path, :group_id, :old_path, :old_full_path)
expect(data[:path]).to eq(group.path)
expect(data[:full_path]).to eq(group.path)
expect(data[:old_path]).to eq(group.path_was)
expect(data[:old_full_path]).to eq(group.path_was)
end
it 'contains old and new full_path for subgroup' do
subgroup = create(:group, parent: group)
allow(subgroup).to receive(:path_was).and_return('old-path')
data = event_data(subgroup, :rename)
expect(data[:full_path]).to eq(subgroup.full_path)
expect(data[:old_path]).to eq('old-path')
end
end
context 'user_rename' do
it 'contains old and new username' do
allow(user).to receive(:username_was).and_return('old-username')
data = event_data(user, :rename)
expect(data).to include(:event_name, :name, :created_at, :updated_at, :email, :user_id, :username, :old_username)
expect(data[:username]).to eq(user.username)
expect(data[:old_username]).to eq(user.username_was)
end
end
end
context 'event names' do
it { expect(event_name(user, :create)).to eq "user_create" }
it { expect(event_name(user, :destroy)).to eq "user_destroy" }
it { expect(event_name(user, :rename)).to eq 'user_rename' }
it { expect(event_name(project, :create)).to eq "project_create" }
it { expect(event_name(project, :destroy)).to eq "project_destroy" }
it { expect(event_name(project, :rename)).to eq "project_rename" }
......@@ -85,6 +122,7 @@ describe SystemHooksService do
it { expect(event_name(key, :destroy)).to eq 'key_destroy' }
it { expect(event_name(group, :create)).to eq 'group_create' }
it { expect(event_name(group, :destroy)).to eq 'group_destroy' }
it { expect(event_name(group, :rename)).to eq 'group_rename' }
it { expect(event_name(group_member, :create)).to eq 'user_add_to_group' }
it { expect(event_name(group_member, :destroy)).to eq 'user_remove_from_group' }
end
......
require 'spec_helper'
describe Geo::PruneEventLogWorker, :geo do
include ::EE::GeoHelpers
subject(:worker) { described_class.new }
set(:primary) { create(:geo_node, :primary, host: 'primary-geo-node') }
set(:secondary) { create(:geo_node) }
before do
allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(true)
end
describe '#perform' do
context 'current node secondary' do
before do
stub_current_geo_node(secondary)
end
it 'does nothing' do
expect(worker).not_to receive(:try_obtain_lease)
worker.perform
end
end
context 'current node primary' do
before do
stub_current_geo_node(primary)
end
it 'logs error when it cannot obtain lease' do
allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain) { nil }
expect(worker).to receive(:log_error).with('Cannot obtain an exclusive lease. There must be another worker already in execution.')
worker.perform
end
context 'no secondary nodes' do
before do
secondary.destroy
end
it 'deletes everything from the Geo event log' do
create_list(:geo_event_log, 2)
expect(worker).to receive(:log_info).with('No secondary nodes, delete all Geo Event Log entries')
expect { worker.perform }.to change { Geo::EventLog.count }.by(-2)
end
end
context 'multiple secondary nodes' do
set(:secondary2) { create(:geo_node) }
let(:healthy_status) { build(:geo_node_status, :healthy) }
let(:unhealthy_status) { build(:geo_node_status, :unhealthy) }
let(:node_status_service) do
service = double
allow(Geo::NodeStatusService).to receive(:new).and_return(service)
service
end
it 'contacts all secondary nodes for their status' do
expect(node_status_service).to receive(:call).twice { healthy_status }
expect(worker).to receive(:log_info).with('Delete Geo Event Log entries up to id', anything)
worker.perform
end
it 'aborts when there are unhealthy nodes' do
create_list(:geo_event_log, 2)
expect(node_status_service).to receive(:call).twice.and_return(healthy_status, unhealthy_status)
expect(worker).to receive(:log_info).with('Could not get status of all nodes, not deleting any entries from Geo Event Log', unhealthy_node_count: 1)
expect { worker.perform }.not_to change { Geo::EventLog.count }
end
it 'takes the integer-minimum value of all cursor_last_event_ids' do
events = create_list(:geo_event_log, 12)
allow(node_status_service).to receive(:call).twice.and_return(
build(:geo_node_status, :healthy, cursor_last_event_id: events[3]),
build(:geo_node_status, :healthy, cursor_last_event_id: events.last)
)
expect(worker).to receive(:log_info).with('Delete Geo Event Log entries up to id', geo_event_log_id: events[3])
expect { worker.perform }.to change { Geo::EventLog.count }.by(-3)
end
end
end
end
describe '#log_error' do
it 'calls the Geo logger' do
expect(Gitlab::Geo::Logger).to receive(:error)
worker.log_error('Something is wrong')
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment