Commit d2d54197 authored by Nick Thomas's avatar Nick Thomas

Remove Geo SSH repo sync support

parent 32ad854e
...@@ -5,28 +5,20 @@ import { ...@@ -5,28 +5,20 @@ import {
import '../flash'; import '../flash';
import Api from '../api'; import Api from '../api';
const onPrimaryCheckboxChange = function onPrimaryCheckboxChange(e, $namespaces, $key, $useSSH) { const onPrimaryCheckboxChange = function onPrimaryCheckboxChange(e, $namespaces) {
const $namespacesSelect = $('.select2', $namespaces); const $namespacesSelect = $('.select2', $namespaces);
$namespacesSelect.select2('data', null); $namespacesSelect.select2('data', null);
$namespaces.toggleClass('hidden', e.currentTarget.checked); $namespaces.toggleClass('hidden', e.currentTarget.checked);
$key.toggleClass('hidden', e.currentTarget.checked || !$useSSH.is(':checked'));
}; };
export default function geoNodeForm($container) { export default function geoNodeForm($container) {
const $namespaces = $('.js-hide-if-geo-primary', $container); const $namespaces = $('.js-hide-if-geo-primary', $container);
const $primaryCheckbox = $('input[type="checkbox"]', $container); const $primaryCheckbox = $('input[type="checkbox"]', $container);
const $select2Dropdown = $('.js-geo-node-namespaces', $container); const $select2Dropdown = $('.js-geo-node-namespaces', $container);
const $useHTTP = $('.js-use-http', $container);
const $useSSH = $('.js-use-ssh', $container);
const $sshKey = $('.js-ssh-key', $container);
$primaryCheckbox.on('change', e => $primaryCheckbox.on('change', e =>
onPrimaryCheckboxChange(e, $namespaces, $sshKey, $useSSH)); onPrimaryCheckboxChange(e, $namespaces));
$useHTTP.on('click', () => $sshKey.toggleClass('hidden', true));
$useSSH.on('click', () => $sshKey.toggleClass('hidden', false));
$select2Dropdown.select2({ $select2Dropdown.select2({
placeholder: s__('Geo|Select groups to replicate.'), placeholder: s__('Geo|Select groups to replicate.'),
......
class GeoNode < ActiveRecord::Base class GeoNode < ActiveRecord::Base
include IgnorableColumn
include Presentable include Presentable
belongs_to :geo_node_key, inverse_of: :geo_node, dependent: :destroy # rubocop: disable Cop/ActiveRecordDependent ignore_column :clone_protocol
ignore_column :geo_node_key_id
belongs_to :oauth_application, class_name: 'Doorkeeper::Application', dependent: :destroy # rubocop: disable Cop/ActiveRecordDependent belongs_to :oauth_application, class_name: 'Doorkeeper::Application', dependent: :destroy # rubocop: disable Cop/ActiveRecordDependent
has_many :geo_node_namespace_links has_many :geo_node_namespace_links
...@@ -9,10 +12,7 @@ class GeoNode < ActiveRecord::Base ...@@ -9,10 +12,7 @@ class GeoNode < ActiveRecord::Base
has_one :status, class_name: 'GeoNodeStatus' has_one :status, class_name: 'GeoNodeStatus'
default_values url: ->(record) { record.class.current_node_url }, default_values url: ->(record) { record.class.current_node_url },
primary: false, primary: false
clone_protocol: 'http'
accepts_nested_attributes_for :geo_node_key
validates :url, presence: true, uniqueness: { case_sensitive: false } validates :url, presence: true, uniqueness: { case_sensitive: false }
validate :check_url_is_valid validate :check_url_is_valid
...@@ -21,12 +21,9 @@ class GeoNode < ActiveRecord::Base ...@@ -21,12 +21,9 @@ class GeoNode < ActiveRecord::Base
validates :access_key, presence: true validates :access_key, presence: true
validates :encrypted_secret_access_key, presence: true validates :encrypted_secret_access_key, presence: true
validates :clone_protocol, presence: true, inclusion: %w(ssh http)
validates :geo_node_key, presence: true, if: :uses_ssh_key?
validate :check_not_adding_primary_as_secondary, if: :secondary? validate :check_not_adding_primary_as_secondary, if: :secondary?
after_initialize :build_dependents
after_save :expire_cache! after_save :expire_cache!
after_destroy :expire_cache! after_destroy :expire_cache!
before_validation :update_dependents_attributes before_validation :update_dependents_attributes
...@@ -246,19 +243,7 @@ class GeoNode < ActiveRecord::Base ...@@ -246,19 +243,7 @@ class GeoNode < ActiveRecord::Base
{ protocol: uri.scheme, host: uri.host, port: uri.port, script_name: uri.path } { protocol: uri.scheme, host: uri.host, port: uri.port, script_name: uri.path }
end end
def build_dependents
build_geo_node_key if new_record? && secondary? && geo_node_key.nil?
end
def update_dependents_attributes def update_dependents_attributes
if primary?
self.geo_node_key = nil
elsif uses_ssh_key?
self.geo_node_key&.title = "Geo node: #{self.url}"
end
self.geo_node_key = nil unless uses_ssh_key? || geo_node_key&.persisted?
if self.primary? if self.primary?
self.oauth_application = nil self.oauth_application = nil
update_clone_url update_clone_url
......
class GeoNodeKey < Key
has_one :geo_node, inverse_of: :geo_node_key
def orphaned?
self.geo_nodes.length == 0
end
def almost_orphaned?
self.geo_nodes.length == 1
end
def destroyed_when_orphaned?
true
end
# Geo secondary nodes use these keys to get read access to all projects.
# If the secondary is promoted to a primary, its key is no longer valid.
#
# This is necessary because keys are placed in the `~git/.ssh` directory;
# repository mirroring and other actions that shell out to SSH make use of
# the same directory. A Geo secondary does not perform any of these actions,
# but if it is made a primary and the keys are not removed, every user on the
# GitLab instance will be able to access every project using this key.
def active?
geo_node&.uses_ssh_key?
end
end
require 'securerandom' require 'securerandom'
module Geo module Geo
# The clone_url_prefix is used to build URLs for the Geo synchronization
# If this is missing from the primary node we raise this exception
EmptyCloneUrlPrefixError = Class.new(StandardError)
class BaseSyncService class BaseSyncService
include ExclusiveLeaseGuard include ExclusiveLeaseGuard
include ::Gitlab::Geo::ProjectLogHelpers include ::Gitlab::Geo::ProjectLogHelpers
...@@ -53,16 +49,6 @@ module Geo ...@@ -53,16 +49,6 @@ module Geo
LEASE_TIMEOUT LEASE_TIMEOUT
end end
def primary_ssh_path_prefix
@primary_ssh_path_prefix ||= Gitlab::Geo.primary_node.clone_url_prefix.tap do |prefix|
raise EmptyCloneUrlPrefixError, 'Missing clone_url_prefix in the primary node' unless prefix.present?
end
end
def primary_http_path_prefix
@primary_http_path_prefix ||= Gitlab::Geo.primary_node.url
end
private private
def retry_count def retry_count
...@@ -90,25 +76,7 @@ module Geo ...@@ -90,25 +76,7 @@ module Geo
end end
def fetch_geo_mirror(repository) def fetch_geo_mirror(repository)
case current_node&.clone_protocol url = Gitlab::Geo.primary_node.url + repository.full_path + '.git'
when 'http'
fetch_http_geo_mirror(repository)
when 'ssh'
fetch_ssh_geo_mirror(repository)
else
raise "Unknown clone protocol: #{current_node&.clone_protocol}"
end
end
def build_repository_url(prefix, repository)
url = prefix
url += '/' unless url.end_with?('/')
url + repository.full_path + '.git'
end
def fetch_http_geo_mirror(repository)
url = build_repository_url(primary_http_path_prefix, repository)
# Fetch the repository, using a JWT header for authentication # Fetch the repository, using a JWT header for authentication
authorization = ::Gitlab::Geo::BaseRequest.new.authorization authorization = ::Gitlab::Geo::BaseRequest.new.authorization
...@@ -117,12 +85,6 @@ module Geo ...@@ -117,12 +85,6 @@ module Geo
repository.with_config(header) { repository.fetch_as_mirror(url, forced: true) } repository.with_config(header) { repository.fetch_as_mirror(url, forced: true) }
end end
def fetch_ssh_geo_mirror(repository)
url = build_repository_url(primary_ssh_path_prefix, repository)
repository.fetch_as_mirror(url, forced: true)
end
def registry def registry
@registry ||= Geo::ProjectRegistry.find_or_initialize_by(project_id: project.id) @registry ||= Geo::ProjectRegistry.find_or_initialize_by(project_id: project.id)
end end
......
...@@ -6,7 +6,6 @@ module Geo ...@@ -6,7 +6,6 @@ module Geo
@geo_node = geo_node @geo_node = geo_node
@old_namespace_ids = geo_node.namespace_ids @old_namespace_ids = geo_node.namespace_ids
@params = params.dup @params = params.dup
@params.delete(:geo_node_key_attributes)
@params[:namespace_ids] = @params[:namespace_ids].to_s.split(',') @params[:namespace_ids] = @params[:namespace_ids].to_s.split(',')
end end
......
...@@ -27,8 +27,7 @@ module Geo ...@@ -27,8 +27,7 @@ module Geo
update_delay_s: update_delay_in_seconds, update_delay_s: update_delay_in_seconds,
download_time_s: download_time_in_seconds) download_time_s: download_time_in_seconds)
rescue Gitlab::Shell::Error, rescue Gitlab::Shell::Error,
Gitlab::Git::RepositoryMirroring::RemoteError, Gitlab::Git::RepositoryMirroring::RemoteError => e
Geo::EmptyCloneUrlPrefixError => e
fail_registry!('Error syncing repository', e) fail_registry!('Error syncing repository', e)
rescue Gitlab::Git::Repository::NoRepository => e rescue Gitlab::Git::Repository::NoRepository => e
log_info('Setting force_to_redownload flag') log_info('Setting force_to_redownload flag')
......
...@@ -28,8 +28,7 @@ module Geo ...@@ -28,8 +28,7 @@ module Geo
download_time_s: download_time_in_seconds) download_time_s: download_time_in_seconds)
rescue Gitlab::Git::RepositoryMirroring::RemoteError, rescue Gitlab::Git::RepositoryMirroring::RemoteError,
Gitlab::Shell::Error, Gitlab::Shell::Error,
ProjectWiki::CouldNotCreateWikiError, ProjectWiki::CouldNotCreateWikiError => e
Geo::EmptyCloneUrlPrefixError => e
fail_registry!('Error syncing wiki repository', e) fail_registry!('Error syncing wiki repository', e)
rescue Gitlab::Git::Repository::NoRepository => e rescue Gitlab::Git::Repository::NoRepository => e
log_info('Setting force_to_redownload flag') log_info('Setting force_to_redownload flag')
......
- disable_key_edit = local_assigns.fetch(:disable_key_edit, false)
= form_errors(geo_node) = form_errors(geo_node)
.form-group .form-group
= form.label :url, 'URL', class: 'control-label' = form.label :url, 'URL', class: 'control-label'
...@@ -13,33 +11,6 @@ ...@@ -13,33 +11,6 @@
= form.check_box :primary = form.check_box :primary
%strong This is a primary node %strong This is a primary node
.form-group.js-hide-if-geo-primary{ class: ('hidden' unless geo_node.secondary?) }
= form.label :clone_protocol, s_('Geo|Repository cloning'), class: 'control-label'
.col-sm-10
.radio
= form.label :clone_protocol_http do
= form.radio_button :clone_protocol, :http, class: 'js-use-http'
.option-title
HTTP/HTTPS
.option-description= _('Clone repositories and wikis from the primary using HTTP/HTTPS.')
.radio
= form.label :clone_protocol_ssh do
= form.radio_button :clone_protocol, :ssh, class: 'js-use-ssh'
.option-title
SSH (deprecated)
.option-description= _('Authentication must be manually configured. Deprecated since GitLab 10.2.')
= form.fields_for :geo_node_key, geo_node.geo_node_key, include_id: !disable_key_edit do |fg|
.form-group.js-ssh-key{ class: ('hidden' unless geo_node.secondary? && geo_node.clone_protocol == 'ssh') }
= fg.label :key, 'Public Key', class: 'control-label'
.col-sm-10
= fg.text_area :key, class: 'form-control thin_area', rows: 5, disabled: disable_key_edit
- unless disable_key_edit
%p.help-block
For SSH authentication, paste the public key used by the node you are adding. Read more about it
= link_to 'here', help_page_path('gitlab-geo/configuration.html', anchor: 'step-5-enabling-the-secondary-gitlab-node')
.form-group.js-hide-if-geo-primary{ class: ('hidden' unless geo_node.secondary?) } .form-group.js-hide-if-geo-primary{ class: ('hidden' unless geo_node.secondary?) }
= form.label :namespace_ids, s_('Geo|Groups to replicate'), class: 'control-label' = form.label :namespace_ids, s_('Geo|Groups to replicate'), class: 'control-label'
.col-sm-10 .col-sm-10
......
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
Edit Geo Node Edit Geo Node
= form_for [:admin, @node], html: { class: 'form-horizontal js-geo-node-form' } do |f| = form_for [:admin, @node], html: { class: 'form-horizontal js-geo-node-form' } do |f|
= render partial: 'form', locals: { form: f, geo_node: @node, disable_key_edit: true } = render partial: 'form', locals: { form: f, geo_node: @node }
.form-actions .form-actions
= f.submit 'Save changes', class: 'btn btn-create' = f.submit 'Save changes', class: 'btn btn-create'
= link_to 'Cancel', admin_geo_nodes_path, class: 'btn btn-cancel' = link_to 'Cancel', admin_geo_nodes_path, class: 'btn btn-cancel'
......
---
title: Remove Geo SSH repo sync support
merge_request: 3553
author:
type: removed
class RemoveGeoSshRepoSync < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
GeoNode = Class.new(ActiveRecord::Base)
Key = Class.new(ActiveRecord::Base)
disable_ddl_transaction!
def up
Key.where(id: GeoNode.all.select(:geo_node_key_id)).delete_all
remove_column :geo_nodes, :clone_protocol
remove_column :geo_nodes, :geo_node_key_id
end
def down
add_column :geo_nodes, :geo_node_key_id, :integer
add_column_with_default :geo_nodes, :clone_protocol, :string, default: 'http', allow_null: false
end
end
...@@ -11,7 +11,7 @@ ...@@ -11,7 +11,7 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20171124165823) do ActiveRecord::Schema.define(version: 20171124182517) do
# These are extensions that must be enabled in order to support this database # These are extensions that must be enabled in order to support this database
enable_extension "plpgsql" enable_extension "plpgsql"
...@@ -960,7 +960,6 @@ ActiveRecord::Schema.define(version: 20171124165823) do ...@@ -960,7 +960,6 @@ ActiveRecord::Schema.define(version: 20171124165823) do
create_table "geo_nodes", force: :cascade do |t| create_table "geo_nodes", force: :cascade do |t|
t.boolean "primary" t.boolean "primary"
t.integer "geo_node_key_id"
t.integer "oauth_application_id" t.integer "oauth_application_id"
t.boolean "enabled", default: true, null: false t.boolean "enabled", default: true, null: false
t.string "access_key" t.string "access_key"
...@@ -969,7 +968,6 @@ ActiveRecord::Schema.define(version: 20171124165823) do ...@@ -969,7 +968,6 @@ ActiveRecord::Schema.define(version: 20171124165823) do
t.string "clone_url_prefix" t.string "clone_url_prefix"
t.integer "files_max_capacity", default: 10, null: false t.integer "files_max_capacity", default: 10, null: false
t.integer "repos_max_capacity", default: 25, null: false t.integer "repos_max_capacity", default: 25, null: false
t.string "clone_protocol", default: "http", null: false
t.string "url", null: false t.string "url", null: false
end end
......
...@@ -24,7 +24,7 @@ Example response: ...@@ -24,7 +24,7 @@ Example response:
"enabled": true, "enabled": true,
"files_max_capacity": 10, "files_max_capacity": 10,
"repos_max_capacity": 25, "repos_max_capacity": 25,
"clone_protocol": "ssh" "clone_protocol": "http"
}, },
{ {
"id": 2, "id": 2,
...@@ -58,7 +58,7 @@ Example response: ...@@ -58,7 +58,7 @@ Example response:
"enabled": true, "enabled": true,
"files_max_capacity": 10, "files_max_capacity": 10,
"repos_max_capacity": 25, "repos_max_capacity": 25,
"clone_protocol": "ssh" "clone_protocol": "http"
} }
``` ```
......
...@@ -38,9 +38,8 @@ and the replicated read-only ones as **secondaries**. ...@@ -38,9 +38,8 @@ and the replicated read-only ones as **secondaries**.
Keep in mind that: Keep in mind that:
- Secondaries talk to primary to get user data for logins (API), to - Secondaries talk to the primary to get user data for logins (API) and to
clone/pull from repositories (SSH) and to retrieve LFS Objects and Attachments replicate repositories, LFS Objects and Attachments (HTTPS + JWT).
(HTTPS + JWT).
- Since GitLab Enterprise Edition Premium 10.0, the primary no longer talks to - Since GitLab Enterprise Edition Premium 10.0, the primary no longer talks to
secondaries to notify for changes (API). secondaries to notify for changes (API).
...@@ -62,7 +61,7 @@ The following diagram illustrates the underlying architecture of GitLab Geo: ...@@ -62,7 +61,7 @@ The following diagram illustrates the underlying architecture of GitLab Geo:
[Source diagram](https://docs.google.com/drawings/d/1Abw0P_H0Ew1-2Lj_xPDRWP87clGIke-1fil7_KQqrtE/edit) [Source diagram](https://docs.google.com/drawings/d/1Abw0P_H0Ew1-2Lj_xPDRWP87clGIke-1fil7_KQqrtE/edit)
In this diagram, there is one Geo primary node and one secondary. The In this diagram, there is one Geo primary node and one secondary. The
secondary clones repositories via git over SSH. Attachments, LFS objects, and secondary clones repositories via git over HTTPS. Attachments, LFS objects, and
other files are downloaded via HTTPS using the GitLab API to authenticate, other files are downloaded via HTTPS using the GitLab API to authenticate,
with a special endpoint protected by JWT. with a special endpoint protected by JWT.
......
...@@ -113,7 +113,14 @@ certificate from the primary and follow ...@@ -113,7 +113,14 @@ certificate from the primary and follow
[these instructions](https://docs.gitlab.com/omnibus/settings/ssl.html) [these instructions](https://docs.gitlab.com/omnibus/settings/ssl.html)
on the secondary. on the secondary.
### Step 4. Managing the secondary GitLab node ### Step 4. Enable Git access over HTTP/HTTPS
GitLab Geo synchronizes repositories over HTTP/HTTPS, and so requires this clone
method to be enabled. Navigate to **Admin Area ➔ Settings**
(`/admin/application_settings`) on the primary node, and set
`Enabled Git access protocols` to `Both SSH and HTTP(S)` or `Only HTTP(S)`.
### Step 5. Managing the secondary GitLab node
You can monitor the status of the syncing process on a secondary node You can monitor the status of the syncing process on a secondary node
by visiting the primary node's **Admin Area ➔ Geo Nodes** (`/admin/geo_nodes`) by visiting the primary node's **Admin Area ➔ Geo Nodes** (`/admin/geo_nodes`)
...@@ -170,42 +177,6 @@ secondary nodes, but repositories that have not been selected will be empty. ...@@ -170,42 +177,6 @@ secondary nodes, but repositories that have not been selected will be empty.
1. Secondary nodes won't pull repositories that do not belong to the selected 1. Secondary nodes won't pull repositories that do not belong to the selected
groups to be replicated. groups to be replicated.
## Replicating wikis and repositories over SSH
>**Warning:**
In GitLab 10.2, replicating repositories and wikis over SSH was deprecated.
Support for SSH replication will be removed in 10.3. These instructions should
only be used if you need to add a new secondary in the short term.
1. SSH into the **secondary** node and login as root:
```bash
sudo -i
```
1. Add the primary's SSH key fingerprint to the `known_hosts` file.
```bash
sudo -u git -H ssh git@<primary-node-url>
```
Replace `<primary-node-url>` with the FQDN of the primary node. You should
manually check the displayed fingerprint against a trusted record of the
expected value before accepting it!
1. Generate a *passphraseless* SSH keypair for the `git` user, and capture the
public component:
```bash
test -e ~git/.ssh/id_rsa || sudo -u git -H ssh-keygen -q -t rsa -b 4096 -f ~git/.ssh/id_rsa
cat ~git/.ssh/id_rsa.pub
```
Follow the steps above to set up the new Geo node. When you reach
[Step 4: Enabling the secondary GitLab node](#step-4-managing-the-secondary-gitlab-node)
select "SSH (deprecated)" instead of "HTTP/HTTPS", and populate the "Public Key"
with the output of the previous command (beginning `ssh-rsa AAAA...`).
### Upgrading Geo ### Upgrading Geo
See the [updating the Geo nodes document](updating_the_geo_nodes.md). See the [updating the Geo nodes document](updating_the_geo_nodes.md).
......
...@@ -123,7 +123,15 @@ cp primary.geo.example.com.crt /usr/local/share/ca-certificates ...@@ -123,7 +123,15 @@ cp primary.geo.example.com.crt /usr/local/share/ca-certificates
update-ca-certificates update-ca-certificates
``` ```
### Step 4. Managing the secondary GitLab node ### Step 4. Enable Git access over HTTP/HTTPS
GitLab Geo synchronizes repositories over HTTP/HTTPS, and so requires this clone
method to be enabled. Navigate to **Admin Area ➔ Settings**
(`/admin/application_settings`) on the primary node, and set
`Enabled Git access protocols` to `Both SSH and HTTP(S)` or `Only HTTP(S)`.
### Step 5. Managing the secondary GitLab node
You can monitor the status of the syncing process on a secondary node You can monitor the status of the syncing process on a secondary node
by visiting the primary node's **Admin Area ➔ Geo Nodes** (`/admin/geo_nodes`) by visiting the primary node's **Admin Area ➔ Geo Nodes** (`/admin/geo_nodes`)
...@@ -173,10 +181,6 @@ Point your users to the ["Using a Geo Server" guide](using_a_geo_server.md). ...@@ -173,10 +181,6 @@ Point your users to the ["Using a Geo Server" guide](using_a_geo_server.md).
Read [Selective replication](configuration.md#selective-replication). Read [Selective replication](configuration.md#selective-replication).
## Replicating wikis and repositories over SSH
Read [Replicating wikis and repositories over SSH](configuration.md#replicating-wikis-and-repositories-over-ssh).
## Troubleshooting ## Troubleshooting
Read the [troubleshooting document](troubleshooting.md). Read the [troubleshooting document](troubleshooting.md).
......
...@@ -44,8 +44,8 @@ The following guide assumes that: ...@@ -44,8 +44,8 @@ The following guide assumes that:
make sure the GitLab version is the same on all nodes. make sure the GitLab version is the same on all nodes.
- The IP of the primary server for our examples will be `1.2.3.4`, whereas the - The IP of the primary server for our examples will be `1.2.3.4`, whereas the
secondary's IP will be `5.6.7.8`. Note that the primary and secondary servers secondary's IP will be `5.6.7.8`. Note that the primary and secondary servers
**must** be able to communicate over these addresses (using HTTPS & SSH). **must** be able to communicate over these addresses. These IP addresses can
These IP addresses can either be public or private. either be public or private.
If your GitLab installation is using external PostgreSQL, the Omnibus roles If your GitLab installation is using external PostgreSQL, the Omnibus roles
will not be able to perform all necessary configuration steps. Refer to will not be able to perform all necessary configuration steps. Refer to
......
...@@ -25,11 +25,6 @@ You must make the changes in the exact specific order: ...@@ -25,11 +25,6 @@ You must make the changes in the exact specific order:
1. Promote the Postgres in your secondary node as primary 1. Promote the Postgres in your secondary node as primary
1. Modify the `gitlab.rb` for both nodes to reflect their new statuses 1. Modify the `gitlab.rb` for both nodes to reflect their new statuses
1. Log-in to your secondary node with a user with `sudo` permission 1. Log-in to your secondary node with a user with `sudo` permission
1. **Remove** the Geo SSH client keys (this is very important!):
```bash
sudo rm ~git/.ssh/id_rsa ~git/.ssh/id_rsa.pub
```
1. Run `sudo gitlab-rake geo:set_secondary_as_primary` 1. Run `sudo gitlab-rake geo:set_secondary_as_primary`
1. Rsync everything in `/var/opt/gitlab/gitlab-rails/uploads` and 1. Rsync everything in `/var/opt/gitlab/gitlab-rails/uploads` and
`/var/opt/gitlab/gitlab-rails/shared` from your old node to the new one. `/var/opt/gitlab/gitlab-rails/shared` from your old node to the new one.
......
...@@ -53,29 +53,6 @@ and expect something like: ...@@ -53,29 +53,6 @@ and expect something like:
By running the command above, `primary` should be `true` when executed in By running the command above, `primary` should be `true` when executed in
the primary node, and `false` on any secondary. the primary node, and `false` on any secondary.
#### Did I define the correct SSH Key for the node?
You must create an SSH Key for `git` user.
This key is the one you have to inform at `Admin > Geo`.
#### Can I SSH from secondary to primary node using `git` user account?
This is the most obvious cause of problems with repository replication issues.
If you haven't added the primary node's key to `known_hosts`, you will end up with
a lot of failed sidekiq jobs with an error similar to:
```
Gitlab::Shell::Error: Host key verification failed. fatal: Could not read from remote repository. Please make sure you have the correct access rights and the repository exists.
```
An easy way to fix this is by logging in as the `git` user in the secondary node and run:
```bash
# remove old entries to your primary gitlab in known_hosts
ssh-keyscan -R your-primary-gitlab.example.com
```
#### How do I fix the message, "ERROR: replication slots can only be used if max_replication_slots > 0"? #### How do I fix the message, "ERROR: replication slots can only be used if max_replication_slots > 0"?
This means that the `max_replication_slots` PostgreSQL variable needs to This means that the `max_replication_slots` PostgreSQL variable needs to
......
...@@ -14,6 +14,20 @@ all you need to do is update GitLab itself: ...@@ -14,6 +14,20 @@ all you need to do is update GitLab itself:
the tracking database is enabled. the tracking database is enabled.
1. [Test](#check-status-after-updating) primary and secondary nodes, and check version in each. 1. [Test](#check-status-after-updating) primary and secondary nodes, and check version in each.
## Upgrading to GitLab 10.3
### Support for SSH repository synchronization removed
In GitLab 10.2, synchronizing secondaries over SSH was deprecated. In 10.3,
support is removed entirely. All installations will switch to the HTTP/HTTPS
cloning method instead. Before upgrading, ensure that all your Geo nodes are
configured to use this method and that it works for your installation. In
particular, ensure that [Git access over HTTP/HTTPS is enabled](configuration.md#step-4-enable-git-access-over-http-https).
Synchronizing repositories over the public Internet using HTTP is insecure, so
you should ensure that you have HTTPS configured before upgrading. Note that
file synchronization is **also** insecure in these cases!
## Upgrading to GitLab 10.2 ## Upgrading to GitLab 10.2
### Secure PostgreSQL replication ### Secure PostgreSQL replication
......
...@@ -86,9 +86,7 @@ class Admin::GeoNodesController < Admin::ApplicationController ...@@ -86,9 +86,7 @@ class Admin::GeoNodesController < Admin::ApplicationController
:primary, :primary,
:namespace_ids, :namespace_ids,
:repos_max_capacity, :repos_max_capacity,
:files_max_capacity, :files_max_capacity
:clone_protocol,
geo_node_key_attributes: [:key]
) )
end end
......
...@@ -12,8 +12,7 @@ module EE ...@@ -12,8 +12,7 @@ module EE
def can_read_project? def can_read_project?
raise NotImplementedError.new unless defined?(super) raise NotImplementedError.new unless defined?(super)
return geo_node_key.active? if geo_node_key? return true if geo?
return true if actor == :geo
super super
end end
...@@ -52,16 +51,8 @@ module EE ...@@ -52,16 +51,8 @@ module EE
end end
end end
def geo_node_key
actor if geo_node_key?
end
def geo_node_key?
actor.is_a?(::GeoNodeKey)
end
def geo? def geo?
geo_node_key? || actor == :geo actor == :geo
end end
end end
end end
......
module SystemCheck
module Geo
class HTTPCloneEnabledCheck < ::SystemCheck::BaseCheck
set_name 'HTTP/HTTPS repository cloning is enabled'
def check?
enabled_git_access_protocol.blank? || enabled_git_access_protocol == 'http'
end
def show_error
try_fixing_it(
'Enable HTTP/HTTPS repository cloning for Geo repository synchronization'
)
for_more_information('doc/gitlab-geo/README.md')
end
private
def enabled_git_access_protocol
Gitlab::CurrentSettings.current_application_settings.enabled_git_access_protocol
end
end
end
end
...@@ -1092,7 +1092,11 @@ module API ...@@ -1092,7 +1092,11 @@ module API
expose :enabled expose :enabled
expose :files_max_capacity expose :files_max_capacity
expose :repos_max_capacity expose :repos_max_capacity
expose :clone_protocol
# Retained for backwards compatibility. Remove in API v5
expose :clone_protocol do |_record, _options|
'http'
end
end end
class PersonalAccessToken < Grape::Entity class PersonalAccessToken < Grape::Entity
......
...@@ -57,8 +57,7 @@ module API ...@@ -57,8 +57,7 @@ module API
gl_repository: gl_repository, gl_repository: gl_repository,
gl_username: user&.username, gl_username: user&.username,
repository_path: repository_path, repository_path: repository_path,
gitaly: gitaly_payload(params[:action]), gitaly: gitaly_payload(params[:action])
geo_node: actor.is_a?(GeoNodeKey)
} }
end end
......
...@@ -11,10 +11,10 @@ module SystemCheck ...@@ -11,10 +11,10 @@ module SystemCheck
].freeze ].freeze
set_name 'Git user has default SSH configuration?' set_name 'Git user has default SSH configuration?'
set_skip_reason 'skipped (Geo uses SSH key, or git user is not present / configured)' set_skip_reason 'skipped (git user is not present / configured)'
def skip? def skip?
::Gitlab::Geo.current_node&.uses_ssh_key? || !home_dir || !File.directory?(home_dir) !home_dir || !File.directory?(home_dir)
end end
def check? def check?
......
...@@ -462,6 +462,7 @@ namespace :gitlab do ...@@ -462,6 +462,7 @@ namespace :gitlab do
SystemCheck::Geo::GeoDatabaseConfiguredCheck, SystemCheck::Geo::GeoDatabaseConfiguredCheck,
SystemCheck::Geo::DatabaseReplicationCheck, SystemCheck::Geo::DatabaseReplicationCheck,
SystemCheck::Geo::HttpConnectionCheck, SystemCheck::Geo::HttpConnectionCheck,
SystemCheck::Geo::HTTPCloneEnabledCheck,
SystemCheck::Geo::ClocksSynchronizationCheck, SystemCheck::Geo::ClocksSynchronizationCheck,
SystemCheck::App::GitUserDefaultSSHConfigCheck SystemCheck::App::GitUserDefaultSSHConfigCheck
] ]
......
...@@ -110,7 +110,7 @@ describe Admin::GeoNodesController, :postgresql do ...@@ -110,7 +110,7 @@ describe Admin::GeoNodesController, :postgresql do
end end
describe '#create' do describe '#create' do
let(:geo_node_attributes) { { url: 'http://example.com', geo_node_key_attributes: { key: SSHKeygen.generate } } } let(:geo_node_attributes) { { url: 'http://example.com' } }
def go def go
post :create, geo_node: geo_node_attributes post :create, geo_node: geo_node_attributes
...@@ -139,16 +139,9 @@ describe Admin::GeoNodesController, :postgresql do ...@@ -139,16 +139,9 @@ describe Admin::GeoNodesController, :postgresql do
end end
describe '#update' do describe '#update' do
let(:geo_node_attributes) do let(:geo_node_attributes) { { url: 'http://example.com' } }
{
url: 'http://example.com',
clone_protocol: 'ssh',
geo_node_key_attributes: attributes_for(:key)
}
end
let(:geo_node) { create(:geo_node, :ssh) } let(:geo_node) { create(:geo_node) }
let!(:original_fingerprint) { geo_node.geo_node_key.fingerprint }
def go def go
post :update, id: geo_node, geo_node: geo_node_attributes post :update, id: geo_node, geo_node: geo_node_attributes
...@@ -168,25 +161,11 @@ describe Admin::GeoNodesController, :postgresql do ...@@ -168,25 +161,11 @@ describe Admin::GeoNodesController, :postgresql do
allow(Gitlab::Geo).to receive(:license_allows?).and_return(true) allow(Gitlab::Geo).to receive(:license_allows?).and_return(true)
end end
it 'updates the node without changing the key' do it 'updates the node' do
go go
geo_node.reload geo_node.reload
expect(geo_node.url.chomp('/')).to eq(geo_node_attributes[:url]) expect(geo_node.url.chomp('/')).to eq(geo_node_attributes[:url])
expect(geo_node.geo_node_key.fingerprint).to eq(original_fingerprint)
end
context 'changing clone protocol' do
let(:geo_node_attributes) { { clone_protocol: 'http' } }
it 'changes the protocol without removing the key' do
go
geo_node.reload
expect(geo_node.clone_protocol).to eq('http')
expect(geo_node.geo_node_key.fingerprint).to eq(original_fingerprint)
end
end end
it 'delegates the update of the Geo node to Geo::NodeUpdateService' do it 'delegates the update of the Geo node to Geo::NodeUpdateService' do
......
require 'spec_helper'
describe SystemCheck::Geo::HTTPCloneEnabledCheck do
describe '#check?' do
subject { described_class.new.check? }
where(:enabled_protocol, :result) do
[
['unknown', false],
['ssh', false],
['http', true],
['', true],
[nil, true]
]
end
with_them do
before do
stub_application_setting(enabled_git_access_protocol: enabled_protocol)
end
it { is_expected.to eq(result) }
end
end
end
require 'spec_helper'
describe API::Internal do
let(:project) { create(:project, :repository) }
let(:secret_token) { Gitlab::Shell.secret_token }
describe "POST /internal/allowed", :clean_gitlab_redis_shared_state do
context 'Geo Node' do
let(:geo_node) { create(:geo_node, :ssh) }
it 'recognizes the Geo Node' do
post(
api("/internal/allowed"),
key_id: geo_node.geo_node_key.id,
project: project.repository.path_to_repo,
action: 'git-upload-pack',
secret_token: secret_token,
protocol: 'ssh')
expect(response.status).to eq(200)
expect(json_response['geo_node']).to be(true)
end
end
context 'user' do
let(:user) { create(:user) }
let(:key) { create(:key, user: user) }
before do
project.team << [user, :developer]
end
it 'does not recognize key as a Geo Node' do
post(
api("/internal/allowed"),
key_id: key.id,
project: project.repository.path_to_repo,
action: 'git-upload-pack',
secret_token: secret_token,
protocol: 'ssh')
expect(response.status).to eq(200)
expect(json_response['geo_node']).to be(false)
end
end
end
end
FactoryGirl.define do
factory :geo_node_key, class: 'GeoNodeKey' do
title
key do
SSHKeygen.generate
end
end
end
...@@ -8,11 +8,6 @@ FactoryGirl.define do ...@@ -8,11 +8,6 @@ FactoryGirl.define do
uri.to_s uri.to_s
end end
trait :ssh do
clone_protocol 'ssh'
association :geo_node_key
end
trait :primary do trait :primary do
primary true primary true
url do url do
......
...@@ -351,32 +351,6 @@ describe Gitlab::GitAccess do ...@@ -351,32 +351,6 @@ describe Gitlab::GitAccess do
end end
end end
describe 'geo node key permissions' do
let(:key) { build(:geo_node_key, geo_node: geo_node) }
let(:actor) { key }
context 'assigned to ssh primary geo node' do
let(:geo_node) { build(:geo_node, :ssh, primary: true) }
it { expect { pull_access_check }.to raise_not_found }
it { expect { push_access_check }.to raise_not_found }
end
context 'assigned to ssh secondary geo node' do
let(:geo_node) { build(:geo_node, :ssh, primary: false) }
it { expect { pull_access_check }.not_to raise_error }
it { expect { push_access_check }.to raise_unauthorized(described_class::ERROR_MESSAGES[:upload]) }
end
context 'assigned to http secondary geo node' do
let(:geo_node) { build(:geo_node, primary: false) }
it { expect { pull_access_check }.to raise_not_found }
it { expect { push_access_check }.to raise_not_found }
end
end
describe 'build authentication_abilities permissions' do describe 'build authentication_abilities permissions' do
let(:authentication_abilities) { build_authentication_abilities } let(:authentication_abilities) { build_authentication_abilities }
...@@ -1036,6 +1010,13 @@ describe Gitlab::GitAccess do ...@@ -1036,6 +1010,13 @@ describe Gitlab::GitAccess do
end end
end end
describe 'Geo system permissions' do
let(:actor) { :geo }
it { expect { pull_access_check }.not_to raise_error }
it { expect { push_access_check }.to raise_unauthorized(Gitlab::GitAccess::ERROR_MESSAGES[:upload]) }
end
private private
def raise_unauthorized(message) def raise_unauthorized(message)
......
...@@ -41,16 +41,6 @@ describe SystemCheck::App::GitUserDefaultSSHConfigCheck do ...@@ -41,16 +41,6 @@ describe SystemCheck::App::GitUserDefaultSSHConfigCheck do
it { is_expected.to eq(expected_result) } it { is_expected.to eq(expected_result) }
end end
it 'skips Geo secondaries with SSH' do
stub_user
stub_home_dir
node = create(:geo_node, :ssh)
stub_current_geo_node(node)
is_expected.to be_truthy
end
end end
describe '#check?' do describe '#check?' do
......
require 'spec_helper'
describe GeoNodeKey do
describe 'Associations' do
it { is_expected.to have_one(:geo_node) }
end
describe '#active?' do
let(:geo_node) { create(:geo_node, :ssh) }
let(:geo_node_key) { geo_node.geo_node_key }
subject { geo_node_key.active? }
it 'returns true for a secondary SSH Geo node' do
is_expected.to be_truthy
end
it 'returns false for a primary SSH Geo node' do
geo_node.primary = true
is_expected.to be_falsy
end
it 'returns false for a secondary HTTP Geo node' do
geo_node.clone_protocol = 'http'
is_expected.to be_falsy
end
end
end
...@@ -15,30 +15,18 @@ describe GeoNode, type: :model do ...@@ -15,30 +15,18 @@ describe GeoNode, type: :model do
let(:api_version) { API::API.version } let(:api_version) { API::API.version }
context 'associations' do context 'associations' do
it { is_expected.to belong_to(:geo_node_key).dependent(:destroy) }
it { is_expected.to belong_to(:oauth_application).dependent(:destroy) } it { is_expected.to belong_to(:oauth_application).dependent(:destroy) }
it { is_expected.to have_many(:geo_node_namespace_links) } it { is_expected.to have_many(:geo_node_namespace_links) }
it { is_expected.to have_many(:namespaces).through(:geo_node_namespace_links) } it { is_expected.to have_many(:namespaces).through(:geo_node_namespace_links) }
end end
context 'validations' do
let(:ssh_node) { build(:geo_node, :ssh) }
it { expect(ssh_node).to validate_presence_of(:geo_node_key) }
it { expect(new_node).not_to validate_presence_of(:geo_node_key) }
it { expect(new_primary_node).not_to validate_presence_of(:geo_node_key) }
end
context 'default values' do context 'default values' do
let(:gitlab_host) { 'gitlabhost' }
where(:attribute, :value) do where(:attribute, :value) do
:url | Gitlab::Routing.url_helpers.root_url :url | Gitlab::Routing.url_helpers.root_url
:primary | false :primary | false
:repos_max_capacity | 25 :repos_max_capacity | 25
:files_max_capacity | 10 :files_max_capacity | 10
:clone_protocol | 'http'
end end
with_them do with_them do
...@@ -57,33 +45,7 @@ describe GeoNode, type: :model do ...@@ -57,33 +45,7 @@ describe GeoNode, type: :model do
end end
context 'dependent models and attributes for GeoNode' do context 'dependent models and attributes for GeoNode' do
let(:geo_node_key_attributes) { FactoryGirl.build(:geo_node_key).attributes }
context 'on initialize' do
it 'initializes a corresponding key' do
expect(empty_node.geo_node_key).to be_present
end
it 'is valid when required attributes are present' do
new_node.clone_protocol = 'ssh'
new_node.geo_node_key_attributes = geo_node_key_attributes
expect(new_node).to be_valid
end
end
context 'on create' do context 'on create' do
context 'SSH node' do
let(:ssh_node) { create(:geo_node, :ssh) }
it 'saves a corresponding key' do
expect(ssh_node.geo_node_key).to be_persisted
end
end
it 'does not save a key' do
expect(node.geo_node_key).to be_nil
end
it 'saves a corresponding oauth application if it is a secondary node' do it 'saves a corresponding oauth application if it is a secondary node' do
expect(node.oauth_application).to be_persisted expect(node.oauth_application).to be_persisted
end end
...@@ -94,7 +56,7 @@ describe GeoNode, type: :model do ...@@ -94,7 +56,7 @@ describe GeoNode, type: :model do
end end
it 'persists current clone_url_prefix' do it 'persists current clone_url_prefix' do
expect(primary_node.clone_url_prefix).to be_present expect(primary_node.clone_url_prefix).to eq(Gitlab.config.gitlab_shell.ssh_path_prefix)
end end
end end
end end
...@@ -144,10 +106,6 @@ describe GeoNode, type: :model do ...@@ -144,10 +106,6 @@ describe GeoNode, type: :model do
end end
context 'when required fields are not filled' do context 'when required fields are not filled' do
it 'returns an initialized Geo node key' do
expect(empty_node.geo_node_key).not_to be_nil
end
it 'returns an URI object' do it 'returns an URI object' do
expect(empty_node.uri).to be_a URI expect(empty_node.uri).to be_a URI
end end
...@@ -404,43 +362,4 @@ describe GeoNode, type: :model do ...@@ -404,43 +362,4 @@ describe GeoNode, type: :model do
end end
end end
end end
describe '#geo_node_key' do
context 'primary node' do
it 'cannot be set' do
node = new_primary_node
expect(node.geo_node_key).to be_nil
node.geo_node_key = build(:geo_node_key)
expect(node).to be_valid
node.save!
expect(node.geo_node_key(true)).to be_nil
end
end
context 'secondary node' do
it 'is not set for HTTP' do
node = build(:geo_node, url: 'http://example.com/')
expect(node.geo_node_key).to be_present
node.save!
expect(node.geo_node_key).to be_nil
end
it 'is automatically set for SSH' do
node = build(:geo_node, :ssh, url: 'http://example.com/')
expect(node.geo_node_key).to be_present
node.save!
expect(node.geo_node_key.title).to eq('Geo node: http://example.com/')
end
end
end
end end
...@@ -15,18 +15,4 @@ describe Geo::BaseSyncService do ...@@ -15,18 +15,4 @@ describe Geo::BaseSyncService do
expect(subject.lease_key).to eq('geo_sync_service:wiki:999') expect(subject.lease_key).to eq('geo_sync_service:wiki:999')
end end
end end
describe '#primary_ssh_path_prefix' do
let!(:primary_node) { create(:geo_node, :primary) }
it 'raises exception when clone_url_prefix is nil' do
allow_any_instance_of(GeoNode).to receive(:clone_url_prefix) { nil }
expect { subject.primary_ssh_path_prefix }.to raise_error Geo::EmptyCloneUrlPrefixError
end
it 'returns the prefix defined in the primary node' do
expect(subject.primary_ssh_path_prefix).to eq('git@localhost:')
end
end
end end
...@@ -7,17 +7,14 @@ describe Geo::NodeUpdateService do ...@@ -7,17 +7,14 @@ describe Geo::NodeUpdateService do
let(:geo_node_with_restrictions) { create(:geo_node, namespace_ids: [groups.first.id]) } let(:geo_node_with_restrictions) { create(:geo_node, namespace_ids: [groups.first.id]) }
describe '#execute' do describe '#execute' do
it 'updates the node without changing the key' do it 'updates the node' do
ssh_node = create(:geo_node, :ssh) params = { url: 'http://example.com' }
original_fingerprint = ssh_node.geo_node_key.fingerprint service = described_class.new(geo_node, params)
params = { url: 'http://example.com', geo_node_key_attributes: attributes_for(:key) }
service = described_class.new(ssh_node, params)
service.execute service.execute
ssh_node.reload geo_node.reload
expect(ssh_node.url.chomp('/')).to eq(params[:url]) expect(geo_node.url.chomp('/')).to eq(params[:url])
expect(ssh_node.geo_node_key.fingerprint).to eq(original_fingerprint)
end end
it 'returns true when update succeeds' do it 'returns true when update succeeds' do
......
...@@ -235,21 +235,5 @@ describe Geo::RepositorySyncService do ...@@ -235,21 +235,5 @@ describe Geo::RepositorySyncService do
expect(registry.repository_retry_at).to be_nil expect(registry.repository_retry_at).to be_nil
end end
end end
context 'secondary replicates over SSH' do
set(:ssh_secondary) { create(:geo_node, :ssh) }
let(:url_to_repo) { "#{primary.clone_url_prefix}/#{project.full_path}.git" }
before do
stub_current_geo_node(ssh_secondary)
end
it 'fetches wiki repository over SSH' do
expect(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true).once
subject.execute
end
end
end end
end end
...@@ -141,21 +141,5 @@ RSpec.describe Geo::WikiSyncService do ...@@ -141,21 +141,5 @@ RSpec.describe Geo::WikiSyncService do
end end
end end
end end
context 'secondary replicates over SSH' do
set(:ssh_secondary) { create(:geo_node, :ssh) }
let(:url_to_repo) { "#{primary.clone_url_prefix}/#{project.full_path}.wiki.git" }
before do
stub_current_geo_node(ssh_secondary)
end
it 'fetches wiki repository over SSH' do
expect(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true).once
subject.execute
end
end
end end
end end
...@@ -22,7 +22,6 @@ describe 'geo rake tasks' do ...@@ -22,7 +22,6 @@ describe 'geo rake tasks' do
expect(node.uri.scheme).to eq('https') expect(node.uri.scheme).to eq('https')
expect(node.primary).to be_truthy expect(node.primary).to be_truthy
expect(node.geo_node_key).to be_nil
end end
end end
......
...@@ -6,8 +6,7 @@ RSpec.describe Geo::MetricsUpdateWorker, :geo do ...@@ -6,8 +6,7 @@ RSpec.describe Geo::MetricsUpdateWorker, :geo do
subject { described_class.new } subject { described_class.new }
describe '#perform' do describe '#perform' do
let(:geo_node_key) { create(:geo_node_key) } let(:secondary) { create(:geo_node) }
let(:secondary) { create(:geo_node, geo_node_key: geo_node_key) }
before do before do
stub_current_geo_node(secondary) stub_current_geo_node(secondary)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment