Commit 4bcc24e6 authored by Nick Thomas's avatar Nick Thomas

Allow Geo repository sync over HTTP/HTTPS

parent 924f2630
......@@ -5,19 +5,28 @@ import {
import '../flash';
import Api from '../api';
const onPrimaryCheckboxChange = function onPrimaryCheckboxChange(e, $namespaces) {
const onPrimaryCheckboxChange = function onPrimaryCheckboxChange(e, $namespaces, $key, $useSSH) {
const $namespacesSelect = $('.select2', $namespaces);
$namespacesSelect.select2('data', null);
$namespaces.toggleClass('hidden', e.currentTarget.checked);
$key.toggleClass('hidden', e.currentTarget.checked || !$useSSH.is(':checked'));
};
export default function geoNodeForm($container) {
const $namespaces = $('.js-hide-if-geo-primary', $container);
const $primaryCheckbox = $('input[type="checkbox"]', $container);
const $select2Dropdown = $('.js-geo-node-namespaces', $container);
const $useHTTP = $('.js-use-http', $container);
const $useSSH = $('.js-use-ssh', $container);
const $sshKey = $('.js-ssh-key', $container);
$primaryCheckbox.on('change', e =>
onPrimaryCheckboxChange(e, $namespaces, $sshKey, $useSSH));
$primaryCheckbox.on('change', e => onPrimaryCheckboxChange(e, $namespaces));
$useHTTP.on('click', () => $sshKey.toggleClass('hidden', true));
$useSSH.on('click', () => $sshKey.toggleClass('hidden', false));
$select2Dropdown.select2({
placeholder: s__('Geo|Select groups to replicate.'),
......
class Projects::GitHttpController < Projects::GitHttpClientController
include WorkhorseRequest
prepend ::EE::Projects::GitHttpController
before_action :access_check
......
......@@ -11,7 +11,8 @@ class GeoNode < ActiveRecord::Base
host: lambda { Gitlab.config.gitlab.host },
port: lambda { Gitlab.config.gitlab.port },
relative_url_root: lambda { Gitlab.config.gitlab.relative_url_root },
primary: false
primary: false,
clone_protocol: 'http'
accepts_nested_attributes_for :geo_node_key
......@@ -21,8 +22,9 @@ class GeoNode < ActiveRecord::Base
validates :relative_url_root, length: { minimum: 0, allow_nil: false }
validates :access_key, presence: true
validates :encrypted_secret_access_key, presence: true
validates :clone_protocol, presence: true, inclusion: %w(ssh http)
validates :geo_node_key, presence: true, if: :secondary?
validates :geo_node_key, presence: true, if: :uses_ssh_key?
validate :check_not_adding_primary_as_secondary, if: :secondary?
after_initialize :build_dependents
......@@ -46,6 +48,10 @@ class GeoNode < ActiveRecord::Base
!primary
end
def uses_ssh_key?
secondary? && clone_protocol == 'ssh'
end
def uri
if relative_url_root
relative_url = relative_url_root.starts_with?('/') ? relative_url_root : "/#{relative_url_root}"
......@@ -205,10 +211,12 @@ class GeoNode < ActiveRecord::Base
def update_dependents_attributes
if primary?
self.geo_node_key = nil
else
elsif uses_ssh_key?
self.geo_node_key&.title = "Geo node: #{self.url}"
end
self.geo_node_key = nil unless uses_ssh_key? || geo_node_key&.persisted?
if self.primary?
self.oauth_application = nil
update_clone_url
......
......@@ -22,6 +22,6 @@ class GeoNodeKey < Key
# but if it is made a primary and the keys are not removed, every user on the
# GitLab instance will be able to access every project using this key.
def active?
geo_node&.secondary?
geo_node&.uses_ssh_key?
end
end
......@@ -37,12 +37,54 @@ module Geo
end
end
def primary_http_path_prefix
@primary_http_path_prefix ||= Gitlab::Geo.primary_node.url
end
private
def sync_repository
raise NotImplementedError, 'This class should implement sync_repository method'
end
def current_node
::Gitlab::Geo.current_node
end
def fetch_geo_mirror(repository)
case current_node&.clone_protocol
when 'http'
fetch_http_geo_mirror(repository)
when 'ssh'
fetch_ssh_geo_mirror(repository)
else
raise "Unknown clone protocol: #{current_node&.clone_protocol}"
end
end
def build_repository_url(prefix, repository)
url = prefix
url += '/' unless url.end_with?('/')
url + repository.full_path + '.git'
end
def fetch_http_geo_mirror(repository)
url = build_repository_url(primary_http_path_prefix, repository)
# Fetch the repository, using a JWT header for authentication
authorization = ::Gitlab::Geo::BaseRequest.new.authorization
header = { "http.#{url}.extraHeader" => "Authorization: #{authorization}" }
repository.with_config(header) { repository.fetch_geo_mirror(url) }
end
def fetch_ssh_geo_mirror(repository)
url = build_repository_url(primary_ssh_path_prefix, repository)
repository.fetch_geo_mirror(url)
end
def registry
@registry ||= Geo::ProjectRegistry.find_or_initialize_by(project_id: project.id)
end
......
......@@ -15,9 +15,9 @@ module Geo
begin
project.ensure_repository
project.repository.fetch_geo_mirror(ssh_url_to_repo)
fetch_geo_mirror(project.repository)
update_registry(finished_at: DateTime.now)
log_info("Finished repository sync",
update_delay_s: update_delay_in_seconds,
download_time_s: download_time_in_seconds)
......
......@@ -14,9 +14,9 @@ module Geo
begin
project.wiki.ensure_repository
project.wiki.repository.fetch_geo_mirror(ssh_url_to_wiki)
fetch_geo_mirror(project.wiki.repository)
update_registry(finished_at: DateTime.now)
log_info("Finished wiki sync",
update_delay_s: update_delay_in_seconds,
download_time_s: download_time_in_seconds)
......
......@@ -12,15 +12,31 @@
.col-sm-10
= form.text_field :url, class: 'form-control'
.form-group.js-hide-if-geo-primary{ class: ('hidden' unless geo_node.secondary?) }
= form.label :clone_protocol, s_('Geo|Repository cloning'), class: 'control-label'
.col-sm-10
.radio
= form.label :clone_protocol_http do
= form.radio_button :clone_protocol, :http, class: 'js-use-http'
.option-title
HTTP/HTTPS
.option-description= _('Clone repositories and wikis from the primary using HTTP/HTTPS.')
.radio
= form.label :clone_protocol_ssh do
= form.radio_button :clone_protocol, :ssh, class: 'js-use-ssh'
.option-title
SSH (deprecated)
.option-description= _('Authentication must be manually configured. Deprecated since GitLab 10.2.')
= form.fields_for :geo_node_key, geo_node.geo_node_key, include_id: !disable_key_edit do |fg|
.form-group.js-hide-if-geo-primary{ class: ('hidden' unless geo_node.secondary?) }
.form-group.js-ssh-key{ class: ('hidden' unless geo_node.secondary? && geo_node.clone_protocol == 'ssh') }
= fg.label :key, 'Public Key', class: 'control-label'
.col-sm-10
= fg.text_area :key, class: 'form-control thin_area', rows: 5, disabled: disable_key_edit
- unless disable_key_edit
%p.help-block
Paste the ssh public key used by the node you are adding. Read more about it
For SSH authentication, paste the public key used by the node you are adding. Read more about it
= link_to 'here', help_page_path('gitlab-geo/configuration.html', anchor: 'step-5-enabling-the-secondary-gitlab-node')
.form-group.js-hide-if-geo-primary{ class: ('hidden' unless geo_node.secondary?) }
......
---
title: Allow Geo repository sync over HTTPS
merge_request: 3341
author:
type: added
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class AddGeoNodeCloneProtocol < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_column_with_default :geo_nodes, :clone_protocol, :string, allow_null: false, default: 'ssh'
change_column_default :geo_nodes, :clone_protocol, 'http'
end
def down
remove_column :geo_nodes, :clone_protocol
end
end
......@@ -802,6 +802,7 @@ ActiveRecord::Schema.define(version: 20171017145932) do
t.string "clone_url_prefix"
t.integer "files_max_capacity", default: 10, null: false
t.integer "repos_max_capacity", default: 25, null: false
t.string "clone_protocol", default: "http", null: false
end
add_index "geo_nodes", ["access_key"], name: "index_geo_nodes_on_access_key", using: :btree
......
......@@ -12,6 +12,8 @@ and there is significant chance of data loss. For the latest updates, check the
We recommend you use it with at least GitLab Enterprise Edition 10.0 for
basic Geo features, or latest version for a better experience.
- You should make sure that all nodes run the same GitLab version.
- GitLab Geo requires PostgreSQL 9.6 and Git 2.9 in addition to GitLab's usual
[minimum requirements](../install/requirements.md)
GitLab Geo allows you to replicate your GitLab instance to other geographical
locations as a read-only fully operational version.
......@@ -54,7 +56,7 @@ The following diagram illustrates the underlying architecture of GitLab Geo:
![GitLab Geo architecture](img/geo-architecture.png)
[Source diagram](https://docs.google.com/drawings/d/1L44flo2Mxng928yAcHduaCJyGtKNEjk2WQkxaCU_cT8/edit)
[Source diagram](https://docs.google.com/drawings/d/1Abw0P_H0Ew1-2Lj_xPDRWP87clGIke-1fil7_KQqrtE/edit)
In this diagram, there is one Geo primary node and one secondary. The
secondary clones repositories via git over SSH. Attachments, LFS objects, and
......
......@@ -46,7 +46,7 @@ first two steps of the [Setup instructions](README.md#setup-instructions):
1. You have already installed on the secondary server the same version of
GitLab Enterprise Edition that is present on the primary server.
1. You have set up the database replication.
1. Your secondary node is allowed to communicate via HTTP/HTTPS and SSH with
1. Your secondary node is allowed to communicate via HTTP/HTTPS with
your primary node (make sure your firewall is not blocking that).
1. Your nodes must have an NTP service running to synchronize the clocks.
You can use different timezones, but the hour relative to UTC can't be more
......@@ -75,31 +75,7 @@ logins opened on all nodes as we will be moving back and forth.
This command will use your defined `external_url` in `gitlab.rb`
### Step 2. Updating the `known_hosts` file of the secondary nodes
1. SSH into the **secondary** node and login as root:
```
sudo -i
```
1. The secondary nodes need to know the SSH fingerprint of the primary node that
will be used for the Git clone/fetch operations. In order to add it to the
`known_hosts` file, run the following command and type `yes` when asked:
```
sudo -u git -H ssh git@<primary-node-url>
```
Replace `<primary-node-url>` with the FQDN of the primary node.
1. Verify that the fingerprint was added by checking `known_hosts`:
```
cat /var/opt/gitlab/.ssh/known_hosts
```
### Step 3. Copying the database encryption key
### Step 2. Copying the database encryption key
GitLab stores a unique encryption key in disk that we use to safely store
sensitive data in the database. Any secondary node must have the
......@@ -137,7 +113,7 @@ sensitive data in the database. Any secondary node must have the
gitlab-ctl reconfigure
```
### Step 4. Enabling hashed storage (from GitLab 10.0)
### Step 3. Enabling hashed storage (from GitLab 10.0)
1. Visit the **primary** node's **Admin Area ➔ Settings**
(`/admin/application_settings`) in your browser
......@@ -149,25 +125,12 @@ Using hashed storage significantly improves Geo replication - project and group
renames no longer require synchronization between nodes - so we recommend it is
used for all GitLab Geo installations.
### Step 5. Enabling the secondary GitLab node
1. SSH into the **secondary** node and login as root:
```
sudo -i
```
1. Get the contents of `id_rsa.pub` key that was pre-generated by Omnibus GitLab
and copy them:
```
sudo -u git cat /var/opt/gitlab/.ssh/id_rsa.pub
```
### Step 4. Enabling the secondary GitLab node
1. Visit the **primary** node's **Admin Area ➔ Geo Nodes** (`/admin/geo_nodes`)
in your browser.
1. Add the secondary node by providing its full URL and the public SSH key
you created previously. **Do NOT** check the box 'This is a primary node'.
1. Add the secondary node by providing its full URL. **Do NOT** check the box
'This is a primary node'.
1. Added in GitLab 9.5: Choose which namespaces should be replicated by the secondary node. Leave blank to replicate all. Read more in [selective replication](#selective-replication).
1. Click the **Add node** button.
1. Restart GitLab on the secondary:
......@@ -175,11 +138,14 @@ used for all GitLab Geo installations.
```
gitlab-ctl restart
```
---
After the **Add Node** button is pressed, the primary node will start to notify
changes to the secondary. Make sure the secondary instance is running and
accessible.
After the **Add Node** button is pressed, the secondary will start automatically
replicating missing data from the primary in a process known as backfill.
Meanwhile, the primary node will start to notify changes to the secondary, which
will act on those notifications immediately. Make sure the secondary instance is
running and accessible.
The two most obvious issues that replication can have here are:
......@@ -187,23 +153,15 @@ The two most obvious issues that replication can have here are:
1. Instance to instance notification not working. In that case, it can be
something of the following:
- You are using a custom certificate or custom CA (see the
[Troubleshooting](#troubleshooting) section)
[Troubleshooting](configuration.md#troubleshooting) section)
- Instance is firewalled (check your firewall rules)
### Step 6. Replicating the repositories data
Getting a new secondary Geo node up and running, will also require the
repositories data to be synced.
With GitLab 9.0 the syncing process starts automatically from the
secondary node after the **Add Node** button is pressed.
Currently, this is what is synced:
* Git repositories
* Wikis
* LFS objects
* Issue, merge request, and comment attachments
* Issue, merge request, snippet and comment attachments
* User, group, and project avatars
You can monitor the status of the syncing process on a secondary node
......@@ -217,39 +175,6 @@ repository shards you must duplicate the same configuration on the secondary.
Disabling a secondary node stops the syncing process.
With GitLab 8.14 this process is started manually from the primary node.
You can start the syncing process by clicking the "Backfill all repositories"
button on `Admin > Geo Nodes` screen.
On previous versions, you can use `rsync` for that:
Make sure `rsync` is installed in both primary and secondary servers and root
SSH access with a password is enabled. Otherwise, you can set up an SSH key-based
connection between the servers.
1. SSH into the **secondary** node and login as root:
```
sudo -i
```
1. Assuming `1.2.3.4` is the IP of the primary node, run the following command
to start the sync:
```bash
# For Omnibus installations
rsync -guavrP root@1.2.3.4:/var/opt/gitlab/git-data/repositories/ /var/opt/gitlab/git-data/repositories/
gitlab-ctl reconfigure # to fix directory permissions
```
If this step is not followed, the secondary node will eventually clone and
fetch every missing repository as they are updated with new commits on the
primary node, so syncing the repositories beforehand will buy you some time.
While active repositories will be eventually replicated, if you don't rsync,
the files, any archived/inactive repositories will not get in the secondary node
as Geo doesn't run any routine task to look for missing repositories.
## Next steps
Your nodes should now be ready to use. You can login to the secondary node
......@@ -280,19 +205,43 @@ groups to be replicated.
## Adding another secondary Geo node
To add another Geo node in an already Geo configured infrastructure, just follow
[the steps starting from step 2](#step-2-updating-the-known_hosts-file-of-the-secondary-nodes).
[the steps starting from step 2](#step-2-copying-the-database-encryption-key)
Just omit the first step that sets up the primary node.
## Additional information for the SSH key pairs
## Replicating wikis and repositories over SSH
In GitLab 10.2, replicating repositories and wikis over SSH was deprecated.
Support for this option will be removed within a few releases, but if you need
to add a new secondary in the short term, you can follow these instructions:
1. SSH into the **secondary** node and login as root:
```bash
sudo -i
```
1. Add the primary's SSH key fingerprint to the `known_hosts` file.
```bash
sudo -u git -H ssh git@<primary-node-url>
```
Replace `<primary-node-url>` with the FQDN of the primary node. You should
manually check the displayed fingerprint against a trusted record of the
expected value before accepting it!
When adding a new **secondary** Geo node, you must provide an SSH public key for
the system user that your GitLab instance runs as (unless changed, should be the
user `git`). This user will act as a "normal user" who fetches from the primary
Geo node.
1. Generate a *passphraseless* SSH keypair for the `git` user, and capture the
public component:
```bash
test -e ~git/.ssh/id_rsa || sudo -u git -H ssh-keygen -q -t rsa -b 4096 -f ~git/.ssh/id_rsa
cat ~git/.ssh/id_rsa.pub
```
Omnibus automatically generates `~git/.ssh/id_rsa` and `~git/.ssh/id_rsa.pub`
files on secondary Geo nodes. Primaries do not need these files, and you should
not create them manually.
Follow the steps above to set up the new Geo node. When you reach
[Step 4: Enabling the secondary GitLab node](#step-4-enabling-the-secondary-gitlab-node)
select "SSH (deprecated)" instead of "HTTP/HTTPS", and populate the "Public Key"
with the output of the previous command (beginning `ssh-rsa AAAA...`).
### Upgrading Geo
......
......@@ -46,7 +46,7 @@ first two steps of the [Setup instructions](README.md#setup-instructions):
1. You have already installed on the secondary server the same version of
GitLab Enterprise Edition that is present on the primary server.
1. You have set up the database replication.
1. Your secondary node is allowed to communicate via HTTP/HTTPS and SSH with
1. Your secondary node is allowed to communicate via HTTP/HTTPS with
your primary node (make sure your firewall is not blocking that).
1. Your nodes must have an NTP service running to synchronize the clocks.
You can use different timezones, but the hour relative to UTC can't be more
......@@ -73,31 +73,7 @@ logins opened on all nodes as we will be moving back and forth.
bundle exec rake geo:set_primary_node
```
### Step 2. Updating the `known_hosts` file of the secondary nodes
1. SSH into the **secondary** node and login as root:
```
sudo -i
```
1. The secondary nodes need to know the SSH fingerprint of the primary node that
will be used for the Git clone/fetch operations. In order to add it to the
`known_hosts` file, run the following command and type `yes` when asked:
```
sudo -u git -H ssh git@<primary-node-url>
```
Replace `<primary-node-url>` with the FQDN of the primary node.
1. Verify that the fingerprint was added by checking `known_hosts`:
```
cat /home/git/.ssh/known_hosts
```
### Step 3. Copying the database encryption key
### Step 2. Copying the database encryption key
GitLab stores a unique encryption key in disk that we use to safely store
sensitive data in the database. Any secondary node must have the
......@@ -130,7 +106,7 @@ sensitive data in the database. Any secondary node must have the
1. Save and close the file.
### Step 4. Enabling hashed storage (from GitLab 10.0)
### Step 3. Enabling hashed storage (from GitLab 10.0)
1. Visit the **primary** node's **Admin Area ➔ Settings**
(`/admin/application_settings`) in your browser
......@@ -143,41 +119,27 @@ renames no longer require synchronization between nodes - so we recommend it is
used for all GitLab Geo installations.
### Step 5. Enabling the secondary GitLab node
1. SSH into the **secondary** node and login as root:
```
sudo -i
```
1. Create a new SSH key pair for the secondary node. Choose the default location
and leave the password blank by hitting 'Enter' three times:
```bash
sudo -u git -H ssh-keygen -b 4096 -C 'Secondary GitLab Geo node'
```
Read more in [additional info for SSH key pairs](#additional-information-for-the-ssh-key-pairs).
1. Get the contents of `id_rsa.pub` the was just created:
```
sudo -u git cat /home/git/.ssh/id_rsa.pub
```
### Step 4. Enabling the secondary GitLab node
1. Visit the **primary** node's **Admin Area ➔ Geo Nodes** (`/admin/geo_nodes`)
in your browser.
1. Add the secondary node by providing its full URL and the public SSH key
you created previously. **Do NOT** check the box 'This is a primary node'.
1. Add the secondary node by providing its full URL. **Do NOT** check the box
'This is a primary node'.
1. Added in GitLab 9.5: Choose which namespaces should be replicated by the secondary node. Leave blank to replicate all. Read more in [selective replication](#selective-replication).
1. Click the **Add node** button.
1. Restart GitLab on the secondary:
```
gitlab-ctl restart
```
---
After the **Add Node** button is pressed, the primary node will start to notify
changes to the secondary. Make sure the secondary instance is running and
accessible.
After the **Add Node** button is pressed, the secondary will start automatically
replicating missing data from the primary in a process known as backfill.
Meanwhile, the primary node will start to notify changes to the secondary, which
will act on those notifications immediately. Make sure the secondary instance is
running and accessible.
The two most obvious issues that replication can have here are:
......@@ -188,20 +150,12 @@ The two most obvious issues that replication can have here are:
[Troubleshooting](configuration.md#troubleshooting) section)
- Instance is firewalled (check your firewall rules)
### Step 6. Replicating the repositories data
Getting a new secondary Geo node up and running, will also require the
repositories data to be synced.
With GitLab 9.0 the syncing process starts automatically from the
secondary node after the **Add Node** button is pressed.
Currently, this is what is synced:
* Git repositories
* Wikis
* LFS objects
* Issue, merge request, and comment attachments
* Issue, merge request, snippet and comment attachments
* User, group, and project avatars
You can monitor the status of the syncing process on a secondary node
......@@ -215,39 +169,6 @@ repository shards you must duplicate the same configuration on the secondary.
Disabling a secondary node stops the syncing process.
With GitLab 8.14 this process is started manually from the primary node.
You can start the syncing process by clicking the "Backfill all repositories"
button on `Admin > Geo Nodes` screen.
On previous versions, you can use `rsync` for that:
Make sure `rsync` is installed in both primary and secondary servers and root
SSH access with a password is enabled. Otherwise, you can set up an SSH key-based
connection between the servers.
1. SSH into the **secondary** node and login as root:
```
sudo -i
```
1. Assuming `1.2.3.4` is the IP of the primary node, run the following command
to start the sync:
```bash
# Installations from source
rsync -guavrP root@1.2.3.4:/home/git/repositories/ /home/git/repositories/
chmod ug+rwX,o-rwx /home/git/repositories
```
If this step is not followed, the secondary node will eventually clone and
fetch every missing repository as they are updated with new commits on the
primary node, so syncing the repositories beforehand will buy you some time.
While active repositories will be eventually replicated, if you don't rsync,
the files, any archived/inactive repositories will not get in the secondary node
as Geo doesn't run any routine task to look for missing repositories.
## Next steps
Your nodes should now be ready to use. You can login to the secondary node
......@@ -267,12 +188,12 @@ Read [Selective replication](configuration.md#selective-replication).
## Adding another secondary Geo node
To add another Geo node in an already Geo configured infrastructure, just follow
[the steps starting from step 2](#step-2-updating-the-known_hosts-file-of-the-secondary-nodes).
[the steps starting from step 2](#step-2-copying-the-database-encryption-key).
Just omit the first step that sets up the primary node.
## Additional information for the SSH key pairs
## Replicating wikis and repositories over SSH
Read [Additional information for the SSH key pairs](configuration.md#additional-information-for-the-ssh-key-pairs).
Read [Replicating wikis and repositories over SSH](configuration.md#replicating-wikis-and-repositories-over-ssh).
## Troubleshooting
......
doc/gitlab-geo/img/geo-architecture.png

59.3 KB | W: | H:

doc/gitlab-geo/img/geo-architecture.png

59.4 KB | W: | H:

doc/gitlab-geo/img/geo-architecture.png
doc/gitlab-geo/img/geo-architecture.png
doc/gitlab-geo/img/geo-architecture.png
doc/gitlab-geo/img/geo-architecture.png
  • 2-up
  • Swipe
  • Onion skin
......@@ -14,6 +14,31 @@ all you need to do is update GitLab itself:
the tracking database is enabled.
1. [Test](#check-status-after-updating) primary and secondary nodes, and check version in each.
## Upgrading to GitLab 10.2
Support for replicating repositories and wikis over HTTP/HTTPS has been added.
Replicating over SSH has been deprecated, and support for this option will be
removed in a future release.
To switch to HTTP/HTTPS replication, log into the primary node as an admin and visit
**Admin Area ➔ Geo Nodes** (`/admin/geo_nodes`). For each secondary listed,
press the "Edit" button, change the "Repository cloning" setting from
"SSH (deprecated)" to "HTTP/HTTPS", and press "Save changes". This should take
effect immediately.
Any new secondaries should be created using HTTP/HTTPS replication - this is the
default setting.
After you've verified that HTTP/HTTPS replication is working, you should remove
the now-unused SSH keys from your secondaries, as they may cause problems if the
secondary if ever promoted to a primary:
1. **[secondary]** Login to **all** your secondary nodes and run:
```ruby
sudo -u git -H rm ~git/.ssh/id_rsa ~git/.ssh/id_rsa.pub
```
## Upgrading to GitLab 10.1
[Hashed storage](../administration/repository_storage_types.md) was introduced
......
......@@ -86,6 +86,7 @@ class Admin::GeoNodesController < Admin::ApplicationController
:namespace_ids,
:repos_max_capacity,
:files_max_capacity,
:clone_protocol,
geo_node_key_attributes: [:key]
)
end
......
module EE
module Projects
module GitHttpController
def render_ok
raise NotImplementedError.new unless defined?(super)
set_workhorse_internal_api_content_type
render json: ::Gitlab::Workhorse.git_http_ok(repository, wiki?, user, action_name, show_all_refs: geo_request?)
end
private
def geo_request?
::Gitlab::Geo::JwtRequestDecoder.geo_auth_attempt?(request.headers['Authorization'])
end
def geo?
authentication_result.geo?(project)
end
def access_actor
raise NotImplementedError.new unless defined?(super)
return :geo if geo?
super
end
def authenticate_user
raise NotImplementedError.new unless defined?(super)
return super unless geo_request?
payload = ::Gitlab::Geo::JwtRequestDecoder.new(request.headers['Authorization']).decode
if payload
@authentication_result = ::Gitlab::Auth::Result.new(nil, project, :geo, [:download_code])
return # grant access
end
render_bad_geo_auth('Bad token')
rescue ::Gitlab::Geo::InvalidDecryptionKeyError
render_bad_geo_auth("Invalid decryption key")
end
def render_bad_geo_auth(message)
render plain: "Geo JWT authentication failed: #{message}", status: 401
end
end
end
end
......@@ -6,6 +6,15 @@ module EE
module Repository
extend ActiveSupport::Concern
# Transiently sets a configuration variable
def with_config(values = {})
values.each { |k, v| rugged.config[k] = v }
yield
ensure
values.keys.each { |key| rugged.config.delete(key) }
end
# Runs code after a repository has been synced.
def after_sync
expire_all_method_caches
......
module EE
module Gitlab
module Auth
module Result
def success?
raise NotImplementedError.new unless defined?(super)
type == :geo || super
end
def geo?(for_project)
type == :geo &&
project &&
project == for_project
end
end
end
end
end
......@@ -13,6 +13,7 @@ module EE
raise NotImplementedError.new unless defined?(super)
return geo_node_key.active? if geo_node_key?
return true if actor == :geo
super
end
......@@ -22,7 +23,7 @@ module EE
def user
raise NotImplementedError.new unless defined?(super)
return nil if actor.is_a?(::GeoNodeKey)
return nil if geo?
super
end
......@@ -32,7 +33,7 @@ module EE
def check_download_access!
raise NotImplementedError.new unless defined?(super)
return if geo_node_key?
return if geo?
super
end
......@@ -40,7 +41,7 @@ module EE
def check_active_user!
raise NotImplementedError.new unless defined?(super)
return if geo_node_key?
return if geo?
super
end
......@@ -58,6 +59,10 @@ module EE
def geo_node_key?
actor.is_a?(::GeoNodeKey)
end
def geo?
geo_node_key? || actor == :geo
end
end
end
end
module Gitlab
module Auth
Result = Struct.new(:actor, :project, :type, :authentication_abilities) do
prepend ::EE::Gitlab::Auth::Result
def ci?(for_project)
type == :ci &&
project &&
......
......@@ -12,10 +12,14 @@ module Gitlab
# Raises GeoNodeNotFoundError if current node is not a Geo node
def headers
{
'Authorization' => geo_auth_token(request_data)
'Authorization' => authorization
}
end
def authorization
geo_auth_token(request_data)
end
private
def geo_auth_token(message)
......
......@@ -5,6 +5,11 @@ module Gitlab
class JwtRequestDecoder
IAT_LEEWAY = 60.seconds.to_i
def self.geo_auth_attempt?(header)
token_type, _ = header&.split(' ', 2)
token_type == ::Gitlab::Geo::BaseRequest::GITLAB_GEO_AUTH_TOKEN_TYPE
end
attr_reader :auth_header
def initialize(auth_header)
......
module SystemCheck
module Geo
class GitVersionCheck < ::SystemCheck::App::GitVersionCheck
set_name -> { "Git version >= #{self.required_version} ?" }
set_check_pass -> { "yes (#{self.current_version})" }
def self.required_version
@required_version ||= Gitlab::VersionInfo.new(2, 9, 5)
end
end
end
end
......@@ -458,6 +458,7 @@ namespace :gitlab do
checks = [
SystemCheck::Geo::LicenseCheck,
SystemCheck::Geo::EnabledCheck,
SystemCheck::Geo::GitVersionCheck,
SystemCheck::Geo::GeoDatabaseConfiguredCheck,
SystemCheck::Geo::DatabaseReplicationCheck,
SystemCheck::Geo::HttpConnectionCheck,
......
......@@ -114,8 +114,15 @@ describe Admin::GeoNodesController, :postgresql do
end
describe '#update' do
let(:geo_node_attributes) { { url: 'http://example.com', geo_node_key_attributes: attributes_for(:key) } }
let(:geo_node) { create(:geo_node) }
let(:geo_node_attributes) do
{
url: 'http://example.com',
clone_protocol: 'ssh',
geo_node_key_attributes: attributes_for(:key)
}
end
let(:geo_node) { create(:geo_node, :ssh) }
let!(:original_fingerprint) { geo_node.geo_node_key.fingerprint }
def go
......@@ -144,6 +151,19 @@ describe Admin::GeoNodesController, :postgresql do
expect(geo_node.geo_node_key.fingerprint).to eq(original_fingerprint)
end
context 'changing clone protocol' do
let(:geo_node_attributes) { { clone_protocol: 'http' } }
it 'changes the protocol without removing the key' do
go
geo_node.reload
expect(geo_node.clone_protocol).to eq('http')
expect(geo_node.geo_node_key.fingerprint).to eq(original_fingerprint)
end
end
it 'delegates the update of the Geo node to Geo::NodeUpdateService' do
expect_any_instance_of(Geo::NodeUpdateService).to receive(:execute).once
......
......@@ -6,7 +6,7 @@ describe API::Internal do
describe "POST /internal/allowed", :clean_gitlab_redis_shared_state do
context 'Geo Node' do
let(:geo_node) { create(:geo_node) }
let(:geo_node) { create(:geo_node, :ssh) }
it 'recognizes the Geo Node' do
post(
......
require 'spec_helper'
describe "Git HTTP requests (Geo)" do
include ::EE::GeoHelpers
include GitHttpHelpers
include WorkhorseHelpers
set(:project) { create(:project, :repository, :private) }
set(:primary) { create(:geo_node, :primary) }
set(:secondary) { create(:geo_node) }
# Ensure the token always comes from the real time of the request
let!(:auth_token) { Gitlab::Geo::BaseRequest.new.authorization }
before do
stub_licensed_features(geo: true)
stub_current_geo_node(secondary)
end
shared_examples_for 'Geo sync request' do
subject do
make_request
response
end
context 'valid Geo JWT token' do
let(:env) { valid_geo_env }
it 'returns an OK response' do
is_expected.to have_gitlab_http_status(:ok)
expect(response.content_type).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE)
expect(json_response).to include('ShowAllRefs' => true)
end
end
context 'post-dated Geo JWT token' do
let(:env) { valid_geo_env }
it { travel_to(2.minutes.ago) { is_expected.to have_gitlab_http_status(:unauthorized) } }
end
xcontext 'expired Geo JWT token' do
let(:env) { valid_geo_env }
it { travel_to(Time.now + 2.minutes) { is_expected.to have_gitlab_http_status(:unauthorized) } }
end
context 'invalid Geo JWT token' do
let(:env) { geo_env("GL-Geo xxyyzz:12345") }
it { is_expected.to have_gitlab_http_status(:unauthorized) }
end
context 'no Geo JWT token' do
let(:env) { workhorse_internal_api_request_header }
it { is_expected.to have_gitlab_http_status(:unauthorized) }
end
context 'Geo is unlicensed' do
let(:env) { valid_geo_env }
before do
stub_licensed_features(geo: false)
end
it { is_expected.to have_gitlab_http_status(:forbidden) }
end
end
describe 'GET info_refs' do
def make_request
get "/#{project.full_path}.git/info/refs", { service: 'git-upload-pack' }, env
end
it_behaves_like 'Geo sync request'
end
describe 'POST upload_pack' do
def make_request
post "/#{project.full_path}.git/git-upload-pack", {}, env
end
it_behaves_like 'Geo sync request'
end
def valid_geo_env
geo_env(auth_token)
end
def geo_env(authorization)
env = workhorse_internal_api_request_header
env['HTTP_AUTHORIZATION'] = authorization
env
end
end
......@@ -2,12 +2,15 @@ FactoryGirl.define do
factory :geo_node do
host { Gitlab.config.gitlab.host }
sequence(:port) {|n| n}
association :geo_node_key
trait :ssh do
clone_protocol 'ssh'
association :geo_node_key
end
trait :primary do
primary true
port { Gitlab.config.gitlab.port }
geo_node_key nil
end
end
end
......@@ -355,19 +355,26 @@ describe Gitlab::GitAccess do
let(:key) { build(:geo_node_key, geo_node: geo_node) }
let(:actor) { key }
context 'assigned to primary geo node' do
let(:geo_node) { build(:geo_node, primary: true) }
context 'assigned to ssh primary geo node' do
let(:geo_node) { build(:geo_node, :ssh, primary: true) }
it { expect { pull_access_check }.to raise_not_found }
it { expect { push_access_check }.to raise_not_found }
end
context 'assigned to secondary geo node' do
let(:geo_node) { build(:geo_node, primary: false) }
context 'assigned to ssh secondary geo node' do
let(:geo_node) { build(:geo_node, :ssh, primary: false) }
it { expect { pull_access_check }.not_to raise_error }
it { expect { push_access_check }.to raise_unauthorized(described_class::ERROR_MESSAGES[:upload]) }
end
context 'assigned to http secondary geo node' do
let(:geo_node) { build(:geo_node, primary: false) }
it { expect { pull_access_check }.to raise_not_found }
it { expect { push_access_check }.to raise_not_found }
end
end
describe 'build authentication_abilities permissions' do
......
require 'spec_helper'
describe SystemCheck::Geo::GitVersionCheck do
describe '#check?' do
subject { described_class.new.check? }
where(:git_version, :result) do
[
['2.8.99', false],
['2.9.0', false],
['2.9.4', false],
['2.9.5', true],
['2.9.55', true],
['10.0.0', true]
]
end
with_them do
before do
stub_git_version(git_version)
end
it { is_expected.to eq(result) }
end
end
def stub_git_version(version)
allow(described_class).to receive(:current_version) { Gitlab::VersionInfo.parse(version) }
end
end
require 'spec_helper'
describe GeoNodeKey do
let(:geo_node) { create(:geo_node) }
let(:geo_node_key) { create(:geo_node_key, geo_nodes: [geo_node]) }
describe 'Associations' do
it { is_expected.to have_one(:geo_node) }
end
describe '#active?' do
let(:geo_node) { create(:geo_node, :ssh) }
let(:geo_node_key) { geo_node.geo_node_key }
subject { geo_node_key.active? }
it 'returns true for a secondary SSH Geo node' do
is_expected.to be_truthy
end
it 'returns false for a primary SSH Geo node' do
geo_node.primary = true
is_expected.to be_falsy
end
it 'returns false for a secondary HTTP Geo node' do
geo_node.clone_protocol = 'http'
is_expected.to be_falsy
end
end
end
......@@ -23,7 +23,10 @@ describe GeoNode, type: :model do
end
context 'validations' do
it { expect(new_node).to validate_presence_of(:geo_node_key) }
let(:ssh_node) { build(:geo_node, :ssh) }
it { expect(ssh_node).to validate_presence_of(:geo_node_key) }
it { expect(new_node).not_to validate_presence_of(:geo_node_key) }
it { expect(new_primary_node).not_to validate_presence_of(:geo_node_key) }
end
......@@ -38,6 +41,7 @@ describe GeoNode, type: :model do
:primary | false
:repos_max_capacity | 25
:files_max_capacity | 10
:clone_protocol | 'http'
end
with_them do
......@@ -69,18 +73,27 @@ describe GeoNode, type: :model do
context 'on initialize' do
it 'initializes a corresponding key' do
expect(new_node.geo_node_key).to be_present
expect(empty_node.geo_node_key).to be_present
end
it 'is valid when required attributes are present' do
new_node.clone_protocol = 'ssh'
new_node.geo_node_key_attributes = geo_node_key_attributes
expect(new_node).to be_valid
end
end
context 'on create' do
it 'saves a corresponding key' do
expect(node.geo_node_key).to be_persisted
context 'SSH node' do
let(:ssh_node) { create(:geo_node, :ssh) }
it 'saves a corresponding key' do
expect(ssh_node.geo_node_key).to be_persisted
end
end
it 'does not save a key' do
expect(node.geo_node_key).to be_nil
end
it 'saves a corresponding oauth application if it is a secondary node' do
......@@ -336,11 +349,20 @@ describe GeoNode, type: :model do
end
context 'secondary node' do
it 'is automatically set' do
it 'is not set for HTTP' do
node = build(:geo_node, url: 'http://example.com/')
expect(node.geo_node_key).to be_present
expect(node.geo_node_key.title).not_to include('example.com')
node.save!
expect(node.geo_node_key).to be_nil
end
it 'is automatically set for SSH' do
node = build(:geo_node, :ssh, url: 'http://example.com/')
expect(node.geo_node_key).to be_present
node.save!
......
......@@ -3,35 +3,32 @@ require 'spec_helper'
describe Geo::NodeCreateService do
describe '#execute' do
it 'creates a new node with valid params' do
params = { url: 'http://example.com', geo_node_key_attributes: attributes_for(:key) }
service = described_class.new(params)
service = described_class.new(url: 'http://example.com')
expect { service.execute }.to change(GeoNode, :count).by(1)
end
it 'does not create a node with invalid params' do
service = described_class.new({ url: 'http://example.com' })
service = described_class.new(url: 'ftp://example.com')
expect { service.execute }.not_to change(GeoNode, :count)
end
it 'returns true when creation succeeds' do
params = { url: 'http://example.com', geo_node_key_attributes: attributes_for(:key) }
service = described_class.new(params)
service = described_class.new(url: 'http://example.com')
expect(service.execute).to eq true
end
it 'returns false when creation fails' do
params = { url: 'http://example.com' }
service = described_class.new(params)
service = described_class.new(url: 'ftp://example.com')
expect(service.execute).to eq false
end
it 'parses the namespace_ids when node have namespace restrictions' do
groups = create_list(:group, 2)
params = { url: 'http://example.com', geo_node_key_attributes: attributes_for(:key), namespace_ids: groups.map(&:id).join(',') }
params = { url: 'http://example.com', namespace_ids: groups.map(&:id).join(',') }
service = described_class.new(params)
service.execute
......
......@@ -8,15 +8,16 @@ describe Geo::NodeUpdateService do
describe '#execute' do
it 'updates the node without changing the key' do
original_fingerprint = geo_node.geo_node_key.fingerprint
ssh_node = create(:geo_node, :ssh)
original_fingerprint = ssh_node.geo_node_key.fingerprint
params = { url: 'http://example.com', geo_node_key_attributes: attributes_for(:key) }
service = described_class.new(geo_node, params)
service = described_class.new(ssh_node, params)
service.execute
geo_node.reload
expect(geo_node.url.chomp('/')).to eq(params[:url])
expect(geo_node.geo_node_key.fingerprint).to eq(original_fingerprint)
ssh_node.reload
expect(ssh_node.url.chomp('/')).to eq(params[:url])
expect(ssh_node.geo_node_key.fingerprint).to eq(original_fingerprint)
end
it 'returns true when update succeeds' do
......
require 'spec_helper'
RSpec.describe Geo::RepositorySyncService do
let!(:primary) { create(:geo_node, :primary, host: 'primary-geo-node') }
include ::EE::GeoHelpers
set(:primary) { create(:geo_node, :primary, host: 'primary-geo-node', relative_url_root: '/gitlab') }
set(:secondary) { create(:geo_node) }
let(:lease) { double(try_obtain: true) }
subject { described_class.new(project) }
before do
stub_current_geo_node(secondary)
end
it_behaves_like 'geo base sync execution'
describe '#execute' do
let(:project) { create(:project_empty_repo) }
let(:repository) { project.repository }
let(:url_to_repo) { "#{primary.clone_url_prefix}#{project.full_path}.git" }
let(:url_to_repo) { "#{primary.url}/#{project.full_path}.git" }
before do
allow(Gitlab::ExclusiveLease).to receive(:new)
......@@ -22,7 +30,8 @@ RSpec.describe Geo::RepositorySyncService do
.and_return(true)
end
it 'fetches project repository' do
it 'fetches project repository with JWT credentials' do
expect(repository).to receive(:with_config).with("http.#{url_to_repo}.extraHeader" => anything).and_call_original
expect(repository).to receive(:fetch_geo_mirror).with(url_to_repo).once
subject.execute
......@@ -101,7 +110,6 @@ RSpec.describe Geo::RepositorySyncService do
context 'when repository sync fail' do
let(:registry) { Geo::ProjectRegistry.find_by(project_id: project.id) }
let(:url_to_repo) { "#{primary.clone_url_prefix}#{project.full_path}.git" }
before do
allow(repository).to receive(:fetch_geo_mirror).with(url_to_repo) { raise Gitlab::Shell::Error }
......@@ -118,5 +126,21 @@ RSpec.describe Geo::RepositorySyncService do
end
end
end
context 'secondary replicates over SSH' do
set(:ssh_secondary) { create(:geo_node, :ssh) }
let(:url_to_repo) { "#{primary.clone_url_prefix}/#{project.full_path}.git" }
before do
stub_current_geo_node(ssh_secondary)
end
it 'fetches wiki repository over SSH' do
expect(repository).to receive(:fetch_geo_mirror).with(url_to_repo).once
subject.execute
end
end
end
end
require 'spec_helper'
RSpec.describe Geo::WikiSyncService do
let!(:primary) { create(:geo_node, :primary, host: 'primary-geo-node') }
include ::EE::GeoHelpers
set(:primary) { create(:geo_node, :primary, host: 'primary-geo-node', relative_url_root: '/gitlab') }
set(:secondary) { create(:geo_node) }
let(:lease) { double(try_obtain: true) }
subject { described_class.new(project) }
before do
stub_current_geo_node(secondary)
end
it_behaves_like 'geo base sync execution'
describe '#execute' do
let(:project) { create(:project_empty_repo) }
let(:repository) { project.wiki.repository }
let(:url_to_repo) { "#{primary.clone_url_prefix}#{project.full_path}.wiki.git" }
let(:url_to_repo) { "#{primary.url}/#{project.full_path}.wiki.git" }
before do
allow(Gitlab::ExclusiveLease).to receive(:new)
......@@ -22,7 +30,8 @@ RSpec.describe Geo::WikiSyncService do
.and_return(true)
end
it 'fetches wiki repository' do
it 'fetches wiki repository with JWT credentials' do
expect(repository).to receive(:with_config).with("http.#{url_to_repo}.extraHeader" => anything).and_call_original
expect(repository).to receive(:fetch_geo_mirror).with(url_to_repo).once
subject.execute
......@@ -107,5 +116,21 @@ RSpec.describe Geo::WikiSyncService do
end
end
end
context 'secondary replicates over SSH' do
set(:ssh_secondary) { create(:geo_node, :ssh) }
let(:url_to_repo) { "#{primary.clone_url_prefix}/#{project.full_path}.wiki.git" }
before do
stub_current_geo_node(ssh_secondary)
end
it 'fetches wiki repository over SSH' do
expect(repository).to receive(:fetch_geo_mirror).with(url_to_repo).once
subject.execute
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment