Commit 8b56fbd9 authored by Valery Sizov's avatar Valery Sizov

Merge branch 'master' of gitlab.com:gitlab-org/gitlab-ee into ce_upstream

parents 22dd32b5 97e7080d
......@@ -5,6 +5,7 @@ v 8.8.0 (unreleased)
v 8.7.0 (unreleased)
- The number of InfluxDB points stored per UDP packet can now be configured
- Fix error when cross-project label reference used with non-existent project
v 8.7.0
- Transactions for /internal/allowed now have an "action" tag set
- Method instrumentation now uses Module#prepend instead of aliasing methods
- Repository.clean_old_archives is now instrumented
......
Please view this file on the master branch, on stable branches it's out of date.
v 8.7.0 (unreleased)
v 8.7.0
- Update GitLab Pages to 0.2.1: support user-defined 404 pages
- Refactor group sync to pull access level logic to its own class. !306
- [Elastic] Stabilize database indexer if database is inconsistent
- Add ability to sync to remote mirrors. !249
- GitLab Geo: Many replication improvements and fixes !354
v 8.6.7
- No EE-specific changes
......
class Admin::GeoNodesController < Admin::ApplicationController
before_action :check_license
def index
@nodes = GeoNode.all
@node = GeoNode.new
......@@ -41,4 +43,11 @@ class Admin::GeoNodesController < Admin::ApplicationController
def geo_node_params
params.require(:geo_node).permit(:url, :primary, geo_node_key_attributes: [:key])
end
def check_license
unless Gitlab::Geo.license_allows?
flash[:alert] = 'You need a diferent license to enable Geo replication'
redirect_to admin_license_path
end
end
end
module Geo
class MoveRepositoryService
include Gitlab::ShellAdapter
attr_reader :id, :name, :old_path_with_namespace, :new_path_with_namespace
def initialize(id, name, old_path_with_namespace, new_path_with_namespace)
@id = id
@name = name
@old_path_with_namespace = old_path_with_namespace
@new_path_with_namespace = new_path_with_namespace
end
def execute
project = Project.find(id)
project.expire_caches_before_rename(old_path_with_namespace)
# Make sure target directory exists (used when transfering repositories)
project.namespace.ensure_dir_exist
if gitlab_shell.mv_repository(old_path_with_namespace, new_path_with_namespace)
# If repository moved successfully we need to send update instructions to users.
# However we cannot allow rollback since we moved repository
# So we basically we mute exceptions in next actions
begin
gitlab_shell.mv_repository("#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
rescue
# Returning false does not rollback after_* transaction but gives
# us information about failing some of tasks
false
end
else
# if we cannot move namespace directory we should rollback
# db changes in order to prevent out of sync between db and fs
raise Exception.new('repository cannot be renamed')
end
end
end
end
module Geo
class ScheduleRepoDestroyService
attr_reader :id, :name, :path_with_namespace
def initialize(params)
@id = params['project_id']
@name = params['name']
@path_with_namespace = params['path_with_namespace']
end
def execute
GeoRepositoryDestroyWorker.perform_async(id, name, path_with_namespace)
end
end
end
module Geo
class ScheduleRepoRenameService
attr_reader :id, :name, :old_path_with_namespace, :path_with_namespace
def initialize(params)
@id = params['project_id']
@name = params['name']
@old_path_with_namespace = params['old_path_with_namespace']
@path_with_namespace = params['path_with_namespace']
end
def execute
GeoRepositoryMoveWorker.perform_async(id, name, old_path_with_namespace, path_with_namespace)
end
end
end
......@@ -17,9 +17,6 @@ module Projects
project.team.truncate
repo_path = project.path_with_namespace
wiki_path = repo_path + '.wiki'
# Flush the cache for both repositories. This has to be done _before_
# removing the physical repositories as some expiration code depends on
# Git data (e.g. a list of branch names).
......@@ -27,14 +24,7 @@ module Projects
Project.transaction do
project.destroy!
unless remove_repository(repo_path)
raise_error('Failed to remove project repository. Please try again or contact administrator')
end
unless remove_repository(wiki_path)
raise_error('Failed to remove wiki repository. Please try again or contact administrator')
end
trash_repositories!
end
log_info("Project \"#{project.name}\" was removed")
......@@ -42,8 +32,39 @@ module Projects
true
end
# Removes physical repository in a Geo replicated secondary node
# There is no need to do any database operation as it will be
# replicated by itself.
def geo_replicate
# Flush the cache for both repositories. This has to be done _before_
# removing the physical repositories as some expiration code depends on
# Git data (e.g. a list of branch names).
flush_caches(project, wiki_path)
trash_repositories!
log_info("Project \"#{project.name}\" was removed")
end
private
def repo_path
project.path_with_namespace
end
def wiki_path
repo_path + '.wiki'
end
def trash_repositories!
unless remove_repository(repo_path)
raise_error('Failed to remove project repository. Please try again or contact administrator')
end
unless remove_repository(wiki_path)
raise_error('Failed to remove wiki repository. Please try again or contact administrator')
end
end
def remove_repository(path)
# Skip repository removal. We use this flag when remove user or group
return true if params[:skip_repo] == true
......
class GeoRepositoryDestroyWorker
include Sidekiq::Worker
sidekiq_options queue: :default
def perform(id, name, path_with_namespace)
# We don't have access to the original model anymore, so we are
# rebuilding only what our service class requires
project = FakeProject.new(id, name, path_with_namespace)
::Projects::DestroyService.new(project, nil).geo_replicate
end
FakeProject = Struct.new(:id, :name, :path_with_namespace) do
def repository
@repository ||= Repository.new(path_with_namespace, self)
end
end
end
class GeoRepositoryMoveWorker
include Sidekiq::Worker
sidekiq_options queue: :default
def perform(id, name, old_path_with_namespace, new_path_with_namespace)
Geo::MoveRepositoryService.new(id, name, old_path_with_namespace, new_path_with_namespace).execute
end
end
......@@ -2,6 +2,7 @@ class HistoricalDataWorker
include Sidekiq::Worker
def perform
return if Gitlab::Geo.secondary?
HistoricalData.track!
end
end
......@@ -4,6 +4,7 @@ class StuckCiBuildsWorker
BUILD_STUCK_TIMEOUT = 1.day
def perform
return if Gitlab::Geo.secondary?
Rails.logger.info 'Cleaning stuck builds'
builds = Ci::Build.running_or_pending.where('updated_at < ?', BUILD_STUCK_TIMEOUT.ago)
......
......@@ -54,7 +54,7 @@ Geo instances. Follow the steps below in the order that they appear:
1. Install GitLab Enterprise Edition on the server that will serve as the
secondary Geo node
1. [Setup a database replication](./database.md) in `master <-> slave` topology
1. [Setup a database replication](./database.md) in `primary <-> secondary (read-only)` topology
1. [Configure GitLab](configuration.md) and set the primary and secondary nodes
After you set up the database replication and configure the GitLab Geo nodes,
there are a few things to consider:
......
......@@ -14,7 +14,8 @@ complete the process.
- [Create SSH key pairs for Geo nodes](#create-ssh-key-pairs-for-geo-nodes)
- [Primary Node GitLab setup](#primary-node-gitlab-setup)
- [Secondary Node GitLab setup](#secondary-node-gitlab-setup)
- [Authorized keys regeneration](#authorized-keys-regeneration)
- [Database Encryptation Key](#database-encryptation-key)
- [Authorized keys regeneration](#authorized-keys-regeneration)
<!-- END doctoc generated TOC please keep comment here to allow auto update -->
......@@ -81,8 +82,8 @@ Host example.com # The FQDN of the primary Geo node
## Primary Node GitLab setup
>**Note:**
You will need to setup your database into a **Master <-> Slave** replication
topology, and your Primary node should always point to a database's Master
You will need to setup your database into a **Primary <-> Secondary (read-only)** replication
topology, and your Primary node should always point to a database's Primary
instance. If you haven't done that already, read [database replication](./database.md).
Go to the server that you chose to be your primary, and visit
......@@ -119,6 +120,18 @@ Edition installation, with some extra requirements:
- Your secondary node should be allowed to communicate via HTTP/HTTPS and
SSH with your primary node (make sure your firewall is not blocking that).
### Database Encryption Key
GitLab stores a unique encryption key in disk that we use to safely store sensitive
data in the database.
Any secondary node must have the exact same value for `db_key_base` as defined in the primary one.
For Omnibus installations it is stored at `/etc/gitlab/gitlab-secrets.json`.
For Source installations it is stored at `/home/git/gitlab/config/secrets.yml`.
### Authorized keys regeneration
The final step will be to regenerate the keys for `.ssh/authorized_keys` using
......
......@@ -4,12 +4,13 @@ This document describes the minimal steps you have to take in order to
replicate your GitLab database into another server. You may have to change
some values according to your database setup, how big it is, etc.
The GitLab primary node where the write operations happen will act as `master`,
and the secondary ones which are read-only will act as `slaves`.
The GitLab primary node where the write operations happen will connect to
`primary` database server, and the secondary ones which are read-only will
connect to `secondary` database servers (which are read-only too).
>**Note:**
To be on par with GitLab's notation, we will use `primary` to denote the `master`
server, and `secondary` for the `slave`.
In many databases documentation you will see `primary` being references as `master`
and `secondary` as either `slave` or `standby` server (read-only).
<!-- START doctoc generated TOC please keep comment here to allow auto update -->
<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
......
Depending on the installation method and your GitLab version, there are multiple update guides. Choose one that fits your needs.
# Updating GitLab
## CE to EE
Depending on the installation method and your GitLab version, there are multiple
update guides.
- [The CE to EE update guides](https://gitlab.com/gitlab-org/gitlab-ee/tree/master/doc/update) the steps are very similar to a version upgrade: stop the server, get the code, update config files for the new functionality, install libs and do migrations, update the init script, start the application and check the application status.
There are currently 3 official ways to install GitLab:
## EE to CE
- Omnibus packages
- Source installation
- Docker installation
- If you need to downgrade your EE installation back to CE, you can follow [this guide](../downgrade_ee_to_ce/README.md) to make
the process as smooth as possible.
Based on your installation, choose a section below that fits your needs.
---
<!-- START doctoc generated TOC please keep comment here to allow auto update -->
<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)*
- [Omnibus Packages](#omnibus-packages)
- [Installation from source](#installation-from-source)
- [Installation using Docker](#installation-using-docker)
- [Upgrading between editions](#upgrading-between-editions)
- [Community to Enterprise Edition](#community-to-enterprise-edition)
- [Enterprise to Community Edition](#enterprise-to-community-edition)
- [Miscellaneous](#miscellaneous)
<!-- END doctoc generated TOC please keep comment here to allow auto update -->
## Omnibus Packages
- [Omnibus update guide](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/doc/update.md) contains the steps needed to update a GitLab [package](https://about.gitlab.com/downloads/).
- The [Omnibus update guide](http://doc.gitlab.com/omnibus/update/README.html)
contains the steps needed to update an Omnibus GitLab package.
## Installation from source
- [The individual upgrade guides](https://gitlab.com/gitlab-org/gitlab-ce/tree/master/doc/update) are for those who have installed GitLab from source.
- [The CE to EE update guides](https://gitlab.com/gitlab-org/gitlab-ee/tree/master/doc/update). The steps are very similar to a version upgrade: stop the server, get the code, update config files for the new functionality, install libs and do migrations, update the init script, start the application and check the application status.
- [Upgrader](upgrader.md) is an automatic ruby script that performs the update for installations from source.
- [Patch versions](patch_versions.md) guide includes the steps needed for a patch version, eg. 6.2.0 to 6.2.1.
- [Upgrading Community Edition from source][source-ce] - The individual
upgrade guides are for those who have installed GitLab CE from source.
- [Upgrading Enterprise Edition from source][source-ee] - The individual
upgrade guides are for those who have installed GitLab EE from source.
- [Patch versions](patch_versions.md) guide includes the steps needed for a
patch version, eg. 6.2.0 to 6.2.1, and apply to both Community and Enterprise
Editions.
## Installation using Docker
GitLab provides official Docker images for both Community and Enterprise
editions. They are based on the Omnibus package and instructions on how to
update them are in [a separate document][omnidocker].
## Upgrading between editions
GitLab comes in two flavors: [Community Edition][ce] which is MIT licensed,
and [Enterprise Edition][ee] which builds on top of the Community Edition and
includes extra features mainly aimed at organizations with more than 100 users.
Below you can find some guides to help you change editions easily.
### Community to Enterprise Edition
>**Note:**
The following guides are for subscribers of the Enterprise Edition only.
If you wish to upgrade your GitLab installation from Community to Enterprise
Edition, follow the guides below based on the installation method:
- [Source CE to EE update guides][source-ee] - Find your version, and follow the
`-ce-to-ee.md` guide. The steps are very similar to a version upgrade: stop
the server, get the code, update config files for the new functionality,
install libraries and do migrations, update the init script, start the
application and check its status.
- [Omnibus CE to EE][omni-ce-ee] - Follow this guide to update your Omnibus
GitLab Community Edition to the Enterprise Edition.
### Enterprise to Community Edition
If you need to downgrade your Enterprise Edition installation back to Community
Edition, you can follow [this guide][ee-ce] to make the process as smooth as
possible.
## Miscellaneous
- [MySQL to PostgreSQL](mysql_to_postgresql.md) guides you through migrating your database from MySQL to PostgreSQL.
- [MySQL installation guide](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/install/database_mysql.md) contains additional information about configuring GitLab to work with a MySQL database.
- [MySQL to PostgreSQL](mysql_to_postgresql.md) guides you through migrating
your database from MySQL to PostgreSQL.
- [MySQL installation guide](../install/database_mysql.md) contains additional
information about configuring GitLab to work with a MySQL database.
- [Restoring from backup after a failed upgrade](restore_after_failure.md)
[omnidocker]: http://doc.gitlab.com/omnibus/docker/README.html
[source-ee]: https://gitlab.com/gitlab-org/gitlab-ee/tree/master/doc/update
[source-ce]: https://gitlab.com/gitlab-org/gitlab-ce/tree/master/doc/update
[ee-ce]: ../downgrade_ee_to_ce/README.md
[ce]: https://about.gitlab.com/features/#community
[ee]: https://about.gitlab.com/features/#enterprise
[omni-ce-ee]: http://doc.gitlab.com/omnibus/update/README.html#from-community-edition-to-enterprise-edition
......@@ -30,6 +30,15 @@ module API
when 'tag_push'
required_attributes! %w(event_name project_id project)
::Geo::ScheduleWikiRepoUpdateService.new(params).execute
when 'project_destroy'
required_attributes! %w(event_name project_id path_with_namespace)
::Geo::ScheduleRepoDestroyService.new(params).execute
when 'project_rename'
required_attributes! %w(event_name project_id path_with_namespace old_path_with_namespace)
::Geo::ScheduleRepoRenameService.new(params).execute
when 'project_transfer'
required_attributes! %w(event_name project_id path_with_namespace old_path_with_namespace)
::Geo::ScheduleRepoRenameService.new(params).execute
end
end
end
......
......@@ -22,6 +22,10 @@ module Gitlab
RequestStore.store[:geo_node_enabled] ||= GeoNode.exists?
end
def self.license_allows?
::License.current && ::License.current.add_on?('GitLab_Geo')
end
def self.primary?
RequestStore.store[:geo_node_primary?] ||= self.enabled? && self.current_node && self.current_node.primary?
end
......
......@@ -70,6 +70,10 @@ module Gitlab
return build_status_object(false, 'The project you were looking for could not be found.')
end
if Gitlab::Geo.secondary? && !Gitlab::Geo.license_allows?
return build_status_object(false, 'Your current license does not have GitLab Geo add-on enabled.')
end
case cmd
when *DOWNLOAD_COMMANDS
download_access_check
......@@ -94,7 +98,7 @@ module Gitlab
def push_access_check(changes)
if Gitlab::Geo.enabled? && Gitlab::Geo.secondary?
if Gitlab::Geo.secondary?
return build_status_object(false, "You can't push code on a secondary GitLab Geo node.")
end
......
......@@ -13,8 +13,8 @@ module Gitlab
@env = env
if disallowed_request? && Gitlab::Geo.secondary?
Rails.logger.debug('Gitlab Geo: preventing possible non readonly operation')
error_message = 'You cannot do writing operations on a secondary Gitlab Geo instance'
Rails.logger.debug('GitLab Geo: preventing possible non readonly operation')
error_message = 'You cannot do writing operations on a secondary GitLab Geo instance'
if json_request?
return [403, { 'Content-Type' => 'application/json' }, [{ 'message' => error_message }.to_json]]
......@@ -60,12 +60,20 @@ module Gitlab
end
def whitelisted_routes
logout_route || WHITELISTED.any? { |path| @request.path.include?(path) }
logout_route || grack_route || WHITELISTED.any? { |path| @request.path.include?(path) } || sidekiq_route
end
def logout_route
route_hash[:controller] == 'sessions' && route_hash[:action] == 'destroy'
end
def sidekiq_route
@request.path.start_with?('/admin/sidekiq')
end
def grack_route
@request.path.end_with?('.git/git-upload-pack')
end
end
end
end
......@@ -68,4 +68,21 @@ describe Gitlab::Geo, lib: true do
expect(described_class.geo_node?(host: 'inexistent', port: 1234)).to be_falsey
end
end
describe 'license_allows?' do
it 'returns true if license has Geo addon' do
allow_any_instance_of(License).to receive(:add_on?).with('GitLab_Geo') { true }
expect(described_class.license_allows?).to be_truthy
end
it 'returns false if license doesnt have Geo addon' do
allow_any_instance_of(License).to receive(:add_on?).with('GitLab_Geo') { false }
expect(described_class.license_allows?).to be_falsey
end
it 'returns false if no license is present' do
allow(License).to receive(:current) { nil }
expect(described_class.license_allows?).to be_falsey
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment