Commit 04e06c46 authored by Douwe Maan's avatar Douwe Maan

Merge branch 'tc-geo-nodes-more-api' into 'master'

Add some API endpoints for Geo admin screen in Vue

Closes #4475

See merge request gitlab-org/gitlab-ee!3923
parents 9fbbbd7f 6405ea83
---
title: Add more endpoints for Geo Nodes API
merge_request: 3923
author:
type: added
......@@ -65,6 +65,60 @@ Example response:
}
```
## Edit a Geo node
Updates an existing Geo secondary node. The primary node cannot be edited.
```
PUT /geo_nodes/:id
```
| Attribute | Type | Required | Description |
|----------------------|---------|----------|---------------------------------------------------------------------------|
| `id` | integer | yes | The ID of the Geo node. |
| `enabled` | boolean | no | Flag indicating if the Geo node is enabled. |
| `url` | string | no | The URL to connect to the Geo node. |
| `files_max_capacity` | integer | no | Control the maximum concurrency of LFS/attachment backfill for this node. |
| `repos_max_capacity` | integer | no | Control the maximum concurrency of repository backfill for this node. |
Example response:
```json
{
"id": 1,
"url": "https://primary.example.com/",
"primary": true,
"enabled": true,
"current": true,
"files_max_capacity": 10,
"repos_max_capacity": 25,
"clone_protocol": "http"
}
```
## Repair a Geo node
To repair the OAuth authentication of a Geo node.
```
PUT /geo_nodes/:id/repair
```
Example response:
```json
{
"id": 1,
"url": "https://primary.example.com/",
"primary": true,
"enabled": true,
"current": true,
"files_max_capacity": 10,
"repos_max_capacity": 25,
"clone_protocol": "http"
}
```
## Retrieve status about all secondary Geo nodes
```
......
......@@ -44,9 +44,9 @@ class Admin::GeoNodesController < Admin::ApplicationController
end
def repair
if @node.primary? || !@node.missing_oauth_application?
if !@node.missing_oauth_application?
flash[:notice] = "This node doesn't need to be repaired."
elsif @node.save
elsif @node.repair
flash[:notice] = 'Node Authentication was successfully repaired.'
else
flash[:alert] = 'There was a problem repairing Node Authentication.'
......
......@@ -26,6 +26,8 @@ class GeoNode < ActiveRecord::Base
before_validation :ensure_access_keys!
alias_method :repair, :save # the `update_dependents_attributes` hook will take care of it
scope :with_url_prefix, ->(prefix) { where('url LIKE ?', "#{prefix}%") }
attr_encrypted :secret_access_key,
......
class GeoNodeStatusEntity < Grape::Entity
include ActionView::Helpers::NumberHelper
expose :geo_node_id
expose :healthy?, as: :healthy
expose :health do |node|
node.healthy? ? 'Healthy' : node.health
end
expose :health_status
expose :missing_oauth_application, as: :missing_oauth_application
expose :attachments_count
expose :attachments_synced_count
expose :attachments_failed_count
expose :attachments_synced_in_percentage do |node|
number_to_percentage(node.attachments_synced_in_percentage, precision: 2)
end
expose :db_replication_lag_seconds
expose :lfs_objects_count
expose :lfs_objects_synced_count
expose :lfs_objects_failed_count
expose :lfs_objects_synced_in_percentage do |node|
number_to_percentage(node.lfs_objects_synced_in_percentage, precision: 2)
end
expose :job_artifacts_count
expose :job_artifacts_synced_count
expose :job_artifacts_failed_count
expose :job_artifacts_synced_in_percentage do |node|
number_to_percentage(node.job_artifacts_synced_in_percentage, precision: 2)
end
expose :repositories_count
expose :repositories_failed_count
expose :repositories_synced_count
expose :repositories_synced_in_percentage do |node|
number_to_percentage(node.repositories_synced_in_percentage, precision: 2)
end
expose :wikis_count
expose :wikis_failed_count
expose :wikis_synced_count
expose :wikis_synced_in_percentage do |node|
number_to_percentage(node.wikis_synced_in_percentage, precision: 2)
end
expose :replication_slots_count
expose :replication_slots_used_count
expose :replication_slots_used_in_percentage do |node|
number_to_percentage(node.replication_slots_used_in_percentage, precision: 2)
end
expose :replication_slots_max_retained_wal_bytes
expose :last_event_id
expose :last_event_timestamp
expose :cursor_last_event_id
expose :cursor_last_event_timestamp
expose :last_successful_status_check_timestamp
expose :version
expose :revision
expose :namespaces, using: NamespaceEntity
# We load GeoNodeStatus data in two ways:
#
# 1. Directly by asking a Geo node via an API call
# 2. Via cached state in the database
#
# We don't yet cached the state of the shard information in the database, so if
# we don't have this information omit from the serialization entirely.
expose :storage_shards, using: StorageShardEntity, if: ->(status, options) do
status.storage_shards.present?
end
expose :storage_shards_match?, as: :storage_shards_match, if: -> (status, options) do
Gitlab::Geo.primary? && status.storage_shards.present?
end
private
def namespaces
object.geo_node.namespaces
end
def missing_oauth_application
object.geo_node.missing_oauth_application?
end
def version
Gitlab::VERSION
end
def revision
Gitlab::REVISION
end
end
class GeoNodeStatusSerializer < BaseSerializer
entity GeoNodeStatusEntity
entity API::Entities::GeoNodeStatus
end
......@@ -1130,6 +1130,8 @@ module API
end
class GeoNode < Grape::Entity
include ::API::Helpers::RelatedResourcesHelpers
expose :id
expose :url
expose :primary?, as: :primary
......@@ -1142,6 +1144,129 @@ module API
expose :clone_protocol do |_record, _options|
'http'
end
expose :_links do
expose :self do |geo_node|
expose_url api_v4_geo_nodes_path(id: geo_node.id)
end
expose :repair do |geo_node|
expose_url api_v4_geo_nodes_repair_path(id: geo_node.id)
end
end
end
class GeoNodeStatus < Grape::Entity
include ::API::Helpers::RelatedResourcesHelpers
include ActionView::Helpers::NumberHelper
expose :geo_node_id
expose :healthy?, as: :healthy
expose :health do |node|
node.healthy? ? 'Healthy' : node.health
end
expose :health_status
expose :missing_oauth_application
expose :attachments_count
expose :attachments_synced_count
expose :attachments_failed_count
expose :attachments_synced_in_percentage do |node|
number_to_percentage(node.attachments_synced_in_percentage, precision: 2)
end
expose :db_replication_lag_seconds
expose :lfs_objects_count
expose :lfs_objects_synced_count
expose :lfs_objects_failed_count
expose :lfs_objects_synced_in_percentage do |node|
number_to_percentage(node.lfs_objects_synced_in_percentage, precision: 2)
end
expose :job_artifacts_count
expose :job_artifacts_synced_count
expose :job_artifacts_failed_count
expose :job_artifacts_synced_in_percentage do |node|
number_to_percentage(node.job_artifacts_synced_in_percentage, precision: 2)
end
expose :repositories_count
expose :repositories_failed_count
expose :repositories_synced_count
expose :repositories_synced_in_percentage do |node|
number_to_percentage(node.repositories_synced_in_percentage, precision: 2)
end
expose :wikis_count
expose :wikis_failed_count
expose :wikis_synced_count
expose :wikis_synced_in_percentage do |node|
number_to_percentage(node.wikis_synced_in_percentage, precision: 2)
end
expose :replication_slots_count
expose :replication_slots_used_count
expose :replication_slots_used_in_percentage do |node|
number_to_percentage(node.replication_slots_used_in_percentage, precision: 2)
end
expose :replication_slots_max_retained_wal_bytes
expose :last_event_id
expose :last_event_timestamp
expose :cursor_last_event_id
expose :cursor_last_event_timestamp
expose :last_successful_status_check_timestamp
expose :version
expose :revision
expose :namespaces, using: NamespaceBasic
# We load GeoNodeStatus data in two ways:
#
# 1. Directly by asking a Geo node via an API call
# 2. Via cached state in the database
#
# We don't yet cached the state of the shard information in the database, so if
# we don't have this information omit from the serialization entirely.
expose :storage_shards, using: StorageShardEntity, if: ->(status, options) do
status.storage_shards.present?
end
expose :storage_shards_match?, as: :storage_shards_match, if: -> (status, options) do
Gitlab::Geo.primary? && status.storage_shards.present?
end
expose :_links do
expose :self do |geo_node_status|
expose_url api_v4_geo_nodes_status_path(id: geo_node_status.geo_node_id)
end
expose :node do |geo_node_status|
expose_url api_v4_geo_nodes_path(id: geo_node_status.geo_node_id)
end
end
private
def namespaces
object.geo_node.namespaces
end
def missing_oauth_application
object.geo_node.missing_oauth_application?
end
def version
Gitlab::VERSION
end
def revision
Gitlab::REVISION
end
end
class PersonalAccessToken < Grape::Entity
......
......@@ -36,7 +36,7 @@ module API
authenticate_by_gitlab_geo_node_token!
status = ::GeoNodeStatus.current_node_status
present status, with: GeoNodeStatusEntity
present status, with: Entities::GeoNodeStatus
end
end
......
......@@ -2,6 +2,7 @@ module API
class GeoNodes < Grape::API
include PaginationParams
include APIGuard
include ::Gitlab::Utils::StrongMemoize
before { authenticated_as_admin! }
......@@ -25,12 +26,12 @@ module API
# Example request:
# GET /geo_nodes/status
desc 'Get status for all Geo nodes' do
success GeoNodeStatusEntity
success Entities::GeoNodeStatus
end
get '/status' do
status = GeoNodeStatus.all
present paginate(status), with: GeoNodeStatusEntity
present paginate(status), with: Entities::GeoNodeStatus
end
# Get project registry failures for the current Geo node
......@@ -55,49 +56,95 @@ module API
present project_registries, with: ::GeoProjectRegistryEntity
end
# Get all Geo node information
#
# Example request:
# GET /geo_nodes/:id
desc 'Get a single GeoNode' do
success Entities::GeoNode
end
params do
requires :id, type: Integer, desc: 'The ID of the node'
end
get ':id' do
node = GeoNode.find_by(id: params[:id])
not_found!('GeoNode') unless node
present node, with: Entities::GeoNode
end
# Get Geo metrics for a single node
#
# Example request:
# GET /geo_nodes/:id/status
desc 'Get metrics for a single Geo node' do
success Entities::GeoNode
end
params do
requires :id, type: Integer, desc: 'The ID of the node'
end
get ':id/status' do
geo_node = GeoNode.find(params[:id])
not_found('Geo node not found') unless geo_node
route_param :id, type: Integer, desc: 'The ID of the node' do
helpers do
def geo_node
strong_memoize(:geo_node) { GeoNode.find(params[:id]) }
end
status =
if geo_node.current?
GeoNodeStatus.current_node_status
def geo_node_status
strong_memoize(:geo_node_status) do
if geo_node.current?
GeoNodeStatus.current_node_status
else
geo_node.status
end
end
end
end
# Get all Geo node information
#
# Example request:
# GET /geo_nodes/:id
desc 'Get a single GeoNode' do
success Entities::GeoNode
end
get do
not_found!('GeoNode') unless geo_node
present geo_node, with: Entities::GeoNode
end
# Get Geo metrics for a single node
#
# Example request:
# GET /geo_nodes/:id/status
desc 'Get metrics for a single Geo node' do
success Entities::GeoNodeStatus
end
get 'status' do
not_found!('GeoNode') unless geo_node
not_found!('Status for Geo node not found') unless geo_node_status
present geo_node_status, with: Entities::GeoNodeStatus
end
# Repair authentication of the Geo node
#
# Example request:
# POST /geo_nodes/:id/repair
desc 'Repair authentication of the Geo node' do
success Entities::GeoNodeStatus
end
post 'repair' do
not_found!('GeoNode') unless geo_node
if !geo_node.missing_oauth_application? || geo_node.repair
status 200
present geo_node_status, with: Entities::GeoNodeStatus
else
geo_node.status
render_validation_error!(geo_node)
end
not_found!('Status for Geo node not found') unless status
present status, with: ::GeoNodeStatusEntity
end
# Edit an existing Geo node
#
# Example request:
# PUT /geo_nodes/:id
desc 'Edit an existing Geo secondary node' do
success Entities::GeoNode
end
params do
optional :enabled, type: Boolean, desc: 'Flag indicating if the Geo node is enabled'
optional :url, type: String, desc: 'The URL to connect to the Geo node'
optional :files_max_capacity, type: Integer, desc: 'Control the maximum concurrency of LFS/attachment backfill for this secondary node'
optional :repos_max_capacity, type: Integer, desc: 'Control the maximum concurrency of repository backfill for this secondary node'
end
put do
not_found!('GeoNode') unless geo_node
update_params = declared_params(include_missing: false)
if geo_node.primary?
forbidden!('Primary node cannot be edited')
elsif geo_node.update_attributes(update_params)
present geo_node, with: Entities::GeoNode
else
render_validation_error!(geo_node)
end
end
end
end
end
......
......@@ -8,7 +8,8 @@
"current",
"files_max_capacity",
"repos_max_capacity",
"clone_protocol"
"clone_protocol",
"_links"
],
"properties" : {
"id": { "type": "integer" },
......@@ -18,7 +19,16 @@
"current": { "type": "boolean" },
"files_max_capacity": { "type": "integer" },
"repos_max_capacity": { "type": "integer" },
"clone_protocol": { "type": ["string"] }
"clone_protocol": { "type": ["string"] },
"_links": {
"type": "object",
"required": ["self", "repair"],
"properties" : {
"self": { "type": "string" },
"repair": { "type": "string" }
},
"additionalProperties": false
}
},
"additionalProperties": false
}
......@@ -32,7 +32,8 @@
"cursor_last_event_timestamp",
"namespaces",
"version",
"revision"
"revision",
"_links"
],
"properties" : {
"geo_node_id": { "type": "integer" },
......@@ -74,7 +75,16 @@
"storage_shards": { "type": "array" },
"storage_shards_match": { "type": "boolean" },
"version": { "type": ["string"] },
"revision": { "type": ["string"] }
"revision": { "type": ["string"] },
"_links": {
"type": "object",
"required": ["self", "node"],
"properties" : {
"self": { "type": "string" },
"node": { "type": "string" }
},
"additionalProperties": false
}
},
"additionalProperties": false
}
......@@ -267,7 +267,7 @@ describe Admin::GeoNodesController, :postgresql do
it 'returns the status' do
get :status, id: geo_node, format: :json
expect(response).to match_response_schema('geo_node_status')
expect(response).to match_response_schema('public_api/v4/geo_node_status', dir: 'ee')
end
end
end
......
......@@ -78,6 +78,17 @@ describe GeoNode, type: :model do
end
end
describe '#repair' do
it 'creates an oauth application for a Geo secondary node' do
stub_current_geo_node(node)
node.update_attribute(:oauth_application, nil)
node.repair
expect(node.oauth_application).to be_present
end
end
describe '#current?' do
it 'returns true when node is the current node' do
node = described_class.new(url: described_class.current_node_url)
......
FactoryBot.define do
factory :geo_node_status do
sequence(:id)
geo_node
storage_shards { StorageShard.all }
......
require 'spec_helper'
describe GeoNodeStatusEntity, :postgresql do
describe API::Entities::GeoNodeStatus, :postgresql do
include ::EE::GeoHelpers
let(:geo_node_status) { build(:geo_node_status) }
......@@ -11,38 +11,6 @@ describe GeoNodeStatusEntity, :postgresql do
before { stub_primary_node }
it { is_expected.to have_key(:geo_node_id) }
it { is_expected.to have_key(:healthy) }
it { is_expected.to have_key(:health) }
it { is_expected.to have_key(:attachments_count) }
it { is_expected.to have_key(:attachments_failed_count) }
it { is_expected.to have_key(:attachments_synced_count) }
it { is_expected.to have_key(:attachments_synced_in_percentage) }
it { is_expected.to have_key(:lfs_objects_count) }
it { is_expected.to have_key(:lfs_objects_failed_count) }
it { is_expected.to have_key(:lfs_objects_synced_count) }
it { is_expected.to have_key(:lfs_objects_synced_in_percentage) }
it { is_expected.to have_key(:job_artifacts_count) }
it { is_expected.to have_key(:job_artifacts_failed_count) }
it { is_expected.to have_key(:job_artifacts_synced_count) }
it { is_expected.to have_key(:job_artifacts_synced_in_percentage) }
it { is_expected.to have_key(:repositories_count) }
it { is_expected.to have_key(:repositories_failed_count) }
it { is_expected.to have_key(:repositories_synced_count)}
it { is_expected.to have_key(:repositories_synced_in_percentage) }
it { is_expected.to have_key(:wikis_count) }
it { is_expected.to have_key(:wikis_failed_count) }
it { is_expected.to have_key(:wikis_synced_count)}
it { is_expected.to have_key(:wikis_synced_in_percentage) }
it { is_expected.to have_key(:replication_slots_count) }
it { is_expected.to have_key(:replication_slots_used_count)}
it { is_expected.to have_key(:replication_slots_used_in_percentage) }
it { is_expected.to have_key(:replication_slots_max_retained_wal_bytes) }
it { is_expected.to have_key(:last_successful_status_check_timestamp) }
it { is_expected.to have_key(:namespaces) }
it { is_expected.to have_key(:storage_shards) }
it { is_expected.to have_key(:storage_shards_match) }
describe '#healthy' do
context 'when node is healthy' do
it 'returns true' do
......
......@@ -6,20 +6,19 @@ describe API::GeoNodes, :geo, api: true do
set(:primary) { create(:geo_node, :primary) }
set(:secondary) { create(:geo_node) }
set(:another_secondary) { create(:geo_node) }
set(:secondary_status) { create(:geo_node_status, :healthy, geo_node: secondary) }
set(:secondary_status) { create(:geo_node_status, :healthy, geo_node_id: secondary.id) }
set(:another_secondary_status) { create(:geo_node_status, :healthy, geo_node_id: another_secondary.id) }
let(:unexisting_node_id) { GeoNode.maximum(:id).to_i.succ }
let(:admin) { create(:admin) }
let(:user) { create(:user) }
set(:admin) { create(:admin) }
set(:user) { create(:user) }
describe 'GET /geo_nodes' do
it 'retrieves the Geo nodes if admin is logged in' do
get api("/geo_nodes", admin)
expect(response).to have_gitlab_http_status(200)
expect(response).to match_response_schema('geo_nodes')
expect(response).to match_response_schema('public_api/v4/geo_nodes', dir: 'ee')
end
it 'denies access if not admin' do
......@@ -34,7 +33,15 @@ describe API::GeoNodes, :geo, api: true do
get api("/geo_nodes/#{primary.id}", admin)
expect(response).to have_gitlab_http_status(200)
expect(response).to match_response_schema('geo_node')
expect(response).to match_response_schema('public_api/v4/geo_node', dir: 'ee')
links = json_response['_links']
expect(links['self']).to end_with("/api/v4/geo_nodes/#{primary.id}")
expect(links['repair']).to end_with("/api/v4/geo_nodes/#{primary.id}/repair")
end
it_behaves_like '404 response' do
let(:request) { get api("/geo_nodes/#{unexisting_node_id}", admin) }
end
it 'denies access if not admin' do
......@@ -49,7 +56,7 @@ describe API::GeoNodes, :geo, api: true do
get api("/geo_nodes/status", admin)
expect(response).to have_gitlab_http_status(200)
expect(response).to match_response_schema('geo_node_statuses')
expect(response).to match_response_schema('public_api/v4/geo_node_statuses', dir: 'ee')
end
it 'denies access if not admin' do
......@@ -68,7 +75,11 @@ describe API::GeoNodes, :geo, api: true do
get api("/geo_nodes/#{secondary.id}/status", admin)
expect(response).to have_gitlab_http_status(200)
expect(response).to match_response_schema('geo_node_status')
expect(response).to match_response_schema('public_api/v4/geo_node_status', dir: 'ee')
links = json_response['_links']
expect(links['self']).to end_with("/api/v4/geo_nodes/#{secondary.id}/status")
expect(links['node']).to end_with("/api/v4/geo_nodes/#{secondary.id}")
end
it 'fetches the current node status' do
......@@ -80,14 +91,82 @@ describe API::GeoNodes, :geo, api: true do
get api("/geo_nodes/#{secondary.id}/status", admin)
expect(response).to have_gitlab_http_status(200)
expect(response).to match_response_schema('geo_node_status')
expect(response).to match_response_schema('public_api/v4/geo_node_status', dir: 'ee')
end
it_behaves_like '404 response' do
let(:request) { get api("/geo_nodes/#{unexisting_node_id}/status", admin) }
end
it 'denies access if not admin' do
get api('/geo_nodes', user)
get api("/geo_nodes/#{secondary.id}/status", user)
expect(response).to have_gitlab_http_status(403)
end
end
describe 'POST /geo_nodes/:id/repair' do
it_behaves_like '404 response' do
let(:request) { post api("/geo_nodes/#{unexisting_node_id}/status", admin) }
end
it 'denies access if not admin' do
post api("/geo_nodes/#{secondary.id}/repair", user)
expect(response).to have_gitlab_http_status(403)
end
it 'returns 200 for the primary node' do
post api("/geo_nodes/#{primary.id}/repair", admin)
expect(response).to have_gitlab_http_status(200)
expect(response).to match_response_schema('public_api/v4/geo_node_status', dir: 'ee')
end
it 'returns 200 when node does not need repairing' do
allow_any_instance_of(GeoNode).to receive(:missing_oauth_application?).and_return(false)
post api("/geo_nodes/#{secondary.id}/repair", admin)
expect(response).to have_gitlab_http_status(200)
expect(response).to match_response_schema('public_api/v4/geo_node_status', dir: 'ee')
end
it 'repairs a secondary with oauth application missing' do
allow_any_instance_of(GeoNode).to receive(:missing_oauth_application?).and_return(true)
post api("/geo_nodes/#{secondary.id}/repair", admin)
expect(response).to have_gitlab_http_status(200)
expect(response).to match_response_schema('public_api/v4/geo_node_status', dir: 'ee')
end
end
describe 'PUT /geo_nodes/:id' do
it_behaves_like '404 response' do
let(:request) { get api("/geo_nodes/#{unexisting_node_id}/status", admin) }
end
it 'denies access if not admin' do
put api("/geo_nodes/#{secondary.id}", user), {}
expect(response).to have_gitlab_http_status(403)
end
it 'updates the parameters' do
params = {
enabled: false,
url: 'https://updated.example.com/',
files_max_capacity: 33,
repos_max_capacity: 44
}.stringify_keys
put api("/geo_nodes/#{secondary.id}", admin), params
expect(response).to have_gitlab_http_status(200)
expect(response).to match_response_schema('public_api/v4/geo_node', dir: 'ee')
expect(json_response).to include(params)
end
end
describe 'GET /geo_nodes/current/failures/:type' do
......@@ -101,7 +180,7 @@ describe API::GeoNodes, :geo, api: true do
get api("/geo_nodes/current/failures", admin)
expect(response).to have_gitlab_http_status(200)
expect(response).to match_response_schema('geo_project_registry')
expect(response).to match_response_schema('public_api/v4/geo_project_registry', dir: 'ee')
end
it 'does not show any registry when there is no failure' do
......
......@@ -185,7 +185,7 @@ describe API::Geo do
get api('/geo/status'), nil, request.headers
expect(response).to have_gitlab_http_status(200)
expect(response).to match_response_schema('geo_node_status')
expect(response).to match_response_schema('public_api/v4/geo_node_status', dir: 'ee')
end
end
......@@ -199,7 +199,7 @@ describe API::Geo do
get api('/geo/status'), nil, request.headers
expect(response).to have_gitlab_http_status(200)
expect(response).to match_response_schema('geo_node_status')
expect(response).to match_response_schema('public_api/v4/geo_node_status', dir: 'ee')
end
end
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment