Commit 604809ec authored by Peter Leitzen's avatar Peter Leitzen

Merge branch 'ak/use-cluster-in-service' into 'master'

Make logs services agnostic from environment

See merge request gitlab-org/gitlab!25252
parents 35cb061f 791c330f
......@@ -4,6 +4,7 @@ module Projects
class LogsController < Projects::ApplicationController
before_action :authorize_read_pod_logs!
before_action :environment
before_action :ensure_deployments, only: %i(k8s elasticsearch)
def index
if environment.nil?
......@@ -27,7 +28,7 @@ module Projects
::Gitlab::UsageCounters::PodLogs.increment(project.id)
::Gitlab::PollingInterval.set_header(response, interval: 3_000)
result = service.new(environment, params: permitted_params).execute
result = service.new(cluster, namespace, params: permitted_params).execute
if result.nil?
head :accepted
......@@ -57,5 +58,22 @@ module Projects
project.default_environment
end
end
def cluster
environment.deployment_platform&.cluster
end
def namespace
environment.deployment_namespace
end
def ensure_deployments
return if cluster && namespace.present?
render status: :bad_request, json: {
status: :error,
message: _('Environment does not have deployments')
}
end
end
end
......@@ -5,7 +5,7 @@ module PodLogs
include ReactiveCaching
include Stepable
attr_reader :environment, :params
attr_reader :cluster, :namespace, :params
CACHE_KEY_GET_POD_LOG = 'get_pod_log'
K8S_NAME_MAX_LENGTH = 253
......@@ -13,24 +13,26 @@ module PodLogs
SUCCESS_RETURN_KEYS = %i(status logs pod_name container_name pods).freeze
def id
environment.id
cluster.id
end
def initialize(environment, params: {})
@environment = environment
def initialize(cluster, namespace, params: {})
@cluster = cluster
@namespace = namespace
@params = filter_params(params.dup.stringify_keys).to_hash
end
def execute
with_reactive_cache(
CACHE_KEY_GET_POD_LOG,
namespace,
params
) do |result|
result
end
end
def calculate_reactive_cache(request, _opts)
def calculate_reactive_cache(request, _namespace, _params)
case request
when CACHE_KEY_GET_POD_LOG
execute_steps
......@@ -47,6 +49,13 @@ module PodLogs
%w(pod_name container_name)
end
def check_arguments(result)
return error(_('Cluster does not exist')) if cluster.nil?
return error(_('Namespace is empty')) if namespace.blank?
success(result)
end
def check_param_lengths(_result)
pod_name = params['pod_name'].presence
container_name = params['container_name'].presence
......@@ -62,17 +71,8 @@ module PodLogs
success(pod_name: pod_name, container_name: container_name)
end
def check_deployment_platform(result)
unless environment.deployment_platform
return error(_('No deployment platform available'))
end
success(result)
end
def get_raw_pods(result)
namespace = environment.deployment_namespace
result[:raw_pods] = environment.deployment_platform.kubeclient.get_pods(namespace: namespace)
result[:raw_pods] = cluster.kubeclient.get_pods(namespace: namespace)
success(result)
end
......@@ -85,7 +85,7 @@ module PodLogs
def check_pod_name(result)
# If pod_name is not received as parameter, get the pod logs of the first
# pod of this environment.
# pod of this namespace.
result[:pod_name] ||= result[:pods].first
unless result[:pod_name]
......
......@@ -2,8 +2,8 @@
module PodLogs
class ElasticsearchService < BaseService
steps :check_param_lengths,
:check_deployment_platform,
steps :check_arguments,
:check_param_lengths,
:get_raw_pods,
:get_pod_names,
:check_pod_name,
......@@ -13,7 +13,7 @@ module PodLogs
:pod_logs,
:filter_return_keys
self.reactive_cache_worker_finder = ->(id, _cache_key, params) { new(Environment.find(id), params: params) }
self.reactive_cache_worker_finder = ->(id, _cache_key, namespace, params) { new(::Clusters::Cluster.find(id), namespace, params: params) }
private
......@@ -37,8 +37,7 @@ module PodLogs
end
def pod_logs(result)
namespace = environment.deployment_namespace
client = environment.deployment_platform.cluster&.application_elastic_stack&.elasticsearch_client
client = cluster&.application_elastic_stack&.elasticsearch_client
return error(_('Unable to connect to Elasticsearch')) unless client
result[:logs] = ::Gitlab::Elasticsearch::Logs.new(client).pod_logs(
......
......@@ -4,8 +4,8 @@ module PodLogs
class KubernetesService < BaseService
LOGS_LIMIT = 500.freeze
steps :check_param_lengths,
:check_deployment_platform,
steps :check_arguments,
:check_param_lengths,
:get_raw_pods,
:get_pod_names,
:check_pod_name,
......@@ -13,14 +13,14 @@ module PodLogs
:pod_logs,
:filter_return_keys
self.reactive_cache_worker_finder = ->(id, _cache_key, params) { new(Environment.find(id), params: params) }
self.reactive_cache_worker_finder = ->(id, _cache_key, namespace, params) { new(::Clusters::Cluster.find(id), namespace, params: params) }
private
def pod_logs(result)
logs = environment.deployment_platform.kubeclient.get_pod_log(
logs = cluster.kubeclient.get_pod_log(
result[:pod_name],
environment.deployment_namespace,
namespace,
container: result[:container_name],
tail_lines: LOGS_LIMIT,
timestamps: true
......
......@@ -58,118 +58,109 @@ describe Projects::LogsController do
end
end
describe "GET #k8s" do
shared_examples 'pod logs service' do |endpoint, service|
let(:service_result) do
{
status: :success,
logs: ['Log 1', 'Log 2', 'Log 3'],
message: 'message',
pods: [pod_name],
pod_name: pod_name,
container_name: container
}
end
let(:service_result_json) { JSON.parse(service_result.to_json) }
let_it_be(:cluster) { create(:cluster, :provided_by_gcp, environment_scope: '*', projects: [project]) }
before do
stub_licensed_features(pod_logs: true)
allow_next_instance_of(::PodLogs::KubernetesService) do |instance|
allow_next_instance_of(service) do |instance|
allow(instance).to receive(:execute).and_return(service_result)
end
end
shared_examples 'resource not found' do |message|
it 'returns 400', :aggregate_failures do
get :k8s, params: environment_params(pod_name: pod_name, format: :json)
it 'returns 404 when unlicensed' do
stub_licensed_features(pod_logs: false)
expect(response).to have_gitlab_http_status(:bad_request)
expect(json_response['message']).to eq(message)
expect(json_response['pods']).to match_array([pod_name])
expect(json_response['pod_name']).to eq(pod_name)
expect(json_response['container_name']).to eq(container)
end
get endpoint, params: environment_params(pod_name: pod_name, format: :json)
expect(response).to have_gitlab_http_status(:not_found)
end
it 'returns the logs for a specific pod', :aggregate_failures do
get :k8s, params: environment_params(pod_name: pod_name, format: :json)
it 'returns the service result' do
get endpoint, params: environment_params(pod_name: pod_name, format: :json)
expect(response).to have_gitlab_http_status(:success)
expect(json_response["logs"]).to match_array(["Log 1", "Log 2", "Log 3"])
expect(json_response["pods"]).to match_array([pod_name])
expect(json_response['message']).to eq(service_result[:message])
expect(json_response['pod_name']).to eq(pod_name)
expect(json_response['container_name']).to eq(container)
expect(json_response).to eq(service_result_json)
end
it 'registers a usage of the endpoint' do
expect(::Gitlab::UsageCounters::PodLogs).to receive(:increment).with(project.id)
get :k8s, params: environment_params(pod_name: pod_name, format: :json)
end
get endpoint, params: environment_params(pod_name: pod_name, format: :json)
context 'when kubernetes API returns error' do
let(:service_result) do
{
status: :error,
message: 'Kubernetes API returned status code: 400',
pods: [pod_name],
pod_name: pod_name,
container_name: container
}
expect(response).to have_gitlab_http_status(:success)
end
it 'returns bad request' do
get :k8s, params: environment_params(pod_name: pod_name, format: :json)
it 'sets the polling header' do
get endpoint, params: environment_params(pod_name: pod_name, format: :json)
expect(response).to have_gitlab_http_status(:bad_request)
expect(json_response["logs"]).to eq(nil)
expect(json_response["pods"]).to match_array([pod_name])
expect(json_response["message"]).to eq('Kubernetes API returned status code: 400')
expect(json_response['pod_name']).to eq(pod_name)
expect(json_response['container_name']).to eq(container)
end
expect(response).to have_gitlab_http_status(:success)
expect(response.headers['Poll-Interval']).to eq('3000')
end
context 'when pod does not exist' do
let(:service_result) do
{
status: :error,
message: 'Pod not found',
pods: [pod_name],
pod_name: pod_name,
container_name: container
}
end
context 'when service is processing' do
let(:service_result) { nil }
it 'returns a 202' do
get endpoint, params: environment_params(pod_name: pod_name, format: :json)
it_behaves_like 'resource not found', 'Pod not found'
expect(response).to have_gitlab_http_status(:accepted)
end
end
context 'when service returns error without pods, pod_name, container_name' do
shared_examples 'unsuccessful execution response' do |message|
let(:service_result) do
{
status: :error,
message: 'No deployment platform'
message: message
}
end
it 'returns the error without pods, pod_name and container_name' do
get :k8s, params: environment_params(pod_name: pod_name, format: :json)
it 'returns the error' do
get endpoint, params: environment_params(pod_name: pod_name, format: :json)
expect(response).to have_gitlab_http_status(:bad_request)
expect(json_response['message']).to eq('No deployment platform')
expect(json_response.keys).to contain_exactly('message', 'status')
expect(json_response).to eq(service_result_json)
end
end
context 'when service is processing (returns nil)' do
let(:service_result) { nil }
context 'when service is failing' do
it_behaves_like 'unsuccessful execution response', 'some error'
end
it 'renders accepted' do
get :k8s, params: environment_params(pod_name: pod_name, format: :json)
context 'when cluster is nil' do
let!(:cluster) { nil }
expect(response).to have_gitlab_http_status(:accepted)
it_behaves_like 'unsuccessful execution response', 'Environment does not have deployments'
end
context 'when namespace is empty' do
before do
allow(environment).to receive(:deployment_namespace).and_return('')
end
it_behaves_like 'unsuccessful execution response', 'Environment does not have deployments'
end
end
describe 'GET #k8s' do
it_behaves_like 'pod logs service', :k8s, PodLogs::KubernetesService
end
describe 'GET #elasticsearch' do
it_behaves_like 'pod logs service', :elasticsearch, PodLogs::ElasticsearchService
end
def environment_params(opts = {})
......
......@@ -5,7 +5,8 @@ require 'spec_helper'
describe ::PodLogs::BaseService do
include KubernetesHelpers
let_it_be(:environment, refind: true) { create(:environment) }
let_it_be(:cluster) { create(:cluster, :provided_by_gcp, environment_scope: '*') }
let(:namespace) { 'autodevops-deploy-9-production' }
let(:pod_name) { 'pod-1' }
let(:container_name) { 'container-0' }
......@@ -16,7 +17,7 @@ describe ::PodLogs::BaseService do
].to_json, object_class: OpenStruct)
end
subject { described_class.new(environment, params: params) }
subject { described_class.new(cluster, namespace, params: params) }
describe '#initialize' do
let(:params) do
......@@ -27,7 +28,8 @@ describe ::PodLogs::BaseService do
end
it 'filters the parameters' do
expect(subject.environment).to eq(environment)
expect(subject.cluster).to eq(cluster)
expect(subject.namespace).to eq(namespace)
expect(subject.params).to eq({
'container_name' => container_name
})
......@@ -35,6 +37,49 @@ describe ::PodLogs::BaseService do
end
end
describe '#check_arguments' do
context 'when cluster and namespace are provided' do
it 'returns success' do
result = subject.send(:check_arguments, {})
expect(result[:status]).to eq(:success)
end
end
context 'when cluster is nil' do
let(:cluster) { nil }
it 'returns an error' do
result = subject.send(:check_arguments, {})
expect(result[:status]).to eq(:error)
expect(result[:message]).to eq('Cluster does not exist')
end
end
context 'when namespace is nil' do
let(:namespace) { nil }
it 'returns an error' do
result = subject.send(:check_arguments, {})
expect(result[:status]).to eq(:error)
expect(result[:message]).to eq('Namespace is empty')
end
end
context 'when namespace is empty' do
let(:namespace) { '' }
it 'returns an error' do
result = subject.send(:check_arguments, {})
expect(result[:status]).to eq(:error)
expect(result[:message]).to eq('Namespace is empty')
end
end
end
describe '#check_param_lengths' do
context 'when pod_name and container_name are provided' do
let(:params) do
......@@ -84,31 +129,11 @@ describe ::PodLogs::BaseService do
end
end
describe '#check_deployment_platform' do
it 'returns success when deployment platform exist' do
create(:cluster, :provided_by_gcp, environment_scope: '*', projects: [environment.project])
create(:deployment, :success, environment: environment)
result = subject.send(:check_deployment_platform, {})
expect(result[:status]).to eq(:success)
end
it 'returns error when deployment platform does not exist' do
result = subject.send(:check_deployment_platform, {})
expect(result[:status]).to eq(:error)
expect(result[:message]).to eq('No deployment platform available')
end
end
describe '#get_raw_pods' do
let(:service) { create(:cluster_platform_kubernetes, :configured) }
it 'returns success with passthrough k8s response' do
create(:cluster, :provided_by_gcp, environment_scope: '*', projects: [environment.project])
create(:deployment, :success, environment: environment)
stub_kubeclient_pods(environment.deployment_namespace)
stub_kubeclient_pods(namespace)
result = subject.send(:get_raw_pods, {})
......
......@@ -3,7 +3,8 @@
require 'spec_helper'
describe ::PodLogs::ElasticsearchService do
let_it_be(:environment, refind: true) { create(:environment) }
let_it_be(:cluster) { create(:cluster, :provided_by_gcp, environment_scope: '*') }
let(:namespace) { 'autodevops-deploy-9-production' }
let(:pod_name) { 'pod-1' }
let(:container_name) { 'container-1' }
......@@ -19,7 +20,7 @@ describe ::PodLogs::ElasticsearchService do
]
end
subject { described_class.new(environment, params: params) }
subject { described_class.new(cluster, namespace, params: params) }
describe '#check_times' do
context 'with start and end provided and valid' do
......@@ -116,7 +117,6 @@ describe ::PodLogs::ElasticsearchService do
end
describe '#pod_logs' do
let(:cluster) { create(:cluster, :provided_by_gcp, environment_scope: '*', projects: [environment.project]) }
let(:result_arg) do
{
pod_name: pod_name,
......@@ -128,7 +128,6 @@ describe ::PodLogs::ElasticsearchService do
end
before do
create(:deployment, :success, environment: environment)
create(:clusters_applications_elastic_stack, :installed, cluster: cluster)
end
......@@ -138,7 +137,7 @@ describe ::PodLogs::ElasticsearchService do
.and_return(Elasticsearch::Transport::Client.new)
allow_any_instance_of(::Gitlab::Elasticsearch::Logs)
.to receive(:pod_logs)
.with(environment.deployment_namespace, pod_name, container_name, search, start_time, end_time)
.with(namespace, pod_name, container_name, search, start_time, end_time)
.and_return(expected_logs)
result = subject.send(:pod_logs, result_arg)
......
......@@ -5,7 +5,8 @@ require 'spec_helper'
describe ::PodLogs::KubernetesService do
include KubernetesHelpers
let_it_be(:environment, refind: true) { create(:environment) }
let_it_be(:cluster) { create(:cluster, :provided_by_gcp, environment_scope: '*') }
let(:namespace) { 'autodevops-deploy-9-production' }
let(:pod_name) { 'pod-1' }
let(:container_name) { 'container-1' }
......@@ -18,7 +19,7 @@ describe ::PodLogs::KubernetesService do
]
end
subject { described_class.new(environment, params: params) }
subject { described_class.new(cluster, namespace, params: params) }
describe '#pod_logs' do
let(:result_arg) do
......@@ -29,13 +30,8 @@ describe ::PodLogs::KubernetesService do
end
let(:service) { create(:cluster_platform_kubernetes, :configured) }
before do
create(:cluster, :provided_by_gcp, environment_scope: '*', projects: [environment.project])
create(:deployment, :success, environment: environment)
end
it 'returns the logs' do
stub_kubeclient_logs(pod_name, environment.deployment_namespace, container: container_name)
stub_kubeclient_logs(pod_name, namespace, container: container_name)
result = subject.send(:pod_logs, result_arg)
......
......@@ -3935,6 +3935,9 @@ msgstr ""
msgid "Cluster cache cleared."
msgstr ""
msgid "Cluster does not exist"
msgstr ""
msgid "ClusterIntegration| %{custom_domain_start}More information%{custom_domain_end}."
msgstr ""
......@@ -7412,6 +7415,9 @@ msgstr ""
msgid "Environment"
msgstr ""
msgid "Environment does not have deployments"
msgstr ""
msgid "Environment variables are applied to environments via the runner. They can be protected by only exposing them to protected branches or tags. Additionally, they can be masked so they are hidden in job logs, though they must match certain regexp requirements to do so. You can use environment variables for passwords, secret keys, or whatever you want."
msgstr ""
......@@ -12659,6 +12665,9 @@ msgstr ""
msgid "Name:"
msgstr ""
msgid "Namespace is empty"
msgstr ""
msgid "Namespace: %{namespace}"
msgstr ""
......@@ -12916,9 +12925,6 @@ msgstr ""
msgid "No data to display"
msgstr ""
msgid "No deployment platform available"
msgstr ""
msgid "No deployments found"
msgstr ""
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment