Commit 65c16bcd authored by Douglas Barbosa Alexandre's avatar Douglas Barbosa Alexandre

Merge branch 'ce-improve-review-apps' into 'master'

[CE] Improve review apps

See merge request gitlab-org/gitlab-ce!21087
parents 09203420 3299680c
# frozen_string_literal: true
require 'time'
require_relative '../gitlab/popen' unless defined?(Gitlab::Popen)
module Quality
class HelmClient
attr_reader :namespace
Release = Struct.new(:name, :revision, :last_update, :status, :chart, :namespace) do
def revision
@revision ||= self[:revision].to_i
end
def last_update
@last_update ||= Time.parse(self[:last_update])
end
end
def initialize(namespace: ENV['KUBE_NAMESPACE'])
@namespace = namespace
end
def releases(args: [])
command = ['list', %(--namespace "#{namespace}"), *args]
run_command(command)
.stdout
.lines
.select { |line| line.include?(namespace) }
.map { |line| Release.new(*line.split(/\t/).map(&:strip)) }
end
def delete(release_name:)
run_command(['delete', '--purge', release_name])
end
private
def run_command(command)
final_command = ['helm', *command].join(' ')
puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output
Gitlab::Popen.popen_with_detail([final_command])
end
end
end
# frozen_string_literal: true
require_relative '../gitlab/popen' unless defined?(Gitlab::Popen)
module Quality
class KubernetesClient
attr_reader :namespace
def initialize(namespace: ENV['KUBE_NAMESPACE'])
@namespace = namespace
end
def cleanup(release_name:)
command = ['kubectl']
command << %(-n "#{namespace}" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa 2>&1)
command << '|' << %(grep "#{release_name}")
command << '|' << "awk '{print $1}'"
command << '|' << %(xargs kubectl -n "#{namespace}" delete)
command << '||' << 'true'
run_command(command)
end
private
def run_command(command)
puts "Running command: `#{command.join(' ')}`" # rubocop:disable Rails/Output
Gitlab::Popen.popen_with_detail(command)
end
end
end
# frozen_string_literal: true
require 'gitlab'
require_relative File.expand_path('../../lib/quality/helm_client.rb', __dir__)
require_relative File.expand_path('../../lib/quality/kubernetes_client.rb', __dir__)
class AutomatedCleanup
attr_reader :project_path, :gitlab_token, :cleaned_up_releases
def initialize(project_path: ENV['CI_PROJECT_PATH'], gitlab_token: ENV['GITLAB_BOT_REVIEW_APPS_CLEANUP_TOKEN'])
@project_path = project_path
@gitlab_token = gitlab_token
@cleaned_up_releases = []
end
def gitlab
@gitlab ||= begin
Gitlab.configure do |config|
config.endpoint = 'https://gitlab.com/api/v4'
# gitlab-bot's token "GitLab review apps cleanup"
config.private_token = gitlab_token
end
Gitlab
end
end
def helm
@helm ||= Quality::HelmClient.new
end
def kubernetes
@kubernetes ||= Quality::KubernetesClient.new
end
def perform_gitlab_environment_cleanup!(days_for_stop:, days_for_delete:)
puts "Checking for review apps not updated in the last #{days_for_stop} days..."
checked_environments = []
delete_threshold = threshold_time(days: days_for_delete)
stop_threshold = threshold_time(days: days_for_stop)
gitlab.deployments(project_path, per_page: 50).auto_paginate do |deployment|
next unless deployment.environment.name.start_with?('review/')
next if checked_environments.include?(deployment.environment.slug)
puts
checked_environments << deployment.environment.slug
deployed_at = Time.parse(deployment.created_at)
if deployed_at < delete_threshold
print_release_state(subject: 'Review app', release_name: deployment.environment.slug, release_date: deployment.created_at, action: 'deleting')
gitlab.delete_environment(project_path, deployment.environment.id)
cleaned_up_releases << deployment.environment.slug
elsif deployed_at < stop_threshold
print_release_state(subject: 'Review app', release_name: deployment.environment.slug, release_date: deployment.created_at, action: 'stopping')
gitlab.stop_environment(project_path, deployment.environment.id)
cleaned_up_releases << deployment.environment.slug
else
print_release_state(subject: 'Review app', release_name: deployment.environment.slug, release_date: deployment.created_at, action: 'leaving')
end
end
end
def perform_helm_releases_cleanup!(days:)
puts "Checking for Helm releases not updated in the last #{days} days..."
threshold_day = threshold_time(days: days)
helm.releases(args: ['--deployed', '--failed', '--date', '--reverse', '--max 25']).each do |release|
next if cleaned_up_releases.include?(release.name)
if release.last_update < threshold_day
print_release_state(subject: 'Release', release_name: release.name, release_date: release.last_update, action: 'cleaning')
helm.delete(release_name: release.name)
kubernetes.cleanup(release_name: release.name)
else
print_release_state(subject: 'Release', release_name: release.name, release_date: release.last_update, action: 'leaving')
end
end
end
def threshold_time(days:)
Time.now - days * 24 * 3600
end
def print_release_state(subject:, release_name:, release_date:, action:)
puts "\n#{subject} '#{release_name}' was last deployed on #{release_date}: #{action} it."
end
end
def timed(task)
start = Time.now
yield(self)
puts "#{task} finished in #{Time.now - start} seconds.\n"
end
automated_cleanup = AutomatedCleanup.new
timed('Review apps cleanup') do
automated_cleanup.perform_gitlab_environment_cleanup!(days_for_stop: 5, days_for_delete: 6)
end
puts
timed('Helm releases cleanup') do
automated_cleanup.perform_helm_releases_cleanup!(days: 7)
end
exit(0)
[[ "$TRACE" ]] && set -x
export TILLER_NAMESPACE="$KUBE_NAMESPACE"
function check_kube_domain() {
if [ -z ${REVIEW_APPS_DOMAIN+x} ]; then
echo "In order to deploy or use Review Apps, REVIEW_APPS_DOMAIN variable must be set"
echo "You can do it in Auto DevOps project settings or defining a variable at group or project level"
echo "You can also manually add it in .gitlab-ci.yml"
false
else
true
fi
}
function download_gitlab_chart() {
curl -o gitlab.tar.bz2 https://gitlab.com/charts/gitlab/-/archive/$GITLAB_HELM_CHART_REF/gitlab-$GITLAB_HELM_CHART_REF.tar.bz2
tar -xjf gitlab.tar.bz2
cd gitlab-$GITLAB_HELM_CHART_REF
helm init --client-only
helm repo add gitlab https://charts.gitlab.io
helm dependency update
helm dependency build
}
function ensure_namespace() {
kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE"
}
function install_tiller() {
echo "Checking Tiller..."
helm init --upgrade
kubectl rollout status -n "$TILLER_NAMESPACE" -w "deployment/tiller-deploy"
if ! helm version --debug; then
echo "Failed to init Tiller."
return 1
fi
echo ""
}
function create_secret() {
echo "Create secret..."
kubectl create secret generic -n "$KUBE_NAMESPACE" \
$CI_ENVIRONMENT_SLUG-gitlab-initial-root-password \
--from-literal=password=$REVIEW_APPS_ROOT_PASSWORD \
--dry-run -o json | kubectl apply -f -
}
function previousDeployFailed() {
set +e
echo "Checking for previous deployment of $CI_ENVIRONMENT_SLUG"
deployment_status=$(helm status $CI_ENVIRONMENT_SLUG >/dev/null 2>&1)
status=$?
# if `status` is `0`, deployment exists, has a status
if [ $status -eq 0 ]; then
echo "Previous deployment found, checking status"
deployment_status=$(helm status $CI_ENVIRONMENT_SLUG | grep ^STATUS | cut -d' ' -f2)
echo "Previous deployment state: $deployment_status"
if [[ "$deployment_status" == "FAILED" || "$deployment_status" == "PENDING_UPGRADE" || "$deployment_status" == "PENDING_INSTALL" ]]; then
status=0;
else
status=1;
fi
else
echo "Previous deployment NOT found."
fi
set -e
return $status
}
function deploy() {
track="${1-stable}"
name="$CI_ENVIRONMENT_SLUG"
if [[ "$track" != "stable" ]]; then
name="$name-$track"
fi
replicas="1"
service_enabled="false"
postgres_enabled="$POSTGRES_ENABLED"
gitlab_migrations_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-rails-ce"
gitlab_sidekiq_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-sidekiq-ce"
gitlab_unicorn_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-unicorn-ce"
gitlab_gitaly_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitaly"
gitlab_shell_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-shell"
gitlab_workhorse_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-workhorse-ce"
if [[ "$CI_PROJECT_NAME" == "gitlab-ee" ]]; then
gitlab_migrations_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-rails-ee"
gitlab_sidekiq_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-sidekiq-ee"
gitlab_unicorn_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-unicorn-ee"
gitlab_workhorse_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-workhorse-ee"
fi
# canary uses stable db
[[ "$track" == "canary" ]] && postgres_enabled="false"
env_track=$( echo $track | tr -s '[:lower:]' '[:upper:]' )
env_slug=$( echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]' )
if [[ "$track" == "stable" ]]; then
# for stable track get number of replicas from `PRODUCTION_REPLICAS`
eval new_replicas=\$${env_slug}_REPLICAS
service_enabled="true"
else
# for all tracks get number of replicas from `CANARY_PRODUCTION_REPLICAS`
eval new_replicas=\$${env_track}_${env_slug}_REPLICAS
fi
if [[ -n "$new_replicas" ]]; then
replicas="$new_replicas"
fi
# Cleanup and previous installs, as FAILED and PENDING_UPGRADE will cause errors with `upgrade`
if [ "$CI_ENVIRONMENT_SLUG" != "production" ] && previousDeployFailed ; then
echo "Deployment in bad state, cleaning up $CI_ENVIRONMENT_SLUG"
delete
cleanup
fi
helm repo add gitlab https://charts.gitlab.io/
helm dep update .
HELM_CMD=$(cat << EOF
helm upgrade --install \
--wait \
--timeout 600 \
--set releaseOverride="$CI_ENVIRONMENT_SLUG" \
--set global.hosts.hostSuffix="$HOST_SUFFIX" \
--set global.hosts.domain="$REVIEW_APPS_DOMAIN" \
--set certmanager.install=false \
--set global.ingress.configureCertmanager=false \
--set global.ingress.tls.secretName=tls-cert \
--set global.ingress.annotations."external-dns\.alpha\.kubernetes\.io/ttl"="10"
--set gitlab.unicorn.resources.requests.cpu=200m \
--set gitlab.sidekiq.resources.requests.cpu=100m \
--set gitlab.gitlab-shell.resources.requests.cpu=100m \
--set redis.resources.requests.cpu=100m \
--set minio.resources.requests.cpu=100m \
--set gitlab.migrations.image.repository="$gitlab_migrations_image_repository" \
--set gitlab.migrations.image.tag="$CI_COMMIT_REF_NAME" \
--set gitlab.sidekiq.image.repository="$gitlab_sidekiq_image_repository" \
--set gitlab.sidekiq.image.tag="$CI_COMMIT_REF_NAME" \
--set gitlab.unicorn.image.repository="$gitlab_unicorn_image_repository" \
--set gitlab.unicorn.image.tag="$CI_COMMIT_REF_NAME" \
--set gitlab.gitaly.image.repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitaly" \
--set gitlab.gitaly.image.tag="v$GITALY_VERSION" \
--set gitlab.gitlab-shell.image.repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-shell" \
--set gitlab.gitlab-shell.image.tag="v$GITLAB_SHELL_VERSION" \
--set gitlab.unicorn.workhorse.image="$gitlab_workhorse_image_repository" \
--set gitlab.unicorn.workhorse.tag="$CI_COMMIT_REF_NAME" \
--namespace="$KUBE_NAMESPACE" \
--version="$CI_PIPELINE_ID-$CI_JOB_ID" \
"$name" \
.
EOF
)
echo "Deploying with:"
echo $HELM_CMD
eval $HELM_CMD
}
function delete() {
track="${1-stable}"
name="$CI_ENVIRONMENT_SLUG"
if [[ "$track" != "stable" ]]; then
name="$name-$track"
fi
echo "Deleting release '$name'..."
helm delete --purge "$name" || true
}
function cleanup() {
echo "Cleaning up $CI_ENVIRONMENT_SLUG..."
kubectl -n "$KUBE_NAMESPACE" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa 2>&1 \
| grep "$CI_ENVIRONMENT_SLUG" \
| awk '{print $1}' \
| xargs kubectl -n "$KUBE_NAMESPACE" delete \
|| true
}
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Quality::HelmClient do
let(:namespace) { 'review-apps-ee' }
let(:release_name) { 'my-release' }
let(:raw_helm_list_result) do
<<~OUTPUT
NAME REVISION UPDATED STATUS CHART NAMESPACE
review-improve-re-2dsd9d 1 Tue Jul 31 15:53:17 2018 FAILED gitlab-0.3.4 #{namespace}
review-11-1-stabl-3r2fso 1 Mon Jul 30 22:44:14 2018 FAILED gitlab-0.3.3 #{namespace}
review-49375-css-fk664j 1 Thu Jul 19 11:01:30 2018 FAILED gitlab-0.2.4 #{namespace}
OUTPUT
end
subject { described_class.new(namespace: namespace) }
describe '#releases' do
it 'calls helm list with default arguments' do
expect(Gitlab::Popen).to receive(:popen_with_detail)
.with([%(helm list --namespace "#{namespace}")])
.and_return(Gitlab::Popen::Result.new([], ''))
subject.releases
end
it 'calls helm list with given arguments' do
expect(Gitlab::Popen).to receive(:popen_with_detail)
.with([%(helm list --namespace "#{namespace}" --deployed)])
.and_return(Gitlab::Popen::Result.new([], ''))
subject.releases(args: ['--deployed'])
end
it 'returns a list of Release objects' do
expect(Gitlab::Popen).to receive(:popen_with_detail)
.with([%(helm list --namespace "#{namespace}" --deployed)])
.and_return(Gitlab::Popen::Result.new([], raw_helm_list_result))
releases = subject.releases(args: ['--deployed'])
expect(releases.size).to eq(3)
expect(releases[0].name).to eq('review-improve-re-2dsd9d')
expect(releases[0].revision).to eq(1)
expect(releases[0].last_update).to eq(Time.parse('Tue Jul 31 15:53:17 2018'))
expect(releases[0].status).to eq('FAILED')
expect(releases[0].chart).to eq('gitlab-0.3.4')
expect(releases[0].namespace).to eq(namespace)
end
end
describe '#delete' do
it 'calls helm delete with default arguments' do
expect(Gitlab::Popen).to receive(:popen_with_detail)
.with(["helm delete --purge #{release_name}"])
.and_return(Gitlab::Popen::Result.new([], '', '', 0))
expect(subject.delete(release_name: release_name).status).to eq(0)
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Quality::KubernetesClient do
subject { described_class.new(namespace: 'review-apps-ee') }
describe '#cleanup' do
it 'calls kubectl with the correct arguments' do
# popen_with_detail will receive an array with a bunch of arguments; we're
# only concerned with it having the correct namespace and release name
expect(Gitlab::Popen).to receive(:popen_with_detail) do |args|
expect(args)
.to satisfy_one { |arg| arg.start_with?('-n "review-apps-ee" get') }
expect(args)
.to satisfy_one { |arg| arg == 'grep "my-release"' }
expect(args)
.to satisfy_one { |arg| arg.end_with?('-n "review-apps-ee" delete') }
end
# We're not verifying the output here, just silencing it
expect { subject.cleanup(release_name: 'my-release') }.to output.to_stdout
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment