Commit 69b91d38 authored by Rémy Coutable's avatar Rémy Coutable Committed by Lin Jen-Shin

Ensure we set 'tiller-namespace' for 'helm' commands

Signed-off-by: default avatarRémy Coutable <remy@rymai.me>
parent d008a870
......@@ -7,7 +7,7 @@ module Quality
class HelmClient
CommandFailedError = Class.new(StandardError)
attr_reader :namespace
attr_reader :tiller_namespace, :namespace
RELEASE_JSON_ATTRIBUTES = %w[Name Revision Updated Status Chart AppVersion Namespace].freeze
......@@ -24,7 +24,8 @@ module Quality
# A single page of data and the corresponding page number.
Page = Struct.new(:releases, :number)
def initialize(namespace:)
def initialize(tiller_namespace:, namespace:)
@tiller_namespace = tiller_namespace
@namespace = namespace
end
......@@ -35,7 +36,7 @@ module Quality
def delete(release_name:)
run_command([
'delete',
%(--tiller-namespace "#{namespace}"),
%(--tiller-namespace "#{tiller_namespace}"),
'--purge',
release_name
])
......@@ -60,7 +61,7 @@ module Quality
command = [
'list',
%(--namespace "#{namespace}"),
%(--tiller-namespace "#{namespace}" --output json),
%(--tiller-namespace "#{tiller_namespace}" --output json),
*args
]
json = JSON.parse(run_command(command))
......
......@@ -25,7 +25,6 @@ class AutomatedCleanup
def initialize(project_path: ENV['CI_PROJECT_PATH'], gitlab_token: ENV['GITLAB_BOT_REVIEW_APPS_CLEANUP_TOKEN'])
@project_path = project_path
@gitlab_token = gitlab_token
ENV['TILLER_NAMESPACE'] ||= review_apps_namespace
end
def gitlab
......@@ -45,7 +44,9 @@ class AutomatedCleanup
end
def helm
@helm ||= Quality::HelmClient.new(namespace: review_apps_namespace)
@helm ||= Quality::HelmClient.new(
tiller_namespace: review_apps_namespace,
namespace: review_apps_namespace)
end
def kubernetes
......
......@@ -2,29 +2,31 @@
function deploy_exists() {
local namespace="${1}"
local deploy="${2}"
echoinfo "Checking if ${deploy} exists in the ${namespace} namespace..." true
local release="${2}"
local deploy_exists
helm status --tiller-namespace "${namespace}" "${deploy}" >/dev/null 2>&1
local deploy_exists=$?
echoinfo "Checking if ${release} exists in the ${namespace} namespace..." true
echoinfo "Deployment status for ${deploy} is ${deploy_exists}"
helm status --tiller-namespace "${namespace}" "${release}" >/dev/null 2>&1
deploy_exists=$?
echoinfo "Deployment status for ${release} is ${deploy_exists}"
return $deploy_exists
}
function previous_deploy_failed() {
local namespace="${1}"
local deploy="${2}"
local release="${2}"
echoinfo "Checking for previous deployment of ${deploy}" true
echoinfo "Checking for previous deployment of ${release}" true
helm status --tiller-namespace "${namespace}" "${deploy}" >/dev/null 2>&1
helm status --tiller-namespace "${namespace}" "${release}" >/dev/null 2>&1
local status=$?
# if `status` is `0`, deployment exists, has a status
if [ $status -eq 0 ]; then
echoinfo "Previous deployment found, checking status..."
deployment_status=$(helm status --tiller-namespace "${namespace}" "${deploy}" | grep ^STATUS | cut -d' ' -f2)
deployment_status=$(helm status --tiller-namespace "${namespace}" "${release}" | grep ^STATUS | cut -d' ' -f2)
echoinfo "Previous deployment state: ${deployment_status}"
if [[ "$deployment_status" == "FAILED" || "$deployment_status" == "PENDING_UPGRADE" || "$deployment_status" == "PENDING_INSTALL" ]]; then
status=0;
......@@ -39,30 +41,33 @@ function previous_deploy_failed() {
function delete_release() {
local namespace="${KUBE_NAMESPACE}"
local deploy="${CI_ENVIRONMENT_SLUG}"
local release="${CI_ENVIRONMENT_SLUG}"
if [ -z "$deploy" ]; then
if [ -z "${release}" ]; then
echoerr "No release given, aborting the delete!"
return
fi
echoinfo "Deleting release '$deploy'..." true
echoinfo "Deleting release '${release}'..." true
helm delete --purge --tiller-namespace "${namespace}" "${deploy}"
helm delete --tiller-namespace "${namespace}" --purge "${release}"
}
function delete_failed_release() {
if [ -z "$CI_ENVIRONMENT_SLUG" ]; then
local namespace="${KUBE_NAMESPACE}"
local release="${CI_ENVIRONMENT_SLUG}"
if [ -z "${release}" ]; then
echoerr "No release given, aborting the delete!"
return
fi
if ! deploy_exists "${KUBE_NAMESPACE}" "${CI_ENVIRONMENT_SLUG}"; then
echoinfo "No Review App with ${CI_ENVIRONMENT_SLUG} is currently deployed."
if ! deploy_exists "${namespace}" "${release}"; then
echoinfo "No Review App with ${release} is currently deployed."
else
# Cleanup and previous installs, as FAILED and PENDING_UPGRADE will cause errors with `upgrade`
if previous_deploy_failed "${KUBE_NAMESPACE}" "$CI_ENVIRONMENT_SLUG" ; then
echoinfo "Review App deployment in bad state, cleaning up $CI_ENVIRONMENT_SLUG"
if previous_deploy_failed "${namespace}" "${release}" ; then
echoinfo "Review App deployment in bad state, cleaning up ${release}"
delete_release
else
echoinfo "Review App deployment in good state"
......@@ -72,9 +77,12 @@ function delete_failed_release() {
function get_pod() {
local namespace="${KUBE_NAMESPACE}"
local release="${CI_ENVIRONMENT_SLUG}"
local app_name="${1}"
local status="${2-Running}"
get_pod_cmd="kubectl get pods -n ${KUBE_NAMESPACE} --field-selector=status.phase=${status} -lapp=${app_name},release=${CI_ENVIRONMENT_SLUG} --no-headers -o=custom-columns=NAME:.metadata.name | tail -n 1"
get_pod_cmd="kubectl get pods --namespace ${namespace} --field-selector=status.phase=${status} -lapp=${app_name},release=${release} --no-headers -o=custom-columns=NAME:.metadata.name | tail -n 1"
echoinfo "Waiting till '${app_name}' pod is ready" true
echoinfo "Running '${get_pod_cmd}'"
......@@ -113,20 +121,24 @@ function check_kube_domain() {
}
function ensure_namespace() {
echoinfo "Ensuring the ${KUBE_NAMESPACE} namespace exists..." true
local namespace="${KUBE_NAMESPACE}"
kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE"
echoinfo "Ensuring the ${namespace} namespace exists..." true
kubectl describe namespace "${namespace}" || kubectl create namespace "${namespace}"
}
function install_tiller() {
local TILLER_NAMESPACE="$KUBE_NAMESPACE"
echoinfo "Checking deployment/tiller-deploy status in the ${TILLER_NAMESPACE} namespace..." true
local namespace="${KUBE_NAMESPACE}"
echoinfo "Checking deployment/tiller-deploy status in the ${namespace} namespace..." true
echoinfo "Initiating the Helm client..."
helm init --client-only
# Set toleration for Tiller to be installed on a specific node pool
helm init \
--tiller-namespace "${namespace}" \
--wait \
--upgrade \
--node-selectors "app=helm" \
......@@ -134,37 +146,40 @@ function install_tiller() {
--override "spec.template.spec.tolerations[0].key"="dedicated" \
--override "spec.template.spec.tolerations[0].operator"="Equal" \
--override "spec.template.spec.tolerations[0].value"="helm" \
--override "spec.template.spec.tolerations[0].effect"="NoSchedule" \
--tiller-namespace "${TILLER_NAMESPACE}"
--override "spec.template.spec.tolerations[0].effect"="NoSchedule"
kubectl rollout status -n "$TILLER_NAMESPACE" -w "deployment/tiller-deploy"
kubectl rollout status --namespace "${namespace}" --watch "deployment/tiller-deploy"
if ! helm version --debug --tiller-namespace "${TILLER_NAMESPACE}"; then
if ! helm version --tiller-namespace "${namespace}" --debug; then
echo "Failed to init Tiller."
return 1
fi
}
function install_external_dns() {
local release_name="dns-gitlab-review-app"
local namespace="${KUBE_NAMESPACE}"
local release="dns-gitlab-review-app"
local domain
domain=$(echo "${REVIEW_APPS_DOMAIN}" | awk -F. '{printf "%s.%s", $(NF-1), $NF}')
echoinfo "Installing external DNS for domain ${domain}..." true
if ! deploy_exists "${KUBE_NAMESPACE}" "${release_name}" || previous_deploy_failed "${KUBE_NAMESPACE}" "${release_name}" ; then
if ! deploy_exists "${namespace}" "${release}" || previous_deploy_failed "${namespace}" "${release}" ; then
echoinfo "Installing external-dns Helm chart"
helm repo update
helm repo update --tiller-namespace "${namespace}"
# Default requested: CPU => 0, memory => 0
helm install stable/external-dns --version '^2.2.1' \
-n "${release_name}" \
--namespace "${KUBE_NAMESPACE}" \
helm install stable/external-dns \
--tiller-namespace "${namespace}" \
--namespace "${namespace}" \
--version '^2.2.1' \
--name "${release}" \
--set provider="aws" \
--set aws.credentials.secretKey="${REVIEW_APPS_AWS_SECRET_KEY}" \
--set aws.credentials.accessKey="${REVIEW_APPS_AWS_ACCESS_KEY}" \
--set aws.zoneType="public" \
--set aws.batchChangeSize=400 \
--set domainFilters[0]="${domain}" \
--set txtOwnerId="${KUBE_NAMESPACE}" \
--set txtOwnerId="${namespace}" \
--set rbac.create="true" \
--set policy="sync" \
--set resources.requests.cpu=50m \
......@@ -177,21 +192,24 @@ function install_external_dns() {
}
function create_application_secret() {
echoinfo "Creating the ${CI_ENVIRONMENT_SLUG}-gitlab-initial-root-password secret in the ${KUBE_NAMESPACE} namespace..." true
local namespace="${KUBE_NAMESPACE}"
local release="${CI_ENVIRONMENT_SLUG}"
kubectl create secret generic -n "$KUBE_NAMESPACE" \
"${CI_ENVIRONMENT_SLUG}-gitlab-initial-root-password" \
echoinfo "Creating the ${release}-gitlab-initial-root-password secret in the ${namespace} namespace..." true
kubectl create secret generic --namespace "${namespace}" \
"${release}-gitlab-initial-root-password" \
--from-literal="password=${REVIEW_APPS_ROOT_PASSWORD}" \
--dry-run -o json | kubectl apply -f -
if [ -z "${REVIEW_APPS_EE_LICENSE}" ]; then echo "License not found" && return; fi
echoinfo "Creating the ${CI_ENVIRONMENT_SLUG}-gitlab-license secret in the ${KUBE_NAMESPACE} namespace..." true
echoinfo "Creating the ${release}-gitlab-license secret in the ${namespace} namespace..." true
echo "${REVIEW_APPS_EE_LICENSE}" > /tmp/license.gitlab
kubectl create secret generic -n "$KUBE_NAMESPACE" \
"${CI_ENVIRONMENT_SLUG}-gitlab-license" \
kubectl create secret generic --namespace "${namespace}" \
"${release}-gitlab-license" \
--from-file=license=/tmp/license.gitlab \
--dry-run -o json | kubectl apply -f -
}
......@@ -217,13 +235,14 @@ function base_config_changed() {
}
function deploy() {
local name="$CI_ENVIRONMENT_SLUG"
local namespace="${KUBE_NAMESPACE}"
local release="${CI_ENVIRONMENT_SLUG}"
local edition="${GITLAB_EDITION-ce}"
local base_config_file_ref="master"
if [[ "$(base_config_changed)" == "true" ]]; then base_config_file_ref="$CI_COMMIT_SHA"; fi
if [[ "$(base_config_changed)" == "true" ]]; then base_config_file_ref="${CI_COMMIT_SHA}"; fi
local base_config_file="https://gitlab.com/gitlab-org/gitlab/raw/${base_config_file_ref}/scripts/review_apps/base-config.yaml"
echoinfo "Deploying ${name}..." true
echoinfo "Deploying ${release}..." true
IMAGE_REPOSITORY="registry.gitlab.com/gitlab-org/build/cng-mirror"
gitlab_migrations_image_repository="${IMAGE_REPOSITORY}/gitlab-rails-${edition}"
......@@ -237,47 +256,49 @@ function deploy() {
create_application_secret
HELM_CMD=$(cat << EOF
helm upgrade --install \
helm upgrade \
--tiller-namespace="${namespace}" \
--namespace="${namespace}" \
--install \
--wait \
--timeout 900 \
--set ci.branch="$CI_COMMIT_REF_NAME" \
--set ci.commit.sha="$CI_COMMIT_SHORT_SHA" \
--set ci.job.url="$CI_JOB_URL" \
--set ci.pipeline.url="$CI_PIPELINE_URL" \
--set releaseOverride="$CI_ENVIRONMENT_SLUG" \
--set global.hosts.hostSuffix="$HOST_SUFFIX" \
--set global.hosts.domain="$REVIEW_APPS_DOMAIN" \
--set gitlab.migrations.image.repository="$gitlab_migrations_image_repository" \
--set gitlab.migrations.image.tag="$CI_COMMIT_REF_SLUG" \
--set gitlab.gitaly.image.repository="$gitlab_gitaly_image_repository" \
--set gitlab.gitaly.image.tag="v$GITALY_VERSION" \
--set gitlab.gitlab-shell.image.repository="$gitlab_shell_image_repository" \
--set gitlab.gitlab-shell.image.tag="v$GITLAB_SHELL_VERSION" \
--set gitlab.sidekiq.image.repository="$gitlab_sidekiq_image_repository" \
--set gitlab.sidekiq.image.tag="$CI_COMMIT_REF_SLUG" \
--set gitlab.unicorn.image.repository="$gitlab_unicorn_image_repository" \
--set gitlab.unicorn.image.tag="$CI_COMMIT_REF_SLUG" \
--set gitlab.unicorn.workhorse.image="$gitlab_workhorse_image_repository" \
--set gitlab.unicorn.workhorse.tag="$CI_COMMIT_REF_SLUG" \
--set gitlab.task-runner.image.repository="$gitlab_task_runner_image_repository" \
--set gitlab.task-runner.image.tag="$CI_COMMIT_REF_SLUG"
--set ci.branch="${CI_COMMIT_REF_NAME}" \
--set ci.commit.sha="${CI_COMMIT_SHORT_SHA}" \
--set ci.job.url="${CI_JOB_URL}" \
--set ci.pipeline.url="${CI_PIPELINE_URL}" \
--set releaseOverride="${release}" \
--set global.hosts.hostSuffix="${HOST_SUFFIX}" \
--set global.hosts.domain="${REVIEW_APPS_DOMAIN}" \
--set gitlab.migrations.image.repository="${gitlab_migrations_image_repository}" \
--set gitlab.migrations.image.tag="${CI_COMMIT_REF_SLUG}" \
--set gitlab.gitaly.image.repository="${gitlab_gitaly_image_repository}" \
--set gitlab.gitaly.image.tag="v${GITALY_VERSION}" \
--set gitlab.gitlab-shell.image.repository="${gitlab_shell_image_repository}" \
--set gitlab.gitlab-shell.image.tag="v${GITLAB_SHELL_VERSION}" \
--set gitlab.sidekiq.image.repository="${gitlab_sidekiq_image_repository}" \
--set gitlab.sidekiq.image.tag="${CI_COMMIT_REF_SLUG}" \
--set gitlab.unicorn.image.repository="${gitlab_unicorn_image_repository}" \
--set gitlab.unicorn.image.tag="${CI_COMMIT_REF_SLUG}" \
--set gitlab.unicorn.workhorse.image="${gitlab_workhorse_image_repository}" \
--set gitlab.unicorn.workhorse.tag="${CI_COMMIT_REF_SLUG}" \
--set gitlab.task-runner.image.repository="${gitlab_task_runner_image_repository}" \
--set gitlab.task-runner.image.tag="${CI_COMMIT_REF_SLUG}"
EOF
)
if [ -n "${REVIEW_APPS_EE_LICENSE}" ]; then
HELM_CMD=$(cat << EOF
${HELM_CMD} \
--set global.gitlab.license.secret="${CI_ENVIRONMENT_SLUG}-gitlab-license"
--set global.gitlab.license.secret="${release}-gitlab-license"
EOF
)
fi
HELM_CMD=$(cat << EOF
${HELM_CMD} \
--namespace="$KUBE_NAMESPACE" \
--version="${CI_PIPELINE_ID}-${CI_JOB_ID}" \
-f "${base_config_file}" \
"${name}" .
"${release}" .
EOF
)
......@@ -288,11 +309,14 @@ EOF
}
function display_deployment_debug() {
local namespace="${KUBE_NAMESPACE}"
local release="${CI_ENVIRONMENT_SLUG}"
# Get all pods for this release
echoinfo "Pods for release ${CI_ENVIRONMENT_SLUG}"
kubectl get pods -n "$KUBE_NAMESPACE" -lrelease=${CI_ENVIRONMENT_SLUG}
echoinfo "Pods for release ${release}"
kubectl get pods --namespace "${namespace}" -lrelease=${release}
# Get all non-completed jobs
echoinfo "Unsuccessful Jobs for release ${CI_ENVIRONMENT_SLUG}"
kubectl get jobs -n "$KUBE_NAMESPACE" -lrelease=${CI_ENVIRONMENT_SLUG} --field-selector=status.successful!=1
echoinfo "Unsuccessful Jobs for release ${release}"
kubectl get jobs --namespace "${namespace}" -lrelease=${release} --field-selector=status.successful!=1
}
......@@ -3,7 +3,8 @@
require 'fast_spec_helper'
RSpec.describe Quality::HelmClient do
let(:namespace) { 'review-apps-ee' }
let(:tiller_namespace) { 'review-apps-ee' }
let(:namespace) { tiller_namespace }
let(:release_name) { 'my-release' }
let(:raw_helm_list_page1) do
<<~OUTPUT
......@@ -30,12 +31,12 @@ RSpec.describe Quality::HelmClient do
OUTPUT
end
subject { described_class.new(namespace: namespace) }
subject { described_class.new(tiller_namespace: tiller_namespace, namespace: namespace) }
describe '#releases' do
it 'raises an error if the Helm command fails' do
expect(Gitlab::Popen).to receive(:popen_with_detail)
.with([%(helm list --namespace "#{namespace}" --tiller-namespace "#{namespace}" --output json)])
.with([%(helm list --namespace "#{namespace}" --tiller-namespace "#{tiller_namespace}" --output json)])
.and_return(Gitlab::Popen::Result.new([], '', '', double(success?: false)))
expect { subject.releases.to_a }.to raise_error(described_class::CommandFailedError)
......@@ -43,7 +44,7 @@ RSpec.describe Quality::HelmClient do
it 'calls helm list with default arguments' do
expect(Gitlab::Popen).to receive(:popen_with_detail)
.with([%(helm list --namespace "#{namespace}" --tiller-namespace "#{namespace}" --output json)])
.with([%(helm list --namespace "#{namespace}" --tiller-namespace "#{tiller_namespace}" --output json)])
.and_return(Gitlab::Popen::Result.new([], '', '', double(success?: true)))
subject.releases.to_a
......@@ -51,7 +52,7 @@ RSpec.describe Quality::HelmClient do
it 'calls helm list with extra arguments' do
expect(Gitlab::Popen).to receive(:popen_with_detail)
.with([%(helm list --namespace "#{namespace}" --tiller-namespace "#{namespace}" --output json --deployed)])
.with([%(helm list --namespace "#{namespace}" --tiller-namespace "#{tiller_namespace}" --output json --deployed)])
.and_return(Gitlab::Popen::Result.new([], '', '', double(success?: true)))
subject.releases(args: ['--deployed']).to_a
......@@ -59,7 +60,7 @@ RSpec.describe Quality::HelmClient do
it 'returns a list of Release objects' do
expect(Gitlab::Popen).to receive(:popen_with_detail)
.with([%(helm list --namespace "#{namespace}" --tiller-namespace "#{namespace}" --output json --deployed)])
.with([%(helm list --namespace "#{namespace}" --tiller-namespace "#{tiller_namespace}" --output json --deployed)])
.and_return(Gitlab::Popen::Result.new([], raw_helm_list_page2, '', double(success?: true)))
releases = subject.releases(args: ['--deployed']).to_a
......@@ -78,10 +79,10 @@ RSpec.describe Quality::HelmClient do
it 'automatically paginates releases' do
expect(Gitlab::Popen).to receive(:popen_with_detail).ordered
.with([%(helm list --namespace "#{namespace}" --tiller-namespace "#{namespace}" --output json)])
.with([%(helm list --namespace "#{namespace}" --tiller-namespace "#{tiller_namespace}" --output json)])
.and_return(Gitlab::Popen::Result.new([], raw_helm_list_page1, '', double(success?: true)))
expect(Gitlab::Popen).to receive(:popen_with_detail).ordered
.with([%(helm list --namespace "#{namespace}" --tiller-namespace "#{namespace}" --output json --offset review-6709-group-t40qbv)])
.with([%(helm list --namespace "#{namespace}" --tiller-namespace "#{tiller_namespace}" --output json --offset review-6709-group-t40qbv)])
.and_return(Gitlab::Popen::Result.new([], raw_helm_list_page2, '', double(success?: true)))
releases = subject.releases.to_a
......@@ -94,7 +95,7 @@ RSpec.describe Quality::HelmClient do
describe '#delete' do
it 'raises an error if the Helm command fails' do
expect(Gitlab::Popen).to receive(:popen_with_detail)
.with([%(helm delete --tiller-namespace "#{namespace}" --purge #{release_name})])
.with([%(helm delete --tiller-namespace "#{tiller_namespace}" --purge #{release_name})])
.and_return(Gitlab::Popen::Result.new([], '', '', double(success?: false)))
expect { subject.delete(release_name: release_name) }.to raise_error(described_class::CommandFailedError)
......@@ -102,7 +103,7 @@ RSpec.describe Quality::HelmClient do
it 'calls helm delete with default arguments' do
expect(Gitlab::Popen).to receive(:popen_with_detail)
.with([%(helm delete --tiller-namespace "#{namespace}" --purge #{release_name})])
.with([%(helm delete --tiller-namespace "#{tiller_namespace}" --purge #{release_name})])
.and_return(Gitlab::Popen::Result.new([], '', '', double(success?: true)))
expect(subject.delete(release_name: release_name)).to eq('')
......@@ -113,7 +114,7 @@ RSpec.describe Quality::HelmClient do
it 'raises an error if the Helm command fails' do
expect(Gitlab::Popen).to receive(:popen_with_detail)
.with([%(helm delete --tiller-namespace "#{namespace}" --purge #{release_name.join(' ')})])
.with([%(helm delete --tiller-namespace "#{tiller_namespace}" --purge #{release_name.join(' ')})])
.and_return(Gitlab::Popen::Result.new([], '', '', double(success?: false)))
expect { subject.delete(release_name: release_name) }.to raise_error(described_class::CommandFailedError)
......@@ -121,7 +122,7 @@ RSpec.describe Quality::HelmClient do
it 'calls helm delete with multiple release names' do
expect(Gitlab::Popen).to receive(:popen_with_detail)
.with([%(helm delete --tiller-namespace "#{namespace}" --purge #{release_name.join(' ')})])
.with([%(helm delete --tiller-namespace "#{tiller_namespace}" --purge #{release_name.join(' ')})])
.and_return(Gitlab::Popen::Result.new([], '', '', double(success?: true)))
expect(subject.delete(release_name: release_name)).to eq('')
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment