review-apps.sh 13.7 KB
Newer Older
1 2 3
[[ "$TRACE" ]] && set -x
export TILLER_NAMESPACE="$KUBE_NAMESPACE"

4
function deploy_exists() {
5 6 7 8 9 10 11 12 13 14 15
  local namespace="${1}"
  local deploy="${2}"
  echoinfo "Checking if ${deploy} exists in the ${namespace} namespace..." true

  helm status --tiller-namespace "${namespace}" "${deploy}" >/dev/null 2>&1
  local deploy_exists=$?

  echoinfo "Deployment status for ${deploy} is ${deploy_exists}"
  return $deploy_exists
}

16
function previous_deploy_failed() {
17 18 19
  local deploy="${1}"
  echoinfo "Checking for previous deployment of ${deploy}" true

20
  helm status "${deploy}" >/dev/null 2>&1
21 22 23 24 25
  local status=$?

  # if `status` is `0`, deployment exists, has a status
  if [ $status -eq 0 ]; then
    echoinfo "Previous deployment found, checking status..."
26
    deployment_status=$(helm status "${deploy}" | grep ^STATUS | cut -d' ' -f2)
27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48
    echoinfo "Previous deployment state: ${deployment_status}"
    if [[ "$deployment_status" == "FAILED" || "$deployment_status" == "PENDING_UPGRADE" || "$deployment_status" == "PENDING_INSTALL" ]]; then
      status=0;
    else
      status=1;
    fi
  else
    echoerr "Previous deployment NOT found."
  fi
  return $status
}

function delete() {
  if [ -z "$CI_ENVIRONMENT_SLUG" ]; then
    echoerr "No release given, aborting the delete!"
    return
  fi

  local name="$CI_ENVIRONMENT_SLUG"

  echoinfo "Deleting release '$name'..." true

49
  helm delete --purge "$name"
50 51 52 53 54
}

function get_pod() {
  local app_name="${1}"
  local status="${2-Running}"
Rémy Coutable's avatar
Rémy Coutable committed
55
  get_pod_cmd="kubectl get pods -n ${KUBE_NAMESPACE} --field-selector=status.phase=${status} -lapp=${app_name},release=${CI_ENVIRONMENT_SLUG} --no-headers -o=custom-columns=NAME:.metadata.name | tail -n 1"
56 57
  echoinfo "Waiting till '${app_name}' pod is ready" true
  echoinfo "Running '${get_pod_cmd}'"
58

59 60 61
  local interval=5
  local elapsed_seconds=0
  local max_seconds=$((2 * 60))
62
  while true; do
63 64
    local pod_name
    pod_name="$(eval "${get_pod_cmd}")"
65 66
    [[ "${pod_name}" == "" ]] || break

67 68
    if [[ "${elapsed_seconds}" -gt "${max_seconds}" ]]; then
      echoerr "The pod name couldn't be found after ${elapsed_seconds} seconds, aborting."
69
      break
70 71 72 73 74
    fi

    printf "."
    let "elapsed_seconds+=interval"
    sleep ${interval}
75 76 77 78 79
  done

  echoinfo "The pod name is '${pod_name}'."
  echo "${pod_name}"
}
80

81
function check_kube_domain() {
82 83
  echoinfo "Checking that Kube domain exists..." true

84 85 86 87 88 89 90 91 92 93 94
  if [ -z ${REVIEW_APPS_DOMAIN+x} ]; then
    echo "In order to deploy or use Review Apps, REVIEW_APPS_DOMAIN variable must be set"
    echo "You can do it in Auto DevOps project settings or defining a variable at group or project level"
    echo "You can also manually add it in .gitlab-ci.yml"
    false
  else
    true
  fi
}

function ensure_namespace() {
95 96
  echoinfo "Ensuring the ${KUBE_NAMESPACE} namespace exists..." true

97 98 99 100
  kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE"
}

function install_tiller() {
101 102 103 104 105
  echoinfo "Checking deployment/tiller-deploy status in the ${TILLER_NAMESPACE} namespace..." true

  echoinfo "Initiating the Helm client..."
  helm init --client-only

106
  # Set toleration for Tiller to be installed on a specific node pool
107
  helm init \
108
    --wait \
109
    --upgrade \
110 111 112 113 114 115
    --node-selectors "app=helm" \
    --replicas 3 \
    --override "spec.template.spec.tolerations[0].key"="dedicated" \
    --override "spec.template.spec.tolerations[0].operator"="Equal" \
    --override "spec.template.spec.tolerations[0].value"="helm" \
    --override "spec.template.spec.tolerations[0].effect"="NoSchedule"
116

117
  kubectl rollout status -n "$TILLER_NAMESPACE" -w "deployment/tiller-deploy"
118

119 120 121 122
  if ! helm version --debug; then
    echo "Failed to init Tiller."
    return 1
  fi
123 124 125 126
}

function install_external_dns() {
  local release_name="dns-gitlab-review-app"
127 128
  local domain
  domain=$(echo "${REVIEW_APPS_DOMAIN}" | awk -F. '{printf "%s.%s", $(NF-1), $NF}')
129 130
  echoinfo "Installing external DNS for domain ${domain}..." true

131
  if ! deploy_exists "${KUBE_NAMESPACE}" "${release_name}" || previous_deploy_failed "${release_name}" ; then
132 133
    echoinfo "Installing external-dns Helm chart"
    helm repo update
134
    # Default requested: CPU => 0, memory => 0
135
    helm install stable/external-dns --version '^2.2.1' \
136 137 138
      -n "${release_name}" \
      --namespace "${KUBE_NAMESPACE}" \
      --set provider="aws" \
139 140
      --set aws.credentials.secretKey="${REVIEW_APPS_AWS_SECRET_KEY}" \
      --set aws.credentials.accessKey="${REVIEW_APPS_AWS_ACCESS_KEY}" \
141
      --set aws.zoneType="public" \
142
      --set aws.batchChangeSize=400 \
143 144 145
      --set domainFilters[0]="${domain}" \
      --set txtOwnerId="${KUBE_NAMESPACE}" \
      --set rbac.create="true" \
146 147 148 149 150
      --set policy="sync" \
      --set resources.requests.cpu=50m \
      --set resources.limits.cpu=100m \
      --set resources.requests.memory=100M \
      --set resources.limits.memory=200M
151 152 153
  else
    echoinfo "The external-dns Helm chart is already successfully deployed."
  fi
154 155
}

156
function create_application_secret() {
157
  echoinfo "Creating the ${CI_ENVIRONMENT_SLUG}-gitlab-initial-root-password secret in the ${KUBE_NAMESPACE} namespace..." true
158 159

  kubectl create secret generic -n "$KUBE_NAMESPACE" \
160 161
    "${CI_ENVIRONMENT_SLUG}-gitlab-initial-root-password" \
    --from-literal="password=${REVIEW_APPS_ROOT_PASSWORD}" \
162 163 164
    --dry-run -o json | kubectl apply -f -
}

165
function download_chart() {
166
  echoinfo "Downloading the GitLab chart..." true
Ian Baum's avatar
Ian Baum committed
167

168
  curl -o gitlab.tar.bz2 "https://gitlab.com/charts/gitlab/-/archive/${GITLAB_HELM_CHART_REF}/gitlab-${GITLAB_HELM_CHART_REF}.tar.bz2"
169
  tar -xjf gitlab.tar.bz2
170
  cd "gitlab-${GITLAB_HELM_CHART_REF}"
171 172 173 174 175 176

  echoinfo "Adding the gitlab repo to Helm..."
  helm repo add gitlab https://charts.gitlab.io

  echoinfo "Building the gitlab chart's dependencies..."
  helm dependency build .
177 178 179
}

function deploy() {
180 181
  local name="$CI_ENVIRONMENT_SLUG"
  echoinfo "Deploying ${name}..." true
182

183 184 185 186 187 188 189 190 191
  IMAGE_REPOSITORY="registry.gitlab.com/gitlab-org/build/cng-mirror"
  IMAGE_VERSION="${CI_PROJECT_NAME#gitlab-}"
  gitlab_migrations_image_repository="${IMAGE_REPOSITORY}/gitlab-rails-${IMAGE_VERSION}"
  gitlab_sidekiq_image_repository="${IMAGE_REPOSITORY}/gitlab-sidekiq-${IMAGE_VERSION}"
  gitlab_unicorn_image_repository="${IMAGE_REPOSITORY}/gitlab-unicorn-${IMAGE_VERSION}"
  gitlab_task_runner_image_repository="${IMAGE_REPOSITORY}/gitlab-task-runner-${IMAGE_VERSION}"
  gitlab_gitaly_image_repository="${IMAGE_REPOSITORY}/gitaly"
  gitlab_shell_image_repository="${IMAGE_REPOSITORY}/gitlab-shell"
  gitlab_workhorse_image_repository="${IMAGE_REPOSITORY}/gitlab-workhorse-${IMAGE_VERSION}"
192 193

  # Cleanup and previous installs, as FAILED and PENDING_UPGRADE will cause errors with `upgrade`
194
  if [ "$CI_ENVIRONMENT_SLUG" != "production" ] && previous_deploy_failed "$CI_ENVIRONMENT_SLUG" ; then
195 196 197
    echo "Deployment in bad state, cleaning up $CI_ENVIRONMENT_SLUG"
    delete
  fi
198

199
  create_application_secret
200 201 202 203

HELM_CMD=$(cat << EOF
  helm upgrade --install \
    --wait \
204
    --timeout 900 \
205
    --set releaseOverride="$CI_ENVIRONMENT_SLUG" \
206
    --set global.appConfig.enableUsagePing=false \
207
    --set global.imagePullPolicy=Always \
208 209 210 211
    --set global.hosts.hostSuffix="$HOST_SUFFIX" \
    --set global.hosts.domain="$REVIEW_APPS_DOMAIN" \
    --set global.ingress.configureCertmanager=false \
    --set global.ingress.tls.secretName=tls-cert \
212
    --set global.ingress.annotations."external-dns\.alpha\.kubernetes\.io/ttl"="10" \
213 214
    --set certmanager.install=false \
    --set prometheus.install=false \
215
    --set nginx-ingress.controller.service.enableHttp=false \
216
    --set nginx-ingress.controller.replicaCount=2 \
217
    --set nginx-ingress.controller.config.ssl-ciphers="ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4" \
218
    --set gitlab.migrations.image.repository="$gitlab_migrations_image_repository" \
219
    --set gitlab.migrations.image.tag="$CI_COMMIT_REF_SLUG" \
220
    --set gitlab.gitaly.image.repository="$gitlab_gitaly_image_repository" \
221
    --set gitlab.gitaly.image.tag="v$GITALY_VERSION" \
222
    --set gitlab.gitlab-shell.image.repository="$gitlab_shell_image_repository" \
223
    --set gitlab.gitlab-shell.image.tag="v$GITLAB_SHELL_VERSION" \
224 225 226 227
    --set gitlab.sidekiq.image.repository="$gitlab_sidekiq_image_repository" \
    --set gitlab.sidekiq.image.tag="$CI_COMMIT_REF_SLUG" \
    --set gitlab.unicorn.image.repository="$gitlab_unicorn_image_repository" \
    --set gitlab.unicorn.image.tag="$CI_COMMIT_REF_SLUG" \
228
    --set gitlab.unicorn.workhorse.image="$gitlab_workhorse_image_repository" \
229
    --set gitlab.unicorn.workhorse.tag="$CI_COMMIT_REF_SLUG" \
230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320
    --set gitlab.task-runner.image.repository="$gitlab_task_runner_image_repository" \
    --set gitlab.task-runner.image.tag="$CI_COMMIT_REF_SLUG"
EOF
)

# Default requested: CPU => 100m, memory => 100Mi
HELM_CMD=$(cat << EOF
  $HELM_CMD \
  --set nginx-ingress.controller.resources.limits.cpu=200m \
  --set nginx-ingress.controller.resources.requests.memory=210M \
  --set nginx-ingress.controller.resources.limits.memory=420M
EOF
)

# Default requested: CPU => 5m, memory => 5Mi
HELM_CMD=$(cat << EOF
  $HELM_CMD \
  --set nginx-ingress.defaultBackend.resources.limits.cpu=10m \
  --set nginx-ingress.defaultBackend.resources.requests.memory=12M \
  --set nginx-ingress.defaultBackend.resources.limits.memory=24M
EOF
)

# Default requested: CPU => 100m, memory => 200Mi
HELM_CMD=$(cat << EOF
  $HELM_CMD \
  --set gitlab.gitaly.resources.requests.cpu=150m \
  --set gitlab.gitaly.resources.limits.cpu=300m \
  --set gitlab.gitaly.resources.limits.memory=420M
EOF
)

# Default requested: CPU => 0, memory => 6M
HELM_CMD=$(cat << EOF
  $HELM_CMD \
  --set gitlab.gitlab-shell.resources.requests.cpu=70m \
  --set gitlab.gitlab-shell.resources.limits.cpu=140m \
  --set gitlab.gitlab-shell.resources.requests.memory=20M \
  --set gitlab.gitlab-shell.resources.limits.memory=40M
EOF
)

# Default requested: CPU => 50m, memory => 650M
HELM_CMD=$(cat << EOF
  $HELM_CMD \
  --set gitlab.sidekiq.resources.requests.cpu=200m \
  --set gitlab.sidekiq.resources.limits.cpu=300m \
  --set gitlab.sidekiq.resources.requests.memory=800M \
  --set gitlab.sidekiq.resources.limits.memory=1.2G
EOF
)

# Default requested: CPU => 300m + 100m (workhorse), memory => 1.2G + 100M (workhorse)
HELM_CMD=$(cat << EOF
  $HELM_CMD \
  --set gitlab.unicorn.resources.limits.cpu=800m \
  --set gitlab.unicorn.resources.limits.memory=2.6G
EOF
)

# Default requested: CPU => 100m, memory => 64Mi
HELM_CMD=$(cat << EOF
  $HELM_CMD \
  --set redis.resources.limits.cpu=200m \
  --set redis.resources.limits.memory=130M
EOF
)

# Default requested: CPU => 100m, memory => 128Mi
HELM_CMD=$(cat << EOF
  $HELM_CMD \
  --set minio.resources.limits.cpu=200m \
  --set minio.resources.limits.memory=280M
EOF
)

# Default requested: CPU => 0, memory => 0
HELM_CMD=$(cat << EOF
  $HELM_CMD \
  --set gitlab-runner.resources.requests.cpu=300m \
  --set gitlab-runner.resources.limits.cpu=600m \
  --set gitlab-runner.resources.requests.memory=300M \
  --set gitlab-runner.resources.limits.memory=600M
EOF
)

HELM_CMD=$(cat << EOF
  $HELM_CMD \
  --namespace="$KUBE_NAMESPACE" \
  --version="$CI_PIPELINE_ID-$CI_JOB_ID" \
  "$name" .
321 322 323
EOF
)

324 325
  echoinfo "Deploying with:"
  echoinfo "${HELM_CMD}"
326

327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347
  eval "${HELM_CMD}"
}

function display_deployment_debug() {
  migrations_pod=$(get_pod "migrations");
  if [ -z "${migrations_pod}" ]; then
    echoerr "Migrations pod not found."
  else
    echoinfo "Logs tail of the ${migrations_pod} pod..."

    kubectl logs -n "$KUBE_NAMESPACE" "${migrations_pod}" | sed "s/${REVIEW_APPS_ROOT_PASSWORD}/[REDACTED]/g"
  fi

  unicorn_pod=$(get_pod "unicorn");
  if [ -z "${unicorn_pod}" ]; then
    echoerr "Unicorn pod not found."
  else
    echoinfo "Logs tail of the ${unicorn_pod} pod..."

    kubectl logs -n "$KUBE_NAMESPACE" -c unicorn "${unicorn_pod}" | sed "s/${REVIEW_APPS_ROOT_PASSWORD}/[REDACTED]/g"
  fi
348 349 350
}

function wait_for_review_app_to_be_accessible() {
351 352
  echoinfo "Waiting for the Review App at ${CI_ENVIRONMENT_URL} to be accessible..." true

353 354
  local interval=5
  local elapsed_seconds=0
355
  local max_seconds=$((2 * 60))
356
  while true; do
357 358 359 360 361
    local review_app_http_code
    review_app_http_code=$(curl --silent --output /dev/null --max-time 5 --write-out "%{http_code}" "${CI_ENVIRONMENT_URL}/users/sign_in")
    if [[ "${review_app_http_code}" -eq "200" ]] || [[ "${elapsed_seconds}" -gt "${max_seconds}" ]]; then
      break
    fi
362 363 364 365 366 367

    printf "."
    let "elapsed_seconds+=interval"
    sleep ${interval}
  done

368 369
  if [[ "${review_app_http_code}" -eq "200" ]]; then
    echoinfo "The Review App at ${CI_ENVIRONMENT_URL} is ready after ${elapsed_seconds} seconds!"
370
  else
371
    echoerr "The Review App at ${CI_ENVIRONMENT_URL} isn't ready after ${max_seconds} seconds of polling..."
372 373
    exit 1
  fi
374 375
}

376 377 378 379 380 381
function add_license() {
  if [ -z "${REVIEW_APPS_EE_LICENSE}" ]; then echo "License not found" && return; fi

  task_runner_pod=$(get_pod "task-runner");
  if [ -z "${task_runner_pod}" ]; then echo "Task runner pod not found" && return; fi

382 383
  echoinfo "Installing license..." true

384
  echo "${REVIEW_APPS_EE_LICENSE}" > /tmp/license.gitlab
385
  kubectl -n "$KUBE_NAMESPACE" cp /tmp/license.gitlab "${task_runner_pod}":/tmp/license.gitlab
386 387
  rm /tmp/license.gitlab

388
  kubectl -n "$KUBE_NAMESPACE" exec -it "${task_runner_pod}" -- /srv/gitlab/bin/rails runner -e production \
389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405
    '
    content = File.read("/tmp/license.gitlab").strip;
    FileUtils.rm_f("/tmp/license.gitlab");

    unless License.where(data:content).empty?
      puts "License already exists";
      Kernel.exit 0;
    end

    unless License.new(data: content).save
      puts "Could not add license";
      Kernel.exit 0;
    end

    puts "License added";
    '
}