Review apps include recent charts architecture

Include workhorse container in deploy.
DRY up use of helm command for reuse with print and execute.
Move automated cleanup job into a separate job and run it
in schedules only at the start of the pipeline.

Calculate cleanup thresholds once and use the following frequency:

Stop the envionments after 5 days.
Delete the envionment after 6 days.
Cleanup Helm installation after 7 days.

.gitlab-ci.yml

@@ -1114,8 +1114,7 @@ review:
   image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
   stage: test
   allow_failure: true
-  before_script:
-    - gem install gitlab --no-document
+  before_script: []
   variables:
     GIT_DEPTH: "1"
     HOST_SUFFIX: "$CI_ENVIRONMENT_SLUG"
@@ -1124,8 +1123,8 @@ review:
   script:
     - export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION)
     - export GITALY_VERSION=$(<GITALY_SERVER_VERSION)
+    - export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION)
     - source ./scripts/review_apps/review-apps.sh
-    - ruby -rrubygems scripts/review_apps/automated_cleanup.rb
     - BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng
     - check_kube_domain
     - download_gitlab_chart
@@ -1139,7 +1138,7 @@ review:
     on_stop: stop_review
   only:
     refs:
-      - branches
+      - branches@gitlab-org/gitlab-ee
     kubernetes: active
   except:
     refs:
@@ -1164,8 +1163,28 @@ stop_review:
     action: stop
   only:
     refs:
-      - branches
+      - branches@gitlab-org/gitlab-ee
     kubernetes: active
   except:
     - master
     - /(^docs[\/-].*|.*-docs$)/
+
+automated_review_cleanup:
+  <<: *dedicated-no-docs-pull-cache-job
+  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
+  stage: build
+  allow_failure: true
+  cache: {}
+  dependencies: []
+  before_script:
+    - gem install gitlab --no-document
+  variables:
+    GIT_DEPTH: "1"
+  script:
+    - ruby -rrubygems scripts/review_apps/automated_cleanup.rb
+  only:
+    refs:
+      - schedules@gitlab-org/gitlab-ee
+  except:
+    - master
+    - /(^docs[\/-].*|.*-docs$)/

lib/quality/kubernetes_client.rb

@@ -11,7 +11,8 @@ module Quality
     end
 
     def cleanup(release_name:)
-      command = [%(-n "#{namespace}" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa 2>&1)]
+      command = ['kubectl']
+      command << %(-n "#{namespace}" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa 2>&1)
       command << '|' << %(grep "#{release_name}")
       command << '|' << "awk '{print $1}'"
       command << '|' << %(xargs kubectl -n "#{namespace}" delete)
@@ -23,10 +24,9 @@ module Quality
     private
 
     def run_command(command)
-      final_command = ['kubectl', *command].join(' ')
-      puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output
+      puts "Running command: `#{command.join(' ')}`" # rubocop:disable Rails/Output
 
-      Gitlab::Popen.popen_with_detail([final_command])
+      Gitlab::Popen.popen_with_detail(command)
     end
   end
 end

scripts/review_apps/automated_cleanup.rb

@@ -5,8 +5,6 @@ require_relative File.expand_path('../../lib/quality/helm_client.rb', __dir__)
 require_relative File.expand_path('../../lib/quality/kubernetes_client.rb', __dir__)
 
 class AutomatedCleanup
-  STALE_REVIEW_APP_DAYS_THRESHOLD = 10.freeze
-
   attr_reader :project_path, :gitlab_token, :cleaned_up_releases
 
   def initialize(project_path: ENV['CI_PROJECT_PATH'], gitlab_token: ENV['GITLAB_BOT_REVIEW_APPS_CLEANUP_TOKEN'])
@@ -35,10 +33,11 @@ class AutomatedCleanup
     @kubernetes ||= Quality::KubernetesClient.new
   end
 
-  def perform_gitlab_environment_cleanup!(days_for_stop: STALE_REVIEW_APP_DAYS_THRESHOLD, days_for_delete: STALE_REVIEW_APP_DAYS_THRESHOLD)
+  def perform_gitlab_environment_cleanup!(days_for_stop:, days_for_delete:)
     puts "Checking for review apps not updated in the last #{days_for_stop} days..."
 
     checked_environments = []
+    threshold_day = threshold_time(days: days_for_delete)
     gitlab.deployments(project_path, per_page: 50).auto_paginate do |deployment|
       next unless deployment.environment.name.start_with?('review/')
       next if checked_environments.include?(deployment.environment.slug)
@@ -48,11 +47,11 @@ class AutomatedCleanup
       checked_environments << deployment.environment.slug
       deployed_at = Time.parse(deployment.created_at)
 
-      if deployed_at < threshold_time(days: days_for_delete)
+      if deployed_at < threshold_day
         print_release_state(subject: 'Review app', release_name: deployment.environment.slug, release_date: deployment.created_at, action: 'deleting')
         gitlab.delete_environment(project_path, deployment.environment.id)
         cleaned_up_releases << deployment.environment.slug
-      elsif deployed_at < threshold_time(days: days_for_stop)
+      elsif deployed_at < threshold_day
         print_release_state(subject: 'Review app', release_name: deployment.environment.slug, release_date: deployment.created_at, action: 'stopping')
         gitlab.stop_environment(project_path, deployment.environment.id)
         cleaned_up_releases << deployment.environment.slug
@@ -62,13 +61,14 @@ class AutomatedCleanup
     end
   end
 
-  def perform_helm_releases_cleanup!(days: STALE_REVIEW_APP_DAYS_THRESHOLD)
+  def perform_helm_releases_cleanup!(days:)
     puts "Checking for Helm releases not updated in the last #{days} days..."
 
+    threshold_day = threshold_time(days: days)
     helm.releases(args: ['--deployed', '--failed', '--date', '--reverse', '--max 25']).each do |release|
       next if cleaned_up_releases.include?(release.name)
 
-      if release.last_update < threshold_time(days: days)
+      if release.last_update < threshold_day
         print_release_state(subject: 'Release', release_name: release.name, release_date: release.last_update, action: 'cleaning')
         helm.delete(release_name: release.name)
         kubernetes.cleanup(release_name: release.name)

scripts/review_apps/review-apps.sh

@@ -85,11 +85,13 @@ function deploy() {
   gitlab_unicorn_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-unicorn-ce"
   gitlab_gitaly_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitaly"
   gitlab_shell_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-shell"
+  gitlab_workhorse_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-workhorse-ce"
 
   if [[ "$CI_PROJECT_NAME" == "gitlab-ee" ]]; then
     gitlab_migrations_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-rails-ee"
     gitlab_sidekiq_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-sidekiq-ee"
     gitlab_unicorn_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-unicorn-ee"
+    gitlab_workhorse_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-workhorse-ee"
   fi
 
   # canary uses stable db
@@ -119,40 +121,7 @@ function deploy() {
   helm repo add gitlab https://charts.gitlab.io/
   helm dep update .
 
-  cat << EOF
-    Deploying:
-
-    helm upgrade --install \
-      --wait \
-      --timeout 600 \
-      --set releaseOverride="$CI_ENVIRONMENT_SLUG" \
-      --set global.hosts.hostSuffix="$HOST_SUFFIX" \
-      --set global.hosts.domain="$REVIEW_APPS_DOMAIN" \
-      --set global.hosts.externalIP="$REVIEW_APPS_DOMAIN_IP" \
-      --set certmanager.install=false \
-      --set global.ingress.configureCertmanager=false \
-      --set global.ingress.tls.secretName=tls-cert \
-      --set gitlab.unicorn.resources.requests.cpu=200m \
-      --set gitlab.sidekiq.resources.requests.cpu=100m \
-      --set gitlab.gitlab-shell.resources.requests.cpu=100m \
-      --set redis.resources.requests.cpu=100m \
-      --set minio.resources.requests.cpu=100m \
-      --set gitlab.migrations.image.repository="$gitlab_migrations_image_repository" \
-      --set gitlab.migrations.image.tag="$CI_COMMIT_REF_NAME" \
-      --set gitlab.sidekiq.image.repository="$gitlab_sidekiq_image_repository" \
-      --set gitlab.sidekiq.image.tag="$CI_COMMIT_REF_NAME" \
-      --set gitlab.unicorn.image.repository="$gitlab_unicorn_image_repository" \
-      --set gitlab.unicorn.image.tag="$CI_COMMIT_REF_NAME" \
-      --set gitlab.gitaly.image.repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitaly" \
-      --set gitlab.gitaly.image.tag="v$GITALY_VERSION" \
-      --set gitlab.gitlab-shell.image.repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-shell" \
-      --set gitlab.gitlab-shell.image.tag="v$GITLAB_SHELL_VERSION" \
-      --namespace="$KUBE_NAMESPACE" \
-      --version="$CI_PIPELINE_ID-$CI_JOB_ID" \
-      "$name" \
-      .
-EOF
-
+HELM_CMD=$(cat << EOF
   helm upgrade --install \
     --wait \
     --timeout 600 \
@@ -178,10 +147,19 @@ EOF
     --set gitlab.gitaly.image.tag="v$GITALY_VERSION" \
     --set gitlab.gitlab-shell.image.repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-shell" \
     --set gitlab.gitlab-shell.image.tag="v$GITLAB_SHELL_VERSION" \
+    --set gitlab.unicorn.workhorse.image="$gitlab_workhorse_image_repository" \
+    --set gitlab.unicorn.workhorse.tag="$CI_COMMIT_REF_NAME" \
     --namespace="$KUBE_NAMESPACE" \
     --version="$CI_PIPELINE_ID-$CI_JOB_ID" \
     "$name" \
     .
+EOF
+)
+
+  echo "Deploying with:"
+  echo $HELM_CMD
+
+  eval $HELM_CMD
 }
 
 function delete() {

spec/lib/quality/helm_client_spec.rb

 # frozen_string_literal: true
 
+require 'spec_helper'
+
 RSpec.describe Quality::HelmClient do
   let(:namespace) { 'review-apps-ee' }
   let(:release_name) { 'my-release' }

spec/lib/quality/kubernetes_client_spec.rb

 # frozen_string_literal: true
 
+require 'spec_helper'
+
 RSpec.describe Quality::KubernetesClient do
   let(:namespace) { 'review-apps-ee' }
   let(:release_name) { 'my-release' }
@@ -10,11 +12,16 @@ RSpec.describe Quality::KubernetesClient do
     it 'calls helm list with default arguments' do
       expect(Gitlab::Popen).to receive(:popen_with_detail)
         .with([
-          %(kubectl -n "#{namespace}" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa 2>&1) +
-          %( | grep "#{release_name}") +
-          " | awk '{print $1}'" +
-          %( | xargs kubectl -n "#{namespace}" delete) +
-          ' || true'
+          %(kubectl),
+          %(-n "#{namespace}" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa 2>&1),
+          '|',
+          %(grep "#{release_name}"),
+          '|',
+          "awk '{print $1}'",
+          '|',
+          %(xargs kubectl -n "#{namespace}" delete),
+          '||',
+          'true'
         ])
         .and_return(Gitlab::Popen::Result.new([], ''))