Merge branch 'improve-review-apps' into 'master'

Improve review apps

Closes #6875

See merge request gitlab-org/gitlab-ee!6665

.gitlab-ci.yml

@@ -1109,39 +1109,23 @@ no_ee_check:
     - //@gitlab-org/gitlab-ce
 
 # GitLab EE Review apps
-review-app-image:
-  <<: *single-script-job
-  stage: test
-  allow_failure: true
-  variables:
-    <<: *single-script-job-variables
-    SCRIPT_NAME: trigger-build
-  script:
-    - BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./$SCRIPT_NAME cng
-  when: manual
-  only:
-    refs:
-      - branches
-  except:
-    refs:
-      - master
-      - /(^docs[\/-].*|.*-docs$)/
-
 review:
-  <<: *single-script-job
-  image: registry.gitlab.com/charts/gitlab:latest
-  stage: post-test
+  <<: *dedicated-no-docs-pull-cache-job
+  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
+  stage: test
   allow_failure: true
+  before_script: []
   variables:
-    <<: *single-script-job-variables
-    SCRIPT_NAME: review-apps.sh
+    GIT_DEPTH: "1"
     HOST_SUFFIX: "$CI_ENVIRONMENT_SLUG"
     DOMAIN: "-$CI_ENVIRONMENT_SLUG.$REVIEW_APPS_DOMAIN"
     GITLAB_HELM_CHART_REF: "master"
   script:
-    - source ./$SCRIPT_NAME
-    - export GITLAB_SHELL_VERSION=$(< GITLAB_SHELL_VERSION)
-    - export GITALY_VERSION=$(< GITALY_SERVER_VERSION)
+    - export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION)
+    - export GITALY_VERSION=$(<GITALY_SERVER_VERSION)
+    - export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION)
+    - source ./scripts/review_apps/review-apps.sh
+    - BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng
     - check_kube_domain
     - download_gitlab_chart
     - ensure_namespace
@@ -1152,10 +1136,9 @@ review:
     name: review/$CI_COMMIT_REF_NAME
     url: https://gitlab-$CI_ENVIRONMENT_SLUG.$REVIEW_APPS_DOMAIN
     on_stop: stop_review
-  when: manual
   only:
     refs:
-      - branches
+      - branches@gitlab-org/gitlab-ee
     kubernetes: active
   except:
     refs:
@@ -1164,24 +1147,44 @@ review:
 
 stop_review:
   <<: *single-script-job
-  image: registry.gitlab.com/charts/gitlab:latest
+  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
   stage: post-cleanup
   allow_failure: true
-  variables:
-    <<: *single-script-job-variables
-    SCRIPT_NAME: review-apps.sh
+  cache: {}
+  dependencies: []
+  before_script: []
   script:
-    - source ./$SCRIPT_NAME
+    - source ./scripts/review_apps/review-apps.sh
     - delete
     - cleanup
+  when: manual
   environment:
     name: review/$CI_COMMIT_REF_NAME
     action: stop
-  when: manual
   only:
     refs:
-      - branches
+      - branches@gitlab-org/gitlab-ee
     kubernetes: active
   except:
     - master
     - /(^docs[\/-].*|.*-docs$)/
+
+automated_review_cleanup:
+  <<: *dedicated-no-docs-pull-cache-job
+  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
+  stage: build
+  allow_failure: true
+  cache: {}
+  dependencies: []
+  before_script:
+    - gem install gitlab --no-document
+  variables:
+    GIT_DEPTH: "1"
+  script:
+    - ruby -rrubygems scripts/review_apps/automated_cleanup.rb
+  only:
+    refs:
+      - schedules@gitlab-org/gitlab-ee
+  except:
+    - master
+    - /(^docs[\/-].*|.*-docs$)/

lib/quality/helm_client.rb

+# frozen_string_literal: true
+
+require 'time'
+require_relative '../gitlab/popen' unless defined?(Gitlab::Popen)
+
+module Quality
+  class HelmClient
+    attr_reader :namespace
+
+    Release = Struct.new(:name, :revision, :last_update, :status, :chart, :namespace) do
+      def revision
+        @revision ||= self[:revision].to_i
+      end
+
+      def last_update
+        @last_update ||= Time.parse(self[:last_update])
+      end
+    end
+
+    def initialize(namespace: ENV['KUBE_NAMESPACE'])
+      @namespace = namespace
+    end
+
+    def releases(args: [])
+      command = ['list', %(--namespace "#{namespace}"), *args]
+
+      run_command(command)
+        .stdout
+        .lines
+        .select { |line| line.include?(namespace) }
+        .map { |line| Release.new(*line.split(/\t/).map(&:strip)) }
+    end
+
+    def delete(release_name:)
+      run_command(['delete', '--purge', release_name])
+    end
+
+    private
+
+    def run_command(command)
+      final_command = ['helm', *command].join(' ')
+      puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output
+
+      Gitlab::Popen.popen_with_detail([final_command])
+    end
+  end
+end

lib/quality/kubernetes_client.rb

+# frozen_string_literal: true
+
+require_relative '../gitlab/popen' unless defined?(Gitlab::Popen)
+
+module Quality
+  class KubernetesClient
+    attr_reader :namespace
+
+    def initialize(namespace: ENV['KUBE_NAMESPACE'])
+      @namespace = namespace
+    end
+
+    def cleanup(release_name:)
+      command = ['kubectl']
+      command << %(-n "#{namespace}" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa 2>&1)
+      command << '|' << %(grep "#{release_name}")
+      command << '|' << "awk '{print $1}'"
+      command << '|' << %(xargs kubectl -n "#{namespace}" delete)
+      command << '||' << 'true'
+
+      run_command(command)
+    end
+
+    private
+
+    def run_command(command)
+      puts "Running command: `#{command.join(' ')}`" # rubocop:disable Rails/Output
+
+      Gitlab::Popen.popen_with_detail(command)
+    end
+  end
+end

scripts/review_apps/automated_cleanup.rb

+# frozen_string_literal: true
+
+require 'gitlab'
+require_relative File.expand_path('../../lib/quality/helm_client.rb', __dir__)
+require_relative File.expand_path('../../lib/quality/kubernetes_client.rb', __dir__)
+
+class AutomatedCleanup
+  attr_reader :project_path, :gitlab_token, :cleaned_up_releases
+
+  def initialize(project_path: ENV['CI_PROJECT_PATH'], gitlab_token: ENV['GITLAB_BOT_REVIEW_APPS_CLEANUP_TOKEN'])
+    @project_path = project_path
+    @gitlab_token = gitlab_token
+    @cleaned_up_releases = []
+  end
+
+  def gitlab
+    @gitlab ||= begin
+      Gitlab.configure do |config|
+        config.endpoint = 'https://gitlab.com/api/v4'
+        # gitlab-bot's token "GitLab review apps cleanup"
+        config.private_token = gitlab_token
+      end
+
+      Gitlab
+    end
+  end
+
+  def helm
+    @helm ||= Quality::HelmClient.new
+  end
+
+  def kubernetes
+    @kubernetes ||= Quality::KubernetesClient.new
+  end
+
+  def perform_gitlab_environment_cleanup!(days_for_stop:, days_for_delete:)
+    puts "Checking for review apps not updated in the last #{days_for_stop} days..."
+
+    checked_environments = []
+    delete_threshold = threshold_time(days: days_for_delete)
+    stop_threshold = threshold_time(days: days_for_stop)
+    gitlab.deployments(project_path, per_page: 50).auto_paginate do |deployment|
+      next unless deployment.environment.name.start_with?('review/')
+      next if checked_environments.include?(deployment.environment.slug)
+
+      puts
+
+      checked_environments << deployment.environment.slug
+      deployed_at = Time.parse(deployment.created_at)
+
+      if deployed_at < delete_threshold
+        print_release_state(subject: 'Review app', release_name: deployment.environment.slug, release_date: deployment.created_at, action: 'deleting')
+        gitlab.delete_environment(project_path, deployment.environment.id)
+        cleaned_up_releases << deployment.environment.slug
+      elsif deployed_at < stop_threshold
+        print_release_state(subject: 'Review app', release_name: deployment.environment.slug, release_date: deployment.created_at, action: 'stopping')
+        gitlab.stop_environment(project_path, deployment.environment.id)
+        cleaned_up_releases << deployment.environment.slug
+      else
+        print_release_state(subject: 'Review app', release_name: deployment.environment.slug, release_date: deployment.created_at, action: 'leaving')
+      end
+    end
+  end
+
+  def perform_helm_releases_cleanup!(days:)
+    puts "Checking for Helm releases not updated in the last #{days} days..."
+
+    threshold_day = threshold_time(days: days)
+    helm.releases(args: ['--deployed', '--failed', '--date', '--reverse', '--max 25']).each do |release|
+      next if cleaned_up_releases.include?(release.name)
+
+      if release.last_update < threshold_day
+        print_release_state(subject: 'Release', release_name: release.name, release_date: release.last_update, action: 'cleaning')
+        helm.delete(release_name: release.name)
+        kubernetes.cleanup(release_name: release.name)
+      else
+        print_release_state(subject: 'Release', release_name: release.name, release_date: release.last_update, action: 'leaving')
+      end
+    end
+  end
+
+  def threshold_time(days:)
+    Time.now - days * 24 * 3600
+  end
+
+  def print_release_state(subject:, release_name:, release_date:, action:)
+    puts "\n#{subject} '#{release_name}' was last deployed on #{release_date}: #{action} it."
+  end
+end
+
+def timed(task)
+  start = Time.now
+  yield(self)
+  puts "#{task} finished in #{Time.now - start} seconds.\n"
+end
+
+automated_cleanup = AutomatedCleanup.new
+
+timed('Review apps cleanup') do
+  automated_cleanup.perform_gitlab_environment_cleanup!(days_for_stop: 5, days_for_delete: 6)
+end
+
+puts
+
+timed('Helm releases cleanup') do
+  automated_cleanup.perform_helm_releases_cleanup!(days: 7)
+end
+
+exit(0)

scripts/review-apps.sh → scripts/review_apps/review-apps.sh

@@ -83,11 +83,15 @@ function deploy() {
   gitlab_migrations_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-rails-ce"
   gitlab_sidekiq_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-sidekiq-ce"
   gitlab_unicorn_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-unicorn-ce"
+  gitlab_gitaly_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitaly"
+  gitlab_shell_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-shell"
+  gitlab_workhorse_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-workhorse-ce"
 
   if [[ "$CI_PROJECT_NAME" == "gitlab-ee" ]]; then
     gitlab_migrations_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-rails-ee"
     gitlab_sidekiq_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-sidekiq-ee"
     gitlab_unicorn_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-unicorn-ee"
+    gitlab_workhorse_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-workhorse-ee"
   fi
 
   # canary uses stable db
@@ -117,6 +121,7 @@ function deploy() {
   helm repo add gitlab https://charts.gitlab.io/
   helm dep update .
 
+HELM_CMD=$(cat << EOF
   helm upgrade --install \
     --wait \
     --timeout 600 \
@@ -142,10 +147,19 @@ function deploy() {
     --set gitlab.gitaly.image.tag="v$GITALY_VERSION" \
     --set gitlab.gitlab-shell.image.repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-shell" \
     --set gitlab.gitlab-shell.image.tag="v$GITLAB_SHELL_VERSION" \
+    --set gitlab.unicorn.workhorse.image="$gitlab_workhorse_image_repository" \
+    --set gitlab.unicorn.workhorse.tag="$CI_COMMIT_REF_NAME" \
     --namespace="$KUBE_NAMESPACE" \
     --version="$CI_PIPELINE_ID-$CI_JOB_ID" \
     "$name" \
     .
+EOF
+)
+
+  echo "Deploying with:"
+  echo $HELM_CMD
+
+  eval $HELM_CMD
 }
 
 function delete() {
@@ -155,10 +169,13 @@ function delete() {
   if [[ "$track" != "stable" ]]; then
     name="$name-$track"
   fi
+
+  echo "Deleting release '$name'..."
   helm delete --purge "$name" || true
 }
 
 function cleanup() {
+  echo "Cleaning up $CI_ENVIRONMENT_SLUG..."
   kubectl -n "$KUBE_NAMESPACE" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa 2>&1 \
     | grep "$CI_ENVIRONMENT_SLUG" \
     | awk '{print $1}' \

scripts/trigger-build

@@ -158,7 +158,8 @@ module Trigger
 
     def status
       req = Net::HTTP::Get.new(@uri)
-      req['PRIVATE-TOKEN'] = ENV['GITLAB_QA_ACCESS_TOKEN']
+      # gitlab-bot's token "GitLab multi-project pipeline polling"
+      req['PRIVATE-TOKEN'] = ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN']
 
       res = Net::HTTP.start(@uri.hostname, @uri.port, use_ssl: true) do |http|
         http.request(req)

spec/lib/quality/helm_client_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Quality::HelmClient do
+  let(:namespace) { 'review-apps-ee' }
+  let(:release_name) { 'my-release' }
+  let(:raw_helm_list_result) do
+    <<~OUTPUT
+    NAME                    	REVISION	UPDATED                 	STATUS	CHART       	NAMESPACE
+    review-improve-re-2dsd9d	1       	Tue Jul 31 15:53:17 2018	FAILED	gitlab-0.3.4	#{namespace}
+    review-11-1-stabl-3r2fso	1       	Mon Jul 30 22:44:14 2018	FAILED	gitlab-0.3.3	#{namespace}
+    review-49375-css-fk664j 	1       	Thu Jul 19 11:01:30 2018	FAILED	gitlab-0.2.4	#{namespace}
+    OUTPUT
+  end
+
+  subject { described_class.new(namespace: namespace) }
+
+  describe '#releases' do
+    it 'calls helm list with default arguments' do
+      expect(Gitlab::Popen).to receive(:popen_with_detail)
+        .with([%(helm list --namespace "#{namespace}")])
+        .and_return(Gitlab::Popen::Result.new([], ''))
+
+      subject.releases
+    end
+
+    it 'calls helm list with given arguments' do
+      expect(Gitlab::Popen).to receive(:popen_with_detail)
+        .with([%(helm list --namespace "#{namespace}" --deployed)])
+        .and_return(Gitlab::Popen::Result.new([], ''))
+
+      subject.releases(args: ['--deployed'])
+    end
+
+    it 'returns a list of Release objects' do
+      expect(Gitlab::Popen).to receive(:popen_with_detail)
+        .with([%(helm list --namespace "#{namespace}" --deployed)])
+        .and_return(Gitlab::Popen::Result.new([], raw_helm_list_result))
+
+      releases = subject.releases(args: ['--deployed'])
+
+      expect(releases.size).to eq(3)
+      expect(releases[0].name).to eq('review-improve-re-2dsd9d')
+      expect(releases[0].revision).to eq(1)
+      expect(releases[0].last_update).to eq(Time.parse('Tue Jul 31 15:53:17 2018'))
+      expect(releases[0].status).to eq('FAILED')
+      expect(releases[0].chart).to eq('gitlab-0.3.4')
+      expect(releases[0].namespace).to eq(namespace)
+    end
+  end
+
+  describe '#delete' do
+    it 'calls helm delete with default arguments' do
+      expect(Gitlab::Popen).to receive(:popen_with_detail)
+        .with(["helm delete --purge #{release_name}"])
+        .and_return(Gitlab::Popen::Result.new([], '', '', 0))
+
+      expect(subject.delete(release_name: release_name).status).to eq(0)
+    end
+  end
+end

spec/lib/quality/kubernetes_client_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Quality::KubernetesClient do
+  subject { described_class.new(namespace: 'review-apps-ee') }
+
+  describe '#cleanup' do
+    it 'calls kubectl with the correct arguments' do
+      # popen_with_detail will receive an array with a bunch of arguments; we're
+      # only concerned with it having the correct namespace and release name
+      expect(Gitlab::Popen).to receive(:popen_with_detail) do |args|
+        expect(args)
+          .to satisfy_one { |arg| arg.start_with?('-n "review-apps-ee" get') }
+        expect(args)
+          .to satisfy_one { |arg| arg == 'grep "my-release"' }
+        expect(args)
+          .to satisfy_one { |arg| arg.end_with?('-n "review-apps-ee" delete') }
+      end
+
+      # We're not verifying the output here, just silencing it
+      expect { subject.cleanup(release_name: 'my-release') }.to output.to_stdout
+    end
+  end
+end