Merge branch 'ak/wait-for-green-es' into 'master'

Wait for elasticsearch to be green on install

See merge request gitlab-org/gitlab!24489

app/models/clusters/applications/elastic_stack.rb

@@ -30,7 +30,8 @@ module Clusters
           version: VERSION,
           rbac: cluster.platform_kubernetes_rbac?,
           chart: chart,
-          files: files
+          files: files,
+          postinstall: post_install_script
         )
       end
 
@@ -43,6 +44,10 @@ module Clusters
         )
       end
 
+      def files
+        super.merge('wait-for-elasticsearch.sh': File.read("#{Rails.root}/vendor/elastic_stack/wait-for-elasticsearch.sh"))
+      end
+
       def elasticsearch_client
         strong_memoize(:elasticsearch_client) do
           next unless kube_client
@@ -69,10 +74,16 @@ module Clusters
 
       private
 
+      def post_install_script
+        [
+          "timeout -t60 sh /data/helm/elastic-stack/config/wait-for-elasticsearch.sh http://elastic-stack-elasticsearch-client:9200"
+        ]
+      end
+
       def post_delete_script
         [
           Gitlab::Kubernetes::KubectlCmd.delete("pvc", "--selector", "release=elastic-stack")
-        ].compact
+        ]
       end
 
       def kube_client

changelogs/unreleased/ak-wait-for-green-es.yml

+---
+title: Wait for elasticsearch to be green on install
+merge_request: 24489
+author:
+type: added

doc/user/clusters/applications.md

@@ -464,8 +464,8 @@ chart is used to install this application with a
 file.
 
 NOTE: **Note:**
-The chart will deploy 4 Elasticsearch nodes: 2 masters, 1 data and 1 client node,
-with resource requests totalling 0.1 CPU and 3GB RAM. Each data node requests 1.5GB of memory,
+The chart will deploy 5 Elasticsearch nodes: 2 masters, 2 data and 1 client node,
+with resource requests totalling 0.125 CPU and 4.5GB RAM. Each data node requests 1.5GB of memory,
 which makes it incompatible with clusters of `f1-micro` and `g1-small` instance types.
 
 ## Install using GitLab CI (alpha)

ee/app/models/ee/clusters/platforms/kubernetes.rb

@@ -100,7 +100,7 @@ module EE
         end
 
         def elastic_stack_available?
-          !!cluster.application_elastic_stack
+          !!cluster.application_elastic_stack&.installed?
         end
 
         private

ee/spec/models/ee/clusters/platforms/kubernetes_spec.rb

@@ -174,7 +174,7 @@ describe Clusters::Platforms::Kubernetes do
 
       context 'when ElasticSearch is enabled' do
         let(:cluster) { create(:cluster, :project, platform_kubernetes: service) }
-        let!(:elastic_stack) { create(:clusters_applications_elastic_stack, cluster: cluster) }
+        let!(:elastic_stack) { create(:clusters_applications_elastic_stack, :installed, cluster: cluster) }
 
         before do
           expect_any_instance_of(::Clusters::Applications::ElasticStack).to receive(:elasticsearch_client).at_least(:once).and_return(Elasticsearch::Transport::Client.new)

vendor/elastic_stack/values.yaml

@@ -8,7 +8,7 @@ elasticsearch:
   client:
     replicas: 1
   data:
-    replicas: 1
+    replicas: 2
 
 kibana:
   enabled: false

vendor/elastic_stack/wait-for-elasticsearch.sh

+#!/bin/bash
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+IFS=$'\n\t'
+set -euo pipefail
+
+HOST="$1"
+
+printf 'Waiting for ES to be reachable ...'
+until $(wget -O- -q "$HOST" &>/dev/null); do
+    printf '.'
+    sleep 1
+done
+echo " OK!"
+
+printf 'Waiting for ES to be healthy ...'
+while : ; do
+    HEALTH="$(wget -O- -q "$HOST/_cat/health?h=status" 2> /dev/null)"
+    HEALTH="$(echo "$HEALTH" | sed -r 's/^[[:space:]]+|[[:space:]]+$//g')" # trim whitespace (otherwise we'll have "green ")
+    ([ "$HEALTH" != "green" ] && printf '.' && sleep 1) || break
+done
+echo " OK!"
+
+echo "Elastic Search is up!"