Merge branch '118662-drop-support-es-v5-support-v7' into 'master'

Add support for ES7 & Drop suport for ES5

Closes #196503

See merge request gitlab-org/gitlab!22859

.gitlab/ci/global.gitlab-ci.yml

@@ -213,7 +213,7 @@
     - name: postgres:9.6
       command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
     - name: redis:alpine
-    - name: elasticsearch:5.6.12
+    - name: elasticsearch:6.4.2
 
 .use-pg10-ee:
   image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.5-golang-1.12-git-2.24-lfs-2.9-chrome-73.0-node-12.x-yarn-1.16-postgresql-10-graphicsmagick-1.3.33"
@@ -221,7 +221,7 @@
     - name: postgres:10.9
       command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
     - name: redis:alpine
-    - name: elasticsearch:5.6.12
+    - name: elasticsearch:6.4.2
 
 .only-ee:
   only:

GITLAB_ELASTICSEARCH_INDEXER_VERSION

-1.5.0
+2.0.0

changelogs/unreleased/118662-drop-support-es-v5-support-v7.yml

+---
+title: Drop support for ES5 add support for ES7
+merge_request: 22859
+author:
+type: added

config/initializers/elastic_client_setup.rb

@@ -18,6 +18,32 @@ Gitlab.ee do
   Elasticsearch::Model::ClassMethods.prepend GemExtensions::Elasticsearch::Model::Client
   Elasticsearch::Model.singleton_class.prepend GemExtensions::Elasticsearch::Model::Client
 
+  # This monkey patch cannot be handled by prepend like the above since this
+  # module is included into other classes.
+  module Elasticsearch
+    module Model
+      module Response
+        module Base
+          if Gem::Version.new(Elasticsearch::Model::VERSION) >= Gem::Version.new('7.0.0')
+            raise "elasticsearch-model was upgraded, please remove this monkey patch in #{__FILE__}"
+          end
+
+          # Handle ES7 API where total is returned as an object. This
+          # change is taken from the V7 gem
+          # https://github.com/elastic/elasticsearch-rails/commit/9c40f630e1b549f0b7889fe33dcd826b485af6fc
+          # and can be removed when we upgrade the gem to V7
+          def total
+            if response.response['hits']['total'].respond_to?(:keys)
+              response.response['hits']['total']['value']
+            else
+              response.response['hits']['total']
+            end
+          end
+        end
+      end
+    end
+  end
+
   ### Modified from elasticsearch-model/lib/elasticsearch/model.rb
 
   [

doc/integration/elasticsearch.md

@@ -19,7 +19,8 @@ special searches:
 | -------------- | --------------------- |
 | GitLab Enterprise Edition 8.4 - 8.17   | Elasticsearch 2.4 with [Delete By Query Plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/2.4/plugins-delete-by-query.html) installed |
 | GitLab Enterprise Edition 9.0 - 11.4   | Elasticsearch 5.1 - 5.5 |
-| GitLab Enterprise Edition 11.5+        | Elasticsearch 5.6 - 6.x |
+| GitLab Enterprise Edition 11.5 - 12.6  | Elasticsearch 5.6 - 6.x |
+| GitLab Enterprise Edition 12.7+        | Elasticsearch 6.x - 7.x |
 
 ## Installing Elasticsearch
 

ee/lib/elastic/latest/config.rb

@@ -21,7 +21,7 @@ module Elastic
             analyzer: {
               default: {
                 tokenizer: 'standard',
-                filter: %w(standard lowercase my_stemmer)
+                filter: %w(lowercase my_stemmer)
               },
               my_ngram_analyzer: {
                 tokenizer: 'my_ngram_tokenizer',

ee/lib/gitlab/elastic/helper.rb

@@ -26,20 +26,29 @@ module Gitlab
         client = proxy.client
         index_name = proxy.index_name
 
-        # ES5.6 needs a setting enabled to support JOIN datatypes that ES6 does not support...
-        if Gitlab::VersionInfo.parse(client.info['version']['number']) < Gitlab::VersionInfo.new(6)
-          settings['index.mapping.single_type'] = true
+        create_index_options = {
+          index: index_name,
+          body: {
+            settings: settings.to_hash,
+            mappings: mappings.to_hash
+          }
+        }
+
+        # include_type_name defaults to false in ES7. This will ensure ES7
+        # behaves like ES6 when creating mappings. See
+        # https://www.elastic.co/blog/moving-from-types-to-typeless-apis-in-elasticsearch-7-0
+        # for more information. We also can't set this for any versions before
+        # 6.8 as this parameter was not supported. Since it defaults to true in
+        # all 6.x it's safe to only set it for 7.x.
+        if Gitlab::VersionInfo.parse(client.info['version']['number']).major == 7
+          create_index_options[:include_type_name] = true
         end
 
         if client.indices.exists? index: index_name
           client.indices.delete index: index_name
         end
 
-        client.indices.create index: index_name,
-                              body: {
-                                settings: settings.to_hash,
-                                mappings: mappings.to_hash
-                              }
+        client.indices.create create_index_options
       end
       # rubocop: enable CodeReuse/ActiveRecord
 

ee/lib/system_check/app/elasticsearch_check.rb

@@ -21,10 +21,10 @@ module SystemCheck
 
       def check?
         case self.class.current_version.major
-        when 5
-          !(1..5).cover?(self.class.current_version.minor)
         when 6
           true
+        when 7
+          true
         else
           false
         end

ee/spec/lib/system_check/app/elasticsearch_check_spec.rb

@@ -32,11 +32,12 @@ describe SystemCheck::App::ElasticsearchCheck do
     where(:version, :result) do
       '2.3.0' | false
       '5.3.1' | false
-      '5.6.0' | true
-      '5.6.6' | true
+      '5.6.0' | false
+      '5.6.6' | false
       '6.0.0' | true
-      '6.3.4' | true
-      '7.1.0' | false
+      '6.4.2' | true
+      '7.1.0' | true
+      '7.5.1' | true
     end
 
     with_them do