Merge branch 'es_common_index' into 'master'

ES common index

This will allow as to use parent/child relationship in ES. Related to https://gitlab.com/gitlab-org/gitlab-ee/issues/375

- [x] Fix settings overlapping 
- [x] Specs
- [x] Test everything
- [x] Change rake tasks
- [x] Change index settings for all models
- [x] Update the install doc
- [x] Update the update doc (I created https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/5545)
- [x] Prepare text for release blog post


Proposed addition to releasepost:

We totally changed the structure of Elasticsearch  index this is why we need to remove whole index and build new one. To be able to use parent/child relationship in ES we decided to move everything in a single index. This solution has some drawback  - if we decide to change the type of some existing field in ES we will need to rebuild whole index for every entity. So it makes it harder to support but we expect to have more performance advantages with parent/child relationships. 

**Migration**

Remove old indexes

```
curl -XDELETE 'http://localhost:9200/_all/'
```

Build new indexes as described in [Elasticsearch integration](../integration/elasticsearch.md#add-gitlabs-data-to-the-elasticsearch-index)



See merge request !598

app/models/concerns/elastic/application_search.rb

@@ -6,7 +6,7 @@ module Elastic
     included do
       include Elasticsearch::Model
 
-      index_name [Rails.application.class.parent_name.downcase, self.name.downcase, Rails.env].join('-')
+      index_name [Rails.application.class.parent_name.downcase, Rails.env].join('-')
 
       settings \
         index: {
@@ -92,7 +92,7 @@ module Elastic
                      end
 
         query_hash[:sort] = [
-          { updated_at_sort: { order: :desc, mode: :min } },
+          { updated_at: { order: :desc } },
           :_score
         ]
 

app/models/concerns/elastic/issues_search.rb

@@ -22,8 +22,6 @@ module Elastic
         indexes :assignee_id, type: :integer
 
         indexes :confidential, type: :boolean
-
-        indexes :updated_at_sort, type: :date,   index: :not_analyzed
       end
 
       def as_indexed_json(options = {})
@@ -35,9 +33,6 @@ module Elastic
           data[attr.to_s] = self.send(attr)
         end
 
-        data['project'] = { 'id' => project_id }
-        data['author'] = { 'id' => author_id }
-        data['updated_at_sort'] = updated_at
         data
       end
 

app/models/concerns/elastic/merge_requests_search.rb

@@ -25,8 +25,6 @@ module Elastic
         indexes :source_project_id, type: :integer
         indexes :target_project_id, type: :integer
         indexes :author_id,         type: :integer
-
-        indexes :updated_at_sort, type: :string, index: 'not_analyzed'
       end
 
       def as_indexed_json(options = {})
@@ -51,8 +49,7 @@ module Elastic
         ].each do |attr|
           data[attr.to_s] = self.send(attr)
         end
-
-        data['updated_at_sort'] = updated_at
+        
         data
       end
 

app/models/concerns/elastic/milestones_search.rb

@@ -13,14 +13,13 @@ module Elastic
                               index_options: 'offsets'
         indexes :project_id,  type: :integer
         indexes :created_at,  type: :date
-
-        indexes :updated_at_sort, type: :string, index: 'not_analyzed'
+        indexes :updated_at,  type: :date
       end
 
       def as_indexed_json(options = {})
         as_json(
-          only: [:id, :title, :description, :project_id, :created_at]
-        ).merge({ updated_at_sort: updated_at })
+          only: [:id, :title, :description, :project_id, :created_at, :updated_at]
+        )
       end
 
       def self.elastic_search(query, options: {})

app/models/concerns/elastic/notes_search.rb

@@ -11,14 +11,13 @@ module Elastic
                               index_options: 'offsets'
         indexes :project_id,  type: :integer
         indexes :created_at,  type: :date
+        indexes :updated_at,  type: :date
 
         indexes :issue do
           indexes :assignee_id, type: :integer
           indexes :author_id, type: :integer
           indexes :confidential, type: :boolean
         end
-
-        indexes :updated_at_sort, type: :string, index: 'not_analyzed'
       end
 
       def as_indexed_json(options = {})
@@ -26,7 +25,7 @@ module Elastic
 
         # We don't use as_json(only: ...) because it calls all virtual and serialized attributtes
         # https://gitlab.com/gitlab-org/gitlab-ee/issues/349
-        [:id, :note, :project_id, :created_at].each do |attr|
+        [:id, :note, :project_id, :created_at, :updated_at].each do |attr|
           data[attr.to_s] = self.send(attr)
         end
 
@@ -38,7 +37,6 @@ module Elastic
           }
         end
 
-        data['updated_at_sort'] = updated_at
         data
       end
 
@@ -62,7 +60,7 @@ module Elastic
         query_hash = confidentiality_filter(query_hash, options[:current_user])
 
         query_hash[:sort] = [
-          { updated_at_sort: { order: :desc, mode: :min } },
+          { updated_at: { order: :desc } },
           :_score
         ]
 

app/models/concerns/elastic/projects_search.rb

@@ -22,6 +22,7 @@ module Elastic
         indexes :namespace_id,        type: :integer
 
         indexes :created_at,          type: :date
+        indexes :updated_at,          type: :date
         indexes :archived,            type: :boolean
         indexes :visibility_level,    type: :integer
         indexes :last_activity_at,    type: :date
@@ -40,6 +41,7 @@ module Elastic
           :description,
           :namespace_id,
           :created_at,
+          :updated_at,
           :archived,
           :visibility_level,
           :last_activity_at,

app/models/concerns/elastic/repositories_search.rb

@@ -5,6 +5,8 @@ module Elastic
     included do
       include Elasticsearch::Git::Repository
 
+      index_name [Rails.application.class.parent_name.downcase, Rails.env].join('-')
+
       def repository_id
         project.id
       end
@@ -18,8 +20,6 @@ module Elastic
       end
 
       def self.import
-        Repository.__elasticsearch__.create_index!
-
         Project.find_each do |project|
           if project.repository.exists? && !project.repository.empty?
             project.repository.index_commits

app/models/concerns/elastic/snippets_search.rb

@@ -21,8 +21,6 @@ module Elastic
         indexes :project_id,       type: :integer
         indexes :author_id,        type: :integer
         indexes :visibility_level, type: :integer
-
-        indexes :updated_at_sort,  type: :date,   index: :not_analyzed
       end
 
       def as_indexed_json(options = {})
@@ -62,7 +60,7 @@ module Elastic
         query_hash = filter(query_hash, options[:user])
 
         query_hash[:sort] = [
-          { updated_at_sort: { order: :desc, mode: :min } },
+          { updated_at: { order: :desc } },
           :_score
         ]
 

app/models/concerns/elastic/wiki_repositories_search.rb

@@ -5,6 +5,8 @@ module Elastic
     included do
       include Elasticsearch::Git::Repository
 
+      index_name [Rails.application.class.parent_name.downcase, Rails.env].join('-')
+
       def repository_id
         "wiki_#{project.id}"
       end
@@ -18,8 +20,6 @@ module Elastic
       end
 
       def self.import
-        ProjectWiki.__elasticsearch__.create_index!
-
         Project.where(wiki_enabled: true).find_each do |project|
           unless project.wiki.empty?
             project.wiki.index_blobs

app/models/personal_snippet.rb

 class PersonalSnippet < Snippet
   # Elastic search configuration (it does not support STI)
   document_type 'snippet'
-  index_name [Rails.application.class.parent_name.downcase, 'snippet', Rails.env].join('-')
+  index_name [Rails.application.class.parent_name.downcase, Rails.env].join('-')
   include Elastic::SnippetsSearch
 end

app/models/project_snippet.rb

 class ProjectSnippet < Snippet
   # Elastic search configuration (it does not support STI)
   document_type 'snippet'
-  index_name [Rails.application.class.parent_name.downcase, 'snippet', Rails.env].join('-')
+  index_name [Rails.application.class.parent_name.downcase, Rails.env].join('-')
   include Elastic::SnippetsSearch
 
   belongs_to :project

bin/elastic_repo_indexer

@@ -20,7 +20,7 @@ ELASTIC_PORT = elastic_connection_info['port']
 class Repository
   include Elasticsearch::Git::Repository
 
-  index_name ['repository', 'index', RAILS_ENV].compact.join('-')
+  index_name ['gitlab', RAILS_ENV].compact.join('-')
 
   self.__elasticsearch__.client = Elasticsearch::Client.new(
     host: ELASTIC_HOST,
@@ -40,8 +40,6 @@ class Repository
   end
 end
 
-Repository.__elasticsearch__.create_index!
-
 repo = Repository.new
 
 params = { from_rev: FROM_SHA, to_rev: TO_SHA }.compact
@@ -52,4 +50,4 @@ puts "Done"
 
 print "Indexing blobs..."
 repo.index_blobs(params)
-puts "Done"
\ No newline at end of file
+puts "Done"

doc/integration/elasticsearch.md

@@ -64,10 +64,10 @@ Configure Elasticsearch's host and port in **Admin > Settings**. Then create emp
 
     ```
     # Omnibus installations
-    sudo gitlab-rake gitlab:elastic:create_empty_indexes
+    sudo gitlab-rake gitlab:elastic:create_empty_index
 
     # Installations from source
-    bundle exec rake gitlab:elastic:create_empty_indexes
+    bundle exec rake gitlab:elastic:create_empty_index
     ```
 
 
@@ -216,10 +216,10 @@ To minimize downtime of the search feature we recommend the following:
 
     ```
     # Omnibus installations
-    sudo gitlab-rake gitlab:elastic:create_empty_indexes
+    sudo gitlab-rake gitlab:elastic:create_empty_index
 
     # Installations from source
-    bundle exec rake gitlab:elastic:create_empty_indexes
+    bundle exec rake gitlab:elastic:create_empty_index
     ```
 
 1. Index all repositories using the `gitlab:elastic:index_repositories` Rake
@@ -227,8 +227,8 @@ To minimize downtime of the search feature we recommend the following:
 
 1. Enable Elasticsearch indexing.
 
-1. Run indexers for database (with the `UPDATE_INDEX=1` parameter), wikis, and
-   repositories. By running  the repository indexer twice you will be sure that
+1. Run indexers for database, wikis, and
+   repositories (with the `UPDATE_INDEX=1` parameter). By running the repository indexer twice you will be sure that
    everything is indexed because some commits could be pushed while you
    performed the initial indexing. The repository indexer will skip
    repositories and commits that are already indexed, so it will be much

features/steps/project/elastic/global_search.rb

@@ -8,15 +8,11 @@ class Spinach::Features::GlobalSearch < Spinach::FeatureSteps
   include StubConfiguration
 
   before do
-    [::Project, Issue, MergeRequest, Milestone].each do |model|
-      model.__elasticsearch__.create_index!
-    end
+    Gitlab::Elastic::Helper.create_empty_index
   end
 
   after do
-    [::Project, Issue, MergeRequest, Milestone].each do |model|
-      model.__elasticsearch__.delete_index!
-    end
+    Gitlab::Elastic::Helper.delete_index
 
     stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
   end

features/steps/project/elastic/project_search.rb

@@ -6,15 +6,11 @@ class Spinach::Features::ProjectSearch < Spinach::FeatureSteps
   include StubConfiguration
 
   before do
-    [::Project, Repository, Note, MergeRequest, Milestone, ::ProjectWiki, Issue].each do |model|
-      model.__elasticsearch__.create_index!
-    end
+    Gitlab::Elastic::Helper.create_empty_index
   end
 
   after do
-    [::Project, Repository, Note, MergeRequest, Milestone, ::ProjectWiki, Issue].each do |model|
-      model.__elasticsearch__.delete_index!
-    end
+    Gitlab::Elastic::Helper.delete_index
 
     stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
   end

features/steps/project/elastic/snippets_search.rb

@@ -6,23 +6,23 @@ class Spinach::Features::SnippetsSearch < Spinach::FeatureSteps
   include StubConfiguration
 
   before do
-    Snippet.__elasticsearch__.create_index!
+    Gitlab::Elastic::Helper.create_empty_index
   end
 
   after do
-    Snippet.__elasticsearch__.delete_index!
+    Gitlab::Elastic::Helper.delete_index
 
     stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
   end
 
   step 'there is a snippet "index" with "php rocks" string' do
     create :personal_snippet, :public, content: "php rocks", title: "index"
-    Snippet.__elasticsearch__.refresh_index!
+    Gitlab::Elastic::Helper.refresh_index
   end
 
   step 'there is a snippet "php" with "benefits" string' do
     create :personal_snippet, :public, content: "benefits", title: "php"
-    Snippet.__elasticsearch__.refresh_index!
+    Gitlab::Elastic::Helper.refresh_index
   end
 
   step 'I search "php"' do

lib/gitlab/elastic/helper.rb

+module Gitlab
+  module Elastic
+    class Helper
+      def self.create_empty_index
+        index_name = Project.index_name
+        settings = {}
+        mappings = {}
+
+        [
+          Project,
+          Issue,
+          MergeRequest,
+          Snippet,
+          Note,
+          Milestone,
+          ProjectWiki,
+          Repository
+        ].each do |klass|
+          settings.deep_merge!(klass.settings.to_hash)
+          mappings.merge!(klass.mappings.to_hash)
+        end
+
+        client = Project.__elasticsearch__.client
+
+        if client.indices.exists? index: index_name
+          client.indices.delete index: index_name
+        end
+
+        client.indices.create index: index_name,
+                              body: {
+                                settings: settings.to_hash,
+                                mappings: mappings.to_hash }
+      end
+
+      def self.delete_index
+        Project.__elasticsearch__.delete_index!
+      end
+
+      def self.refresh_index
+        Project.__elasticsearch__.refresh_index!
+      end
+    end
+  end
+end

lib/tasks/gitlab/elastic.rake

 namespace :gitlab do
   namespace :elastic do
-    desc "GitLab | Update Elasticsearch indexes"
+    desc "GitLab | Elasticsearch | Index eveything at once"
     task :index do
+      Rake::Task["gitlab:elastic:create_empty_index"].invoke
+      Rake::Task["gitlab:elastic:clear_index_status"].invoke
       Rake::Task["gitlab:elastic:index_repositories"].invoke
       Rake::Task["gitlab:elastic:index_wikis"].invoke
       Rake::Task["gitlab:elastic:index_database"].invoke
     end
 
-    desc "GitLab | Update Elasticsearch indexes for project repositories"
+    desc "GitLab | Elasticsearch | Index project repositories"
     task index_repositories: :environment  do
-      Repository.__elasticsearch__.create_index!
-
       projects = if ENV['UPDATE_INDEX']
                    Project
                  else
@@ -56,10 +56,8 @@ namespace :gitlab do
       end
     end
 
-    desc "GitLab | Update Elasticsearch indexes for wiki repositories"
+    desc "GitLab | Elasticsearch | Index wiki repositories"
     task index_wikis: :environment  do
-      ProjectWiki.__elasticsearch__.create_index!
-
       projects = apply_project_filters(Project.where(wiki_enabled: true))
 
       projects.find_each do |project|
@@ -75,11 +73,9 @@ namespace :gitlab do
       end
     end
 
-    desc "GitLab | Update Elasticsearch indexes for all database objects"
+    desc "GitLab | Elasticsearch | Index all database objects"
     task index_database: :environment do
       [Project, Issue, MergeRequest, Snippet, Note, Milestone].each do |klass|
-        klass.__elasticsearch__.create_index!
-
         print "Indexing #{klass} records... "
 
         if klass == Note
@@ -92,72 +88,28 @@ namespace :gitlab do
       end
     end
 
-    desc "GitLab | Recreate Elasticsearch indexes for particular model"
-    task reindex_model: :environment do
-      model_name = ENV['MODEL']
-
-      unless %w(Project Issue MergeRequest Snippet Note Milestone).include?(model_name)
-        raise "Please pass MODEL variable"
-      end
-
-      klass = model_name.constantize
-      klass.__elasticsearch__.create_index! force: true
-
-      print "Reindexing #{klass} records... "
-
-      if klass == Note
-        Note.searchable.import
-      else
-        klass.import
-      end
-
-      puts "done".color(:green)
-    end
-
-    desc "GitLab | Create empty Elasticsearch indexes"
-    task create_empty_indexes: :environment do
-      [
-        Project,
-        Issue,
-        MergeRequest,
-        Snippet,
-        Note,
-        Milestone,
-        ProjectWiki,
-        Repository
-      ].each do |klass|
-        print "Creating index for #{klass}... "
-
-        klass.__elasticsearch__.create_index!
-
-        puts "done".color(:green)
-      end
+    desc "GitLab | Elasticsearch | Create empty index"
+    task create_empty_index: :environment do
+      Gitlab::Elastic::Helper.create_empty_index
+      puts "Index created".color(:green)
     end
 
-    desc "GitLab | Clear Elasticsearch indexing status"
+    desc "GitLab | Elasticsearch | Clear indexing status"
     task clear_index_status: :environment do
       IndexStatus.destroy_all
-      puts "Done".color(:green)
+      puts "Index status has been reset".color(:green)
     end
 
-    desc "GitLab | Delete Elasticsearch indexes"
-    task delete_indexes: :environment do
-      [
-        Project,
-        Issue,
-        MergeRequest,
-        Snippet,
-        Note,
-        Milestone,
-        ProjectWiki,
-        Repository
-      ].each do |klass|
-        print "Delete index for #{klass}... "
-
-        klass.__elasticsearch__.delete_index!
+    desc "GitLab | Elasticsearch | Delete index"
+    task delete_index: :environment do
+      Gitlab::Elastic::Helper.delete_index
+      puts "Index deleted".color(:green)
+    end
 
-        puts "done".color(:green)
-      end
+    desc "GitLab | Elasticsearch | Recreate index"
+    task recreate_index: :environment do
+      Gitlab::Elastic::Helper.create_empty_index
+      puts "Index recreated".color(:green)
     end
 
     def apply_project_filters(projects)

spec/features/projects/elastic/global_search_spec.rb

@@ -6,30 +6,22 @@ feature 'Global elastic search', feature: true do
 
   before do
     stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
+    Gitlab::Elastic::Helper.create_empty_index
 
     project.team << [user, :master]
     login_with(user)
   end
 
   after do
+    Gitlab::Elastic::Helper.delete_index
     stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
   end
 
   describe 'I search through the issues and I see pagination' do
     before do
-      [::Project, Issue, MergeRequest, Milestone].each do |model|
-        model.__elasticsearch__.create_index!
-      end
-
       create_list(:issue, 21, project: project, title: 'initial')
 
-      Issue.__elasticsearch__.refresh_index!
-    end
-
-    after do
-      [::Project, Issue, MergeRequest, Milestone].each do |model|
-        model.__elasticsearch__.delete_index!
-      end
+      Gitlab::Elastic::Helper.refresh_index
     end
 
     it "has a pagination" do

spec/lib/gitlab/elastic/project_search_results_spec.rb

@@ -7,14 +7,12 @@ describe Gitlab::Elastic::ProjectSearchResults, lib: true do
 
   before do
     stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
-    Project.__elasticsearch__.create_index!
-    Issue.__elasticsearch__.create_index!
+    Gitlab::Elastic::Helper.create_empty_index
   end
 
   after do
+    Gitlab::Elastic::Helper.delete_index
     stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
-    Project.__elasticsearch__.delete_index!
-    Issue.__elasticsearch__.delete_index!
   end
 
   describe 'initialize with empty ref' do
@@ -53,7 +51,7 @@ describe Gitlab::Elastic::ProjectSearchResults, lib: true do
       project1.wiki.create_page("index_page", " term")
       project1.wiki.index_blobs
 
-      Project.__elasticsearch__.refresh_index!
+      Gitlab::Elastic::Helper.refresh_index
 
       result = Gitlab::Elastic::ProjectSearchResults.new(user, project.id, "term")
       expect(result.notes_count).to eq(1)
@@ -77,7 +75,7 @@ describe Gitlab::Elastic::ProjectSearchResults, lib: true do
     let!(:security_issue_2) { create(:issue, :confidential, title: 'Security issue 2', project: project, assignee: assignee) }
 
     before do
-      Issue.__elasticsearch__.refresh_index!
+      Gitlab::Elastic::Helper.refresh_index
     end
 
     it 'should not list project confidential issues for non project members' do

spec/lib/gitlab/elastic/search_results_spec.rb

@@ -3,9 +3,11 @@ require 'spec_helper'
 describe Gitlab::Elastic::SearchResults, lib: true do
   before do
     stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
+    Gitlab::Elastic::Helper.create_empty_index
   end
 
   after do
+    Gitlab::Elastic::Helper.delete_index
     stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
   end
 
@@ -16,8 +18,6 @@ describe Gitlab::Elastic::SearchResults, lib: true do
 
   describe 'issues' do
     before do
-      Issue.__elasticsearch__.create_index!
-
       @issue_1 = create(
         :issue,
         project: project_1,
@@ -38,11 +38,7 @@ describe Gitlab::Elastic::SearchResults, lib: true do
         iid: 2
       )
 
-      Issue.__elasticsearch__.refresh_index!
-    end
-
-    after do
-      Issue.__elasticsearch__.delete_index!
+      Gitlab::Elastic::Helper.refresh_index
     end
 
     it 'should list issues that title or description contain the query' do
@@ -91,8 +87,6 @@ describe Gitlab::Elastic::SearchResults, lib: true do
     let(:admin) { create(:admin) }
 
     before do
-      Issue.__elasticsearch__.create_index!
-
       @issue = create(:issue, project: project_1, title: 'Issue 1', iid: 1)
       @security_issue_1 = create(:issue, :confidential, project: project_1, title: 'Security issue 1', author: author, iid: 2)
       @security_issue_2 = create(:issue, :confidential, title: 'Security issue 2', project: project_1, assignee: assignee, iid: 3)
@@ -100,7 +94,7 @@ describe Gitlab::Elastic::SearchResults, lib: true do
       @security_issue_4 = create(:issue, :confidential, project: project_3, title: 'Security issue 4', assignee: assignee, iid: 1)
       @security_issue_5 = create(:issue, :confidential, project: project_4, title: 'Security issue 5', iid: 1)
 
-      Issue.__elasticsearch__.refresh_index!
+      Gitlab::Elastic::Helper.refresh_index
     end
 
     context 'search by term' do
@@ -276,8 +270,6 @@ describe Gitlab::Elastic::SearchResults, lib: true do
 
   describe 'merge requests' do
     before do
-      MergeRequest.__elasticsearch__.create_index!
-
       @merge_request_1 = create(
         :merge_request,
         source_project: project_1,
@@ -302,11 +294,7 @@ describe Gitlab::Elastic::SearchResults, lib: true do
         iid: 2
       )
 
-      MergeRequest.__elasticsearch__.refresh_index!
-    end
-
-    after do
-      MergeRequest.__elasticsearch__.delete_index!
+      Gitlab::Elastic::Helper.refresh_index
     end
 
     it 'should list merge requests that title or description contain the query' do
@@ -345,18 +333,6 @@ describe Gitlab::Elastic::SearchResults, lib: true do
   end
 
   describe 'project scoping' do
-    before do
-      [Project, MergeRequest, Issue, Milestone].each do |model|
-        model.__elasticsearch__.create_index!
-      end
-    end
-
-    after do
-      [Project, MergeRequest, Issue, Milestone].each do |model|
-        model.__elasticsearch__.delete_index!
-      end
-    end
-
     it "returns items for project" do
       project = create :project, name: "term"
 
@@ -380,9 +356,7 @@ describe Gitlab::Elastic::SearchResults, lib: true do
       # The Milestone you have no access to
       create :milestone, title: 'bla-bla term'
 
-      [Project, MergeRequest, Issue, Milestone].each do |model|
-        model.__elasticsearch__.refresh_index!
-      end
+      Gitlab::Elastic::Helper.refresh_index
 
       result = Gitlab::Elastic::SearchResults.new(user, [project.id], 'term')
 

spec/models/concerns/elastic/issue_spec.rb

@@ -3,11 +3,11 @@ require 'spec_helper'
 describe Issue, elastic: true do
   before do
     stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
-    described_class.__elasticsearch__.create_index!
+    Gitlab::Elastic::Helper.create_empty_index
   end
 
   after do
-    described_class.__elasticsearch__.delete_index!
+    Gitlab::Elastic::Helper.delete_index
     stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
   end
 
@@ -21,7 +21,7 @@ describe Issue, elastic: true do
     # The issue I have no access to
     create :issue, title: 'bla-bla term'
 
-    described_class.__elasticsearch__.refresh_index!
+    Gitlab::Elastic::Helper.refresh_index
 
     options = { project_ids: [project.id] }
 
@@ -36,10 +36,6 @@ describe Issue, elastic: true do
                                                 'updated_at', 'state', 'project_id', 'author_id',
                                                 'assignee_id', 'confidential')
 
-    expected_hash['project'] = { "id" => project.id }
-    expected_hash['author'] = { "id" => issue.author_id }
-    expected_hash['updated_at_sort'] = issue.updated_at
-
     expect(issue.as_indexed_json).to eq(expected_hash)
   end
 end

spec/models/concerns/elastic/merge_request_spec.rb

@@ -3,11 +3,11 @@ require 'spec_helper'
 describe MergeRequest, elastic: true do
   before do
     stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
-    described_class.__elasticsearch__.create_index!
+    Gitlab::Elastic::Helper.create_empty_index
   end
 
   after do
-    described_class.__elasticsearch__.delete_index!
+    Gitlab::Elastic::Helper.delete_index
     stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
   end
 
@@ -21,7 +21,7 @@ describe MergeRequest, elastic: true do
     # The merge request you have no access to
     create :merge_request, title: 'also with term'
 
-    described_class.__elasticsearch__.refresh_index!
+    Gitlab::Elastic::Helper.refresh_index
 
     options = { project_ids: [project.id] }
 
@@ -47,8 +47,6 @@ describe MergeRequest, elastic: true do
       'author_id'
     )
 
-    expected_hash['updated_at_sort'] = merge_request.updated_at
-
     expect(merge_request.as_indexed_json).to eq(expected_hash)
   end
 end

spec/models/concerns/elastic/milestone_spec.rb

@@ -3,11 +3,11 @@ require 'spec_helper'
 describe Milestone, elastic: true do
   before do
     stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
-    described_class.__elasticsearch__.create_index!
+    Gitlab::Elastic::Helper.create_empty_index
   end
 
   after do
-    described_class.__elasticsearch__.delete_index!
+    Gitlab::Elastic::Helper.delete_index
     stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
   end
 
@@ -21,7 +21,7 @@ describe Milestone, elastic: true do
     # The milestone you have no access to
     create :milestone, title: 'bla-bla term'
 
-    described_class.__elasticsearch__.refresh_index!
+    Gitlab::Elastic::Helper.refresh_index
 
     options = { project_ids: [project.id] }
 
@@ -36,11 +36,10 @@ describe Milestone, elastic: true do
       'title',
       'description',
       'project_id',
-      'created_at'
+      'created_at',
+      'updated_at'
     )
 
-    expected_hash[:updated_at_sort] = milestone.updated_at
-
     expect(milestone.as_indexed_json).to eq(expected_hash)
   end
 end

spec/models/concerns/elastic/note_spec.rb

@@ -3,11 +3,11 @@ require 'spec_helper'
 describe Note, elastic: true do
   before do
     stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
-    described_class.__elasticsearch__.create_index!
+    Gitlab::Elastic::Helper.create_empty_index
   end
 
   after do
-    described_class.__elasticsearch__.delete_index!
+    Gitlab::Elastic::Helper.delete_index
     stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
   end
 
@@ -20,7 +20,7 @@ describe Note, elastic: true do
     # The note in the project you have no access to
     create :note, note: 'bla-bla term'
 
-    described_class.__elasticsearch__.refresh_index!
+    Gitlab::Elastic::Helper.refresh_index
 
     options = { project_ids: [issue.project.id] }
 
@@ -35,8 +35,9 @@ describe Note, elastic: true do
       'note',
       'project_id',
       'created_at',
-      'issue',
-      'updated_at_sort'
+      'updated_at',
+      'issue'
+
     ]
 
     expect(note.as_indexed_json.keys).to eq(expected_hash_keys)
@@ -58,7 +59,7 @@ describe Note, elastic: true do
       create :note, note: 'bla-bla term', project: issue.project, noteable: issue
       create :note, project: issue.project, noteable: issue
 
-      Note.__elasticsearch__.refresh_index!
+      Gitlab::Elastic::Helper.refresh_index
 
       options = { project_ids: [issue.project.id] }
 
@@ -72,7 +73,7 @@ describe Note, elastic: true do
       create :note, note: 'bla-bla term', project: issue.project, noteable: issue
       create :note, project: issue.project, noteable: issue
 
-      Note.__elasticsearch__.refresh_index!
+      Gitlab::Elastic::Helper.refresh_index
 
       options = { project_ids: [issue.project.id], current_user: user }
 
@@ -89,7 +90,7 @@ describe Note, elastic: true do
       create :note, note: 'bla-bla term', project: issue.project, noteable: issue
       create :note, project: issue.project, noteable: issue
 
-      Note.__elasticsearch__.refresh_index!
+      Gitlab::Elastic::Helper.refresh_index
 
       options = { project_ids: [issue.project.id], current_user: member }
 
@@ -106,7 +107,7 @@ describe Note, elastic: true do
       create :note, note: 'bla-bla term', project: issue.project, noteable: issue
       create :note, project: issue.project, noteable: issue
 
-      Note.__elasticsearch__.refresh_index!
+      Gitlab::Elastic::Helper.refresh_index
 
       options = { project_ids: [issue.project.id], current_user: member }
 

spec/models/concerns/elastic/project_spec.rb

@@ -3,11 +3,11 @@ require 'spec_helper'
 describe Project, elastic: true do
   before do
     stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
-    described_class.__elasticsearch__.create_index!
+    Gitlab::Elastic::Helper.create_empty_index
   end
 
   after do
-    described_class.__elasticsearch__.delete_index!
+    Gitlab::Elastic::Helper.delete_index
     stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
   end
 
@@ -18,7 +18,7 @@ describe Project, elastic: true do
     create :empty_project, path: 'someone_elses_project'
     project_ids = [project.id, project1.id, project2.id]
 
-    described_class.__elasticsearch__.refresh_index!
+    Gitlab::Elastic::Helper.refresh_index
 
     expect(described_class.elastic_search('test', options: { pids: project_ids }).total_count).to eq(1)
     expect(described_class.elastic_search('test1', options: { pids: project_ids }).total_count).to eq(1)
@@ -36,6 +36,7 @@ describe Project, elastic: true do
       'namespace_id',
       'created_at',
       'archived',
+      'updated_at',
       'visibility_level',
       'last_activity_at'
     )

spec/models/concerns/elastic/repositories_spec.rb

@@ -3,11 +3,11 @@ require 'spec_helper'
 describe Repository, elastic: true do
   before do
     stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
-    described_class.__elasticsearch__.create_index!
+    Gitlab::Elastic::Helper.create_empty_index
   end
 
   after do
-    described_class.__elasticsearch__.delete_index!
+    Gitlab::Elastic::Helper.delete_index
     stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
   end
 
@@ -17,7 +17,7 @@ describe Repository, elastic: true do
     project.repository.index_blobs
     project.repository.index_commits
 
-    described_class.__elasticsearch__.refresh_index!
+    Gitlab::Elastic::Helper.refresh_index
 
     expect(project.repository.search('def popen')[:blobs][:total_count]).to eq(1)
     expect(project.repository.search('initial')[:commits][:total_count]).to eq(1)

spec/models/concerns/elastic/snippet_spec.rb

@@ -3,11 +3,11 @@ require 'spec_helper'
 describe Snippet, elastic: true do
   before do
     stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
-    described_class.__elasticsearch__.create_index!
+    Gitlab::Elastic::Helper.create_empty_index
   end
 
   after do
-    described_class.__elasticsearch__.delete_index!
+    Gitlab::Elastic::Helper.delete_index
     stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
   end
 
@@ -24,7 +24,7 @@ describe Snippet, elastic: true do
     let!(:project_private_snippet)  { create(:snippet, :private, project: project, content: 'password: XXX') }
 
     before do
-      described_class.__elasticsearch__.refresh_index!
+      Gitlab::Elastic::Helper.refresh_index
     end
 
     it 'returns only public snippets when user is blank' do
@@ -78,7 +78,7 @@ describe Snippet, elastic: true do
     create(:snippet, :public, file_name: 'index.php')
     create(:snippet)
 
-    described_class.__elasticsearch__.refresh_index!
+    Gitlab::Elastic::Helper.refresh_index
 
     options = { user: user }
 

spec/models/concerns/elastic/wiki_reposotories_spec.rb

@@ -3,11 +3,11 @@ require 'spec_helper'
 describe ProjectWiki, elastic: true do
   before do
     stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
-    described_class.__elasticsearch__.create_index!
+    Gitlab::Elastic::Helper.create_empty_index
   end
 
   after do
-    described_class.__elasticsearch__.delete_index!
+    Gitlab::Elastic::Helper.delete_index
     stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
   end
 
@@ -18,7 +18,7 @@ describe ProjectWiki, elastic: true do
 
     project.wiki.index_blobs
 
-    described_class.__elasticsearch__.refresh_index!
+    Gitlab::Elastic::Helper.refresh_index
 
     expect(project.wiki.search('bla', type: :blob)[:blobs][:total_count]).to eq(1)
   end

spec/models/repository_spec.rb

@@ -956,11 +956,11 @@ describe Repository, models: true do
   describe "Elastic search", elastic: true do
     before do
       stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
-      Repository.__elasticsearch__.create_index!
+      Gitlab::Elastic::Helper.create_empty_index
     end
 
     after do
-      Repository.__elasticsearch__.delete_index!
+      Gitlab::Elastic::Helper.delete_index
       stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
     end
 
@@ -970,7 +970,7 @@ describe Repository, models: true do
 
         project.repository.index_commits
 
-        Repository.__elasticsearch__.refresh_index!
+        Gitlab::Elastic::Helper.refresh_index
 
         expect(project.repository.find_commits_by_message_with_elastic('initial').first).to be_a(Commit)
         expect(project.repository.find_commits_by_message_with_elastic('initial').count).to eq(1)
@@ -983,7 +983,7 @@ describe Repository, models: true do
 
         project.repository.index_blobs
 
-        Repository.__elasticsearch__.refresh_index!
+        Gitlab::Elastic::Helper.refresh_index
 
         result = project.repository.search(
           'def popen',