5.1 update: Fix removing project data on project removing

It also adds the custom ES API action because elasticsearch-ruby is not fully compatible with ES 5

app/workers/elastic_indexer_worker.rb

+require 'gitlab/elastic/delete_by_query'
+
 class ElasticIndexerWorker
   include Sidekiq::Worker
   include Elasticsearch::Model::Client::ClassMethods
@@ -36,7 +38,7 @@ class ElasticIndexerWorker
         client.delete index: klass.index_name, type: klass.document_type, id: record_id
       end
 
-      clear_project_indexes(record_id) if klass == Project
+      clear_project_data(record_id) if klass == Project
     end
   rescue Elasticsearch::Transport::Transport::Errors::NotFound, ActiveRecord::RecordNotFound
     # These errors can happen in several cases, including:
@@ -56,44 +58,34 @@ class ElasticIndexerWorker
     end
   end
 
-  def clear_project_indexes(record_id)
-    remove_repository_index(record_id)
-    remove_wiki_index(record_id)
-    remove_nested_content(record_id)
+  def clear_project_data(record_id)
+    remove_children_documents(Repository.document_type, record_id)
+    remove_children_documents(ProjectWiki.document_type, record_id)
+    remove_children_documents(MergeRequest.document_type, record_id)
+    remove_documents_by_project_id(record_id)
   end
 
-  def remove_repository_index(record_id)
-    client.delete_by_query({
-      index: Repository.__elasticsearch__.index_name,
+  def remove_documents_by_project_id(record_id)
+    client.gitlab_delete_by_query({
+      index: Project.__elasticsearch__.index_name,
       body: {
         query: {
-          or: [
-            { term: { "commit.rid" => record_id } },
-            { term: { "blob.rid" => record_id } }
-          ]
+          term: { "project_id" => record_id }
         }
       }
     })
   end
 
-  def remove_nested_content(record_id)
-    client.delete_by_query({
+  def remove_children_documents(document_type, parent_record_id)
+    client.gitlab_delete_by_query({
       index: Project.__elasticsearch__.index_name,
       body: {
         query: {
-          term: { "_parent" => record_id }
+          parent_id: {
+            type: document_type,
+            id: parent_record_id
           }
         }
-    })
-  end
-
-  def remove_wiki_index(record_id)
-    client.delete_by_query({
-      index: ProjectWiki.__elasticsearch__.index_name,
-      body: {
-        query: {
-          term: { "blob.rid" => "wiki_#{record_id}" }
-        }
       }
     })
   end

lib/gitlab/elastic/delete_by_query.rb

+module Elasticsearch
+  module API
+    module Actions
+      # The built-in action https://github.com/elastic/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/delete_by_query.rb
+      # does not work with Elasticsearch 5.0 yet. There is an issue for that https://github.com/elastic/elasticsearch-ruby/issues/387
+      # but until it's not fixed we can use our own action for that
+      def gitlab_delete_by_query(arguments={})
+        raise ArgumentError, "Required argument 'index' missing" unless arguments[:index]
+
+        valid_params = [
+          :analyzer,
+          :consistency,
+          :default_operator,
+          :df,
+          :ignore_indices,
+          :ignore_unavailable,
+          :allow_no_indices,
+          :expand_wildcards,
+          :replication,
+          :q,
+          :routing,
+          :source,
+          :timeout ]
+
+        method = HTTP_POST
+        path   = Utils.__pathify Utils.__listify(arguments[:index]),
+                                 Utils.__listify(arguments[:type]),
+                                 '/_delete_by_query'
+
+        params = Utils.__validate_and_extract_params arguments, valid_params
+        body   = arguments[:body]
+
+        perform_request(method, path, params, body).body
+      end
+    end
+  end
+end