Merge branch '1606-prefer-es-git-go' into 'master'

elasticsearch: Add support for an experimental repository indexer

Closes #1606

See merge request !1483

app/controllers/admin/application_settings_controller.rb

@@ -166,7 +166,8 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
       :shared_runners_minutes,
       :usage_ping_enabled,
       :minimum_mirror_sync_time,
-      :geo_status_timeout
+      :geo_status_timeout,
+      :elasticsearch_experimental_indexer,
     ]
   end
 end

app/views/admin/application_settings/_form.html.haml

@@ -469,6 +469,13 @@
             = f.check_box :elasticsearch_indexing
             Elasticsearch indexing
 
+    .form-group
+      .col-sm-offset-2.col-sm-10
+        .checkbox
+          = f.label :elasticsearch_experimental_indexer do
+            = f.check_box :elasticsearch_experimental_indexer
+            Use experimental repository indexer
+
     .form-group
       .col-sm-offset-2.col-sm-10
         .checkbox

changelogs/unreleased-ee/1606-prefer-es-git-go.yml

+---
+title: 'elasticsearch: Add support for an experimental repository indexer'
+merge_request: 1483
+author:

db/migrate/20170403141442_add_elasticsearch_experimental_indexer_to_application_settings.rb

+class AddElasticsearchExperimentalIndexerToApplicationSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column :application_settings, :elasticsearch_experimental_indexer, :boolean
+  end
+
+  def down
+    remove_column :application_settings, :elasticsearch_experimental_indexer
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20170402231018) do
+ActiveRecord::Schema.define(version: 20170403141442) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -131,6 +131,7 @@ ActiveRecord::Schema.define(version: 20170402231018) do
     t.integer "geo_status_timeout", default: 10
     t.string "uuid"
     t.decimal "polling_interval_multiplier", default: 1.0, null: false
+    t.boolean "elasticsearch_experimental_indexer"
   end
 
   create_table "approvals", force: :cascade do |t|

doc/integration/elasticsearch.md

@@ -55,6 +55,7 @@ The following Elasticsearch settings are available:
 | Parameter                           | Description |
 | ---------                           | ----------- |
 | `Elasticsearch indexing`            | Enables/disables Elasticsearch indexing. You may want to enable indexing but disable search in order to give the index time to be fully completed, for example. Also keep in mind that this option doesn't have any impact on existing data, this only enables/disables background indexer which tracks data changes. So by enabling this you will not get your existing data indexed, use special rake task for that as explained in [Add GitLab's data to the Elasticsearch index](#add-gitlabs-data-to-the-elasticsearch-index). |
+| `Use experimental repository indexer` | Perform repository indexing using [GitLab Elasticsearch Indexer](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer). |
 | `Search with Elasticsearch enabled` | Enables/disables using Elasticsearch in search. |
 | `URL`                              | The URL to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://host1, https://host2:9200"). |
 | `Using AWS hosted Elasticsearch with IAM credentials` | Sign your Elasticsearch requests using [AWS IAM authorization][aws-iam]. The access key must be allowed to perform `es:*` actions. |

lib/gitlab/elastic/indexer.rb

@@ -6,6 +6,8 @@ module Gitlab
     class Indexer
       include Gitlab::CurrentSettings
 
+      EXPERIMENTAL_INDEXER = 'gitlab-elasticsearch-indexer'.freeze
+
       Error = Class.new(StandardError)
 
       attr_reader :project
@@ -44,7 +46,11 @@ module Gitlab
       end
 
       def path_to_indexer
-        File.join(Rails.root, 'bin/elastic_repo_indexer')
+        if current_application_settings.elasticsearch_experimental_indexer?
+          EXPERIMENTAL_INDEXER
+        else
+          Rails.root.join('bin', 'elastic_repo_indexer').to_s
+        end
       end
 
       def run_indexer!(from_sha, to_sha)

spec/lib/gitlab/elastic/indexer_spec.rb

@@ -86,7 +86,19 @@ describe Gitlab::Elastic::Indexer do
     end
   end
 
-  def expect_popen(*with)
+  context 'experimental indexer present' do
+    before do
+      stub_application_setting(elasticsearch_experimental_indexer: true)
+    end
+
+    it 'uses the experimental indexer' do
+      expect_popen.with(['gitlab-elasticsearch-indexer', anything, anything], anything, anything).and_return(popen_success)
+
+      indexer.run
+    end
+  end
+
+  def expect_popen
     expect(Gitlab::Popen).to receive(:popen)
   end