Merge branch '1181-elasticsearch-client-params' into 'master'

Add support for HTTPS + AWS ElasticSearch

Closes #1181

See merge request !1305

.gitlab-ci.yml

@@ -9,7 +9,7 @@ variables:
   MYSQL_ALLOW_EMPTY_PASSWORD: "1"
   # retry tests only in CI environment
   RSPEC_RETRY_RETRY_COUNT: "3"
-  ELASTIC_HOST: "elasticsearch"
+  ELASTIC_URL: "http://elasticsearch:9200"
   RAILS_ENV: "test"
   SIMPLECOV: "true"
   SETUP_DB: "true"

Gemfile

@@ -108,6 +108,8 @@ gem 'elasticsearch-model', '~> 0.1.9'
 gem 'elasticsearch-rails', '~> 0.1.9'
 gem 'elasticsearch-api',   '5.0.3'
 gem 'gitlab-elasticsearch-git', '1.1.1', require: "elasticsearch/git"
+gem 'aws-sdk'
+gem 'faraday_middleware-aws-signers-v4'
 
 # Markdown and HTML processing
 gem 'html-pipeline',        '~> 1.11.0'

Gemfile.lock

@@ -67,6 +67,14 @@ GEM
       execjs
       json
     awesome_print (1.2.0)
+    aws-sdk (2.7.8)
+      aws-sdk-resources (= 2.7.8)
+    aws-sdk-core (2.7.8)
+      aws-sigv4 (~> 1.0)
+      jmespath (~> 1.0)
+    aws-sdk-resources (2.7.8)
+      aws-sdk-core (= 2.7.8)
+    aws-sigv4 (1.0.0)
     axiom-types (0.1.1)
       descendants_tracker (~> 0.0.4)
       ice_nine (~> 0.11.0)
@@ -203,6 +211,9 @@ GEM
       multipart-post (>= 1.2, < 3)
     faraday_middleware (0.10.0)
       faraday (>= 0.7.4, < 0.10)
+    faraday_middleware-aws-signers-v4 (0.1.5)
+      aws-sdk (~> 2.1)
+      faraday (~> 0.9)
     faraday_middleware-multi_json (0.0.6)
       faraday_middleware
       multi_json
@@ -393,6 +404,7 @@ GEM
     jira-ruby (1.1.2)
       activesupport
       oauth (~> 0.5, >= 0.5.0)
+    jmespath (1.3.1)
     jquery-atwho-rails (1.3.2)
     jquery-rails (4.1.1)
       rails-dom-testing (>= 1, < 3)
@@ -862,6 +874,7 @@ DEPENDENCIES
   asciidoctor-plantuml (= 0.0.7)
   attr_encrypted (~> 3.0.0)
   awesome_print (~> 1.2.0)
+  aws-sdk
   babosa (~> 1.0.2)
   base32 (~> 0.3.0)
   benchmark-ips (~> 2.3.0)
@@ -896,6 +909,7 @@ DEPENDENCIES
   email_reply_trimmer (~> 0.1)
   email_spec (~> 1.6.0)
   factory_girl_rails (~> 4.7.0)
+  faraday_middleware-aws-signers-v4
   ffaker (~> 2.4)
   flay (~> 2.6.1)
   fog-aws (~> 0.9)

app/controllers/admin/application_settings_controller.rb

@@ -161,9 +161,12 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
   def application_setting_params_ee
     [
       :help_text,
-      :elasticsearch_host,
+      :elasticsearch_url,
       :elasticsearch_indexing,
-      :elasticsearch_port,
+      :elasticsearch_aws,
+      :elasticsearch_aws_access_key,
+      :elasticsearch_aws_secret_access_key,
+      :elasticsearch_aws_region,
       :elasticsearch_search,
       :repository_size_limit,
       :shared_runners_minutes,

app/models/application_setting.rb

@@ -91,13 +91,13 @@ class ApplicationSetting < ActiveRecord::Base
             presence: true,
             numericality: { only_integer: true, greater_than: 0 }
 
-  validates :elasticsearch_host,
+  validates :elasticsearch_url,
             presence: { message: "can't be blank when indexing is enabled" },
             if: :elasticsearch_indexing?
 
-  validates :elasticsearch_port,
-            presence: { message: "can't be blank when indexing is enabled" },
-            if: :elasticsearch_indexing?
+  validates :elasticsearch_aws_region,
+            presence: { message: "can't be blank when using aws hosted elasticsearch" },
+            if: ->(setting) { setting.elasticsearch_indexing? && setting.elasticsearch_aws? }
 
   validates :repository_storages, presence: true
   validate :check_repository_storages
@@ -236,8 +236,9 @@ class ApplicationSetting < ActiveRecord::Base
 
   def self.defaults_ee
     {
-      elasticsearch_host: ENV['ELASTIC_HOST'] || 'localhost',
-      elasticsearch_port: ENV['ELASTIC_PORT'] || '9200',
+      elasticsearch_url: ENV['ELASTIC_URL'] || 'http://localhost:9200',
+      elasticsearch_aws: false,
+      elasticsearch_aws_region: ENV['ELASTIC_REGION'] || 'us-east-1',
       usage_ping_enabled: true,
       minimum_mirror_sync_time: Gitlab::Mirror::FIFTEEN
     }
@@ -268,8 +269,18 @@ class ApplicationSetting < ActiveRecord::Base
     Gitlab::Mirror.configure_cron_jobs!
   end
 
-  def elasticsearch_host
-    read_attribute(:elasticsearch_host).split(',').map(&:strip)
+  def elasticsearch_url
+    read_attribute(:elasticsearch_url).split(',').map(&:strip)
+  end
+
+  def elasticsearch_config
+    {
+      url:                   elasticsearch_url,
+      aws:                   elasticsearch_aws,
+      aws_access_key:        elasticsearch_aws_access_key,
+      aws_secret_access_key: elasticsearch_aws_secret_access_key,
+      aws_region:            elasticsearch_aws_region,
+    }
   end
 
   def home_page_url_column_exist

app/views/admin/application_settings/_form.html.haml

@@ -454,16 +454,40 @@
             Search with Elasticsearch enabled
 
     .form-group
-      = f.label :elasticsearch_host, 'Host', class: 'control-label col-sm-2'
+      = f.label :elasticsearch_url, 'URL', class: 'control-label col-sm-2'
       .col-sm-10
-        = f.text_field :elasticsearch_host, value: @application_setting.elasticsearch_host.join(', '), class: 'form-control', placeholder: 'localhost'
+        = f.text_field :elasticsearch_url, value: @application_setting.elasticsearch_url.join(', '), class: 'form-control', placeholder: 'http://localhost:9200'
         .help-block
-          The TCP/IP host to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "host1, host2").
+          The url to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://localhost:9200, http://localhost:9201").
 
+  %fieldset
+    %legend Elasticsearch AWS IAM credentials
+    .form-group
+      .col-sm-offset-2.col-sm-10
+        .checkbox
+          = f.label :elasticsearch_aws do
+            = f.check_box :elasticsearch_aws
+            Using AWS hosted Elasticsearch with IAM credentials
     .form-group
-      = f.label :elasticsearch_port, 'Port', class: 'control-label col-sm-2'
+      = f.label :elasticsearch_aws_region, 'AWS region', class: 'control-label col-sm-2'
       .col-sm-10
-        = f.text_field :elasticsearch_port, class: 'form-control', placeholder: ApplicationSetting.current.elasticsearch_port
+        = f.text_field :elasticsearch_aws_region, value: @application_setting.elasticsearch_aws_region, class: 'form-control'
+        .help-block
+          Region that elasticsearch is configured
+
+    .form-group
+      = f.label :elasticsearch_aws_access_key, 'AWS Access Key', class: 'control-label col-sm-2'
+      .col-sm-10
+        = f.text_field :elasticsearch_aws_access_key, value: @application_setting.elasticsearch_aws_access_key, class: 'form-control'
+        .help-block
+          AWS Access Key.  Only required if not using role instance credentials
+
+    .form-group
+      = f.label :elasticsearch_aws_secret_access_key, 'AWS Secret Access Key', class: 'control-label col-sm-2'
+      .col-sm-10
+        = f.password_field :elasticsearch_aws_secret_access_key, value: @application_setting.elasticsearch_aws_secret_access_key, class: 'form-control'
+        .help-block
+          AWS Secret Access Key.  Only required if not using role instance credentials
 
   %fieldset
     %legend Koding

bin/elastic_repo_indexer

@@ -8,7 +8,9 @@ require 'active_support'
 require 'active_support/core_ext'
 require 'benchmark'
 
-path_to_log_file = File.expand_path('../../log/es-indexer.log', __FILE__)
+require File.expand_path('../lib/gitlab/elastic/client', File.dirname(__FILE__))
+
+path_to_log_file = File.expand_path('../log/es-indexer.log', File.dirname(__FILE__))
 LOGGER = Logger.new(path_to_log_file)
 
 PROJECT_ID = ARGV.shift
@@ -17,21 +19,21 @@ FROM_SHA = ENV['FROM_SHA']
 TO_SHA = ENV['TO_SHA']
 RAILS_ENV = ENV['RAILS_ENV']
 
-LOGGER.info("Has been scheduled for project #{REPO_PATH} with SHA range #{FROM_SHA}:#{TO_SHA}")
+# Symbols get stringified when passed through JSON
+elastic = {}
+JSON.parse(ENV['ELASTIC_CONNECTION_INFO']).each { |k, v| elastic[k.to_sym] = v }
+ELASTIC_CONFIG = elastic
 
-elastic_connection_info = JSON.parse ENV['ELASTIC_CONNECTION_INFO']
-ELASTIC_HOST = elastic_connection_info['host']
-ELASTIC_PORT = elastic_connection_info['port']
+LOGGER.info("Has been scheduled for project #{REPO_PATH} with SHA range #{FROM_SHA}:#{TO_SHA}")
 
 class Repository
   include Elasticsearch::Git::Repository
 
   index_name ['gitlab', RAILS_ENV].compact.join('-')
 
-  self.__elasticsearch__.client = Elasticsearch::Client.new(
-    host: ELASTIC_HOST,
-    port: ELASTIC_PORT
-  )
+  def initialize
+    self.__elasticsearch__.client = ::Gitlab::Elastic::Client.build(ELASTIC_CONFIG)
+  end
 
   def client_for_indexing
     self.__elasticsearch__.client

changelogs/unreleased-ee/1181-elasticsearch-client-params.yml

+---
+title: Expose elasticsearch client params for AWS signing and HTTPS
+merge_request: 1305
+author: Matt Gresko

config/initializers/1_settings.rb

@@ -277,8 +277,7 @@ Settings.gitlab['no_todos_messages'] ||= YAML.load_file(Rails.root.join('config'
 #
 Settings['elasticsearch'] ||= Settingslogic.new({})
 Settings.elasticsearch['enabled'] = false if Settings.elasticsearch['enabled'].nil?
-Settings.elasticsearch['host'] ||=  ENV['ELASTIC_HOST'] || "localhost"
-Settings.elasticsearch['port'] ||= ENV['ELASTIC_PORT'] || 9200
+Settings.elasticsearch['url'] = ENV['ELASTIC_URL'] || "http://localhost:9200"
 
 #
 # CI

config/initializers/elastic_client_setup.rb

@@ -10,21 +10,15 @@ module Elasticsearch
 
         def client(client = nil)
           if @client.nil? || es_configuration_changed?
-            @es_host = current_application_settings.elasticsearch_host
-            @es_port = current_application_settings.elasticsearch_port
-
-            @client = Elasticsearch::Client.new(
-              host: current_application_settings.elasticsearch_host,
-              port: current_application_settings.elasticsearch_port
-            )
+            @es_config = current_application_settings.elasticsearch_config
+            @client = ::Gitlab::Elastic::Client.build(@es_config)
           end
 
           @client
         end
 
         def es_configuration_changed?
-          @es_host != current_application_settings.elasticsearch_host ||
-            @es_port != current_application_settings.elasticsearch_port
+          @es_config != current_application_settings.elasticsearch_config
         end
       end
     end

db/migrate/20170215151539_add_aws_elasticsearch.rb

+class AddAwsElasticsearch < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :application_settings, :elasticsearch_url, :string, default: 'http://localhost:9200'
+    add_column :application_settings, :elasticsearch_aws, :boolean, default: false, null: false
+    add_column :application_settings, :elasticsearch_aws_region, :string, default: 'us-east-1'
+    add_column :application_settings, :elasticsearch_aws_access_key, :string
+    add_column :application_settings, :elasticsearch_aws_secret_access_key, :string
+  end
+end

db/migrate/20170215151540_migrate_old_elasticsearch_settings.rb

+class MigrateOldElasticsearchSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    settings = Arel::Table.new(:application_settings)
+
+    finder =
+      settings
+        .where(settings[:elasticsearch_host].not_eq(nil))
+        .project(:id, :elasticsearch_host, :elasticsearch_port)
+
+    result = connection.exec_query(finder.to_sql)
+
+    # There are only a few rows in the `application_settings` table
+    result.rows.each do |id, hosts, port|
+      # elasticsearch_host may look like "1.example.com,2.example.com, 3.example.com"
+      urls = hosts.split(',').map do |host|
+        url = URI.parse('http://' + host.strip)
+        url.port = port
+        url.to_s
+      end
+
+      updater =
+        Arel::UpdateManager.new(ActiveRecord::Base)
+          .table(settings)
+          .set(settings[:elasticsearch_url] => urls.join(','))
+          .where(settings[:id].eq(id))
+
+      connection.exec_update(updater.to_sql)
+    end
+  end
+
+  def down
+    # no-op
+  end
+end

db/migrate/20170215151541_remove_old_elasticsearch_settings.rb

+class RemoveOldElasticsearchSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = true
+
+  DOWNTIME_REASON = 'Removing two columns from application_settings'
+
+  def change
+    remove_column :application_settings, :elasticsearch_host, :string, default: 'localhost'
+    remove_column :application_settings, :elasticsearch_port, :string, default: '9200'
+  end
+end

db/schema.rb

@@ -93,8 +93,6 @@ ActiveRecord::Schema.define(version: 20170224075132) do
     t.boolean "user_default_external", default: false, null: false
     t.boolean "elasticsearch_indexing", default: false, null: false
     t.boolean "elasticsearch_search", default: false, null: false
-    t.string "elasticsearch_host", default: "localhost"
-    t.string "elasticsearch_port", default: "9200"
     t.string "repository_storages", default: "default"
     t.string "enabled_git_access_protocol"
     t.boolean "domain_blacklist_enabled", default: false
@@ -121,7 +119,12 @@ ActiveRecord::Schema.define(version: 20170224075132) do
     t.integer "repository_size_limit", limit: 8, default: 0
     t.integer "terminal_max_session_time", default: 0, null: false
     t.integer "minimum_mirror_sync_time", default: 15, null: false
-    t.string "default_artifacts_expire_in", default: '0', null: false
+    t.string "default_artifacts_expire_in", default: "0", null: false
+    t.string "elasticsearch_url", default: "http://localhost:9200"
+    t.boolean "elasticsearch_aws", default: false, null: false
+    t.string "elasticsearch_aws_region", default: "us-east-1"
+    t.string "elasticsearch_aws_access_key"
+    t.string "elasticsearch_aws_secret_access_key"
   end
 
   create_table "approvals", force: :cascade do |t|

doc/api/settings.md

@@ -86,8 +86,11 @@ PUT /application/settings
 | `help_text` | string | no | GitLab server administrator information |
 | `elasticsearch_indexing` | boolean | no | Enable Elasticsearch indexing |
 | `elasticsearch_search` | boolean | no | Enable Elasticsearch search |
-| `elasticsearch_host` | string | no | The TCP/IP host to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "host1, host2") |
-| `elasticsearch_port` | integer | no | The TCP/IP port that Elasticsearch listens to. The default value is 9200 |
+| `elasticsearch_url` | string | no | The url to use for connecting to Elasticsearch. Use a comma-separated list to support cluster (e.g., "http://localhost:9200, http://localhost:9201") |
+| `elasticsearch_aws` | boolean | no | Enable the use of AWS hosted Elasticsearch |
+| `elasticsearch_aws_region` | string | no | The AWS region the elasticsearch domain is configured |
+| `elasticsearch_aws_access_key` | string | no | AWS IAM access key |
+| `elasticsearch_aws_secret_access_key` | string | no | AWS IAM secret access key |
 | `usage_ping_enabled` | boolean | no | Every week GitLab will report license usage back to GitLab, Inc.|
 | `repository_size_limit` | integer | no | Size limit per repository (MB) |
 | `plantuml_enabled` | boolean | no | Enable PlantUML integration. Default is `false`. |

doc/api/v3_to_v4.md

@@ -58,3 +58,4 @@ changes are in V4:
 
 - Remove the ProjectGitHook API. Use the ProjectPushRule API instead [!1301](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/1301)
 - Removed `repository_storage` from `PUT /application/settings` and `GET /application/settings` (use `repository_storages` instead) [!1307](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/1307)
+- Removed `elasticsearch_host` and `elasticsearch_port` from `PUT /application/settings` (use `elasticsearch_url` instead) [!1305](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/1305) 

doc/integration/elasticsearch.md

 # Elasticsearch integration
 
->[Introduced][ee-109] in GitLab EE 8.4.
+> Elasticsearch was [Introduced][ee-109] in GitLab EE 8.4. Support for
+> [Amazon Elasticsearch][aws-elasticsearch] was [introduced][ee-1305] in GitLab
+> EE 9.0.
 
 [Elasticsearch] is a flexible, scalable and powerful search service.
 
@@ -23,7 +25,8 @@ searching in:
 Once the data is added to the database or repository and [Elasticsearch is
 enabled in the admin area](#enable-elasticsearch) the search index will be
 updated automatically. Elasticsearch can be installed on the same machine as
-GitLab, or on a separate server.
+GitLab, or on a separate server, or you can use the [Amazon Elasticsearch][aws-elastic]
+service.
 
 ## Requirements
 
@@ -53,10 +56,11 @@ The following Elasticsearch settings are available:
 | ---------                           | ----------- |
 | `Elasticsearch indexing`            | Enables/disables Elasticsearch indexing. You may want to enable indexing but disable search in order to give the index time to be fully completed, for example. Also keep in mind that this option doesn't have any impact on existing data, this only enables/disables background indexer which tracks data changes. So by enabling this you will not get your existing data indexed, use special rake task for that as explained in [Add GitLab's data to the Elasticsearch index](#add-gitlabs-data-to-the-elasticsearch-index). |
 | `Search with Elasticsearch enabled` | Enables/disables using Elasticsearch in search. |
-| `Host`                              | The TCP/IP host to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "host1, host2"). |
-| `Port`                              | The TCP port that Elasticsearch listens to. The default value is 9200  |
-
-
+| `URL`                              | The URL to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://host1, https://host2:9200"). |
+| `Using AWS hosted Elasticsearch with IAM credentials` | Sign your Elasticsearch requests using [AWS IAM authorization][aws-iam]. The access key must be allowed to perform `es:*` actions. |
+| `AWS Region` | The AWS region your Elasticsearch service is located in. |
+| `AWS Access Key` | The AWS access key. |
+| `AWS Secret Access Key` | The AWS secret access key. |
 
 ## Add GitLab's data to the Elasticsearch index
 
@@ -298,12 +302,28 @@ see details in the [8-11-to-8-12 update guide](https://gitlab.com/gitlab-org/git
 If you have this exception (just like in the case above but the actual message is different) please check if you have the correct Elasticsearch version and you met the other [requirements](#requirements).
 There is also an easy way to check it automatically with `sudo gitlab-rake gitlab:check` command.
 
-### I indexed all the repositories but I can't find anything
+### Exception Elasticsearch::Transport::Transport::Errors::RequestEntityTooLarge
 
-Make sure you indexed all the database data as stated above (`sudo gitlab-rake gitlab:elastic:index`)
+```
+[413] {"Message":"Request size exceeded 10485760 bytes"}
+```
+
+This exception is seen when your Elasticsearch cluster is configured to reject
+requests above a certain size (10MiB in this case). This corresponds to the
+`http.max_content_length` setting in `elasticsearch.yml`. Increase it to a
+larger size and restart your Elasticsearch cluster.
 
+AWS has [fixed limits](http://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/aes-limits.html)
+for this setting ("Maximum Size of HTTP Request Payloads"), based on the size of
+the underlying instance.
 
+### I indexed all the repositories but I can't find anything
+
+Make sure you indexed all the database data as stated above (`sudo gitlab-rake gitlab:elastic:index`)
 
+[ee-1305]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/1305
+[aws-elasticsearch]: http://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-gsg.html
+[aws-iam]: http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html
 [ee-109]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/109 "Elasticsearch Merge Request"
 [elasticsearch]: https://www.elastic.co/products/elasticsearch "Elasticsearch website"
 [install]: https://www.elastic.co/guide/en/elasticsearch/reference/current/_installation.html "Elasticsearch installation documentation"

lib/api/settings.rb

@@ -115,8 +115,13 @@ module API
       optional :elasticsearch_indexing, type: Boolean, desc: 'Enable Elasticsearch indexing'
       given elasticsearch_indexing: ->(val) { val } do
         optional :elasticsearch_search, type: Boolean, desc: 'Enable Elasticsearch search'
-        requires :elasticsearch_host, type: String, desc: 'The TCP/IP host to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "host1, host2")'
-        requires :elasticsearch_port, type: Integer, desc: 'The TCP/IP port that Elasticsearch listens to. The default value is 9200'
+        requires :elasticsearch_url, type: String, desc: 'The url to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://localhost:9200, http://localhost:9201")'
+      end
+      optional :elasticsearch_aws, type: Boolean, desc: 'Enable support for AWS hosted elasticsearch'
+      given elasticsearch_aws: ->(val) { val } do
+        requires :elasticsearch_aws_region, type: String, desc: 'The AWS region the elasticsearch domain is configured'
+        optional :elasticsearch_aws_access_key, type: String, desc: 'AWS IAM access key'
+        optional :elasticsearch_aws_secret_access_key, type: String, desc: 'AWS IAM secret access key'
       end
       optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.'
       optional :repository_storages, type: Array[String], desc: 'A list of names of enabled storage paths, taken from `gitlab.yml`. New projects will be created in one of these stores, chosen at random.'

lib/api/v3/settings.rb

@@ -115,8 +115,7 @@ module API
         optional :elasticsearch_indexing, type: Boolean, desc: 'Enable Elasticsearch indexing'
         given elasticsearch_indexing: ->(val) { val } do
           optional :elasticsearch_search, type: Boolean, desc: 'Enable Elasticsearch search'
-          requires :elasticsearch_host, type: String, desc: 'The TCP/IP host to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "host1, host2")'
-          requires :elasticsearch_port, type: Integer, desc: 'The TCP/IP port that Elasticsearch listens to. The default value is 9200'
+          requires :elasticsearch_url, type: String, desc: 'The url to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://localhost:9200, http://localhost:9201")'
         end
         optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.'
         optional :repository_storage, type: String, desc: 'The first entry in `repository_storages`. Deprecated, but retained for compatibility reasons'

lib/gitlab/elastic/client.rb

+# This file is required by `bin/elastic_repo_indexer` as well as from within
+# Rails, so needs to explicitly require its dependencies
+require 'elasticsearch'
+require 'aws-sdk'
+require 'faraday_middleware/aws_signers_v4'
+
+module Gitlab
+  module Elastic
+    module Client
+      # Takes a hash as returned by `ApplicationSetting#elasticsearch_config`,
+      # and configures itself based on those parameters
+      def self.build(config)
+        base_config = { urls: config[:url] }
+
+        if config[:aws]
+          creds = Aws::Credentials.new(config[:aws_access_key], config[:aws_secret_access_key])
+          region = config[:aws_region]
+
+          ::Elasticsearch::Client.new(base_config) do |fmid|
+            fmid.request(:aws_signers_v4, credentials: creds, service_name: 'es', region: region)
+          end
+        else
+          ::Elasticsearch::Client.new(base_config)
+        end
+      end
+    end
+  end
+end

lib/gitlab/elastic/indexer.rb

@@ -13,15 +13,10 @@ module Gitlab
       def initialize(project)
         @project = project
 
-        connection_info = {
-          host: current_application_settings.elasticsearch_host,
-          port: current_application_settings.elasticsearch_port
-        }.to_json
-
         # We accept any form of settings, including string and array
         # This is why JSON is needed
         @vars = {
-          'ELASTIC_CONNECTION_INFO' => connection_info,
+          'ELASTIC_CONNECTION_INFO' => current_application_settings.elasticsearch_config.to_json,
           'RAILS_ENV'               => Rails.env
         }
       end

lib/tasks/gitlab/check.rake

@@ -1072,8 +1072,7 @@ namespace :gitlab do
   end
 
   def check_elasticsearch
-    client = Elasticsearch::Client.new(host: ApplicationSetting.current.elasticsearch_host,
-                                       port: ApplicationSetting.current.elasticsearch_port)
+    client = Gitlab::Elastic::Client.build(ApplicationSetting.current.elasticsearch_config)
 
     print "Elasticsearch version 5.1.x? ... "
 

spec/lib/gitlab/elastic/client_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Elastic::Client do
+  describe 'build' do
+    let(:client) { described_class.build(params) }
+
+    context 'without credentials' do
+      let(:params) { { url: 'http://dummy-elastic:9200' } }
+
+      it 'makes unsigned requests' do
+        stub_request(:get, 'http://dummy-elastic:9200/foo/_all/1')
+          .with(headers: { 'Content-Type' => 'application/json' })
+          .to_return(status: 200, body: [:fake_response])
+
+        expect(client.get(index: 'foo', id: 1)).to eq([:fake_response])
+      end
+    end
+
+    context 'with AWS IAM credentials' do
+      let(:params) do
+        {
+          url: 'http://example-elastic:9200',
+          aws: true,
+          aws_region: 'us-east-1',
+          aws_access_key: '0',
+          aws_secret_access_key: '0',
+        }
+      end
+
+      it 'signs requests' do
+        travel_to(Time.parse('20170303T133952Z')) do
+          stub_request(:get, 'http://example-elastic:9200/foo/_all/1')
+            .with(
+              headers: {
+               'Authorization'        => 'AWS4-HMAC-SHA256 Credential=0/20170303/us-east-1/es/aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=4ba2aae19a476152dacf5a2191da67b0cf81b9d7152dab5c42b1bba701da19f1',
+               'Content-Type'         => 'application/json',
+               'X-Amz-Content-Sha256' => 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
+               'X-Amz-Date'           => '20170303T133952Z',
+            })
+            .to_return(status: 200, body: [:fake_response])
+
+          expect(client.get(index: 'foo', id: 1)).to eq([:fake_response])
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/elastic/indexer_spec.rb

@@ -5,7 +5,7 @@ describe Gitlab::Elastic::Indexer do
 
   before do
     stub_env('IN_MEMORY_APPLICATION_SETTINGS', 'true')
-    stub_application_setting(es_host: ['elastic-host1', 'elastic-host2'])
+    stub_application_setting(elasticsearch_url: ['http://localhost:9200'])
   end
 
   let(:project)  { create(:project) }
@@ -16,13 +16,6 @@ describe Gitlab::Elastic::Indexer do
   let(:popen_success) { [[''], 0] }
   let(:popen_failure) { [['error'], 1] }
 
-  let(:elastic_connection_info) do
-    {
-      host: current_application_settings.elasticsearch_host,
-      port: current_application_settings.elasticsearch_port,
-    }
-  end
-
   context 'empty project' do
     let(:project) { create(:empty_project) }
 
@@ -58,7 +51,7 @@ describe Gitlab::Elastic::Indexer do
         ],
         nil,
         hash_including(
-          'ELASTIC_CONNECTION_INFO' => elastic_connection_info.to_json,
+          'ELASTIC_CONNECTION_INFO' => current_application_settings.elasticsearch_config.to_json,
           'RAILS_ENV'               => Rails.env,
           'FROM_SHA'                => from_sha,
           'TO_SHA'                  => to_sha

spec/models/application_setting_spec.rb

@@ -294,4 +294,44 @@ describe ApplicationSetting, models: true do
       expect(setting.repository_size_limit).to eql(8.exabytes - 1)
     end
   end
+
+  describe '#elasticsearch_url' do
+    it 'presents a single URL as a one-element array' do
+      setting.elasticsearch_url = 'http://example.com'
+
+      expect(setting.elasticsearch_url).to eq(%w[http://example.com])
+    end
+
+    it 'presents multiple URLs as a many-element array' do
+      setting.elasticsearch_url = 'http://example.com,https://invalid.invalid:9200'
+
+      expect(setting.elasticsearch_url).to eq(%w[http://example.com https://invalid.invalid:9200])
+    end
+
+    it 'strips whitespace from around URLs' do
+      setting.elasticsearch_url = ' http://example.com, https://invalid.invalid:9200 '
+
+      expect(setting.elasticsearch_url).to eq(%w[http://example.com https://invalid.invalid:9200])
+    end
+  end
+
+  describe '#elasticsearch_config' do
+    it 'places all elasticsearch configuration values into a hash' do
+      setting.update!(
+        elasticsearch_url: 'http://example.com:9200',
+        elasticsearch_aws: false,
+        elasticsearch_aws_region:     'test-region',
+        elasticsearch_aws_access_key: 'test-access-key',
+        elasticsearch_aws_secret_access_key: 'test-secret-access-key'
+      )
+
+      expect(setting.elasticsearch_config).to eq(
+        url: ['http://example.com:9200'],
+        aws: false,
+        aws_region:     'test-region',
+        aws_access_key: 'test-access-key',
+        aws_secret_access_key: 'test-secret-access-key'
+      )
+    end
+  end
 end

spec/support/webmock.rb

 require 'webmock'
 require 'webmock/rspec'
 
-WebMock.disable_net_connect!(
-  allow_localhost: true,
-  allow: ['elasticsearch', 'registry.gitlab.com-gitlab-org-test-elastic-image']
-)
+allowed = %w[elasticsearch registry.gitlab.com-gitlab-org-test-elastic-image]
+
+if ENV.has_key?('ELASTIC_URL')
+  url = URI.parse(ENV['ELASTIC_URL'])
+  allowed << url.host
+  allowed.uniq!
+end
+
+WebMock.disable_net_connect!(allow_localhost: true, allow: allowed)

spec/workers/elastic_indexer_worker_spec.rb

@@ -5,14 +5,12 @@ describe ElasticIndexerWorker, elastic: true do
   subject { described_class.new }
 
   before do
-    Elasticsearch::Model.client = Elasticsearch::Client.new(
-      host: current_application_settings.elasticsearch_host,
-      port: current_application_settings.elasticsearch_port
-    )
+    stub_application_setting(elasticsearch_indexing: true)
 
-    Gitlab::Elastic::Helper.create_empty_index
+    Elasticsearch::Model.client =
+      Gitlab::Elastic::Client.build(current_application_settings.elasticsearch_config)
 
-    stub_application_setting(elasticsearch_indexing: true)
+    Gitlab::Elastic::Helper.create_empty_index
   end
 
   after do