[1181] Expose low level elasticsearch client params

  * replace elasticsearch_host and elasticsearch_port with elasticsearch_url
  * Add support for AWS Elasticsearch Service
    * created universal gitlab elasticsearch client
    * add ability to sign requests with aws_signers_v4
    * expose elasticsearch_aws_region param
    * expose elasticsearch_aws_access_key param
    * expose elasticsearch_aws_secret_access_key param
    * If using AWS instance credentials they will automatically be picked up by client

.gitlab-ci.yml

@@ -9,7 +9,7 @@ variables:
   MYSQL_ALLOW_EMPTY_PASSWORD: "1"
   # retry tests only in CI environment
   RSPEC_RETRY_RETRY_COUNT: "3"
-  ELASTIC_HOST: "elasticsearch"
+  ELASTIC_URL: "http://elasticsearch:9200"
   RAILS_ENV: "test"
   SIMPLECOV: "true"
   SETUP_DB: "true"

Gemfile

@@ -108,6 +108,8 @@ gem 'elasticsearch-model', '~> 0.1.9'
 gem 'elasticsearch-rails', '~> 0.1.9'
 gem 'elasticsearch-api',   '5.0.3'
 gem 'gitlab-elasticsearch-git', '1.1.1', require: "elasticsearch/git"
+gem 'aws-sdk'
+gem 'faraday_middleware-aws-signers-v4'
 
 # Markdown and HTML processing
 gem 'html-pipeline',        '~> 1.11.0'

Gemfile.lock

@@ -67,6 +67,14 @@ GEM
       execjs
       json
     awesome_print (1.2.0)
+    aws-sdk (2.7.8)
+      aws-sdk-resources (= 2.7.8)
+    aws-sdk-core (2.7.8)
+      aws-sigv4 (~> 1.0)
+      jmespath (~> 1.0)
+    aws-sdk-resources (2.7.8)
+      aws-sdk-core (= 2.7.8)
+    aws-sigv4 (1.0.0)
     axiom-types (0.1.1)
       descendants_tracker (~> 0.0.4)
       ice_nine (~> 0.11.0)
@@ -203,6 +211,9 @@ GEM
       multipart-post (>= 1.2, < 3)
     faraday_middleware (0.10.0)
       faraday (>= 0.7.4, < 0.10)
+    faraday_middleware-aws-signers-v4 (0.1.5)
+      aws-sdk (~> 2.1)
+      faraday (~> 0.9)
     faraday_middleware-multi_json (0.0.6)
       faraday_middleware
       multi_json
@@ -393,6 +404,7 @@ GEM
     jira-ruby (1.1.2)
       activesupport
       oauth (~> 0.5, >= 0.5.0)
+    jmespath (1.3.1)
     jquery-atwho-rails (1.3.2)
     jquery-rails (4.1.1)
       rails-dom-testing (>= 1, < 3)
@@ -862,6 +874,7 @@ DEPENDENCIES
   asciidoctor-plantuml (= 0.0.7)
   attr_encrypted (~> 3.0.0)
   awesome_print (~> 1.2.0)
+  aws-sdk
   babosa (~> 1.0.2)
   base32 (~> 0.3.0)
   benchmark-ips (~> 2.3.0)
@@ -896,6 +909,7 @@ DEPENDENCIES
   email_reply_trimmer (~> 0.1)
   email_spec (~> 1.6.0)
   factory_girl_rails (~> 4.7.0)
+  faraday_middleware-aws-signers-v4
   ffaker (~> 2.4)
   flay (~> 2.6.1)
   fog-aws (~> 0.9)

app/controllers/admin/application_settings_controller.rb

@@ -161,9 +161,12 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
   def application_setting_params_ee
     [
       :help_text,
-      :elasticsearch_host,
+      :elasticsearch_url,
       :elasticsearch_indexing,
-      :elasticsearch_port,
+      :elasticsearch_aws,
+      :elasticsearch_aws_access_key,
+      :elasticsearch_aws_secret_access_key,
+      :elasticsearch_aws_region,
       :elasticsearch_search,
       :repository_size_limit,
       :shared_runners_minutes,

app/models/application_setting.rb

@@ -91,13 +91,13 @@ class ApplicationSetting < ActiveRecord::Base
             presence: true,
             numericality: { only_integer: true, greater_than: 0 }
 
-  validates :elasticsearch_host,
+  validates :elasticsearch_url,
             presence: { message: "can't be blank when indexing is enabled" },
             if: :elasticsearch_indexing?
 
-  validates :elasticsearch_port,
-            presence: { message: "can't be blank when indexing is enabled" },
-            if: :elasticsearch_indexing?
+  validates :elasticsearch_aws_region,
+            presence: { message: "can't be blank when using aws hosted elasticsearch" },
+            if: ->(setting) { setting.elasticsearch_indexing? && setting.elasticsearch_aws? }
 
   validates :repository_storages, presence: true
   validate :check_repository_storages
@@ -236,8 +236,9 @@ class ApplicationSetting < ActiveRecord::Base
 
   def self.defaults_ee
     {
-      elasticsearch_host: ENV['ELASTIC_HOST'] || 'localhost',
-      elasticsearch_port: ENV['ELASTIC_PORT'] || '9200',
+      elasticsearch_url: ENV['ELASTIC_URL'] || 'http://localhost:9200',
+      elasticsearch_aws: false,
+      elasticsearch_aws_region: ENV['ELASTIC_REGION'] || 'us-east-1',
       usage_ping_enabled: true,
       minimum_mirror_sync_time: Gitlab::Mirror::FIFTEEN
     }
@@ -268,8 +269,8 @@ class ApplicationSetting < ActiveRecord::Base
     Gitlab::Mirror.configure_cron_jobs!
   end
 
-  def elasticsearch_host
-    read_attribute(:elasticsearch_host).split(',').map(&:strip)
+  def elasticsearch_url
+    read_attribute(:elasticsearch_url).split(',').map(&:strip)
   end
 
   def home_page_url_column_exist

app/views/admin/application_settings/_form.html.haml

@@ -454,16 +454,41 @@
             Search with Elasticsearch enabled
 
     .form-group
-      = f.label :elasticsearch_host, 'Host', class: 'control-label col-sm-2'
+      = f.label :elasticsearch_url, 'URL', class: 'control-label col-sm-2'
       .col-sm-10
-        = f.text_field :elasticsearch_host, value: @application_setting.elasticsearch_host.join(', '), class: 'form-control', placeholder: 'localhost'
+        = f.text_field :elasticsearch_url, value: @application_setting.elasticsearch_url.join(', '), class: 'form-control', placeholder: 'http://localhost:9200'
         .help-block
-          The TCP/IP host to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "host1, host2").
+          The url to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://localhost:9200, http://localhost:9201").
+
+  %fieldset
+    %legend Elasticsearch AWS
+    .form-group
+      .col-sm-offset-2.col-sm-10
+        .checkbox
+          = f.label :elasticsearch_aws do
+            = f.check_box :elasticsearch_aws
+            Using AWS hosted Elasticsearch
+
+    .form-group
+      = f.label :elasticsearch_aws_region, 'AWS region', class: 'control-label col-sm-2'
+      .col-sm-10
+        = f.text_field :elasticsearch_aws_region, value: @application_setting.elasticsearch_aws_region, class: 'form-control'
+        .help-block
+          Region that elasticsearch is configured
 
     .form-group
-      = f.label :elasticsearch_port, 'Port', class: 'control-label col-sm-2'
+      = f.label :elasticsearch_aws_access_key, 'AWS Access Key', class: 'control-label col-sm-2'
       .col-sm-10
-        = f.text_field :elasticsearch_port, class: 'form-control', placeholder: ApplicationSetting.current.elasticsearch_port
+        = f.text_field :elasticsearch_aws_access_key, value: @application_setting.elasticsearch_aws_access_key, class: 'form-control'
+        .help-block
+          AWS Access Key.  Only required if not using role instance credentials
+
+    .form-group
+      = f.label :elasticsearch_aws_secret_access_key, 'AWS Secret Access Key', class: 'control-label col-sm-2'
+      .col-sm-10
+        = f.password_field :elasticsearch_aws_secret_access_key, value: @application_setting.elasticsearch_aws_secret_access_key, class: 'form-control'
+        .help-block
+          AWS Secret Access Key.  Only required if not using role instance credentials
 
   %fieldset
     %legend Koding

bin/elastic_repo_indexer

@@ -8,6 +8,8 @@ require 'active_support'
 require 'active_support/core_ext'
 require 'benchmark'
 
+load File.join(File.dirname(__FILE__), '..', '/lib/gitlab/elastic/client.rb')
+
 path_to_log_file = File.expand_path('../../log/es-indexer.log', __FILE__)
 LOGGER = Logger.new(path_to_log_file)
 
@@ -20,18 +22,25 @@ RAILS_ENV = ENV['RAILS_ENV']
 LOGGER.info("Has been scheduled for project #{REPO_PATH} with SHA range #{FROM_SHA}:#{TO_SHA}")
 
 elastic_connection_info = JSON.parse ENV['ELASTIC_CONNECTION_INFO']
-ELASTIC_HOST = elastic_connection_info['host']
-ELASTIC_PORT = elastic_connection_info['port']
+ELASTIC_URL = elastic_connection_info['url']
+ELASTIC_AWS = elastic_connection_info['aws']
+ELASTIC_AWS_ACCESS_KEY = elastic_connection_info['aws_access_key']
+ELASTIC_AWS_SECRET_ACCESS_KEY = elastic_connection_info['aws_secret_access_key']
+ELASTIC_AWS_REGION = elastic_connection_info['aws_region']
 
 class Repository
   include Elasticsearch::Git::Repository
+  include Gitlab::Elastic
 
   index_name ['gitlab', RAILS_ENV].compact.join('-')
 
-  self.__elasticsearch__.client = Elasticsearch::Client.new(
-    host: ELASTIC_HOST,
-    port: ELASTIC_PORT
-  )
+  def initialize
+    if ELASTIC_AWS
+      self.__elasticsearch__.client = AWSClient(ELASTIC_URL, ELASTIC_REGION, ELASTIC_AWS_ACCESS_KEY, ELASTIC_AWS_SECRET_ACCESS_KEY).client
+    else
+      self.__elasticsearch__.client = BaseClient.new(ELASTIC_URL).client
+    end
+  end
 
   def client_for_indexing
     self.__elasticsearch__.client

changelogs/unreleased-ee/1181-elasticsearch-client-params.yml

+---
+title: Expose elasticsearch client params for AWS signing and HTTPS
+merge_request: 1281
+author:

config/initializers/1_settings.rb

@@ -277,8 +277,7 @@ Settings.gitlab['no_todos_messages'] ||= YAML.load_file(Rails.root.join('config'
 #
 Settings['elasticsearch'] ||= Settingslogic.new({})
 Settings.elasticsearch['enabled'] = false if Settings.elasticsearch['enabled'].nil?
-Settings.elasticsearch['host'] ||=  ENV['ELASTIC_HOST'] || "localhost"
-Settings.elasticsearch['port'] ||= ENV['ELASTIC_PORT'] || 9200
+Settings.elasticsearch['url'] = ENV['ELASTIC_URL'] || "http://localhost:9200"
 
 #
 # CI

config/initializers/elastic_client_setup.rb

@@ -7,24 +7,28 @@ module Elasticsearch
     module Client
       module ClassMethods
         include Gitlab::CurrentSettings
-
+        include Gitlab::Elastic
         def client(client = nil)
           if @client.nil? || es_configuration_changed?
-            @es_host = current_application_settings.elasticsearch_host
-            @es_port = current_application_settings.elasticsearch_port
+            @es_url = current_application_settings.elasticsearch_url
+            @es_aws = current_application_settings.elasticsearch_aws
 
-            @client = Elasticsearch::Client.new(
-              host: current_application_settings.elasticsearch_host,
-              port: current_application_settings.elasticsearch_port
-            )
+            if @es_aws
+              # AWS specific handling
+              @es_region = current_application_settings.elasticsearch_aws_region
+              @es_access_key = current_application_settings.elasticsearch_aws_access_key
+              @es_secret_access_key = current_application_settings.elasticsearch_aws_secret_access_key
+              @client = AWSClient.new(@es_url, @es_region, @es_access_key, @es_secret_access_key).client
+            else
+              @client = BaseClient.new(@es_url).client
+            end
           end
-
           @client
         end
 
         def es_configuration_changed?
-          @es_host != current_application_settings.elasticsearch_host ||
-            @es_port != current_application_settings.elasticsearch_port
+          @es_url != current_application_settings.elasticsearch_url ||
+            @es_aws != current_application_settings.elasticsearch_aws
         end
       end
     end

db/migrate/20170215151539_add_aws_elasticsearch.rb

+class AddAwsElasticsearch < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :application_settings, :elasticsearch_url, :string, default: 'http://localhost:9200'
+    add_column :application_settings, :elasticsearch_aws, :boolean, default: false, null: false
+    add_column :application_settings, :elasticsearch_aws_region, :string, default: 'us-east-1'
+    add_column :application_settings, :elasticsearch_aws_access_key, :string
+    add_column :application_settings, :elasticsearch_aws_secret_access_key, :string
+  end
+end

db/migrate/20170215151540_migrate_old_elasticsearch_settings.rb

+class MigrateOldElasticsearchSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    settings = Arel::Table.new(:application_settings)
+
+    finder =
+      settings
+        .where(settings[:elasticsearch_host].not_eq(nil))
+        .project(:id, :elasticsearch_host, :elasticsearch_port)
+
+    result = connection.exec_query(finder.to_sql)
+
+    # There are only a few rows in the `application_settings` table
+    result.rows.each do |id, hosts, port|
+      # elasticsearch_host may look like "1.example.com,2.example.com, 3.example.com"
+      urls = hosts.split(',').map do |host|
+        url = URI.parse('http://' + host.strip)
+        url.port = port
+        url.to_s
+      end
+
+      updater =
+        Arel::UpdateManager.new(ActiveRecord::Base)
+          .table(settings)
+          .set(settings[:elasticsearch_url] => urls.join(','))
+          .where(settings[:id].eq(id))
+
+      connection.exec_update(updater.to_sql)
+    end
+  end
+
+  def down
+    # no-op
+  end
+end

db/migrate/20170215151541_remove_old_elasticsearch_settings.rb

+class RemoveOldElasticsearchSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = true
+
+  DOWNTIME_REASON = 'Removing two columns from application_settings'
+
+  def change
+    remove_column :application_settings, :elasticsearch_host, :string, default: 'localhost'
+    remove_column :application_settings, :elasticsearch_port, :string, default: '9200'
+  end
+end

db/schema.rb

@@ -93,8 +93,6 @@ ActiveRecord::Schema.define(version: 20170217151947) do
     t.boolean "user_default_external", default: false, null: false
     t.boolean "elasticsearch_indexing", default: false, null: false
     t.boolean "elasticsearch_search", default: false, null: false
-    t.string "elasticsearch_host", default: "localhost"
-    t.string "elasticsearch_port", default: "9200"
     t.string "repository_storages", default: "default"
     t.string "enabled_git_access_protocol"
     t.boolean "domain_blacklist_enabled", default: false
@@ -121,7 +119,12 @@ ActiveRecord::Schema.define(version: 20170217151947) do
     t.integer "repository_size_limit", limit: 8, default: 0
     t.integer "terminal_max_session_time", default: 0, null: false
     t.integer "minimum_mirror_sync_time", default: 15, null: false
-    t.string "default_artifacts_expire_in", default: '0', null: false
+    t.string "default_artifacts_expire_in", default: "0", null: false
+    t.string "elasticsearch_url", default: "http://localhost:9200"
+    t.boolean "elasticsearch_aws", default: false, null: false
+    t.string "elasticsearch_aws_region", default: "us-east-1"
+    t.string "elasticsearch_aws_access_key"
+    t.string "elasticsearch_aws_secret_access_key"
   end
 
   create_table "approvals", force: :cascade do |t|

doc/api/settings.md

@@ -86,8 +86,11 @@ PUT /application/settings
 | `help_text` | string | no | GitLab server administrator information |
 | `elasticsearch_indexing` | boolean | no | Enable Elasticsearch indexing |
 | `elasticsearch_search` | boolean | no | Enable Elasticsearch search |
-| `elasticsearch_host` | string | no | The TCP/IP host to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "host1, host2") |
-| `elasticsearch_port` | integer | no | The TCP/IP port that Elasticsearch listens to. The default value is 9200 |
+| `elasticsearch_url` | string | no | The url to use for connecting to Elasticsearch. Use a comma-separated list to support cluster (e.g., "http://localhost:9200, http://localhost:9201") |
+| `elasticsearch_aws` | boolean | no | Enable the use of AWS hosted Elasticsearch |
+| `elasticsearch_aws_region` | string | no | The AWS region the elasticsearch domain is configured |
+| `elasticsearch_aws_access_key` | string | no | AWS IAM access key |
+| `elasticsearch_aws_secret_access_key` | string | no | AWS IAM secret access key |
 | `usage_ping_enabled` | boolean | no | Every week GitLab will report license usage back to GitLab, Inc.|
 | `repository_size_limit` | integer | no | Size limit per repository (MB) |
 | `plantuml_enabled` | boolean | no | Enable PlantUML integration. Default is `false`. |

lib/api/settings.rb

@@ -115,8 +115,13 @@ module API
       optional :elasticsearch_indexing, type: Boolean, desc: 'Enable Elasticsearch indexing'
       given elasticsearch_indexing: ->(val) { val } do
         optional :elasticsearch_search, type: Boolean, desc: 'Enable Elasticsearch search'
-        requires :elasticsearch_host, type: String, desc: 'The TCP/IP host to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "host1, host2")'
-        requires :elasticsearch_port, type: Integer, desc: 'The TCP/IP port that Elasticsearch listens to. The default value is 9200'
+        requires :elasticsearch_url, type: String, desc: 'The url to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://localhost:9200, http://localhost:9201")'
+      end
+      optional :elasticsearch_aws, type: Boolean, desc: 'Enable support for AWS hosted elasticsearch'
+      given elasticsearch_aws: ->(val) { val } do
+        requires :elasticsearch_aws_region, type: String, desc: 'The AWS region the elasticsearch domain is configured'
+        optional :elasticsearch_aws_access_key, type: String, desc: 'AWS IAM access key'
+        optional :elasticsearch_aws_secret_access_key, type: String, desc: 'AWS IAM secret access key'
       end
       optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.'
       optional :repository_storages, type: Array[String], desc: 'A list of names of enabled storage paths, taken from `gitlab.yml`. New projects will be created in one of these stores, chosen at random.'

lib/api/v3/settings.rb

@@ -115,8 +115,7 @@ module API
         optional :elasticsearch_indexing, type: Boolean, desc: 'Enable Elasticsearch indexing'
         given elasticsearch_indexing: ->(val) { val } do
           optional :elasticsearch_search, type: Boolean, desc: 'Enable Elasticsearch search'
-          requires :elasticsearch_host, type: String, desc: 'The TCP/IP host to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "host1, host2")'
-          requires :elasticsearch_port, type: Integer, desc: 'The TCP/IP port that Elasticsearch listens to. The default value is 9200'
+          requires :elasticsearch_url, type: String, desc: 'The url to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://localhost:9200, http://localhost:9201")'
         end
         optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.'
         optional :repository_storage, type: String, desc: 'The first entry in `repository_storages`. Deprecated, but retained for compatibility reasons'

lib/gitlab/elastic/client.rb

+require 'elasticsearch'
+require 'aws-sdk'
+require 'faraday_middleware/aws_signers_v4'
+
+module Gitlab
+  module Elastic
+    class BaseClient
+      attr_accessor :client
+
+      def initialize(urls)
+        @urls = urls
+        @client = Elasticsearch::Client.new urls: @urls
+      end
+    end
+  end
+end
+
+module Gitlab
+  module Elastic
+    class AWSClient < BaseClient
+      def initialize(urls, region, access_key = nil, secret_access_key = nil)
+        @urls = urls
+        @region = region
+        @access_key = access_key
+        @secret_access_key = secret_access_key
+
+        if @access_key.nil? || @secret_access_key.nil?
+          @credentials = Aws::Credentials.new()
+        else
+          @credentials = Aws::Credentials.new(@access_key, @secret_access_key)
+        end
+
+        @client = Elasticsearch::Client.new urls: @urls do |fmid|
+          fmid.request :aws_signers_v4,
+            credentials: @credentials,
+            service_name: 'es',
+            region: @region
+        end
+      end
+    end
+  end
+end

lib/gitlab/elastic/indexer.rb

@@ -14,8 +14,11 @@ module Gitlab
         @project = project
 
         connection_info = {
-          host: current_application_settings.elasticsearch_host,
-          port: current_application_settings.elasticsearch_port
+          url: current_application_settings.elasticsearch_url,
+          aws: current_application_settings.elasticsearch_aws,
+          aws_access_key: current_application_settings.elasticsearch_aws_access_key,
+          aws_secret_access_key: current_application_settings.elasticsearch_aws_secret_access_key,
+          aws_region: current_application_settings.elasticsearch_aws_region
         }.to_json
 
         # We accept any form of settings, including string and array

lib/tasks/gitlab/check.rake

@@ -1072,8 +1072,16 @@ namespace :gitlab do
   end
 
   def check_elasticsearch
-    client = Elasticsearch::Client.new(host: ApplicationSetting.current.elasticsearch_host,
-                                       port: ApplicationSetting.current.elasticsearch_port)
+    if ApplicationSetting.current.elasticsearch_aws
+      client = Gitlab::Elastic::AWSClient.new(
+        ApplicationSetting.current.elasticsearch_url,
+        ApplicationSetting.current.elasticsearch_aws_region,
+        ApplicationSetting.current.elasticsearch_aws_access_key,
+        ApplicationSetting.current.elasticsearch_aws_secret_access_key
+      ).client
+    else
+      client = Gitlab::Elastic::BaseClient.new(ApplicationSetting.current.elasticsearch_url).client
+    end
 
     print "Elasticsearch version 5.1.x? ... "
 

spec/lib/gitlab/elastic/indexer_spec.rb

@@ -5,7 +5,7 @@ describe Gitlab::Elastic::Indexer do
 
   before do
     stub_env('IN_MEMORY_APPLICATION_SETTINGS', 'true')
-    stub_application_setting(es_host: ['elastic-host1', 'elastic-host2'])
+    stub_application_setting(es_url: ['http://localhost:9200', 'http://localhost:9201'])
   end
 
   let(:project)  { create(:project) }
@@ -18,8 +18,7 @@ describe Gitlab::Elastic::Indexer do
 
   let(:elastic_connection_info) do
     {
-      host: current_application_settings.elasticsearch_host,
-      port: current_application_settings.elasticsearch_port,
+      url: current_application_settings.elasticsearch_url
     }
   end
 

spec/workers/elastic_indexer_worker_spec.rb

@@ -6,8 +6,7 @@ describe ElasticIndexerWorker, elastic: true do
 
   before do
     Elasticsearch::Model.client = Elasticsearch::Client.new(
-      host: current_application_settings.elasticsearch_host,
-      port: current_application_settings.elasticsearch_port
+      url: current_application_settings.elasticsearch_url
     )
 
     Gitlab::Elastic::Helper.create_empty_index