Merge branch '195788-community-contribution-es-ecs' into 'master'

Support AWS IAM role for ECS tasks in elastic search client

Closes #195788

See merge request gitlab-org/gitlab!33456

ee/changelogs/unreleased/195788-community-contribution-es-ecs.yml

+---
+title: Support AWS IAM role for ECS tasks in Elasticsearch client
+merge_request: 33456
+author:
+type: added

ee/lib/gitlab/elastic/client.rb

@@ -32,10 +32,12 @@ module Gitlab
 
         return static_credentials if static_credentials&.set?
 
-        # Instantiating this will perform an API call, so only do so if the
-        # static credentials did not work
-        instance_credentials = Aws::InstanceProfileCredentials.new
-
+        # When static credentials are not configured, Aws::CredentialProviderChain API
+        # will be used to retrieve credentials. It will check AWS access credential environment
+        # variables, AWS credential profile, ECS credential service and EC2 credential service.
+        # Please see aws-sdk-core/lib/aws-sdk-core/credential_provider_chain.rb for details of
+        # the possible providers and order of the providers.
+        instance_credentials = Aws::CredentialProviderChain.new.resolve
         instance_credentials if instance_credentials&.set?
       end
     end

ee/lib/gitlab/elastic/indexer.rb

@@ -109,8 +109,9 @@ module Gitlab
           'SSL_CERT_DIR'            => OpenSSL::X509::DEFAULT_CERT_DIR
         }
 
-        # Users can override default SSL certificate path via these envs
-        %w(SSL_CERT_FILE SSL_CERT_DIR).each_with_object(vars) do |key, hash|
+        # Users can override default SSL certificate path via SSL_CERT_FILE SSL_CERT_DIR
+        # AWS_CONTAINER_CREDENTIALS_RELATIVE_URI is used in AWS ECS to get credentials when making AWS API calls
+        %w(SSL_CERT_FILE SSL_CERT_DIR AWS_CONTAINER_CREDENTIALS_RELATIVE_URI).each_with_object(vars) do |key, hash|
           hash[key] = ENV[key] if ENV.key?(key)
         end
       end

ee/spec/lib/gitlab/elastic/client_spec.rb

@@ -3,33 +3,6 @@
 require 'spec_helper'
 
 RSpec.describe Gitlab::Elastic::Client do
-  let(:creds_valid_response) do
-    '{
-      "Code": "Success",
-      "Type": "AWS-HMAC",
-      "AccessKeyId": "0",
-      "SecretAccessKey": "0",
-      "Token": "token",
-      "Expiration": "2018-12-16T01:51:37Z",
-      "LastUpdated": "2009-11-23T0:00:00Z"
-    }'
-  end
-
-  let(:creds_fail_response) do
-    '{
-      "Code": "ErrorCode",
-      "Message": "ErrorMsg",
-      "LastUpdated": "2009-11-23T0:00:00Z"
-    }'
-  end
-
-  def stub_instance_credentials(creds_response)
-    stub_request(:get, "http://169.254.169.254/latest/meta-data/iam/security-credentials/")
-      .to_return(status: 200, body: "RoleName", headers: {})
-    stub_request(:get, "http://169.254.169.254/latest/meta-data/iam/security-credentials/RoleName")
-      .to_return(status: 200, body: creds_response, headers: {})
-  end
-
   describe '.build' do
     let(:client) { described_class.build(params) }
 
@@ -60,7 +33,6 @@ RSpec.describe Gitlab::Elastic::Client do
         # Mock the correlation ID (passed as header) to have deterministic signature
         allow(Labkit::Correlation::CorrelationId).to receive(:current_or_new_id).and_return('new-correlation-id')
 
-        stub_instance_credentials(creds_fail_response)
         travel_to(Time.parse('20170303T133952Z')) do
           stub_request(:get, 'http://example-elastic:9200/foo/_all/1')
             .with(
@@ -99,7 +71,6 @@ RSpec.describe Gitlab::Elastic::Client do
     end
 
     context 'when the AWS IAM static credentials are invalid' do
-      context 'with AWS ec2 instance profile' do
       let(:params) do
         {
           url: 'http://example-elastic:9200',
@@ -107,27 +78,30 @@ RSpec.describe Gitlab::Elastic::Client do
           aws_region: 'us-east-1'
         }
       end
+      let(:credentials) { double(:aws_credentials, set?: true) }
 
-        it 'returns credentials from ec2 instance profile' do
-          stub_instance_credentials(creds_valid_response)
-
-          expect(creds.credentials.access_key_id).to eq '0'
-          expect(creds.credentials.secret_access_key).to eq '0'
+      before do
+        allow_next_instance_of(Aws::CredentialProviderChain) do |instance|
+          allow(instance).to receive(:resolve).and_return(credentials)
         end
       end
 
-      context 'with AWS no credentials' do
-        let(:params) do
-          {
-            url: 'http://example-elastic:9200',
-            aws: true,
-            aws_region: 'us-east-1'
-          }
+      it 'returns credentials from Aws::CredentialProviderChain' do
+        expect(creds).to eq credentials
       end
 
+      context 'when Aws::CredentialProviderChain returns unset credentials' do
+        let(:credentials) { double(:aws_credentials, set?: false) }
+
         it 'returns nil' do
-          stub_instance_credentials(creds_fail_response)
+          expect(creds).to be_nil
+        end
+      end
 
+      context 'when Aws::CredentialProviderChain returns nil' do
+        let(:credentials) { nil }
+
+        it 'returns nil' do
           expect(creds).to be_nil
         end
       end

ee/spec/lib/gitlab/elastic/indexer_spec.rb

@@ -317,6 +317,22 @@ RSpec.describe Gitlab::Elastic::Indexer do
     end
   end
 
+  context 'when AWS_CONTAINER_CREDENTIALS_RELATIVE_URI is set' do
+    let(:aws_cred_relative_uri) { '/ecs/relative/cred/uri'}
+
+    before do
+      stub_env('AWS_CONTAINER_CREDENTIALS_RELATIVE_URI', aws_cred_relative_uri)
+    end
+
+    context 'when building env vars for child process' do
+      subject { envvars }
+
+      it 'AWS_CONTAINER_CREDENTIALS_RELATIVE_URI env vars will be included' do
+        expect(subject).to include('AWS_CONTAINER_CREDENTIALS_RELATIVE_URI' => aws_cred_relative_uri)
+      end
+    end
+  end
+
   def expect_popen
     expect(Gitlab::Popen).to receive(:popen)
   end