Merge branch...

Merge branch '333562-setup-gitlab-mailgun-endpoint-for-syncing-bounced-invite-emails-2' into 'master'

Setup GitLab MailGun endpoint for permanent failure on invite emails

See merge request gitlab-org/gitlab!65078

app/controllers/members/mailgun/permanent_failures_controller.rb

+# frozen_string_literal: true
+
+module Members
+  module Mailgun
+    class PermanentFailuresController < ApplicationController
+      respond_to :json
+
+      skip_before_action :authenticate_user!
+      skip_before_action :verify_authenticity_token
+
+      before_action :ensure_feature_enabled!
+      before_action :authenticate_signature!
+      before_action :validate_invite_email!
+
+      feature_category :authentication_and_authorization
+
+      def create
+        webhook_processor.execute
+
+        head :ok
+      end
+
+      private
+
+      def ensure_feature_enabled!
+        render_406 unless Gitlab::CurrentSettings.mailgun_events_enabled?
+      end
+
+      def authenticate_signature!
+        access_denied! unless valid_signature?
+      end
+
+      def valid_signature?
+        return false if Gitlab::CurrentSettings.mailgun_signing_key.blank?
+
+        # per this guide: https://documentation.mailgun.com/en/latest/user_manual.html#webhooks
+        digest = OpenSSL::Digest.new('SHA256')
+        data = [params.dig(:signature, :timestamp), params.dig(:signature, :token)].join
+
+        hmac_digest = OpenSSL::HMAC.hexdigest(digest, Gitlab::CurrentSettings.mailgun_signing_key, data)
+
+        ActiveSupport::SecurityUtils.secure_compare(params.dig(:signature, :signature), hmac_digest)
+      end
+
+      def validate_invite_email!
+        # permanent_failures webhook does not provide a way to filter failures, so we'll get them all on this endpoint
+        # and we only care about our invite_emails
+        render_406 unless payload[:tags]&.include?(::Members::Mailgun::INVITE_EMAIL_TAG)
+      end
+
+      def webhook_processor
+        ::Members::Mailgun::ProcessWebhookService.new(payload)
+      end
+
+      def payload
+        @payload ||= params.permit!['event-data']
+      end
+
+      def render_406
+        # failure to stop retries per https://documentation.mailgun.com/en/latest/user_manual.html#webhooks
+        head :not_acceptable
+      end
+    end
+  end
+end

app/mailers/emails/members.rb

@@ -150,10 +150,10 @@ module Emails
     end
 
     def invite_email_headers
-      if Gitlab.dev_env_or_com?
+      if Gitlab::CurrentSettings.mailgun_events_enabled?
         {
-          'X-Mailgun-Tag' => 'invite_email',
-          'X-Mailgun-Variables' => { 'invite_token' => @token }.to_json
+          'X-Mailgun-Tag' => ::Members::Mailgun::INVITE_EMAIL_TAG,
+          'X-Mailgun-Variables' => { ::Members::Mailgun::INVITE_EMAIL_TOKEN_KEY => @token }.to_json
         }
       else
         {}

app/services/members/mailgun.rb

+# frozen_string_literal: true
+
+module Members
+  module Mailgun
+    INVITE_EMAIL_TAG = 'invite_email'
+    INVITE_EMAIL_TOKEN_KEY = :invite_token
+  end
+end

app/services/members/mailgun/process_webhook_service.rb

+# frozen_string_literal: true
+
+module Members
+  module Mailgun
+    class ProcessWebhookService
+      ProcessWebhookServiceError = Class.new(StandardError)
+
+      def initialize(payload)
+        @payload = payload
+      end
+
+      def execute
+        @member = Member.find_by_invite_token(invite_token)
+        update_member_and_log if member
+      rescue ProcessWebhookServiceError => e
+        Gitlab::ErrorTracking.track_exception(e)
+      end
+
+      private
+
+      attr_reader :payload, :member
+
+      def update_member_and_log
+        log_update_event if member.update(invite_email_success: false)
+      end
+
+      def log_update_event
+        Gitlab::AppLogger.info "UPDATED MEMBER INVITE_EMAIL_SUCCESS: member_id: #{member.id}"
+      end
+
+      def invite_token
+        # may want to validate schema in some way using ::JSONSchemer.schema(SCHEMA_PATH).valid?(message) if this
+        # gets more complex
+        payload.dig('user-variables', ::Members::Mailgun::INVITE_EMAIL_TOKEN_KEY) ||
+          raise(ProcessWebhookServiceError, "Failed to receive #{::Members::Mailgun::INVITE_EMAIL_TOKEN_KEY} in user-variables: #{payload}")
+      end
+    end
+  end
+end

app/views/admin/application_settings/_mailgun.html.haml

-- return unless Feature.enabled?(:mailgun_events_receiver)
-
 - expanded = integration_expanded?('mailgun_')
 %section.settings.as-mailgun.no-animate#js-mailgun-settings{ class: ('expanded' if expanded) }
   .settings-header

config/feature_flags/development/mailgun_events_receiver.yml

----
-name: mailgun_events_receiver
-introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/64249
-rollout_issue_url:
-milestone: '14.1'
-type: development
-group: group::expansion
-default_enabled: false

config/routes.rb

@@ -222,6 +222,7 @@ Rails.application.routes.draw do
 
     draw :snippets
     draw :profile
+    draw :members
 
     # Product analytics collector
     match '/collector/i', to: ProductAnalytics::CollectorApp.new, via: :all

config/routes/members.rb

+# frozen_string_literal: true
+
+namespace :members do
+  namespace :mailgun do
+    resources :permanent_failures, only: [:create]
+  end
+end

db/migrate/20210630144339_add_invite_email_success_to_members.rb

+# frozen_string_literal: true
+
+class AddInviteEmailSuccessToMembers < ActiveRecord::Migration[6.1]
+  def change
+    add_column :members, :invite_email_success, :boolean, null: false, default: true
+  end
+end

db/schema_migrations/20210630144339

+8d1777941e1a4b5f9f8f5f5e3ae416d6d02aaee1174eff1f9b4b38a6cdf0103a
\ No newline at end of file

db/structure.sql

@@ -14662,7 +14662,8 @@ CREATE TABLE members (
     requested_at timestamp without time zone,
     expires_at date,
     ldap boolean DEFAULT false NOT NULL,
-    override boolean DEFAULT false NOT NULL
+    override boolean DEFAULT false NOT NULL,
+    invite_email_success boolean DEFAULT true NOT NULL
 );
 
 CREATE SEQUENCE members_id_seq

doc/administration/integration/mailgun.md

+---
+stage: Growth
+group: Expansion
+info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
+type: reference, howto
+---
+
+# Mailgun and GitLab **(FREE SELF)**
+
+When you use Mailgun to send emails for your GitLab instance and [Mailgun](https://www.mailgun.com/)
+integration is enabled and configured in GitLab, you can receive their webhook for
+permanent invite email failures. To set up the integration, you must:
+
+1. [Configure your Mailgun domain](#configure-your-mailgun-domain).
+1. [Enable Mailgun integration](#enable-mailgun-integration).
+
+After completing the integration, Mailgun `permanent_failure` webhooks are sent to your GitLab instance.
+
+## Configure your Mailgun domain
+
+Before you can enable Mailgun in GitLab, set up your own Mailgun permanent failure endpoint to receive the webhooks.
+
+Using the [Mailgun webhook guide](https://www.mailgun.com/blog/a-guide-to-using-mailguns-webhooks/):
+
+1. Add a webhook with the **Event type** set to **Permanent Failure**.
+1. Fill in the URL of your instance and include the `/-/members/mailgun/permanent_failures` path.
+   - Example: `https://myinstance.gitlab.com/-/members/mailgun/permanent_failures`
+
+## Enable Mailgun integration
+
+After configuring your Mailgun domain for the permanent failures endpoint,
+you're ready to enable the Mailgun integration:
+
+1. Sign in to GitLab as an [Administrator](../../user/permissions.md) user.
+1. On the top bar, select **Menu >** **{admin}** **Admin**.
+1. In the left sidebar, go to **Settings > General** and expand the **Mailgun** section.
+1. Select the **Enable Mailgun** check box.
+1. Enter the Mailgun HTTP webhook signing key as described in
+   [the Mailgun documentation](https://documentation.mailgun.com/en/latest/user_manual.html#webhooks) and
+   shown in the [API security](https://app.mailgun.com/app/account/security/api_keys) section for your Mailgun account.
+1. Select **Save changes**.

doc/api/settings.md

@@ -328,7 +328,7 @@ listed in the descriptions of the relevant settings.
 | `issues_create_limit`                    | integer          | no                                   | Max number of issue creation requests per minute per user. Disabled by default.|
 | `keep_latest_artifact`                   | boolean          | no                                   | Prevent the deletion of the artifacts from the most recent successful jobs, regardless of the expiry time. Enabled by default. |
 | `local_markdown_version`                 | integer          | no                                   | Increase this value when any cached Markdown should be invalidated. |
-| `mailgun_signing_key`                    | string           | no                                   | The Mailgun HTTP webhook signing key for receiving events from webhook |
+| `mailgun_signing_key`                    | string           | no                                   | The Mailgun HTTP webhook signing key for receiving events from webhook. |
 | `mailgun_events_enabled`                 | boolean          | no                                   | Enable Mailgun event receiver. |
 | `maintenance_mode_message`               | string           | no                                   | **(PREMIUM)** Message displayed when instance is in maintenance mode. |
 | `maintenance_mode`                       | boolean          | no                                   | **(PREMIUM)** When instance is in maintenance mode, non-administrative users can sign in with read-only access and make read-only API requests. |

doc/user/admin_area/settings/index.md

@@ -39,6 +39,7 @@ To access the default page for Admin Area settings:
 | ------ | ----------- |
 | [Elasticsearch](../../../integration/elasticsearch.md#enabling-advanced-search) | Elasticsearch integration. Elasticsearch AWS IAM. |
 | [Kroki](../../../administration/integration/kroki.md#enable-kroki-in-gitlab) | Allow rendering of diagrams in AsciiDoc and Markdown documents using [kroki.io](https://kroki.io). |
+| [Mailgun](../../../administration/integration/mailgun.md) | Enable your GitLab instance to receive invite email bounce events from Mailgun, if it is your email provider. |
 | [PlantUML](../../../administration/integration/plantuml.md) | Allow rendering of PlantUML diagrams in documents. |
 | [Slack application](../../../user/project/integrations/gitlab_slack_application.md#configuration) **(FREE SAAS)** | Slack integration allows you to interact with GitLab via slash commands in a chat window. This option is only available on GitLab.com, though it may be [available for self-managed instances in the future](https://gitlab.com/gitlab-org/gitlab/-/issues/28164). |
 | [Third party offers](third_party_offers.md) | Control the display of third party offers. |

spec/features/admin/admin_settings_spec.rb

@@ -269,10 +269,7 @@ RSpec.describe 'Admin updates settings' do
     end
 
     context 'Integrations page' do
-      let(:mailgun_events_receiver_enabled) { true }
-
       before do
-        stub_feature_flags(mailgun_events_receiver: mailgun_events_receiver_enabled)
         visit general_admin_application_settings_path
       end
 
@@ -286,26 +283,16 @@ RSpec.describe 'Admin updates settings' do
         expect(current_settings.hide_third_party_offers).to be true
       end
 
-      context 'when mailgun_events_receiver feature flag is enabled' do
-        it 'enabling Mailgun events', :aggregate_failures do
-          page.within('.as-mailgun') do
-            check 'Enable Mailgun event receiver'
-            fill_in 'Mailgun HTTP webhook signing key', with: 'MAILGUN_SIGNING_KEY'
-            click_button 'Save changes'
-          end
-
-          expect(page).to have_content 'Application settings saved successfully'
-          expect(current_settings.mailgun_events_enabled).to be true
-          expect(current_settings.mailgun_signing_key).to eq 'MAILGUN_SIGNING_KEY'
+      it 'enabling Mailgun events', :aggregate_failures do
+        page.within('.as-mailgun') do
+          check 'Enable Mailgun event receiver'
+          fill_in 'Mailgun HTTP webhook signing key', with: 'MAILGUN_SIGNING_KEY'
+          click_button 'Save changes'
         end
-      end
-
-      context 'when mailgun_events_receiver feature flag is disabled' do
-        let(:mailgun_events_receiver_enabled) { false }
 
-        it 'does not have mailgun' do
-          expect(page).not_to have_selector('.as-mailgun')
-        end
+        expect(page).to have_content 'Application settings saved successfully'
+        expect(current_settings.mailgun_events_enabled).to be true
+        expect(current_settings.mailgun_signing_key).to eq 'MAILGUN_SIGNING_KEY'
       end
     end
 

spec/lib/gitlab/import_export/safe_model_attributes.yml

@@ -167,6 +167,7 @@ ProjectMember:
 - expires_at
 - ldap
 - override
+- invite_email_success
 User:
 - id
 - username

spec/mailers/notify_spec.rb

@@ -827,15 +827,15 @@ RSpec.describe Notify do
         end
       end
 
-      context 'when on gitlab.com' do
+      context 'when mailgun events are enabled' do
         before do
-          allow(Gitlab).to receive(:dev_env_or_com?).and_return(true)
+          stub_application_setting(mailgun_events_enabled: true)
         end
 
         it 'has custom headers' do
           aggregate_failures do
-            expect(subject).to have_header('X-Mailgun-Tag', 'invite_email')
-            expect(subject).to have_header('X-Mailgun-Variables', { 'invite_token' => project_member.invite_token }.to_json)
+            expect(subject).to have_header('X-Mailgun-Tag', ::Members::Mailgun::INVITE_EMAIL_TAG)
+            expect(subject).to have_header('X-Mailgun-Variables', { ::Members::Mailgun::INVITE_EMAIL_TOKEN_KEY => project_member.invite_token }.to_json)
           end
         end
       end

spec/requests/members/mailgun/permanent_failure_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe 'receive a permanent failure' do
+  describe 'POST /members/mailgun/permanent_failures', :aggregate_failures do
+    let_it_be(:member) { create(:project_member, :invited) }
+
+    let(:raw_invite_token) { member.raw_invite_token }
+    let(:mailgun_events) { true }
+    let(:mailgun_signing_key) { 'abc123' }
+
+    subject(:post_request) { post members_mailgun_permanent_failures_path(standard_params) }
+
+    before do
+      stub_application_setting(mailgun_events_enabled: mailgun_events, mailgun_signing_key: mailgun_signing_key)
+    end
+
+    it 'marks the member invite email success as false' do
+      expect { post_request }.to change { member.reload.invite_email_success }.from(true).to(false)
+
+      expect(response).to have_gitlab_http_status(:ok)
+    end
+
+    context 'when the change to a member is not made' do
+      context 'with incorrect signing key' do
+        context 'with incorrect signing key' do
+          let(:mailgun_signing_key) { '_foobar_' }
+
+          it 'does not change member status and responds as not_found' do
+            expect { post_request }.not_to change { member.reload.invite_email_success }
+
+            expect(response).to have_gitlab_http_status(:not_found)
+          end
+        end
+
+        context 'with nil signing key' do
+          let(:mailgun_signing_key) { nil }
+
+          it 'does not change member status and responds as not_found' do
+            expect { post_request }.not_to change { member.reload.invite_email_success }
+
+            expect(response).to have_gitlab_http_status(:not_found)
+          end
+        end
+      end
+
+      context 'when the feature is not enabled' do
+        let(:mailgun_events) { false }
+
+        it 'does not change member status and responds as expected' do
+          expect { post_request }.not_to change { member.reload.invite_email_success }
+
+          expect(response).to have_gitlab_http_status(:not_acceptable)
+        end
+      end
+
+      context 'when it is not an invite email' do
+        before do
+          stub_const('::Members::Mailgun::INVITE_EMAIL_TAG', '_foobar_')
+        end
+
+        it 'does not change member status and responds as expected' do
+          expect { post_request }.not_to change { member.reload.invite_email_success }
+
+          expect(response).to have_gitlab_http_status(:not_acceptable)
+        end
+      end
+    end
+
+    def standard_params
+      {
+        "signature": {
+          "timestamp": "1625056677",
+          "token": "eb944d0ace7227667a1b97d2d07276ae51d2b849ed2cfa68f3",
+          "signature": "9790cc6686eb70f0b1f869180d906870cdfd496d27fee81da0aa86b9e539e790"
+        },
+        "event-data": {
+          "severity": "permanent",
+          "tags": ["invite_email"],
+          "timestamp": 1521233195.375624,
+          "storage": {
+            "url": "_anything_",
+            "key": "_anything_"
+          },
+          "log-level": "error",
+          "id": "_anything_",
+          "campaigns": [],
+          "reason": "suppress-bounce",
+          "user-variables": {
+            "invite_token": raw_invite_token
+          },
+          "flags": {
+            "is-routed": false,
+            "is-authenticated": true,
+            "is-system-test": false,
+            "is-test-mode": false
+          },
+          "recipient-domain": "example.com",
+          "envelope": {
+            "sender": "bob@mg.gitlab.com",
+            "transport": "smtp",
+            "targets": "alice@example.com"
+          },
+          "message": {
+            "headers": {
+              "to": "Alice <alice@example.com>",
+              "message-id": "20130503192659.13651.20287@mg.gitlab.com",
+              "from": "Bob <bob@mg.gitlab.com>",
+              "subject": "Test permanent_fail webhook"
+            },
+            "attachments": [],
+            "size": 111
+          },
+          "recipient": "alice@example.com",
+          "event": "failed",
+          "delivery-status": {
+            "attempt-no": 1,
+            "message": "",
+            "code": 605,
+            "description": "Not delivering to previously bounced address",
+            "session-seconds": 0
+          }
+        }
+      }
+    end
+  end
+end

spec/services/members/mailgun/process_webhook_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Members::Mailgun::ProcessWebhookService do
+  describe '#execute', :aggregate_failures do
+    let_it_be(:member) { create(:project_member, :invited) }
+
+    let(:raw_invite_token) { member.raw_invite_token }
+    let(:payload) { { 'user-variables' => { ::Members::Mailgun::INVITE_EMAIL_TOKEN_KEY => raw_invite_token } } }
+
+    subject(:service) { described_class.new(payload).execute }
+
+    it 'marks the member invite email success as false' do
+      expect(Gitlab::AppLogger).to receive(:info).with(/^UPDATED MEMBER INVITE_EMAIL_SUCCESS/).and_call_original
+
+      expect { service }.to change { member.reload.invite_email_success }.from(true).to(false)
+    end
+
+    context 'when member can not be found' do
+      let(:raw_invite_token) { '_foobar_' }
+
+      it 'does not change member status' do
+        expect(Gitlab::AppLogger).not_to receive(:info).with(/^UPDATED MEMBER INVITE_EMAIL_SUCCESS/)
+
+        expect { service }.not_to change { member.reload.invite_email_success }
+      end
+    end
+
+    context 'when invite token is not found in payload' do
+      let(:payload) { {} }
+
+      it 'does not change member status and logs an error' do
+        expect(Gitlab::AppLogger).not_to receive(:info).with(/^UPDATED MEMBER INVITE_EMAIL_SUCCESS/)
+        expect(Gitlab::ErrorTracking).to receive(:track_exception).with(
+          an_instance_of(described_class::ProcessWebhookServiceError))
+
+        expect { service }.not_to change { member.reload.invite_email_success }
+      end
+    end
+  end
+end