Make Reply by email easier to configure

.gitignore

@@ -25,7 +25,6 @@ config/initializers/rack_attack.rb
 config/initializers/smtp_settings.rb
 config/resque.yml
 config/unicorn.rb
-config/mail_room.yml
 config/secrets.yml
 coverage/*
 db/*.sqlite3

Gemfile

@@ -290,7 +290,7 @@ gem 'newrelic-grape'
 
 gem 'octokit', '~> 3.7.0'
 
-gem "mail_room", "~> 0.6.0"
+gem "mail_room", "~> 0.6.1"
 
 gem 'email_reply_parser', '~> 0.5.8'
 

Gemfile.lock

@@ -392,7 +392,7 @@ GEM
       systemu (~> 2.6.2)
     mail (2.6.3)
       mime-types (>= 1.16, < 3)
-    mail_room (0.6.0)
+    mail_room (0.6.1)
     method_source (0.8.2)
     mime-types (1.25.1)
     mimemagic (0.3.0)
@@ -854,7 +854,7 @@ DEPENDENCIES
   jquery-ui-rails (~> 4.2.1)
   kaminari (~> 0.16.3)
   letter_opener (~> 1.1.2)
-  mail_room (~> 0.6.0)
+  mail_room (~> 0.6.1)
   minitest (~> 5.7.0)
   mousetrap-rails (~> 1.4.6)
   mysql2 (~> 0.3.16)

config/gitlab.yml.example

@@ -99,7 +99,29 @@ production: &base
   # For documentation on how to set this up, see http://doc.gitlab.com/ce/incoming_email/README.html
   incoming_email:
     enabled: false
-    address: "incoming+%{key}@gitlab.example.com"
+
+    # The email address including the `%{key}` placeholder that will be replaced to reference the item being replied to.
+    # The `%{key}` placeholder is added after the user part, after a `+` character, before the `@`.
+    address: "gitlab-incoming+%{key}@gmail.com"
+
+    # Email account username
+    # With third party providers, this is usually the full email address.
+    # With self-hosted email servers, this is usually the user part of the email address.
+    user: "gitlab-incoming@gmail.com"
+    # Email account password
+    password: "[REDACTED]"
+
+    # IMAP server host
+    host: "imap.gmail.com"
+    # IMAP server port
+    port: 993
+    # Whether the IMAP server uses SSL
+    ssl: true
+    # Whether the IMAP server uses StartTLS
+    start_tls: false
+
+    # The mailbox where incoming mail will end up. Usually "inbox".
+    mailbox: "inbox"
 
   ## Gravatar
   ## For Libravatar see: http://doc.gitlab.com/ce/customization/libravatar.html

config/initializers/1_settings.rb

@@ -188,6 +188,8 @@ Settings.gitlab_ci['builds_path']         = File.expand_path(Settings.gitlab_ci[
 #
 Settings['incoming_email'] ||= Settingslogic.new({})
 Settings.incoming_email['enabled']  = false if Settings.incoming_email['enabled'].nil?
+Settings.incoming_email['port']     = 143 if Settings.incoming_email['port'].nil?
+Settings.incoming_email['mailbox']  = "inbox" if Settings.incoming_email['mailbox'].nil?
 
 #
 # Gravatar

config/mail_room.yml

+:mailboxes:
+<%
+require_relative 'config/environment.rb'
+
+if Gitlab::IncomingEmail.enabled? 
+  config = Gitlab::IncomingEmail.config
+
+  redis_config_file = "config/resque.yml"
+  redis_url = 
+    if File.exists?(redis_config_file)
+      YAML.load_file(redis_config_file)[Rails.env]
+    else
+      "redis://localhost:6379"
+    end
+  %>
+  -
+    :host: <%= config.host.to_json %>
+    :port: <%= config.port.to_json %>
+    :ssl: <%= config.ssl.to_json %>
+    :start_tls: <%= config.start_tls.to_json %>
+    :email: <%= config.user.to_json %>
+    :password: <%= config.password.to_json %>
+
+    :name: <%= config.mailbox.to_json %>
+
+    :delete_after_delivery: true
+
+    :delivery_method: sidekiq
+    :delivery_options:
+      :redis_url: <%= redis_url.to_json %>
+      :namespace: resque:gitlab
+      :queue: incoming_email
+      :worker: EmailReceiverWorker
+
+    :arbitration_method: redis
+    :arbitration_options:
+      :redis_url: <%= redis_url.to_json %>
+      :namespace: mail_room:gitlab
+<% end %>

config/mail_room.yml.example

-:mailboxes:
-  -
-    # # IMAP server host
-    # :host: "imap.gmail.com"
-    # # IMAP server port
-    # :port: 993
-    # # Whether the IMAP server uses SSL
-    # :ssl: true
-    # # Whether the IMAP server uses StartTLS
-    # :start_tls: false
-    # # Email account username. Usually the full email address.
-    # :email: "gitlab-incoming@gmail.com"
-    # # Email account password
-    # :password: "password"
-
-    # # The name of the mailbox where incoming mail will end up. Usually "inbox".
-    # :name: "inbox"
-
-    # # Always "sidekiq".
-    # :delivery_method: sidekiq
-    # # Always true.
-    # :delete_after_delivery: true
-    # :delivery_options:
-    #   # The URL to the Redis server used by Sidekiq. Should match the URL in config/resque.yml.
-    #   :redis_url: redis://localhost:6379
-    #   # Always "resque:gitlab".
-    #   :namespace: resque:gitlab
-    #   # Always "incoming_email".
-    #   :queue: incoming_email
-    #   # Always "EmailReceiverWorker".
-    #   :worker: EmailReceiverWorker
-
-    # # Always "redis".
-    # :arbitration_method: redis
-    # :arbitration_options:
-    #   # The URL to the Redis server. Should match the URL in config/resque.yml.
-    #   :redis_url: redis://localhost:6379
-    #   # Always "mail_room:gitlab".
-    #   :namespace: mail_room:gitlab

doc/incoming_email/README.md

@@ -4,9 +4,9 @@ GitLab can be set up to allow users to comment on issues and merge requests by r
 
 ## Get a mailbox
 
-Reply by email requires an IMAP-enabled email account, with a provider or server that supports [email sub-addressing](https://en.wikipedia.org/wiki/Email_address#Sub-addressing). Sub-addressing is a feature where any email to `user+some_arbitrary_tag@example.com` will end up in the mailbox for `user@example.com`, and is supported by providers such as Gmail, Yahoo! Mail, Outlook.com and iCloud, as well as the Postfix mail server which you can run on-premises.
+Reply by email requires an IMAP-enabled email account, with a provider or server that supports [email sub-addressing](https://en.wikipedia.org/wiki/Email_address#Sub-addressing). Sub-addressing is a feature where any email to `user+some_arbitrary_tag@example.com` will end up in the mailbox for `user@example.com`, and is supported by providers such as Gmail, Google Apps, Yahoo! Mail, Outlook.com and iCloud, as well as the Postfix mail server which you can run on-premises.
 
-If you want to use Gmail with Reply by email, make sure you have [IMAP access enabled](https://support.google.com/mail/troubleshooter/1668960?hl=en#ts=1665018) and [allow less secure apps to access the account](https://support.google.com/accounts/answer/6010255).
+If you want to use Gmail / Google Apps with Reply by email, make sure you have [IMAP access enabled](https://support.google.com/mail/troubleshooter/1668960?hl=en#ts=1665018) and [allow less secure apps to access the account](https://support.google.com/accounts/answer/6010255).
 
 To set up a basic Postfix mail server with IMAP access on Ubuntu, follow [these instructions](./postfix.md).
 
@@ -14,30 +14,62 @@ To set up a basic Postfix mail server with IMAP access on Ubuntu, follow [these
 
 ### Omnibus package installations
 
-1. Find the `incoming_email` section in `/etc/gitlab/gitlab.rb`, enable the feature, enter the email address including a placeholder for the `key` that references the item being replied to and fill in the details for your specific IMAP server and email account:
+1. Find the `incoming_email` section in `/etc/gitlab/gitlab.rb`, enable the feature and fill in the details for your specific IMAP server and email account:
 
     ```ruby
-    # Postfix mail server, assumes mailbox incoming@gitlab.example.com
+    # Configuration for Postfix mail server, assumes mailbox incoming@gitlab.example.com
     gitlab_rails['incoming_email_enabled'] = true
+    
+    # The email address including a placeholder for the key that references the item being replied to.
+    # The `%{key}` placeholder is added after the user part, before the `@`.
     gitlab_rails['incoming_email_address'] = "incoming+%{key}@gitlab.example.com"
-    gitlab_rails['incoming_email_host'] = "gitlab.example.com" # IMAP server host
-    gitlab_rails['incoming_email_port'] = 143 # IMAP server port
-    gitlab_rails['incoming_email_ssl'] = false # Whether the IMAP server uses SSL
-    gitlab_rails['incoming_email_email'] = "incoming"  # Email account username. Usually the full email address.
-    gitlab_rails['incoming_email_password'] = "[REDACTED]" # Email account password
-    gitlab_rails['incoming_email_mailbox_name'] = "inbox" # The name of the mailbox where incoming mail will end up. Usually "inbox".
+    
+    # Email account username
+    # With third party providers, this is usually the full email address.
+    # With self-hosted email servers, this is usually the user part of the email address.
+    gitlab_rails['incoming_email_email'] = "incoming"
+    # Email account password
+    gitlab_rails['incoming_email_password'] = "[REDACTED]"
+    
+    # IMAP server host
+    gitlab_rails['incoming_email_host'] = "gitlab.example.com"
+    # IMAP server port
+    gitlab_rails['incoming_email_port'] = 143
+    # Whether the IMAP server uses SSL
+    gitlab_rails['incoming_email_ssl'] = false
+    # Whether the IMAP server uses StartTLS
+    gitlab_rails['incoming_email_start_tls'] = false
+
+    # The mailbox where incoming mail will end up. Usually "inbox".
+    gitlab_rails['incoming_email_mailbox_name'] = "inbox"
     ```
 
     ```ruby
-    # Gmail / Google Apps, assumes mailbox gitlab-incoming@gmail.com
+    # Configuration for Gmail / Google Apps, assumes mailbox gitlab-incoming@gmail.com
     gitlab_rails['incoming_email_enabled'] = true
+    
+    # The email address including the `%{key}` placeholder that will be replaced to reference the item being replied to.
+    # The `%{key}` placeholder is added after the user part, after a `+` character, before the `@`.
     gitlab_rails['incoming_email_address'] = "gitlab-incoming+%{key}@gmail.com"
-    gitlab_rails['incoming_email_host'] = "imap.gmail.com" # IMAP server host
-    gitlab_rails['incoming_email_port'] = 993 # IMAP server port
-    gitlab_rails['incoming_email_ssl'] = true # Whether the IMAP server uses SSL
-    gitlab_rails['incoming_email_email'] = "gitlab-incoming@gmail.com"  # Email account username. Usually the full email address.
-    gitlab_rails['incoming_email_password'] = "[REDACTED]" # Email account password
-    gitlab_rails['incoming_email_mailbox_name'] = "inbox" # The name of the mailbox where incoming mail will end up. Usually "inbox".
+    
+    # Email account username
+    # With third party providers, this is usually the full email address.
+    # With self-hosted email servers, this is usually the user part of the email address.
+    gitlab_rails['incoming_email_email'] = "gitlab-incoming@gmail.com"
+    # Email account password
+    gitlab_rails['incoming_email_password'] = "[REDACTED]"
+    
+    # IMAP server host
+    gitlab_rails['incoming_email_host'] = "imap.gmail.com"
+    # IMAP server port
+    gitlab_rails['incoming_email_port'] = 993
+    # Whether the IMAP server uses SSL
+    gitlab_rails['incoming_email_ssl'] = true
+    # Whether the IMAP server uses StartTLS
+    gitlab_rails['incoming_email_start_tls'] = false
+
+    # The mailbox where incoming mail will end up. Usually "inbox".
+    gitlab_rails['incoming_email_mailbox_name'] = "inbox"
     ```
 
     As mentioned, the part after `+` in the address is ignored, and any email sent here will end up in the mailbox for `incoming@gitlab.example.com`/`gitlab-incoming@gmail.com`.
@@ -64,229 +96,146 @@ To set up a basic Postfix mail server with IMAP access on Ubuntu, follow [these
     cd /home/git/gitlab
     ```
 
-1. Find the `incoming_email` section in `config/gitlab.yml`, enable the feature and enter the email address including a placeholder for the `key` that references the item being replied to:
+1. Find the `incoming_email` section in `config/gitlab.yml`, enable the feature and fill in the details for your specific IMAP server and email account:
 
     ```sh
     sudo editor config/gitlab.yml
     ```
 
     ```yaml
-    # Postfix mail server, assumes mailbox incoming@gitlab.example.com
+    # Configuration for Postfix mail server, assumes mailbox incoming@gitlab.example.com
     incoming_email:
       enabled: true
-      address: "incoming+%{key}@gitlab.example.com"
-    ```
-
-    ```yaml
-    # Gmail / Google Apps, assumes mailbox gitlab-incoming@gmail.com
-    incoming_email:
-      enabled: true
-      address: "gitlab-incoming+%{key}@gmail.com"
-    ```
 
-    As mentioned, the part after `+` in the address is ignored, and any email sent here will end up in the mailbox for `incoming@gitlab.example.com`/`gitlab-incoming@gmail.com`.
-
-2. Copy `config/mail_room.yml.example` to `config/mail_room.yml`:
-
-    ```sh
-    sudo cp config/mail_room.yml.example config/mail_room.yml
-    ```
-
-3. Uncomment the configuration options in `config/mail_room.yml` and fill in the details for your specific IMAP server and email account:
+      # The email address including the `%{key}` placeholder that will be replaced to reference the item being replied to.
+      # The `%{key}` placeholder is added after the user part, after a `+` character, before the `@`.
+      address: "incoming+%{key}@gitlab.example.com"
 
-    ```sh
-    sudo editor config/mail_room.yml
-    ```
+      # Email account username
+      # With third party providers, this is usually the full email address.
+      # With self-hosted email servers, this is usually the user part of the email address.
+      user: "incoming"
+      # Email account password
+      password: "[REDACTED]"
 
-    ```yaml
-    # Postfix mail server
-    :mailboxes:
-      -
       # IMAP server host
-        :host: "gitlab.example.com"
+      host: "gitlab.example.com"
       # IMAP server port
-        :port: 143
+      port: 143
       # Whether the IMAP server uses SSL
-        :ssl: false
+      ssl: false
       # Whether the IMAP server uses StartTLS
-        :start_tls: false
-        # Email account username. Usually the full email address.
-        :email: "incoming"
-        # Email account password
-        :password: "[REDACTED]"
-
-        # The name of the mailbox where incoming mail will end up. Usually "inbox".
-        :name: "inbox"
-
-        # Always "sidekiq".
-        :delivery_method: sidekiq
-        # Always true.
-        :delete_after_delivery: true
-        :delivery_options:
-          # The URL to the Redis server used by Sidekiq. Should match the URL in config/resque.yml.
-          :redis_url: redis://localhost:6379
-          # Always "resque:gitlab".
-          :namespace: resque:gitlab
-          # Always "incoming_email".
-          :queue: incoming_email
-          # Always "EmailReceiverWorker"
-          :worker: EmailReceiverWorker
-
-        # Always "redis".
-        :arbitration_method: redis
-        :arbitration_options:
-          # The URL to the Redis server. Should match the URL in config/resque.yml.
-          :redis_url: redis://localhost:6379
-          # Always "mail_room:gitlab".
-          :namespace: mail_room:gitlab
+      start_tls: false
+
+      # The mailbox where incoming mail will end up. Usually "inbox".
+      mailbox: "inbox"
     ```
 
     ```yaml
-    # Gmail / Google Apps
-    :mailboxes:
-      -
+    # Configuration for Gmail / Google Apps, assumes mailbox gitlab-incoming@gmail.com
+    incoming_email:
+      enabled: true
+
+      # The email address including the `%{key}` placeholder that will be replaced to reference the item being replied to.
+      # The `%{key}` placeholder is added after the user part, after a `+` character, before the `@`.
+      address: "gitlab-incoming+%{key}@gmail.com"
+
+      # Email account username
+      # With third party providers, this is usually the full email address.
+      # With self-hosted email servers, this is usually the user part of the email address.
+      user: "gitlab-incoming@gmail.com"
+      # Email account password
+      password: "[REDACTED]"
+
       # IMAP server host
-        :host: "imap.gmail.com"
+      host: "imap.gmail.com"
       # IMAP server port
-        :port: 993
+      port: 993
       # Whether the IMAP server uses SSL
-        :ssl: true
+      ssl: true
       # Whether the IMAP server uses StartTLS
-        :start_tls: false
-        # Email account username. Usually the full email address.
-        :email: "gitlab-incoming@gmail.com"
-        # Email account password
-        :password: "[REDACTED]"
-
-        # The name of the mailbox where incoming mail will end up. Usually "inbox".
-        :name: "inbox"
-
-        # Always "sidekiq".
-        :delivery_method: sidekiq
-        # Always true.
-        :delete_after_delivery: true
-        :delivery_options:
-          # The URL to the Redis server used by Sidekiq. Should match the URL in config/resque.yml.
-          :redis_url: redis://localhost:6379
-          # Always "resque:gitlab".
-          :namespace: resque:gitlab
-          # Always "incoming_email".
-          :queue: incoming_email
-          # Always "EmailReceiverWorker"
-          :worker: EmailReceiverWorker
-
-        # Always "redis".
-        :arbitration_method: redis
-        :arbitration_options:
-          # The URL to the Redis server. Should match the URL in config/resque.yml.
-          :redis_url: redis://localhost:6379
-          # Always "mail_room:gitlab".
-          :namespace: mail_room:gitlab
+      start_tls: false
+
+      # The mailbox where incoming mail will end up. Usually "inbox".
+      mailbox: "inbox"
     ```
 
-5. Edit the init script configuration at `/etc/default/gitlab` to enable `mail_room`:
+    As mentioned, the part after `+` in the address is ignored, and any email sent here will end up in the mailbox for `incoming@gitlab.example.com`/`gitlab-incoming@gmail.com`.
+
+1. Enable `mail_room` in the init script at `/etc/default/gitlab`:
 
     ```sh
     sudo mkdir -p /etc/default
     echo 'mail_room_enabled=true' | sudo tee -a /etc/default/gitlab
     ```
 
-6. Restart GitLab:
+1. Restart GitLab:
 
     ```sh
     sudo service gitlab restart
     ```
 
-7. Verify that everything is configured correctly:
+1. Verify that everything is configured correctly:
 
     ```sh
     sudo -u git -H bundle exec rake gitlab:incoming_email:check RAILS_ENV=production
     ```
 
-8. Reply by email should now be working.
+1. Reply by email should now be working.
 
 ### Development
 
 1. Go to the GitLab installation directory.
 
-1. Find the `incoming_email` section in `config/gitlab.yml`, enable the feature and enter the email address including a placeholder for the `key` that references the item being replied to:
+1. Find the `incoming_email` section in `config/gitlab.yml`, enable the feature and fill in the details for your specific IMAP server and email account:
 
     ```yaml
-    # Gmail / Google Apps, assumes mailbox gitlab-incoming@gmail.com
+    # Configuration for Gmail / Google Apps, assumes mailbox gitlab-incoming@gmail.com
     incoming_email:
       enabled: true
-      address: "gitlab-incoming+%{key}@gmail.com"
-    ```
-
-    As mentioned, the part after `+` is ignored, and this will end up in the mailbox for `gitlab-incoming@gmail.com`.
 
-2. Copy `config/mail_room.yml.example` to `config/mail_room.yml`:
-
-    ```sh
-    sudo cp config/mail_room.yml.example config/mail_room.yml
-    ```
+      # The email address including a placeholder for the key that references the item being replied to.
+      # The `%{key}` placeholder is added after the user part, before the `@`.
+      address: "gitlab-incoming+%{key}@gmail.com"
 
-3. Uncomment the configuration options in `config/mail_room.yml` and fill in the details for your specific IMAP server and email account:
+      # Email account username
+      # With third party providers, this is usually the full email address.
+      # With self-hosted email servers, this is usually the user part of the email address.
+      user: "gitlab-incoming@gmail.com"
+      # Email account password
+      password: "[REDACTED]"
 
-    ```yaml
-    # Gmail / Google Apps, assumes mailbox gitlab-incoming@gmail.com
-    :mailboxes:
-      -
       # IMAP server host
-        :host: "imap.gmail.com"
+      host: "imap.gmail.com"
       # IMAP server port
-        :port: 993
+      port: 993
       # Whether the IMAP server uses SSL
-        :ssl: true
+      ssl: true
       # Whether the IMAP server uses StartTLS
-        :start_tls: false
-        # Email account username. Usually the full email address.
-        :email: "gitlab-incoming@gmail.com"
-        # Email account password
-        :password: "[REDACTED]"
-
-        # The name of the mailbox where incoming mail will end up. Usually "inbox".
-        :name: "inbox"
-
-        # Always "sidekiq".
-        :delivery_method: sidekiq
-        # Always true.
-        :delete_after_delivery: true
-        :delivery_options:
-          # The URL to the Redis server used by Sidekiq. Should match the URL in config/resque.yml.
-          :redis_url: redis://localhost:6379
-          # Always "resque:gitlab".
-          :namespace: resque:gitlab
-          # Always "incoming_email".
-          :queue: incoming_email
-          # Always "EmailReceiverWorker"
-          :worker: EmailReceiverWorker
-
-        # Always "redis".
-        :arbitration_method: redis
-        :arbitration_options:
-          # The URL to the Redis server. Should match the URL in config/resque.yml.
-          :redis_url: redis://localhost:6379
-          # Always "mail_room:gitlab".
-          :namespace: mail_room:gitlab
+      start_tls: false
+
+      # The mailbox where incoming mail will end up. Usually "inbox".
+      mailbox: "inbox"
     ```
 
-4. Uncomment the `mail_room` line in your `Procfile`:
+    As mentioned, the part after `+` is ignored, and this will end up in the mailbox for `gitlab-incoming@gmail.com`.
+
+1. Uncomment the `mail_room` line in your `Procfile`:
 
     ```yaml
     mail_room: bundle exec mail_room -q -c config/mail_room.yml
     ```
 
-6. Restart GitLab:
+1. Restart GitLab:
 
     ```sh
     bundle exec foreman start
     ```
 
-7. Verify that everything is configured correctly:
+1. Verify that everything is configured correctly:
 
     ```sh
     bundle exec rake gitlab:incoming_email:check RAILS_ENV=development
     ```
 
-8. Reply by email should now be working.
+1. Reply by email should now be working.

lib/gitlab/incoming_email.rb

@@ -24,12 +24,12 @@ module Gitlab
         match[1]
       end
 
-      private
-
       def config
         Gitlab.config.incoming_email
       end
 
+      private
+
       def address_regex
         wildcard_address = config.address
         return nil unless wildcard_address

lib/tasks/gitlab/check.rake

@@ -642,7 +642,6 @@ namespace :gitlab do
 
       if Gitlab.config.incoming_email.enabled
         check_address_formatted_correctly
-        check_mail_room_config_exists
         check_imap_authentication
 
         if Rails.env.production?
@@ -744,42 +743,16 @@ namespace :gitlab do
       end
     end
 
-    def check_mail_room_config_exists
-      print "MailRoom config exists? ... "
-
-      mail_room_config_file = Rails.root.join("config", "mail_room.yml")
-
-      if File.exists?(mail_room_config_file)
-        puts "yes".green
-      else
-        puts "no".red
-        try_fixing_it(
-          "Copy config/mail_room.yml.example to config/mail_room.yml",
-          "Check that the information in config/mail_room.yml is correct"
-        )
-        for_more_information(
-          "doc/incoming_email/README.md"
-        )
-        fix_and_rerun
-      end
-    end
-
     def check_imap_authentication
       print "IMAP server credentials are correct? ... "
 
-      mail_room_config_file = Rails.root.join("config", "mail_room.yml")
-
-      unless File.exists?(mail_room_config_file)
-        puts "can't check because of previous errors".magenta
-        return
-      end
-
-      config = YAML.load_file(mail_room_config_file)[:mailboxes].first rescue nil
+      config = Gitlab.config.incoming_email
 
       if config
         begin
-          imap = Net::IMAP.new(config[:host], port: config[:port], ssl: config[:ssl])
-          imap.login(config[:email], config[:password])
+          imap = Net::IMAP.new(config.host, port: config.port, ssl: config.ssl)
+          imap.starttls if config.start_tls
+          imap.login(config.user, config.password)
           connected = true
         rescue
           connected = false
@@ -791,7 +764,7 @@ namespace :gitlab do
       else
         puts "no".red
         try_fixing_it(
-          "Check that the information in config/mail_room.yml is correct"
+          "Check that the information in config/gitlab.yml is correct"
         )
         for_more_information(
           "doc/incoming_email/README.md"