Merge branch '29497-pages-custom-domain-dns-verification' into 'master'

Resolve "Domain validation for Pages custom domains" (master branch)

Closes #29497, #42971, and #42083

See merge request gitlab-org/gitlab-ce!17311

app/controllers/projects/pages_domains_controller.rb

@@ -3,7 +3,7 @@ class Projects::PagesDomainsController < Projects::ApplicationController
 
   before_action :require_pages_enabled!
   before_action :authorize_update_pages!, except: [:show]
-  before_action :domain, only: [:show, :destroy]
+  before_action :domain, only: [:show, :destroy, :verify]
 
   def show
   end
@@ -12,11 +12,23 @@ class Projects::PagesDomainsController < Projects::ApplicationController
     @domain = @project.pages_domains.new
   end
 
+  def verify
+    result = VerifyPagesDomainService.new(@domain).execute
+
+    if result[:status] == :success
+      flash[:notice] = 'Successfully verified domain ownership'
+    else
+      flash[:alert] = 'Failed to verify domain ownership'
+    end
+
+    redirect_to project_pages_domain_path(@project, @domain)
+  end
+
   def create
     @domain = @project.pages_domains.create(pages_domain_params)
 
     if @domain.valid?
-      redirect_to project_pages_path(@project)
+      redirect_to project_pages_domain_path(@project, @domain)
     else
       render 'new'
     end
@@ -46,6 +58,6 @@ class Projects::PagesDomainsController < Projects::ApplicationController
   end
 
   def domain
-    @domain ||= @project.pages_domains.find_by(domain: params[:id].to_s)
+    @domain ||= @project.pages_domains.find_by!(domain: params[:id].to_s)
   end
 end

app/helpers/application_settings_helper.rb

@@ -199,6 +199,7 @@ module ApplicationSettingsHelper
       :metrics_port,
       :metrics_sample_interval,
       :metrics_timeout,
+      :pages_domain_verification_enabled,
       :password_authentication_enabled_for_web,
       :password_authentication_enabled_for_git,
       :performance_bar_allowed_group_id,

app/mailers/emails/pages_domains.rb

+module Emails
+  module PagesDomains
+    def pages_domain_enabled_email(domain, recipient)
+      @domain = domain
+      @project = domain.project
+
+      mail(
+        to: recipient.notification_email,
+        subject: subject("GitLab Pages domain '#{domain.domain}' has been enabled")
+      )
+    end
+
+    def pages_domain_disabled_email(domain, recipient)
+      @domain = domain
+      @project = domain.project
+
+      mail(
+        to: recipient.notification_email,
+        subject: subject("GitLab Pages domain '#{domain.domain}' has been disabled")
+      )
+    end
+
+    def pages_domain_verification_succeeded_email(domain, recipient)
+      @domain = domain
+      @project = domain.project
+
+      mail(
+        to: recipient.notification_email,
+        subject: subject("Verification succeeded for GitLab Pages domain '#{domain.domain}'")
+      )
+    end
+
+    def pages_domain_verification_failed_email(domain, recipient)
+      @domain = domain
+      @project = domain.project
+
+      mail(
+        to: recipient.notification_email,
+        subject: subject("ACTION REQUIRED: Verification failed for GitLab Pages domain '#{domain.domain}'")
+      )
+    end
+  end
+end

app/mailers/notify.rb

@@ -5,6 +5,7 @@ class Notify < BaseMailer
   include Emails::Issues
   include Emails::MergeRequests
   include Emails::Notes
+  include Emails::PagesDomains
   include Emails::Projects
   include Emails::Profile
   include Emails::Pipelines

app/models/pages_domain.rb

 class PagesDomain < ActiveRecord::Base
+  VERIFICATION_KEY = 'gitlab-pages-verification-code'.freeze
+  VERIFICATION_THRESHOLD = 3.days.freeze
+
   belongs_to :project
 
   validates :domain, hostname: { allow_numeric_hostname: true }
   validates :domain, uniqueness: { case_sensitive: false }
   validates :certificate, certificate: true, allow_nil: true, allow_blank: true
   validates :key, certificate_key: true, allow_nil: true, allow_blank: true
+  validates :verification_code, presence: true, allow_blank: false
 
   validate :validate_pages_domain
   validate :validate_matching_key, if: ->(domain) { domain.certificate.present? || domain.key.present? }
@@ -16,10 +20,32 @@ class PagesDomain < ActiveRecord::Base
     key: Gitlab::Application.secrets.db_key_base,
     algorithm: 'aes-256-cbc'
 
+  after_initialize :set_verification_code
   after_create :update_daemon
-  after_save :update_daemon
+  after_update :update_daemon, if: :pages_config_changed?
   after_destroy :update_daemon
 
+  scope :enabled, -> { where('enabled_until >= ?', Time.now ) }
+  scope :needs_verification, -> do
+    verified_at = arel_table[:verified_at]
+    enabled_until = arel_table[:enabled_until]
+    threshold = Time.now + VERIFICATION_THRESHOLD
+
+    where(verified_at.eq(nil).or(enabled_until.eq(nil).or(enabled_until.lt(threshold))))
+  end
+
+  def verified?
+    !!verified_at
+  end
+
+  def unverified?
+    !verified?
+  end
+
+  def enabled?
+    !Gitlab::CurrentSettings.pages_domain_verification_enabled? || enabled_until.present?
+  end
+
   def to_param
     domain
   end
@@ -84,12 +110,49 @@ class PagesDomain < ActiveRecord::Base
     @certificate_text ||= x509.try(:to_text)
   end
 
+  # Verification codes may be TXT records for domain or verification_domain, to
+  # support the use of CNAME records on domain.
+  def verification_domain
+    return unless domain.present?
+
+    "_#{VERIFICATION_KEY}.#{domain}"
+  end
+
+  def keyed_verification_code
+    return unless verification_code.present?
+
+    "#{VERIFICATION_KEY}=#{verification_code}"
+  end
+
   private
 
+  def set_verification_code
+    return if self.verification_code.present?
+
+    self.verification_code = SecureRandom.hex(16)
+  end
+
   def update_daemon
     ::Projects::UpdatePagesConfigurationService.new(project).execute
   end
 
+  def pages_config_changed?
+    project_id_changed? ||
+      domain_changed? ||
+      certificate_changed? ||
+      key_changed? ||
+      became_enabled? ||
+      became_disabled?
+  end
+
+  def became_enabled?
+    enabled_until.present? && !enabled_until_was.present?
+  end
+
+  def became_disabled?
+    !enabled_until.present? && enabled_until_was.present?
+  end
+
   def validate_matching_key
     unless has_matching_key?
       self.errors.add(:key, "doesn't match the certificate")

app/services/notification_service.rb

@@ -339,6 +339,30 @@ class NotificationService
     end
   end
 
+  def pages_domain_verification_succeeded(domain)
+    recipients_for_pages_domain(domain).each do |user|
+      mailer.pages_domain_verification_succeeded_email(domain, user).deliver_later
+    end
+  end
+
+  def pages_domain_verification_failed(domain)
+    recipients_for_pages_domain(domain).each do |user|
+      mailer.pages_domain_verification_failed_email(domain, user).deliver_later
+    end
+  end
+
+  def pages_domain_enabled(domain)
+    recipients_for_pages_domain(domain).each do |user|
+      mailer.pages_domain_enabled_email(domain, user).deliver_later
+    end
+  end
+
+  def pages_domain_disabled(domain)
+    recipients_for_pages_domain(domain).each do |user|
+      mailer.pages_domain_disabled_email(domain, user).deliver_later
+    end
+  end
+
   protected
 
   def new_resource_email(target, method)
@@ -433,6 +457,14 @@ class NotificationService
 
   private
 
+  def recipients_for_pages_domain(domain)
+    project = domain.project
+
+    return [] unless project
+
+    notifiable_users(project.team.masters, :watch, target: project)
+  end
+
   def notifiable?(*args)
     NotificationRecipientService.notifiable?(*args)
   end

app/services/projects/update_pages_configuration_service.rb

@@ -23,7 +23,7 @@ module Projects
     end
 
     def pages_domains_config
-      project.pages_domains.map do |domain|
+      enabled_pages_domains.map do |domain|
         {
           domain: domain.domain,
           certificate: domain.certificate,
@@ -32,6 +32,14 @@ module Projects
       end
     end
 
+    def enabled_pages_domains
+      if Gitlab::CurrentSettings.pages_domain_verification_enabled?
+        project.pages_domains.enabled
+      else
+        project.pages_domains
+      end
+    end
+
     def reload_daemon
       # GitLab Pages daemon constantly watches for modification time of `pages.path`
       # It reloads configuration when `pages.path` is modified

app/services/verify_pages_domain_service.rb

+require 'resolv'
+
+class VerifyPagesDomainService < BaseService
+  # The maximum number of seconds to be spent on each DNS lookup
+  RESOLVER_TIMEOUT_SECONDS = 15
+
+  # How long verification lasts for
+  VERIFICATION_PERIOD = 7.days
+
+  attr_reader :domain
+
+  def initialize(domain)
+    @domain = domain
+  end
+
+  def execute
+    return error("No verification code set for #{domain.domain}") unless domain.verification_code.present?
+
+    if !verification_enabled? || dns_record_present?
+      verify_domain!
+    elsif expired?
+      disable_domain!
+    else
+      unverify_domain!
+    end
+  end
+
+  private
+
+  def verify_domain!
+    was_disabled = !domain.enabled?
+    was_unverified = domain.unverified?
+
+    # Prevent any pre-existing grace period from being truncated
+    reverify = [domain.enabled_until, VERIFICATION_PERIOD.from_now].compact.max
+
+    domain.update!(verified_at: Time.now, enabled_until: reverify)
+
+    if was_disabled
+      notify(:enabled)
+    elsif was_unverified
+      notify(:verification_succeeded)
+    end
+
+    success
+  end
+
+  def unverify_domain!
+    if domain.verified?
+      domain.update!(verified_at: nil)
+      notify(:verification_failed)
+    end
+
+    error("Couldn't verify #{domain.domain}")
+  end
+
+  def disable_domain!
+    domain.update!(verified_at: nil, enabled_until: nil)
+
+    notify(:disabled)
+
+    error("Couldn't verify #{domain.domain}. It is now disabled.")
+  end
+
+  # A domain is only expired until `disable!` has been called
+  def expired?
+    domain.enabled_until && domain.enabled_until < Time.now
+  end
+
+  def dns_record_present?
+    Resolv::DNS.open do |resolver|
+      resolver.timeouts = RESOLVER_TIMEOUT_SECONDS
+
+      check(domain.domain, resolver) || check(domain.verification_domain, resolver)
+    end
+  end
+
+  def check(domain_name, resolver)
+    records = parse(txt_records(domain_name, resolver))
+
+    records.any? do |record|
+      record == domain.keyed_verification_code || record == domain.verification_code
+    end
+  rescue => err
+    log_error("Failed to check TXT records on #{domain_name} for #{domain.domain}: #{err}")
+    false
+  end
+
+  def txt_records(domain_name, resolver)
+    resolver.getresources(domain_name, Resolv::DNS::Resource::IN::TXT)
+  end
+
+  def parse(records)
+    records.flat_map(&:strings).flat_map(&:split)
+  end
+
+  def verification_enabled?
+    Gitlab::CurrentSettings.pages_domain_verification_enabled?
+  end
+
+  def notify(type)
+    return unless verification_enabled?
+
+    Gitlab::AppLogger.info("Pages domain '#{domain.domain}' changed state to '#{type}'")
+    notification_service.public_send("pages_domain_#{type}", domain) # rubocop:disable GitlabSecurity/PublicSend
+  end
+end

app/views/admin/application_settings/_form.html.haml

@@ -237,6 +237,17 @@
       .col-sm-10
         = f.number_field :max_pages_size, class: 'form-control'
         .help-block 0 for unlimited
+    .form-group
+      .col-sm-offset-2.col-sm-10
+        .checkbox
+          = f.label :pages_domain_verification_enabled do
+            = f.check_box :pages_domain_verification_enabled
+            Require users to prove ownership of custom domains
+        .help-block
+          Domain verification is an essential security measure for public GitLab
+          sites. Users are required to demonstrate they control a domain before
+          it is enabled
+          = link_to icon('question-circle'), help_page_path('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
 
   %fieldset
     %legend Continuous Integration and Deployment

app/views/notify/pages_domain_disabled_email.html.haml

+%p
+  Following a verification check, your GitLab Pages custom domain has been
+  %strong disabled.
+  This means that your content is no longer visible at #{link_to @domain.url, @domain.url}
+%p
+  Project: #{link_to @project.human_name, project_url(@project)}
+%p
+  Domain: #{link_to @domain.domain, project_pages_domain_url(@project, @domain)}
+%p
+  If this domain has been disabled in error, please follow
+  = link_to 'these instructions', help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+  to verify and re-enable your domain.
+%p
+  If you no longer wish to use this domain with GitLab Pages, please remove it
+  from your GitLab project and delete any related DNS records.

app/views/notify/pages_domain_disabled_email.text.haml

+Following a verification check, your GitLab Pages custom domain has been
+**disabled**. This means that your content is no longer visible at #{@domain.url}
+
+Project: #{@project.human_name} (#{project_url(@project)})
+Domain:  #{@domain.domain} (#{project_pages_domain_url(@project, @domain)})
+
+If this domain has been disabled in error, please follow these instructions
+to verify and re-enable your domain:
+
+= help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+
+If you no longer wish to use this domain with GitLab Pages, please remove it
+from your GitLab project and delete any related DNS records.

app/views/notify/pages_domain_enabled_email.html.haml

+%p
+  Following a verification check, your GitLab Pages custom domain has been
+  enabled. You should now be able to view your content at #{link_to @domain.url, @domain.url}
+%p
+  Project: #{link_to @project.human_name, project_url(@project)}
+%p
+  Domain: #{link_to @domain.domain, project_pages_domain_url(@project, @domain)}
+%p
+  Please visit
+  = link_to 'these instructions', help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+  for more information about custom domain verification.

app/views/notify/pages_domain_enabled_email.text.haml

+Following a verification check, your GitLab Pages custom domain has been
+enabled. You should now be able to view your content at #{@domain.url}
+
+Project: #{@project.human_name} (#{project_url(@project)})
+Domain:  #{@domain.domain} (#{project_pages_domain_url(@project, @domain)})
+
+Please visit
+= help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+for more information about custom domain verification.

app/views/notify/pages_domain_verification_failed_email.html.haml

+%p
+  Verification has failed for one of your GitLab Pages custom domains!
+%p
+  Project: #{link_to @project.human_name, project_url(@project)}
+%p
+  Domain: #{link_to @domain.domain, project_pages_domain_url(@project, @domain)}
+%p
+  Unless you take action, it will be disabled on
+  %strong= @domain.enabled_until.strftime('%F %T.')
+  Until then, you can view your content at #{link_to @domain.url, @domain.url}
+%p
+  Please visit
+  = link_to 'these instructions', help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+  for more information about custom domain verification.
+%p
+  If you no longer wish to use this domain with GitLab Pages, please remove it
+  from your GitLab project and delete any related DNS records.

app/views/notify/pages_domain_verification_failed_email.text.haml

+Verification has failed for one of your GitLab Pages custom domains!
+
+Project: #{@project.human_name} (#{project_url(@project)})
+Domain:  #{@domain.domain} (#{project_pages_domain_url(@project, @domain)})
+
+Unless you take action, it will be disabled on *#{@domain.enabled_until.strftime('%F %T')}*.
+Until then, you can view your content at #{@domain.url}
+
+Please visit
+= help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+for more information about custom domain verification.
+
+If you no longer wish to use this domain with GitLab Pages, please remove it
+from your GitLab project and delete any related DNS records.

app/views/notify/pages_domain_verification_succeeded_email.html.haml

+%p
+  One of your GitLab Pages custom domains has been successfully verified!
+%p
+  Project: #{link_to @project.human_name, project_url(@project)}
+%p
+  Domain: #{link_to @domain.domain, project_pages_domain_url(@project, @domain)}
+%p
+  This is a notification. No action is required on your part. You can view your
+  content at #{link_to @domain.url, @domain.url}
+%p
+  Please visit
+  = link_to 'these instructions', help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+  for more information about custom domain verification.

app/views/notify/pages_domain_verification_succeeded_email.text.haml

+One of your GitLab Pages custom domains has been successfully verified!
+
+Project: #{@project.human_name} (#{project_url(@project)})
+Domain:  #{@domain.domain} (#{project_pages_domain_url(@project, @domain)})
+
+No action is required on your part. You can view your content at #{@domain.url}
+
+Please visit
+= help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+for more information about custom domain verification.

app/views/projects/pages/_list.html.haml

@@ -3,15 +3,26 @@
     .panel-heading
       Domains (#{@domains.count})
     %ul.well-list
+      - verification_enabled = Gitlab::CurrentSettings.pages_domain_verification_enabled?
       - @domains.each do |domain|
         %li
           .pull-right
             = link_to 'Details', project_pages_domain_path(@project, domain), class: "btn btn-sm btn-grouped"
             = link_to 'Remove', project_pages_domain_path(@project, domain), data: { confirm: 'Are you sure?'}, method: :delete, class: "btn btn-remove btn-sm btn-grouped"
           .clearfix
-            %span= link_to domain.domain, domain.url
+            - if verification_enabled
+              - tooltip, status = domain.unverified? ? ['Unverified', 'failed'] : ['Verified', 'success']
+              = link_to domain.url, title: tooltip, class: 'has-tooltip' do
+                = sprite_icon("status_#{status}", size: 16, css_class: "has-tooltip ci-status-icon ci-status-icon-#{status}")
+                = domain.domain
+            - else
+              = link_to domain.domain, domain.url
           %p
             - if domain.subject
               %span.label.label-gray Certificate: #{domain.subject}
             - if domain.expired?
               %span.label.label-danger Expired
+        - if verification_enabled && domain.unverified?
+          %li.warning-row
+            #{domain.domain} is not verified. To learn how to verify ownership, visit your
+            = link_to 'domain details', project_pages_domain_path(@project, domain)

app/views/projects/pages_domains/show.html.haml

 - page_title "#{@domain.domain}", 'Pages Domains'
+- verification_enabled = Gitlab::CurrentSettings.pages_domain_verification_enabled?
+- if verification_enabled && @domain.unverified?
+  %p.alert.alert-warning
+    %strong
+      This domain is not verified. You will need to verify ownership before
+      access is enabled.
 
 %h3.page-title
   Pages Domain
@@ -15,9 +21,26 @@
         DNS
       %td
         %p
-          To access the domain create a new DNS record:
+          To access this domain create a new DNS record:
         %pre
           #{@domain.domain} CNAME #{@domain.project.pages_subdomain}.#{Settings.pages.host}.
+    - if verification_enabled
+      %tr
+        %td
+          Verification status
+        %td
+          %p
+            - help_link = help_page_path('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+            To #{link_to 'verify ownership', help_link} of your domain, create
+            this DNS record:
+          %pre
+            #{@domain.verification_domain} TXT #{@domain.keyed_verification_code}
+          %p
+            - if @domain.verified?
+              #{@domain.domain} has been successfully verified.
+            - else
+              = button_to 'Verify ownership', verify_project_pages_domain_path(@project, @domain), class: 'btn btn-save btn-sm'
+
     %tr
       %td
         Certificate

app/workers/all_queues.yml

@@ -3,6 +3,7 @@
 - cronjob:expire_build_artifacts
 - cronjob:gitlab_usage_ping
 - cronjob:import_export_project_cleanup
+- cronjob:pages_domain_verification_cron
 - cronjob:pipeline_schedule
 - cronjob:prune_old_events
 - cronjob:remove_expired_group_links
@@ -82,6 +83,7 @@
 - new_merge_request
 - new_note
 - pages
+- pages_domain_verification
 - post_receive
 - process_commit
 - project_cache

app/workers/pages_domain_verification_cron_worker.rb

+class PagesDomainVerificationCronWorker
+  include ApplicationWorker
+  include CronjobQueue
+
+  def perform
+    PagesDomain.needs_verification.find_each do |domain|
+      PagesDomainVerificationWorker.perform_async(domain.id)
+    end
+  end
+end

app/workers/pages_domain_verification_worker.rb

+class PagesDomainVerificationWorker
+  include ApplicationWorker
+
+  def perform(domain_id)
+    domain = PagesDomain.find_by(id: domain_id)
+
+    return unless domain
+
+    VerifyPagesDomainService.new(domain).execute
+  end
+end

changelogs/unreleased/29497-pages-custom-domain-dns-verification.yml

+---
+title: Add verification for GitLab Pages custom domains
+merge_request:
+author:
+type: security

config/gitlab.yml.example

@@ -214,6 +214,10 @@ production: &base
     repository_archive_cache_worker:
       cron: "0 * * * *"
 
+    # Verify custom GitLab Pages domains
+    pages_domain_verification_cron_worker:
+      cron: "*/15 * * * *"
+
   registry:
     # enabled: true
     # host: registry.example.com

config/initializers/1_settings.rb

@@ -427,6 +427,10 @@ Settings.cron_jobs['stuck_merge_jobs_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['stuck_merge_jobs_worker']['cron'] ||= '0 */2 * * *'
 Settings.cron_jobs['stuck_merge_jobs_worker']['job_class'] = 'StuckMergeJobsWorker'
 
+Settings.cron_jobs['pages_domain_verification_cron_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['pages_domain_verification_cron_worker']['cron'] ||= '*/15 * * * *'
+Settings.cron_jobs['pages_domain_verification_cron_worker']['job_class'] = 'PagesDomainVerificationCronWorker'
+
 #
 # GitLab Shell
 #

config/routes/project.rb

@@ -55,7 +55,11 @@ constraints(ProjectUrlConstrainer.new) do
       end
 
       resource :pages, only: [:show, :destroy] do
-        resources :domains, only: [:show, :new, :create, :destroy], controller: 'pages_domains', constraints: { id: %r{[^/]+} }
+        resources :domains, only: [:show, :new, :create, :destroy], controller: 'pages_domains', constraints: { id: %r{[^/]+} } do
+          member do
+            post :verify
+          end
+        end
       end
 
       resources :snippets, concerns: :awardable, constraints: { id: /\d+/ } do

config/sidekiq_queues.yml

@@ -67,3 +67,4 @@
   - [gcp_cluster, 1]
   - [project_migrate_hashed_storage, 1]
   - [storage_migrator, 1]
+  - [pages_domain_verification, 1]

db/migrate/20180216120000_add_pages_domain_verification.rb

+class AddPagesDomainVerification < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    add_column :pages_domains, :verified_at, :datetime_with_timezone
+    add_column :pages_domains, :verification_code, :string
+  end
+end

db/migrate/20180216120010_add_pages_domain_verified_at_index.rb

+class AddPagesDomainVerifiedAtIndex < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :pages_domains, :verified_at
+  end
+
+  def down
+    remove_concurrent_index :pages_domains, :verified_at
+  end
+end

db/migrate/20180216120020_allow_domain_verification_to_be_disabled.rb

+class AllowDomainVerificationToBeDisabled < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    add_column :application_settings, :pages_domain_verification_enabled, :boolean, default: true, null: false
+  end
+end

db/migrate/20180216120030_add_pages_domain_enabled_until.rb

+class AddPagesDomainEnabledUntil < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    add_column :pages_domains, :enabled_until, :datetime_with_timezone
+  end
+end

db/migrate/20180216120040_add_pages_domain_enabled_until_index.rb

+class AddPagesDomainEnabledUntilIndex < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :pages_domains, [:project_id, :enabled_until]
+    add_concurrent_index :pages_domains, [:verified_at, :enabled_until]
+  end
+
+  def down
+    remove_concurrent_index :pages_domains, [:verified_at, :enabled_until]
+    remove_concurrent_index :pages_domains, [:project_id, :enabled_until]
+  end
+end

db/migrate/20180216120050_pages_domains_verification_grace_period.rb

+class PagesDomainsVerificationGracePeriod < ActiveRecord::Migration
+  DOWNTIME = false
+
+  class PagesDomain < ActiveRecord::Base
+    include EachBatch
+  end
+
+  # Allow this migration to resume if it fails partway through
+  disable_ddl_transaction!
+
+  def up
+    now = Time.now
+    grace = now + 30.days
+
+    PagesDomain.each_batch do |relation|
+      relation.update_all(verified_at: now, enabled_until: grace)
+
+      # Sleep 2 minutes between batches to not overload the DB with dead tuples
+      sleep(2.minutes) unless relation.reorder(:id).last == PagesDomain.reorder(:id).last
+    end
+  end
+
+  def down
+    # no-op
+  end
+end

db/post_migrate/20180216121020_fill_pages_domain_verification_code.rb

+class FillPagesDomainVerificationCode < ActiveRecord::Migration
+  DOWNTIME = false
+
+  class PagesDomain < ActiveRecord::Base
+    include EachBatch
+  end
+
+  # Allow this migration to resume if it fails partway through
+  disable_ddl_transaction!
+
+  def up
+    PagesDomain.where(verification_code: [nil, '']).each_batch do |relation|
+      connection.execute(set_codes_sql(relation))
+
+      # Sleep 2 minutes between batches to not overload the DB with dead tuples
+      sleep(2.minutes) unless relation.reorder(:id).last == PagesDomain.reorder(:id).last
+    end
+
+    change_column_null(:pages_domains, :verification_code, false)
+  end
+
+  def down
+    change_column_null(:pages_domains, :verification_code, true)
+  end
+
+  private
+
+  def set_codes_sql(relation)
+    ids = relation.pluck(:id)
+    whens = ids.map { |id| "WHEN #{id} THEN '#{SecureRandom.hex(16)}'" }
+
+    <<~SQL
+      UPDATE pages_domains
+      SET verification_code =
+        CASE id
+        #{whens.join("\n")}
+        END
+      WHERE id IN(#{ids.join(',')})
+    SQL
+  end
+end

db/post_migrate/20180216121030_enqueue_verify_pages_domain_workers.rb

+class EnqueueVerifyPagesDomainWorkers < ActiveRecord::Migration
+  class PagesDomain < ActiveRecord::Base
+    include EachBatch
+  end
+
+  def up
+    PagesDomain.each_batch do |relation|
+      ids = relation.pluck(:id).map { |id| [id] }
+      PagesDomainVerificationWorker.bulk_perform_async(ids)
+    end
+  end
+
+  def down
+    # no-op
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180215181245) do
+ActiveRecord::Schema.define(version: 20180216121030) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -156,6 +156,7 @@ ActiveRecord::Schema.define(version: 20180215181245) do
     t.integer "gitaly_timeout_fast", default: 10, null: false
     t.boolean "authorized_keys_enabled", default: true, null: false
     t.string "auto_devops_domain"
+    t.boolean "pages_domain_verification_enabled", default: true, null: false
   end
 
   create_table "audit_events", force: :cascade do |t|
@@ -1313,10 +1314,16 @@ ActiveRecord::Schema.define(version: 20180215181245) do
     t.string "encrypted_key_iv"
     t.string "encrypted_key_salt"
     t.string "domain"
+    t.datetime_with_timezone "verified_at"
+    t.string "verification_code", null: false
+    t.datetime_with_timezone "enabled_until"
   end
 
   add_index "pages_domains", ["domain"], name: "index_pages_domains_on_domain", unique: true, using: :btree
+  add_index "pages_domains", ["project_id", "enabled_until"], name: "index_pages_domains_on_project_id_and_enabled_until", using: :btree
   add_index "pages_domains", ["project_id"], name: "index_pages_domains_on_project_id", using: :btree
+  add_index "pages_domains", ["verified_at", "enabled_until"], name: "index_pages_domains_on_verified_at_and_enabled_until", using: :btree
+  add_index "pages_domains", ["verified_at"], name: "index_pages_domains_on_verified_at", using: :btree
 
   create_table "personal_access_tokens", force: :cascade do |t|
     t.integer "user_id", null: false

doc/administration/pages/index.md

@@ -226,6 +226,18 @@ world. Custom domains and TLS are supported.
 
 1. [Reconfigure GitLab][reconfigure]
 
+### Custom domain verification
+
+To prevent malicious users from hijacking domains that don't belong to them,
+GitLab supports [custom domain verification](../../user/project/pages/getting_started_part_three.md#dns-txt-record).
+When adding a custom domain, users will be required to prove they own it by
+adding a GitLab-controlled verification code to the DNS records for that domain.
+
+If your userbase is private or otherwise trusted, you can disable the
+verification requirement. Navigate to `Admin area ➔ Settings` and uncheck
+**Require users to prove ownership of custom domains** in the Pages section.
+This setting is enabled by default.
+
 ## Change storage path
 
 Follow the steps below to change the default path where GitLab Pages' contents

doc/user/project/pages/getting_started_part_three.md

@@ -62,7 +62,7 @@ for the most popular hosting services:
 - [Microsoft](https://msdn.microsoft.com/en-us/library/bb727018.aspx)
 
 If your hosting service is not listed above, you can just try to
-search the web for "how to add dns record on <my hosting service>".
+search the web for `how to add dns record on <my hosting service>`.
 
 ### DNS A record
 
@@ -95,12 +95,32 @@ without any `/project-name`.
 
 ![DNS CNAME record pointing to GitLab.com project](img/dns_cname_record_example.png)
 
-### TL;DR
+#### DNS TXT record
+
+Unless your GitLab administrator has [disabled custom domain verification](../../../administration/pages/index.md#custom-domain-verification),
+you'll have to prove that you own the domain by creating a `TXT` record
+containing a verification code. The code will be displayed after you
+[add your custom domain to GitLab Pages settings](#add-your-custom-domain-to-gitlab-pages-settings).
+
+If using a [DNS A record](#dns-a-record), you can place the TXT record directly
+under the domain. If using a [DNS CNAME record](#dns-cname-record), the two record types won't
+co-exist, so you need to place the TXT record in a special subdomain of its own.
+
+#### TL;DR
+
+If the domain has multiple uses (e.g., you host email on it as well):
 
 | From | DNS Record | To |
 | ---- | ---------- | -- |
 | domain.com | A | 52.167.214.135 |
-| subdomain.domain.com | CNAME | namespace.gitlab.io |
+| domain.com | TXT | gitlab-pages-verification-code=00112233445566778899aabbccddeeff |
+
+If the domain is dedicated to GitLab Pages use and no other services run on it:
+
+| From | DNS Record | To |
+| ---- | ---------- | -- |
+| subdomain.domain.com | CNAME | gitlab.io |
+| _gitlab-pages-verification-code.subdomain.domain.com | TXT | gitlab-pages-verification-code=00112233445566778899aabbccddeeff |
 
 > **Notes**:
 >
@@ -121,6 +141,17 @@ your site will be accessible only via HTTP:
 
 ![Add new domain](img/add_certificate_to_pages.png)
 
+Once you have added a new domain, you will need to **verify your ownership**
+(unless the GitLab administrator has disabled this feature). A verification code
+will be shown to you; add it as a [DNS TXT record](#dns-txt-record), then press
+the "Verify ownership" button to activate your new domain:
+
+![Verify your domain](img/verify_your_domain.png)
+
+Once your domain has been verified, leave the verification record in place -
+your domain will be periodically reverified, and may be disabled if the record
+is removed.
+
 You can add more than one alias (custom domains and subdomains) to the same project.
 An alias can be understood as having many doors leading to the same room.
 
@@ -128,8 +159,8 @@ All the aliases you've set to your site will be listed on **Setting > Pages**.
 From that page, you can view, add, and remove them.
 
 Note that [DNS propagation may take some time (up to 24h)](http://www.inmotionhosting.com/support/domain-names/dns-nameserver-changes/domain-names-dns-changes),
-although it's usually a matter of minutes to complete. Until it does, visit attempts
-to your domain will respond with a 404.
+although it's usually a matter of minutes to complete. Until it does, verification
+will fail and attempts to visit your domain will respond with a 404.
 
 Read through the [general documentation on GitLab Pages](introduction.md#add-a-custom-domain-to-your-pages-website) to learn more about adding
 custom domains to GitLab Pages sites.

lib/api/entities.rb

@@ -1154,6 +1154,10 @@ module API
       expose :domain
       expose :url
       expose :project_id
+      expose :verified?, as: :verified
+      expose :verification_code, as: :verification_code
+      expose :enabled_until
+
       expose :certificate,
         as: :certificate_expiration,
         if: ->(pages_domain, _) { pages_domain.certificate? },
@@ -1165,6 +1169,10 @@ module API
     class PagesDomain < Grape::Entity
       expose :domain
       expose :url
+      expose :verified?, as: :verified
+      expose :verification_code, as: :verification_code
+      expose :enabled_until
+
       expose :certificate,
         if: ->(pages_domain, _) { pages_domain.certificate? },
         using: PagesDomainCertificate do |pages_domain|

spec/controllers/projects/pages_domains_controller_spec.rb

@@ -46,7 +46,46 @@ describe Projects::PagesDomainsController do
         post(:create, request_params.merge(pages_domain: pages_domain_params))
       end.to change { PagesDomain.count }.by(1)
 
-      expect(response).to redirect_to(project_pages_path(project))
+      created_domain = PagesDomain.reorder(:id).last
+
+      expect(created_domain).to be_present
+      expect(response).to redirect_to(project_pages_domain_path(project, created_domain))
+    end
+  end
+
+  describe 'POST verify' do
+    let(:params) { request_params.merge(id: pages_domain.domain) }
+
+    def stub_service
+      service = double(:service)
+
+      expect(VerifyPagesDomainService).to receive(:new) { service }
+
+      service
+    end
+
+    it 'handles verification success' do
+      expect(stub_service).to receive(:execute).and_return(status: :success)
+
+      post :verify, params
+
+      expect(response).to redirect_to project_pages_domain_path(project, pages_domain)
+      expect(flash[:notice]).to eq('Successfully verified domain ownership')
+    end
+
+    it 'handles verification failure' do
+      expect(stub_service).to receive(:execute).and_return(status: :failed)
+
+      post :verify, params
+
+      expect(response).to redirect_to project_pages_domain_path(project, pages_domain)
+      expect(flash[:alert]).to eq('Failed to verify domain ownership')
+    end
+
+    it 'returns a 404 response for an unknown domain' do
+      post :verify, request_params.merge(id: 'unknown-domain')
+
+      expect(response).to have_gitlab_http_status(404)
     end
   end
 

spec/factories/pages_domains.rb

 FactoryBot.define do
   factory :pages_domain, class: 'PagesDomain' do
-    domain 'my.domain.com'
+    sequence(:domain) { |n| "my#{n}.domain.com" }
+    verified_at { Time.now }
+    enabled_until { 1.week.from_now }
+
+    trait :disabled do
+      verified_at nil
+      enabled_until nil
+    end
+
+    trait :unverified do
+      verified_at nil
+    end
+
+    trait :reverify do
+      enabled_until { 1.hour.from_now }
+    end
+
+    trait :expired do
+      enabled_until { 1.hour.ago }
+    end
 
     trait :with_certificate do
       certificate '-----BEGIN CERTIFICATE-----

spec/features/projects/pages_spec.rb

@@ -60,7 +60,6 @@ feature 'Pages' do
             fill_in 'Domain', with: 'my.test.domain.com'
             click_button 'Create New Domain'
 
-            expect(page).to have_content('Domains (1)')
             expect(page).to have_content('my.test.domain.com')
           end
         end
@@ -159,7 +158,6 @@ feature 'Pages' do
           fill_in 'Key (PEM)', with: certificate_key
           click_button 'Create New Domain'
 
-          expect(page).to have_content('Domains (1)')
           expect(page).to have_content('my.test.domain.com')
         end
       end

spec/fixtures/api/schemas/public_api/v4/pages_domain/basic.json

@@ -4,6 +4,9 @@
     "domain": { "type": "string" },
     "url": { "type": "uri" },
     "project_id": { "type": "integer" },
+    "verified": { "type": "boolean" },
+    "verification_code": { "type": ["string", "null"] },
+    "enabled_until": { "type": ["date", "null"] },
     "certificate_expiration": {
       "type": "object",
       "properties": {
@@ -14,6 +17,6 @@
       "additionalProperties": false
     }
   },
-  "required": ["domain", "url", "project_id"],
+  "required": ["domain", "url", "project_id", "verified", "verification_code", "enabled_until"],
   "additionalProperties": false
 }

spec/fixtures/api/schemas/public_api/v4/pages_domain/detail.json

@@ -3,6 +3,9 @@
   "properties": {
     "domain": { "type": "string" },
     "url": { "type": "uri" },
+    "verified": { "type": "boolean" },
+    "verification_code": { "type": ["string", "null"] },
+    "enabled_until": { "type": ["date", "null"] },
     "certificate": {
       "type": "object",
       "properties": {
@@ -15,6 +18,6 @@
       "additionalProperties": false
     }
   },
-  "required": ["domain", "url"],
+  "required": ["domain", "url", "verified", "verification_code", "enabled_until"],
   "additionalProperties": false
 }

spec/mailers/emails/pages_domains_spec.rb

+require 'spec_helper'
+require 'email_spec'
+
+describe Emails::PagesDomains do
+  include EmailSpec::Matchers
+  include_context 'gitlab email notification'
+
+  set(:project) { create(:project) }
+  set(:domain) { create(:pages_domain, project: project)  }
+  set(:user) { project.owner }
+
+  shared_examples 'a pages domain email' do
+    it_behaves_like 'an email sent from GitLab'
+    it_behaves_like 'it should not have Gmail Actions links'
+    it_behaves_like 'a user cannot unsubscribe through footer link'
+
+    it 'has the expected content' do
+      aggregate_failures do
+        is_expected.to have_subject(email_subject)
+        is_expected.to have_body_text(project.human_name)
+        is_expected.to have_body_text(domain.domain)
+        is_expected.to have_body_text domain.url
+        is_expected.to have_body_text project_pages_domain_url(project, domain)
+        is_expected.to have_body_text help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+      end
+    end
+  end
+
+  describe '#pages_domain_enabled_email' do
+    let(:email_subject) { "#{project.path} | GitLab Pages domain '#{domain.domain}' has been enabled" }
+
+    subject { Notify.pages_domain_enabled_email(domain, user) }
+
+    it_behaves_like 'a pages domain email'
+
+    it { is_expected.to have_body_text 'has been enabled' }
+  end
+
+  describe '#pages_domain_disabled_email' do
+    let(:email_subject) { "#{project.path} | GitLab Pages domain '#{domain.domain}' has been disabled" }
+
+    subject { Notify.pages_domain_disabled_email(domain, user) }
+
+    it_behaves_like 'a pages domain email'
+
+    it { is_expected.to have_body_text 'has been disabled' }
+  end
+
+  describe '#pages_domain_verification_succeeded_email' do
+    let(:email_subject) { "#{project.path} | Verification succeeded for GitLab Pages domain '#{domain.domain}'" }
+
+    subject { Notify.pages_domain_verification_succeeded_email(domain, user) }
+
+    it_behaves_like 'a pages domain email'
+
+    it { is_expected.to have_body_text 'successfully verified' }
+  end
+
+  describe '#pages_domain_verification_failed_email' do
+    let(:email_subject) { "#{project.path} | ACTION REQUIRED: Verification failed for GitLab Pages domain '#{domain.domain}'" }
+
+    subject { Notify.pages_domain_verification_failed_email(domain, user) }
+
+    it_behaves_like 'a pages domain email'
+
+    it 'says verification has failed and when the domain is enabled until' do
+      is_expected.to have_body_text 'Verification has failed'
+      is_expected.to have_body_text domain.enabled_until.strftime('%F %T')
+    end
+  end
+end

spec/migrations/enqueue_verify_pages_domain_workers_spec.rb

+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20180216121030_enqueue_verify_pages_domain_workers')
+
+describe EnqueueVerifyPagesDomainWorkers, :sidekiq, :migration do
+  around do |example|
+    Sidekiq::Testing.fake! do
+      example.run
+    end
+  end
+
+  describe '#up' do
+    it 'enqueues a verification worker for every domain' do
+      domains = 1.upto(3).map { |i| PagesDomain.create!(domain: "my#{i}.domain.com") }
+
+      expect { migrate! }.to change(PagesDomainVerificationWorker.jobs, :size).by(3)
+
+      enqueued_ids = PagesDomainVerificationWorker.jobs.map { |job| job['args'] }
+      expected_ids = domains.map { |domain| [domain.id] }
+
+      expect(enqueued_ids).to match_array(expected_ids)
+    end
+  end
+end

spec/models/pages_domain_spec.rb

 require 'spec_helper'
 
 describe PagesDomain do
+  using RSpec::Parameterized::TableSyntax
+
+  subject(:pages_domain) { described_class.new }
+
   describe 'associations' do
     it { is_expected.to belong_to(:project) }
   end
@@ -64,19 +68,51 @@ describe PagesDomain do
     end
   end
 
+  describe 'validations' do
+    it { is_expected.to validate_presence_of(:verification_code) }
+  end
+
+  describe '#verification_code' do
+    subject { pages_domain.verification_code }
+
+    it 'is set automatically with 128 bits of SecureRandom data' do
+      expect(SecureRandom).to receive(:hex).with(16) { 'verification code' }
+
+      is_expected.to eq('verification code')
+    end
+  end
+
+  describe '#keyed_verification_code' do
+    subject { pages_domain.keyed_verification_code }
+
+    it { is_expected.to eq("gitlab-pages-verification-code=#{pages_domain.verification_code}") }
+  end
+
+  describe '#verification_domain' do
+    subject { pages_domain.verification_domain }
+
+    it { is_expected.to be_nil }
+
+    it 'is a well-known subdomain if the domain is present' do
+      pages_domain.domain = 'example.com'
+
+      is_expected.to eq('_gitlab-pages-verification-code.example.com')
+    end
+  end
+
   describe '#url' do
     subject { domain.url }
 
     context 'without the certificate' do
       let(:domain) { build(:pages_domain, certificate: '') }
 
-      it { is_expected.to eq('http://my.domain.com') }
+      it { is_expected.to eq("http://#{domain.domain}") }
     end
 
     context 'with a certificate' do
       let(:domain) { build(:pages_domain, :with_certificate) }
 
-      it { is_expected.to eq('https://my.domain.com') }
+      it { is_expected.to eq("https://#{domain.domain}") }
     end
   end
 
@@ -154,4 +190,108 @@ describe PagesDomain do
     # We test only existence of output, since the output is long
     it { is_expected.not_to be_empty }
   end
+
+  describe '#update_daemon' do
+    it 'runs when the domain is created' do
+      domain = build(:pages_domain)
+
+      expect(domain).to receive(:update_daemon)
+
+      domain.save!
+    end
+
+    it 'runs when the domain is destroyed' do
+      domain = create(:pages_domain)
+
+      expect(domain).to receive(:update_daemon)
+
+      domain.destroy!
+    end
+
+    it 'delegates to Projects::UpdatePagesConfigurationService' do
+      service = instance_double('Projects::UpdatePagesConfigurationService')
+      expect(Projects::UpdatePagesConfigurationService).to receive(:new) { service }
+      expect(service).to receive(:execute)
+
+      create(:pages_domain)
+    end
+
+    context 'configuration updates when attributes change' do
+      set(:project1) { create(:project) }
+      set(:project2) { create(:project) }
+      set(:domain) { create(:pages_domain) }
+
+      where(:attribute, :old_value, :new_value, :update_expected) do
+        now = Time.now
+        future = now + 1.day
+
+        :project | nil       | :project1 | true
+        :project | :project1 | :project1 | false
+        :project | :project1 | :project2 | true
+        :project | :project1 | nil       | true
+
+        # domain can't be set to nil
+        :domain | 'a.com' | 'a.com' | false
+        :domain | 'a.com' | 'b.com' | true
+
+        # verification_code can't be set to nil
+        :verification_code | 'foo' | 'foo'  | false
+        :verification_code | 'foo' | 'bar'  | false
+
+        :verified_at | nil | now    | false
+        :verified_at | now | now    | false
+        :verified_at | now | future | false
+        :verified_at | now | nil    | false
+
+        :enabled_until | nil | now    | true
+        :enabled_until | now | now    | false
+        :enabled_until | now | future | false
+        :enabled_until | now | nil    | true
+      end
+
+      with_them do
+        it 'runs if a relevant attribute has changed' do
+          a = old_value.is_a?(Symbol) ? send(old_value) : old_value
+          b = new_value.is_a?(Symbol) ? send(new_value) : new_value
+
+          domain.update!(attribute => a)
+
+          if update_expected
+            expect(domain).to receive(:update_daemon)
+          else
+            expect(domain).not_to receive(:update_daemon)
+          end
+
+          domain.update!(attribute => b)
+        end
+      end
+
+      context 'TLS configuration' do
+        set(:domain_with_tls) { create(:pages_domain, :with_key, :with_certificate) }
+
+        let(:cert1) { domain_with_tls.certificate }
+        let(:cert2) { cert1 + ' ' }
+        let(:key1) { domain_with_tls.key }
+        let(:key2) { key1 + ' ' }
+
+        it 'updates when added' do
+          expect(domain).to receive(:update_daemon)
+
+          domain.update!(key: key1, certificate: cert1)
+        end
+
+        it 'updates when changed' do
+          expect(domain_with_tls).to receive(:update_daemon)
+
+          domain_with_tls.update!(key: key2, certificate: cert2)
+        end
+
+        it 'updates when removed' do
+          expect(domain_with_tls).to receive(:update_daemon)
+
+          domain_with_tls.update!(key: nil, certificate: nil)
+        end
+      end
+    end
+  end
 end

spec/services/notification_service_spec.rb

@@ -1678,6 +1678,78 @@ describe NotificationService, :mailer do
     end
   end
 
+  describe 'Pages domains' do
+    set(:project) { create(:project) }
+    set(:domain) { create(:pages_domain, project: project) }
+    set(:u_blocked) { create(:user, :blocked) }
+    set(:u_silence) { create_user_with_notification(:disabled, 'silent', project) }
+    set(:u_owner)   { project.owner }
+    set(:u_master1) { create(:user) }
+    set(:u_master2) { create(:user) }
+    set(:u_developer) { create(:user) }
+
+    before do
+      project.add_master(u_blocked)
+      project.add_master(u_silence)
+      project.add_master(u_master1)
+      project.add_master(u_master2)
+      project.add_developer(u_developer)
+
+      reset_delivered_emails!
+    end
+
+    %i[
+      pages_domain_enabled
+      pages_domain_disabled
+      pages_domain_verification_succeeded
+      pages_domain_verification_failed
+    ].each do |sym|
+      describe "##{sym}" do
+        subject(:notify!) { notification.send(sym, domain) }
+
+        it 'emails current watching masters' do
+          expect(Notify).to receive(:"#{sym}_email").at_least(:once).and_call_original
+
+          notify!
+
+          should_only_email(u_master1, u_master2, u_owner)
+        end
+
+        it 'emails nobody if the project is missing' do
+          domain.project = nil
+
+          notify!
+
+          should_not_email_anyone
+        end
+      end
+    end
+
+    describe '#pages_domain_verification_failed' do
+      it 'emails current watching masters' do
+        notification.pages_domain_verification_failed(domain)
+
+        should_only_email(u_master1, u_master2, u_owner)
+      end
+    end
+
+    describe '#pages_domain_enabled' do
+      it 'emails current watching masters' do
+        notification.pages_domain_enabled(domain)
+
+        should_only_email(u_master1, u_master2, u_owner)
+      end
+    end
+
+    describe '#pages_domain_disabled' do
+      it 'emails current watching masters' do
+        notification.pages_domain_disabled(domain)
+
+        should_only_email(u_master1, u_master2, u_owner)
+      end
+    end
+  end
+
   def build_team(project)
     @u_watcher               = create_global_setting_for(create(:user), :watch)
     @u_participating         = create_global_setting_for(create(:user), :participating)

spec/services/verify_pages_domain_service_spec.rb

+require 'spec_helper'
+
+describe VerifyPagesDomainService do
+  using RSpec::Parameterized::TableSyntax
+  include EmailHelpers
+
+  let(:error_status) { { status: :error, message: "Couldn't verify #{domain.domain}" } }
+
+  subject(:service) { described_class.new(domain) }
+
+  describe '#execute' do
+    context 'verification code recognition (verified domain)' do
+      where(:domain_sym, :code_sym) do
+        :domain | :verification_code
+        :domain | :keyed_verification_code
+
+        :verification_domain | :verification_code
+        :verification_domain | :keyed_verification_code
+      end
+
+      with_them do
+        set(:domain) { create(:pages_domain) }
+
+        let(:domain_name) { domain.send(domain_sym) }
+        let(:verification_code) { domain.send(code_sym) }
+
+        it 'verifies and enables the domain' do
+          stub_resolver(domain_name => ['something else', verification_code])
+
+          expect(service.execute).to eq(status: :success)
+          expect(domain).to be_verified
+          expect(domain).to be_enabled
+        end
+
+        it 'verifies and enables when the code is contained partway through a TXT record' do
+          stub_resolver(domain_name => "something #{verification_code} else")
+
+          expect(service.execute).to eq(status: :success)
+          expect(domain).to be_verified
+          expect(domain).to be_enabled
+        end
+
+        it 'does not verify when the code is not present' do
+          stub_resolver(domain_name => 'something else')
+
+          expect(service.execute).to eq(error_status)
+
+          expect(domain).not_to be_verified
+          expect(domain).to be_enabled
+        end
+      end
+
+      context 'verified domain' do
+        set(:domain) { create(:pages_domain) }
+
+        it 'unverifies (but does not disable) when the right code is not present' do
+          stub_resolver(domain.domain => 'something else')
+
+          expect(service.execute).to eq(error_status)
+          expect(domain).not_to be_verified
+          expect(domain).to be_enabled
+        end
+
+        it 'unverifies (but does not disable) when no records are present' do
+          stub_resolver
+
+          expect(service.execute).to eq(error_status)
+          expect(domain).not_to be_verified
+          expect(domain).to be_enabled
+        end
+      end
+
+      context 'expired domain' do
+        set(:domain) { create(:pages_domain, :expired) }
+
+        it 'verifies and enables when the right code is present' do
+          stub_resolver(domain.domain => domain.keyed_verification_code)
+
+          expect(service.execute).to eq(status: :success)
+
+          expect(domain).to be_verified
+          expect(domain).to be_enabled
+        end
+
+        it 'disables when the right code is not present' do
+          error_status[:message] += '. It is now disabled.'
+
+          stub_resolver
+
+          expect(service.execute).to eq(error_status)
+
+          expect(domain).not_to be_verified
+          expect(domain).not_to be_enabled
+        end
+      end
+    end
+
+    context 'timeout behaviour' do
+      let(:domain) { create(:pages_domain) }
+
+      it 'sets a timeout on the DNS query' do
+        expect(stub_resolver).to receive(:timeouts=).with(described_class::RESOLVER_TIMEOUT_SECONDS)
+
+        service.execute
+      end
+    end
+
+    context 'email notifications' do
+      let(:notification_service) { instance_double('NotificationService') }
+
+      where(:factory, :verification_succeeds, :expected_notification) do
+        nil         | true  | nil
+        nil         | false | :verification_failed
+        :reverify   | true  | nil
+        :reverify   | false | :verification_failed
+        :unverified | true  | :verification_succeeded
+        :unverified | false | nil
+        :expired    | true  | nil
+        :expired    | false | :disabled
+        :disabled   | true  | :enabled
+        :disabled   | false | nil
+      end
+
+      with_them do
+        let(:domain) { create(:pages_domain, *[factory].compact) }
+
+        before do
+          allow(service).to receive(:notification_service) { notification_service }
+
+          if verification_succeeds
+            stub_resolver(domain.domain => domain.verification_code)
+          else
+            stub_resolver
+          end
+        end
+
+        it 'sends a notification if appropriate' do
+          if expected_notification
+            expect(notification_service).to receive(:"pages_domain_#{expected_notification}").with(domain)
+          end
+
+          service.execute
+        end
+      end
+
+      context 'pages verification disabled' do
+        let(:domain) { create(:pages_domain, :disabled) }
+
+        before do
+          stub_application_setting(pages_domain_verification_enabled: false)
+          allow(service).to receive(:notification_service) { notification_service }
+        end
+
+        it 'skips email notifications' do
+          expect(notification_service).not_to receive(:pages_domain_enabled)
+
+          service.execute
+        end
+      end
+    end
+
+    context 'pages configuration updates' do
+      context 'enabling a disabled domain' do
+        let(:domain) { create(:pages_domain, :disabled) }
+
+        it 'schedules an update' do
+          stub_resolver(domain.domain => domain.verification_code)
+
+          expect(domain).to receive(:update_daemon)
+
+          service.execute
+        end
+      end
+
+      context 'verifying an enabled domain' do
+        let(:domain) { create(:pages_domain) }
+
+        it 'schedules an update' do
+          stub_resolver(domain.domain => domain.verification_code)
+
+          expect(domain).not_to receive(:update_daemon)
+
+          service.execute
+        end
+      end
+
+      context 'disabling an expired domain' do
+        let(:domain) { create(:pages_domain, :expired) }
+
+        it 'schedules an update' do
+          stub_resolver
+
+          expect(domain).to receive(:update_daemon)
+
+          service.execute
+        end
+      end
+
+      context 'failing to verify a disabled domain' do
+        let(:domain) { create(:pages_domain, :disabled) }
+
+        it 'does not schedule an update' do
+          stub_resolver
+
+          expect(domain).not_to receive(:update_daemon)
+
+          service.execute
+        end
+      end
+    end
+
+    context 'no verification code' do
+      let(:domain) { create(:pages_domain) }
+
+      it 'returns an error' do
+        domain.verification_code = ''
+
+        disallow_resolver!
+
+        expect(service.execute).to eq(status: :error, message: "No verification code set for #{domain.domain}")
+      end
+    end
+
+    context 'pages domain verification is disabled' do
+      let(:domain) { create(:pages_domain, :disabled) }
+
+      before do
+        stub_application_setting(pages_domain_verification_enabled: false)
+      end
+
+      it 'extends domain validity by unconditionally reverifying' do
+        disallow_resolver!
+
+        service.execute
+
+        expect(domain).to be_verified
+        expect(domain).to be_enabled
+      end
+
+      it 'does not shorten any grace period' do
+        grace = Time.now + 1.year
+        domain.update!(enabled_until: grace)
+        disallow_resolver!
+
+        service.execute
+
+        expect(domain.enabled_until).to be_like_time(grace)
+      end
+    end
+  end
+
+  def disallow_resolver!
+    expect(Resolv::DNS).not_to receive(:open)
+  end
+
+  def stub_resolver(stubbed_lookups = {})
+    resolver = instance_double('Resolv::DNS')
+    allow(resolver).to receive(:timeouts=)
+
+    expect(Resolv::DNS).to receive(:open).and_yield(resolver)
+
+    allow(resolver).to receive(:getresources) { [] }
+    stubbed_lookups.each do |domain, records|
+      records = Array(records).map { |txt| Resolv::DNS::Resource::IN::TXT.new(txt) }
+      allow(resolver).to receive(:getresources).with(domain, Resolv::DNS::Resource::IN::TXT) { records }
+    end
+
+    resolver
+  end
+end

spec/workers/pages_domain_verification_cron_worker_spec.rb

+require 'spec_helper'
+
+describe PagesDomainVerificationCronWorker do
+  subject(:worker) { described_class.new }
+
+  describe '#perform' do
+    it 'enqueues a PagesDomainVerificationWorker for domains needing verification' do
+      verified = create(:pages_domain)
+      reverify = create(:pages_domain, :reverify)
+      disabled = create(:pages_domain, :disabled)
+
+      [reverify, disabled].each do |domain|
+        expect(PagesDomainVerificationWorker).to receive(:perform_async).with(domain.id)
+      end
+
+      expect(PagesDomainVerificationWorker).not_to receive(:perform_async).with(verified.id)
+
+      worker.perform
+    end
+  end
+end

spec/workers/pages_domain_verification_worker_spec.rb

+require 'spec_helper'
+
+describe PagesDomainVerificationWorker do
+  subject(:worker) { described_class.new }
+
+  let(:domain) { create(:pages_domain) }
+
+  describe '#perform' do
+    it 'does nothing for a non-existent domain' do
+      domain.destroy
+
+      expect(VerifyPagesDomainService).not_to receive(:new)
+
+      expect { worker.perform(domain.id) }.not_to raise_error
+    end
+
+    it 'delegates to VerifyPagesDomainService' do
+      service = double(:service)
+      expected_domain = satisfy { |obj| obj == domain }
+
+      expect(VerifyPagesDomainService).to receive(:new).with(expected_domain) { service }
+      expect(service).to receive(:execute)
+
+      worker.perform(domain.id)
+    end
+  end
+end