Merge branch 'pages-domains-certificates-auto-renew-mysql' into 'master'

Renew pages domain's certificates through Let's Encrypt

See merge request gitlab-org/gitlab-ce!29363

app/models/pages_domain.rb

@@ -3,6 +3,7 @@
 class PagesDomain < ApplicationRecord
   VERIFICATION_KEY = 'gitlab-pages-verification-code'.freeze
   VERIFICATION_THRESHOLD = 3.days.freeze
+  SSL_RENEWAL_THRESHOLD = 30.days.freeze
 
   enum certificate_source: { user_provided: 0, gitlab_provided: 1 }, _prefix: :certificate
 
@@ -41,6 +42,15 @@ class PagesDomain < ApplicationRecord
     where(verified_at.eq(nil).or(enabled_until.eq(nil).or(enabled_until.lt(threshold))))
   end
 
+  scope :need_auto_ssl_renewal, -> do
+    expiring = where(certificate_valid_not_after: nil).or(
+      where(arel_table[:certificate_valid_not_after].lt(SSL_RENEWAL_THRESHOLD.from_now)))
+
+    user_provided_or_expiring = certificate_user_provided.or(expiring)
+
+    where(auto_ssl_enabled: true).merge(user_provided_or_expiring)
+  end
+
   scope :for_removal, -> { where("remove_at < ?", Time.now) }
 
   def verified?

app/workers/all_queues.yml

@@ -9,6 +9,7 @@
 - cronjob:import_export_project_cleanup
 - cronjob:pages_domain_verification_cron
 - cronjob:pages_domain_removal_cron
+- cronjob:pages_domain_ssl_renewal_cron
 - cronjob:pipeline_schedule
 - cronjob:prune_old_events
 - cronjob:remove_expired_group_links
@@ -133,6 +134,7 @@
 - new_note
 - pages
 - pages_domain_verification
+- pages_domain_ssl_renewal
 - plugin
 - post_receive
 - process_commit

app/workers/pages_domain_ssl_renewal_cron_worker.rb

+# frozen_string_literal: true
+
+class PagesDomainSslRenewalCronWorker
+  include ApplicationWorker
+  include CronjobQueue
+
+  def perform
+    return unless ::Gitlab::LetsEncrypt::Client.new.enabled?
+
+    PagesDomain.need_auto_ssl_renewal.find_each do |domain|
+      PagesDomainSslRenewalWorker.perform_async(domain.id)
+    end
+  end
+end

app/workers/pages_domain_ssl_renewal_worker.rb

+# frozen_string_literal: true
+
+class PagesDomainSslRenewalWorker
+  include ApplicationWorker
+
+  def perform(domain_id)
+    return unless ::Gitlab::LetsEncrypt::Client.new.enabled?
+
+    domain = PagesDomain.find_by_id(domain_id)
+
+    return unless domain
+
+    ::PagesDomains::ObtainLetsEncryptCertificateService.new(domain).execute
+  end
+end

config/initializers/1_settings.rb

@@ -367,6 +367,10 @@ Settings.cron_jobs['pages_domain_removal_cron_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['pages_domain_removal_cron_worker']['cron'] ||= '47 0 * * *'
 Settings.cron_jobs['pages_domain_removal_cron_worker']['job_class'] = 'PagesDomainRemovalCronWorker'
 
+Settings.cron_jobs['pages_domain_ssl_renewal_cron_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['pages_domain_ssl_renewal_cron_worker']['cron'] ||= '*/5 * * * *'
+Settings.cron_jobs['pages_domain_ssl_renewal_cron_worker']['job_class'] = 'PagesDomainSslRenewalCronWorker'
+
 Settings.cron_jobs['issue_due_scheduler_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['issue_due_scheduler_worker']['cron'] ||= '50 00 * * *'
 Settings.cron_jobs['issue_due_scheduler_worker']['job_class'] = 'IssueDueSchedulerWorker'

config/sidekiq_queues.yml

@@ -72,6 +72,7 @@
   - [project_rollback_hashed_storage, 1]
   - [hashed_storage, 1]
   - [pages_domain_verification, 1]
+  - [pages_domain_ssl_renewal, 1]
   - [object_storage_upload, 1]
   - [object_storage, 1]
   - [plugin, 1]

db/migrate/20190607145325_add_pages_domains_ssl_renew_index.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddPagesDomainsSslRenewIndex < ActiveRecord::Migration[5.1]
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  INDEX_NAME = 'index_pages_domains_need_auto_ssl_renewal'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index(:pages_domains, [:certificate_source, :certificate_valid_not_after],
+                         where: "auto_ssl_enabled = #{::Gitlab::Database.true_value}", name: INDEX_NAME)
+  end
+
+  def down
+    remove_concurrent_index(:pages_domains, [:certificate_source, :certificate_valid_not_after],
+                            where: "auto_ssl_enabled = #{::Gitlab::Database.true_value}", name: INDEX_NAME)
+  end
+end

db/schema.rb

@@ -2334,6 +2334,7 @@ ActiveRecord::Schema.define(version: 20190620112608) do
     t.datetime_with_timezone "certificate_valid_not_before"
     t.datetime_with_timezone "certificate_valid_not_after"
     t.integer "certificate_source", limit: 2, default: 0, null: false
+    t.index ["certificate_source", "certificate_valid_not_after"], name: "index_pages_domains_need_auto_ssl_renewal", where: "(auto_ssl_enabled = true)", using: :btree
     t.index ["domain"], name: "index_pages_domains_on_domain", unique: true, using: :btree
     t.index ["project_id", "enabled_until"], name: "index_pages_domains_on_project_id_and_enabled_until", using: :btree
     t.index ["project_id"], name: "index_pages_domains_on_project_id", using: :btree

spec/factories/pages_domains.rb

@@ -182,6 +182,7 @@ ZDXgrA==
     end
 
     trait :letsencrypt do
+      auto_ssl_enabled { true }
       certificate_source { :gitlab_provided }
     end
   end

spec/models/pages_domain_spec.rb

@@ -479,4 +479,30 @@ describe PagesDomain do
       end
     end
   end
+
+  describe '.need_auto_ssl_renewal' do
+    subject { described_class.need_auto_ssl_renewal }
+
+    let!(:domain_with_user_provided_certificate) { create(:pages_domain) }
+    let!(:domain_with_expired_user_provided_certificate) do
+      create(:pages_domain, :with_expired_certificate)
+    end
+    let!(:domain_with_user_provided_certificate_and_auto_ssl) do
+      create(:pages_domain, auto_ssl_enabled: true)
+    end
+
+    let!(:domain_with_gitlab_provided_certificate) { create(:pages_domain, :letsencrypt) }
+    let!(:domain_with_expired_gitlab_provided_certificate) do
+      create(:pages_domain, :letsencrypt, :with_expired_certificate)
+    end
+
+    it 'contains only domains needing verification' do
+      is_expected.to(
+        contain_exactly(
+          domain_with_user_provided_certificate_and_auto_ssl,
+          domain_with_expired_gitlab_provided_certificate
+        )
+      )
+    end
+  end
 end

spec/workers/pages_domain_ssl_renewal_cron_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe PagesDomainSslRenewalCronWorker do
+  include LetsEncryptHelpers
+
+  subject(:worker) { described_class.new }
+
+  before do
+    stub_lets_encrypt_settings
+  end
+
+  describe '#perform' do
+    let!(:domain) { create(:pages_domain) }
+    let!(:domain_with_enabled_auto_ssl) { create(:pages_domain, auto_ssl_enabled: true) }
+    let!(:domain_with_obtained_letsencrypt) { create(:pages_domain, :letsencrypt, auto_ssl_enabled: true) }
+    let!(:domain_without_auto_certificate) do
+      create(:pages_domain, :without_certificate, :without_key, auto_ssl_enabled: true)
+    end
+
+    let!(:domain_with_expired_auto_ssl) do
+      create(:pages_domain, :letsencrypt, :with_expired_certificate)
+    end
+
+    it 'enqueues a PagesDomainSslRenewalWorker for domains needing renewal' do
+      [domain_without_auto_certificate,
+       domain_with_enabled_auto_ssl,
+       domain_with_expired_auto_ssl].each do |domain|
+        expect(PagesDomainSslRenewalWorker).to receive(:perform_async).with(domain.id)
+      end
+
+      [domain,
+       domain_with_obtained_letsencrypt].each do |domain|
+        expect(PagesDomainVerificationWorker).not_to receive(:perform_async).with(domain.id)
+      end
+
+      worker.perform
+    end
+
+    shared_examples 'does nothing' do
+      it 'does nothing' do
+        expect(PagesDomainSslRenewalWorker).not_to receive(:perform_async)
+
+        worker.perform
+      end
+    end
+
+    context 'when letsencrypt integration is disabled' do
+      before do
+        stub_application_setting(
+          lets_encrypt_terms_of_service_accepted: false
+        )
+      end
+
+      include_examples 'does nothing'
+    end
+  end
+end

spec/workers/pages_domain_ssl_renewal_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe PagesDomainSslRenewalWorker do
+  include LetsEncryptHelpers
+
+  subject(:worker) { described_class.new }
+
+  let(:domain) { create(:pages_domain) }
+
+  before do
+    stub_lets_encrypt_settings
+  end
+
+  describe '#perform' do
+    it 'delegates to ObtainLetsEncryptCertificateService' do
+      service = double(:service)
+      expect(::PagesDomains::ObtainLetsEncryptCertificateService).to receive(:new).with(domain).and_return(service)
+      expect(service).to receive(:execute)
+
+      worker.perform(domain.id)
+    end
+
+    context 'when domain was deleted' do
+      before do
+        domain.destroy!
+      end
+
+      it 'does nothing' do
+        expect(::PagesDomains::ObtainLetsEncryptCertificateService).not_to receive(:new)
+      end
+    end
+  end
+end