Merge branch 'pages_lets_encrypt_orders' into 'master'

Add pages domains acme orders

See merge request gitlab-org/gitlab-ce!27811

app/controllers/acme_challenges_controller.rb

+# frozen_string_literal: true
+
+class AcmeChallengesController < ActionController::Base
+  def show
+    if acme_order
+      render plain: acme_order.challenge_file_content, content_type: 'text/plain'
+    else
+      head :not_found
+    end
+  end
+
+  private
+
+  def acme_order
+    @acme_order ||= PagesDomainAcmeOrder.find_by_domain_and_token(params[:domain], params[:token])
+  end
+end

app/models/pages_domain.rb

@@ -5,6 +5,7 @@ class PagesDomain < ApplicationRecord
   VERIFICATION_THRESHOLD = 3.days.freeze
 
   belongs_to :project
+  has_many :acme_orders, class_name: "PagesDomainAcmeOrder"
 
   validates :domain, hostname: { allow_numeric_hostname: true }
   validates :domain, uniqueness: { case_sensitive: false }

app/models/pages_domain_acme_order.rb

+# frozen_string_literal: true
+
+class PagesDomainAcmeOrder < ApplicationRecord
+  belongs_to :pages_domain
+
+  scope :expired, -> { where("expires_at < ?", Time.now) }
+
+  validates :pages_domain, presence: true
+  validates :expires_at, presence: true
+  validates :url, presence: true
+  validates :challenge_token, presence: true
+  validates :challenge_file_content, presence: true
+  validates :private_key, presence: true
+
+  attr_encrypted :private_key,
+                 mode: :per_attribute_iv,
+                 key: Settings.attr_encrypted_db_key_base_truncated,
+                 algorithm: 'aes-256-gcm',
+                 encode: true
+
+  def self.find_by_domain_and_token(domain_name, challenge_token)
+    joins(:pages_domain).find_by(pages_domains: { domain: domain_name }, challenge_token: challenge_token)
+  end
+end

app/services/pages_domains/create_acme_order_service.rb

+# frozen_string_literal: true
+
+module PagesDomains
+  class CreateAcmeOrderService
+    attr_reader :pages_domain
+
+    def initialize(pages_domain)
+      @pages_domain = pages_domain
+    end
+
+    def execute
+      lets_encrypt_client = Gitlab::LetsEncrypt::Client.new
+      order = lets_encrypt_client.new_order(pages_domain.domain)
+
+      challenge = order.new_challenge
+
+      private_key = OpenSSL::PKey::RSA.new(4096)
+      saved_order = pages_domain.acme_orders.create!(
+        url: order.url,
+        expires_at: order.expires,
+        private_key: private_key.to_pem,
+
+        challenge_token: challenge.token,
+        challenge_file_content: challenge.file_content
+      )
+
+      challenge.request_validation
+      saved_order
+    end
+  end
+end

app/services/pages_domains/obtain_lets_encrypt_certificate_service.rb

+# frozen_string_literal: true
+
+module PagesDomains
+  class ObtainLetsEncryptCertificateService
+    attr_reader :pages_domain
+
+    def initialize(pages_domain)
+      @pages_domain = pages_domain
+    end
+
+    def execute
+      pages_domain.acme_orders.expired.delete_all
+      acme_order = pages_domain.acme_orders.first
+
+      unless acme_order
+        ::PagesDomains::CreateAcmeOrderService.new(pages_domain).execute
+        return
+      end
+
+      api_order = ::Gitlab::LetsEncrypt::Client.new.load_order(acme_order.url)
+
+      # https://tools.ietf.org/html/rfc8555#section-7.1.6 - statuses diagram
+      case api_order.status
+      when 'ready'
+        api_order.request_certificate(private_key: acme_order.private_key, domain: pages_domain.domain)
+      when 'valid'
+        save_certificate(acme_order.private_key, api_order)
+        acme_order.destroy!
+        # when 'invalid'
+        # TODO: implement error handling
+      end
+    end
+
+    private
+
+    def save_certificate(private_key, api_order)
+      certificate = api_order.certificate
+      pages_domain.update!(key: private_key, certificate: certificate)
+    end
+  end
+end

config/routes.rb

@@ -75,6 +75,8 @@ Rails.application.routes.draw do
       resources :issues, module: :boards, only: [:index, :update]
     end
 
+    get 'acme-challenge/' => 'acme_challenges#show'
+
     # UserCallouts
     resources :user_callouts, only: [:create]
 

db/migrate/20190429082448_create_pages_domain_acme_orders.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CreatePagesDomainAcmeOrders < ActiveRecord::Migration[5.1]
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def change
+    create_table :pages_domain_acme_orders do |t|
+      t.references :pages_domain, null: false, index: true, foreign_key: { on_delete: :cascade }, type: :integer
+
+      t.datetime_with_timezone :expires_at, null: false
+      t.timestamps_with_timezone null: false
+
+      t.string :url, null: false
+
+      t.string :challenge_token, null: false, index: true
+      t.text :challenge_file_content, null: false
+
+      t.text :encrypted_private_key, null: false
+      t.text :encrypted_private_key_iv, null: false
+    end
+  end
+end

db/schema.rb

@@ -1571,6 +1571,20 @@ ActiveRecord::Schema.define(version: 20190530154715) do
     t.index ["access_grant_id"], name: "index_oauth_openid_requests_on_access_grant_id", using: :btree
   end
 
+  create_table "pages_domain_acme_orders", force: :cascade do |t|
+    t.integer "pages_domain_id", null: false
+    t.datetime_with_timezone "expires_at", null: false
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.string "url", null: false
+    t.string "challenge_token", null: false
+    t.text "challenge_file_content", null: false
+    t.text "encrypted_private_key", null: false
+    t.text "encrypted_private_key_iv", null: false
+    t.index ["challenge_token"], name: "index_pages_domain_acme_orders_on_challenge_token", using: :btree
+    t.index ["pages_domain_id"], name: "index_pages_domain_acme_orders_on_pages_domain_id", using: :btree
+  end
+
   create_table "pages_domains", id: :serial, force: :cascade do |t|
     t.integer "project_id"
     t.text "certificate"
@@ -2560,6 +2574,7 @@ ActiveRecord::Schema.define(version: 20190530154715) do
   add_foreign_key "notes", "projects", name: "fk_99e097b079", on_delete: :cascade
   add_foreign_key "notification_settings", "users", name: "fk_0c95e91db7", on_delete: :cascade
   add_foreign_key "oauth_openid_requests", "oauth_access_grants", column: "access_grant_id", name: "fk_oauth_openid_requests_oauth_access_grants_access_grant_id"
+  add_foreign_key "pages_domain_acme_orders", "pages_domains", on_delete: :cascade
   add_foreign_key "pages_domains", "projects", name: "fk_ea2f6dfc6f", on_delete: :cascade
   add_foreign_key "personal_access_tokens", "users"
   add_foreign_key "pool_repositories", "projects", column: "source_project_id", on_delete: :nullify

lib/gitlab/lets_encrypt/challenge.rb

@@ -7,7 +7,7 @@ module Gitlab
         @acme_challenge = acme_challenge
       end
 
-      delegate :url, :token, :file_content, :status, :request_validation, to: :acme_challenge
+      delegate :token, :file_content, :status, :request_validation, to: :acme_challenge
 
       private
 

lib/gitlab/lets_encrypt/order.rb

@@ -13,7 +13,16 @@ module Gitlab
         ::Gitlab::LetsEncrypt::Challenge.new(challenge)
       end
 
-      delegate :url, :status, to: :acme_order
+      def request_certificate(domain:, private_key:)
+        csr = ::Acme::Client::CertificateRequest.new(
+          private_key: OpenSSL::PKey.read(private_key),
+          subject: { common_name: domain }
+        )
+
+        acme_order.finalize(csr: csr)
+      end
+
+      delegate :url, :status, :expires, :certificate, to: :acme_order
 
       private
 

spec/controllers/acme_challenges_controller_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe AcmeChallengesController do
+  describe '#show' do
+    let!(:acme_order) { create(:pages_domain_acme_order) }
+
+    def make_request(domain, token)
+      get(:show, params: { domain: domain, token: token })
+    end
+
+    before do
+      make_request(domain, token)
+    end
+
+    context 'with right domain and token' do
+      let(:domain) { acme_order.pages_domain.domain }
+      let(:token) { acme_order.challenge_token }
+
+      it 'renders acme challenge file content' do
+        expect(response.body).to eq(acme_order.challenge_file_content)
+      end
+    end
+
+    context 'when domain is invalid' do
+      let(:domain) { 'somewrongdomain.com' }
+      let(:token) { acme_order.challenge_token }
+
+      it 'renders not found' do
+        expect(response).to have_gitlab_http_status(404)
+      end
+    end
+
+    context 'when token is invalid' do
+      let(:domain) { acme_order.pages_domain.domain }
+      let(:token) { 'wrongtoken' }
+
+      it 'renders not found' do
+        expect(response).to have_gitlab_http_status(404)
+      end
+    end
+  end
+end

spec/factories/pages_domain_acme_orders.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :pages_domain_acme_order do
+    pages_domain
+    url { 'https://example.com/' }
+    expires_at { 1.day.from_now }
+    challenge_token { 'challenge_token' }
+    challenge_file_content { 'filecontent' }
+
+    private_key { OpenSSL::PKey::RSA.new(4096).to_pem }
+
+    trait :expired do
+      expires_at { 1.day.ago }
+    end
+  end
+end

spec/lib/gitlab/lets_encrypt/challenge_spec.rb

@@ -3,23 +3,11 @@
 require 'spec_helper'
 
 describe ::Gitlab::LetsEncrypt::Challenge do
-  delegated_methods = {
-    url: 'https://example.com/',
-    status: 'pending',
-    token: 'tokenvalue',
-    file_content: 'hereisfilecontent',
-    request_validation: true
-  }
+  include LetsEncryptHelpers
 
-  let(:acme_challenge) do
-    acme_challenge = instance_double('Acme::Client::Resources::Challenge')
-    allow(acme_challenge).to receive_messages(delegated_methods)
-    acme_challenge
-  end
-
-  let(:challenge) { described_class.new(acme_challenge) }
+  let(:challenge) { described_class.new(acme_challenge_double) }
 
-  delegated_methods.each do |method, value|
+  LetsEncryptHelpers::ACME_CHALLENGE_METHODS.each do |method, value|
     describe "##{method}" do
       it 'delegates to Acme::Client::Resources::Challenge' do
         expect(challenge.public_send(method)).to eq(value)

spec/lib/gitlab/lets_encrypt/order_spec.rb

@@ -3,20 +3,13 @@
 require 'spec_helper'
 
 describe ::Gitlab::LetsEncrypt::Order do
-  delegated_methods = {
-    url: 'https://example.com/',
-    status: 'valid'
-  }
+  include LetsEncryptHelpers
 
-  let(:acme_order) do
-    acme_order = instance_double('Acme::Client::Resources::Order')
-    allow(acme_order).to receive_messages(delegated_methods)
-    acme_order
-  end
+  let(:acme_order) { acme_order_double }
 
   let(:order) { described_class.new(acme_order) }
 
-  delegated_methods.each do |method, value|
+  LetsEncryptHelpers::ACME_ORDER_METHODS.each do |method, value|
     describe "##{method}" do
       it 'delegates to Acme::Client::Resources::Order' do
         expect(order.public_send(method)).to eq(value)
@@ -25,15 +18,24 @@ describe ::Gitlab::LetsEncrypt::Order do
   end
 
   describe '#new_challenge' do
-    before do
-      challenge = instance_double('Acme::Client::Resources::Challenges::HTTP01')
-      authorization = instance_double('Acme::Client::Resources::Authorization')
-      allow(authorization).to receive(:http).and_return(challenge)
-      allow(acme_order).to receive(:authorizations).and_return([authorization])
-    end
-
     it 'returns challenge' do
       expect(order.new_challenge).to be_a(::Gitlab::LetsEncrypt::Challenge)
     end
   end
+
+  describe '#request_certificate' do
+    let(:private_key) do
+      OpenSSL::PKey::RSA.new(4096).to_pem
+    end
+
+    it 'generates csr and finalizes order' do
+      expect(acme_order).to receive(:finalize) do |csr:|
+        expect do
+          csr.csr # it's being evaluated lazily
+        end.not_to raise_error
+      end
+
+      order.request_certificate(domain: 'example.com', private_key: private_key)
+    end
+  end
 end

spec/models/pages_domain_acme_order_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe PagesDomainAcmeOrder do
+  using RSpec::Parameterized::TableSyntax
+
+  describe '.expired' do
+    let!(:not_expired_order) { create(:pages_domain_acme_order) }
+    let!(:expired_order) { create(:pages_domain_acme_order, :expired) }
+
+    it 'returns only expired orders' do
+      expect(described_class.count).to eq(2)
+      expect(described_class.expired).to eq([expired_order])
+    end
+  end
+
+  describe '.find_by_domain_and_token' do
+    let!(:domain) { create(:pages_domain, domain: 'test.com') }
+    let!(:acme_order) { create(:pages_domain_acme_order, challenge_token: 'righttoken', pages_domain: domain) }
+
+    where(:domain_name, :challenge_token, :present) do
+      'test.com' | 'righttoken' | true
+      'test.com' | 'wrongtoken' | false
+      'test.org' | 'righttoken' | false
+    end
+
+    with_them do
+      subject { described_class.find_by_domain_and_token(domain_name, challenge_token).present? }
+
+      it { is_expected.to eq(present) }
+    end
+  end
+
+  subject { create(:pages_domain_acme_order) }
+
+  describe 'associations' do
+    it { is_expected.to belong_to(:pages_domain) }
+  end
+
+  describe 'validations' do
+    it { is_expected.to validate_presence_of(:pages_domain) }
+    it { is_expected.to validate_presence_of(:expires_at) }
+    it { is_expected.to validate_presence_of(:url) }
+    it { is_expected.to validate_presence_of(:challenge_token) }
+    it { is_expected.to validate_presence_of(:challenge_file_content) }
+    it { is_expected.to validate_presence_of(:private_key) }
+  end
+end

spec/services/pages_domains/create_acme_order_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe PagesDomains::CreateAcmeOrderService do
+  include LetsEncryptHelpers
+
+  let(:pages_domain) { create(:pages_domain) }
+
+  let(:challenge) { ::Gitlab::LetsEncrypt::Challenge.new(acme_challenge_double) }
+
+  let(:order_double) do
+    Gitlab::LetsEncrypt::Order.new(acme_order_double).tap do |order|
+      allow(order).to receive(:new_challenge).and_return(challenge)
+    end
+  end
+
+  let(:lets_encrypt_client) do
+    instance_double('Gitlab::LetsEncrypt::Client').tap do |client|
+      allow(client).to receive(:new_order).with(pages_domain.domain)
+        .and_return(order_double)
+    end
+  end
+
+  let(:service) { described_class.new(pages_domain) }
+
+  before do
+    allow(::Gitlab::LetsEncrypt::Client).to receive(:new).and_return(lets_encrypt_client)
+  end
+
+  it 'saves order to database before requesting validation' do
+    allow(pages_domain.acme_orders).to receive(:create!).and_call_original
+    allow(challenge).to receive(:request_validation).and_call_original
+
+    service.execute
+
+    expect(pages_domain.acme_orders).to have_received(:create!).ordered
+    expect(challenge).to have_received(:request_validation).ordered
+  end
+
+  it 'generates and saves private key' do
+    service.execute
+
+    saved_order = PagesDomainAcmeOrder.last
+    expect { OpenSSL::PKey::RSA.new(saved_order.private_key) }.not_to raise_error
+  end
+
+  it 'properly saves order attributes' do
+    service.execute
+
+    saved_order = PagesDomainAcmeOrder.last
+    expect(saved_order.url).to eq(order_double.url)
+    expect(saved_order.expires_at).to be_like_time(order_double.expires)
+  end
+
+  it 'properly saves challenge attributes' do
+    service.execute
+
+    saved_order = PagesDomainAcmeOrder.last
+    expect(saved_order.challenge_token).to eq(challenge.token)
+    expect(saved_order.challenge_file_content).to eq(challenge.file_content)
+  end
+end

spec/services/pages_domains/obtain_lets_encrypt_certificate_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe PagesDomains::ObtainLetsEncryptCertificateService do
+  include LetsEncryptHelpers
+
+  let(:pages_domain) { create(:pages_domain, :without_certificate, :without_key) }
+  let(:service) { described_class.new(pages_domain) }
+
+  before do
+    stub_lets_encrypt_settings
+  end
+
+  def expect_to_create_acme_challenge
+    expect(::PagesDomains::CreateAcmeOrderService).to receive(:new).with(pages_domain)
+      .and_wrap_original do |m, *args|
+      create_service = m.call(*args)
+
+      expect(create_service).to receive(:execute)
+
+      create_service
+    end
+  end
+
+  def stub_lets_encrypt_order(url, status)
+    order = ::Gitlab::LetsEncrypt::Order.new(acme_order_double(status: status))
+
+    allow_any_instance_of(::Gitlab::LetsEncrypt::Client).to(
+      receive(:load_order).with(url).and_return(order)
+    )
+
+    order
+  end
+
+  context 'when there is no acme order' do
+    it 'creates acme order' do
+      expect_to_create_acme_challenge
+
+      service.execute
+    end
+  end
+
+  context 'when there is expired acme order' do
+    let!(:existing_order) do
+      create(:pages_domain_acme_order, :expired, pages_domain: pages_domain)
+    end
+
+    it 'removes acme order and creates new one' do
+      expect_to_create_acme_challenge
+
+      service.execute
+
+      expect(PagesDomainAcmeOrder.find_by_id(existing_order.id)).to be_nil
+    end
+  end
+
+  %w(pending processing).each do |status|
+    context "there is an order in '#{status}' status" do
+      let(:existing_order) do
+        create(:pages_domain_acme_order, pages_domain: pages_domain)
+      end
+
+      before do
+        stub_lets_encrypt_order(existing_order.url, status)
+      end
+
+      it 'does not raise errors' do
+        expect do
+          service.execute
+        end.not_to raise_error
+      end
+    end
+  end
+
+  context 'when order is ready' do
+    let(:existing_order) do
+      create(:pages_domain_acme_order, pages_domain: pages_domain)
+    end
+
+    let!(:api_order) do
+      stub_lets_encrypt_order(existing_order.url, 'ready')
+    end
+
+    it 'request certificate' do
+      expect(api_order).to receive(:request_certificate).and_call_original
+
+      service.execute
+    end
+  end
+
+  context 'when order is valid' do
+    let(:existing_order) do
+      create(:pages_domain_acme_order, pages_domain: pages_domain)
+    end
+
+    let!(:api_order) do
+      stub_lets_encrypt_order(existing_order.url, 'valid')
+    end
+
+    let(:certificate) do
+      key = OpenSSL::PKey.read(existing_order.private_key)
+
+      subject = "/C=BE/O=Test/OU=Test/CN=#{pages_domain.domain}"
+
+      cert = OpenSSL::X509::Certificate.new
+      cert.subject = cert.issuer = OpenSSL::X509::Name.parse(subject)
+      cert.not_before = Time.now
+      cert.not_after = 1.year.from_now
+      cert.public_key = key.public_key
+      cert.serial = 0x0
+      cert.version = 2
+
+      ef = OpenSSL::X509::ExtensionFactory.new
+      ef.subject_certificate = cert
+      ef.issuer_certificate = cert
+      cert.extensions = [
+        ef.create_extension("basicConstraints", "CA:TRUE", true),
+        ef.create_extension("subjectKeyIdentifier", "hash")
+      ]
+      cert.add_extension ef.create_extension("authorityKeyIdentifier",
+                                             "keyid:always,issuer:always")
+
+      cert.sign key, OpenSSL::Digest::SHA1.new
+
+      cert.to_pem
+    end
+
+    before do
+      expect(api_order).to receive(:certificate) { certificate }
+    end
+
+    it 'saves private_key and certificate for domain' do
+      service.execute
+
+      expect(pages_domain.key).to be_present
+      expect(pages_domain.certificate).to eq(certificate)
+    end
+
+    it 'removes order from database' do
+      service.execute
+
+      expect(PagesDomainAcmeOrder.find_by_id(existing_order.id)).to be_nil
+    end
+  end
+end

spec/support/helpers/lets_encrypt_helpers.rb

 # frozen_string_literal: true
 
 module LetsEncryptHelpers
+  ACME_ORDER_METHODS = {
+    url: 'https://example.com/',
+    status: 'valid',
+    expires: 2.days.from_now
+  }.freeze
+
+  ACME_CHALLENGE_METHODS = {
+    status: 'pending',
+    token: 'tokenvalue',
+    file_content: 'hereisfilecontent',
+    request_validation: true
+  }.freeze
+
+  def stub_lets_encrypt_settings
+    stub_application_setting(
+      lets_encrypt_notification_email: 'myemail@test.example.com',
+      lets_encrypt_terms_of_service_accepted: true
+    )
+  end
+
   def stub_lets_encrypt_client
     client = instance_double('Acme::Client')
 
@@ -16,4 +36,24 @@ module LetsEncryptHelpers
 
     client
   end
+
+  def acme_challenge_double
+    challenge = instance_double('Acme::Client::Resources::Challenges::HTTP01')
+    allow(challenge).to receive_messages(ACME_CHALLENGE_METHODS)
+    challenge
+  end
+
+  def acme_authorization_double
+    authorization = instance_double('Acme::Client::Resources::Authorization')
+    allow(authorization).to receive(:http).and_return(acme_challenge_double)
+    authorization
+  end
+
+  def acme_order_double(attributes = {})
+    acme_order = instance_double('Acme::Client::Resources::Order')
+    allow(acme_order).to receive_messages(ACME_ORDER_METHODS.merge(attributes))
+    allow(acme_order).to receive(:authorizations).and_return([acme_authorization_double])
+    allow(acme_order).to receive(:finalize)
+    acme_order
+  end
 end