Automatic merge of gitlab-org/gitlab-ce master

.gitlab/ci/memory.gitlab-ci.yml

@@ -5,15 +5,15 @@ memory-static:
 
     # Loads each of gems in the Gemfile and checks how much memory they consume when they are required.
     # 'derailed_benchmarks' internally uses 'get_process_mem'
-    - scripts/memory-static 'tmp/memory_static_full_report.txt' 'tmp/memory_static_metrics.txt'
+    - bundle exec derailed bundle:mem > tmp/memory_bundle_mem.txt
+    - scripts/generate-gems-size-metrics-static tmp/memory_bundle_mem.txt >> 'tmp/memory_metrics.txt'
 
     # Outputs detailed information about objects created while gems are loaded.
     # 'derailed_benchmarks' internally uses 'memory_profiler'
-    - scripts/memory-static-objects 'tmp/memory_static_objects_full_report.txt' 'tmp/memory_static_metrics.txt'
+    - bundle exec derailed bundle:objects > tmp/memory_bundle_objects.txt
+    - scripts/generate-gems-memory-metrics-static tmp/memory_bundle_objects.txt >> 'tmp/memory_metrics.txt'
   artifacts:
     paths:
-      - tmp/memory_static_full_report.txt
-      - tmp/memory_static_objects_full_report.txt
-      - tmp/memory_static_metrics.txt
+      - tmp/memory_*.txt
     reports:
-      metrics: tmp/memory_static_metrics.txt
+      metrics: tmp/memory_metrics.txt

app/views/projects/pages_domains/_form.html.haml

@@ -11,7 +11,7 @@
 
 - if Gitlab.config.pages.external_https
 
-  - auto_ssl_available = ::Gitlab::LetsEncrypt::Client.new.enabled?
+  - auto_ssl_available = ::Gitlab::LetsEncrypt.enabled?(@domain)
   - auto_ssl_enabled = @domain.auto_ssl_enabled?
   - auto_ssl_available_and_enabled = auto_ssl_available && auto_ssl_enabled
 

app/workers/pages_domain_ssl_renewal_cron_worker.rb

@@ -5,9 +5,9 @@ class PagesDomainSslRenewalCronWorker
   include CronjobQueue
 
   def perform
-    return unless ::Gitlab::LetsEncrypt::Client.new.enabled?
-
     PagesDomain.need_auto_ssl_renewal.find_each do |domain|
+      next unless ::Gitlab::LetsEncrypt.enabled?(domain)
+
       PagesDomainSslRenewalWorker.perform_async(domain.id)
     end
   end

app/workers/pages_domain_ssl_renewal_worker.rb

@@ -4,11 +4,9 @@ class PagesDomainSslRenewalWorker
   include ApplicationWorker
 
   def perform(domain_id)
-    return unless ::Gitlab::LetsEncrypt::Client.new.enabled?
-
     domain = PagesDomain.find_by_id(domain_id)
-
-    return unless domain
+    return unless domain&.enabled?
+    return unless ::Gitlab::LetsEncrypt.enabled?(domain)
 
     ::PagesDomains::ObtainLetsEncryptCertificateService.new(domain).execute
   end

lib/gitlab/lets_encrypt.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module LetsEncrypt
+    def self.enabled?(pages_domain = nil)
+      return false unless Gitlab::CurrentSettings.lets_encrypt_terms_of_service_accepted
+
+      return false unless Feature.enabled?(:pages_auto_ssl)
+
+      # If no domain is passed, just check whether we're enabled globally
+      return true unless pages_domain
+
+      !!pages_domain.project && Feature.enabled?(:pages_auto_ssl_for_project, pages_domain.project)
+    end
+  end
+end

lib/gitlab/lets_encrypt/client.rb

@@ -34,14 +34,6 @@ module Gitlab
         acme_client.terms_of_service
       end
 
-      def enabled?
-        return false unless Feature.enabled?(:pages_auto_ssl)
-
-        return false unless private_key
-
-        Gitlab::CurrentSettings.lets_encrypt_terms_of_service_accepted
-      end
-
       private
 
       def acme_client
@@ -65,7 +57,7 @@ module Gitlab
       end
 
       def ensure_account
-        raise 'Acme integration is disabled' unless enabled?
+        raise 'Acme integration is disabled' unless ::Gitlab::LetsEncrypt.enabled?
 
         @acme_account ||= acme_client.new_account(contact: contact, terms_of_service_agreed: true)
       end

scripts/generate-gems-memory-metrics-static

+#!/usr/bin/env ruby
+
+abort "usage: #{__FILE__} <memory_bundle_objects_file_name>" unless ARGV.length == 1
+memory_bundle_objects_file_name = ARGV.first
+
+full_report = File.readlines(memory_bundle_objects_file_name)
+
+allocated_str = full_report[1]
+retained_str = full_report[2]
+allocated_stats = /Total allocated: (?<bytes>.*) bytes \((?<objects>.*) objects\)/.match(allocated_str)
+retained_stats = /Total retained: (?<bytes>.*) bytes \((?<objects>.*) objects\)/.match(retained_str)
+
+abort 'failed to process the benchmark output' unless allocated_stats && retained_stats
+
+puts "memory_static_objects_allocated_mb #{(allocated_stats[:bytes].to_f / (1024 * 1024)).round(1)}"
+puts "memory_static_objects_retained_mb #{(retained_stats[:bytes].to_f / (1024 * 1024)).round(1)}"
+puts "memory_static_objects_allocated_items #{allocated_stats[:objects]}"
+puts "memory_static_objects_retained_items #{retained_stats[:objects]}"

scripts/generate-gems-size-metrics-static

+#!/usr/bin/env ruby
+
+abort "usage: #{__FILE__} <memory_bundle_mem_file_name>" unless ARGV.length == 1
+memory_bundle_mem_file_name = ARGV.first
+
+full_report = File.readlines(memory_bundle_mem_file_name)
+
+def total_size(memory_bundle_mem_report)
+  stats = /TOP: (?<total_mibs_str>.*) MiB/.match(memory_bundle_mem_report.first)
+  abort 'failed to process the benchmark output' unless stats
+  "gem_total_size_mb #{stats[:total_mibs_str].to_f.round(1)}"
+end
+
+TOP_LEVEL_GEM_LOG_FORMAT = /^  (?<gem_name>\S.*):\s*(?<gem_size>\d[.\d]*)\s*MiB/.freeze
+def all_gems(memory_bundle_mem_report)
+  memory_bundle_mem_report.map do |line|
+    TOP_LEVEL_GEM_LOG_FORMAT.match(line)
+  end.compact
+end
+
+def gems_as_metrics(gems_match_data)
+  gems_match_data.map do |gem|
+    gem_name = gem[:gem_name]
+    gem_size_mb = gem[:gem_size].to_f.round(1)
+    "gem_size_mb{name=\"#{gem_name}\"} #{gem_size_mb}"
+  end
+end
+
+puts total_size(full_report)
+puts gems_as_metrics(all_gems(full_report)).sort(&:casecmp)

scripts/memory-static

-#!/usr/bin/env ruby
-
-require_relative '../lib/gitlab/popen'
-
-full_report_filename, metrics_filename = ARGV
-abort 'usage: memory-static <full_report_filename> <metrics_filename>' unless full_report_filename && metrics_filename
-
-full_report, status = Gitlab::Popen.popen(%w(bundle exec derailed bundle:mem))
-abort 'failed to execute the benchmark' unless status.zero?
-
-File.open(full_report_filename, 'w') do |f|
-  f.write(full_report)
-end
-
-stats = /TOP: (?<total_mibs_str>.*) MiB/.match(full_report.lines.first)
-abort 'failed to process the benchmark output' unless stats
-
-File.open(metrics_filename, 'a') do |f|
-  f.puts "memory_static_total_mb #{stats[:total_mibs_str].to_f.round(1)}"
-end

scripts/memory-static-objects

-#!/usr/bin/env ruby
-
-require_relative '../lib/gitlab/popen'
-
-full_report_filename, metrics_filename = ARGV
-abort 'usage: memory-static-objects <full_report_filename> <metrics_filename>' unless full_report_filename && metrics_filename
-
-full_report, status = Gitlab::Popen.popen(%w(bundle exec derailed bundle:objects))
-abort 'failed to execute the benchmark' unless status.zero?
-
-File.open(full_report_filename, 'w') do |f|
-  f.write(full_report)
-end
-
-allocated_str = full_report.lines[1]
-retained_str = full_report.lines[2]
-allocated_stats = /Total allocated: (?<bytes>.*) bytes \((?<objects>.*) objects\)/.match(allocated_str)
-retained_stats = /Total retained: (?<bytes>.*) bytes \((?<objects>.*) objects\)/.match(retained_str)
-
-abort 'failed to process the benchmark output' unless allocated_stats && retained_stats
-
-File.open(metrics_filename, 'a') do |f|
-  f.puts "memory_static_objects_allocated_mb #{(allocated_stats[:bytes].to_f / (1024 * 1024)).round(1)}"
-  f.puts "memory_static_objects_retained_mb #{(retained_stats[:bytes].to_f / (104 * 1024)).round(1)}"
-  f.puts "memory_static_objects_allocated_items #{allocated_stats[:objects]}"
-  f.puts "memory_static_objects_retained_items #{retained_stats[:objects]}"
-end

spec/lib/gitlab/lets_encrypt/client_spec.rb

@@ -116,42 +116,6 @@ describe ::Gitlab::LetsEncrypt::Client do
     end
   end
 
-  describe '#enabled?' do
-    subject { client.enabled? }
-
-    context 'when terms of service are accepted' do
-      it { is_expected.to eq(true) }
-
-      context "when private_key isn't present and database is read only" do
-        before do
-          allow(::Gitlab::Database).to receive(:read_only?).and_return(true)
-        end
-
-        it 'returns false' do
-          expect(::Gitlab::CurrentSettings.lets_encrypt_private_key).to eq(nil)
-
-          is_expected.to eq(false)
-        end
-      end
-
-      context 'when feature flag is disabled' do
-        before do
-          stub_feature_flags(pages_auto_ssl: false)
-        end
-
-        it { is_expected.to eq(false) }
-      end
-    end
-
-    context 'when terms of service are not accepted' do
-      before do
-        stub_application_setting(lets_encrypt_terms_of_service_accepted: false)
-      end
-
-      it { is_expected.to eq(false) }
-    end
-  end
-
   describe '#terms_of_service_url' do
     subject { client.terms_of_service_url }
 

spec/lib/gitlab/lets_encrypt_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe ::Gitlab::LetsEncrypt do
+  include LetsEncryptHelpers
+
+  before do
+    stub_lets_encrypt_settings
+  end
+
+  describe '.enabled?' do
+    let(:project) { create(:project) }
+    let(:pages_domain) { create(:pages_domain, project: project) }
+
+    subject { described_class.enabled?(pages_domain) }
+
+    context 'when terms of service are accepted' do
+      it { is_expected.to eq(true) }
+
+      context 'when feature flag is disabled' do
+        before do
+          stub_feature_flags(pages_auto_ssl: false)
+        end
+
+        it { is_expected.to eq(false) }
+      end
+    end
+
+    context 'when terms of service are not accepted' do
+      before do
+        stub_application_setting(lets_encrypt_terms_of_service_accepted: false)
+      end
+
+      it { is_expected.to eq(false) }
+    end
+
+    context 'when feature flag for project is disabled' do
+      before do
+        stub_feature_flags(pages_auto_ssl_for_project: false)
+      end
+
+      it 'returns false' do
+        is_expected.to eq(false)
+      end
+    end
+
+    context 'when domain has not project' do
+      let(:pages_domain) { create(:pages_domain) }
+
+      it 'returns false' do
+        is_expected.to eq(false)
+      end
+    end
+  end
+end

spec/workers/pages_domain_ssl_renewal_cron_worker_spec.rb

@@ -12,15 +12,18 @@ describe PagesDomainSslRenewalCronWorker do
   end
 
   describe '#perform' do
-    let!(:domain) { create(:pages_domain) }
-    let!(:domain_with_enabled_auto_ssl) { create(:pages_domain, auto_ssl_enabled: true) }
-    let!(:domain_with_obtained_letsencrypt) { create(:pages_domain, :letsencrypt, auto_ssl_enabled: true) }
+    let(:project) { create :project }
+    let!(:domain) { create(:pages_domain, project: project) }
+    let!(:domain_with_enabled_auto_ssl) { create(:pages_domain, project: project, auto_ssl_enabled: true) }
+    let!(:domain_with_obtained_letsencrypt) do
+      create(:pages_domain, :letsencrypt, project: project, auto_ssl_enabled: true)
+    end
     let!(:domain_without_auto_certificate) do
-      create(:pages_domain, :without_certificate, :without_key, auto_ssl_enabled: true)
+      create(:pages_domain, :without_certificate, :without_key, project: project, auto_ssl_enabled: true)
     end
 
     let!(:domain_with_expired_auto_ssl) do
-      create(:pages_domain, :letsencrypt, :with_expired_certificate)
+      create(:pages_domain, :letsencrypt, :with_expired_certificate, project: project)
     end
 
     it 'enqueues a PagesDomainSslRenewalWorker for domains needing renewal' do

spec/workers/pages_domain_ssl_renewal_worker_spec.rb

@@ -7,7 +7,8 @@ describe PagesDomainSslRenewalWorker do
 
   subject(:worker) { described_class.new }
 
-  let(:domain) { create(:pages_domain) }
+  let(:project) { create(:project) }
+  let(:domain) { create(:pages_domain, project: project) }
 
   before do
     stub_lets_encrypt_settings
@@ -22,14 +23,24 @@ describe PagesDomainSslRenewalWorker do
       worker.perform(domain.id)
     end
 
+    shared_examples 'does nothing' do
+      it 'does nothing' do
+        expect(::PagesDomains::ObtainLetsEncryptCertificateService).not_to receive(:new)
+      end
+    end
+
     context 'when domain was deleted' do
       before do
         domain.destroy!
       end
 
-      it 'does nothing' do
-        expect(::PagesDomains::ObtainLetsEncryptCertificateService).not_to receive(:new)
-      end
+      include_examples 'does nothing'
+    end
+
+    context 'when domain is disabled' do
+      let(:domain) { create(:pages_domain, :disabled) }
+
+      include_examples 'does nothing'
     end
   end
 end