Merge remote-tracking branch 'gitlab-ce/5029-support-cluster-metrics-ce' into...

Merge remote-tracking branch 'gitlab-ce/5029-support-cluster-metrics-ce' into 5029-support-cluster-metrics

# Conflicts:
#	app/controllers/projects/prometheus/metrics_controller.rb
#	app/controllers/projects/prometheus_controller.rb
#	app/models/environment.rb
#	config/gitlab.yml.example
#	config/sidekiq_queues.yml
#	db/schema.rb
#	spec/controllers/projects/prometheus/metrics_controller_spec.rb
#	spec/models/environment_spec.rb
#	spec/models/project_services/prometheus_service_spec.rb

app/controllers/projects/clusters/gcp_controller.rb

@@ -40,9 +40,9 @@ class Projects::Clusters::GcpController < Projects::ApplicationController
   def verify_billing
     case google_project_billing_status
     when nil
-      flash[:alert] = _('We could not verify that one of your projects on GCP has billing enabled. Please try again.')
+      flash.now[:alert] = _('We could not verify that one of your projects on GCP has billing enabled. Please try again.')
     when false
-      flash[:alert] = _('Please <a href=%{link_to_billing} target="_blank" rel="noopener noreferrer">enable billing for one of your projects to be able to create a Kubernetes cluster</a>, then try again.').html_safe % { link_to_billing: "https://console.cloud.google.com/freetrial?utm_campaign=2018_cpanel&utm_source=gitlab&utm_medium=referral" }
+      flash.now[:alert] = _('Please <a href=%{link_to_billing} target="_blank" rel="noopener noreferrer">enable billing for one of your projects to be able to create a Kubernetes cluster</a>, then try again.').html_safe % { link_to_billing: "https://console.cloud.google.com/freetrial?utm_campaign=2018_cpanel&utm_source=gitlab&utm_medium=referral" }
     when true
       return
     end

app/controllers/projects/prometheus/metrics_controller.rb

@@ -21,7 +21,7 @@ module Projects
       private
 
       def prometheus_adapter
-        project.prometheus_service
+        @prometheus_adapter ||= ::Prometheus::AdapterService.new(project).prometheus_adapter
       end
 
       def require_prometheus_metrics!

app/controllers/projects/prometheus_controller.rb

-class Projects::PrometheusController < Projects::ApplicationController
-  before_action :authorize_read_project!
-  before_action :require_prometheus_metrics!
-
-  def active_metrics
-    respond_to do |format|
-      format.json do
-        matched_metrics = prometheus_adapter.query(:matched_metrics) || {}
-
-        if matched_metrics.any?
-          render json: matched_metrics
-        else
-          head :no_content
-        end
-      end
-    end
-  end
-
-  private
-
-  def prometheus_adapter
-    project.prometheus_service
-  end
-
-  def require_prometheus_metrics!
-    render_404 unless prometheus_adapter.can_query?
-  end
-end

app/models/environment.rb

@@ -165,6 +165,10 @@ class Environment < ActiveRecord::Base
     prometheus_adapter.query(:additional_metrics_environment, self) if has_metrics?
   end
 
+  def prometheus_adapter
+    @prometheus_adapter ||= Prometheus::AdapterService.new(project, deployment_platform).prometheus_adapter
+  end
+
   def slug
     super.presence || generate_slug
   end
@@ -228,25 +232,8 @@ class Environment < ActiveRecord::Base
     self.environment_type || self.name
   end
 
-  def prometheus_adapter
-    @prometheus_adapter ||= if service_prometheus_adapter.can_query?
-                              service_prometheus_adapter
-                            else
-                              cluster_prometheus_adapter
-                            end
-  end
-
-  def service_prometheus_adapter
-    project.find_or_initialize_service('prometheus')
-  end
-
-  def cluster_prometheus_adapter
-    return unless deployment_platform.respond_to?(:cluster)
-
-    cluster = deployment_platform.cluster
-    return unless cluster.application_prometheus&.installed?
-
-    cluster.application_prometheus
+  def deployment_platform
+    project.deployment_platform
   end
 
   private

app/services/prometheus/adapter_service.rb

+module Prometheus
+  class AdapterService
+    def initialize(project, deployment_platform = nil)
+      @project = project
+
+      @deployment_platform = if deployment_platform
+                               deployment_platform
+                             else
+                               project.deployment_platform
+                             end
+    end
+
+    attr_reader :deployment_platform, :project
+
+    def prometheus_adapter
+      @prometheus_adapter ||= if service_prometheus_adapter.can_query?
+                                service_prometheus_adapter
+                              else
+                                cluster_prometheus_adapter
+                              end
+    end
+
+    def service_prometheus_adapter
+      project.find_or_initialize_service('prometheus')
+    end
+
+    def cluster_prometheus_adapter
+      return unless deployment_platform.respond_to?(:cluster)
+
+      cluster = deployment_platform.cluster
+      return unless cluster.application_prometheus&.installed?
+
+      cluster.application_prometheus
+    end
+  end
+end

app/views/admin/runners/index.html.haml

@@ -35,9 +35,8 @@
           method: :put, class: 'btn btn-default',
           data: { confirm: _("Are you sure you want to reset registration token?") }
 
-  = render partial: 'ci/runner/how_to_setup_runner',
-           locals: { registration_token: Gitlab::CurrentSettings.runners_registration_token,
-                     type: 'shared' }
+  = render partial: 'ci/runner/how_to_setup_shared_runner',
+           locals: { registration_token: Gitlab::CurrentSettings.runners_registration_token }
 
   .append-bottom-20.clearfix
     .pull-left

app/views/ci/runner/_how_to_setup_runner.html.haml

 - link = link_to _("GitLab Runner section"), 'https://about.gitlab.com/gitlab-ci/#gitlab-runner', target: '_blank'
-.bs-callout.help-callout
-  %h4= _("How to setup a #{type} Runner for a new project")
+.append-bottom-10
+  %h4= _("Setup a #{type} Runner manually")
 
-  %ol
-    %li
-      = _("Install a Runner compatible with GitLab CI")
-      = (_("(checkout the %{link} for information on how to install it).") % { link: link }).html_safe
-    %li
-      = _("Specify the following URL during the Runner setup:")
-      %code#coordinator_address= root_url(only_path: false)
-    %li
-      = _("Use the following registration token during setup:")
-      %code#registration_token= registration_token
-    %li
-      = _("Start the Runner!")
+%ol
+  %li
+    = _("Install a Runner compatible with GitLab CI")
+    = (_("(checkout the %{link} for information on how to install it).") % { link: link }).html_safe
+  %li
+    = _("Specify the following URL during the Runner setup:")
+    %code#coordinator_address= root_url(only_path: false)
+  %li
+    = _("Use the following registration token during setup:")
+    %code#registration_token= registration_token
+  %li
+    = _("Start the Runner!")

app/views/ci/runner/_how_to_setup_shared_runner.html.haml

+.bs-callout.help-callout
+  = render partial: 'ci/runner/how_to_setup_runner',
+           locals: { registration_token: registration_token, type: 'shared' }

app/views/ci/runner/_how_to_setup_specific_runner.html.haml

+.bs-callout.help-callout
+  .append-bottom-10
+    %h4= _('Setup a specific Runner automatically')
+
+  %p
+    - link_to_help_page = link_to(_('Learn more about Kubernetes'),
+                                  help_page_path('user/project/clusters/index'),
+                                  target: '_blank',
+                                  rel: 'noopener noreferrer')
+
+    = _('You can easily install a Runner on a Kubernetes cluster. %{link_to_help_page}').html_safe % { link_to_help_page: link_to_help_page }
+
+  %ol
+    %li
+      = _('Click the button below to begin the install process by navigating to the Kubernetes page')
+    %li
+      = _('Select an existing Kubernetes cluster or create a new one')
+    %li
+      = _('From the Kubernetes cluster details view, install Runner from the applications list')
+
+  = link_to _('Install Runner on Kubernetes'),
+    project_clusters_path(@project),
+    class: 'btn btn-info'
+  %hr
+  = render partial: 'ci/runner/how_to_setup_runner',
+           locals: { registration_token: registration_token, type: 'specific' }

app/views/projects/clusters/_empty_state.html.haml

@@ -4,7 +4,7 @@
   .col-xs-12
     .text-content
       %h4.text-center= s_('ClusterIntegration|Integrate Kubernetes cluster automation')
-      - link_to_help_page = link_to(s_('ClusterIntegration|Learn more about Kubernetes'), help_page_path('user/project/clusters/index'), target: '_blank', rel: 'noopener noreferrer')
+      - link_to_help_page = link_to(_('Learn more about Kubernetes'), help_page_path('user/project/clusters/index'), target: '_blank', rel: 'noopener noreferrer')
       %p= s_('ClusterIntegration|Kubernetes clusters allow you to use review apps, deploy your applications, run your pipelines, and much more in an easy way. %{link_to_help_page}').html_safe % { link_to_help_page: link_to_help_page}
 
       .text-center

app/views/projects/runners/_shared_runners.html.haml

 %h3 Shared Runners
 
-.bs-callout.bs-callout-warning.shared-runners-description
+.bs-callout.shared-runners-description
   - if Gitlab::CurrentSettings.shared_runners_text.present?
     = markdown_field(Gitlab::CurrentSettings.current_application_settings, :shared_runners_text)
   - else
@@ -9,7 +9,7 @@
     on GitLab.com).
   %hr
   - if @project.shared_runners_enabled?
-    = link_to toggle_shared_runners_project_runners_path(@project), class: 'btn btn-warning', method: :post do
+    = link_to toggle_shared_runners_project_runners_path(@project), class: 'btn btn-close', method: :post do
       Disable shared Runners
   - else
     = link_to toggle_shared_runners_project_runners_path(@project), class: 'btn btn-success', method: :post do

app/views/projects/runners/_specific_runners.html.haml

 %h3 Specific Runners
 
-= render partial: 'ci/runner/how_to_setup_runner',
-         locals: { registration_token: @project.runners_token,
-                   type: 'specific' }
+= render partial: 'ci/runner/how_to_setup_specific_runner',
+         locals: { registration_token: @project.runners_token }
 
 - if @project_runners.any?
   %h4.underlined-title Runners activated for this project

changelogs/unreleased/42044-osw-add-button-to-deploy-runner-to-kubernetes.yml

+---
+title: Add a button to deploy a runner to a Kubernetes cluster in the settings page
+merge_request: 17278
+author:
+type: changed

changelogs/unreleased/43496-error-message-for-gke-clusters-persists-in-the-next-page.yml

+---
+title: Do not persist Google Project verification flash errors after a page reload
+merge_request: 17299
+author:
+type: fixed

doc/user/project/milestones/index.md

@@ -8,8 +8,8 @@ Milestones allow you to organize issues and merge requests into a cohesive group
 
 ## Project milestones and group milestones
 
-- **Project milestones** can be assigned to issues or merge requests in that project only. 
-- **Group milestones** can be assigned to any issue or merge request of any project in that group. 
+- **Project milestones** can be assigned to issues or merge requests in that project only.
+- **Group milestones** can be assigned to any issue or merge request of any project in that group.
 - In the [future](https://gitlab.com/gitlab-org/gitlab-ce/issues/36862), you will be able to assign group milestones to issues and merge reqeusts of projects in [subgroups](../../group/subgroups/index.md).
 
 ## Creating milestones

spec/controllers/projects/clusters/gcp_controller_spec.rb

@@ -161,7 +161,7 @@ describe Projects::Clusters::GcpController do
           it 'renders the cluster form with an error' do
             go
 
-            expect(response).to set_flash[:alert]
+            expect(response).to set_flash.now[:alert]
             expect(response).to render_template('new')
           end
         end

spec/features/admin/admin_runners_spec.rb

@@ -19,7 +19,7 @@ describe "Admin Runners" do
       end
 
       it 'has all necessary texts' do
-        expect(page).to have_text "How to setup"
+        expect(page).to have_text "Setup a shared Runner manually"
         expect(page).to have_text "Runners with last contact more than a minute ago: 1"
       end
 
@@ -54,7 +54,7 @@ describe "Admin Runners" do
       end
 
       it 'has all necessary texts including no runner message' do
-        expect(page).to have_text "How to setup"
+        expect(page).to have_text "Setup a shared Runner manually"
         expect(page).to have_text "Runners with last contact more than a minute ago: 0"
         expect(page).to have_text 'No runners found'
       end

spec/features/runners_spec.rb

@@ -7,6 +7,20 @@ feature 'Runners' do
     sign_in(user)
   end
 
+  context 'when user opens runners page' do
+    given(:project) { create(:project) }
+
+    background do
+      project.add_master(user)
+    end
+
+    scenario 'user can see a button to install runners on kubernetes clusters' do
+      visit runners_path(project)
+
+      expect(page).to have_link('Install Runner on Kubernetes', href: project_clusters_path(project))
+    end
+  end
+
   context 'when a project has enabled shared_runners' do
     given(:project) { create(:project) }
 

spec/models/concerns/prometheus_adapter_spec.rb

+require 'spec_helper'
+
+describe PrometheusAdapter, :use_clean_rails_memory_store_caching do
+  include PrometheusHelpers
+  include ReactiveCachingHelpers
+
+  class TestClass
+    include PrometheusAdapter
+  end
+
+  let(:project) { create(:prometheus_project) }
+  let(:service) { project.prometheus_service }
+
+  let(:described_class) { TestClass }
+  let(:environment_query) { Gitlab::Prometheus::Queries::EnvironmentQuery }
+
+  describe '#query' do
+    describe 'environment' do
+      let(:environment) { build_stubbed(:environment, slug: 'env-slug') }
+
+      around do |example|
+        Timecop.freeze { example.run }
+      end
+
+      context 'with valid data' do
+        subject { service.query(:environment, environment) }
+
+        before do
+          stub_reactive_cache(service, prometheus_data, environment_query, environment.id)
+        end
+
+        it 'returns reactive data' do
+          is_expected.to eq(prometheus_metrics_data)
+        end
+      end
+    end
+
+    describe 'matched_metrics' do
+      let(:matched_metrics_query) { Gitlab::Prometheus::Queries::MatchedMetricQuery }
+      let(:prometheus_client_wrapper) { double(:prometheus_client_wrapper, label_values: nil) }
+
+      context 'with valid data' do
+        subject { service.query(:matched_metrics) }
+
+        before do
+          allow(service).to receive(:prometheus_client_wrapper).and_return(prometheus_client_wrapper)
+          synchronous_reactive_cache(service)
+        end
+
+        it 'returns reactive data' do
+          expect(subject[:success]).to be_truthy
+          expect(subject[:data]).to eq([])
+        end
+      end
+    end
+
+    describe 'deployment' do
+      let(:deployment) { build_stubbed(:deployment) }
+      let(:deployment_query) { Gitlab::Prometheus::Queries::DeploymentQuery }
+
+      around do |example|
+        Timecop.freeze { example.run }
+      end
+
+      context 'with valid data' do
+        subject { service.query(:deployment, deployment) }
+
+        before do
+          stub_reactive_cache(service, prometheus_data, deployment_query, deployment.id)
+        end
+
+        it 'returns reactive data' do
+          expect(subject).to eq(prometheus_metrics_data)
+        end
+      end
+    end
+  end
+
+  describe '#calculate_reactive_cache' do
+    let(:environment) { create(:environment, slug: 'env-slug') }
+    before do
+      service.manual_configuration = true
+      service.active = true
+    end
+
+    subject do
+      service.calculate_reactive_cache(environment_query.name, environment.id)
+    end
+
+    around do |example|
+      Timecop.freeze { example.run }
+    end
+
+    context 'when service is inactive' do
+      before do
+        service.active = false
+      end
+
+      it { is_expected.to be_nil }
+    end
+
+    context 'when Prometheus responds with valid data' do
+      before do
+        stub_all_prometheus_requests(environment.slug)
+      end
+
+      it { expect(subject.to_json).to eq(prometheus_data.to_json) }
+      it { expect(subject.to_json).to eq(prometheus_data.to_json) }
+    end
+
+    [404, 500].each do |status|
+      context "when Prometheus responds with #{status}" do
+        before do
+          stub_all_prometheus_requests(environment.slug, status: status, body: "QUERY FAILED!")
+        end
+
+        it { is_expected.to eq(success: false, result: %(#{status} - "QUERY FAILED!")) }
+      end
+    end
+  end
+end

spec/models/environment_spec.rb

@@ -495,7 +495,7 @@ describe Environment do
       it 'returns the metrics from the deployment service' do
         expect(environment.prometheus_adapter)
           .to receive(:query).with(:environment, environment)
-                .and_return(:fake_metrics)
+          .and_return(:fake_metrics)
 
         is_expected.to eq(:fake_metrics)
       end
@@ -552,66 +552,6 @@ describe Environment do
         allow(environment).to receive(:has_metrics?).and_return(true)
       end
 
-      it 'returns the metrics from the deployment service' do
-        expect(environment.prometheus_adapter)
-          .to receive(:query).with(:environment, environment)
-          .and_return(:fake_metrics)
-
-        is_expected.to eq(:fake_metrics)
-      end
-    end
-
-    context 'when the environment does not have metrics' do
-      before do
-        allow(environment).to receive(:has_metrics?).and_return(false)
-      end
-
-      it { is_expected.to be_nil }
-    end
-  end
-
-  describe '#has_metrics?' do
-    subject { environment.has_metrics? }
-
-    context 'when the enviroment is available' do
-      context 'with a deployment service' do
-        let(:project) { create(:prometheus_project) }
-
-        context 'and a deployment' do
-          let!(:deployment) { create(:deployment, environment: environment) }
-          it { is_expected.to be_truthy }
-        end
-
-        context 'but no deployments' do
-          it { is_expected.to be_falsy }
-        end
-      end
-
-      context 'without a monitoring service' do
-        it { is_expected.to be_falsy }
-      end
-    end
-
-    context 'when the environment is unavailable' do
-      let(:project) { create(:prometheus_project) }
-
-      before do
-        environment.stop
-      end
-
-      it { is_expected.to be_falsy }
-    end
-  end
-
-  describe '#additional_metrics' do
-    let(:project) { create(:prometheus_project) }
-    subject { environment.additional_metrics }
-
-    context 'when the environment has additional metrics' do
-      before do
-        allow(environment).to receive(:has_metrics?).and_return(true)
-      end
-
       it 'returns the additional metrics from the deployment service' do
         expect(environment.prometheus_adapter).to receive(:query)
                                                 .with(:additional_metrics_environment, environment)
@@ -724,36 +664,10 @@ describe Environment do
   end
 
   describe '#prometheus_adapter' do
-    let(:cluster) { create(:cluster, :provided_by_user, environment_scope: '*', projects: [project]) }
-
-    context 'prometheus service can execute queries' do
-      let(:prometheus_service) { double(:prometheus_service, can_query?: true) }
-
-      before do
-        allow(environment.project).to receive(:find_or_initialize_service).with('prometheus').and_return prometheus_service
-      end
-
-      it 'return prometheus service as prometheus adapter' do
-        expect(environment.prometheus_adapter).to eq(prometheus_service)
-      end
-    end
-
-    context "prometheus service can't execute queries" do
-      let(:prometheus_service) { double(:prometheus_service, can_query?: false) }
+    it 'calls prometheus adapter service' do
+      expect_any_instance_of(Prometheus::AdapterService).to receive(:prometheus_adapter)
 
-      context 'with cluster with prometheus installed' do
-        let!(:prometheus) { create(:clusters_applications_prometheus, :installed, cluster: cluster) }
-
-        it 'returns application handling all environments' do
-          expect(environment.prometheus_adapter).to eq(prometheus)
-        end
-      end
-
-      context 'with cluster without prometheus installed' do
-        it 'returns nil' do
-          expect(environment.prometheus_adapter).to be_nil
-        end
-      end
+      subject.prometheus_adapter
     end
   end
 end

spec/models/project_services/prometheus_service_spec.rb

@@ -6,7 +6,6 @@ describe PrometheusService, :use_clean_rails_memory_store_caching do
 
   let(:project) { create(:prometheus_project) }
   let(:service) { project.prometheus_service }
-  let(:environment_query) { Gitlab::Prometheus::Queries::EnvironmentQuery }
 
   describe "Associations" do
     it { is_expected.to belong_to :project }

spec/services/prometheus/adapter_service_spec.rb

+require 'spec_helper'
+
+describe Prometheus::AdapterService do
+  let(:project) { create(:project) }
+  subject { described_class.new(project) }
+
+  describe '#prometheus_adapter' do
+    let(:cluster) { create(:cluster, :provided_by_user, environment_scope: '*', projects: [project]) }
+
+    context 'prometheus service can execute queries' do
+      let(:prometheus_service) { double(:prometheus_service, can_query?: true) }
+
+      before do
+        allow(project).to receive(:find_or_initialize_service).with('prometheus').and_return prometheus_service
+      end
+
+      it 'return prometheus service as prometheus adapter' do
+        expect(subject.prometheus_adapter).to eq(prometheus_service)
+      end
+    end
+
+    context "prometheus service can't execute queries" do
+      let(:prometheus_service) { double(:prometheus_service, can_query?: false) }
+
+      context 'with cluster with prometheus installed' do
+        let!(:prometheus) { create(:clusters_applications_prometheus, :installed, cluster: cluster) }
+
+        it 'returns application handling all environments' do
+          expect(subject.prometheus_adapter).to eq(prometheus)
+        end
+      end
+
+      context 'with cluster without prometheus installed' do
+        it 'returns nil' do
+          expect(subject.prometheus_adapter).to be_nil
+        end
+      end
+    end
+  end
+end