Add QA test for prometheus alerts

The configuration of alerts depends on a lot of moving parts and has
been prone to issue. This test sets up a cluster, creates a project,
connects the cluster to the project, sets up Auto Devops, deploys the
application, then performs basic CRUD operations on alerts from the
metrics dashboard. Alerts are configurated by updating Prometheus, so
a future test could build upon the assertions by ensuring that the alert
can indeed be fired.

app/assets/javascripts/monitoring/components/charts/time_series.vue

@@ -311,7 +311,10 @@ export default {
       <gl-tooltip :target="() => $refs.graphTitle" :disabled="!showTitleTooltip">
         {{ graphData.title }}
       </gl-tooltip>
-      <div class="prometheus-graph-widgets js-graph-widgets flex-fill">
+      <div
+        class="prometheus-graph-widgets js-graph-widgets flex-fill"
+        data-qa-selector="prometheus_graph_widgets"
+      >
         <slot></slot>
       </div>
     </div>

app/assets/javascripts/monitoring/components/dashboard.vue

@@ -353,7 +353,7 @@ export default {
 </script>
 
 <template>
-  <div class="prometheus-graphs">
+  <div class="prometheus-graphs" data-qa-selector="prometheus_graphs">
     <div
       v-if="showHeader"
       ref="prometheusGraphsHeader"

app/assets/javascripts/monitoring/components/panel_type.vue

@@ -138,6 +138,7 @@ export default {
         v-gl-tooltip
         class="ml-auto mx-3"
         toggle-class="btn btn-transparent border-0"
+        data-qa-selector="prometheus_widgets_dropdown"
         :right="true"
         :no-caret="true"
         :title="__('More actions')"
@@ -161,7 +162,11 @@ export default {
         >
           {{ __('Generate link to chart') }}
         </gl-dropdown-item>
-        <gl-dropdown-item v-if="alertWidgetAvailable" v-gl-modal="`alert-modal-${index}`">
+        <gl-dropdown-item
+          v-if="alertWidgetAvailable"
+          v-gl-modal="`alert-modal-${index}`"
+          data-qa-selector="alert_widget_menu_item"
+        >
           {{ __('Alerts') }}
         </gl-dropdown-item>
       </gl-dropdown>

app/assets/javascripts/pipelines/components/pipelines_table_row.vue

@@ -331,6 +331,7 @@ export default {
           :loading="isRetrying"
           :disabled="isRetrying"
           container-class="js-pipelines-retry-button btn btn-default btn-retry"
+          data-qa-selector="pipeline_retry_button"
           @click="handleRetryClick"
         >
           <icon name="repeat" />

app/views/layouts/nav/sidebar/_project.html.haml

@@ -240,7 +240,7 @@
 
             - if project_nav_tab? :environments
               = nav_link(controller: :environments, action: [:metrics, :metrics_redirect]) do
-                = link_to metrics_project_environments_path(@project), title: _('Metrics'), class: 'shortcuts-metrics' do
+                = link_to metrics_project_environments_path(@project), title: _('Metrics'), class: 'shortcuts-metrics', data: { qa_selector: 'operations_metrics_link' } do
                   %span
                     = _('Metrics')
 

ee/app/assets/javascripts/monitoring/components/alert_widget_form.vue

@@ -239,11 +239,12 @@ export default {
         <gl-dropdown
           id="alert-query-dropdown"
           :text="queryDropdownLabel"
-          toggle-class="dropdown-menu-toggle"
+          toggle-class="dropdown-menu-toggle qa-alert-query-dropdown"
         >
           <gl-dropdown-item
             v-for="query in relevantQueries"
             :key="query.metricId"
+            data-qa-selector="alert_query_option"
             @click="selectQuery(query.metricId)"
           >
             {{ query.label }}
@@ -282,6 +283,7 @@ export default {
           v-model.number="threshold"
           :disabled="formDisabled"
           type="number"
+          data-qa-selector="alert_threshold_field"
         />
       </gl-form-group>
     </div>

qa/qa.rb

@@ -289,6 +289,8 @@ module QA
           autoload :AddExisting, 'qa/page/project/operations/kubernetes/add_existing'
           autoload :Show, 'qa/page/project/operations/kubernetes/show'
         end
+
+        autoload :Metrics, 'qa/page/project/operations/metrics'
       end
 
       module Wiki

qa/qa/ee.rb

@@ -128,6 +128,8 @@ module QA
           module Kubernetes
             autoload :Show, 'qa/ee/page/project/operations/kubernetes/show'
           end
+
+          autoload :Metrics, 'qa/ee/page/project/operations/metrics'
         end
 
         module Packages

qa/qa/ee/page/project/operations/metrics.rb

+# frozen_string_literal: true
+
+module QA
+  module EE
+    module Page
+      module Project
+        module Operations
+          module Metrics
+            EXPECTED_LABEL = 'Total (GB)'
+
+            def self.prepended(page)
+              page.module_eval do
+                view 'ee/app/assets/javascripts/monitoring/components/alert_widget_form.vue' do
+                  element :alert_query_dropdown
+                  element :alert_query_option
+                  element :alert_threshold_field
+                end
+              end
+            end
+
+            def wait_for_alert(operator = '>', threshold = 0)
+              wait_until(reload: false) { has_alert?(operator, threshold) }
+            end
+
+            def has_alert?(operator = '>', threshold = 0)
+              within_element :prometheus_graphs do
+                has_text?([EXPECTED_LABEL, operator, threshold].join(' '))
+              end
+            end
+
+            def write_first_alert(operator = '>', threshold = 0)
+              open_first_alert_modal
+              click_on operator
+              fill_element :alert_threshold_field, threshold
+
+              within('.modal-content') { click_button(class: 'btn-success') }
+            end
+
+            def delete_first_alert
+              open_first_alert_modal
+
+              within('.modal-content') { click_button(class: 'btn-danger') }
+              wait_for_requests
+            end
+
+            def open_first_alert_modal
+              all_elements(:prometheus_widgets_dropdown, minimum: 1).first.click
+              click_element :alert_widget_menu_item
+
+              click_element :alert_query_dropdown unless has_element?(:alert_query_option, wait: 3)
+              all_elements(:alert_query_option, minimum: 1).first.click
+            end
+          end
+        end
+      end
+    end
+  end
+end

qa/qa/fixtures/monitored_auto_devops/.gitlab-ci.yml

+# This is stripped down version of the .gitlab-ci.yml found
+# here: https://gitlab.com/joshlambert/autodevops-deploy.
+#
+# It performs only the deploy stage.
+
+image: alpine:latest
+
+variables:
+  # AUTO_DEVOPS_DOMAIN is the application deployment domain and should be set as a variable at the group or project level.
+  AUTO_DEVOPS_DOMAIN: my-fake-domain.com
+
+  POSTGRES_USER: user
+  POSTGRES_PASSWORD: testing-password
+  POSTGRES_ENABLED: 'false'
+  POSTGRES_DB: $CI_ENVIRONMENT_SLUG
+
+  KUBERNETES_VERSION: 1.11.6
+  HELM_VERSION: 2.12.2
+
+  DOCKER_DRIVER: overlay2
+
+stages:
+  - production
+
+# This job continuously deploys to production on every push to `master`.
+
+production:
+  stage: production
+  script:
+    - check_kube_domain
+    - install_dependencies
+    - download_chart
+    - ensure_namespace
+    - initialize_tiller
+    - create_secret
+    - deploy
+    - persist_environment_url
+  environment:
+    name: production
+    url: http://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN
+  artifacts:
+    paths: [environment_url.txt]
+  only:
+    refs:
+      - master
+    kubernetes: active
+
+# ---------------------------------------------------------------------------
+
+.auto_devops: &auto_devops |
+  # Auto DevOps variables and functions
+  [[ "$TRACE" ]] && set -x
+  auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB}
+  export DATABASE_URL=${DATABASE_URL-$auto_database_url}
+  export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG
+  export CI_APPLICATION_TAG=$CI_COMMIT_SHA
+  export CI_CONTAINER_NAME=ci_job_build_${CI_JOB_ID}
+  export TILLER_NAMESPACE=$KUBE_NAMESPACE
+  # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" for Security Products
+  export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
+
+
+  function get_replicas() {
+    track="${1:-stable}"
+    percentage="${2:-100}"
+
+    env_track=$( echo $track | tr -s  '[:lower:]'  '[:upper:]' )
+    env_slug=$( echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s  '[:lower:]'  '[:upper:]' )
+
+    if [[ "$track" == "stable" ]] || [[ "$track" == "rollout" ]]; then
+      # for stable track get number of replicas from `PRODUCTION_REPLICAS`
+      eval new_replicas=\$${env_slug}_REPLICAS
+      if [[ -z "$new_replicas" ]]; then
+        new_replicas=$REPLICAS
+      fi
+    else
+      # for all tracks get number of replicas from `CANARY_PRODUCTION_REPLICAS`
+      eval new_replicas=\$${env_track}_${env_slug}_REPLICAS
+      if [[ -z "$new_replicas" ]]; then
+        eval new_replicas=\${env_track}_REPLICAS
+      fi
+    fi
+
+    replicas="${new_replicas:-1}"
+    replicas="$(($replicas * $percentage / 100))"
+
+    # always return at least one replicas
+    if [[ $replicas -gt 0 ]]; then
+      echo "$replicas"
+    else
+      echo 1
+    fi
+  }
+
+
+  # Extracts variables prefixed with K8S_SECRET_
+  # and creates a Kubernetes secret.
+  #
+  # e.g. If we have the following environment variables:
+  #   K8S_SECRET_A=value1
+  #   K8S_SECRET_B=multi\ word\ value
+  #
+  # Then we will create a secret with the following key-value pairs:
+  #   data:
+  #     A: dmFsdWUxCg==
+  #     B: bXVsdGkgd29yZCB2YWx1ZQo=
+  function create_application_secret() {
+    track="${1-stable}"
+    export APPLICATION_SECRET_NAME=$(application_secret_name "$track")
+
+    env | sed -n "s/^K8S_SECRET_\(.*\)$/\1/p" > k8s_prefixed_variables
+
+    kubectl create secret \
+      -n "$KUBE_NAMESPACE" generic "$APPLICATION_SECRET_NAME" \
+      --from-env-file k8s_prefixed_variables -o yaml --dry-run |
+      kubectl replace -n "$KUBE_NAMESPACE" --force -f -
+
+    export APPLICATION_SECRET_CHECKSUM=$(cat k8s_prefixed_variables | sha256sum | cut -d ' ' -f 1)
+
+    rm k8s_prefixed_variables
+  }
+
+  function deploy_name() {
+    name="$CI_ENVIRONMENT_SLUG"
+    track="${1-stable}"
+
+    if [[ "$track" != "stable" ]]; then
+      name="$name-$track"
+    fi
+
+    echo $name
+  }
+
+  function application_secret_name() {
+    track="${1-stable}"
+    name=$(deploy_name "$track")
+
+    echo "${name}-secret"
+  }
+
+
+  function deploy() {
+    track="${1-stable}"
+    percentage="${2:-100}"
+    name=$(deploy_name "$track")
+
+    replicas="1"
+    service_enabled="true"
+    postgres_enabled="$POSTGRES_ENABLED"
+
+    # if track is different than stable,
+    # re-use all attached resources
+    if [[ "$track" != "stable" ]]; then
+      service_enabled="false"
+      postgres_enabled="false"
+    fi
+
+    replicas=$(get_replicas "$track" "$percentage")
+
+    if [[ "$CI_PROJECT_VISIBILITY" != "public" ]]; then
+      secret_name='gitlab-registry'
+    else
+      secret_name=''
+    fi
+
+    create_application_secret "$track"
+
+    env_slug=$(echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]')
+    eval env_ADDITIONAL_HOSTS=\$${env_slug}_ADDITIONAL_HOSTS
+    if [ -n "$env_ADDITIONAL_HOSTS" ]; then
+      additional_hosts="{$env_ADDITIONAL_HOSTS}"
+    elif [ -n "$ADDITIONAL_HOSTS" ]; then
+      additional_hosts="{$ADDITIONAL_HOSTS}"
+    fi
+
+    if [[ -n "$DB_INITIALIZE" && -z "$(helm ls -q "^$name$")" ]]; then
+      echo "Deploying first release with database initialization..."
+      helm upgrade --install \
+        --wait \
+        --set service.enabled="$service_enabled" \
+        --set releaseOverride="$CI_ENVIRONMENT_SLUG" \
+        --set image.repository="registry.gitlab.com/joshlambert/ruby-gke/master" \
+        --set image.tag="63492726c2264a0277141d6a6573c3d22ecd7de3" \
+        --set image.pullPolicy=IfNotPresent \
+        --set image.secrets[0].name="$secret_name" \
+        --set application.track="$track" \
+        --set application.database_url="$DATABASE_URL" \
+        --set application.secretName="$APPLICATION_SECRET_NAME" \
+        --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \
+        --set service.url="$CI_ENVIRONMENT_URL" \
+        --set service.additionalHosts="$additional_hosts" \
+        --set replicaCount="$replicas" \
+        --set postgresql.enabled="$postgres_enabled" \
+        --set postgresql.nameOverride="postgres" \
+        --set postgresql.postgresUser="$POSTGRES_USER" \
+        --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \
+        --set postgresql.postgresDatabase="$POSTGRES_DB" \
+        --set application.initializeCommand="$DB_INITIALIZE" \
+        --set gitlab.app="$CI_PROJECT_PATH_SLUG" \
+        --set gitlab.env="$CI_ENVIRONMENT_SLUG" \
+        --namespace="$KUBE_NAMESPACE" \
+        "$name" \
+        chart/
+
+      echo "Deploying second release..."
+      helm upgrade --reuse-values \
+        --wait \
+        --set application.initializeCommand="" \
+        --set application.migrateCommand="$DB_MIGRATE" \
+        --namespace="$KUBE_NAMESPACE" \
+        "$name" \
+        chart/
+    else
+      echo "Deploying new release..."
+      helm upgrade --install \
+        --wait \
+        --set service.enabled="$service_enabled" \
+        --set releaseOverride="$CI_ENVIRONMENT_SLUG" \
+        --set image.repository="registry.gitlab.com/joshlambert/ruby-gke/master" \
+        --set image.tag="63492726c2264a0277141d6a6573c3d22ecd7de3" \
+        --set image.pullPolicy=IfNotPresent \
+        --set image.secrets[0].name="$secret_name" \
+        --set application.track="$track" \
+        --set application.database_url="$DATABASE_URL" \
+        --set application.secretName="$APPLICATION_SECRET_NAME" \
+        --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \
+        --set service.url="$CI_ENVIRONMENT_URL" \
+        --set service.additionalHosts="$additional_hosts" \
+        --set replicaCount="$replicas" \
+        --set postgresql.enabled="$postgres_enabled" \
+        --set postgresql.nameOverride="postgres" \
+        --set postgresql.postgresUser="$POSTGRES_USER" \
+        --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \
+        --set postgresql.postgresDatabase="$POSTGRES_DB" \
+        --set application.migrateCommand="$DB_MIGRATE" \
+        --set gitlab.app="$CI_PROJECT_PATH_SLUG" \
+        --set gitlab.env="$CI_ENVIRONMENT_SLUG" \
+        --namespace="$KUBE_NAMESPACE" \
+        "$name" \
+        chart/
+    fi
+
+    kubectl rollout status -n "$KUBE_NAMESPACE" -w "deployment/$name"
+  }
+
+
+  function install_dependencies() {
+    apk add -U openssl curl tar gzip bash ca-certificates git
+    wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub
+    wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.23-r3/glibc-2.23-r3.apk
+    apk add glibc-2.23-r3.apk
+    rm glibc-2.23-r3.apk
+
+    curl "https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz" | tar zx
+    mv linux-amd64/helm /usr/bin/
+    mv linux-amd64/tiller /usr/bin/
+    helm version --client
+    tiller -version
+
+    curl -L -o /usr/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl"
+    chmod +x /usr/bin/kubectl
+    kubectl version --client
+  }
+
+  function download_chart() {
+    if [[ ! -d chart ]]; then
+      auto_chart=${AUTO_DEVOPS_CHART:-gitlab/auto-deploy-app}
+      auto_chart_name=$(basename $auto_chart)
+      auto_chart_name=${auto_chart_name%.tgz}
+    else
+      auto_chart="chart"
+      auto_chart_name="chart"
+    fi
+
+    helm init --client-only
+    helm repo add gitlab https://charts.gitlab.io
+    if [[ ! -d "$auto_chart" ]]; then
+      helm fetch ${auto_chart} --untar
+    fi
+    if [ "$auto_chart_name" != "chart" ]; then
+      mv ${auto_chart_name} chart
+    fi
+
+    helm dependency update chart/
+    helm dependency build chart/
+  }
+
+  function ensure_namespace() {
+    kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE"
+  }
+
+  function check_kube_domain() {
+    if [ -z ${AUTO_DEVOPS_DOMAIN+x} ]; then
+      echo "In order to deploy or use Review Apps, AUTO_DEVOPS_DOMAIN variable must be set"
+      echo "You can do it in Auto DevOps project settings or defining a secret variable at group or project level"
+      echo "You can also manually add it in .gitlab-ci.yml"
+      false
+    else
+      true
+    fi
+  }
+
+  function initialize_tiller() {
+    echo "Checking Tiller..."
+
+    export HELM_HOST="localhost:44134"
+    tiller -listen ${HELM_HOST} -alsologtostderr > /dev/null 2>&1 &
+    echo "Tiller is listening on ${HELM_HOST}"
+
+    if ! helm version --debug; then
+     echo "Failed to init Tiller."
+     return 1
+    fi
+    echo ""
+  }
+
+  function create_secret() {
+    echo "Create secret..."
+    if [[ "$CI_PROJECT_VISIBILITY" == "public" ]]; then
+      return
+    fi
+
+    kubectl create secret -n "$KUBE_NAMESPACE" \
+      docker-registry gitlab-registry \
+      --docker-server="$CI_REGISTRY" \
+      --docker-username="$CI_REGISTRY_USER" \
+      --docker-password="$CI_REGISTRY_PASSWORD" \
+      --docker-email="$GITLAB_USER_EMAIL" \
+      -o yaml --dry-run | kubectl replace -n "$KUBE_NAMESPACE" --force -f -
+  }
+
+  function persist_environment_url() {
+      echo $CI_ENVIRONMENT_URL > environment_url.txt
+  }
+
+before_script:
+  - *auto_devops

qa/qa/page/project/operations/metrics.rb

+# frozen_string_literal: true
+
+module QA
+  module Page
+    module Project
+      module Operations
+        class Metrics < Page::Base
+          EXPECTED_TITLE = 'Memory Usage (Total)'
+          LOADING_MESSAGE = 'Waiting for performance data'
+
+          view 'app/assets/javascripts/monitoring/components/dashboard.vue' do
+            element :prometheus_graphs
+          end
+
+          view 'app/assets/javascripts/monitoring/components/charts/time_series.vue' do
+            element :prometheus_graph_widgets
+          end
+
+          view 'app/assets/javascripts/monitoring/components/panel_type.vue' do
+            element :prometheus_widgets_dropdown
+            element :alert_widget_menu_item
+          end
+
+          def wait_for_metrics
+            wait_for_data
+            return if has_metrics?
+
+            wait_until(max_duration: 180) do
+              wait_for_data
+              has_metrics?
+            end
+          end
+
+          def wait_for_data
+            wait_until(reload: false) { !has_text?(LOADING_MESSAGE) } if has_text?(LOADING_MESSAGE)
+          end
+
+          def has_metrics?
+            within_element :prometheus_graphs do
+              has_text?(EXPECTED_TITLE)
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+QA::Page::Project::Operations::Metrics.prepend_if_ee('QA::EE::Page::Project::Operations::Metrics')

qa/qa/page/project/pipeline/index.rb

@@ -9,6 +9,7 @@ module QA::Page
 
       view 'app/assets/javascripts/pipelines/components/pipelines_table_row.vue' do
         element :pipeline_commit_status
+        element :pipeline_retry_button
       end
 
       def click_on_latest_pipeline
@@ -18,11 +19,26 @@ module QA::Page
       end
 
       def wait_for_latest_pipeline_success
+        wait_for_latest_pipeline_status { has_text?('passed') }
+      end
+
+      def wait_for_latest_pipeline_completion
+        wait_for_latest_pipeline_status { has_text?('passed') || has_text?('failed') }
+      end
+
+      def wait_for_latest_pipeline_status
         wait_until(reload: false, max_duration: 300) do
-          within_element_by_index(:pipeline_commit_status, 0) do
-            has_text?('passed')
+          within_element_by_index(:pipeline_commit_status, 0) { yield }
         end
       end
+
+      def wait_for_latest_pipeline_success_or_retry
+        wait_for_latest_pipeline_completion
+
+        if has_text?('failed')
+          click_element :pipeline_retry_button
+          wait_for_latest_pipeline_success
+        end
       end
     end
   end

qa/qa/page/project/sub_menus/operations.rb

@@ -12,6 +12,7 @@ module QA
               view 'app/views/layouts/nav/sidebar/_project.html.haml' do
                 element :link_operations
                 element :operations_environments_link
+                element :operations_metrics_link
               end
             end
           end
@@ -24,6 +25,14 @@ module QA
             end
           end
 
+          def go_to_operations_metrics
+            hover_operations do
+              within_submenu do
+                click_element(:operations_metrics_link)
+              end
+            end
+          end
+
           def go_to_operations_kubernetes
             hover_operations do
               within_submenu do

qa/qa/specs/features/ee/browser_ui/8_monitor/health/alerting_spec.rb

+# frozen_string_literal: true
+
+module QA
+  context 'Monitor' do
+    describe 'Alerts', :orchestrated, :kubernetes do
+      before do
+        @cluster = Service::KubernetesCluster.new.create!
+      end
+
+      after do
+        @cluster&.remove!
+      end
+
+      it 'allows configuration of alerts' do
+        Flow::Login.sign_in
+        project = create_project
+        create_kubernetes_cluster(project, @cluster)
+        push_repository(project)
+        wait_for_deployment
+
+        Page::Project::Operations::Metrics.perform do |metrics|
+          verify_metrics(metrics)
+          verify_add_alert(metrics)
+          verify_edit_alert(metrics)
+          verify_persist_alert(metrics)
+          verify_delete_alert(metrics)
+        end
+      end
+
+      private
+
+      def create_project
+        Resource::Project.fabricate_via_api! do |p|
+          p.name = 'alerts'
+          p.description = 'Project with alerting configured'
+        end
+      end
+
+      def create_kubernetes_cluster(project, cluster)
+        Resource::KubernetesCluster.fabricate_via_browser_ui! do |c|
+          c.project = project
+          c.cluster = cluster
+          c.install_helm_tiller = true
+          c.install_prometheus = true
+          c.install_runner = true
+        end
+      end
+
+      def push_repository(project)
+        Resource::Repository::ProjectPush.fabricate! do |push|
+          push.project = project
+          push.directory = Pathname
+            .new(__dir__)
+            .join('../../../../../../fixtures/monitored_auto_devops')
+          push.commit_message = 'Create Auto DevOps compatible gitlab-ci.yml'
+        end
+      end
+
+      def wait_for_deployment
+        Page::Project::Menu.perform(&:click_ci_cd_pipelines)
+        Page::Project::Pipeline::Index.perform(&:wait_for_latest_pipeline_success_or_retry)
+        Page::Project::Menu.perform(&:go_to_operations_metrics)
+      end
+
+      def verify_metrics(metrics)
+        metrics.wait_for_metrics
+
+        expect(metrics).to have_metrics
+        expect(metrics).not_to have_alert
+      end
+
+      def verify_add_alert(metrics)
+        metrics.write_first_alert('>', 0)
+
+        expect(metrics).to have_alert
+      end
+
+      def verify_edit_alert(metrics)
+        metrics.write_first_alert('<', 0)
+
+        expect(metrics).to have_alert('<')
+      end
+
+      def verify_persist_alert(metrics)
+        metrics.refresh
+        metrics.wait_for_metrics
+        metrics.wait_for_alert('<')
+
+        expect(metrics).to have_alert('<')
+      end
+
+      def verify_delete_alert(metrics)
+        metrics.delete_first_alert
+
+        expect(metrics).not_to have_alert('<')
+      end
+    end
+  end
+end