Merge branch 'generate-sample-metrics-intervals' into 'master'

Generate Prometheus sample metrics over pre-set intervals

Closes #36251

See merge request gitlab-org/gitlab!22066

app/controllers/projects/environments/sample_metrics_controller.rb

@@ -2,7 +2,7 @@
 
 class Projects::Environments::SampleMetricsController < Projects::ApplicationController
   def query
-    result = Metrics::SampleMetricsService.new(params[:identifier]).query
+    result = Metrics::SampleMetricsService.new(params[:identifier], range_start: params[:start], range_end: params[:end]).query
 
     if result
       render json: { "status": "success", "data": { "resultType": "matrix", "result": result } }

app/services/metrics/sample_metrics_service.rb

@@ -4,16 +4,17 @@ module Metrics
   class SampleMetricsService
     DIRECTORY = "sample_metrics"
 
-    attr_reader :identifier
+    attr_reader :identifier, :range_minutes
 
-    def initialize(identifier)
+    def initialize(identifier, range_start:, range_end:)
       @identifier = identifier
+      @range_minutes = convert_range_minutes(range_start, range_end)
     end
 
     def query
       return unless identifier && File.exist?(file_location)
 
-      YAML.load_file(File.expand_path(file_location, __dir__))
+      query_interval
     end
 
     private
@@ -22,5 +23,14 @@ module Metrics
       sanitized_string = identifier.gsub(/[^0-9A-Za-z_]/, '')
       File.join(Rails.root, DIRECTORY, "#{sanitized_string}.yml")
     end
+
+    def query_interval
+      result = YAML.load_file(File.expand_path(file_location, __dir__))
+      result[range_minutes]
+    end
+
+    def convert_range_minutes(range_start, range_end)
+      ((range_end.to_time - range_start.to_time) / 1.minute).to_i
+    end
   end
 end

changelogs/unreleased/generate-sample-metrics-intervals.yml

+---
+title: Generate Prometheus sample metrics over pre-set intervals
+merge_request: 22066
+author:
+type: added

doc/raketasks/generate_sample_prometheus_data.md

 # Generate Sample Prometheus Data
 
 This command will run Prometheus queries for each of the metrics of a specific environment
-for a default time interval of 7 days ago to now. The results of each of query are stored
-under a `sample_metrics` directory as a yaml file named by the metric's `identifier`.
-When the environmental variable `USE_SAMPLE_METRICS` is set, the Prometheus API query is
-re-routed to `Projects::Environments::SampleMetricsController` which loads the appropriate
-data set if it is present within the `sample_metrics` directory.
+for a series of time intervals: 30 minutes, 3 hours, 8 hours, 24 hours, 72 hours, and 7 days
+to now. The results of each of query are stored under a `sample_metrics` directory as a yaml
+file named by the metric's `identifier`. When the environmental variable `USE_SAMPLE_METRICS`
+is set, the Prometheus API query is re-routed to `Projects::Environments::SampleMetricsController`
+which loads the appropriate data set if it is present within the `sample_metrics` directory.
 
 - This command requires an id from an Environment with an available Prometheus installation.
 

lib/tasks/gitlab/generate_sample_prometheus_data.rake

@@ -8,12 +8,17 @@ namespace :gitlab do
     sample_metrics_directory_name = Metrics::SampleMetricsService::DIRECTORY
     FileUtils.mkdir_p(sample_metrics_directory_name)
 
+    sample_metrics_intervals = [30.minutes, 180.minutes, 8.hours, 24.hours, 72.hours, 7.days]
+
     metrics.each do |metric|
       query = metric.query % query_variables
-      result = environment.prometheus_adapter.prometheus_client.query_range(query, start: 7.days.ago)
 
       next unless metric.identifier
 
+      result = sample_metrics_intervals.each_with_object({}) do |interval, memo|
+        memo[interval.to_i / 60] = environment.prometheus_adapter.prometheus_client.query_range(query, start: interval.ago)
+      end
+
       File.write("#{sample_metrics_directory_name}/#{metric.identifier}.yml", result.to_yaml)
     end
   end

spec/controllers/projects/environments/sample_metrics_controller_spec.rb

@@ -58,7 +58,9 @@ describe Projects::Environments::SampleMetricsController do
       id: environment.id.to_s,
       namespace_id: project.namespace.full_path,
       project_id: project.name,
-      identifier: 'sample_metric_query_result'
+      identifier: 'sample_metric_query_result',
+      start: '2019-12-02T23:31:45.000Z',
+      end: '2019-12-03T00:01:45.000Z'
     }.merge(params)
   end
 end

spec/services/metrics/sample_metrics_service_spec.rb

@@ -4,7 +4,10 @@ require 'spec_helper'
 
 describe Metrics::SampleMetricsService do
   describe 'query' do
-    subject { described_class.new(identifier).query }
+    let(:range_start) { '2019-12-02T23:31:45.000Z' }
+    let(:range_end) { '2019-12-03T00:01:45.000Z' }
+
+    subject { described_class.new(identifier, range_start: range_start, range_end: range_end).query }
 
     context 'when the file is not found' do
       let(:identifier) { nil }
@@ -26,10 +29,10 @@ describe Metrics::SampleMetricsService do
         FileUtils.rm(destination)
       end
 
-      subject { described_class.new(identifier).query }
+      subject { described_class.new(identifier, range_start: range_start, range_end: range_end).query }
 
       it 'loads data from the sample file correctly' do
-        expect(subject).to eq(YAML.load_file(source))
+        expect(subject).to eq(YAML.load_file(source)[30])
       end
     end
 

spec/tasks/gitlab/generate_sample_prometheus_data_spec.rb

@@ -17,7 +17,7 @@ describe 'gitlab:generate_sample_prometheus_data rake task' do
   it 'creates the file correctly' do
     Rake.application.rake_require 'tasks/gitlab/generate_sample_prometheus_data'
     allow(Environment).to receive(:find).and_return(environment)
-    allow(environment).to receive_message_chain(:prometheus_adapter, :prometheus_client, :query_range) { sample_query_result }
+    allow(environment).to receive_message_chain(:prometheus_adapter, :prometheus_client, :query_range) { sample_query_result[30] }
     run_rake_task('gitlab:generate_sample_prometheus_data', [environment.id])
 
     expect(File.exist?(sample_query_file)).to be true