Merge branch 'ci-platform-metrics' into 'master'

Adds CI Platform Metrics bookkeeping model

See merge request gitlab-org/gitlab!40036

app/models/ci_platform_metric.rb

+# frozen_string_literal: true
+
+class CiPlatformMetric < ApplicationRecord
+  validates :recorded_at, presence: true
+  validates :platform_target, presence: true, length: { maximum: 255 }
+  validates :count, presence: true
+
+  CI_VARIABLE_KEY = "AUTO_DEVOPS_PLATFORM_TARGET"
+
+  def self.update!
+    # This work can NOT be done in-database because value is encrypted.
+    # However, for "AUTO_DEVOPS_PLATFORM_TARGET", these values are only
+    # encrypted as a matter of course, rather than as a need for secrecy.
+    # So this is not a security risk, but exposing other keys possibly could be.
+    variables = Ci::Variable.by_key(CI_VARIABLE_KEY)
+    update_recorded_at = Time.zone.now
+    counts = variables.group_by(&:value).map do |value, variables|
+      {
+        recorded_at: update_recorded_at,
+        platform_target: value,
+        count: variables.count
+      }
+    end
+
+    create(counts)
+  end
+end

changelogs/unreleased/ci-platform-metrics.yml

+---
+title: Adds CI Platform Metrics bookkeeping model
+merge_request: 40036
+author:
+type: added

db/migrate/20200820204041_create_ci_platform_metrics.rb

+# frozen_string_literal: true
+
+class CreateCiPlatformMetrics < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    unless table_exists?(:ci_platform_metrics)
+      create_table :ci_platform_metrics do |t|
+        t.datetime_with_timezone :recorded_at, null: false
+        t.text :platform_target, null: false
+        t.integer :count, null: false
+      end
+    end
+
+    add_text_limit :ci_platform_metrics, :platform_target, 255
+    add_concurrent_index :ci_variables, :key
+  end
+
+  def down
+    drop_table :ci_platform_metrics
+    remove_concurrent_index :ci_variables, :key
+  end
+end

db/schema_migrations/20200820204041

+ce4d108c6587943ab3740dcc39298d6877d7317ec1023d8d263cecd9f1e0f478
\ No newline at end of file

db/structure.sql

@@ -10233,6 +10233,23 @@ CREATE SEQUENCE public.ci_pipelines_id_seq
 
 ALTER SEQUENCE public.ci_pipelines_id_seq OWNED BY public.ci_pipelines.id;
 
+CREATE TABLE public.ci_platform_metrics (
+    id bigint NOT NULL,
+    recorded_at timestamp with time zone NOT NULL,
+    platform_target text NOT NULL,
+    count integer NOT NULL,
+    CONSTRAINT check_f922abc32b CHECK ((char_length(platform_target) <= 255))
+);
+
+CREATE SEQUENCE public.ci_platform_metrics_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE public.ci_platform_metrics_id_seq OWNED BY public.ci_platform_metrics.id;
+
 CREATE TABLE public.ci_refs (
     id bigint NOT NULL,
     project_id bigint NOT NULL,
@@ -16857,6 +16874,8 @@ ALTER TABLE ONLY public.ci_pipelines ALTER COLUMN id SET DEFAULT nextval('public
 
 ALTER TABLE ONLY public.ci_pipelines_config ALTER COLUMN pipeline_id SET DEFAULT nextval('public.ci_pipelines_config_pipeline_id_seq'::regclass);
 
+ALTER TABLE ONLY public.ci_platform_metrics ALTER COLUMN id SET DEFAULT nextval('public.ci_platform_metrics_id_seq'::regclass);
+
 ALTER TABLE ONLY public.ci_refs ALTER COLUMN id SET DEFAULT nextval('public.ci_refs_id_seq'::regclass);
 
 ALTER TABLE ONLY public.ci_resource_groups ALTER COLUMN id SET DEFAULT nextval('public.ci_resource_groups_id_seq'::regclass);
@@ -17821,6 +17840,9 @@ ALTER TABLE ONLY public.ci_pipelines_config
 ALTER TABLE ONLY public.ci_pipelines
     ADD CONSTRAINT ci_pipelines_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY public.ci_platform_metrics
+    ADD CONSTRAINT ci_platform_metrics_pkey PRIMARY KEY (id);
+
 ALTER TABLE ONLY public.ci_refs
     ADD CONSTRAINT ci_refs_pkey PRIMARY KEY (id);
 
@@ -19403,6 +19425,8 @@ CREATE INDEX index_ci_triggers_on_owner_id ON public.ci_triggers USING btree (ow
 
 CREATE INDEX index_ci_triggers_on_project_id ON public.ci_triggers USING btree (project_id);
 
+CREATE INDEX index_ci_variables_on_key ON public.ci_variables USING btree (key);
+
 CREATE UNIQUE INDEX index_ci_variables_on_project_id_and_key_and_environment_scope ON public.ci_variables USING btree (project_id, key, environment_scope);
 
 CREATE INDEX index_cluster_agent_tokens_on_agent_id ON public.cluster_agent_tokens USING btree (agent_id);

spec/factories/ci_platform_metrics.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :ci_platform_metric do
+    recorded_at { Time.zone.now }
+    platform_target { generate(:title) }
+    count { SecureRandom.random_number(100) }
+  end
+end

spec/models/ci_platform_metric_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe CiPlatformMetric do
+  subject { build(:ci_platform_metric) }
+
+  describe 'validations' do
+    it { is_expected.to validate_presence_of(:recorded_at) }
+    it { is_expected.to validate_presence_of(:count) }
+    it { is_expected.to validate_presence_of(:platform_target) }
+    it { is_expected.to validate_length_of(:platform_target).is_at_most(255) }
+  end
+
+  describe '.update!' do
+    def platform_target_counts_by_day
+      report = Hash.new { |hash, key| hash[key] = {} }
+      CiPlatformMetric.all.each do |metric|
+        date = metric.recorded_at.to_date
+        report[date][metric.platform_target] = metric.count
+      end
+      report
+    end
+
+    context "when there is already existing metrics data" do
+      let!(:metric_1) { create(:ci_platform_metric) }
+      let!(:metric_2) { create(:ci_platform_metric) }
+
+      it "does not erase any existing data" do
+        CiPlatformMetric.update!
+
+        expect(CiPlatformMetric.all.to_a).to contain_exactly(metric_1, metric_2)
+      end
+    end
+
+    context "when there are multiple platform target variables" do
+      let(:today) { Time.zone.local(1982, 4, 24) }
+      let(:tomorrow) { today + 1.day }
+
+      it "updates platform target counts for that day" do
+        Timecop.freeze(today) do
+          create(:ci_variable, key: described_class::CI_VARIABLE_KEY, value: "aws")
+          create(:ci_variable, key: described_class::CI_VARIABLE_KEY, value: "aws")
+          create(:ci_variable, key: described_class::CI_VARIABLE_KEY, value: "fargate")
+          create(:ci_variable, key: described_class::CI_VARIABLE_KEY, value: "fargate")
+          create(:ci_variable, key: described_class::CI_VARIABLE_KEY, value: "fargate")
+          CiPlatformMetric.update!
+        end
+        Timecop.freeze(tomorrow) do
+          create(:ci_variable, key: described_class::CI_VARIABLE_KEY, value: "fargate")
+          CiPlatformMetric.update!
+        end
+
+        expect(platform_target_counts_by_day).to eq({
+          today.to_date => { "aws" => 2, "fargate" => 3 },
+          tomorrow.to_date => { "aws" => 2, "fargate" => 4 }
+        })
+      end
+    end
+
+    context "when there are no platform target variables" do
+      it "does not generate any new platform metrics" do
+        create(:ci_variable, key: "KEY_WHATEVER", value: "aws")
+        CiPlatformMetric.update!
+
+        expect(platform_target_counts_by_day).to eq({})
+      end
+    end
+  end
+end