Merge branch 'sh-store-rspec-profiling-csv' into 'master'

Batch insert CI profiling spec data

Closes gitlab-ee#10154

See merge request gitlab-org/gitlab-ce!25937

.gitlab-ci.yml

@@ -66,6 +66,7 @@ stages:
     paths:
       - knapsack/
       - rspec_flaky/
+      - rspec_profiling/
 
 .use-pg: &use-pg
   services:
@@ -159,6 +160,7 @@ stages:
       - coverage/
       - knapsack/
       - rspec_flaky/
+      - rspec_profiling/
       - tmp/capybara/
     reports:
       junit: junit_rspec.xml
@@ -336,6 +338,7 @@ retrieve-tests-metadata:
     - wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $KNAPSACK_RSPEC_SUITE_REPORT_PATH
     - '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
     - mkdir -p rspec_flaky/
+    - mkdir -p rspec_profiling/
     - wget -O $FLAKY_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$FLAKY_RSPEC_SUITE_REPORT_PATH || rm $FLAKY_RSPEC_SUITE_REPORT_PATH
     - '[[ -f $FLAKY_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_SUITE_REPORT_PATH}'
 
@@ -350,7 +353,7 @@ update-tests-metadata:
       - rspec_flaky/
     policy: push
   script:
-    - retry gem install fog-aws mime-types activesupport --no-document
+    - retry gem install fog-aws mime-types activesupport rspec_profiling postgres-copy --no-document
     - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec-pg_node_*.json
     - scripts/merge-reports ${FLAKY_RSPEC_SUITE_REPORT_PATH} rspec_flaky/all_*_*.json
     - FLAKY_RSPEC_GENERATE_REPORT=1 scripts/prune-old-flaky-specs ${FLAKY_RSPEC_SUITE_REPORT_PATH}
@@ -358,6 +361,7 @@ update-tests-metadata:
     - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $FLAKY_RSPEC_SUITE_REPORT_PATH'
     - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
     - rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json
+    - scripts/insert-rspec-profiling-data
 
 flaky-examples-check:
   <<: *dedicated-runner

config/initializers/rspec_profiling.rb

+# frozen_string_literal: true
+
+return unless Rails.env.test?
+
 module RspecProfilingExt
-  module PSQL
-    def establish_connection
-      ::RspecProfiling::Collectors::PSQL::Result.establish_connection(ENV['RSPEC_PROFILING_POSTGRES_URL'])
+  module Collectors
+    class CSVWithTimestamps < ::RspecProfiling::Collectors::CSV
+      TIMESTAMP_FIELDS = %w(created_at updated_at).freeze
+      HEADERS = (::RspecProfiling::Collectors::CSV::HEADERS + TIMESTAMP_FIELDS).freeze
+
+      def insert(attributes)
+        output << HEADERS.map do |field|
+          if TIMESTAMP_FIELDS.include?(field)
+            Time.now
+          else
+            attributes.fetch(field.to_sym)
+          end
+        end
+      end
+
+      private
+
+      def output
+        @output ||= ::CSV.open(path, "w").tap { |csv| csv << HEADERS }
+      end
     end
   end
 
@@ -10,8 +31,12 @@ module RspecProfilingExt
       if ENV['CI_COMMIT_REF_NAME']
         "#{defined?(Gitlab::License) ? 'ee' : 'ce'}:#{ENV['CI_COMMIT_REF_NAME']}"
       else
-        super
+        super&.chomp
+      end
     end
+
+    def sha
+      super&.chomp
     end
   end
 
@@ -30,16 +55,11 @@ module RspecProfilingExt
   end
 end
 
-if Rails.env.test?
-  RspecProfiling.configure do |config|
-    if ENV['RSPEC_PROFILING_POSTGRES_URL'].present?
-      RspecProfiling::Collectors::PSQL.prepend(RspecProfilingExt::PSQL)
-      config.collector = RspecProfiling::Collectors::PSQL
-    end
-
-    if ENV.key?('CI')
+RspecProfiling.configure do |config|
+  if ENV.key?('CI') || ENV.key?('RSPEC_PROFILING')
     RspecProfiling::VCS::Git.prepend(RspecProfilingExt::Git)
     RspecProfiling::Run.prepend(RspecProfilingExt::Run)
-    end
+    config.collector = RspecProfilingExt::Collectors::CSVWithTimestamps
+    config.csv_path = -> { "rspec_profiling/#{Time.now.to_i}-#{SecureRandom.hex(8)}-rspec-data.csv" }
   end
 end

scripts/insert-rspec-profiling-data

+#!/usr/bin/env ruby
+
+require 'csv'
+require 'rspec_profiling'
+require 'postgres-copy'
+
+module RspecProfiling
+  module Collectors
+    class PSQL
+      def establish_connection
+        # This disables the automatic creation of the database and
+        # table. In the future, we may want a way to specify the host of
+        # the database to connect so that we can call #install.
+        Result.establish_connection(results_url)
+      end
+
+      def prepared?
+        connection.data_source_exists?(table)
+      end
+
+      def results_url
+        ENV['RSPEC_PROFILING_POSTGRES_URL']
+      end
+
+      class Result < ActiveRecord::Base
+        acts_as_copy_target
+      end
+    end
+  end
+end
+
+def insert_data(path)
+  puts "#{Time.now} Inserting CI stats..."
+
+  collector = RspecProfiling::Collectors::PSQL.new
+  collector.install
+
+  files = Dir[File.join(path, "*.csv")]
+
+  files.each do |filename|
+    puts "#{Time.now} Inserting #{filename}..."
+    result = RspecProfiling::Collectors::PSQL::Result.copy_from(filename)
+    puts "#{Time.now} Inserted #{result.cmd_tuples} lines in #{filename}, DB response: #{result.cmd_status}"
+  end
+end
+
+insert_data('rspec_profiling') if ENV['RSPEC_PROFILING_POSTGRES_URL'].present?