Track unique number of test cases parsed

Adds a new event `i_testing_test_case_parsed` which is tracked using
HLL Redis Counter to be able to count the unique number of
test cases parsed weekly.

Also, adds new add_pipelined method to HLL.

app/services/ci/build_report_result_service.rb

@@ -2,12 +2,20 @@
 
 module Ci
   class BuildReportResultService
+    include Gitlab::Utils::UsageData
+
+    EVENT_NAME = 'i_testing_test_case_parsed'
+
     def execute(build)
       return unless build.has_test_reports?
 
+      test_suite = generate_test_suite_report(build)
+
+      track_test_cases(build, test_suite)
+
       build.report_results.create!(
         project_id: build.project_id,
-        data: tests_params(build)
+        data: tests_params(test_suite)
       )
     end
 
@@ -17,9 +25,7 @@ module Ci
       build.collect_test_reports!(Gitlab::Ci::Reports::TestReports.new)
     end
 
-    def tests_params(build)
-      test_suite = generate_test_suite_report(build)
-
+    def tests_params(test_suite)
       {
         tests: {
           name: test_suite.name,
@@ -31,5 +37,20 @@ module Ci
         }
       }
     end
+
+    def track_test_cases(build, test_suite)
+      return if Feature.disabled?(:track_unique_test_cases_parsed, build.project)
+
+      track_usage_event(EVENT_NAME, test_case_hashes(build, test_suite))
+    end
+
+    def test_case_hashes(build, test_suite)
+      [].tap do |hashes|
+        test_suite.each_test_case do |test_case|
+          key = "#{build.project_id}-#{test_suite.name}-#{test_case.key}"
+          hashes << Digest::SHA256.hexdigest(key)
+        end
+      end
+    end
   end
 end

changelogs/unreleased/eb-unit-tests-parsed-usage-ping.yml

+---
+title: Track unique number of test cases parsed
+merge_request: 41918
+author:
+type: added

config/feature_flags/development/track_unique_test_cases_parsed.yml

+---
+name: track_unique_test_cases_parsed
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41918
+rollout_issue_url:
+group: group::testing
+type: development
+default_enabled: false

config/feature_flags/development/usage_data_i_testing_test_case_parsed.yml

+---
+name: usage_data_i_testing_test_case_parsed
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41918
+rollout_issue_url:
+group: group::testing
+type: development
+default_enabled: true

lib/gitlab/ci/reports/test_case.rb

@@ -23,7 +23,7 @@ module Gitlab
           @attachment = params.fetch(:attachment, nil)
           @job = params.fetch(:job, nil)
 
-          @key = sanitize_key_name("#{classname}_#{name}")
+          @key = hash_key("#{classname}_#{name}")
         end
 
         def has_attachment?
@@ -42,8 +42,8 @@ module Gitlab
 
         private
 
-        def sanitize_key_name(key)
-          key.gsub(/[^0-9A-Za-z]/, '-')
+        def hash_key(key)
+          Digest::SHA256.hexdigest(key)
         end
       end
     end

lib/gitlab/ci/reports/test_suite.rb

@@ -12,18 +12,24 @@ module Gitlab
         def initialize(name = nil)
           @name = name
           @test_cases = {}
+          @all_test_cases = []
           @total_time = 0.0
-          @duplicate_cases = []
         end
 
         def add_test_case(test_case)
-          @duplicate_cases << test_case if existing_key?(test_case)
-
           @test_cases[test_case.status] ||= {}
           @test_cases[test_case.status][test_case.key] = test_case
           @total_time += test_case.execution_time
         end
 
+        def each_test_case
+          @test_cases.each do |status, test_cases|
+            test_cases.values.each do |test_case|
+              yield test_case
+            end
+          end
+        end
+
         # rubocop: disable CodeReuse/ActiveRecord
         def total_count
           return 0 if suite_error
@@ -86,10 +92,6 @@ module Gitlab
 
         private
 
-        def existing_key?(test_case)
-          @test_cases[test_case.status]&.key?(test_case.key)
-        end
-
         def sort_by_status
           @test_cases = @test_cases.sort_by { |status, _| Gitlab::Ci::Reports::TestCase::STATUS_TYPES.index(status) }.to_h
         end

lib/gitlab/redis/hll.rb

@@ -3,6 +3,7 @@
 module Gitlab
   module Redis
     class HLL
+      BATCH_SIZE = 300
       KEY_REGEX = %r{\A(\w|-|:)*\{\w*\}(\w|-|:)*\z}.freeze
       KeyFormatError = Class.new(StandardError)
 
@@ -29,17 +30,24 @@ module Gitlab
       #   2020-216-{project_action}
       #   i_{analytics}_dev_ops_score-2020-32
       def add(key:, value:, expiry:)
-        unless KEY_REGEX.match?(key)
-          raise KeyFormatError.new("Invalid key format. #{key} key should have changeable parts in curly braces. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands")
-        end
+        validate_key!(key)
 
         Gitlab::Redis::SharedState.with do |redis|
           redis.multi do |multi|
-            multi.pfadd(key, value)
+            Array.wrap(value).each_slice(BATCH_SIZE) { |batch| multi.pfadd(key, batch) }
+
             multi.expire(key, expiry)
           end
         end
       end
+
+      private
+
+      def validate_key!(key)
+        return if KEY_REGEX.match?(key)
+
+        raise KeyFormatError.new("Invalid key format. #{key} key should have changeable parts in curly braces. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands")
+      end
     end
   end
 end

lib/gitlab/usage_data_counters/known_events.yml

@@ -185,6 +185,11 @@
   redis_slot: incident_management
   category: incident_management
   aggregation: weekly
+# Testing category
+- name: i_testing_test_case_parsed
+  category: testing
+  redis_slot: testing
+  aggregation: weekly
 # Project Management group
 - name: g_project_management_issue_title_changed
   category: issues_edit

spec/lib/gitlab/ci/reports/test_suite_spec.rb

@@ -229,6 +229,20 @@ RSpec.describe Gitlab::Ci::Reports::TestSuite do
     end
   end
 
+  describe '#each_test_case' do
+    before do
+      test_suite.add_test_case(test_case_success)
+      test_suite.add_test_case(test_case_failed)
+      test_suite.add_test_case(test_case_skipped)
+      test_suite.add_test_case(test_case_error)
+    end
+
+    it 'yields each test case to given block' do
+      expect { |b| test_suite.each_test_case(&b) }
+        .to yield_successive_args(test_case_success, test_case_failed, test_case_skipped, test_case_error)
+    end
+  end
+
   Gitlab::Ci::Reports::TestCase::STATUS_TYPES.each do |status_type|
     describe "##{status_type}_count" do
       subject { test_suite.public_send("#{status_type}_count") }

spec/lib/gitlab/redis/hll_spec.rb

@@ -39,6 +39,24 @@ RSpec.describe Gitlab::Redis::HLL, :clean_gitlab_redis_shared_state do
         end
       end
     end
+
+    context 'when adding entries' do
+      let(:metric) { 'test-{metric}' }
+
+      it 'supports single value' do
+        track_event(metric, 1)
+
+        expect(count_unique_events([metric])).to eq(1)
+      end
+
+      it 'supports multiple values' do
+        stub_const("#{described_class.name}::HLL_BATCH_SIZE", 2)
+
+        track_event(metric, [1, 2, 3, 4, 5])
+
+        expect(count_unique_events([metric])).to eq(5)
+      end
+    end
   end
 
   describe '.count' do
@@ -94,13 +112,13 @@ RSpec.describe Gitlab::Redis::HLL, :clean_gitlab_redis_shared_state do
 
       expect(unique_counts).to eq(4)
     end
+  end
 
-    def track_event(key, value, expiry = 1.day)
-      described_class.add(key: key, value: value, expiry: expiry)
-    end
+  def track_event(key, value, expiry = 1.day)
+    described_class.add(key: key, value: value, expiry: expiry)
+  end
 
-    def count_unique_events(keys)
-      described_class.count(keys: keys)
-    end
+  def count_unique_events(keys)
+    described_class.count(keys: keys)
   end
 end

spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb

@@ -20,7 +20,7 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
 
   describe '.categories' do
     it 'gets all unique category names' do
-      expect(described_class.categories).to contain_exactly('analytics', 'compliance', 'ide_edit', 'search', 'source_code', 'incident_management', 'issues_edit')
+      expect(described_class.categories).to contain_exactly('analytics', 'compliance', 'ide_edit', 'search', 'source_code', 'incident_management', 'issues_edit', 'testing')
     end
   end
 

spec/lib/gitlab/usage_data_spec.rb

@@ -1178,9 +1178,9 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
     subject { described_class.redis_hll_counters }
 
     let(:categories) { ::Gitlab::UsageDataCounters::HLLRedisCounter.categories }
-    let(:ineligible_total_categories) { ['source_code'] }
+    let(:ineligible_total_categories) { %w[source_code testing] }
 
-    it 'has all know_events' do
+    it 'has all known_events' do
       expect(subject).to have_key(:redis_hll_counters)
 
       expect(subject[:redis_hll_counters].keys).to match_array(categories)

spec/services/ci/build_report_result_service_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 RSpec.describe Ci::BuildReportResultService do
-  describe "#execute" do
+  describe '#execute', :clean_gitlab_redis_shared_state do
     subject(:build_report_result) { described_class.new.execute(build) }
 
     context 'when build is finished' do
@@ -17,6 +17,25 @@ RSpec.describe Ci::BuildReportResultService do
         expect(build_report_result.tests_skipped).to eq(0)
         expect(build_report_result.tests_duration).to eq(0.010284)
         expect(Ci::BuildReportResult.count).to eq(1)
+
+        unique_test_cases_parsed = Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(
+          event_names: described_class::EVENT_NAME,
+          start_date: 2.weeks.ago,
+          end_date: 2.weeks.from_now
+        )
+        expect(unique_test_cases_parsed).to eq(4)
+      end
+
+      context 'when feature flag for tracking is disabled' do
+        before do
+          stub_feature_flags(track_unique_test_cases_parsed: false)
+        end
+
+        it 'creates the report but does not track the event' do
+          expect(Gitlab::UsageDataCounters::HLLRedisCounter).not_to receive(:track_event)
+          expect(build_report_result.tests_name).to eq("test")
+          expect(Ci::BuildReportResult.count).to eq(1)
+        end
       end
 
       context 'when data has already been persisted' do