Merge branch '239449-set-42-days-expiration-time-for-redis-hll-metrics' into 'master'

Set 42 days expiration time for Redis HLL metrics

Closes #239449

See merge request gitlab-org/gitlab!40159

lib/gitlab/usage_data_counters/known_events.yml

@@ -3,86 +3,69 @@
 - name: g_compliance_dashboard
   redis_slot: compliance
   category: compliance
-  expiry: 84 # expiration time in days, equivalent to 12 weeks
   aggregation: weekly
 - name: g_compliance_audit_events
   category: compliance
   redis_slot: compliance
-  expiry: 84
   aggregation: weekly
 - name: i_compliance_audit_events
   category: compliance
   redis_slot: compliance
-  expiry: 84
   aggregation: weekly
 - name: i_compliance_credential_inventory
   category: compliance
   redis_slot: compliance
-  expiry: 84
   aggregation: weekly
 # Analytics category
 - name: g_analytics_contribution
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
 - name: g_analytics_insights
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
 - name: g_analytics_issues
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
 - name: g_analytics_productivity
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
 - name: g_analytics_valuestream
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
 - name: p_analytics_pipelines
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
 - name: p_analytics_code_reviews
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
 - name: p_analytics_valuestream
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
 - name: p_analytics_insights
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
 - name: p_analytics_issues
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
 - name: p_analytics_repo
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
 - name: i_analytics_cohorts
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
 - name: i_analytics_dev_ops_score
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly

spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb

@@ -9,17 +9,23 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
   let(:entity4) { '8b9a2671-2abf-4bec-a682-22f6a8f7bf31' }
 
   let(:weekly_event) { 'g_analytics_contribution' }
-  let(:daily_event) { 'g_search' }
+  let(:daily_event) { 'g_analytics_search' }
+  let(:analytics_slot_event) { 'g_analytics_contribution' }
+  let(:compliance_slot_event) { 'g_compliance_dashboard' }
+  let(:category_analytics) { 'g_analytics_search' }
+  let(:category_productivity) { 'g_analytics_productivity' }
+  let(:no_slot) { 'no_slot' }
   let(:different_aggregation) { 'different_aggregation' }
+  let(:custom_daily_event) { 'g_analytics_custom' }
 
   let(:known_events) do
     [
-      { name: "g_analytics_contribution", redis_slot: "analytics", category: "analytics", expiry: 84, aggregation: "weekly" },
-      { name: "g_analytics_valuestream", redis_slot: "analytics", category: "analytics", expiry: 84, aggregation: "daily" },
-      { name: "g_analytics_productivity", redis_slot: "analytics", category: "productivity", expiry: 84, aggregation: "weekly" },
-      { name: "g_compliance_dashboard", redis_slot: "compliance", category: "compliance", aggregation: "weekly" },
-      { name: "g_search", category: "global", aggregation: "daily" },
-      { name: "different_aggregation", category: "global", aggregation: "monthly" }
+      { name: weekly_event, redis_slot: "analytics", category: "analytics", expiry: 84, aggregation: "weekly" },
+      { name: daily_event, redis_slot: "analytics", category: "analytics", expiry: 84, aggregation: "daily" },
+      { name: category_productivity, redis_slot: "analytics", category: "productivity", aggregation: "weekly" },
+      { name: compliance_slot_event, redis_slot: "compliance", category: "compliance", aggregation: "weekly" },
+      { name: no_slot, category: "global", aggregation: "daily" },
+      { name: different_aggregation, category: "global", aggregation: "monthly" }
     ].map(&:with_indifferent_access)
   end
 
@@ -46,7 +52,8 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
       expect { described_class.track_event(entity1, 'unknown', Date.current) } .to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::UnknownEvent)
     end
 
-    it 'sets the keys in Redis to expire automatically after 12 weeks' do
+    context 'for weekly events' do
+      it 'sets the keys in Redis to expire automatically after the given expiry time' do
         described_class.track_event(entity1, "g_analytics_contribution")
 
         Gitlab::Redis::SharedState.with do |redis|
@@ -73,6 +80,35 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
       end
     end
 
+    context 'for daily events' do
+      it 'sets the keys in Redis to expire after the given expiry time' do
+        described_class.track_event(entity1, "g_analytics_search")
+
+        Gitlab::Redis::SharedState.with do |redis|
+          keys = redis.scan_each(match: "*-g_{analytics}_search").to_a
+          expect(keys).not_to be_empty
+
+          keys.each do |key|
+            expect(redis.ttl(key)).to be_within(5.seconds).of(84.days)
+          end
+        end
+      end
+
+      it 'sets the keys in Redis to expire after 29 days by default' do
+        described_class.track_event(entity1, "no_slot")
+
+        Gitlab::Redis::SharedState.with do |redis|
+          keys = redis.scan_each(match: "*-{no_slot}").to_a
+          expect(keys).not_to be_empty
+
+          keys.each do |key|
+            expect(redis.ttl(key)).to be_within(5.seconds).of(29.days)
+          end
+        end
+      end
+    end
+  end
+
   describe '.unique_events' do
     before do
       # events in current week, should not be counted as week is not complete
@@ -82,6 +118,7 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
       # Events last week
       described_class.track_event(entity1, weekly_event, 2.days.ago)
       described_class.track_event(entity1, weekly_event, 2.days.ago)
+      described_class.track_event(entity1, no_slot, 2.days.ago)
 
       # Events 2 weeks ago
       described_class.track_event(entity1, weekly_event, 2.weeks.ago)
@@ -107,15 +144,15 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
     end
 
     it 'raise error if metrics are not in the same slot' do
-      expect { described_class.unique_events(event_names: %w(g_analytics_contribution g_compliance_dashboard), start_date: 4.weeks.ago, end_date: Date.current) }.to raise_error('Events should be in same slot')
+      expect { described_class.unique_events(event_names: [compliance_slot_event, analytics_slot_event], start_date: 4.weeks.ago, end_date: Date.current) }.to raise_error('Events should be in same slot')
     end
 
     it 'raise error if metrics are not in the same category' do
-      expect { described_class.unique_events(event_names: %w(g_analytics_contribution g_analytics_productivity), start_date: 4.weeks.ago, end_date: Date.current) }.to raise_error('Events should be in same category')
+      expect { described_class.unique_events(event_names: [category_analytics, category_productivity], start_date: 4.weeks.ago, end_date: Date.current) }.to raise_error('Events should be in same category')
     end
 
     it "raise error if metrics don't have same aggregation" do
-      expect { described_class.unique_events(event_names: %w(g_analytics_contribution g_analytics_valuestream), start_date: 4.weeks.ago, end_date: Date.current) }.to raise_error('Events should have same aggregation level')
+      expect { described_class.unique_events(event_names: [daily_event, weekly_event], start_date: 4.weeks.ago, end_date: Date.current) }.to raise_error('Events should have same aggregation level')
     end
 
     context 'when data for the last complete week' do
@@ -135,5 +172,9 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
       it { expect(described_class.unique_events(event_names: daily_event, start_date: 28.days.ago, end_date: Date.current)).to eq(3) }
       it { expect(described_class.unique_events(event_names: daily_event, start_date: 28.days.ago, end_date: 21.days.ago)).to eq(1) }
     end
+
+    context 'when no slot is set' do
+      it { expect(described_class.unique_events(event_names: no_slot, start_date: 7.days.ago, end_date: Date.current)).to eq(1) }
+    end
   end
 end