Add new ScheduleUpdateUserActivityWorker and UpdateUserActivityWorker

Signed-off-by: Rémy Coutable <remy@rymai.me>

app/workers/schedule_update_user_activity_worker.rb

+class ScheduleUpdateUserActivityWorker
+  include Sidekiq::Worker
+  include CronjobQueue
+
+  def perform(batch_size = 500)
+    return if Gitlab::Geo.secondary?
+
+    Gitlab::UserActivities.new.each_slice(batch_size) do |batch|
+      UpdateUserActivityWorker.perform_async(Hash[batch])
+    end
+  end
+end

app/workers/update_user_activity_worker.rb

+class UpdateUserActivityWorker
+  include Sidekiq::Worker
+  include DedicatedSidekiqQueue
+
+  def perform(pairs)
+    return if Gitlab::Geo.secondary?
+
+    pairs = cast_data(pairs)
+    ids = pairs.keys
+    conditions = 'WHEN id = ? THEN ? ' * ids.length
+
+    User.where(id: ids).
+      update_all([
+        "last_activity_on = CASE #{conditions} ELSE last_activity_on END",
+        *pairs.to_a.flatten
+      ])
+
+    Gitlab::UserActivities.new.delete(*ids)
+  end
+
+  private
+
+  def cast_data(pairs)
+    pairs.each_with_object({}) do |(key, value), new_pairs|
+      new_pairs[key.to_i] = Time.at(value.to_i).to_s(:db)
+    end
+  end
+end

changelogs/unreleased-ee/27790-periodically-save-last-activity-date-data-from-redis-to-the-database.yml

+---
+title: Periodically persists users activity to users.last_activity_on
+merge_request: 1597
+author:

config/initializers/1_settings.rb

@@ -367,6 +367,11 @@ Settings.cron_jobs['gitlab_usage_ping_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['gitlab_usage_ping_worker']['cron'] ||= Settings.send(:cron_random_weekly_time)
 Settings.cron_jobs['gitlab_usage_ping_worker']['job_class'] = 'GitlabUsagePingWorker'
 
+# Every day at 00:30
+Settings.cron_jobs['schedule_update_user_activity_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['schedule_update_user_activity_worker']['cron'] ||= '30 0 * * *'
+Settings.cron_jobs['schedule_update_user_activity_worker']['job_class'] = 'ScheduleUpdateUserActivityWorker'
+
 #
 # GitLab Shell
 #

config/sidekiq_queues.yml

@@ -53,3 +53,4 @@
   - [default, 1]
   - [pages, 1]
   - [system_hook_push, 1]
+  - [update_user_activity, 1]

db/migrate/20170307125949_add_last_activity_on_to_users.rb

+class AddLastActivityOnToUsers < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :users, :last_activity_on, :date
+  end
+end

db/schema.rb

@@ -1303,6 +1303,7 @@ ActiveRecord::Schema.define(version: 20170408033905) do
     t.string "organization"
     t.boolean "authorized_projects_populated"
     t.boolean "ghost"
+    t.date "last_activity_on"
     t.boolean "notified_of_own_activity"
     t.boolean "require_two_factor_authentication_from_group", default: false, null: false
     t.integer "two_factor_grace_period", default: 48, null: false

spec/workers/schedule_update_user_activity_worker_spec.rb

+require 'spec_helper'
+
+describe ScheduleUpdateUserActivityWorker, :redis do
+  let(:now) { Time.now }
+
+  before do
+    Gitlab::UserActivities.record('1', now)
+    Gitlab::UserActivities.record('2', now)
+  end
+
+  it 'schedules UpdateUserActivityWorker once' do
+    expect(UpdateUserActivityWorker).to receive(:perform_async).with({ '1' => now.to_i.to_s, '2' => now.to_i.to_s })
+
+    subject.perform
+  end
+
+  context 'when specifying a batch size' do
+    it 'schedules UpdateUserActivityWorker twice' do
+      expect(UpdateUserActivityWorker).to receive(:perform_async).with({ '1' => now.to_i.to_s })
+      expect(UpdateUserActivityWorker).to receive(:perform_async).with({ '2' => now.to_i.to_s })
+
+      subject.perform(1)
+    end
+  end
+end

spec/workers/update_user_activity_worker_spec.rb

+require 'spec_helper'
+
+describe UpdateUserActivityWorker, :redis do
+  let(:user_active_2_days_ago) { create(:user, current_sign_in_at: 10.months.ago) }
+  let(:user_active_yesterday_1) { create(:user) }
+  let(:user_active_yesterday_2) { create(:user) }
+  let(:user_active_today) { create(:user) }
+  let(:data) do
+    {
+      user_active_2_days_ago.id.to_s => 2.days.ago.at_midday.to_i.to_s,
+      user_active_yesterday_1.id.to_s => 1.day.ago.at_midday.to_i.to_s,
+      user_active_yesterday_2.id.to_s => 1.day.ago.at_midday.to_i.to_s,
+      user_active_today.id.to_s => Time.now.to_i.to_s
+    }
+  end
+
+  it 'updates users.last_activity_on' do
+    subject.perform(data)
+
+    aggregate_failures do
+      expect(user_active_2_days_ago.reload.last_activity_on).to eq(2.days.ago.to_date)
+      expect(user_active_yesterday_1.reload.last_activity_on).to eq(1.day.ago.to_date)
+      expect(user_active_yesterday_2.reload.last_activity_on).to eq(1.day.ago.to_date)
+      expect(user_active_today.reload.reload.last_activity_on).to eq(Date.today)
+    end
+  end
+
+  it 'deletes the pairs from Redis' do
+    data.each { |id, time| Gitlab::UserActivities.record(id, time) }
+
+    subject.perform(data)
+
+    expect(Gitlab::UserActivities.new.to_a).to be_empty
+  end
+end