Commit 40a97205 authored by Sean McGivern's avatar Sean McGivern

Merge branch 'usage-ping-port' into 'master'

Usage ping port

Closes #27750

See merge request !10481
parents a9da3743 5f570690
......@@ -367,6 +367,9 @@ const ShortcutsBlob = require('./shortcuts_blob');
case 'admin':
new Admin();
switch (path[1]) {
case 'cohorts':
new gl.UsagePing();
break;
case 'groups':
new UsersSelect();
break;
......
......@@ -165,6 +165,7 @@ import './syntax_highlight';
import './task_list';
import './todos';
import './tree';
import './usage_ping';
import './user';
import './user_tabs';
import './username_validator';
......
function UsagePing() {
const usageDataUrl = $('.usage-data').data('endpoint');
$.ajax({
type: 'GET',
url: usageDataUrl,
dataType: 'html',
success(html) {
$('.usage-data').html(html);
},
});
}
window.gl = window.gl || {};
window.gl.UsagePing = UsagePing;
......@@ -17,6 +17,18 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
end
end
def usage_data
respond_to do |format|
format.html do
usage_data = Gitlab::UsageData.data
usage_data_json = params[:pretty] ? JSON.pretty_generate(usage_data) : usage_data.to_json
render html: Gitlab::Highlight.highlight('payload.json', usage_data_json)
end
format.json { render json: Gitlab::UsageData.to_json }
end
end
def reset_runners_token
@application_setting.reset_runners_registration_token!
flash[:notice] = 'New runners registration token has been generated!'
......@@ -135,6 +147,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
:version_check_enabled,
:terminal_max_session_time,
:polling_interval_multiplier,
:usage_ping_enabled,
disabled_oauth_sign_in_sources: [],
import_sources: [],
......
class Admin::CohortsController < Admin::ApplicationController
def index
if current_application_settings.usage_ping_enabled
cohorts_results = Rails.cache.fetch('cohorts', expires_in: 1.day) do
CohortsService.new.execute
end
@cohorts = CohortsSerializer.new.represent(cohorts_results)
end
end
end
......@@ -5,6 +5,8 @@ class Projects::GitHttpController < Projects::GitHttpClientController
# GET /foo/bar.git/info/refs?service=git-receive-pack (git push)
def info_refs
if upload_pack? && upload_pack_allowed?
log_user_activity
render_ok
elsif receive_pack? && receive_pack_allowed?
render_ok
......@@ -106,4 +108,8 @@ class Projects::GitHttpController < Projects::GitHttpClientController
def access_klass
@access_klass ||= wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess
end
def log_user_activity
Users::ActivityService.new(user, 'pull').execute
end
end
......@@ -35,6 +35,7 @@ class SessionsController < Devise::SessionsController
# hide the signed-in notification
flash[:notice] = nil
log_audit_event(current_user, with: authentication_method)
log_user_activity(current_user)
end
end
......@@ -123,6 +124,10 @@ class SessionsController < Devise::SessionsController
for_authentication.security_event
end
def log_user_activity(user)
Users::ActivityService.new(user, 'login').execute
end
def load_recaptcha
Gitlab::Recaptcha.load_configurations!
end
......
......@@ -238,7 +238,8 @@ class ApplicationSetting < ActiveRecord::Base
terminal_max_session_time: 0,
two_factor_grace_period: 48,
user_default_external: false,
polling_interval_multiplier: 1
polling_interval_multiplier: 1,
usage_ping_enabled: true
}
end
......
class CohortActivityMonthEntity < Grape::Entity
include ActionView::Helpers::NumberHelper
expose :total do |cohort_activity_month|
number_with_delimiter(cohort_activity_month[:total])
end
expose :percentage do |cohort_activity_month|
number_to_percentage(cohort_activity_month[:percentage], precision: 0)
end
end
class CohortEntity < Grape::Entity
include ActionView::Helpers::NumberHelper
expose :registration_month do |cohort|
cohort[:registration_month].strftime('%b %Y')
end
expose :total do |cohort|
number_with_delimiter(cohort[:total])
end
expose :inactive do |cohort|
number_with_delimiter(cohort[:inactive])
end
expose :activity_months, using: CohortActivityMonthEntity
end
class CohortsEntity < Grape::Entity
expose :months_included
expose :cohorts, using: CohortEntity
end
class CohortsSerializer < AnalyticsGenericSerializer
entity CohortsEntity
end
class CohortsService
MONTHS_INCLUDED = 12
def execute
{
months_included: MONTHS_INCLUDED,
cohorts: cohorts
}
end
# Get an array of hashes that looks like:
#
# [
# {
# registration_month: Date.new(2017, 3),
# activity_months: [3, 2, 1],
# total: 3
# inactive: 0
# },
# etc.
#
# The `months` array is always from oldest to newest, so it's always
# non-strictly decreasing from left to right.
def cohorts
months = Array.new(MONTHS_INCLUDED) { |i| i.months.ago.beginning_of_month.to_date }
Array.new(MONTHS_INCLUDED) do
registration_month = months.last
activity_months = running_totals(months, registration_month)
# Even if no users registered in this month, we always want to have a
# value to fill in the table.
inactive = counts_by_month[[registration_month, nil]].to_i
months.pop
{
registration_month: registration_month,
activity_months: activity_months,
total: activity_months.first[:total],
inactive: inactive
}
end
end
private
# Calculate a running sum of active users, so users active in later months
# count as active in this month, too. Start with the most recent month first,
# for calculating the running totals, and then reverse for displaying in the
# table.
#
# Each month has a total, and a percentage of the overall total, as keys.
def running_totals(all_months, registration_month)
month_totals =
all_months
.map { |activity_month| counts_by_month[[registration_month, activity_month]] }
.reduce([]) { |result, total| result << result.last.to_i + total.to_i }
.reverse
overall_total = month_totals.first
month_totals.map do |total|
{ total: total, percentage: total.zero? ? 0 : 100 * total / overall_total }
end
end
# Get a hash that looks like:
#
# {
# [created_at_month, last_activity_on_month] => count,
# [created_at_month, last_activity_on_month_2] => count_2,
# # etc.
# }
#
# created_at_month can never be nil, but last_activity_on_month can (when a
# user has never logged in, just been created). This covers the last
# MONTHS_INCLUDED months.
def counts_by_month
@counts_by_month ||=
begin
created_at_month = column_to_date('created_at')
last_activity_on_month = column_to_date('last_activity_on')
User
.where('created_at > ?', MONTHS_INCLUDED.months.ago.end_of_month)
.group(created_at_month, last_activity_on_month)
.reorder("#{created_at_month} ASC", "#{last_activity_on_month} ASC")
.count
end
end
def column_to_date(column)
if Gitlab::Database.postgresql?
"CAST(DATE_TRUNC('month', #{column}) AS date)"
else
"STR_TO_DATE(DATE_FORMAT(#{column}, '%Y-%m-01'), '%Y-%m-%d')"
end
end
end
......@@ -72,6 +72,8 @@ class EventCreateService
def push(project, current_user, push_data)
create_event(project, current_user, Event::PUSHED, data: push_data)
Users::ActivityService.new(current_user, 'push').execute
end
private
......
module Users
class ActivityService
def initialize(author, activity)
@author = author.respond_to?(:user) ? author.user : author
@activity = activity
end
def execute
return unless @author && @author.is_a?(User)
record_activity
end
private
def record_activity
Gitlab::UserActivities.record(@author.id)
Rails.logger.debug("Recorded activity: #{@activity} for User ID: #{@author.id} (username: #{@author.username}")
end
end
end
......@@ -477,7 +477,7 @@
diagrams in Asciidoc documents using an external PlantUML service.
%fieldset
%legend Usage statistics
%legend#usage-statistics Usage statistics
.form-group
.col-sm-offset-2.col-sm-10
.checkbox
......@@ -486,6 +486,19 @@
Version check enabled
.help-block
Let GitLab inform you when an update is available.
.form-group
.col-sm-offset-2.col-sm-10
.checkbox
= f.label :usage_ping_enabled do
= f.check_box :usage_ping_enabled
Usage ping enabled
= link_to icon('question-circle'), help_page_path("user/admin_area/settings/usage_statistics", anchor: "usage-data")
.help-block
Every week GitLab will report license usage back to GitLab, Inc.
Disable this option if you do not want this to occur. To see the
JSON payload that will be sent, visit the
= succeed '.' do
= link_to "Cohorts page", admin_cohorts_path(anchor: 'usage-ping')
%fieldset
%legend Email
......
.bs-callout.clearfix
%p
User cohorts are shown for the last #{@cohorts[:months_included]}
months. Only users with activity are counted in the cohort total; inactive
users are counted separately.
= link_to icon('question-circle'), help_page_path('user/admin_area/user_cohorts', anchor: 'cohorts'), title: 'About this feature', target: '_blank'
.table-holder
%table.table
%thead
%tr
%th Registration month
%th Inactive users
%th Cohort total
- @cohorts[:months_included].times do |i|
%th Month #{i}
%tbody
- @cohorts[:cohorts].each do |cohort|
%tr
%td= cohort[:registration_month]
%td= cohort[:inactive]
%td= cohort[:total]
- cohort[:activity_months].each do |activity_month|
%td
- next if cohort[:total] == '0'
= activity_month[:percentage]
%br
= activity_month[:total]
%h2#usage-ping Usage ping
.bs-callout.clearfix
%p
User cohorts are shown because the usage ping is enabled. The data sent with
this is shown below. To disable this, visit
= succeed '.' do
= link_to 'application settings', admin_application_settings_path(anchor: 'usage-statistics')
%pre.usage-data.js-syntax-highlight.code.highlight{ data: { endpoint: usage_data_admin_application_settings_path(format: :html, pretty: true) } }
- @no_container = true
= render "admin/dashboard/head"
%div{ class: container_class }
- if @cohorts
= render 'cohorts_table'
= render 'usage_ping'
- else
.bs-callout.bs-callout-warning.clearfix
%p
User cohorts are only shown when the
= link_to 'usage ping', help_page_path('user/admin_area/usage_statistics'), target: '_blank'
is enabled. To enable it and see user cohorts,
visit
= succeed '.' do
= link_to 'application settings', admin_application_settings_path(anchor: 'usage-statistics')
......@@ -27,3 +27,7 @@
= link_to admin_runners_path, title: 'Runners' do
%span
Runners
= nav_link path: 'cohorts#index' do
= link_to admin_cohorts_path, title: 'Cohorts' do
%span
Cohorts
class GitlabUsagePingWorker
LEASE_TIMEOUT = 86400
include Sidekiq::Worker
include CronjobQueue
include HTTParty
def perform
return unless current_application_settings.usage_ping_enabled
# Multiple Sidekiq workers could run this. We should only do this at most once a day.
return unless try_obtain_lease
begin
HTTParty.post(url,
body: Gitlab::UsageData.to_json(force_refresh: true),
headers: { 'Content-type' => 'application/json' }
)
rescue HTTParty::Error => e
Rails.logger.info "Unable to contact GitLab, Inc.: #{e}"
end
end
def try_obtain_lease
Gitlab::ExclusiveLease.new('gitlab_usage_ping_worker:ping', timeout: LEASE_TIMEOUT).try_obtain
end
def url
'https://version.gitlab.com/usage_data'
end
end
class ScheduleUpdateUserActivityWorker
include Sidekiq::Worker
include CronjobQueue
def perform(batch_size = 500)
Gitlab::UserActivities.new.each_slice(batch_size) do |batch|
UpdateUserActivityWorker.perform_async(Hash[batch])
end
end
end
class UpdateUserActivityWorker
include Sidekiq::Worker
include DedicatedSidekiqQueue
def perform(pairs)
pairs = cast_data(pairs)
ids = pairs.keys
conditions = 'WHEN id = ? THEN ? ' * ids.length
User.where(id: ids).
update_all([
"last_activity_on = CASE #{conditions} ELSE last_activity_on END",
*pairs.to_a.flatten
])
Gitlab::UserActivities.new.delete(*ids)
end
private
def cast_data(pairs)
pairs.each_with_object({}) do |(key, value), new_pairs|
new_pairs[key.to_i] = Time.at(value.to_i).to_s(:db)
end
end
end
---
title: Add usage ping to CE
merge_request:
author:
......@@ -110,6 +110,14 @@ class Settings < Settingslogic
URI.parse(url_without_path).host
end
# Random cron time every Sunday to load balance usage pings
def cron_random_weekly_time
hour = rand(24)
minute = rand(60)
"#{minute} #{hour} * * 0"
end
end
end
......@@ -204,8 +212,8 @@ Settings.gitlab['email_from'] ||= ENV['GITLAB_EMAIL_FROM'] || "gitlab@#{Settings
Settings.gitlab['email_display_name'] ||= ENV['GITLAB_EMAIL_DISPLAY_NAME'] || 'GitLab'
Settings.gitlab['email_reply_to'] ||= ENV['GITLAB_EMAIL_REPLY_TO'] || "noreply@#{Settings.gitlab.host}"
Settings.gitlab['email_subject_suffix'] ||= ENV['GITLAB_EMAIL_SUBJECT_SUFFIX'] || ""
Settings.gitlab['base_url'] ||= Settings.send(:build_base_gitlab_url)
Settings.gitlab['url'] ||= Settings.send(:build_gitlab_url)
Settings.gitlab['base_url'] ||= Settings.__send__(:build_base_gitlab_url)
Settings.gitlab['url'] ||= Settings.__send__(:build_gitlab_url)
Settings.gitlab['user'] ||= 'git'
Settings.gitlab['user_home'] ||= begin
Etc.getpwnam(Settings.gitlab['user']).dir
......@@ -215,7 +223,7 @@ end
Settings.gitlab['time_zone'] ||= nil
Settings.gitlab['signup_enabled'] ||= true if Settings.gitlab['signup_enabled'].nil?
Settings.gitlab['signin_enabled'] ||= true if Settings.gitlab['signin_enabled'].nil?
Settings.gitlab['restricted_visibility_levels'] = Settings.send(:verify_constant_array, Gitlab::VisibilityLevel, Settings.gitlab['restricted_visibility_levels'], [])
Settings.gitlab['restricted_visibility_levels'] = Settings.__send__(:verify_constant_array, Gitlab::VisibilityLevel, Settings.gitlab['restricted_visibility_levels'], [])
Settings.gitlab['username_changing_enabled'] = true if Settings.gitlab['username_changing_enabled'].nil?
Settings.gitlab['issue_closing_pattern'] = '((?:[Cc]los(?:e[sd]?|ing)|[Ff]ix(?:e[sd]|ing)?|[Rr]esolv(?:e[sd]?|ing))(:?) +(?:(?:issues? +)?%{issue_ref}(?:(?:, *| +and +)?)|([A-Z][A-Z0-9_]+-\d+))+)' if Settings.gitlab['issue_closing_pattern'].nil?
Settings.gitlab['default_projects_features'] ||= {}
......@@ -228,7 +236,7 @@ Settings.gitlab.default_projects_features['wiki'] = true if Settin
Settings.gitlab.default_projects_features['snippets'] = true if Settings.gitlab.default_projects_features['snippets'].nil?
Settings.gitlab.default_projects_features['builds'] = true if Settings.gitlab.default_projects_features['builds'].nil?
Settings.gitlab.default_projects_features['container_registry'] = true if Settings.gitlab.default_projects_features['container_registry'].nil?
Settings.gitlab.default_projects_features['visibility_level'] = Settings.send(:verify_constant, Gitlab::VisibilityLevel, Settings.gitlab.default_projects_features['visibility_level'], Gitlab::VisibilityLevel::PRIVATE)
Settings.gitlab.default_projects_features['visibility_level'] = Settings.__send__(:verify_constant, Gitlab::VisibilityLevel, Settings.gitlab.default_projects_features['visibility_level'], Gitlab::VisibilityLevel::PRIVATE)
Settings.gitlab['domain_whitelist'] ||= []
Settings.gitlab['import_sources'] ||= %w[github bitbucket gitlab google_code fogbugz git gitlab_project gitea]
Settings.gitlab['trusted_proxies'] ||= []
......@@ -242,7 +250,7 @@ Settings.gitlab_ci['shared_runners_enabled'] = true if Settings.gitlab_ci['share
Settings.gitlab_ci['all_broken_builds'] = true if Settings.gitlab_ci['all_broken_builds'].nil?
Settings.gitlab_ci['add_pusher'] = false if Settings.gitlab_ci['add_pusher'].nil?
Settings.gitlab_ci['builds_path'] = Settings.absolute(Settings.gitlab_ci['builds_path'] || "builds/")
Settings.gitlab_ci['url'] ||= Settings.send(:build_gitlab_ci_url)
Settings.gitlab_ci['url'] ||= Settings.__send__(:build_gitlab_ci_url)
#
# Reply by email
......@@ -281,7 +289,7 @@ Settings.pages['https'] = false if Settings.pages['https'].nil?
Settings.pages['host'] ||= "example.com"
Settings.pages['port'] ||= Settings.pages.https ? 443 : 80
Settings.pages['protocol'] ||= Settings.pages.https ? "https" : "http"
Settings.pages['url'] ||= Settings.send(:build_pages_url)
Settings.pages['url'] ||= Settings.__send__(:build_pages_url)
Settings.pages['external_http'] ||= false unless Settings.pages['external_http'].present?
Settings.pages['external_https'] ||= false unless Settings.pages['external_https'].present?
......@@ -355,6 +363,14 @@ Settings.cron_jobs['remove_unreferenced_lfs_objects_worker']['job_class'] = 'Rem
Settings.cron_jobs['stuck_import_jobs_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['stuck_import_jobs_worker']['cron'] ||= '15 * * * *'
Settings.cron_jobs['stuck_import_jobs_worker']['job_class'] = 'StuckImportJobsWorker'
Settings.cron_jobs['gitlab_usage_ping_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['gitlab_usage_ping_worker']['cron'] ||= Settings.__send__(:cron_random_weekly_time)
Settings.cron_jobs['gitlab_usage_ping_worker']['job_class'] = 'GitlabUsagePingWorker'
# Every day at 00:30
Settings.cron_jobs['schedule_update_user_activity_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['schedule_update_user_activity_worker']['cron'] ||= '30 0 * * *'
Settings.cron_jobs['schedule_update_user_activity_worker']['job_class'] = 'ScheduleUpdateUserActivityWorker'
#
# GitLab Shell
......@@ -369,7 +385,7 @@ Settings.gitlab_shell['ssh_host'] ||= Settings.gitlab.ssh_host
Settings.gitlab_shell['ssh_port'] ||= 22
Settings.gitlab_shell['ssh_user'] ||= Settings.gitlab.user
Settings.gitlab_shell['owner_group'] ||= Settings.gitlab.user
Settings.gitlab_shell['ssh_path_prefix'] ||= Settings.send(:build_gitlab_shell_ssh_path_prefix)
Settings.gitlab_shell['ssh_path_prefix'] ||= Settings.__send__(:build_gitlab_shell_ssh_path_prefix)
#
# Repositories
......
......@@ -91,6 +91,7 @@ namespace :admin do
resource :application_settings, only: [:show, :update] do
resources :services, only: [:index, :edit, :update]
get :usage_data
put :reset_runners_token
put :reset_health_check_token
put :clear_repository_check_states
......@@ -105,6 +106,8 @@ namespace :admin do
end
end
resources :cohorts, only: :index
resources :builds, only: :index do
collection do
post :cancel_all
......
......@@ -53,3 +53,4 @@
- [default, 1]
- [pages, 1]
- [system_hook_push, 1]
- [update_user_activity, 1]
class AddUsagePingToApplicationSettings < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
add_column :application_settings, :usage_ping_enabled, :boolean, default: true, null: false
end
end
class CreateUserActivities < ActiveRecord::Migration
DOWNTIME = false
# This migration is a no-op. It just exists to match EE.
def change
end
end
class AddLastActivityOnToUsers < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
add_column :users, :last_activity_on, :date
end
end
class AddUuidToApplicationSettings < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_column :application_settings, :uuid, :string
execute("UPDATE application_settings SET uuid = #{quote(SecureRandom.uuid)}")
end
def down
remove_column :application_settings, :uuid
end
end
class DropUserActivitiesTable < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
# This migration is a no-op. It just exists to match EE.
def change
end
end
class MigrateUserActivitiesToUsersLastActivityOn < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
disable_ddl_transaction!
DOWNTIME = false
USER_ACTIVITY_SET_KEY = 'user/activities'.freeze
ACTIVITIES_PER_PAGE = 100
TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED = Time.utc(2016, 12, 1)
def up
return if activities_count(TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED, Time.now).zero?
day = Time.at(activities(TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED, Time.now).first.second)
transaction do
while day <= Time.now.utc.tomorrow
persist_last_activity_on(day: day)
day = day.tomorrow
end
end
end
def down
# This ensures we don't lock all users for the duration of the migration.
update_column_in_batches(:users, :last_activity_on, nil) do |table, query|
query.where(table[:last_activity_on].not_eq(nil))
end
end
private
def persist_last_activity_on(day:, page: 1)
activities_count = activities_count(day.at_beginning_of_day, day.at_end_of_day)
return if activities_count.zero?
activities = activities(day.at_beginning_of_day, day.at_end_of_day, page: page)
update_sql =
Arel::UpdateManager.new(ActiveRecord::Base).
table(users_table).
set(users_table[:last_activity_on] => day.to_date).
where(users_table[:username].in(activities.map(&:first))).
to_sql
connection.exec_update(update_sql, self.class.name, [])
unless last_page?(page, activities_count)
persist_last_activity_on(day: day, page: page + 1)
end
end
def users_table
@users_table ||= Arel::Table.new(:users)
end
def activities(from, to, page: 1)
Gitlab::Redis.with do |redis|
redis.zrangebyscore(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i,
with_scores: true,
limit: limit(page))
end
end
def activities_count(from, to)
Gitlab::Redis.with do |redis|
redis.zcount(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i)
end
end
def limit(page)
[offset(page), ACTIVITIES_PER_PAGE]
end
def total_pages(count)
(count.to_f / ACTIVITIES_PER_PAGE).ceil
end
def last_page?(page, count)
page >= total_pages(count)
end
def offset(page)
(page - 1) * ACTIVITIES_PER_PAGE
end
end
......@@ -116,6 +116,8 @@ ActiveRecord::Schema.define(version: 20170418103908) do
t.integer "unique_ips_limit_time_window"
t.boolean "unique_ips_limit_enabled", default: false, null: false
t.decimal "polling_interval_multiplier", default: 1.0, null: false
t.boolean "usage_ping_enabled", default: true, null: false
t.string "uuid"
end
create_table "audit_events", force: :cascade do |t|
......@@ -1301,6 +1303,7 @@ ActiveRecord::Schema.define(version: 20170418103908) do
t.string "organization"
t.boolean "authorized_projects_populated"
t.boolean "ghost"
t.date "last_activity_on"
t.boolean "notified_of_own_activity"
t.boolean "require_two_factor_authentication_from_group", default: false, null: false
t.integer "two_factor_grace_period", default: 48, null: false
......
......@@ -69,6 +69,7 @@ All technical content published by GitLab lives in the documentation, including:
- [Sidekiq Troubleshooting](administration/troubleshooting/sidekiq.md) Debug when Sidekiq appears hung and is not processing jobs.
- [System hooks](system_hooks/system_hooks.md) Notifications when users, projects and keys are changed.
- [Update](update/README.md) Update guides to upgrade your installation.
- [User cohorts](user/admin_area/user_cohorts.md) View user activity over time.
- [Web terminals](administration/integration/terminal.md) Provide terminal access to environments from within GitLab.
- [Welcome message](customization/welcome_message.md) Add a custom welcome message to the sign-in page.
......
......@@ -72,6 +72,7 @@ GET /users
"organization": "",
"last_sign_in_at": "2012-06-01T11:41:01Z",
"confirmed_at": "2012-05-23T09:05:22Z",
"last_activity_on": "2012-05-23",
"color_scheme_id": 2,
"projects_limit": 100,
"current_sign_in_at": "2012-06-02T06:36:55Z",
......@@ -104,6 +105,7 @@ GET /users
"organization": "",
"last_sign_in_at": null,
"confirmed_at": "2012-05-30T16:53:06.148Z",
"last_activity_on": "2012-05-23",
"color_scheme_id": 3,
"projects_limit": 100,
"current_sign_in_at": "2014-03-19T17:54:13Z",
......@@ -196,6 +198,7 @@ Parameters:
"organization": "",
"last_sign_in_at": "2012-06-01T11:41:01Z",
"confirmed_at": "2012-05-23T09:05:22Z",
"last_activity_on": "2012-05-23",
"color_scheme_id": 2,
"projects_limit": 100,
"current_sign_in_at": "2012-06-02T06:36:55Z",
......@@ -320,6 +323,7 @@ GET /user
"organization": "",
"last_sign_in_at": "2012-06-01T11:41:01Z",
"confirmed_at": "2012-05-23T09:05:22Z",
"last_activity_on": "2012-05-23",
"color_scheme_id": 2,
"projects_limit": 100,
"current_sign_in_at": "2012-06-02T06:36:55Z",
......@@ -365,6 +369,7 @@ GET /user
"organization": "",
"last_sign_in_at": "2012-06-01T11:41:01Z",
"confirmed_at": "2012-05-23T09:05:22Z",
"last_activity_on": "2012-05-23",
"color_scheme_id": 2,
"projects_limit": 100,
"current_sign_in_at": "2012-06-02T06:36:55Z",
......@@ -986,3 +991,55 @@ Parameters:
| --------- | ---- | -------- | ----------- |
| `user_id` | integer | yes | The ID of the user |
| `impersonation_token_id` | integer | yes | The ID of the impersonation token |
### Get user activities (admin only)
>**Note:** This API endpoint is only available on 8.15 (EE) and 9.1 (CE) and above.
Get the last activity date for all users, sorted from oldest to newest.
The activities that update the timestamp are:
- Git HTTP/SSH activities (such as clone, push)
- User logging in into GitLab
By default, it shows the activity for all users in the last 6 months, but this can be
amended by using the `from` parameter.
```
GET /user/activities
```
Parameters:
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `from` | string | no | Date string in the format YEAR-MONTH-DAY, e.g. `2016-03-11`. Defaults to 6 months ago. |
```bash
curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v3/user/activities
```
Example response:
```json
[
{
"username": "user1",
"last_activity_on": "2015-12-14",
"last_activity_at": "2015-12-14"
},
{
"username": "user2",
"last_activity_on": "2015-12-15",
"last_activity_at": "2015-12-15"
},
{
"username": "user3",
"last_activity_on": "2015-12-16",
"last_activity_at": "2015-12-16"
}
]
```
Please note that `last_activity_at` is deprecated, please use `last_activity_on`.
# Usage statistics
GitLab Inc. will periodically collect information about your instance in order
to perform various actions.
All statistics are opt-in and you can always disable them from the admin panel.
## Version check
GitLab can inform you when an update is available and the importance of it.
No information other than the GitLab version and the instance's domain name
are collected.
In the **Overview** tab you can see if your GitLab version is up to date. There
are three cases: 1) you are up to date (green), 2) there is an update available
(yellow) and 3) your version is vulnerable and a security fix is released (red).
In any case, you will see a message informing you of the state and the
importance of the update.
If enabled, the version status will also be shown in the help page (`/help`)
for all signed in users.
## Usage ping
> [Introduced][ee-557] in GitLab Enterprise Edition 8.10. More statistics
[were added][ee-735] in GitLab Enterprise Edition
8.12. [Moved to GitLab Community Edition][ce-23361] in 9.1.
GitLab Inc. can collect non-sensitive information about how GitLab users
use their GitLab instance upon the activation of a ping feature
located in the admin panel (`/admin/application_settings`).
You can see the **exact** JSON payload that your instance sends to GitLab
in the "Usage statistics" section of the admin panel.
Nothing qualitative is collected. Only quantitative. That means no project
names, author names, comment bodies, names of labels, etc.
The usage ping is sent in order for GitLab Inc. to have a better understanding
of how our users use our product, and to be more data-driven when creating or
changing features.
The total number of the following is sent back to GitLab Inc.:
- Comments
- Groups
- Users
- Projects
- Issues
- Labels
- CI builds
- Snippets
- Milestones
- Todos
- Pushes
- Merge requests
- Environments
- Triggers
- Deploy keys
- Pages
- Project Services
- Projects using the Prometheus service
- Issue Boards
- CI Runners
- Deployments
- Geo Nodes
- LDAP Groups
- LDAP Keys
- LDAP Users
- LFS objects
- Protected branches
- Releases
- Remote mirrors
- Uploads
- Web hooks
Also, we track if you've installed Mattermost with GitLab.
For example: `"mattermost_enabled":true"`.
More data will be added over time. The goal of this ping is to be as light as
possible, so it won't have any performance impact on your installation when
the calculation is made.
### Deactivate the usage ping
By default, usage ping is opt-out. If you want to deactivate this feature, go to
the Settings page of your administration panel and uncheck the Usage ping
checkbox.
## Privacy policy
GitLab Inc. does **not** collect any sensitive information, like project names
or the content of the comments. GitLab Inc. does not disclose or otherwise make
available any of the data collected on a customer specific basis.
Read more about this in the [Privacy policy](https://about.gitlab.com/privacy).
[ee-557]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/557
[ee-735]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/735
[ce-23361]: https://gitlab.com/gitlab-org/gitlab-ce/issues/23361
# Cohorts
> **Notes:**
- [Introduced][ce-23361] in GitLab 9.1.
As a benefit of having the [usage ping active](settings/usage_statistics.md),
GitLab lets you analyze the users' activities of your GitLab installation.
Under `/admin/cohorts`, when the usage ping is active, GitLab will show the
monthly cohorts of new users and their activities over time.
How do we read the user cohorts table? Let's take an example with the following
user cohorts.
![User cohort example](img/cohorts.png)
For the cohort of June 2016, 163 users have been created on this server. One
month later, in July 2016, 155 users (or 95% of the June cohort) are still
active. Two months later, 139 users (or 85%) are still active. 9 months later,
we can see that only 6% of this cohort are still active.
How do we measure the activity of users? GitLab considers a user active if:
* the user signs in
* the user has Git activity (whether push or pull).
### Setup
1. Activate the usage ping as defined [in the documentation](settings/usage_statistics.md)
2. Go to `/admin/cohorts` to see the user cohorts of the server
[ce-23361]: https://gitlab.com/gitlab-org/gitlab-ce/issues/23361
......@@ -18,6 +18,12 @@ module API
expose :bio, :location, :skype, :linkedin, :twitter, :website_url, :organization
end
class UserActivity < Grape::Entity
expose :username
expose :last_activity_on
expose :last_activity_on, as: :last_activity_at # Back-compat
end
class Identity < Grape::Entity
expose :provider, :extern_uid
end
......@@ -25,6 +31,7 @@ module API
class UserPublic < User
expose :last_sign_in_at
expose :confirmed_at
expose :last_activity_on
expose :email
expose :color_scheme_id, :projects_limit, :current_sign_in_at
expose :identities, using: Entities::Identity
......
......@@ -60,6 +60,12 @@ module API
rescue JSON::ParserError
{}
end
def log_user_activity(actor)
commands = Gitlab::GitAccess::DOWNLOAD_COMMANDS
::Users::ActivityService.new(actor, 'Git SSH').execute if commands.include?(params[:action])
end
end
end
end
......@@ -40,6 +40,8 @@ module API
response = { status: access_status.status, message: access_status.message }
if access_status.status
log_user_activity(actor)
# Return the repository full path so that gitlab-shell has it when
# handling ssh commands
response[:repository_path] =
......
......@@ -35,7 +35,7 @@ module API
optional :assignee_id, type: Integer, desc: 'The ID of a user to assign issue'
optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign issue'
optional :labels, type: String, desc: 'Comma-separated list of label names'
optional :due_date, type: String, desc: 'Date time string in the format YEAR-MONTH-DAY'
optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY'
optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential'
end
......
......@@ -534,6 +534,21 @@ module API
email.destroy
current_user.update_secondary_emails!
end
desc 'Get a list of user activities'
params do
optional :from, type: DateTime, default: 6.months.ago, desc: 'Date string in the format YEAR-MONTH-DAY'
use :pagination
end
get "activities" do
authenticated_as_admin!
activities = User.
where(User.arel_table[:last_activity_on].gteq(params[:from])).
reorder(last_activity_on: :asc)
present paginate(activities), with: Entities::UserActivity
end
end
end
end
module Gitlab
class UsageData
include Gitlab::CurrentSettings
class << self
def data(force_refresh: false)
Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data }
end
def uncached_data
license_usage_data.merge(system_usage_data)
end
def to_json(force_refresh: false)
data(force_refresh: force_refresh).to_json
end
def system_usage_data
{
counts: {
boards: Board.count,
ci_builds: ::Ci::Build.count,
ci_pipelines: ::Ci::Pipeline.count,
ci_runners: ::Ci::Runner.count,
ci_triggers: ::Ci::Trigger.count,
deploy_keys: DeployKey.count,
deployments: Deployment.count,
environments: Environment.count,
groups: Group.count,
issues: Issue.count,
keys: Key.count,
labels: Label.count,
lfs_objects: LfsObject.count,
merge_requests: MergeRequest.count,
milestones: Milestone.count,
notes: Note.count,
pages_domains: PagesDomain.count,
projects: Project.count,
projects_prometheus_active: PrometheusService.active.count,
protected_branches: ProtectedBranch.count,
releases: Release.count,
services: Service.where(active: true).count,
snippets: Snippet.count,
todos: Todo.count,
uploads: Upload.count,
web_hooks: WebHook.count
}
}
end
def license_usage_data
usage_data = {
uuid: current_application_settings.uuid,
version: Gitlab::VERSION,
active_user_count: User.active.count,
recorded_at: Time.now,
mattermost_enabled: Gitlab.config.mattermost.enabled,
edition: 'CE'
}
usage_data
end
end
end
end
module Gitlab
class UserActivities
include Enumerable
KEY = 'users:activities'.freeze
BATCH_SIZE = 500
def self.record(key, time = Time.now)
Gitlab::Redis.with do |redis|
redis.hset(KEY, key, time.to_i)
end
end
def delete(*keys)
Gitlab::Redis.with do |redis|
redis.hdel(KEY, keys)
end
end
def each
cursor = 0
loop do
cursor, pairs =
Gitlab::Redis.with do |redis|
redis.hscan(KEY, cursor, count: BATCH_SIZE)
end
Hash[pairs].each { |pair| yield pair }
break if cursor == '0'
end
end
end
end
......@@ -3,12 +3,49 @@ require 'spec_helper'
describe Admin::ApplicationSettingsController do
include StubENV
let(:group) { create(:group) }
let(:project) { create(:project, namespace: group) }
let(:admin) { create(:admin) }
let(:user) { create(:user)}
before do
stub_env('IN_MEMORY_APPLICATION_SETTINGS', 'false')
end
describe 'GET #usage_data with no access' do
before do
sign_in(user)
end
it 'returns 404' do
get :usage_data, format: :html
expect(response.status).to eq(404)
end
end
describe 'GET #usage_data' do
before do
sign_in(admin)
end
it 'returns HTML data' do
get :usage_data, format: :html
expect(response.body).to start_with('<span')
expect(response.status).to eq(200)
end
it 'returns JSON data' do
get :usage_data, format: :json
body = JSON.parse(response.body)
expect(body["version"]).to eq(Gitlab::VERSION)
expect(body).to include('counts')
expect(response.status).to eq(200)
end
end
describe 'PUT #update' do
before do
sign_in(admin)
......
......@@ -16,7 +16,9 @@ describe SessionsController do
end
end
context 'when using valid password' do
context 'when using valid password', :redis do
include UserActivitiesHelpers
let(:user) { create(:user) }
it 'authenticates user correctly' do
......@@ -37,6 +39,12 @@ describe SessionsController do
subject.sign_out user
end
end
it 'updates the user activity' do
expect do
post(:create, user: { login: user.username, password: user.password })
end.to change { user_activity(user) }
end
end
end
......
require 'spec_helper'
describe Gitlab::UsageData do
let!(:project) { create(:empty_project) }
let!(:project2) { create(:empty_project) }
let!(:board) { create(:board, project: project) }
describe '#data' do
subject { Gitlab::UsageData.data }
it "gathers usage data" do
expect(subject.keys).to match_array(%i(
active_user_count
counts
recorded_at
mattermost_enabled
edition
version
uuid
))
end
it "gathers usage counts" do
count_data = subject[:counts]
expect(count_data[:boards]).to eq(1)
expect(count_data[:projects]).to eq(2)
expect(count_data.keys).to match_array(%i(
boards
ci_builds
ci_pipelines
ci_runners
ci_triggers
deploy_keys
deployments
environments
groups
issues
keys
labels
lfs_objects
merge_requests
milestones
notes
projects
projects_prometheus_active
pages_domains
protected_branches
releases
services
snippets
todos
uploads
web_hooks
))
end
end
describe '#license_usage_data' do
subject { Gitlab::UsageData.license_usage_data }
it "gathers license data" do
expect(subject[:uuid]).to eq(current_application_settings.uuid)
expect(subject[:version]).to eq(Gitlab::VERSION)
expect(subject[:active_user_count]).to eq(User.active.count)
expect(subject[:recorded_at]).to be_a(Time)
end
end
end
require 'spec_helper'
describe Gitlab::UserActivities, :redis, lib: true do
let(:now) { Time.now }
describe '.record' do
context 'with no time given' do
it 'uses Time.now and records an activity in Redis' do
Timecop.freeze do
now # eager-load now
described_class.record(42)
end
Gitlab::Redis.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]])
end
end
end
context 'with a time given' do
it 'uses the given time and records an activity in Redis' do
described_class.record(42, now)
Gitlab::Redis.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]])
end
end
end
end
describe '.delete' do
context 'with a single key' do
context 'and key exists' do
it 'removes the pair from Redis' do
described_class.record(42, now)
Gitlab::Redis.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]])
end
subject.delete(42)
Gitlab::Redis.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []])
end
end
end
context 'and key does not exist' do
it 'removes the pair from Redis' do
Gitlab::Redis.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []])
end
subject.delete(42)
Gitlab::Redis.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []])
end
end
end
end
context 'with multiple keys' do
context 'and all keys exist' do
it 'removes the pair from Redis' do
described_class.record(41, now)
described_class.record(42, now)
Gitlab::Redis.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['41', now.to_i.to_s], ['42', now.to_i.to_s]]])
end
subject.delete(41, 42)
Gitlab::Redis.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []])
end
end
end
context 'and some keys does not exist' do
it 'removes the existing pair from Redis' do
described_class.record(42, now)
Gitlab::Redis.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]])
end
subject.delete(41, 42)
Gitlab::Redis.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []])
end
end
end
end
end
describe 'Enumerable' do
before do
described_class.record(40, now)
described_class.record(41, now)
described_class.record(42, now)
end
it 'allows to read the activities sequentially' do
expected = { '40' => now.to_i.to_s, '41' => now.to_i.to_s, '42' => now.to_i.to_s }
actual = described_class.new.each_with_object({}) do |(key, time), actual|
actual[key] = time
end
expect(actual).to eq(expected)
end
context 'with many records' do
before do
1_000.times { |i| described_class.record(i, now) }
end
it 'is possible to loop through all the records' do
expect(described_class.new.count).to eq(1_000)
end
end
end
end
# encoding: utf-8
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20170324160416_migrate_user_activities_to_users_last_activity_on.rb')
describe MigrateUserActivitiesToUsersLastActivityOn, :redis do
let(:migration) { described_class.new }
let!(:user_active_1) { create(:user) }
let!(:user_active_2) { create(:user) }
def record_activity(user, time)
Gitlab::Redis.with do |redis|
redis.zadd(described_class::USER_ACTIVITY_SET_KEY, time.to_i, user.username)
end
end
around do |example|
Timecop.freeze { example.run }
end
before do
record_activity(user_active_1, described_class::TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED + 2.months)
record_activity(user_active_2, described_class::TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED + 3.months)
mute_stdout { migration.up }
end
describe '#up' do
it 'fills last_activity_on from the legacy Redis Sorted Set' do
expect(user_active_1.reload.last_activity_on).to eq((described_class::TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED + 2.months).to_date)
expect(user_active_2.reload.last_activity_on).to eq((described_class::TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED + 3.months).to_date)
end
end
describe '#down' do
it 'sets last_activity_on to NULL for all users' do
mute_stdout { migration.down }
expect(user_active_1.reload.last_activity_on).to be_nil
expect(user_active_2.reload.last_activity_on).to be_nil
end
end
def mute_stdout
orig_stdout = $stdout
$stdout = StringIO.new
yield
$stdout = orig_stdout
end
end
......@@ -147,10 +147,15 @@ describe API::Internal, api: true do
end
end
describe "POST /internal/allowed" do
describe "POST /internal/allowed", :redis do
context "access granted" do
before do
project.team << [user, :developer]
Timecop.freeze
end
after do
Timecop.return
end
context 'with env passed as a JSON' do
......@@ -176,6 +181,7 @@ describe API::Internal, api: true do
expect(response).to have_http_status(200)
expect(json_response["status"]).to be_truthy
expect(json_response["repository_path"]).to eq(project.wiki.repository.path_to_repo)
expect(user).not_to have_an_activity_record
end
end
......@@ -186,6 +192,7 @@ describe API::Internal, api: true do
expect(response).to have_http_status(200)
expect(json_response["status"]).to be_truthy
expect(json_response["repository_path"]).to eq(project.wiki.repository.path_to_repo)
expect(user).to have_an_activity_record
end
end
......@@ -196,6 +203,7 @@ describe API::Internal, api: true do
expect(response).to have_http_status(200)
expect(json_response["status"]).to be_truthy
expect(json_response["repository_path"]).to eq(project.repository.path_to_repo)
expect(user).to have_an_activity_record
end
end
......@@ -206,6 +214,7 @@ describe API::Internal, api: true do
expect(response).to have_http_status(200)
expect(json_response["status"]).to be_truthy
expect(json_response["repository_path"]).to eq(project.repository.path_to_repo)
expect(user).not_to have_an_activity_record
end
context 'project as /namespace/project' do
......@@ -241,6 +250,7 @@ describe API::Internal, api: true do
expect(response).to have_http_status(200)
expect(json_response["status"]).to be_falsey
expect(user).not_to have_an_activity_record
end
end
......@@ -250,6 +260,7 @@ describe API::Internal, api: true do
expect(response).to have_http_status(200)
expect(json_response["status"]).to be_falsey
expect(user).not_to have_an_activity_record
end
end
end
......@@ -267,6 +278,7 @@ describe API::Internal, api: true do
expect(response).to have_http_status(200)
expect(json_response["status"]).to be_falsey
expect(user).not_to have_an_activity_record
end
end
......@@ -276,6 +288,7 @@ describe API::Internal, api: true do
expect(response).to have_http_status(200)
expect(json_response["status"]).to be_falsey
expect(user).not_to have_an_activity_record
end
end
end
......
require 'spec_helper'
describe API::Users, api: true do
describe API::Users, api: true do
include ApiHelpers
let(:user) { create(:user) }
let(:user) { create(:user) }
let(:admin) { create(:admin) }
let(:key) { create(:key, user: user) }
let(:email) { create(:email, user: user) }
let(:key) { create(:key, user: user) }
let(:email) { create(:email, user: user) }
let(:omniauth_user) { create(:omniauth_user) }
let(:ldap_user) { create(:omniauth_user, provider: 'ldapmain') }
let(:ldap_blocked_user) { create(:omniauth_user, provider: 'ldapmain', state: 'ldap_blocked') }
......@@ -129,7 +129,7 @@ describe API::Users, api: true do
end
describe "POST /users" do
before{ admin }
before { admin }
it "creates user" do
expect do
......@@ -214,9 +214,9 @@ describe API::Users, api: true do
it "does not create user with invalid email" do
post api('/users', admin),
email: 'invalid email',
password: 'password',
name: 'test'
email: 'invalid email',
password: 'password',
name: 'test'
expect(response).to have_http_status(400)
end
......@@ -242,12 +242,12 @@ describe API::Users, api: true do
it 'returns 400 error if user does not validate' do
post api('/users', admin),
password: 'pass',
email: 'test@example.com',
username: 'test!',
name: 'test',
bio: 'g' * 256,
projects_limit: -1
password: 'pass',
email: 'test@example.com',
username: 'test!',
name: 'test',
bio: 'g' * 256,
projects_limit: -1
expect(response).to have_http_status(400)
expect(json_response['message']['password']).
to eq(['is too short (minimum is 8 characters)'])
......@@ -267,19 +267,19 @@ describe API::Users, api: true do
context 'with existing user' do
before do
post api('/users', admin),
email: 'test@example.com',
password: 'password',
username: 'test',
name: 'foo'
email: 'test@example.com',
password: 'password',
username: 'test',
name: 'foo'
end
it 'returns 409 conflict error if user with same email exists' do
expect do
post api('/users', admin),
name: 'foo',
email: 'test@example.com',
password: 'password',
username: 'foo'
name: 'foo',
email: 'test@example.com',
password: 'password',
username: 'foo'
end.to change { User.count }.by(0)
expect(response).to have_http_status(409)
expect(json_response['message']).to eq('Email has already been taken')
......@@ -288,10 +288,10 @@ describe API::Users, api: true do
it 'returns 409 conflict error if same username exists' do
expect do
post api('/users', admin),
name: 'foo',
email: 'foo@example.com',
password: 'password',
username: 'test'
name: 'foo',
email: 'foo@example.com',
password: 'password',
username: 'test'
end.to change { User.count }.by(0)
expect(response).to have_http_status(409)
expect(json_response['message']).to eq('Username has already been taken')
......@@ -416,12 +416,12 @@ describe API::Users, api: true do
it 'returns 400 error if user does not validate' do
put api("/users/#{user.id}", admin),
password: 'pass',
email: 'test@example.com',
username: 'test!',
name: 'test',
bio: 'g' * 256,
projects_limit: -1
password: 'pass',
email: 'test@example.com',
username: 'test!',
name: 'test',
bio: 'g' * 256,
projects_limit: -1
expect(response).to have_http_status(400)
expect(json_response['message']['password']).
to eq(['is too short (minimum is 8 characters)'])
......@@ -488,7 +488,7 @@ describe API::Users, api: true do
key_attrs = attributes_for :key
expect do
post api("/users/#{user.id}/keys", admin), key_attrs
end.to change{ user.keys.count }.by(1)
end.to change { user.keys.count }.by(1)
end
it "returns 400 for invalid ID" do
......@@ -580,7 +580,7 @@ describe API::Users, api: true do
email_attrs = attributes_for :email
expect do
post api("/users/#{user.id}/emails", admin), email_attrs
end.to change{ user.emails.count }.by(1)
end.to change { user.emails.count }.by(1)
end
it "returns a 400 for invalid ID" do
......@@ -842,7 +842,7 @@ describe API::Users, api: true do
key_attrs = attributes_for :key
expect do
post api("/user/keys", user), key_attrs
end.to change{ user.keys.count }.by(1)
end.to change { user.keys.count }.by(1)
expect(response).to have_http_status(201)
end
......@@ -880,7 +880,7 @@ describe API::Users, api: true do
delete api("/user/keys/#{key.id}", user)
expect(response).to have_http_status(204)
end.to change{user.keys.count}.by(-1)
end.to change { user.keys.count}.by(-1)
end
it "returns 404 if key ID not found" do
......@@ -963,7 +963,7 @@ describe API::Users, api: true do
email_attrs = attributes_for :email
expect do
post api("/user/emails", user), email_attrs
end.to change{ user.emails.count }.by(1)
end.to change { user.emails.count }.by(1)
expect(response).to have_http_status(201)
end
......@@ -989,7 +989,7 @@ describe API::Users, api: true do
delete api("/user/emails/#{email.id}", user)
expect(response).to have_http_status(204)
end.to change{user.emails.count}.by(-1)
end.to change { user.emails.count}.by(-1)
end
it "returns 404 if email ID not found" do
......@@ -1158,6 +1158,49 @@ describe API::Users, api: true do
end
end
context "user activities", :redis do
let!(:old_active_user) { create(:user, last_activity_on: Time.utc(2000, 1, 1)) }
let!(:newly_active_user) { create(:user, last_activity_on: 2.days.ago.midday) }
context 'last activity as normal user' do
it 'has no permission' do
get api("/user/activities", user)
expect(response).to have_http_status(403)
end
end
context 'as admin' do
it 'returns the activities from the last 6 months' do
get api("/user/activities", admin)
expect(response).to include_pagination_headers
expect(json_response.size).to eq(1)
activity = json_response.last
expect(activity['username']).to eq(newly_active_user.username)
expect(activity['last_activity_on']).to eq(2.days.ago.to_date.to_s)
expect(activity['last_activity_at']).to eq(2.days.ago.to_date.to_s)
end
context 'passing a :from parameter' do
it 'returns the activities from the given date' do
get api("/user/activities?from=2000-1-1", admin)
expect(response).to include_pagination_headers
expect(json_response.size).to eq(2)
activity = json_response.first
expect(activity['username']).to eq(old_active_user.username)
expect(activity['last_activity_on']).to eq(Time.utc(2000, 1, 1).to_date.to_s)
expect(activity['last_activity_at']).to eq(Time.utc(2000, 1, 1).to_date.to_s)
end
end
end
end
describe 'GET /users/:user_id/impersonation_tokens' do
let!(:active_personal_access_token) { create(:personal_access_token, user: user) }
let!(:revoked_personal_access_token) { create(:personal_access_token, :revoked, user: user) }
......
......@@ -3,6 +3,7 @@ require "spec_helper"
describe 'Git HTTP requests', lib: true do
include GitHttpHelpers
include WorkhorseHelpers
include UserActivitiesHelpers
it "gives WWW-Authenticate hints" do
clone_get('doesnt/exist.git')
......@@ -255,6 +256,14 @@ describe 'Git HTTP requests', lib: true do
expect(response.content_type.to_s).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE)
end
end
it 'updates the user last activity', :redis do
expect(user_activity(user)).to be_nil
download(path, env) do |response|
expect(user_activity(user)).to be_present
end
end
end
context "when an oauth token is provided" do
......
require 'spec_helper'
describe CohortsService do
describe '#execute' do
def month_start(months_ago)
months_ago.months.ago.beginning_of_month.to_date
end
# In the interests of speed and clarity, this example has minimal data.
it 'returns a list of user cohorts' do
6.times do |months_ago|
months_ago_time = (months_ago * 2).months.ago
create(:user, created_at: months_ago_time, last_activity_on: Time.now)
create(:user, created_at: months_ago_time, last_activity_on: months_ago_time)
end
create(:user) # this user is inactive and belongs to the current month
expected_cohorts = [
{
registration_month: month_start(11),
activity_months: Array.new(12) { { total: 0, percentage: 0 } },
total: 0,
inactive: 0
},
{
registration_month: month_start(10),
activity_months: [{ total: 2, percentage: 100 }] + Array.new(10) { { total: 1, percentage: 50 } },
total: 2,
inactive: 0
},
{
registration_month: month_start(9),
activity_months: Array.new(10) { { total: 0, percentage: 0 } },
total: 0,
inactive: 0
},
{
registration_month: month_start(8),
activity_months: [{ total: 2, percentage: 100 }] + Array.new(8) { { total: 1, percentage: 50 } },
total: 2,
inactive: 0
},
{
registration_month: month_start(7),
activity_months: Array.new(8) { { total: 0, percentage: 0 } },
total: 0,
inactive: 0
},
{
registration_month: month_start(6),
activity_months: [{ total: 2, percentage: 100 }] + Array.new(6) { { total: 1, percentage: 50 } },
total: 2,
inactive: 0
},
{
registration_month: month_start(5),
activity_months: Array.new(6) { { total: 0, percentage: 0 } },
total: 0,
inactive: 0
},
{
registration_month: month_start(4),
activity_months: [{ total: 2, percentage: 100 }] + Array.new(4) { { total: 1, percentage: 50 } },
total: 2,
inactive: 0
},
{
registration_month: month_start(3),
activity_months: Array.new(4) { { total: 0, percentage: 0 } },
total: 0,
inactive: 0
},
{
registration_month: month_start(2),
activity_months: [{ total: 2, percentage: 100 }] + Array.new(2) { { total: 1, percentage: 50 } },
total: 2,
inactive: 0
},
{
registration_month: month_start(1),
activity_months: Array.new(2) { { total: 0, percentage: 0 } },
total: 0,
inactive: 0
},
{
registration_month: month_start(0),
activity_months: [{ total: 2, percentage: 100 }],
total: 2,
inactive: 1
},
]
expect(described_class.new.execute).to eq(months_included: 12,
cohorts: expected_cohorts)
end
end
end
require 'spec_helper'
describe EventCreateService, services: true do
include UserActivitiesHelpers
let(:service) { EventCreateService.new }
describe 'Issues' do
......@@ -111,6 +113,19 @@ describe EventCreateService, services: true do
end
end
describe '#push', :redis do
let(:project) { create(:empty_project) }
let(:user) { create(:user) }
it 'creates a new event' do
expect { service.push(project, user, {}) }.to change { Event.count }
end
it 'updates user last activity' do
expect { service.push(project, user, {}) }.to change { user_activity(user) }
end
end
describe 'Project' do
let(:user) { create :user }
let(:project) { create(:empty_project) }
......
require 'spec_helper'
describe Users::ActivityService, services: true do
include UserActivitiesHelpers
let(:user) { create(:user) }
subject(:service) { described_class.new(user, 'type') }
describe '#execute', :redis do
context 'when last activity is nil' do
before do
service.execute
end
it 'sets the last activity timestamp for the user' do
expect(last_hour_user_ids).to eq([user.id])
end
it 'updates the same user' do
service.execute
expect(last_hour_user_ids).to eq([user.id])
end
it 'updates the timestamp of an existing user' do
Timecop.freeze(Date.tomorrow) do
expect { service.execute }.to change { user_activity(user) }.to(Time.now.to_i.to_s)
end
end
describe 'other user' do
it 'updates other user' do
other_user = create(:user)
described_class.new(other_user, 'type').execute
expect(last_hour_user_ids).to match_array([user.id, other_user.id])
end
end
end
end
def last_hour_user_ids
Gitlab::UserActivities.new.
select { |k, v| v >= 1.hour.ago.to_i.to_s }.
map { |k, _| k.to_i }
end
end
RSpec::Matchers.define :have_an_activity_record do |expected|
match do |user|
expect(Gitlab::UserActivities.new.find { |k, _| k == user.id.to_s }).to be_present
end
end
module UserActivitiesHelpers
def user_activity(user)
Gitlab::UserActivities.new.
find { |k, _| k == user.id.to_s }&.
second
end
end
require 'spec_helper'
describe GitlabUsagePingWorker do
subject { GitlabUsagePingWorker.new }
it "sends POST request" do
stub_application_setting(usage_ping_enabled: true)
stub_request(:post, "https://version.gitlab.com/usage_data").
to_return(status: 200, body: '', headers: {})
expect(Gitlab::UsageData).to receive(:to_json).with({ force_refresh: true }).and_call_original
expect(subject).to receive(:try_obtain_lease).and_return(true)
expect(subject.perform.response.code.to_i).to eq(200)
end
it "does not run if usage ping is disabled" do
stub_application_setting(usage_ping_enabled: false)
expect(subject).not_to receive(:try_obtain_lease)
expect(subject).not_to receive(:perform)
end
end
require 'spec_helper'
describe ScheduleUpdateUserActivityWorker, :redis do
let(:now) { Time.now }
before do
Gitlab::UserActivities.record('1', now)
Gitlab::UserActivities.record('2', now)
end
it 'schedules UpdateUserActivityWorker once' do
expect(UpdateUserActivityWorker).to receive(:perform_async).with({ '1' => now.to_i.to_s, '2' => now.to_i.to_s })
subject.perform
end
context 'when specifying a batch size' do
it 'schedules UpdateUserActivityWorker twice' do
expect(UpdateUserActivityWorker).to receive(:perform_async).with({ '1' => now.to_i.to_s })
expect(UpdateUserActivityWorker).to receive(:perform_async).with({ '2' => now.to_i.to_s })
subject.perform(1)
end
end
end
require 'spec_helper'
describe UpdateUserActivityWorker, :redis do
let(:user_active_2_days_ago) { create(:user, current_sign_in_at: 10.months.ago) }
let(:user_active_yesterday_1) { create(:user) }
let(:user_active_yesterday_2) { create(:user) }
let(:user_active_today) { create(:user) }
let(:data) do
{
user_active_2_days_ago.id.to_s => 2.days.ago.at_midday.to_i.to_s,
user_active_yesterday_1.id.to_s => 1.day.ago.at_midday.to_i.to_s,
user_active_yesterday_2.id.to_s => 1.day.ago.at_midday.to_i.to_s,
user_active_today.id.to_s => Time.now.to_i.to_s
}
end
it 'updates users.last_activity_on' do
subject.perform(data)
aggregate_failures do
expect(user_active_2_days_ago.reload.last_activity_on).to eq(2.days.ago.to_date)
expect(user_active_yesterday_1.reload.last_activity_on).to eq(1.day.ago.to_date)
expect(user_active_yesterday_2.reload.last_activity_on).to eq(1.day.ago.to_date)
expect(user_active_today.reload.reload.last_activity_on).to eq(Date.today)
end
end
it 'deletes the pairs from Redis' do
data.each { |id, time| Gitlab::UserActivities.record(id, time) }
subject.perform(data)
expect(Gitlab::UserActivities.new.to_a).to be_empty
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment