Merge branch 'ee-usage-ping' into 'master'

Add EE usage ping

This MR implements the EE usage ping as a sidekiq-cron async task as discussed in #380. Currently we report:

* Active user count
* Max historical active user count
* License data (e.g. licensee, add-ons, etc.)

Screenshot:

![image](/uploads/552f5445c0034b604c35c2783161cbc1/image.png)


See merge request !557

app/controllers/admin/application_settings_controller.rb

@@ -118,6 +118,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
       :elasticsearch_search,
       :elasticsearch_host,
       :elasticsearch_port,
+      :usage_ping_enabled,
       :repository_storage,
       :enabled_git_access_protocol,
       restricted_visibility_levels: [],

app/helpers/license_helper.rb

@@ -3,6 +3,11 @@ module LicenseHelper
     User.active.count
   end
 
+  def max_historical_user_count
+    date_range = (Date.today - 1.year)..Date.today
+    HistoricalData.during(date_range).maximum(:active_user_count) || 0
+  end
+
   def license_message(signed_in: signed_in?, is_admin: (current_user && current_user.is_admin?))
     @license_message ||=
       if License.current

app/models/application_setting.rb

@@ -151,6 +151,7 @@ class ApplicationSetting < ActiveRecord::Base
       container_registry_token_expire_delay: 5,
       elasticsearch_host: ENV['ELASTIC_HOST'] || 'localhost',
       elasticsearch_port: ENV['ELASTIC_PORT'] || '9200',
+      usage_ping_enabled: true,
       repository_storage: 'default',
       user_default_external: false,
     )

app/views/admin/application_settings/_form.html.haml

@@ -55,6 +55,15 @@
           = f.label :version_check_enabled do
             = f.check_box :version_check_enabled
             Version check enabled
+    .form-group
+      .col-sm-offset-2.col-sm-10
+        .checkbox
+          = f.label :usage_ping_enabled do
+            = f.check_box :usage_ping_enabled
+            Usage ping enabled
+          .help-block
+            Every week GitLab will report license usage back to GitLab, Inc.
+            Disable this option if you do not want this to occur.
     .form-group
       .col-sm-offset-2.col-sm-10
         .checkbox

app/views/admin/licenses/_breakdown.html.haml

@@ -4,8 +4,7 @@
 - else
   - licensed_users = 'Unlimited'
 
-- date_range = (Date.today - 1.year)..Date.today
-- historical = HistoricalData.during(date_range).maximum(:active_user_count) || 0
+- historical = max_historical_user_count
 - if historical && restricted && historical > restricted
   - users_over_license = historical - restricted
 - else

app/workers/gitlab_usage_ping_worker.rb

+class GitlabUsagePingWorker
+  LEASE_TIMEOUT = 86400
+
+  include LicenseHelper
+  include Sidekiq::Worker
+  include HTTParty
+
+  # This is not guaranteed to succeed, so don't retry on failure
+  sidekiq_options queue: :default, retry: false
+
+  def perform
+    return unless current_application_settings.usage_ping_enabled
+
+    # Multiple Sidekiq workers could run this. We should only do this at most once a day.
+    return unless try_obtain_lease
+
+    begin
+      HTTParty.post(url,
+                    body: data.to_json,
+                    headers: { 'Content-type' => 'application/json' }
+                   )
+    rescue HTTParty::Error => e
+      Rails.logger.info "Unable to contact GitLab, Inc.: #{e}"
+    end
+  end
+
+  def try_obtain_lease
+    Gitlab::ExclusiveLease.new('gitlab_usage_ping_worker:ping', timeout: LEASE_TIMEOUT).try_obtain
+  end
+
+  def data
+    usage_data = { version: Gitlab::VERSION,
+                   active_user_count: current_active_user_count }
+    license = License.current
+
+    if license
+      usage_data[:license_md5] = Digest::MD5.hexdigest(license.data)
+      usage_data[:historical_max_users] = max_historical_user_count
+      usage_data[:licensee] = license.licensee
+      usage_data[:license_user_count] = license.user_count
+      usage_data[:license_starts_at] = license.starts_at
+      usage_data[:license_expires_at] = license.expires_at
+      usage_data[:license_add_ons] = license.add_ons
+      usage_data[:recorded_at] = Time.now
+    end
+
+    usage_data
+  end
+
+  def url
+    'https://version.gitlab.com/usage_data'
+  end
+end

config/initializers/1_settings.rb

@@ -125,6 +125,14 @@ class Settings < Settingslogic
 
       URI.parse(url_without_path).host
     end
+
+    # Random cron time every Sunday to load balance usage pings
+    def cron_random_weekly_time
+      hour = rand(24)
+      minute = rand(60)
+
+      "#{minute} #{hour} * * 0"
+    end
   end
 end
 
@@ -370,6 +378,9 @@ Settings.cron_jobs['geo_bulk_notify_worker']['job_class'] ||= 'GeoBulkNotifyWork
 Settings.cron_jobs['gitlab_remove_project_export_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['gitlab_remove_project_export_worker']['cron'] ||= '0 * * * *'
 Settings.cron_jobs['gitlab_remove_project_export_worker']['job_class'] = 'GitlabRemoveProjectExportWorker'
+Settings.cron_jobs['gitlab_usage_ping_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['gitlab_usage_ping_worker']['cron'] ||= Settings.send(:cron_random_weekly_time)
+Settings.cron_jobs['gitlab_usage_ping_worker']['job_class'] = 'GitlabUsagePingWorker'
 
 #
 # GitLab Shell

db/migrate/20160713222618_add_usage_ping_to_application_settings.rb

+class AddUsagePingToApplicationSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  def change
+    add_column :application_settings, :usage_ping_enabled, :boolean, default: true, null: false
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20160712171823) do
+ActiveRecord::Schema.define(version: 20160713222618) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -95,6 +95,7 @@ ActiveRecord::Schema.define(version: 20160712171823) do
     t.string   "elasticsearch_port",                    default: "9200"
     t.string   "repository_storage",                    default: "default"
     t.string   "enabled_git_access_protocol"
+    t.boolean  "usage_ping_enabled",                    default: true,        null: false
   end
 
   create_table "approvals", force: :cascade do |t|

spec/workers/gitlab_usage_ping_worker_spec.rb

+require 'spec_helper'
+
+describe GitlabUsagePingWorker do
+  subject { GitlabUsagePingWorker.new }
+
+  it "gathers license data" do
+    data = subject.data
+    license = License.current
+
+    expect(data[:license_md5]).to eq(Digest::MD5.hexdigest(license.data))
+    expect(data[:version]).to eq(Gitlab::VERSION)
+    expect(data[:licensee]).to eq(license.licensee)
+    expect(data[:active_user_count]).to eq(User.active.count)
+    expect(data[:licensee]).to eq(license.licensee)
+    expect(data[:license_user_count]).to eq(license.user_count)
+    expect(data[:license_starts_at]).to eq(license.starts_at)
+    expect(data[:license_expires_at]).to eq(license.expires_at)
+    expect(data[:license_add_ons]).to eq(license.add_ons)
+    expect(data[:recorded_at]).to be_a(Time)
+  end
+
+  it "sends POST request" do
+    stub_application_setting(usage_ping_enabled: true)
+
+    stub_request(:post, "https://version.gitlab.com/usage_data").
+        to_return(status: 200, body: '', headers: {})
+    expect(subject).to receive(:try_obtain_lease).and_return(true)
+
+    expect(subject.perform.response.code.to_i).to eq(200)
+  end
+
+  it "does not run if usage ping is disabled" do
+    stub_application_setting(usage_ping_enabled: false)
+
+    expect(subject).not_to receive(:try_obtain_lease)
+    expect(subject).not_to receive(:perform)
+  end
+end