Merge branch 'sh-improve-ee-usage-ping' into 'master'

Add more usage data to EE ping

This MR loads the EE usage ping asynchronously on the admin application settings page and now includes the counts of the following items:

* Comments
* Groups
* Users
* Projects
* Issues
* Labels
* CI builds
* Snippets
* Milestones
* Todos
* Pushes
* Merge requests
* Environments
* Triggers
* Deploy keys
* Pages
* Project Services
* Issue Boards
* CI Runners
* Deployments
* Geo Nodes
* LDAP Groups
* LDAP Keys
* LDAP Users
* LFS objects
* Protected branches
* Releases
* Remote mirrors
* Web hooks

Closes #997

See merge request !735

CHANGELOG-EE

 Please view this file on the master branch, on stable branches it's out of date.
 v 8.12.0 (Unreleased)
+  - Include more data in EE usage ping
   - Reduce UPDATE queries when moving between import states on projects
   - [ES] Instrument Elasticsearch::Git::Repository
   - Request only the LDAP attributes we need

app/assets/javascripts/application_settings.js.es6

+(global => {
+  global.gl = global.gl || {};
+
+  gl.ApplicationSettings = function() {
+    var usage_data_url = $('.usage-data').data('endpoint');
+
+    $.ajax({
+      type: "GET",
+      url: usage_data_url,
+      dataType: "html",
+      success: function (html) {
+        $(".usage-data").html(html);
+      }
+    });
+  }
+})(window);

app/assets/javascripts/dispatcher.js

@@ -197,6 +197,9 @@
         case 'admin':
           new Admin();
           switch (path[1]) {
+            case 'application_settings':
+              new gl.ApplicationSettings();
+              break;
             case 'groups':
               new UsersSelect();
               break;

app/controllers/admin/application_settings_controller.rb

@@ -13,6 +13,13 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
     end
   end
 
+  def usage_data
+    respond_to do |format|
+      format.html { render html: Gitlab::Highlight.highlight('payload.json', Gitlab::UsageData.to_json) }
+      format.json { render json: Gitlab::UsageData.to_json }
+    end
+  end
+
   def reset_runners_token
     @application_setting.reset_runners_registration_token!
     flash[:notice] = 'New runners registration token has been generated!'

app/helpers/license_helper.rb

@@ -4,8 +4,7 @@ module LicenseHelper
   end
 
   def max_historical_user_count
-    date_range = (Date.today - 1.year)..Date.today
-    HistoricalData.during(date_range).maximum(:active_user_count) || 0
+    HistoricalData.max_historical_user_count
   end
 
   def license_message(signed_in: signed_in?, is_admin: (current_user && current_user.is_admin?))
@@ -17,25 +16,6 @@ module LicenseHelper
       end
   end
 
-  def license_usage_data
-    usage_data = { version: Gitlab::VERSION,
-                   active_user_count: current_active_user_count }
-    license = License.current
-
-    if license
-      usage_data[:license_md5] = Digest::MD5.hexdigest(license.data)
-      usage_data[:historical_max_users] = max_historical_user_count
-      usage_data[:licensee] = license.licensee
-      usage_data[:license_user_count] = license.user_count
-      usage_data[:license_starts_at] = license.starts_at
-      usage_data[:license_expires_at] = license.expires_at
-      usage_data[:license_add_ons] = license.add_ons
-      usage_data[:recorded_at] = Time.now
-    end
-
-    usage_data
-  end
-
   private
 
   def no_license_message(signed_in, is_admin)

app/models/historical_data.rb

@@ -18,5 +18,9 @@ class HistoricalData < ActiveRecord::Base
     def at(date)
       find_by(date: date)
     end
+
+    def max_historical_user_count
+      HistoricalData.during(1.year.ago..Date.today).maximum(:active_user_count) || 0
+    end
   end
 end

app/views/admin/application_settings/_form.html.haml

@@ -61,10 +61,11 @@
           = f.label :usage_ping_enabled do
             = f.check_box :usage_ping_enabled
             Usage ping enabled
-          .help-block
-            Every week GitLab will report license usage back to GitLab, Inc.
-            Disable this option if you do not want this to occur. This is the JSON payload that will be sent:
-            %pre.js-syntax-highlight.code.highlight= Gitlab::Highlight.highlight('payload.json', license_usage_data.to_json)
+          .container
+            .help-block
+              Every week GitLab will report license usage back to GitLab, Inc.
+              Disable this option if you do not want this to occur. This is the JSON payload that will be sent:
+              %pre.usage-data.js-syntax-highlight.code.highlight{ "data-endpoint" => usage_data_admin_application_settings_path(format: :html) }
     .form-group
       .col-sm-offset-2.col-sm-10
         .checkbox

app/workers/gitlab_usage_ping_worker.rb

 class GitlabUsagePingWorker
   LEASE_TIMEOUT = 86400
 
-  include LicenseHelper
   include Sidekiq::Worker
   include HTTParty
 
@@ -16,7 +15,7 @@ class GitlabUsagePingWorker
 
     begin
       HTTParty.post(url,
-                    body: license_usage_data.to_json,
+                    body: Gitlab::UsageData.to_json,
                     headers: { 'Content-type' => 'application/json' }
                    )
     rescue HTTParty::Error => e

config/routes.rb

@@ -320,6 +320,7 @@ Rails.application.routes.draw do
 
     resource :application_settings, only: [:show, :update] do
       resources :services, only: [:index, :edit, :update]
+      get :usage_data
       put :reset_runners_token
       put :reset_health_check_token
       put :clear_repository_check_states

lib/gitlab/usage_data.rb

+module Gitlab
+  class UsageData
+    class << self
+      def data
+        Rails.cache.fetch('usage_data', expires_in: 1.hour) { uncached_data }
+      end
+
+      def uncached_data
+        license_usage_data.merge(system_usage_data)
+      end
+
+      def to_json
+        data.to_json
+      end
+
+      def system_usage_data
+        {
+          counts: {
+            boards: Board.count,
+            ci_builds: ::Ci::Build.count,
+            ci_pipelines: ::Ci::Pipeline.count,
+            ci_runners: ::Ci::Runner.count,
+            ci_triggers: ::Ci::Trigger.count,
+            deploy_keys: DeployKey.count,
+            deployments: Deployment.count,
+            environments: Environment.count,
+            geo_nodes: GeoNode.count,
+            groups: Group.count,
+            issues: Issue.count,
+            keys: Key.count,
+            labels: Label.count,
+            ldap_group_links: LdapGroupLink.count,
+            ldap_keys: LDAPKey.count,
+            ldap_users: User.ldap.count,
+            lfs_objects: LfsObject.count,
+            merge_requests: MergeRequest.count,
+            milestones: Milestone.count,
+            notes: Note.count,
+            pushes: Event.code_push.count,
+            pages_domains: PagesDomain.count,
+            projects: Project.count,
+            protected_branchess: ProtectedBranch.count,
+            releases: Release.count,
+            remote_mirrors: RemoteMirror.count,
+            services: Service.where(active: true).count,
+            snippets: Snippet.count,
+            todos: Todo.count,
+            web_hooks: WebHook.count
+          }
+        }
+      end
+
+      def license_usage_data
+        usage_data = { version: Gitlab::VERSION,
+                       active_user_count: User.active.count }
+        license = ::License.current
+
+        if license
+          usage_data[:license_md5] = Digest::MD5.hexdigest(license.data)
+          usage_data[:historical_max_users] = ::HistoricalData.max_historical_user_count
+          usage_data[:licensee] = license.licensee
+          usage_data[:license_user_count] = license.user_count
+          usage_data[:license_starts_at] = license.starts_at
+          usage_data[:license_expires_at] = license.expires_at
+          usage_data[:license_add_ons] = license.add_ons
+          usage_data[:recorded_at] = Time.now
+        end
+
+        usage_data
+      end
+    end
+  end
+end

spec/controllers/admin/application_settings_controller_spec.rb

+require 'spec_helper'
+
+describe Admin::ApplicationSettingsController do
+  let(:group) { create(:group) }
+  let(:project) { create(:project, namespace: group) }
+  let(:admin) { create(:admin) }
+  let(:user) { create(:user)}
+
+  describe 'GET #usage_data with no access' do
+    before do
+      sign_in(user)
+    end
+
+    it 'returns 404' do
+      get :usage_data, format: :html
+
+      expect(response.status).to eq(404)
+    end
+  end
+
+  describe 'GET #usage_data' do
+    before do
+      sign_in(admin)
+    end
+
+    it 'returns HTML data' do
+      get :usage_data, format: :html
+
+      expect(response.body).to start_with('<span')
+      expect(response.status).to eq(200)
+    end
+
+    it 'returns JSON data' do
+      get :usage_data, format: :json
+
+      body = JSON.parse(response.body)
+      expect(body["version"]).to eq(Gitlab::VERSION)
+      expect(body).to include('counts')
+      expect(response.status).to eq(200)
+    end
+  end
+end

spec/helpers/license_helper_spec.rb

@@ -20,22 +20,4 @@ describe LicenseHelper do
       end
     end
   end
-
-  describe '#license_usage_data' do
-    it "gathers license data" do
-      data = license_usage_data
-      license = License.current
-
-      expect(data[:license_md5]).to eq(Digest::MD5.hexdigest(license.data))
-      expect(data[:version]).to eq(Gitlab::VERSION)
-      expect(data[:licensee]).to eq(license.licensee)
-      expect(data[:active_user_count]).to eq(User.active.count)
-      expect(data[:licensee]).to eq(license.licensee)
-      expect(data[:license_user_count]).to eq(license.user_count)
-      expect(data[:license_starts_at]).to eq(license.starts_at)
-      expect(data[:license_expires_at]).to eq(license.expires_at)
-      expect(data[:license_add_ons]).to eq(license.add_ons)
-      expect(data[:recorded_at]).to be_a(Time)
-    end
-  end
 end

spec/lib/gitlab/usage_data_spec.rb

+require 'spec_helper'
+
+describe Gitlab::UsageData do
+  let!(:project) { create(:project) }
+  let!(:project2) { create(:project) }
+  let!(:board) { create(:board, project: project) }
+
+  describe '#data' do
+    subject { Gitlab::UsageData.data }
+
+    it "gathers usage data" do
+      expect(subject.keys).to match_array(%i(
+        active_user_count
+        counts
+        historical_max_users
+        license_add_ons
+        license_expires_at
+        license_starts_at
+        license_user_count
+        licensee
+        license_md5
+        recorded_at
+        version
+      ))
+    end
+
+    it "gathers usage counts" do
+      count_data = subject[:counts]
+
+      expect(count_data[:boards]).to eq(1)
+      expect(count_data[:projects]).to eq(2)
+
+      expect(count_data.keys).to match_array(%i(
+        boards
+        ci_builds
+        ci_pipelines
+        ci_runners
+        ci_triggers
+        deploy_keys
+        deployments
+        environments
+        geo_nodes
+        groups
+        issues
+        keys
+        labels
+        ldap_group_links
+        ldap_keys
+        ldap_users
+        lfs_objects
+        merge_requests
+        milestones
+        notes
+        projects
+        pushes
+        pages_domains
+        protected_branchess
+        releases
+        remote_mirrors
+        services
+        snippets
+        todos
+        web_hooks
+      ))
+    end
+  end
+
+  describe '#license_usage_data' do
+    subject { Gitlab::UsageData.license_usage_data }
+
+    it "gathers license data" do
+      license = ::License.current
+
+      expect(subject[:license_md5]).to eq(Digest::MD5.hexdigest(license.data))
+      expect(subject[:version]).to eq(Gitlab::VERSION)
+      expect(subject[:licensee]).to eq(license.licensee)
+      expect(subject[:active_user_count]).to eq(User.active.count)
+      expect(subject[:licensee]).to eq(license.licensee)
+      expect(subject[:license_user_count]).to eq(license.user_count)
+      expect(subject[:license_starts_at]).to eq(license.starts_at)
+      expect(subject[:license_expires_at]).to eq(license.expires_at)
+      expect(subject[:license_add_ons]).to eq(license.add_ons)
+      expect(subject[:recorded_at]).to be_a(Time)
+    end
+  end
+end

spec/models/historical_data_spec.rb

@@ -38,4 +38,16 @@ describe HistoricalData do
       expect(data.active_user_count).to eq(5)
     end
   end
+
+  describe ".max_historical_user_count" do
+    before do
+      (1..3).each do |i|
+        HistoricalData.create!(date: Time.now - i.days, active_user_count: i * 100)
+      end
+    end
+
+    it "returns max user count for the past year" do
+      expect(HistoricalData.max_historical_user_count).to eq(300)
+    end
+  end
 end