Merge branch '5621_usage_ping_for_security_products' into 'master'

Usage data for security products

Closes #5621

See merge request gitlab-org/gitlab-ee!6602

db/schema.rb

@@ -430,6 +430,7 @@ ActiveRecord::Schema.define(version: 20180803001726) do
   add_index "ci_builds", ["commit_id", "status", "type"], name: "index_ci_builds_on_commit_id_and_status_and_type", using: :btree
   add_index "ci_builds", ["commit_id", "type", "name", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_name_and_ref", using: :btree
   add_index "ci_builds", ["commit_id", "type", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_ref", using: :btree
+  add_index "ci_builds", ["name"], name: "index_ci_builds_on_name_for_security_products_values", where: "((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, ('dast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('license_management'::character varying)::text, ('sast'::character varying)::text]))", using: :btree
   add_index "ci_builds", ["project_id", "id"], name: "index_ci_builds_on_project_id_and_id", using: :btree
   add_index "ci_builds", ["protected"], name: "index_ci_builds_on_protected", using: :btree
   add_index "ci_builds", ["runner_id"], name: "index_ci_builds_on_runner_id", using: :btree

doc/user/admin_area/settings/usage_statistics.md

@@ -33,7 +33,8 @@ disable the version check at **Admin area > Settings > Usage statistics**.
 
 > [Introduced][ee-557] in GitLab Enterprise Edition 8.10. More statistics
 [were added][ee-735] in GitLab Enterprise Edition
-8.12. [Moved to GitLab Community Edition][ce-23361] in 9.1.
+8.12. [Moved to GitLab Community Edition][ce-23361] in 9.1. More statistics
+[were added][[ee-6602] in 11.2
 
 GitLab sends a weekly payload containing usage data to GitLab Inc. The usage
 ping uses high-level data to help our product, support, and sales teams. It does
@@ -70,3 +71,4 @@ production: &base
 [ee-557]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/557
 [ee-735]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/735
 [ce-23361]: https://gitlab.com/gitlab-org/gitlab-ce/issues/23361
+[ee-6602]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/6602

ee/app/models/license.rb

@@ -307,6 +307,19 @@ class License < ActiveRecord::Base
     restricted_attr(:plan).presence || STARTER_PLAN
   end
 
+  def edition
+    case restricted_attr(:plan)
+    when 'ultimate'
+      'EEU'
+    when 'premium'
+      'EEP'
+    when 'starter'
+      'EES'
+    else # Older licenses
+      'EE'
+    end
+  end
+
   def current_active_users_count
     @current_active_users_count ||= begin
       if exclude_guests_from_active_count?

ee/changelogs/unreleased/5621_usage_ping_for_security_products.yml

+---
+title: Add security products to usage ping
+merge_request: 6602
+author:
+type: changed

ee/db/migrate/20180720082636_add_name_index_to_ci_builds.rb

+# frozen_string_literal: true
+
+class AddNameIndexToCiBuilds < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :ci_builds,
+      [:name],
+      name: 'index_ci_builds_on_name_for_security_products_values',
+      where: "name IN ('container_scanning','dast','dependency_scanning','license_management','sast')"
+  end
+
+  def down
+    remove_concurrent_index :ci_builds, [:name], name: 'index_ci_builds_on_name_for_security_products_values'
+  end
+end

ee/lib/ee/gitlab/usage_data.rb

+# frozen_string_literal: true
+
+module EE
+  module Gitlab
+    module UsageData
+      extend ::Gitlab::Utils::Override
+
+      override :features_usage_data
+      def features_usage_data
+        super.merge(features_usage_data_ee)
+      end
+
+      def features_usage_data_ee
+        {
+          elasticsearch_enabled: ::Gitlab::CurrentSettings.elasticsearch_search?,
+          geo_enabled: ::Gitlab::Geo.enabled?
+        }
+      end
+
+      override :license_usage_data
+      def license_usage_data
+        usage_data = super
+        license = ::License.current
+        usage_data[:edition] =
+          if license
+            license.edition
+          else
+            'EE Free'
+          end
+
+        if license
+          usage_data[:license_md5] = license.md5
+          usage_data[:license_id] = license.license_id
+          usage_data[:historical_max_users] = ::HistoricalData.max_historical_user_count
+          usage_data[:licensee] = license.licensee
+          usage_data[:license_user_count] = license.restricted_user_count
+          usage_data[:license_starts_at] = license.starts_at
+          usage_data[:license_expires_at] = license.expires_at
+          usage_data[:license_plan] = license.plan
+          usage_data[:license_add_ons] = license.add_ons
+          usage_data[:license_trial] = license.trial?
+        end
+
+        usage_data
+      end
+
+      def projects_mirrored_with_pipelines_enabled
+        ::Project.joins(:project_feature).where(
+          mirror: true,
+          mirror_trigger_builds: true,
+          project_features: {
+            builds_access_level: ::ProjectFeature::ENABLED
+          }
+        ).count
+      end
+
+      def service_desk_counts
+        return {} unless ::License.feature_available?(:service_desk)
+
+        projects_with_service_desk = ::Project.where(service_desk_enabled: true)
+
+        {
+          service_desk_enabled_projects: projects_with_service_desk.count,
+          service_desk_issues: ::Issue.where(
+            project: projects_with_service_desk,
+            author: ::User.support_bot,
+            confidential: true
+          ).count
+        }
+      end
+
+      def security_products_usage
+        types = {
+          container_scanning: :container_scanning_jobs,
+          dast: :dast_jobs,
+          dependency_scanning: :dependency_scanning_jobs,
+          license_management: :license_management_jobs,
+          sast: :sast_jobs
+        }
+
+        results = ::Ci::Build.where(name: types.keys).group(:name).count
+        results.each_with_object({}) { |(key, value), response| response[types[key.to_sym]] = value  }
+      end
+
+      override :system_usage_data
+      def system_usage_data
+        usage_data = super
+
+        usage_data[:counts] = usage_data[:counts].merge({
+          epics: ::Epic.count,
+          geo_nodes: ::GeoNode.count,
+          ldap_group_links: ::LdapGroupLink.count,
+          ldap_keys: ::LDAPKey.count,
+          ldap_users: ::User.ldap.count,
+          projects_reporting_ci_cd_back_to_github: ::GithubService.without_defaults.active.count,
+          projects_mirrored_with_pipelines_enabled: projects_mirrored_with_pipelines_enabled
+        }).merge(service_desk_counts).merge(security_products_usage)
+
+        usage_data
+      end
+    end
+  end
+end

ee/spec/lib/gitlab/usage_data_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::UsageData do
+  let(:projects) { create_list(:project, 3) }
+  let!(:board) { create(:board, project: projects[0]) }
+
+  describe '#data' do
+    before do
+      pipeline = create(:ci_pipeline, project: projects[0])
+      create(:ci_build, name: 'container_scanning', pipeline: pipeline)
+      create(:ci_build, name: 'dast', pipeline: pipeline)
+      create(:ci_build, name: 'dependency_scanning', pipeline: pipeline)
+      create(:ci_build, name: 'license_management', pipeline: pipeline)
+      create(:ci_build, name: 'sast', pipeline: pipeline)
+    end
+
+    subject { described_class.data }
+
+    it "gathers usage data" do
+      expect(subject.keys).to include(*%i(
+        historical_max_users
+        license_add_ons
+        license_plan
+        license_expires_at
+        license_starts_at
+        license_user_count
+        license_trial
+        licensee
+        license_md5
+        license_id
+        elasticsearch_enabled
+        geo_enabled
+      ))
+    end
+
+    it "gathers usage counts" do
+      count_data = subject[:counts]
+
+      expect(count_data[:boards]).to eq(1)
+      expect(count_data[:projects]).to eq(3)
+
+      expect(count_data.keys).to include(*%i(
+        projects_mirrored_with_pipelines_enabled
+        epics
+        geo_nodes
+        ldap_group_links
+        ldap_keys
+        ldap_users
+        projects_reporting_ci_cd_back_to_github
+        container_scanning_jobs
+        dast_jobs
+        dependency_scanning_jobs
+        license_management_jobs
+        sast_jobs
+      ))
+    end
+
+    it 'gathers security products usage data' do
+      count_data = subject[:counts]
+
+      expect(count_data[:container_scanning_jobs]).to eq(1)
+      expect(count_data[:dast_jobs]).to eq(1)
+      expect(count_data[:dependency_scanning_jobs]).to eq(1)
+      expect(count_data[:license_management_jobs]).to eq(1)
+      expect(count_data[:sast_jobs]).to eq(1)
+    end
+  end
+
+  describe '#features_usage_data_ee' do
+    subject { described_class.features_usage_data_ee }
+
+    it 'gathers feature usage data of EE' do
+      expect(subject[:elasticsearch_enabled]).to eq(Gitlab::CurrentSettings.elasticsearch_search?)
+      expect(subject[:geo_enabled]).to eq(Gitlab::Geo.enabled?)
+    end
+  end
+
+  describe 'License edition names' do
+    let(:ultimate) { create(:license, plan: 'ultimate') }
+    let(:premium) { create(:license, plan: 'premium') }
+    let(:starter) { create(:license, plan: 'starter') }
+    let(:old) { create(:license, plan: 'other') }
+
+    it "have expected values" do
+      expect(ultimate.edition).to eq('EEU')
+      expect(premium.edition).to eq('EEP')
+      expect(starter.edition).to eq('EES')
+      expect(old.edition).to eq('EE')
+    end
+  end
+
+  describe '#license_usage_data' do
+    subject { described_class.license_usage_data }
+
+    it "gathers license data" do
+      license = ::License.current
+
+      expect(subject[:license_md5]).to eq(Digest::MD5.hexdigest(license.data))
+      expect(subject[:license_id]).to eq(license.license_id)
+      expect(subject[:historical_max_users]).to eq(::HistoricalData.max_historical_user_count)
+      expect(subject[:licensee]).to eq(license.licensee)
+      expect(subject[:license_user_count]).to eq(license.restricted_user_count)
+      expect(subject[:license_starts_at]).to eq(license.starts_at)
+      expect(subject[:license_expires_at]).to eq(license.expires_at)
+      expect(subject[:license_add_ons]).to eq(license.add_ons)
+      expect(subject[:license_trial]).to eq(license.trial?)
+    end
+  end
+
+  describe '.service_desk_counts' do
+    subject { described_class.service_desk_counts }
+
+    before do
+      Project.update_all(service_desk_enabled: true)
+    end
+
+    context 'when Service Desk is disabled' do
+      it 'returns an empty hash' do
+        stub_licensed_features(service_desk: false)
+
+        expect(subject).to eq({})
+      end
+    end
+
+    context 'when there is no license' do
+      it 'returns an empty hash' do
+        allow(License).to receive(:current).and_return(nil)
+
+        expect(subject).to eq({})
+      end
+    end
+
+    context 'when Service Desk is enabled' do
+      it 'gathers Service Desk data' do
+        create_list(:issue, 3, confidential: true, author: User.support_bot, project: projects[0])
+
+        stub_licensed_features(service_desk: true)
+        allow(::EE::Gitlab::ServiceDesk).to receive(:enabled?).with(anything).and_return(true)
+
+        expect(subject).to eq(service_desk_enabled_projects: 3,
+                              service_desk_issues: 3)
+      end
+    end
+  end
+end

lib/gitlab/usage_data.rb

 module Gitlab
   class UsageData
     class << self
+      prepend EE::Gitlab::UsageData
+
       def data(force_refresh: false)
         Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data }
       end
@@ -27,23 +29,6 @@ module Gitlab
           edition: 'EE'
         }
 
-        license = ::License.current
-
-        usage_data[:edition] = license_edition(license)
-
-        if license
-          usage_data[:license_md5] = license.md5
-          usage_data[:license_id] = license.license_id
-          usage_data[:historical_max_users] = ::HistoricalData.max_historical_user_count
-          usage_data[:licensee] = license.licensee
-          usage_data[:license_user_count] = license.restricted_user_count
-          usage_data[:license_starts_at] = license.starts_at
-          usage_data[:license_expires_at] = license.expires_at
-          usage_data[:license_plan] = license.plan
-          usage_data[:license_add_ons] = license.add_ons
-          usage_data[:license_trial] = license.trial?
-        end
-
         usage_data
       end
 
@@ -65,8 +50,6 @@ module Gitlab
             deploy_keys: DeployKey.count,
             deployments: Deployment.count,
             environments: ::Environment.count,
-            epics: ::Epic.count,
-            geo_nodes: GeoNode.count,
             clusters: ::Clusters::Cluster.count,
             clusters_enabled: ::Clusters::Cluster.enabled.count,
             clusters_disabled: ::Clusters::Cluster.disabled.count,
@@ -81,9 +64,6 @@ module Gitlab
             issues: Issue.count,
             keys: Key.count,
             labels: Label.count,
-            ldap_group_links: LdapGroupLink.count,
-            ldap_keys: LDAPKey.count,
-            ldap_users: User.ldap.count,
             lfs_objects: LfsObject.count,
             merge_requests: MergeRequest.count,
             milestones: Milestone.count,
@@ -91,8 +71,6 @@ module Gitlab
             pages_domains: PagesDomain.count,
             projects: Project.count,
             projects_imported_from_github: Project.where(import_type: 'github').count,
-            projects_reporting_ci_cd_back_to_github: GithubService.without_defaults.active.count,
-            projects_mirrored_with_pipelines_enabled: projects_mirrored_with_pipelines_enabled,
             protected_branches: ProtectedBranch.count,
             releases: Release.count,
             remote_mirrors: RemoteMirror.count,
@@ -100,20 +78,7 @@ module Gitlab
             todos: Todo.count,
             uploads: Upload.count,
             web_hooks: WebHook.count
-          }.merge(service_desk_counts).merge(services_usage)
-        }
-      end
-
-      def service_desk_counts
-        return {} unless ::License.feature_available?(:service_desk)
-
-        projects_with_service_desk = Project.where(service_desk_enabled: true)
-
-        {
-          service_desk_enabled_projects: projects_with_service_desk.count,
-          service_desk_issues: Issue.where(project: projects_with_service_desk,
-                                           author: User.support_bot,
-                                           confidential: true).count
+          }.merge(services_usage)
         }
       end
 
@@ -122,7 +87,7 @@ module Gitlab
       end
 
       def features_usage_data
-        features_usage_data_ce.merge(features_usage_data_ee)
+        features_usage_data_ce
       end
 
       def features_usage_data_ce
@@ -138,13 +103,6 @@ module Gitlab
         }
       end
 
-      def features_usage_data_ee
-        {
-          elasticsearch_enabled: Gitlab::CurrentSettings.elasticsearch_search?,
-          geo_enabled: Gitlab::Geo.enabled?
-        }
-      end
-
       def components_usage_data
         {
           gitlab_pages: { enabled: Gitlab.config.pages.enabled, version: Gitlab::Pages::VERSION },
@@ -153,21 +111,6 @@ module Gitlab
         }
       end
 
-      def license_edition(license)
-        return 'EE Free' unless license
-
-        case license.plan
-        when 'ultimate'
-          'EEU'
-        when 'premium'
-          'EEP'
-        when 'starter'
-          'EES'
-        else # Older licenses
-          'EE'
-        end
-      end
-
       def services_usage
         types = {
           JiraService: :projects_jira_active,
@@ -179,16 +122,6 @@ module Gitlab
         results = Service.unscoped.where(type: types.keys, active: true).group(:type).count
         results.each_with_object({}) { |(key, value), response| response[types[key.to_sym]] = value  }
       end
-
-      def projects_mirrored_with_pipelines_enabled
-        Project.joins(:project_feature).where(
-          mirror: true,
-          mirror_trigger_builds: true,
-          project_features: {
-            builds_access_level: ProjectFeature::ENABLED
-          }
-        ).count
-      end
     end
   end
 end

spec/lib/gitlab/usage_data_spec.rb

@@ -26,19 +26,9 @@ describe Gitlab::UsageData do
     subject { described_class.data }
 
     it "gathers usage data" do
-      expect(subject.keys).to match_array(%i(
+      expect(subject.keys).to include(*%i(
         active_user_count
         counts
-        historical_max_users
-        license_add_ons
-        license_plan
-        license_expires_at
-        license_starts_at
-        license_user_count
-        license_trial
-        licensee
-        license_md5
-        license_id
         recorded_at
         edition
         version
@@ -53,8 +43,6 @@ describe Gitlab::UsageData do
         reply_by_email_enabled
         container_registry_enabled
         gitlab_shared_runners_enabled
-        elasticsearch_enabled
-        geo_enabled
         gitlab_pages
         git
         database
@@ -68,10 +56,9 @@ describe Gitlab::UsageData do
       expect(count_data[:boards]).to eq(1)
       expect(count_data[:projects]).to eq(3)
 
-      expect(count_data.keys).to match_array(%i(
+      expect(count_data.keys).to include(*%i(
         boards
         ci_builds
-        projects_mirrored_with_pipelines_enabled
         ci_internal_pipelines
         ci_external_pipelines
         ci_pipeline_config_auto_devops
@@ -84,8 +71,6 @@ describe Gitlab::UsageData do
         deploy_keys
         deployments
         environments
-        epics
-        geo_nodes
         clusters
         clusters_enabled
         clusters_disabled
@@ -100,16 +85,12 @@ describe Gitlab::UsageData do
         issues
         keys
         labels
-        ldap_group_links
-        ldap_keys
-        ldap_users
         lfs_objects
         merge_requests
         milestones
         notes
         projects
         projects_imported_from_github
-        projects_reporting_ci_cd_back_to_github
         projects_jira_active
         projects_slack_notifications_active
         projects_slack_slash_active
@@ -160,15 +141,6 @@ describe Gitlab::UsageData do
     end
   end
 
-  describe '#features_usage_data_ee' do
-    subject { described_class.features_usage_data_ee }
-
-    it 'gathers feature usage data of EE' do
-      expect(subject[:elasticsearch_enabled]).to eq(Gitlab::CurrentSettings.elasticsearch_search?)
-      expect(subject[:geo_enabled]).to eq(Gitlab::Geo.enabled?)
-    end
-  end
-
   describe '#components_usage_data' do
     subject { described_class.components_usage_data }
 
@@ -185,58 +157,11 @@ describe Gitlab::UsageData do
     subject { described_class.license_usage_data }
 
     it "gathers license data" do
-      license = ::License.current
-
       expect(subject[:uuid]).to eq(Gitlab::CurrentSettings.uuid)
-      expect(subject[:license_md5]).to eq(Digest::MD5.hexdigest(license.data))
-      expect(subject[:license_id]).to eq(license.license_id)
       expect(subject[:version]).to eq(Gitlab::VERSION)
-      expect(subject[:licensee]).to eq(license.licensee)
       expect(subject[:installation_type]).to eq(Gitlab::INSTALLATION_TYPE)
       expect(subject[:active_user_count]).to eq(User.active.count)
-      expect(subject[:licensee]).to eq(license.licensee)
-      expect(subject[:license_user_count]).to eq(license.restricted_user_count)
-      expect(subject[:license_starts_at]).to eq(license.starts_at)
-      expect(subject[:license_expires_at]).to eq(license.expires_at)
-      expect(subject[:license_add_ons]).to eq(license.add_ons)
-      expect(subject[:license_trial]).to eq(license.trial?)
       expect(subject[:recorded_at]).to be_a(Time)
     end
   end
-
-  describe '.service_desk_counts' do
-    subject { described_class.service_desk_counts }
-
-    before do
-      Project.update_all(service_desk_enabled: true)
-    end
-
-    context 'when Service Desk is disabled' do
-      it 'returns an empty hash' do
-        allow(License).to receive(:feature_available?).with(:service_desk).and_return(false)
-
-        expect(subject).to eq({})
-      end
-    end
-
-    context 'when there is no license' do
-      it 'returns an empty hash' do
-        allow(License).to receive(:current).and_return(nil)
-
-        expect(subject).to eq({})
-      end
-    end
-
-    context 'when Service Desk is enabled' do
-      it 'gathers Service Desk data' do
-        create_list(:issue, 3, confidential: true, author: User.support_bot, project: project)
-
-        allow(License).to receive(:feature_available?).with(:service_desk).and_return(true)
-        allow(::EE::Gitlab::ServiceDesk).to receive(:enabled?).with(anything).and_return(true)
-
-        expect(subject).to eq(service_desk_enabled_projects: 4,
-                              service_desk_issues: 3)
-      end
-    end
-  end
 end