Commit bf6029d4 authored by Robert Speicher's avatar Robert Speicher

Merge branch...

Merge branch 'ee-45938-postgres-timeout-when-counting-number-of-ci-builds-for-usage-ping' into 'master'

Resolve "Postgres timeout when counting number of CI builds for usage ping"

See merge request gitlab-org/gitlab-ee!7251
parents fd40843a 078b2440
---
title: Handle database statement timeouts in usage ping
merge_request: 21523
author:
type: fixed
...@@ -45,13 +45,13 @@ module EE ...@@ -45,13 +45,13 @@ module EE
end end
def projects_mirrored_with_pipelines_enabled def projects_mirrored_with_pipelines_enabled
::Project.joins(:project_feature).where( count(::Project.joins(:project_feature).where(
mirror: true, mirror: true,
mirror_trigger_builds: true, mirror_trigger_builds: true,
project_features: { project_features: {
builds_access_level: ::ProjectFeature::ENABLED builds_access_level: ::ProjectFeature::ENABLED
} }
).count ))
end end
def service_desk_counts def service_desk_counts
...@@ -60,12 +60,12 @@ module EE ...@@ -60,12 +60,12 @@ module EE
projects_with_service_desk = ::Project.where(service_desk_enabled: true) projects_with_service_desk = ::Project.where(service_desk_enabled: true)
{ {
service_desk_enabled_projects: projects_with_service_desk.count, service_desk_enabled_projects: count(projects_with_service_desk),
service_desk_issues: ::Issue.where( service_desk_issues: count(::Issue.where(
project: projects_with_service_desk, project: projects_with_service_desk,
author: ::User.support_bot, author: ::User.support_bot,
confidential: true confidential: true
).count ))
} }
end end
...@@ -78,7 +78,7 @@ module EE ...@@ -78,7 +78,7 @@ module EE
sast: :sast_jobs sast: :sast_jobs
} }
results = ::Ci::Build.where(name: types.keys).group(:name).count results = count(::Ci::Build.where(name: types.keys).group(:name))
results.each_with_object({}) { |(key, value), response| response[types[key.to_sym]] = value } results.each_with_object({}) { |(key, value), response| response[types[key.to_sym]] = value }
end end
...@@ -87,12 +87,12 @@ module EE ...@@ -87,12 +87,12 @@ module EE
usage_data = super usage_data = super
usage_data[:counts] = usage_data[:counts].merge({ usage_data[:counts] = usage_data[:counts].merge({
epics: ::Epic.count, epics: count(::Epic),
geo_nodes: ::GeoNode.count, geo_nodes: count(::GeoNode),
ldap_group_links: ::LdapGroupLink.count, ldap_group_links: count(::LdapGroupLink),
ldap_keys: ::LDAPKey.count, ldap_keys: count(::LDAPKey),
ldap_users: ::User.ldap.count, ldap_users: count(::User.ldap),
projects_reporting_ci_cd_back_to_github: ::GithubService.without_defaults.active.count, projects_reporting_ci_cd_back_to_github: count(::GithubService.without_defaults.active),
projects_mirrored_with_pipelines_enabled: projects_mirrored_with_pipelines_enabled projects_mirrored_with_pipelines_enabled: projects_mirrored_with_pipelines_enabled
}).merge(service_desk_counts).merge(security_products_usage) }).merge(service_desk_counts).merge(security_products_usage)
......
...@@ -24,7 +24,7 @@ module Gitlab ...@@ -24,7 +24,7 @@ module Gitlab
hostname: Gitlab.config.gitlab.host, hostname: Gitlab.config.gitlab.host,
version: Gitlab::VERSION, version: Gitlab::VERSION,
installation_type: Gitlab::INSTALLATION_TYPE, installation_type: Gitlab::INSTALLATION_TYPE,
active_user_count: User.active.count, active_user_count: count(User.active),
recorded_at: Time.now, recorded_at: Time.now,
edition: 'EE' edition: 'EE'
} }
...@@ -36,51 +36,51 @@ module Gitlab ...@@ -36,51 +36,51 @@ module Gitlab
def system_usage_data def system_usage_data
{ {
counts: { counts: {
assignee_lists: List.assignee.count, assignee_lists: count(List.assignee),
boards: Board.count, boards: count(Board),
ci_builds: ::Ci::Build.count, ci_builds: count(::Ci::Build),
ci_internal_pipelines: ::Ci::Pipeline.internal.count, ci_internal_pipelines: count(::Ci::Pipeline.internal),
ci_external_pipelines: ::Ci::Pipeline.external.count, ci_external_pipelines: count(::Ci::Pipeline.external),
ci_pipeline_config_auto_devops: ::Ci::Pipeline.auto_devops_source.count, ci_pipeline_config_auto_devops: count(::Ci::Pipeline.auto_devops_source),
ci_pipeline_config_repository: ::Ci::Pipeline.repository_source.count, ci_pipeline_config_repository: count(::Ci::Pipeline.repository_source),
ci_runners: ::Ci::Runner.count, ci_runners: count(::Ci::Runner),
ci_triggers: ::Ci::Trigger.count, ci_triggers: count(::Ci::Trigger),
ci_pipeline_schedules: ::Ci::PipelineSchedule.count, ci_pipeline_schedules: count(::Ci::PipelineSchedule),
auto_devops_enabled: ::ProjectAutoDevops.enabled.count, auto_devops_enabled: count(::ProjectAutoDevops.enabled),
auto_devops_disabled: ::ProjectAutoDevops.disabled.count, auto_devops_disabled: count(::ProjectAutoDevops.disabled),
deploy_keys: DeployKey.count, deploy_keys: count(DeployKey),
deployments: Deployment.count, deployments: count(Deployment),
environments: ::Environment.count, environments: count(::Environment),
clusters: ::Clusters::Cluster.count, clusters: count(::Clusters::Cluster),
clusters_enabled: ::Clusters::Cluster.enabled.count, clusters_enabled: count(::Clusters::Cluster.enabled),
clusters_disabled: ::Clusters::Cluster.disabled.count, clusters_disabled: count(::Clusters::Cluster.disabled),
clusters_platforms_gke: ::Clusters::Cluster.gcp_installed.enabled.count, clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled),
clusters_platforms_user: ::Clusters::Cluster.user_provided.enabled.count, clusters_platforms_user: count(::Clusters::Cluster.user_provided.enabled),
clusters_applications_helm: ::Clusters::Applications::Helm.installed.count, clusters_applications_helm: count(::Clusters::Applications::Helm.installed),
clusters_applications_ingress: ::Clusters::Applications::Ingress.installed.count, clusters_applications_ingress: count(::Clusters::Applications::Ingress.installed),
clusters_applications_prometheus: ::Clusters::Applications::Prometheus.installed.count, clusters_applications_prometheus: count(::Clusters::Applications::Prometheus.installed),
clusters_applications_runner: ::Clusters::Applications::Runner.installed.count, clusters_applications_runner: count(::Clusters::Applications::Runner.installed),
in_review_folder: ::Environment.in_review_folder.count, in_review_folder: count(::Environment.in_review_folder),
groups: Group.count, groups: count(Group),
issues: Issue.count, issues: count(Issue),
keys: Key.count, keys: count(Key),
label_lists: List.label.count, label_lists: count(List.label),
labels: Label.count, labels: count(Label),
lfs_objects: LfsObject.count, lfs_objects: count(LfsObject),
merge_requests: MergeRequest.count, merge_requests: count(MergeRequest),
milestone_lists: List.milestone.count, milestone_lists: count(List.milestone),
milestones: Milestone.count, milestones: count(Milestone),
notes: Note.count, notes: count(Note),
pages_domains: PagesDomain.count, pages_domains: count(PagesDomain),
projects: Project.count, projects: count(Project),
projects_imported_from_github: Project.where(import_type: 'github').count, projects_imported_from_github: count(Project.where(import_type: 'github')),
protected_branches: ProtectedBranch.count, protected_branches: count(ProtectedBranch),
releases: Release.count, releases: count(Release),
remote_mirrors: RemoteMirror.count, remote_mirrors: count(RemoteMirror),
snippets: Snippet.count, snippets: count(Snippet),
todos: Todo.count, todos: count(Todo),
uploads: Upload.count, uploads: count(Upload),
web_hooks: WebHook.count web_hooks: count(WebHook)
}.merge(services_usage) }.merge(services_usage)
} }
end end
...@@ -122,8 +122,14 @@ module Gitlab ...@@ -122,8 +122,14 @@ module Gitlab
PrometheusService: :projects_prometheus_active PrometheusService: :projects_prometheus_active
} }
results = Service.unscoped.where(type: types.keys, active: true).group(:type).count results = count(Service.unscoped.where(type: types.keys, active: true).group(:type), fallback: Hash.new(-1))
results.each_with_object({}) { |(key, value), response| response[types[key.to_sym]] = value } types.each_with_object({}) { |(klass, key), response| response[key] = results[klass.to_s] || 0 }
end
def count(relation, fallback: -1)
relation.count
rescue ActiveRecord::StatementInvalid
fallback
end end
end end
end end
......
...@@ -167,4 +167,20 @@ describe Gitlab::UsageData do ...@@ -167,4 +167,20 @@ describe Gitlab::UsageData do
expect(subject[:recorded_at]).to be_a(Time) expect(subject[:recorded_at]).to be_a(Time)
end end
end end
describe '#count' do
let(:relation) { double(:relation) }
it 'returns the count when counting succeeds' do
allow(relation).to receive(:count).and_return(1)
expect(described_class.count(relation)).to eq(1)
end
it 'returns the fallback value when counting fails' do
allow(relation).to receive(:count).and_raise(ActiveRecord::StatementInvalid.new(''))
expect(described_class.count(relation, fallback: 15)).to eq(15)
end
end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment