Commit 3ef4f74b authored by Markus Koller's avatar Markus Koller

Add more storage statistics

This adds counters for build artifacts and LFS objects, and moves
the preexisting repository_size and commit_count from the projects
table into a new project_statistics table.

The counters are displayed in the administration area for projects
and groups, and also available through the API for admins (on */all)
and normal users (on */owned)

The statistics are updated through ProjectCacheWorker, which can now
do more granular updates with the new :statistics argument.
parent 6fd58ee4
class Admin::GroupsController < Admin::ApplicationController class Admin::GroupsController < Admin::ApplicationController
before_action :group, only: [:edit, :show, :update, :destroy, :project_update, :members_update] before_action :group, only: [:edit, :update, :destroy, :project_update, :members_update]
def index def index
@groups = Group.all @groups = Group.with_statistics
@groups = @groups.sort(@sort = params[:sort]) @groups = @groups.sort(@sort = params[:sort])
@groups = @groups.search(params[:name]) if params[:name].present? @groups = @groups.search(params[:name]) if params[:name].present?
@groups = @groups.page(params[:page]) @groups = @groups.page(params[:page])
end end
def show def show
@group = Group.with_statistics.find_by_full_path(params[:id])
@members = @group.members.order("access_level DESC").page(params[:members_page]) @members = @group.members.order("access_level DESC").page(params[:members_page])
@requesters = AccessRequestsFinder.new(@group).execute(current_user) @requesters = AccessRequestsFinder.new(@group).execute(current_user)
@projects = @group.projects.page(params[:projects_page]) @projects = @group.projects.with_statistics.page(params[:projects_page])
end end
def new def new
......
...@@ -3,7 +3,7 @@ class Admin::ProjectsController < Admin::ApplicationController ...@@ -3,7 +3,7 @@ class Admin::ProjectsController < Admin::ApplicationController
before_action :group, only: [:show, :transfer] before_action :group, only: [:show, :transfer]
def index def index
@projects = Project.all @projects = Project.with_statistics
@projects = @projects.in_namespace(params[:namespace_id]) if params[:namespace_id].present? @projects = @projects.in_namespace(params[:namespace_id]) if params[:namespace_id].present?
@projects = @projects.where(visibility_level: params[:visibility_level]) if params[:visibility_level].present? @projects = @projects.where(visibility_level: params[:visibility_level]) if params[:visibility_level].present?
@projects = @projects.with_push if params[:with_push].present? @projects = @projects.with_push if params[:with_push].present?
......
...@@ -75,7 +75,7 @@ class GroupsController < Groups::ApplicationController ...@@ -75,7 +75,7 @@ class GroupsController < Groups::ApplicationController
end end
def projects def projects
@projects = @group.projects.page(params[:page]) @projects = @group.projects.with_statistics.page(params[:page])
end end
def update def update
......
...@@ -246,11 +246,6 @@ module ProjectsHelper ...@@ -246,11 +246,6 @@ module ProjectsHelper
end end
end end
def repository_size(project = @project)
size_in_bytes = project.repository_size * 1.megabyte
number_to_human_size(size_in_bytes, delimiter: ',', precision: 2)
end
def default_url_to_repo(project = @project) def default_url_to_repo(project = @project)
case default_clone_protocol case default_clone_protocol
when 'ssh' when 'ssh'
......
...@@ -11,6 +11,7 @@ module SortingHelper ...@@ -11,6 +11,7 @@ module SortingHelper
sort_value_due_date_soon => sort_title_due_date_soon, sort_value_due_date_soon => sort_title_due_date_soon,
sort_value_due_date_later => sort_title_due_date_later, sort_value_due_date_later => sort_title_due_date_later,
sort_value_largest_repo => sort_title_largest_repo, sort_value_largest_repo => sort_title_largest_repo,
sort_value_largest_group => sort_title_largest_group,
sort_value_recently_signin => sort_title_recently_signin, sort_value_recently_signin => sort_title_recently_signin,
sort_value_oldest_signin => sort_title_oldest_signin, sort_value_oldest_signin => sort_title_oldest_signin,
sort_value_downvotes => sort_title_downvotes, sort_value_downvotes => sort_title_downvotes,
...@@ -92,6 +93,10 @@ module SortingHelper ...@@ -92,6 +93,10 @@ module SortingHelper
'Largest repository' 'Largest repository'
end end
def sort_title_largest_group
'Largest group'
end
def sort_title_recently_signin def sort_title_recently_signin
'Recent sign in' 'Recent sign in'
end end
...@@ -193,7 +198,11 @@ module SortingHelper ...@@ -193,7 +198,11 @@ module SortingHelper
end end
def sort_value_largest_repo def sort_value_largest_repo
'repository_size_desc' 'storage_size_desc'
end
def sort_value_largest_group
'storage_size_desc'
end end
def sort_value_recently_signin def sort_value_recently_signin
......
module StorageHelper
def storage_counter(size_in_bytes)
precision = size_in_bytes < 1.megabyte ? 0 : 1
number_to_human_size(size_in_bytes, delimiter: ',', precision: precision, significant: false)
end
end
...@@ -43,6 +43,8 @@ module Ci ...@@ -43,6 +43,8 @@ module Ci
before_destroy { project } before_destroy { project }
after_create :execute_hooks after_create :execute_hooks
after_save :update_project_statistics, if: :artifacts_size_changed?
after_destroy :update_project_statistics
class << self class << self
def first_pending def first_pending
...@@ -584,5 +586,9 @@ module Ci ...@@ -584,5 +586,9 @@ module Ci
Ci::MaskSecret.mask!(trace, token) Ci::MaskSecret.mask!(trace, token)
trace trace
end end
def update_project_statistics
ProjectCacheWorker.perform_async(project_id, [], [:build_artifacts_size])
end
end end
end end
...@@ -48,8 +48,14 @@ class Group < Namespace ...@@ -48,8 +48,14 @@ class Group < Namespace
end end
def sort(method) def sort(method)
if method == 'storage_size_desc'
# storage_size is a virtual column so we need to
# pass a string to avoid AR adding the table name
reorder('storage_size DESC, namespaces.id DESC')
else
order_by(method) order_by(method)
end end
end
def reference_prefix def reference_prefix
User.reference_prefix User.reference_prefix
......
...@@ -5,4 +5,13 @@ class LfsObjectsProject < ActiveRecord::Base ...@@ -5,4 +5,13 @@ class LfsObjectsProject < ActiveRecord::Base
validates :lfs_object_id, presence: true validates :lfs_object_id, presence: true
validates :lfs_object_id, uniqueness: { scope: [:project_id], message: "already exists in project" } validates :lfs_object_id, uniqueness: { scope: [:project_id], message: "already exists in project" }
validates :project_id, presence: true validates :project_id, presence: true
after_create :update_project_statistics
after_destroy :update_project_statistics
private
def update_project_statistics
ProjectCacheWorker.perform_async(project_id, [], [:lfs_objects_size])
end
end end
...@@ -9,6 +9,7 @@ class Namespace < ActiveRecord::Base ...@@ -9,6 +9,7 @@ class Namespace < ActiveRecord::Base
cache_markdown_field :description, pipeline: :description cache_markdown_field :description, pipeline: :description
has_many :projects, dependent: :destroy has_many :projects, dependent: :destroy
has_many :project_statistics
belongs_to :owner, class_name: "User" belongs_to :owner, class_name: "User"
belongs_to :parent, class_name: "Namespace" belongs_to :parent, class_name: "Namespace"
...@@ -38,6 +39,18 @@ class Namespace < ActiveRecord::Base ...@@ -38,6 +39,18 @@ class Namespace < ActiveRecord::Base
scope :root, -> { where('type IS NULL') } scope :root, -> { where('type IS NULL') }
scope :with_statistics, -> do
joins('LEFT JOIN project_statistics ps ON ps.namespace_id = namespaces.id')
.group('namespaces.id')
.select(
'namespaces.*',
'COALESCE(SUM(ps.storage_size), 0) AS storage_size',
'COALESCE(SUM(ps.repository_size), 0) AS repository_size',
'COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size',
'COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size',
)
end
class << self class << self
def by_path(path) def by_path(path)
find_by('lower(path) = :value', value: path.downcase) find_by('lower(path) = :value', value: path.downcase)
......
...@@ -44,6 +44,7 @@ class Project < ActiveRecord::Base ...@@ -44,6 +44,7 @@ class Project < ActiveRecord::Base
after_create :ensure_dir_exist after_create :ensure_dir_exist
after_create :create_project_feature, unless: :project_feature after_create :create_project_feature, unless: :project_feature
after_save :ensure_dir_exist, if: :namespace_id_changed? after_save :ensure_dir_exist, if: :namespace_id_changed?
after_save :update_project_statistics, if: :namespace_id_changed?
# set last_activity_at to the same as created_at # set last_activity_at to the same as created_at
after_create :set_last_activity_at after_create :set_last_activity_at
...@@ -151,6 +152,7 @@ class Project < ActiveRecord::Base ...@@ -151,6 +152,7 @@ class Project < ActiveRecord::Base
has_one :import_data, dependent: :destroy, class_name: "ProjectImportData" has_one :import_data, dependent: :destroy, class_name: "ProjectImportData"
has_one :project_feature, dependent: :destroy has_one :project_feature, dependent: :destroy
has_one :statistics, class_name: 'ProjectStatistics', dependent: :delete
has_many :commit_statuses, dependent: :destroy, foreign_key: :gl_project_id has_many :commit_statuses, dependent: :destroy, foreign_key: :gl_project_id
has_many :pipelines, dependent: :destroy, class_name: 'Ci::Pipeline', foreign_key: :gl_project_id has_many :pipelines, dependent: :destroy, class_name: 'Ci::Pipeline', foreign_key: :gl_project_id
...@@ -220,6 +222,7 @@ class Project < ActiveRecord::Base ...@@ -220,6 +222,7 @@ class Project < ActiveRecord::Base
scope :with_push, -> { joins(:events).where('events.action = ?', Event::PUSHED) } scope :with_push, -> { joins(:events).where('events.action = ?', Event::PUSHED) }
scope :with_project_feature, -> { joins('LEFT JOIN project_features ON projects.id = project_features.project_id') } scope :with_project_feature, -> { joins('LEFT JOIN project_features ON projects.id = project_features.project_id') }
scope :with_statistics, -> { includes(:statistics) }
# "enabled" here means "not disabled". It includes private features! # "enabled" here means "not disabled". It includes private features!
scope :with_feature_enabled, ->(feature) { scope :with_feature_enabled, ->(feature) {
...@@ -332,8 +335,10 @@ class Project < ActiveRecord::Base ...@@ -332,8 +335,10 @@ class Project < ActiveRecord::Base
end end
def sort(method) def sort(method)
if method == 'repository_size_desc' if method == 'storage_size_desc'
reorder(repository_size: :desc, id: :desc) # storage_size is a joined column so we need to
# pass a string to avoid AR adding the table name
reorder('project_statistics.storage_size DESC, projects.id DESC')
else else
order_by(method) order_by(method)
end end
...@@ -1036,14 +1041,6 @@ class Project < ActiveRecord::Base ...@@ -1036,14 +1041,6 @@ class Project < ActiveRecord::Base
forked? && project == forked_from_project forked? && project == forked_from_project
end end
def update_repository_size
update_attribute(:repository_size, repository.size)
end
def update_commit_count
update_attribute(:commit_count, repository.commit_count)
end
def forks_count def forks_count
forks.count forks.count
end end
...@@ -1322,4 +1319,9 @@ class Project < ActiveRecord::Base ...@@ -1322,4 +1319,9 @@ class Project < ActiveRecord::Base
def full_path_changed? def full_path_changed?
path_changed? || namespace_id_changed? path_changed? || namespace_id_changed?
end end
def update_project_statistics
stats = statistics || build_statistics
stats.update(namespace_id: namespace_id)
end
end end
class ProjectStatistics < ActiveRecord::Base
belongs_to :project
belongs_to :namespace
before_save :update_storage_size
STORAGE_COLUMNS = [:repository_size, :lfs_objects_size, :build_artifacts_size]
STATISTICS_COLUMNS = [:commit_count] + STORAGE_COLUMNS
def total_repository_size
repository_size + lfs_objects_size
end
def refresh!(only: nil)
STATISTICS_COLUMNS.each do |column, generator|
if only.blank? || only.include?(column)
public_send("update_#{column}")
end
end
save!
end
def update_commit_count
self.commit_count = project.repository.commit_count
end
def update_repository_size
self.repository_size = project.repository.size
end
def update_lfs_objects_size
self.lfs_objects_size = project.lfs_objects.sum(:size)
end
def update_build_artifacts_size
self.build_artifacts_size = project.builds.sum(:artifacts_size)
end
def update_storage_size
self.storage_size = STORAGE_COLUMNS.sum(&method(:read_attribute))
end
end
...@@ -74,7 +74,7 @@ class GitPushService < BaseService ...@@ -74,7 +74,7 @@ class GitPushService < BaseService
types = [] types = []
end end
ProjectCacheWorker.perform_async(@project.id, types) ProjectCacheWorker.perform_async(@project.id, types, [:commit_count, :repository_size])
end end
protected protected
......
...@@ -12,7 +12,7 @@ class GitTagPushService < BaseService ...@@ -12,7 +12,7 @@ class GitTagPushService < BaseService
project.execute_hooks(@push_data.dup, :tag_push_hooks) project.execute_hooks(@push_data.dup, :tag_push_hooks)
project.execute_services(@push_data.dup, :tag_push_hooks) project.execute_services(@push_data.dup, :tag_push_hooks)
Ci::CreatePipelineService.new(project, current_user, @push_data).execute Ci::CreatePipelineService.new(project, current_user, @push_data).execute
ProjectCacheWorker.perform_async(project.id) ProjectCacheWorker.perform_async(project.id, [], [:commit_count, :repository_size])
true true
end end
......
...@@ -5,6 +5,9 @@ ...@@ -5,6 +5,9 @@
= link_to 'Edit', admin_group_edit_path(group), id: "edit_#{dom_id(group)}", class: 'btn' = link_to 'Edit', admin_group_edit_path(group), id: "edit_#{dom_id(group)}", class: 'btn'
= link_to 'Delete', [:admin, group], data: { confirm: "Are you sure you want to remove #{group.name}?" }, method: :delete, class: 'btn btn-remove' = link_to 'Delete', [:admin, group], data: { confirm: "Are you sure you want to remove #{group.name}?" }, method: :delete, class: 'btn btn-remove'
.stats .stats
%span.badge
= storage_counter(group.storage_size)
%span %span
= icon('bookmark') = icon('bookmark')
= number_with_delimiter(group.projects.count) = number_with_delimiter(group.projects.count)
......
...@@ -27,6 +27,8 @@ ...@@ -27,6 +27,8 @@
= sort_title_recently_updated = sort_title_recently_updated
= link_to admin_groups_path(sort: sort_value_oldest_updated, name: project_name) do = link_to admin_groups_path(sort: sort_value_oldest_updated, name: project_name) do
= sort_title_oldest_updated = sort_title_oldest_updated
= link_to admin_groups_path(sort: sort_value_largest_group, name: project_name) do
= sort_title_largest_group
= link_to new_admin_group_path, class: "btn btn-new" do = link_to new_admin_group_path, class: "btn btn-new" do
New Group New Group
%ul.content-list %ul.content-list
......
...@@ -38,6 +38,18 @@ ...@@ -38,6 +38,18 @@
%strong %strong
= @group.created_at.to_s(:medium) = @group.created_at.to_s(:medium)
%li
%span.light Storage:
%strong= storage_counter(@group.storage_size)
(
= storage_counter(@group.repository_size)
repositories,
= storage_counter(@group.build_artifacts_size)
build artifacts,
= storage_counter(@group.lfs_objects_size)
LFS
)
%li %li
%span.light Group Git LFS status: %span.light Group Git LFS status:
%strong %strong
...@@ -55,8 +67,8 @@ ...@@ -55,8 +67,8 @@
%li %li
%strong %strong
= link_to project.name_with_namespace, [:admin, project.namespace.becomes(Namespace), project] = link_to project.name_with_namespace, [:admin, project.namespace.becomes(Namespace), project]
%span.label.label-gray %span.badge
= repository_size(project) = storage_counter(project.statistics.storage_size)
%span.pull-right.light %span.pull-right.light
%span.monospace= project.path_with_namespace + ".git" %span.monospace= project.path_with_namespace + ".git"
.panel-footer .panel-footer
...@@ -73,8 +85,8 @@ ...@@ -73,8 +85,8 @@
%li %li
%strong %strong
= link_to project.name_with_namespace, [:admin, project.namespace.becomes(Namespace), project] = link_to project.name_with_namespace, [:admin, project.namespace.becomes(Namespace), project]
%span.label.label-gray %span.badge
= repository_size(project) = storage_counter(project.statistics.storage_size)
%span.pull-right.light %span.pull-right.light
%span.monospace= project.path_with_namespace + ".git" %span.monospace= project.path_with_namespace + ".git"
......
...@@ -69,8 +69,8 @@ ...@@ -69,8 +69,8 @@
.controls .controls
- if project.archived - if project.archived
%span.label.label-warning archived %span.label.label-warning archived
%span.label.label-gray %span.badge
= repository_size(project) = storage_counter(project.statistics.storage_size)
= link_to 'Edit', edit_namespace_project_path(project.namespace, project), id: "edit_#{dom_id(project)}", class: "btn" = link_to 'Edit', edit_namespace_project_path(project.namespace, project), id: "edit_#{dom_id(project)}", class: "btn"
= link_to 'Delete', [project.namespace.becomes(Namespace), project], data: { confirm: remove_project_message(project) }, method: :delete, class: "btn btn-remove" = link_to 'Delete', [project.namespace.becomes(Namespace), project], data: { confirm: remove_project_message(project) }, method: :delete, class: "btn btn-remove"
.title .title
......
...@@ -65,9 +65,16 @@ ...@@ -65,9 +65,16 @@
= @project.repository.path_to_repo = @project.repository.path_to_repo
%li %li
%span.light Size %span.light Storage:
%strong %strong= storage_counter(@project.statistics.storage_size)
= repository_size(@project) (
= storage_counter(@project.statistics.repository_size)
repository,
= storage_counter(@project.statistics.build_artifacts_size)
build artifacts,
= storage_counter(@project.statistics.lfs_objects_size)
LFS
)
%li %li
%span.light last commit: %span.light last commit:
......
...@@ -18,8 +18,8 @@ ...@@ -18,8 +18,8 @@
.pull-right .pull-right
- if project.archived - if project.archived
%span.label.label-warning archived %span.label.label-warning archived
%span.label.label-gray %span.badge
= repository_size(project) = storage_counter(project.statistics.storage_size)
= link_to 'Members', namespace_project_project_members_path(project.namespace, project), id: "edit_#{dom_id(project)}", class: "btn btn-sm" = link_to 'Members', namespace_project_project_members_path(project.namespace, project), id: "edit_#{dom_id(project)}", class: "btn btn-sm"
= link_to 'Edit', edit_namespace_project_path(project.namespace, project), id: "edit_#{dom_id(project)}", class: "btn btn-sm" = link_to 'Edit', edit_namespace_project_path(project.namespace, project), id: "edit_#{dom_id(project)}", class: "btn btn-sm"
= link_to 'Remove', project, data: { confirm: remove_project_message(project)}, method: :delete, class: "btn btn-sm btn-remove" = link_to 'Remove', project, data: { confirm: remove_project_message(project)}, method: :delete, class: "btn btn-sm btn-remove"
......
...@@ -17,10 +17,10 @@ ...@@ -17,10 +17,10 @@
%ul.nav %ul.nav
%li %li
= link_to project_files_path(@project) do = link_to project_files_path(@project) do
Files (#{repository_size}) Files (#{storage_counter(@project.statistics.total_repository_size)})
%li %li
= link_to namespace_project_commits_path(@project.namespace, @project, current_ref) do = link_to namespace_project_commits_path(@project.namespace, @project, current_ref) do
#{'Commit'.pluralize(@project.commit_count)} (#{number_with_delimiter(@project.commit_count)}) #{'Commit'.pluralize(@project.statistics.commit_count)} (#{number_with_delimiter(@project.statistics.commit_count)})
%li %li
= link_to namespace_project_branches_path(@project.namespace, @project) do = link_to namespace_project_branches_path(@project.namespace, @project) do
#{'Branch'.pluralize(@repository.branch_count)} (#{number_with_delimiter(@repository.branch_count)}) #{'Branch'.pluralize(@repository.branch_count)} (#{number_with_delimiter(@repository.branch_count)})
......
...@@ -6,26 +6,27 @@ class ProjectCacheWorker ...@@ -6,26 +6,27 @@ class ProjectCacheWorker
LEASE_TIMEOUT = 15.minutes.to_i LEASE_TIMEOUT = 15.minutes.to_i
# project_id - The ID of the project for which to flush the cache. # project_id - The ID of the project for which to flush the cache.
# refresh - An Array containing extra types of data to refresh such as # files - An Array containing extra types of files to refresh such as
# `:readme` to flush the README and `:changelog` to flush the # `:readme` to flush the README and `:changelog` to flush the
# CHANGELOG. # CHANGELOG.
def perform(project_id, refresh = []) # statistics - An Array containing columns from ProjectStatistics to
# refresh, if empty all columns will be refreshed
def perform(project_id, files = [], statistics = [])
project = Project.find_by(id: project_id) project = Project.find_by(id: project_id)
return unless project && project.repository.exists? return unless project && project.repository.exists?
update_repository_size(project) update_statistics(project, statistics.map(&:to_sym))
project.update_commit_count
project.repository.refresh_method_caches(refresh.map(&:to_sym)) project.repository.refresh_method_caches(files.map(&:to_sym))
end end
def update_repository_size(project) def update_statistics(project, statistics = [])
return unless try_obtain_lease_for(project.id, :update_repository_size) return unless try_obtain_lease_for(project.id, :update_statistics)
Rails.logger.info("Updating repository size for project #{project.id}") Rails.logger.info("Updating statistics for project #{project.id}")
project.update_repository_size project.statistics.refresh!(only: statistics)
end end
private private
......
---
title: Add more storage statistics
merge_request: 7754
author: Markus Koller
...@@ -10,5 +10,5 @@ ...@@ -10,5 +10,5 @@
# end # end
# #
ActiveSupport::Inflector.inflections do |inflect| ActiveSupport::Inflector.inflections do |inflect|
inflect.uncountable %w(award_emoji) inflect.uncountable %w(award_emoji project_statistics)
end end
class CreateProjectStatistics < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
# use bigint columns to support values >2GB
counter_column = { limit: 8, null: false, default: 0 }
create_table :project_statistics do |t|
t.references :project, null: false, index: { unique: true }, foreign_key: { on_delete: :cascade }
t.references :namespace, null: false, index: true
t.integer :commit_count, counter_column
t.integer :storage_size, counter_column
t.integer :repository_size, counter_column
t.integer :lfs_objects_size, counter_column
t.integer :build_artifacts_size, counter_column
end
end
end
class MigrateProjectStatistics < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = true
DOWNTIME_REASON = 'Removes two columns from the projects table'
def up
# convert repository_size in float (megabytes) to integer (bytes),
# initialize total storage_size with repository_size
execute <<-EOF
INSERT INTO project_statistics (project_id, namespace_id, commit_count, storage_size, repository_size)
SELECT id, namespace_id, commit_count, (repository_size * 1024 * 1024), (repository_size * 1024 * 1024) FROM projects
EOF
remove_column :projects, :repository_size
remove_column :projects, :commit_count
end
def down
add_column_with_default :projects, :repository_size, :float, default: 0.0
add_column_with_default :projects, :commit_count, :integer, default: 0
end
end
...@@ -901,6 +901,19 @@ ActiveRecord::Schema.define(version: 20161220141214) do ...@@ -901,6 +901,19 @@ ActiveRecord::Schema.define(version: 20161220141214) do
add_index "project_import_data", ["project_id"], name: "index_project_import_data_on_project_id", using: :btree add_index "project_import_data", ["project_id"], name: "index_project_import_data_on_project_id", using: :btree
create_table "project_statistics", force: :cascade do |t|
t.integer "project_id", null: false
t.integer "namespace_id", null: false
t.integer "commit_count", limit: 8, default: 0, null: false
t.integer "storage_size", limit: 8, default: 0, null: false
t.integer "repository_size", limit: 8, default: 0, null: false
t.integer "lfs_objects_size", limit: 8, default: 0, null: false
t.integer "build_artifacts_size", limit: 8, default: 0, null: false
end
add_index "project_statistics", ["namespace_id"], name: "index_project_statistics_on_namespace_id", using: :btree
add_index "project_statistics", ["project_id"], name: "index_project_statistics_on_project_id", unique: true, using: :btree
create_table "projects", force: :cascade do |t| create_table "projects", force: :cascade do |t|
t.string "name" t.string "name"
t.string "path" t.string "path"
...@@ -915,11 +928,9 @@ ActiveRecord::Schema.define(version: 20161220141214) do ...@@ -915,11 +928,9 @@ ActiveRecord::Schema.define(version: 20161220141214) do
t.boolean "archived", default: false, null: false t.boolean "archived", default: false, null: false
t.string "avatar" t.string "avatar"
t.string "import_status" t.string "import_status"
t.float "repository_size", default: 0.0
t.integer "star_count", default: 0, null: false t.integer "star_count", default: 0, null: false
t.string "import_type" t.string "import_type"
t.string "import_source" t.string "import_source"
t.integer "commit_count", default: 0
t.text "import_error" t.text "import_error"
t.integer "ci_id" t.integer "ci_id"
t.boolean "shared_runners_enabled", default: true, null: false t.boolean "shared_runners_enabled", default: true, null: false
...@@ -1288,6 +1299,7 @@ ActiveRecord::Schema.define(version: 20161220141214) do ...@@ -1288,6 +1299,7 @@ ActiveRecord::Schema.define(version: 20161220141214) do
add_foreign_key "personal_access_tokens", "users" add_foreign_key "personal_access_tokens", "users"
add_foreign_key "project_authorizations", "projects", on_delete: :cascade add_foreign_key "project_authorizations", "projects", on_delete: :cascade
add_foreign_key "project_authorizations", "users", on_delete: :cascade add_foreign_key "project_authorizations", "users", on_delete: :cascade
add_foreign_key "project_statistics", "projects", on_delete: :cascade
add_foreign_key "protected_branch_merge_access_levels", "protected_branches" add_foreign_key "protected_branch_merge_access_levels", "protected_branches"
add_foreign_key "protected_branch_push_access_levels", "protected_branches" add_foreign_key "protected_branch_push_access_levels", "protected_branches"
add_foreign_key "subscriptions", "projects", on_delete: :cascade add_foreign_key "subscriptions", "projects", on_delete: :cascade
......
...@@ -88,3 +88,9 @@ artifacts through the [Admin area settings](../user/admin_area/settings/continuo ...@@ -88,3 +88,9 @@ artifacts through the [Admin area settings](../user/admin_area/settings/continuo
[reconfigure gitlab]: restart_gitlab.md "How to restart GitLab" [reconfigure gitlab]: restart_gitlab.md "How to restart GitLab"
[restart gitlab]: restart_gitlab.md "How to restart GitLab" [restart gitlab]: restart_gitlab.md "How to restart GitLab"
## Storage statistics
You can see the total storage used for build artifacts on groups and projects
in the administration area, as well as through the [groups](../api/groups.md)
and [projects APIs](../api/projects.md).
...@@ -13,6 +13,7 @@ Parameters: ...@@ -13,6 +13,7 @@ Parameters:
| `search` | string | no | Return list of authorized groups matching the search criteria | | `search` | string | no | Return list of authorized groups matching the search criteria |
| `order_by` | string | no | Order groups by `name` or `path`. Default is `name` | | `order_by` | string | no | Order groups by `name` or `path`. Default is `name` |
| `sort` | string | no | Order groups in `asc` or `desc` order. Default is `asc` | | `sort` | string | no | Order groups in `asc` or `desc` order. Default is `asc` |
| `statistics` | boolean | no | Include group statistics (admins only) |
``` ```
GET /groups GET /groups
...@@ -31,7 +32,6 @@ GET /groups ...@@ -31,7 +32,6 @@ GET /groups
You can search for groups by name or path, see below. You can search for groups by name or path, see below.
=======
## List owned groups ## List owned groups
Get a list of groups which are owned by the authenticated user. Get a list of groups which are owned by the authenticated user.
...@@ -40,6 +40,12 @@ Get a list of groups which are owned by the authenticated user. ...@@ -40,6 +40,12 @@ Get a list of groups which are owned by the authenticated user.
GET /groups/owned GET /groups/owned
``` ```
Parameters:
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `statistics` | boolean | no | Include group statistics |
## List a group's projects ## List a group's projects
Get a list of projects in this group. Get a list of projects in this group.
......
...@@ -307,6 +307,8 @@ Parameters: ...@@ -307,6 +307,8 @@ Parameters:
| `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` | | `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` |
| `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` | | `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` |
| `search` | string | no | Return list of authorized projects matching the search criteria | | `search` | string | no | Return list of authorized projects matching the search criteria |
| `simple` | boolean | no | Return only the ID, URL, name, and path of each project |
| `statistics` | boolean | no | Include project statistics |
### List starred projects ### List starred projects
...@@ -325,6 +327,7 @@ Parameters: ...@@ -325,6 +327,7 @@ Parameters:
| `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` | | `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` |
| `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` | | `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` |
| `search` | string | no | Return list of authorized projects matching the search criteria | | `search` | string | no | Return list of authorized projects matching the search criteria |
| `simple` | boolean | no | Return only the ID, URL, name, and path of each project |
### List ALL projects ### List ALL projects
...@@ -343,6 +346,7 @@ Parameters: ...@@ -343,6 +346,7 @@ Parameters:
| `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` | | `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` |
| `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` | | `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` |
| `search` | string | no | Return list of authorized projects matching the search criteria | | `search` | string | no | Return list of authorized projects matching the search criteria |
| `statistics` | boolean | no | Include project statistics |
### Get single project ### Get single project
......
...@@ -40,6 +40,12 @@ In `config/gitlab.yml`: ...@@ -40,6 +40,12 @@ In `config/gitlab.yml`:
storage_path: /mnt/storage/lfs-objects storage_path: /mnt/storage/lfs-objects
``` ```
## Storage statistics
You can see the total storage used for LFS objects on groups and projects
in the administration area, as well as through the [groups](../api/groups.md)
and [projects APIs](../api/projects.md).
## Known limitations ## Known limitations
* Currently, storing GitLab Git LFS objects on a non-local storage (like S3 buckets) * Currently, storing GitLab Git LFS objects on a non-local storage (like S3 buckets)
...@@ -47,3 +53,5 @@ In `config/gitlab.yml`: ...@@ -47,3 +53,5 @@ In `config/gitlab.yml`:
* Currently, removing LFS objects from GitLab Git LFS storage is not supported * Currently, removing LFS objects from GitLab Git LFS storage is not supported
* LFS authentications via SSH was added with GitLab 8.12 * LFS authentications via SSH was added with GitLab 8.12
* Only compatible with the GitLFS client versions 1.1.0 and up, or 1.0.2. * Only compatible with the GitLFS client versions 1.1.0 and up, or 1.0.2.
* The storage statistics currently count each LFS object multiple times for
every project linking to it
...@@ -101,6 +101,16 @@ module API ...@@ -101,6 +101,16 @@ module API
expose :only_allow_merge_if_build_succeeds expose :only_allow_merge_if_build_succeeds
expose :request_access_enabled expose :request_access_enabled
expose :only_allow_merge_if_all_discussions_are_resolved expose :only_allow_merge_if_all_discussions_are_resolved
expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics
end
class ProjectStatistics < Grape::Entity
expose :commit_count
expose :storage_size
expose :repository_size
expose :lfs_objects_size
expose :build_artifacts_size
end end
class Member < UserBasic class Member < UserBasic
...@@ -127,6 +137,15 @@ module API ...@@ -127,6 +137,15 @@ module API
expose :avatar_url expose :avatar_url
expose :web_url expose :web_url
expose :request_access_enabled expose :request_access_enabled
expose :statistics, if: :statistics do
with_options format_with: -> (value) { value.to_i } do
expose :storage_size
expose :repository_size
expose :lfs_objects_size
expose :build_artifacts_size
end
end
end end
class GroupDetail < Group class GroupDetail < Group
......
...@@ -11,6 +11,20 @@ module API ...@@ -11,6 +11,20 @@ module API
optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group'
optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access'
end end
params :statistics_params do
optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
end
def present_groups(groups, options = {})
options = options.reverse_merge(
with: Entities::Group,
current_user: current_user,
)
groups = groups.with_statistics if options[:statistics]
present paginate(groups), options
end
end end
resource :groups do resource :groups do
...@@ -18,6 +32,7 @@ module API ...@@ -18,6 +32,7 @@ module API
success Entities::Group success Entities::Group
end end
params do params do
use :statistics_params
optional :skip_groups, type: Array[Integer], desc: 'Array of group ids to exclude from list' optional :skip_groups, type: Array[Integer], desc: 'Array of group ids to exclude from list'
optional :all_available, type: Boolean, desc: 'Show all group that you have access to' optional :all_available, type: Boolean, desc: 'Show all group that you have access to'
optional :search, type: String, desc: 'Search for a specific group' optional :search, type: String, desc: 'Search for a specific group'
...@@ -38,7 +53,7 @@ module API ...@@ -38,7 +53,7 @@ module API
groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present?
groups = groups.reorder(params[:order_by] => params[:sort]) groups = groups.reorder(params[:order_by] => params[:sort])
present paginate(groups), with: Entities::Group, current_user: current_user present_groups groups, statistics: params[:statistics] && current_user.is_admin?
end end
desc 'Get list of owned groups for authenticated user' do desc 'Get list of owned groups for authenticated user' do
...@@ -46,10 +61,10 @@ module API ...@@ -46,10 +61,10 @@ module API
end end
params do params do
use :pagination use :pagination
use :statistics_params
end end
get '/owned' do get '/owned' do
groups = current_user.owned_groups present_groups current_user.owned_groups, statistics: params[:statistics]
present paginate(groups), with: Entities::Group, current_user: current_user
end end
desc 'Create a group. Available only for users who can create groups.' do desc 'Create a group. Available only for users who can create groups.' do
......
...@@ -248,7 +248,7 @@ module API ...@@ -248,7 +248,7 @@ module API
rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500) rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500)
end end
# Projects helpers # project helpers
def filter_projects(projects) def filter_projects(projects)
if params[:search].present? if params[:search].present?
......
...@@ -40,6 +40,15 @@ module API ...@@ -40,6 +40,15 @@ module API
resource :projects do resource :projects do
helpers do helpers do
params :collection_params do
use :sort_params
use :filter_params
use :pagination
optional :simple, type: Boolean, default: false,
desc: 'Return only the ID, URL, name, and path of each project'
end
params :sort_params do params :sort_params do
optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at], optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at],
default: 'created_at', desc: 'Return projects ordered by field' default: 'created_at', desc: 'Return projects ordered by field'
...@@ -52,97 +61,94 @@ module API ...@@ -52,97 +61,94 @@ module API
optional :visibility, type: String, values: %w[public internal private], optional :visibility, type: String, values: %w[public internal private],
desc: 'Limit by visibility' desc: 'Limit by visibility'
optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria' optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria'
use :sort_params end
params :statistics_params do
optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
end end
params :create_params do params :create_params do
optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.' optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.'
optional :import_url, type: String, desc: 'URL from which the project is imported' optional :import_url, type: String, desc: 'URL from which the project is imported'
end end
def present_projects(projects, options = {})
options = options.reverse_merge(
with: Entities::Project,
current_user: current_user,
simple: params[:simple],
)
projects = filter_projects(projects)
projects = projects.with_statistics if options[:statistics]
options[:with] = Entities::BasicProjectDetails if options[:simple]
present paginate(projects), options
end
end end
desc 'Get a list of visible projects for authenticated user' do desc 'Get a list of visible projects for authenticated user' do
success Entities::BasicProjectDetails success Entities::BasicProjectDetails
end end
params do params do
optional :simple, type: Boolean, default: false, use :collection_params
desc: 'Return only the ID, URL, name, and path of each project'
use :filter_params
use :pagination
end end
get '/visible' do get '/visible' do
projects = ProjectsFinder.new.execute(current_user) entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails
projects = filter_projects(projects) present_projects ProjectsFinder.new.execute(current_user), with: entity
entity = params[:simple] || !current_user ? Entities::BasicProjectDetails : Entities::ProjectWithAccess
present paginate(projects), with: entity, current_user: current_user
end end
desc 'Get a projects list for authenticated user' do desc 'Get a projects list for authenticated user' do
success Entities::BasicProjectDetails success Entities::BasicProjectDetails
end end
params do params do
optional :simple, type: Boolean, default: false, use :collection_params
desc: 'Return only the ID, URL, name, and path of each project'
use :filter_params
use :pagination
end end
get do get do
authenticate! authenticate!
projects = current_user.authorized_projects present_projects current_user.authorized_projects,
projects = filter_projects(projects) with: Entities::ProjectWithAccess
entity = params[:simple] ? Entities::BasicProjectDetails : Entities::ProjectWithAccess
present paginate(projects), with: entity, current_user: current_user
end end
desc 'Get an owned projects list for authenticated user' do desc 'Get an owned projects list for authenticated user' do
success Entities::BasicProjectDetails success Entities::BasicProjectDetails
end end
params do params do
use :filter_params use :collection_params
use :pagination use :statistics_params
end end
get '/owned' do get '/owned' do
authenticate! authenticate!
projects = current_user.owned_projects present_projects current_user.owned_projects,
projects = filter_projects(projects) with: Entities::ProjectWithAccess,
statistics: params[:statistics]
present paginate(projects), with: Entities::ProjectWithAccess, current_user: current_user
end end
desc 'Gets starred project for the authenticated user' do desc 'Gets starred project for the authenticated user' do
success Entities::BasicProjectDetails success Entities::BasicProjectDetails
end end
params do params do
use :filter_params use :collection_params
use :pagination
end end
get '/starred' do get '/starred' do
authenticate! authenticate!
projects = current_user.viewable_starred_projects present_projects current_user.viewable_starred_projects
projects = filter_projects(projects)
present paginate(projects), with: Entities::Project, current_user: current_user
end end
desc 'Get all projects for admin user' do desc 'Get all projects for admin user' do
success Entities::BasicProjectDetails success Entities::BasicProjectDetails
end end
params do params do
use :filter_params use :collection_params
use :pagination use :statistics_params
end end
get '/all' do get '/all' do
authenticated_as_admin! authenticated_as_admin!
projects = Project.all present_projects Project.all, with: Entities::ProjectWithAccess, statistics: params[:statistics]
projects = filter_projects(projects)
present paginate(projects), with: Entities::ProjectWithAccess, current_user: current_user
end end
desc 'Search for projects the current user has access to' do desc 'Search for projects the current user has access to' do
......
...@@ -63,8 +63,7 @@ namespace :gitlab do ...@@ -63,8 +63,7 @@ namespace :gitlab do
if project.persisted? if project.persisted?
puts " * Created #{project.name} (#{repo_path})".color(:green) puts " * Created #{project.name} (#{repo_path})".color(:green)
project.update_repository_size ProjectCacheWorker.perform(project.id)
project.update_commit_count
else else
puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red)
puts " Errors: #{project.errors.messages}".color(:red) puts " Errors: #{project.errors.messages}".color(:red)
......
...@@ -2,7 +2,7 @@ include ActionDispatch::TestProcess ...@@ -2,7 +2,7 @@ include ActionDispatch::TestProcess
FactoryGirl.define do FactoryGirl.define do
factory :lfs_object do factory :lfs_object do
oid "b68143e6463773b1b6c6fd009a76c32aeec041faff32ba2ed42fd7f708a17f80" sequence(:oid) { |n| "b68143e6463773b1b6c6fd009a76c32aeec041faff32ba2ed42fd7f708a%05x" % n }
size 499013 size 499013
end end
......
FactoryGirl.define do
factory :project_statistics do
project { create :project }
namespace { project.namespace }
end
end
require 'spec_helper'
describe StorageHelper do
describe '#storage_counter' do
it 'formats bytes to one decimal place' do
expect(helper.storage_counter(1.23.megabytes)).to eq '1.2 MB'
end
it 'does not add decimals for sizes < 1 MB' do
expect(helper.storage_counter(23.5.kilobytes)).to eq '24 KB'
end
it 'does not add decimals for zeroes' do
expect(helper.storage_counter(2.megabytes)).to eq '2 MB'
end
it 'uses commas as thousands separator' do
expect(helper.storage_counter(100_000_000_000_000_000)).to eq '90,949.5 TB'
end
end
end
...@@ -192,6 +192,7 @@ project: ...@@ -192,6 +192,7 @@ project:
- authorized_users - authorized_users
- project_authorizations - project_authorizations
- route - route
- statistics
award_emoji: award_emoji:
- awardable - awardable
- user - user
......
...@@ -85,4 +85,30 @@ describe Ci::Build, models: true do ...@@ -85,4 +85,30 @@ describe Ci::Build, models: true do
it { expect(build.trace_file_path).to eq(build.old_path_to_trace) } it { expect(build.trace_file_path).to eq(build.old_path_to_trace) }
end end
end end
describe '#update_project_statistics' do
let!(:build) { create(:ci_build, artifacts_size: 23) }
it 'updates project statistics when the artifact size changes' do
expect(ProjectCacheWorker).to receive(:perform_async)
.with(build.project_id, [], [:build_artifacts_size])
build.artifacts_size = 42
build.save!
end
it 'does not update project statistics when the artifact size stays the same' do
expect(ProjectCacheWorker).not_to receive(:perform_async)
build.name = 'changed'
build.save!
end
it 'updates project statistics when the build is destroyed' do
expect(ProjectCacheWorker).to receive(:perform_async)
.with(build.project_id, [], [:build_artifacts_size])
build.destroy
end
end
end end
require 'spec_helper'
describe LfsObjectsProject, models: true do
subject { create(:lfs_objects_project, project: project) }
let(:project) { create(:empty_project) }
describe 'associations' do
it { is_expected.to belong_to(:project) }
it { is_expected.to belong_to(:lfs_object) }
end
describe 'validation' do
it { is_expected.to validate_presence_of(:lfs_object_id) }
it { is_expected.to validate_uniqueness_of(:lfs_object_id).scoped_to(:project_id).with_message("already exists in project") }
it { is_expected.to validate_presence_of(:project_id) }
end
describe '#update_project_statistics' do
it 'updates project statistics when the object is added' do
expect(ProjectCacheWorker).to receive(:perform_async)
.with(project.id, [], [:lfs_objects_size])
subject.save!
end
it 'updates project statistics when the object is removed' do
subject.save!
expect(ProjectCacheWorker).to receive(:perform_async)
.with(project.id, [], [:lfs_objects_size])
subject.destroy
end
end
end
...@@ -4,6 +4,7 @@ describe Namespace, models: true do ...@@ -4,6 +4,7 @@ describe Namespace, models: true do
let!(:namespace) { create(:namespace) } let!(:namespace) { create(:namespace) }
it { is_expected.to have_many :projects } it { is_expected.to have_many :projects }
it { is_expected.to have_many :project_statistics }
it { is_expected.to validate_presence_of(:name) } it { is_expected.to validate_presence_of(:name) }
it { is_expected.to validate_uniqueness_of(:name).scoped_to(:parent_id) } it { is_expected.to validate_uniqueness_of(:name).scoped_to(:parent_id) }
...@@ -57,6 +58,50 @@ describe Namespace, models: true do ...@@ -57,6 +58,50 @@ describe Namespace, models: true do
end end
end end
describe '.with_statistics' do
let(:namespace) { create :namespace }
let(:project1) do
create(:empty_project,
namespace: namespace,
statistics: build(:project_statistics,
storage_size: 606,
repository_size: 101,
lfs_objects_size: 202,
build_artifacts_size: 303))
end
let(:project2) do
create(:empty_project,
namespace: namespace,
statistics: build(:project_statistics,
storage_size: 60,
repository_size: 10,
lfs_objects_size: 20,
build_artifacts_size: 30))
end
it "sums all project storage counters in the namespace" do
project1
project2
statistics = Namespace.with_statistics.find(namespace.id)
expect(statistics.storage_size).to eq 666
expect(statistics.repository_size).to eq 111
expect(statistics.lfs_objects_size).to eq 222
expect(statistics.build_artifacts_size).to eq 333
end
it "correctly handles namespaces without projects" do
statistics = Namespace.with_statistics.find(namespace.id)
expect(statistics.storage_size).to eq 0
expect(statistics.repository_size).to eq 0
expect(statistics.lfs_objects_size).to eq 0
expect(statistics.build_artifacts_size).to eq 0
end
end
describe '#move_dir' do describe '#move_dir' do
before do before do
@namespace = create :namespace @namespace = create :namespace
......
...@@ -49,6 +49,7 @@ describe Project, models: true do ...@@ -49,6 +49,7 @@ describe Project, models: true do
it { is_expected.to have_one(:gitlab_issue_tracker_service).dependent(:destroy) } it { is_expected.to have_one(:gitlab_issue_tracker_service).dependent(:destroy) }
it { is_expected.to have_one(:external_wiki_service).dependent(:destroy) } it { is_expected.to have_one(:external_wiki_service).dependent(:destroy) }
it { is_expected.to have_one(:project_feature).dependent(:destroy) } it { is_expected.to have_one(:project_feature).dependent(:destroy) }
it { is_expected.to have_one(:statistics).class_name('ProjectStatistics').dependent(:delete) }
it { is_expected.to have_one(:import_data).class_name('ProjectImportData').dependent(:destroy) } it { is_expected.to have_one(:import_data).class_name('ProjectImportData').dependent(:destroy) }
it { is_expected.to have_one(:last_event).class_name('Event') } it { is_expected.to have_one(:last_event).class_name('Event') }
it { is_expected.to have_one(:forked_from_project).through(:forked_project_link) } it { is_expected.to have_one(:forked_from_project).through(:forked_project_link) }
...@@ -1729,6 +1730,26 @@ describe Project, models: true do ...@@ -1729,6 +1730,26 @@ describe Project, models: true do
end end
end end
describe '#update_project_statistics' do
let(:project) { create(:empty_project) }
it "is called after creation" do
expect(project.statistics).to be_a ProjectStatistics
expect(project.statistics).to be_persisted
end
it "copies the namespace_id" do
expect(project.statistics.namespace_id).to eq project.namespace_id
end
it "updates the namespace_id when changed" do
namespace = create(:namespace)
project.update(namespace: namespace)
expect(project.statistics.namespace_id).to eq namespace.id
end
end
def enable_lfs def enable_lfs
allow(Gitlab.config.lfs).to receive(:enabled).and_return(true) allow(Gitlab.config.lfs).to receive(:enabled).and_return(true)
end end
......
require 'rails_helper'
describe ProjectStatistics, models: true do
let(:project) { create :empty_project }
let(:statistics) { project.statistics }
describe 'constants' do
describe 'STORAGE_COLUMNS' do
it 'is an array of symbols' do
expect(described_class::STORAGE_COLUMNS).to be_kind_of Array
expect(described_class::STORAGE_COLUMNS.map(&:class).uniq).to eq [Symbol]
end
end
describe 'STATISTICS_COLUMNS' do
it 'is an array of symbols' do
expect(described_class::STATISTICS_COLUMNS).to be_kind_of Array
expect(described_class::STATISTICS_COLUMNS.map(&:class).uniq).to eq [Symbol]
end
it 'includes all storage columns' do
expect(described_class::STATISTICS_COLUMNS & described_class::STORAGE_COLUMNS).to eq described_class::STORAGE_COLUMNS
end
end
end
describe 'associations' do
it { is_expected.to belong_to(:project) }
it { is_expected.to belong_to(:namespace) }
end
describe 'statistics columns' do
it "support values up to 8 exabytes" do
statistics.update!(
commit_count: 8.exabytes - 1,
repository_size: 2.exabytes,
lfs_objects_size: 2.exabytes,
build_artifacts_size: 4.exabytes - 1,
)
statistics.reload
expect(statistics.commit_count).to eq(8.exabytes - 1)
expect(statistics.repository_size).to eq(2.exabytes)
expect(statistics.lfs_objects_size).to eq(2.exabytes)
expect(statistics.build_artifacts_size).to eq(4.exabytes - 1)
expect(statistics.storage_size).to eq(8.exabytes - 1)
end
end
describe '#total_repository_size' do
it "sums repository and LFS object size" do
statistics.repository_size = 2
statistics.lfs_objects_size = 3
statistics.build_artifacts_size = 4
expect(statistics.total_repository_size).to eq 5
end
end
describe '#refresh!' do
before do
allow(statistics).to receive(:update_commit_count)
allow(statistics).to receive(:update_repository_size)
allow(statistics).to receive(:update_lfs_objects_size)
allow(statistics).to receive(:update_build_artifacts_size)
allow(statistics).to receive(:update_storage_size)
end
context "without arguments" do
before do
statistics.refresh!
end
it "sums all counters" do
expect(statistics).to have_received(:update_commit_count)
expect(statistics).to have_received(:update_repository_size)
expect(statistics).to have_received(:update_lfs_objects_size)
expect(statistics).to have_received(:update_build_artifacts_size)
end
end
context "when passing an only: argument" do
before do
statistics.refresh! only: [:lfs_objects_size]
end
it "only updates the given columns" do
expect(statistics).to have_received(:update_lfs_objects_size)
expect(statistics).not_to have_received(:update_commit_count)
expect(statistics).not_to have_received(:update_repository_size)
expect(statistics).not_to have_received(:update_build_artifacts_size)
end
end
end
describe '#update_commit_count' do
before do
allow(project.repository).to receive(:commit_count).and_return(23)
statistics.update_commit_count
end
it "stores the number of commits in the repository" do
expect(statistics.commit_count).to eq 23
end
end
describe '#update_repository_size' do
before do
allow(project.repository).to receive(:size).and_return(12.megabytes)
statistics.update_repository_size
end
it "stores the size of the repository" do
expect(statistics.repository_size).to eq 12.megabytes
end
end
describe '#update_lfs_objects_size' do
let!(:lfs_object1) { create(:lfs_object, size: 23.megabytes) }
let!(:lfs_object2) { create(:lfs_object, size: 34.megabytes) }
let!(:lfs_objects_project1) { create(:lfs_objects_project, project: project, lfs_object: lfs_object1) }
let!(:lfs_objects_project2) { create(:lfs_objects_project, project: project, lfs_object: lfs_object2) }
before do
statistics.update_lfs_objects_size
end
it "stores the size of related LFS objects" do
expect(statistics.lfs_objects_size).to eq 57.megabytes
end
end
describe '#update_build_artifacts_size' do
let!(:pipeline) { create(:ci_pipeline, project: project) }
let!(:build1) { create(:ci_build, pipeline: pipeline, artifacts_size: 45.megabytes) }
let!(:build2) { create(:ci_build, pipeline: pipeline, artifacts_size: 56.megabytes) }
before do
statistics.update_build_artifacts_size
end
it "stores the size of related build artifacts" do
expect(statistics.build_artifacts_size).to eq 101.megabytes
end
end
describe '#update_storage_size' do
it "sums all storage counters" do
statistics.update!(
repository_size: 2,
lfs_objects_size: 3,
)
statistics.reload
expect(statistics.storage_size).to eq 5
end
end
end
...@@ -35,6 +35,14 @@ describe API::Groups, api: true do ...@@ -35,6 +35,14 @@ describe API::Groups, api: true do
expect(json_response.length).to eq(1) expect(json_response.length).to eq(1)
expect(json_response.first['name']).to eq(group1.name) expect(json_response.first['name']).to eq(group1.name)
end end
it "does not include statistics" do
get api("/groups", user1), statistics: true
expect(response).to have_http_status(200)
expect(json_response).to be_an Array
expect(json_response.first).not_to include 'statistics'
end
end end
context "when authenticated as admin" do context "when authenticated as admin" do
...@@ -44,6 +52,31 @@ describe API::Groups, api: true do ...@@ -44,6 +52,31 @@ describe API::Groups, api: true do
expect(json_response).to be_an Array expect(json_response).to be_an Array
expect(json_response.length).to eq(2) expect(json_response.length).to eq(2)
end end
it "does not include statistics by default" do
get api("/groups", admin)
expect(response).to have_http_status(200)
expect(json_response).to be_an Array
expect(json_response.first).not_to include('statistics')
end
it "includes statistics if requested" do
attributes = {
storage_size: 702,
repository_size: 123,
lfs_objects_size: 234,
build_artifacts_size: 345,
}
project1.statistics.update!(attributes)
get api("/groups", admin), statistics: true
expect(response).to have_http_status(200)
expect(json_response).to be_an Array
expect(json_response.first['statistics']).to eq attributes.stringify_keys
end
end end
context "when using skip_groups in request" do context "when using skip_groups in request" do
......
...@@ -49,7 +49,7 @@ describe API::Projects, api: true do ...@@ -49,7 +49,7 @@ describe API::Projects, api: true do
end end
end end
context 'when authenticated' do context 'when authenticated as regular user' do
it 'returns an array of projects' do it 'returns an array of projects' do
get api('/projects', user) get api('/projects', user)
expect(response).to have_http_status(200) expect(response).to have_http_status(200)
...@@ -172,6 +172,22 @@ describe API::Projects, api: true do ...@@ -172,6 +172,22 @@ describe API::Projects, api: true do
end end
end end
end end
it "does not include statistics by default" do
get api('/projects/all', admin)
expect(response).to have_http_status(200)
expect(json_response).to be_an Array
expect(json_response.first).not_to include('statistics')
end
it "includes statistics if requested" do
get api('/projects/all', admin), statistics: true
expect(response).to have_http_status(200)
expect(json_response).to be_an Array
expect(json_response.first).to include 'statistics'
end
end end
end end
...@@ -196,6 +212,32 @@ describe API::Projects, api: true do ...@@ -196,6 +212,32 @@ describe API::Projects, api: true do
expect(json_response.first['name']).to eq(project4.name) expect(json_response.first['name']).to eq(project4.name)
expect(json_response.first['owner']['username']).to eq(user4.username) expect(json_response.first['owner']['username']).to eq(user4.username)
end end
it "does not include statistics by default" do
get api('/projects/owned', user4)
expect(response).to have_http_status(200)
expect(json_response).to be_an Array
expect(json_response.first).not_to include('statistics')
end
it "includes statistics if requested" do
attributes = {
commit_count: 23,
storage_size: 702,
repository_size: 123,
lfs_objects_size: 234,
build_artifacts_size: 345,
}
project4.statistics.update!(attributes)
get api('/projects/owned', user4), statistics: true
expect(response).to have_http_status(200)
expect(json_response).to be_an Array
expect(json_response.first['statistics']).to eq attributes.stringify_keys
end
end end
end end
......
...@@ -583,7 +583,7 @@ describe GitPushService, services: true do ...@@ -583,7 +583,7 @@ describe GitPushService, services: true do
service.push_commits = [commit] service.push_commits = [commit]
expect(ProjectCacheWorker).to receive(:perform_async). expect(ProjectCacheWorker).to receive(:perform_async).
with(project.id, %i(readme)) with(project.id, %i(readme), %i(commit_count repository_size))
service.update_caches service.update_caches
end end
...@@ -596,7 +596,7 @@ describe GitPushService, services: true do ...@@ -596,7 +596,7 @@ describe GitPushService, services: true do
it 'does not flush any conditional caches' do it 'does not flush any conditional caches' do
expect(ProjectCacheWorker).to receive(:perform_async). expect(ProjectCacheWorker).to receive(:perform_async).
with(project.id, []). with(project.id, [], %i(commit_count repository_size)).
and_call_original and_call_original
service.update_caches service.update_caches
......
require 'spec_helper' require 'spec_helper'
describe ProjectCacheWorker do describe ProjectCacheWorker do
let(:project) { create(:project) }
let(:worker) { described_class.new } let(:worker) { described_class.new }
let(:project) { create(:project) }
let(:statistics) { project.statistics }
describe '#perform' do describe '#perform' do
before do before do
...@@ -12,7 +13,7 @@ describe ProjectCacheWorker do ...@@ -12,7 +13,7 @@ describe ProjectCacheWorker do
context 'with a non-existing project' do context 'with a non-existing project' do
it 'does nothing' do it 'does nothing' do
expect(worker).not_to receive(:update_repository_size) expect(worker).not_to receive(:update_statistics)
worker.perform(-1) worker.perform(-1)
end end
...@@ -22,24 +23,19 @@ describe ProjectCacheWorker do ...@@ -22,24 +23,19 @@ describe ProjectCacheWorker do
it 'does nothing' do it 'does nothing' do
allow_any_instance_of(Repository).to receive(:exists?).and_return(false) allow_any_instance_of(Repository).to receive(:exists?).and_return(false)
expect(worker).not_to receive(:update_repository_size) expect(worker).not_to receive(:update_statistics)
worker.perform(project.id) worker.perform(project.id)
end end
end end
context 'with an existing project' do context 'with an existing project' do
it 'updates the repository size' do it 'updates the project statistics' do
expect(worker).to receive(:update_repository_size).and_call_original expect(worker).to receive(:update_statistics)
.with(kind_of(Project), %i(repository_size))
worker.perform(project.id) .and_call_original
end
it 'updates the commit count' do
expect_any_instance_of(Project).to receive(:update_commit_count).
and_call_original
worker.perform(project.id) worker.perform(project.id, [], %w(repository_size))
end end
it 'refreshes the method caches' do it 'refreshes the method caches' do
...@@ -47,33 +43,35 @@ describe ProjectCacheWorker do ...@@ -47,33 +43,35 @@ describe ProjectCacheWorker do
with(%i(readme)). with(%i(readme)).
and_call_original and_call_original
worker.perform(project.id, %i(readme)) worker.perform(project.id, %w(readme))
end end
end end
end end
describe '#update_repository_size' do describe '#update_statistics' do
context 'when a lease could not be obtained' do context 'when a lease could not be obtained' do
it 'does not update the repository size' do it 'does not update the repository size' do
allow(worker).to receive(:try_obtain_lease_for). allow(worker).to receive(:try_obtain_lease_for).
with(project.id, :update_repository_size). with(project.id, :update_statistics).
and_return(false) and_return(false)
expect(project).not_to receive(:update_repository_size) expect(statistics).not_to receive(:refresh!)
worker.update_repository_size(project) worker.update_statistics(project)
end end
end end
context 'when a lease could be obtained' do context 'when a lease could be obtained' do
it 'updates the repository size' do it 'updates the project statistics' do
allow(worker).to receive(:try_obtain_lease_for). allow(worker).to receive(:try_obtain_lease_for).
with(project.id, :update_repository_size). with(project.id, :update_statistics).
and_return(true) and_return(true)
expect(project).to receive(:update_repository_size).and_call_original expect(statistics).to receive(:refresh!)
.with(only: %i(repository_size))
.and_call_original
worker.update_repository_size(project) worker.update_statistics(project, %i(repository_size))
end end
end end
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment