Merge branch 'fj-223712-include-snippets-size-in-project-statistics' into 'master'

Include snippets size in project statistics

See merge request gitlab-org/gitlab!35120

app/models/namespace.rb

@@ -79,6 +79,7 @@ class Namespace < ApplicationRecord
         'COALESCE(SUM(ps.storage_size), 0) AS storage_size',
         'COALESCE(SUM(ps.repository_size), 0) AS repository_size',
         'COALESCE(SUM(ps.wiki_size), 0) AS wiki_size',
+        'COALESCE(SUM(ps.snippets_size), 0) AS snippets_size',
         'COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size',
         'COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size',
         'COALESCE(SUM(ps.packages_size), 0) AS packages_size'

app/models/project_statistics.rb

@@ -7,15 +7,11 @@ class ProjectStatistics < ApplicationRecord
   belongs_to :namespace
 
   default_value_for :wiki_size, 0
-
-  # older migrations fail due to non-existent attribute without this
-  def wiki_size
-    has_attribute?(:wiki_size) ? super : 0
-  end
+  default_value_for :snippets_size, 0
 
   before_save :update_storage_size
 
-  COLUMNS_TO_REFRESH = [:repository_size, :wiki_size, :lfs_objects_size, :commit_count].freeze
+  COLUMNS_TO_REFRESH = [:repository_size, :wiki_size, :lfs_objects_size, :commit_count, :snippets_size].freeze
   INCREMENTABLE_COLUMNS = { build_artifacts_size: %i[storage_size], packages_size: %i[storage_size] }.freeze
   NAMESPACE_RELATABLE_COLUMNS = [:repository_size, :wiki_size, :lfs_objects_size].freeze
 
@@ -54,17 +50,32 @@ class ProjectStatistics < ApplicationRecord
     self.wiki_size = project.wiki.repository.size * 1.megabyte
   end
 
+  def update_snippets_size
+    self.snippets_size = project.snippets.with_statistics.sum(:repository_size)
+  end
+
   def update_lfs_objects_size
     self.lfs_objects_size = project.lfs_objects.sum(:size)
   end
 
-  # older migrations fail due to non-existent attribute without this
-  def packages_size
-    has_attribute?(:packages_size) ? super : 0
+  # `wiki_size` and `snippets_size` have no default value in the database
+  # and the column can be nil.
+  # This means that, when the columns were added, all rows had nil
+  # values on them.
+  # Therefore, any call to any of those methods will return nil instead
+  # of 0, because `default_value_for` works with new records, not existing ones.
+  #
+  # These two methods provide consistency and avoid returning nil.
+  def wiki_size
+    super.to_i
+  end
+
+  def snippets_size
+    super.to_i
   end
 
   def update_storage_size
-    self.storage_size = repository_size + wiki_size.to_i + lfs_objects_size + build_artifacts_size + packages_size
+    self.storage_size = repository_size + wiki_size + lfs_objects_size + build_artifacts_size + packages_size + snippets_size
   end
 
   # Since this incremental update method does not call update_storage_size above,

app/models/snippet.rb

@@ -79,6 +79,7 @@ class Snippet < ApplicationRecord
   scope :fresh, -> { order("created_at DESC") }
   scope :inc_author, -> { includes(:author) }
   scope :inc_relations_for_view, -> { includes(author: :status) }
+  scope :with_statistics, -> { joins(:statistics) }
 
   attr_mentionable :description
 

changelogs/unreleased/fj-223712-include-snippets-size-in-project-statistics.yml

+---
+title: Include snippets size in project statistics
+merge_request: 35120
+author:
+type: changed

spec/factories/project_statistics.rb

@@ -21,6 +21,7 @@ FactoryBot.define do
         project_statistics.lfs_objects_size = evaluator.size_multiplier * 3
         project_statistics.build_artifacts_size = evaluator.size_multiplier * 4
         project_statistics.packages_size = evaluator.size_multiplier * 5
+        project_statistics.snippets_size = evaluator.size_multiplier * 6
       end
     end
   end

spec/models/namespace_spec.rb

@@ -153,7 +153,8 @@ RSpec.describe Namespace do
                                wiki_size:            505,
                                lfs_objects_size:     202,
                                build_artifacts_size: 303,
-                               packages_size:        404))
+                               packages_size:        404,
+                               snippets_size:        605))
     end
 
     let(:project2) do
@@ -164,7 +165,8 @@ RSpec.describe Namespace do
                                wiki_size:            50,
                                lfs_objects_size:     20,
                                build_artifacts_size: 30,
-                               packages_size:        40))
+                               packages_size:        40,
+                               snippets_size:        60))
     end
 
     it "sums all project storage counters in the namespace" do
@@ -172,12 +174,13 @@ RSpec.describe Namespace do
       project2
       statistics = described_class.with_statistics.find(namespace.id)
 
-      expect(statistics.storage_size).to eq 1665
+      expect(statistics.storage_size).to eq 2330
       expect(statistics.repository_size).to eq 111
       expect(statistics.wiki_size).to eq 555
       expect(statistics.lfs_objects_size).to eq 222
       expect(statistics.build_artifacts_size).to eq 333
       expect(statistics.packages_size).to eq 444
+      expect(statistics.snippets_size).to eq 665
     end
 
     it "correctly handles namespaces without projects" do
@@ -189,6 +192,7 @@ RSpec.describe Namespace do
       expect(statistics.lfs_objects_size).to eq 0
       expect(statistics.build_artifacts_size).to eq 0
       expect(statistics.packages_size).to eq 0
+      expect(statistics.snippets_size).to eq 0
     end
   end
 

spec/models/project_statistics_spec.rb

@@ -32,7 +32,8 @@ RSpec.describe ProjectStatistics do
         repository_size: 2.exabytes,
         wiki_size: 1.exabytes,
         lfs_objects_size: 2.exabytes,
-        build_artifacts_size: 3.exabytes - 1
+        build_artifacts_size: 2.exabytes - 1,
+        snippets_size: 1.exabyte
       )
 
       statistics.reload
@@ -41,8 +42,9 @@ RSpec.describe ProjectStatistics do
       expect(statistics.repository_size).to eq(2.exabytes)
       expect(statistics.wiki_size).to eq(1.exabytes)
       expect(statistics.lfs_objects_size).to eq(2.exabytes)
-      expect(statistics.build_artifacts_size).to eq(3.exabytes - 1)
+      expect(statistics.build_artifacts_size).to eq(2.exabytes - 1)
       expect(statistics.storage_size).to eq(8.exabytes - 1)
+      expect(statistics.snippets_size).to eq(1.exabyte)
     end
   end
 
@@ -52,23 +54,47 @@ RSpec.describe ProjectStatistics do
       statistics.wiki_size = 6
       statistics.lfs_objects_size = 3
       statistics.build_artifacts_size = 4
+      statistics.snippets_size = 5
 
       expect(statistics.total_repository_size).to eq 5
     end
   end
 
   describe '#wiki_size' do
-    it "is initialized with not null value" do
+    it 'is initialized with not null value' do
+      expect(statistics.attributes['wiki_size']).to be_zero
+      expect(statistics.wiki_size).to be_zero
+    end
+
+    it 'coerces any nil value to 0' do
+      statistics.update!(wiki_size: nil)
+
+      expect(statistics.attributes['wiki_size']).to be_nil
       expect(statistics.wiki_size).to eq 0
     end
   end
 
+  describe '#snippets_size' do
+    it 'is initialized with not null value' do
+      expect(statistics.attributes['snippets_size']).to be_zero
+      expect(statistics.snippets_size).to be_zero
+    end
+
+    it 'coerces any nil value to 0' do
+      statistics.update!(snippets_size: nil)
+
+      expect(statistics.attributes['snippets_size']).to be_nil
+      expect(statistics.snippets_size).to eq 0
+    end
+  end
+
   describe '#refresh!' do
     before do
       allow(statistics).to receive(:update_commit_count)
       allow(statistics).to receive(:update_repository_size)
       allow(statistics).to receive(:update_wiki_size)
       allow(statistics).to receive(:update_lfs_objects_size)
+      allow(statistics).to receive(:update_snippets_size)
       allow(statistics).to receive(:update_storage_size)
     end
 
@@ -82,6 +108,7 @@ RSpec.describe ProjectStatistics do
         expect(statistics).to have_received(:update_repository_size)
         expect(statistics).to have_received(:update_wiki_size)
         expect(statistics).to have_received(:update_lfs_objects_size)
+        expect(statistics).to have_received(:update_snippets_size)
       end
     end
 
@@ -95,6 +122,7 @@ RSpec.describe ProjectStatistics do
         expect(statistics).not_to have_received(:update_commit_count)
         expect(statistics).not_to have_received(:update_repository_size)
         expect(statistics).not_to have_received(:update_wiki_size)
+        expect(statistics).not_to have_received(:update_snippets_size)
       end
     end
 
@@ -108,9 +136,11 @@ RSpec.describe ProjectStatistics do
         expect(statistics).to have_received(:update_commit_count)
         expect(statistics).to have_received(:update_repository_size)
         expect(statistics).to have_received(:update_wiki_size)
+        expect(statistics).to have_received(:update_snippets_size)
         expect(statistics.repository_size).to eq(0)
         expect(statistics.commit_count).to eq(0)
         expect(statistics.wiki_size).to eq(0)
+        expect(statistics.snippets_size).to eq(0)
       end
     end
 
@@ -130,9 +160,11 @@ RSpec.describe ProjectStatistics do
         expect(statistics).to have_received(:update_commit_count)
         expect(statistics).to have_received(:update_repository_size)
         expect(statistics).to have_received(:update_wiki_size)
+        expect(statistics).to have_received(:update_snippets_size)
         expect(statistics.repository_size).to eq(0)
         expect(statistics.commit_count).to eq(0)
         expect(statistics.wiki_size).to eq(0)
+        expect(statistics.snippets_size).to eq(0)
       end
     end
 
@@ -202,6 +234,33 @@ RSpec.describe ProjectStatistics do
     end
   end
 
+  describe '#update_snippets_size' do
+    before do
+      create_list(:project_snippet, 2, project: project)
+      SnippetStatistics.update_all(repository_size: 10)
+    end
+
+    it 'stores the size of snippets' do
+      # Snippet not associated with the project
+      snippet = create(:project_snippet)
+      snippet.statistics.update!(repository_size: 40)
+
+      statistics.update_snippets_size
+
+      expect(statistics.update_snippets_size).to eq 20
+    end
+
+    context 'when not all snippets has statistics' do
+      it 'stores the size of snippets with statistics' do
+        SnippetStatistics.last.delete
+
+        statistics.update_snippets_size
+
+        expect(statistics.update_snippets_size).to eq 10
+      end
+    end
+  end
+
   describe '#update_lfs_objects_size' do
     let!(:lfs_object1) { create(:lfs_object, size: 23.megabytes) }
     let!(:lfs_object2) { create(:lfs_object, size: 34.megabytes) }
@@ -222,12 +281,13 @@ RSpec.describe ProjectStatistics do
       statistics.update!(
         repository_size: 2,
         wiki_size: 4,
-        lfs_objects_size: 3
+        lfs_objects_size: 3,
+        snippets_size: 2
       )
 
       statistics.reload
 
-      expect(statistics.storage_size).to eq 9
+      expect(statistics.storage_size).to eq 11
     end
 
     it 'works during wiki_size backfill' do
@@ -241,6 +301,21 @@ RSpec.describe ProjectStatistics do
 
       expect(statistics.storage_size).to eq 5
     end
+
+    context 'when nullable columns are nil' do
+      it 'does not raise any error' do
+        expect do
+          statistics.update!(
+            repository_size: 2,
+            wiki_size: nil,
+            lfs_objects_size: 3,
+            snippets_size: nil
+          )
+        end.not_to raise_error
+
+        expect(statistics.storage_size).to eq 5
+      end
+    end
   end
 
   describe '.increment_statistic' do