Add job artifacts to /admin/geo_nodes

db/migrate/20180105233807_add_job_artifact_counts_to_geo_node_statuses.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddJobArtifactCountsToGeoNodeStatuses < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :geo_node_statuses, :job_artifacts_count, :integer
+    add_column :geo_node_statuses, :job_artifacts_synced_count, :integer
+    add_column :geo_node_statuses, :job_artifacts_failed_count, :integer
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180104001824) do
+ActiveRecord::Schema.define(version: 20180105233807) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -994,6 +994,9 @@ ActiveRecord::Schema.define(version: 20180104001824) do
     t.integer "wikis_count"
     t.integer "wikis_synced_count"
     t.integer "wikis_failed_count"
+    t.integer "job_artifacts_count"
+    t.integer "job_artifacts_synced_count"
+    t.integer "job_artifacts_failed_count"
   end
 
   add_index "geo_node_statuses", ["geo_node_id"], name: "index_geo_node_statuses_on_geo_node_id", unique: true, using: :btree

doc/api/geo_nodes.md

@@ -94,6 +94,10 @@ Example response:
     "lfs_objects_synced_count": 0,
     "lfs_objects_failed_count": 0,
     "lfs_objects_synced_in_percentage": "0.00%",
+    "job_artifacts_count": 2,
+    "job_artifacts_synced_count": 1,
+    "job_artifacts_failed_count": 1,
+    "job_artifacts_synced_in_percentage": "50.00%",
     "repositories_count": 41,
     "repositories_failed_count": 1,
     "repositories_synced_count": 40,
@@ -141,6 +145,10 @@ Example response:
   "lfs_objects_synced_count": 0,
   "lfs_objects_failed_count": 0,
   "lfs_objects_synced_in_percentage": "0.00%",
+  "job_artifacts_count": 2,
+  "job_artifacts_synced_count": 1,
+  "job_artifacts_failed_count": 1,
+  "job_artifacts_synced_in_percentage": "50.00%",
   "repositories_count": 41,
   "repositories_failed_count": 1,
   "repositories_synced_count": 40,

ee/app/assets/javascripts/geo_nodes/components/geo_node_details.vue

@@ -56,6 +56,11 @@
             itemValue: this.nodeDetails.lfs,
             itemValueType: VALUE_TYPE.GRAPH,
           },
+          {
+            itemTitle: s__('GeoNodes|Job artifacts:'),
+            itemValue: this.nodeDetails.jobArtifacts,
+            itemValueType: VALUE_TYPE.GRAPH,
+          },
           {
             itemTitle: s__('GeoNodes|Attachments:'),
             itemValue: this.nodeDetails.attachments,

ee/app/assets/javascripts/geo_nodes/store/geo_nodes_store.js

@@ -60,8 +60,13 @@ export default class GeoNodesStore {
       },
       lfs: {
         totalCount: rawNodeDetails.lfs_objects_count,
-        successCount: rawNodeDetails.lfs_objects_failed_count,
-        failureCount: rawNodeDetails.lfs_objects_synced_count,
+        successCount: rawNodeDetails.lfs_objects_synced_count,
+        failureCount: rawNodeDetails.lfs_objects_failed_count,
+      },
+      jobArtifacts: {
+        totalCount: rawNodeDetails.job_artifacts_count,
+        successCount: rawNodeDetails.job_artifacts_synced_count,
+        failureCount: rawNodeDetails.job_artifacts_failed_count,
       },
       attachments: {
         totalCount: rawNodeDetails.attachments_count,

ee/app/models/geo_node_status.rb

@@ -18,6 +18,9 @@ class GeoNodeStatus < ActiveRecord::Base
     lfs_objects_count: 'Total number of LFS objects available on primary',
     lfs_objects_synced_count: 'Number of LFS objects synced on secondary',
     lfs_objects_failed_count: 'Number of LFS objects failed to sync on secondary',
+    job_artifacts_count: 'Total number of job artifacts available on primary',
+    job_artifacts_synced_count: 'Number of job artifacts synced on secondary',
+    job_artifacts_failed_count: 'Number of job artifacts failed to sync on secondary',
     attachments_count: 'Total number of file attachments available on primary',
     attachments_synced_count: 'Number of attachments synced on secondary',
     attachments_failed_count: 'Number of attachments failed to sync on secondary',
@@ -73,16 +76,26 @@ class GeoNodeStatus < ActiveRecord::Base
     self.repositories_count = projects_finder.count_repositories
     self.wikis_count = projects_finder.count_wikis
     self.lfs_objects_count = lfs_objects_finder.count_lfs_objects
+    self.job_artifacts_count = job_artifacts_finder.count_job_artifacts
     self.attachments_count = attachments_finder.count_attachments
     self.last_successful_status_check_at = Time.now
     self.storage_shards = StorageShard.all
 
+    load_primary_data
+    load_secondary_data
+
+    self
+  end
+
+  def load_primary_data
     if Gitlab::Geo.primary?
       self.replication_slots_count = geo_node.replication_slots_count
       self.replication_slots_used_count = geo_node.replication_slots_used_count
       self.replication_slots_max_retained_wal_bytes = geo_node.replication_slots_max_retained_wal_bytes
     end
+  end
 
+  def load_secondary_data
     if Gitlab::Geo.secondary?
       self.db_replication_lag_seconds = Gitlab::Geo::HealthCheck.db_replication_lag_seconds
       self.cursor_last_event_id = Geo::EventLogState.last_processed&.event_id
@@ -93,11 +106,11 @@ class GeoNodeStatus < ActiveRecord::Base
       self.wikis_failed_count = projects_finder.count_failed_wikis
       self.lfs_objects_synced_count = lfs_objects_finder.count_synced_lfs_objects
       self.lfs_objects_failed_count = lfs_objects_finder.count_failed_lfs_objects
+      self.job_artifacts_synced_count = job_artifacts_finder.count_synced_job_artifacts
+      self.job_artifacts_failed_count = job_artifacts_finder.count_failed_job_artifacts
       self.attachments_synced_count = attachments_finder.count_synced_attachments
       self.attachments_failed_count = attachments_finder.count_failed_attachments
     end
-
-    self
   end
 
   alias_attribute :health, :status_message
@@ -146,6 +159,10 @@ class GeoNodeStatus < ActiveRecord::Base
     calc_percentage(lfs_objects_count, lfs_objects_synced_count)
   end
 
+  def job_artifacts_synced_in_percentage
+    calc_percentage(job_artifacts_count, job_artifacts_synced_count)
+  end
+
   def attachments_synced_in_percentage
     calc_percentage(attachments_count, attachments_synced_count)
   end
@@ -196,6 +213,10 @@ class GeoNodeStatus < ActiveRecord::Base
     @lfs_objects_finder ||= Geo::LfsObjectRegistryFinder.new(current_node: geo_node)
   end
 
+  def job_artifacts_finder
+    @job_artifacts_finder ||= Geo::JobArtifactRegistryFinder.new(current_node: geo_node)
+  end
+
   def projects_finder
     @projects_finder ||= Geo::ProjectRegistryFinder.new(current_node: geo_node)
   end

ee/app/serializers/geo_node_status_entity.rb

@@ -26,6 +26,13 @@ class GeoNodeStatusEntity < Grape::Entity
     number_to_percentage(node.lfs_objects_synced_in_percentage, precision: 2)
   end
 
+  expose :job_artifacts_count
+  expose :job_artifacts_synced_count
+  expose :job_artifacts_failed_count
+  expose :job_artifacts_synced_in_percentage do |node|
+    number_to_percentage(node.job_artifacts_synced_in_percentage, precision: 2)
+  end
+
   expose :repositories_count
   expose :repositories_failed_count
   expose :repositories_synced_count

spec/ee/spec/models/geo_node_status_spec.rb

@@ -198,6 +198,61 @@ describe GeoNodeStatus, :geo do
     end
   end
 
+  describe '#job_artifacts_synced_count' do
+    it 'counts synced job artifacts' do
+      # These should be ignored
+      create(:geo_file_registry, success: false)
+      create(:geo_file_registry, :avatar, success: false)
+      create(:geo_file_registry, file_type: :attachment, success: true)
+      create(:geo_file_registry, :lfs, :with_file, success: true)
+      create(:geo_file_registry, :job_artifact, :with_file, success: false)
+
+      create(:geo_file_registry, :job_artifact, :with_file, success: true)
+
+      expect(subject.job_artifacts_synced_count).to eq(1)
+    end
+  end
+
+  describe '#job_artifacts_failed_count' do
+    it 'counts failed job artifacts' do
+      # These should be ignored
+      create(:geo_file_registry, success: false)
+      create(:geo_file_registry, :avatar, success: false)
+      create(:geo_file_registry, file_type: :attachment, success: false)
+      create(:geo_file_registry, :job_artifact, :with_file, success: true)
+
+      create(:geo_file_registry, :job_artifact, :with_file, success: false)
+
+      expect(subject.job_artifacts_failed_count).to eq(1)
+    end
+  end
+
+  describe '#job_artifacts_synced_in_percentage' do
+    context 'when artifacts are available' do
+      before do
+        [project_1, project_2, project_3, project_4].each_with_index do |project, index|
+          build = create(:ci_build, project: project)
+          job_artifact = create(:ci_job_artifact, job: build)
+          create(:geo_file_registry, :job_artifact, success: index.even?, file_id: job_artifact.id)
+        end
+      end
+
+      it 'returns the right percentage with no group restrictions' do
+        expect(subject.job_artifacts_synced_in_percentage).to be_within(0.0001).of(50)
+      end
+
+      it 'returns the right percentage with group restrictions' do
+        secondary.update_attribute(:namespaces, [group])
+
+        expect(subject.job_artifacts_synced_in_percentage).to be_within(0.0001).of(50)
+      end
+    end
+
+    it 'returns 0 when no artifacts are available' do
+      expect(subject.job_artifacts_synced_in_percentage).to eq(0)
+    end
+  end
+
   describe '#repositories_failed_count' do
     before do
       create(:geo_project_registry, :sync_failed, project: project_1)

spec/ee/spec/serializers/geo_node_status_entity_spec.rb

@@ -22,6 +22,10 @@ describe GeoNodeStatusEntity, :postgresql do
   it { is_expected.to have_key(:lfs_objects_failed_count) }
   it { is_expected.to have_key(:lfs_objects_synced_count) }
   it { is_expected.to have_key(:lfs_objects_synced_in_percentage) }
+  it { is_expected.to have_key(:job_artifacts_count) }
+  it { is_expected.to have_key(:job_artifacts_failed_count) }
+  it { is_expected.to have_key(:job_artifacts_synced_count) }
+  it { is_expected.to have_key(:job_artifacts_synced_in_percentage) }
   it { is_expected.to have_key(:repositories_count) }
   it { is_expected.to have_key(:repositories_failed_count) }
   it { is_expected.to have_key(:repositories_synced_count)}
@@ -99,6 +103,16 @@ describe GeoNodeStatusEntity, :postgresql do
     end
   end
 
+  describe '#job_artifacts_synced_in_percentage' do
+    it 'formats as percentage' do
+      geo_node_status.assign_attributes(job_artifacts_count: 256,
+                                        job_artifacts_failed_count: 12,
+                                        job_artifacts_synced_count: 123)
+
+      expect(subject[:job_artifacts_synced_in_percentage]).to eq '48.05%'
+    end
+  end
+
   describe '#repositories_synced_in_percentage' do
     it 'formats as percentage' do
       geo_node_status.assign_attributes(repositories_count: 10,

spec/ee/spec/services/geo/metrics_update_service_spec.rb

@@ -25,6 +25,9 @@ describe Geo::MetricsUpdateService, :geo do
       lfs_objects_count: 100,
       lfs_objects_synced_count: 50,
       lfs_objects_failed_count: 12,
+      job_artifacts_count: 100,
+      job_artifacts_synced_count: 50,
+      job_artifacts_failed_count: 12,
       attachments_count: 30,
       attachments_synced_count: 30,
       attachments_failed_count: 25,
@@ -110,6 +113,9 @@ describe Geo::MetricsUpdateService, :geo do
         expect(metric_value(:geo_lfs_objects)).to eq(100)
         expect(metric_value(:geo_lfs_objects_synced)).to eq(50)
         expect(metric_value(:geo_lfs_objects_failed)).to eq(12)
+        expect(metric_value(:geo_job_artifacts)).to eq(100)
+        expect(metric_value(:geo_job_artifacts_synced)).to eq(50)
+        expect(metric_value(:geo_job_artifacts_failed)).to eq(12)
         expect(metric_value(:geo_attachments)).to eq(30)
         expect(metric_value(:geo_attachments_synced)).to eq(30)
         expect(metric_value(:geo_attachments_failed)).to eq(25)

spec/ee/spec/services/geo/node_status_fetch_service_spec.rb

@@ -60,6 +60,9 @@ describe Geo::NodeStatusFetchService, :geo do
                lfs_objects_count: 100,
                lfs_objects_synced_count: 50,
                lfs_objects_failed_count: 12,
+               job_artifacts_count: 100,
+               job_artifacts_synced_count: 50,
+               job_artifacts_failed_count: 12,
                attachments_count: 30,
                attachments_synced_count: 30,
                attachments_failed_count: 25,
@@ -136,6 +139,9 @@ describe Geo::NodeStatusFetchService, :geo do
       expect(status.lfs_objects_count).to eq(db_status.lfs_objects_count)
       expect(status.lfs_objects_failed_count).to eq(db_status.lfs_objects_failed_count)
       expect(status.lfs_objects_synced_count).to eq(db_status.lfs_objects_synced_count)
+      expect(status.job_artifacts_count).to eq(db_status.job_artifacts_count)
+      expect(status.job_artifacts_failed_count).to eq(db_status.job_artifacts_failed_count)
+      expect(status.job_artifacts_synced_count).to eq(db_status.job_artifacts_synced_count)
       expect(status.repositories_count).to eq(db_status.repositories_count)
       expect(status.repositories_synced_count).to eq(db_status.repositories_synced_count)
       expect(status.repositories_failed_count).to eq(db_status.repositories_failed_count)

spec/factories/geo/file_registry.rb

@@ -21,6 +21,8 @@ FactoryBot.define do
         file =
           if registry.file_type.to_sym == :lfs
             create(:lfs_object)
+          elsif registry.file_type.to_sym == :job_artifact
+            create(:ci_job_artifact)
           else
             create(:upload)
           end

spec/factories/geo_node_statuses.rb

@@ -12,6 +12,9 @@ FactoryBot.define do
       lfs_objects_count 256
       lfs_objects_failed_count 12
       lfs_objects_synced_count 123
+      job_artifacts_count 580
+      job_artifacts_failed_count 3
+      job_artifacts_synced_count 577
       repositories_count 10
       repositories_synced_count 5
       repositories_failed_count 0

spec/fixtures/api/schemas/geo_node_status.json

@@ -12,6 +12,9 @@
     "lfs_objects_count",
     "lfs_objects_failed_count",
     "lfs_objects_synced_count",
+    "job_artifacts_count",
+    "job_artifacts_failed_count",
+    "job_artifacts_synced_count",
     "db_replication_lag_seconds",
     "repositories_count",
     "repositories_failed_count",
@@ -46,6 +49,10 @@
     "lfs_objects_failed_count": { "type": "integer" },
     "lfs_objects_synced_count": { "type": "integer" },
     "lfs_objects_synced_in_percentage": { "type": "string" },
+    "job_artifacts_count": { "type": "integer" },
+    "job_artifacts_failed_count": { "type": "integer" },
+    "job_artifacts_synced_count": { "type": "integer" },
+    "job_artifacts_synced_in_percentage": { "type": "string" },
     "repositories_count": { "type": "integer" },
     "repositories_failed_count": { "type": "integer" },
     "repositories_synced_count": { "type": "integer" },

spec/javascripts/geo_nodes/mock_data.js

@@ -43,6 +43,10 @@ export const rawMockNodeDetails = {
   lfs_objects_synced_count: 0,
   lfs_objects_failed_count: 0,
   lfs_objects_synced_in_percentage: '0.00%',
+  job_artifacts_count: 0,
+  job_artifacts_synced_count: 0,
+  job_artifacts_failed_count: 0,
+  job_artifacts_synced_in_percentage: '0.00%',
   repositories_count: 12,
   repositories_failed_count: 0,
   repositories_synced_count: 12,
@@ -129,6 +133,11 @@ export const mockNodeDetails = {
     successCount: 0,
     failureCount: 0,
   },
+  job_artifacts: {
+    totalCount: 0,
+    successCount: 0,
+    failureCount: 0,
+  },
   attachments: {
     totalCount: 0,
     successCount: 0,