Geo: Verify GitLab Pages Deployments [RUN ALL RSPEC] [RUN AS-IF-FOSS]

config/feature_flags/development/geo_pages_deployment_verification.yml

+---
+name: geo_pages_deployment_verification
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/74905
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/346754
+milestone: '14.6'
+type: development
+group: group::geo
+default_enabled: false

db/migrate/20211119154221_create_pages_deployment_states.rb

+# frozen_string_literal: true
+
+class CreatePagesDeploymentStates < Gitlab::Database::Migration[1.0]
+  VERIFICATION_STATE_INDEX_NAME = "index_pages_deployment_states_on_verification_state"
+  PENDING_VERIFICATION_INDEX_NAME = "index_pages_deployment_states_pending_verification"
+  FAILED_VERIFICATION_INDEX_NAME = "index_pages_deployment_states_failed_verification"
+  NEEDS_VERIFICATION_INDEX_NAME = "index_pages_deployment_states_needs_verification"
+
+  disable_ddl_transaction!
+
+  def up
+    unless table_exists?(:pages_deployment_states)
+      with_lock_retries do
+        create_table :pages_deployment_states, id: false do |t|
+          t.references :pages_deployment, primary_key: true, null: false, foreign_key: { on_delete: :cascade }
+          t.integer :verification_state, default: 0, limit: 2, null: false
+          t.column :verification_started_at, :datetime_with_timezone
+          t.datetime_with_timezone :verification_retry_at
+          t.datetime_with_timezone :verified_at
+          t.integer :verification_retry_count, limit: 2
+          t.binary :verification_checksum, using: 'verification_checksum::bytea'
+          t.text :verification_failure
+
+          t.index :verification_state, name: VERIFICATION_STATE_INDEX_NAME
+          t.index :verified_at, where: "(verification_state = 0)", order: { verified_at: 'ASC NULLS FIRST' }, name: PENDING_VERIFICATION_INDEX_NAME
+          t.index :verification_retry_at, where: "(verification_state = 3)", order: { verification_retry_at: 'ASC NULLS FIRST' }, name: FAILED_VERIFICATION_INDEX_NAME
+          t.index :verification_state, where: "(verification_state = 0 OR verification_state = 3)", name: NEEDS_VERIFICATION_INDEX_NAME
+        end
+      end
+    end
+
+    add_text_limit :pages_deployment_states, :verification_failure, 255
+  end
+
+  def down
+    drop_table :pages_deployment_states
+  end
+end

db/schema_migrations/20211119154221

+020e17ffd6851fb861a17c1b120ca7cdfa300434d4a9ec923a4edcaa7f951b31
\ No newline at end of file

db/structure.sql

@@ -17492,6 +17492,27 @@ CREATE SEQUENCE packages_tags_id_seq
 
 ALTER SEQUENCE packages_tags_id_seq OWNED BY packages_tags.id;
 
+CREATE TABLE pages_deployment_states (
+    pages_deployment_id bigint NOT NULL,
+    verification_state smallint DEFAULT 0 NOT NULL,
+    verification_started_at timestamp with time zone,
+    verification_retry_at timestamp with time zone,
+    verified_at timestamp with time zone,
+    verification_retry_count smallint,
+    verification_checksum bytea,
+    verification_failure text,
+    CONSTRAINT check_15217e8c3a CHECK ((char_length(verification_failure) <= 255))
+);
+
+CREATE SEQUENCE pages_deployment_states_pages_deployment_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE pages_deployment_states_pages_deployment_id_seq OWNED BY pages_deployment_states.pages_deployment_id;
+
 CREATE TABLE pages_deployments (
     id bigint NOT NULL,
     created_at timestamp with time zone NOT NULL,
@@ -21953,6 +21974,8 @@ ALTER TABLE ONLY packages_packages ALTER COLUMN id SET DEFAULT nextval('packages
 
 ALTER TABLE ONLY packages_tags ALTER COLUMN id SET DEFAULT nextval('packages_tags_id_seq'::regclass);
 
+ALTER TABLE ONLY pages_deployment_states ALTER COLUMN pages_deployment_id SET DEFAULT nextval('pages_deployment_states_pages_deployment_id_seq'::regclass);
+
 ALTER TABLE ONLY pages_deployments ALTER COLUMN id SET DEFAULT nextval('pages_deployments_id_seq'::regclass);
 
 ALTER TABLE ONLY pages_domain_acme_orders ALTER COLUMN id SET DEFAULT nextval('pages_domain_acme_orders_id_seq'::regclass);
@@ -23792,6 +23815,9 @@ ALTER TABLE ONLY packages_rubygems_metadata
 ALTER TABLE ONLY packages_tags
     ADD CONSTRAINT packages_tags_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY pages_deployment_states
+    ADD CONSTRAINT pages_deployment_states_pkey PRIMARY KEY (pages_deployment_id);
+
 ALTER TABLE ONLY pages_deployments
     ADD CONSTRAINT pages_deployments_pkey PRIMARY KEY (id);
 
@@ -27021,6 +27047,16 @@ CREATE INDEX index_packages_tags_on_package_id ON packages_tags USING btree (pac
 
 CREATE INDEX index_packages_tags_on_package_id_and_updated_at ON packages_tags USING btree (package_id, updated_at DESC);
 
+CREATE INDEX index_pages_deployment_states_failed_verification ON pages_deployment_states USING btree (verification_retry_at NULLS FIRST) WHERE (verification_state = 3);
+
+CREATE INDEX index_pages_deployment_states_needs_verification ON pages_deployment_states USING btree (verification_state) WHERE ((verification_state = 0) OR (verification_state = 3));
+
+CREATE INDEX index_pages_deployment_states_on_pages_deployment_id ON pages_deployment_states USING btree (pages_deployment_id);
+
+CREATE INDEX index_pages_deployment_states_on_verification_state ON pages_deployment_states USING btree (verification_state);
+
+CREATE INDEX index_pages_deployment_states_pending_verification ON pages_deployment_states USING btree (verified_at NULLS FIRST) WHERE (verification_state = 0);
+
 CREATE INDEX index_pages_deployments_on_ci_build_id ON pages_deployments USING btree (ci_build_id);
 
 CREATE INDEX index_pages_deployments_on_file_store_and_id ON pages_deployments USING btree (file_store, id);
@@ -31567,6 +31603,9 @@ ALTER TABLE ONLY project_tracing_settings
 ALTER TABLE ONLY resource_label_events
     ADD CONSTRAINT fk_rails_fe91ece594 FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL;
 
+ALTER TABLE ONLY pages_deployment_states
+    ADD CONSTRAINT fk_rails_ff6ca551a4 FOREIGN KEY (pages_deployment_id) REFERENCES pages_deployments(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY ci_builds_metadata
     ADD CONSTRAINT fk_rails_ffcf702a02 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
 

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -262,15 +262,15 @@ configuration option in `gitlab.yml`. These metrics are served from the
 | `geo_group_wiki_repositories_failed`           | Gauge   | 13.10 | Number of syncable group wikis failed on secondary | `url` |
 | `geo_group_wiki_repositories_registry`         | Gauge   | 13.10 | Number of syncable group wikis in the registry | `url` |
 | `geo_pages_deployments`                        | Gauge   | 14.3  | Number of pages deployments on primary | `url` |
-| `geo_pages_deployments_checksum_total`         | Gauge   | 14.3  | Number of pages deployments tried to checksum on primary | `url` |
-| `geo_pages_deployments_checksummed`            | Gauge   | 14.3  | Number of pages deployments successfully checksummed on primary | `url` |
-| `geo_pages_deployments_checksum_failed`        | Gauge   | 14.3  | Number of pages deployments failed to calculate the checksum on primary | `url` |
+| `geo_pages_deployments_checksum_total`         | Gauge   | 14.6  | Number of pages deployments tried to checksum on primary | `url` |
+| `geo_pages_deployments_checksummed`            | Gauge   | 14.6  | Number of pages deployments successfully checksummed on primary | `url` |
+| `geo_pages_deployments_checksum_failed`        | Gauge   | 14.6  | Number of pages deployments failed to calculate the checksum on primary | `url` |
 | `geo_pages_deployments_synced`                 | Gauge   | 14.3  | Number of syncable pages deployments synced on secondary | `url` |
 | `geo_pages_deployments_failed`                 | Gauge   | 14.3  | Number of syncable pages deployments failed to sync on secondary | `url` |
 | `geo_pages_deployments_registry`               | Gauge   | 14.3  | Number of pages deployments in the registry | `url` |
-| `geo_pages_deployments_verification_total`     | Gauge   | 14.3  | Number of pages deployments verifications tried on secondary | `url` |
-| `geo_pages_deployments_verified`               | Gauge   | 14.3  | Number of pages deployments verified on secondary | `url` |
-| `geo_pages_deployments_verification_failed`    | Gauge   | 14.3  | Number of pages deployments verifications failed on secondary | `url` |
+| `geo_pages_deployments_verification_total`     | Gauge   | 14.6  | Number of pages deployments verifications tried on secondary | `url` |
+| `geo_pages_deployments_verified`               | Gauge   | 14.6  | Number of pages deployments verified on secondary | `url` |
+| `geo_pages_deployments_verification_failed`    | Gauge   | 14.6  | Number of pages deployments verifications failed on secondary | `url` |
 | `limited_capacity_worker_running_jobs`         | Gauge   | 13.5  | Number of running jobs | `worker` |
 | `limited_capacity_worker_max_running_jobs`     | Gauge   | 13.5  | Maximum number of running jobs | `worker` |
 | `limited_capacity_worker_remaining_work_count` | Gauge   | 13.5  | Number of jobs waiting to be enqueued | `worker` |

ee/app/models/ee/pages_deployment.rb

@@ -6,11 +6,37 @@ module EE
 
     prepended do
       include ::Gitlab::Geo::ReplicableModel
+      include ::Gitlab::Geo::VerificationState
 
       with_replicator ::Geo::PagesDeploymentReplicator
+
+      has_one :pages_deployment_state, autosave: false, inverse_of: :pages_deployment, class_name: '::Geo::PagesDeploymentState'
+
+      after_save :save_verification_details
+
+      delegate :verification_retry_at, :verification_retry_at=,
+         :verified_at, :verified_at=,
+         :verification_checksum, :verification_checksum=,
+         :verification_failure, :verification_failure=,
+         :verification_retry_count, :verification_retry_count=,
+         :verification_state=, :verification_state,
+         :verification_started_at=, :verification_started_at,
+         to: :pages_deployment_state
+
+      scope :with_verification_state, ->(state) { joins(:pages_deployment_state).where(pages_deployment_states: { verification_state: verification_state_value(state) }) }
+      scope :checksummed, -> { joins(:pages_deployment_state).where.not(pages_deployment_states: { verification_checksum: nil } ) }
+      scope :not_checksummed, -> { joins(:pages_deployment_state).where(pages_deployment_states: { verification_checksum: nil } ) }
+
+      scope :available_verifiables, -> { joins(:pages_deployment_state) }
+
+      def verification_state_object
+        pages_deployment_state
+      end
     end
 
     class_methods do
+      extend ::Gitlab::Utils::Override
+
       def replicables_for_current_secondary(primary_key_in)
         node = ::Gitlab::Geo.current_node
 
@@ -19,6 +45,12 @@ module EE
           .merge(object_storage_scope(node))
       end
 
+      override :verification_state_table_class
+
+      def verification_state_table_class
+        ::Geo::PagesDeploymentState
+      end
+
       private
 
       def object_storage_scope(node)
@@ -34,6 +66,10 @@ module EE
       end
     end
 
+    def pages_deployment_state
+      super || build_pages_deployment_state
+    end
+
     def log_geo_deleted_event
       # Keep empty for now. Should be addressed in future
       # by https://gitlab.com/gitlab-org/gitlab/-/issues/232917

ee/app/models/geo/pages_deployment_registry.rb

@@ -2,6 +2,7 @@
 
 class Geo::PagesDeploymentRegistry < Geo::BaseRegistry
   include ::Geo::ReplicableRegistry
+  include ::Geo::VerifiableRegistry
 
   MODEL_CLASS = ::PagesDeployment
   MODEL_FOREIGN_KEY = :pages_deployment_id

ee/app/models/geo/pages_deployment_state.rb

+# frozen_string_literal: true
+
+module Geo
+  class PagesDeploymentState < ApplicationRecord
+    self.primary_key = :pages_deployment_id
+
+    belongs_to :pages_deployment, inverse_of: :pages_deployment_state
+  end
+end

ee/app/replicators/geo/pages_deployment_replicator.rb

@@ -15,7 +15,7 @@ module Geo
 
     override :verification_feature_flag_enabled?
     def self.verification_feature_flag_enabled?
-      false
+      Feature.enabled?(:geo_pages_deployment_verification, default_enabled: :yaml)
     end
   end
 end

ee/db/geo/migrate/20211119152539_add_verification_to_pages_deployment_registry.rb

+# frozen_string_literal: true
+
+class AddVerificationToPagesDeploymentRegistry < Gitlab::Database::Migration[1.0]
+  disable_ddl_transaction!
+
+  def change
+    add_column :pages_deployment_registry, :verification_started_at, :datetime_with_timezone
+    add_column :pages_deployment_registry, :verified_at, :datetime_with_timezone
+    add_column :pages_deployment_registry, :verification_retry_at, :datetime_with_timezone
+    add_column :pages_deployment_registry, :verification_retry_count, :integer, default: 0, limit: 2, null: false
+    add_column :pages_deployment_registry, :verification_state, :integer, limit: 2, default: 0, null: false
+    add_column :pages_deployment_registry, :checksum_mismatch, :boolean, default: false, null: false
+    add_column :pages_deployment_registry, :verification_checksum, :binary
+    add_column :pages_deployment_registry, :verification_checksum_mismatched, :binary
+    # limit is added in 20211207175940_add_text_limit_to_pages_deployment_registry_verification_failure.rb
+    add_column :pages_deployment_registry, :verification_failure, :text # rubocop:disable Migration/AddLimitToTextColumns
+  end
+end

ee/db/geo/migrate/20211207162157_add_indexes_to_pages_deployment_registry.rb

+# frozen_string_literal: true
+
+class AddIndexesToPagesDeploymentRegistry < Gitlab::Database::Migration[1.0]
+  PAGES_DEPLOYMENT_ID_INDEX_NAME = "index_pages_deployment_registry_on_pages_deployment_id"
+  FAILED_VERIFICATION_INDEX_NAME = "pages_deployment_registry_failed_verification"
+  NEEDS_VERIFICATION_INDEX_NAME = "pages_deployment_registry_needs_verification"
+  PENDING_VERIFICATION_INDEX_NAME = "pages_deployment_registry_pending_verification"
+
+  REGISTRY_TABLE = :pages_deployment_registry
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index REGISTRY_TABLE, :pages_deployment_id, name: PAGES_DEPLOYMENT_ID_INDEX_NAME, unique: true
+    add_concurrent_index REGISTRY_TABLE, :verification_retry_at, name: FAILED_VERIFICATION_INDEX_NAME, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 3))"
+    add_concurrent_index REGISTRY_TABLE, :verification_state, name: NEEDS_VERIFICATION_INDEX_NAME, where: "((state = 2) AND (verification_state = ANY (ARRAY[0, 3])))"
+    add_concurrent_index REGISTRY_TABLE, :verified_at, name: PENDING_VERIFICATION_INDEX_NAME, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 0))"
+  end
+
+  def down
+    remove_concurrent_index_by_name REGISTRY_TABLE, name: PAGES_DEPLOYMENT_ID_INDEX_NAME
+    remove_concurrent_index_by_name REGISTRY_TABLE, name: FAILED_VERIFICATION_INDEX_NAME
+    remove_concurrent_index_by_name REGISTRY_TABLE, name: NEEDS_VERIFICATION_INDEX_NAME
+    remove_concurrent_index_by_name REGISTRY_TABLE, name: PENDING_VERIFICATION_INDEX_NAME
+  end
+end

ee/db/geo/migrate/20211207175940_add_text_limit_to_pages_deployment_registry_verification_failure.rb

+# frozen_string_literal: true
+
+class AddTextLimitToPagesDeploymentRegistryVerificationFailure < Gitlab::Database::Migration[1.0]
+  disable_ddl_transaction!
+
+  def up
+    add_text_limit :pages_deployment_registry, :verification_failure, 255
+  end
+
+  def down
+    remove_text_limit :pages_deployment_registry, :verification_failure
+  end
+end

ee/db/geo/schema_migrations/20211119152539

+dc07ed722ba700f3a184b5105a2efedf1a3ec44addd43c2f078b35ff8f3bc384
\ No newline at end of file

ee/db/geo/schema_migrations/20211207162157

+66cc76c52fd028489281dc86f41bc8b6a08a80e0fb36573fb357e2b143222cc7
\ No newline at end of file

ee/db/geo/schema_migrations/20211207175940

+f6c87150f59bdfe84c35c124c0f7f7898e1314877dc5b34d81363828c2324199
\ No newline at end of file

ee/db/geo/structure.sql

@@ -240,7 +240,17 @@ CREATE TABLE pages_deployment_registry (
     retry_at timestamp with time zone,
     state smallint DEFAULT 0 NOT NULL,
     retry_count smallint DEFAULT 0 NOT NULL,
-    last_sync_failure character varying(255)
+    last_sync_failure character varying(255),
+    verification_started_at timestamp with time zone,
+    verified_at timestamp with time zone,
+    verification_retry_at timestamp with time zone,
+    verification_retry_count smallint DEFAULT 0 NOT NULL,
+    verification_state smallint DEFAULT 0 NOT NULL,
+    checksum_mismatch boolean DEFAULT false NOT NULL,
+    verification_checksum bytea,
+    verification_checksum_mismatched bytea,
+    verification_failure text,
+    CONSTRAINT check_7eb0430eff CHECK ((char_length(verification_failure) <= 255))
 );
 
 CREATE SEQUENCE pages_deployment_registry_id_seq
@@ -624,6 +634,12 @@ CREATE INDEX package_file_registry_needs_verification ON package_file_registry U
 
 CREATE INDEX package_file_registry_pending_verification ON package_file_registry USING btree (verified_at NULLS FIRST) WHERE ((state = 2) AND (verification_state = 0));
 
+CREATE INDEX pages_deployment_registry_failed_verification ON pages_deployment_registry USING btree (verification_retry_at NULLS FIRST) WHERE ((state = 2) AND (verification_state = 3));
+
+CREATE INDEX pages_deployment_registry_needs_verification ON pages_deployment_registry USING btree (verification_state) WHERE ((state = 2) AND (verification_state = ANY (ARRAY[0, 3])));
+
+CREATE INDEX pages_deployment_registry_pending_verification ON pages_deployment_registry USING btree (verified_at NULLS FIRST) WHERE ((state = 2) AND (verification_state = 0));
+
 CREATE INDEX pipeline_artifact_registry_failed_verification ON pipeline_artifact_registry USING btree (verification_retry_at NULLS FIRST) WHERE ((state = 2) AND (verification_state = 3));
 
 CREATE INDEX pipeline_artifact_registry_needs_verification ON pipeline_artifact_registry USING btree (verification_state) WHERE ((state = 2) AND (verification_state = ANY (ARRAY[0, 3])));

ee/spec/factories/geo/pages_deployment_registry.rb

@@ -22,5 +22,11 @@ FactoryBot.define do
       last_synced_at { 1.day.ago }
       retry_count { 0 }
     end
+
+    trait :verification_succeeded do
+      verification_checksum { 'e079a831cab27bcda7d81cd9b48296d0c3dd92ef' }
+      verification_state { Geo::PagesDeploymentRegistry.verification_state_value(:verification_succeeded) }
+      verified_at { 5.days.ago }
+    end
   end
 end

ee/spec/factories/geo/pages_deployment_states.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :geo_pages_deployment_state, class: 'Geo::PagesDeploymentState' do
+    pages_deployment
+
+    trait(:checksummed) do
+      verification_checksum { 'abc' }
+    end
+
+    trait(:checksum_failure) do
+      verification_failure { 'Could not calculate the checksum' }
+    end
+  end
+end

ee/spec/factories/pages_deployments.rb

+# frozen_string_literal: true
+
+FactoryBot.modify do
+  factory :lfs_object do
+    trait(:verification_succeeded) do
+      with_file
+      verification_checksum { 'abc' }
+      verification_state { PagesDeployment.verification_state_value(:verification_succeeded) }
+    end
+
+    trait(:verification_failed) do
+      with_file
+      verification_failure { 'Could not calculate the checksum' }
+      verification_state { PagesDeployment.verification_state_value(:verification_failed) }
+    end
+  end
+end

ee/spec/models/ee/pages_deployment_spec.rb

@@ -9,6 +9,17 @@ RSpec.describe PagesDeployment do
   let_it_be(:group) { create(:group) }
   let_it_be(:project) { create(:project, group: group) }
 
+  describe '#save_verification_details' do
+    let(:verifiable_model_record) { build(:pages_deployment) }
+    let(:verification_state_table_class) { verifiable_model_record.class.verification_state_table_class }
+
+    context 'when model_record is part of available_verifiables scope' do
+      it 'creates verification details' do
+        expect { verifiable_model_record.save! }.to change { verification_state_table_class.count }.by(1)
+      end
+    end
+  end
+
   describe '.replicables_for_current_secondary' do
     where(:selective_sync_enabled, :object_storage_sync_enabled, :pages_object_storage_enabled, :synced_pages) do
       true  | true  | true  | 5

ee/spec/models/geo/pages_deployment_registry_spec.rb

@@ -10,4 +10,5 @@ RSpec.describe Geo::PagesDeploymentRegistry, :geo, type: :model do
   end
 
   include_examples 'a Geo framework registry'
+  include_examples 'a Geo verifiable registry'
 end

ee/spec/replicators/geo/pages_deployment_replicator_spec.rb

@@ -6,4 +6,5 @@ RSpec.describe Geo::PagesDeploymentReplicator do
   let(:model_record) { build(:pages_deployment) }
 
   include_examples 'a blob replicator'
+  include_examples 'a verifiable replicator'
 end

lib/gitlab/database/gitlab_schemas.yml

@@ -371,6 +371,7 @@ packages_pypi_metadata: :gitlab_main
 packages_rubygems_metadata: :gitlab_main
 packages_tags: :gitlab_main
 pages_deployments: :gitlab_main
+pages_deployment_states: :gitlab_main
 pages_domain_acme_orders: :gitlab_main
 pages_domains: :gitlab_main
 partitioned_foreign_keys: :gitlab_main