Geo: Add verification for terraform states

- Add verification through self-service framework
for Terraform States, using terraform_state_versions
table

- Add related metrics to API and Prometheus

- Update metrics doc

Changelog: changed

app/models/terraform/state.rb

@@ -104,3 +104,5 @@ module Terraform
     end
   end
 end
+
+Terraform::State.prepend_if_ee('EE::Terraform::State')

changelogs/unreleased/ag-verify-terraform-state.yml

+---
+title: 'Geo: Add verification for Terraform States'
+merge_request: 58800
+author:
+type: changed

db/migrate/20210407140539_add_verification_state_and_started_at_to_terraform_state_version_table.rb

+# frozen_string_literal: true
+
+class AddVerificationStateAndStartedAtToTerraformStateVersionTable < ActiveRecord::Migration[6.0]
+  def change
+    change_table(:terraform_state_versions) do |t|
+      t.column :verification_started_at, :datetime_with_timezone
+      t.integer :verification_state, default: 0, limit: 2, null: false
+    end
+  end
+end

db/migrate/20210420173030_add_verification_indexes_to_terraform_state_versions.rb

+# frozen_string_literal: true
+
+class AddVerificationIndexesToTerraformStateVersions < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  VERIFICATION_STATE_INDEX_NAME = "index_terraform_state_versions_on_verification_state"
+  PENDING_VERIFICATION_INDEX_NAME = "index_terraform_state_versions_pending_verification"
+  FAILED_VERIFICATION_INDEX_NAME = "index_terraform_state_versions_failed_verification"
+  NEEDS_VERIFICATION_INDEX_NAME = "index_terraform_state_versions_needs_verification"
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :terraform_state_versions, :verification_state, name: VERIFICATION_STATE_INDEX_NAME
+    add_concurrent_index :terraform_state_versions, :verified_at, where: "(verification_state = 0)", order: { verified_at: 'ASC NULLS FIRST' }, name: PENDING_VERIFICATION_INDEX_NAME
+    add_concurrent_index :terraform_state_versions, :verification_retry_at, where: "(verification_state = 3)", order: { verification_retry_at: 'ASC NULLS FIRST' }, name: FAILED_VERIFICATION_INDEX_NAME
+    add_concurrent_index :terraform_state_versions, :verification_state, where: "(verification_state = 0 OR verification_state = 3)", name: NEEDS_VERIFICATION_INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :terraform_state_versions, VERIFICATION_STATE_INDEX_NAME
+    remove_concurrent_index_by_name :terraform_state_versions, PENDING_VERIFICATION_INDEX_NAME
+    remove_concurrent_index_by_name :terraform_state_versions, FAILED_VERIFICATION_INDEX_NAME
+    remove_concurrent_index_by_name :terraform_state_versions, NEEDS_VERIFICATION_INDEX_NAME
+  end
+end

db/schema_migrations/20210407140539

+9f19b44a4ef3131e6ddd9cfea0d8b1eb4499754f2200bea90b5ed41eb688f622
\ No newline at end of file

db/schema_migrations/20210420173030

+3a223c462b10edb9eb68fc0adf42f046a45f554f35b4b4ee64a834cd7372f827
\ No newline at end of file

db/structure.sql

@@ -18069,6 +18069,8 @@ CREATE TABLE terraform_state_versions (
     verification_checksum bytea,
     verification_failure text,
     ci_build_id bigint,
+    verification_started_at timestamp with time zone,
+    verification_state smallint DEFAULT 0 NOT NULL,
     CONSTRAINT check_0824bb7bbd CHECK ((char_length(file) <= 255)),
     CONSTRAINT tf_state_versions_verification_failure_text_limit CHECK ((char_length(verification_failure) <= 255))
 );
@@ -24279,12 +24281,20 @@ CREATE INDEX index_term_agreements_on_term_id ON term_agreements USING btree (te
 
 CREATE INDEX index_term_agreements_on_user_id ON term_agreements USING btree (user_id);
 
+CREATE INDEX index_terraform_state_versions_failed_verification ON terraform_state_versions USING btree (verification_retry_at NULLS FIRST) WHERE (verification_state = 3);
+
+CREATE INDEX index_terraform_state_versions_needs_verification ON terraform_state_versions USING btree (verification_state) WHERE ((verification_state = 0) OR (verification_state = 3));
+
 CREATE INDEX index_terraform_state_versions_on_ci_build_id ON terraform_state_versions USING btree (ci_build_id);
 
 CREATE INDEX index_terraform_state_versions_on_created_by_user_id ON terraform_state_versions USING btree (created_by_user_id);
 
 CREATE UNIQUE INDEX index_terraform_state_versions_on_state_id_and_version ON terraform_state_versions USING btree (terraform_state_id, version);
 
+CREATE INDEX index_terraform_state_versions_on_verification_state ON terraform_state_versions USING btree (verification_state);
+
+CREATE INDEX index_terraform_state_versions_pending_verification ON terraform_state_versions USING btree (verified_at NULLS FIRST) WHERE (verification_state = 0);
+
 CREATE INDEX index_terraform_states_on_file_store ON terraform_states USING btree (file_store);
 
 CREATE INDEX index_terraform_states_on_locked_by_user_id ON terraform_states USING btree (locked_by_user_id);

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -215,11 +215,15 @@ configuration option in `gitlab.yml`. These metrics are served from the
 | `geo_package_files_failed`                     | Gauge   | 13.3  | Number of syncable package files failed to sync on secondary | `url` |
 | `geo_package_files_registry`                   | Gauge   | 13.3  | Number of package files in the registry | `url` |
 | `geo_terraform_state_versions`                 | Gauge   | 13.5  | Number of terraform state versions on primary | `url` |
-| `geo_terraform_state_versions_checksummed`     | Gauge   | 13.5  | Number of terraform state versions checksummed on primary | `url` |
+| `geo_terraform_state_versions_checksummed`     | Gauge   | 13.5  | Number of terraform state versions checksummed successfully on primary | `url` |
 | `geo_terraform_state_versions_checksum_failed` | Gauge   | 13.5  | Number of terraform state versions failed to calculate the checksum on primary | `url` |
+| `geo_terraform_state_versions_checksum_total`  | Gauge   | 13.12  | Number of terraform state versions tried to checksum on primary | `url` |
 | `geo_terraform_state_versions_synced`          | Gauge   | 13.5  | Number of syncable terraform state versions synced on secondary | `url` |
 | `geo_terraform_state_versions_failed`          | Gauge   | 13.5  | Number of syncable terraform state versions failed to sync on secondary | `url` |
 | `geo_terraform_state_versions_registry`        | Gauge   | 13.5  | Number of terraform state versions in the registry | `url` |
+| `geo_terraform_state_versions_verified`        | Gauge   | 13.12  | Number of terraform state versions verified on secondary | `url` |
+| `geo_terraform_state_versions_verification_failed` | Gauge   | 13.12  | Number of terraform state versions verifications failed on secondary | `url` |
+| `geo_terraform_state_versions_verification_total` | Gauge   | 13.12  | Number of terraform state versions verifications tried on secondary | `url` |
 | `global_search_bulk_cron_queue_size`           | Gauge   | 12.10 | Number of database records waiting to be synchronized to Elasticsearch | |
 | `global_search_awaiting_indexing_queue_size`   | Gauge   | 13.2  | Number of database updates waiting to be synchronized to Elasticsearch while indexing is paused | |
 | `geo_merge_request_diffs`                      | Gauge   | 13.4  | Number of merge request diffs on primary | `url` |

ee/app/models/ee/terraform/state.rb

+# frozen_string_literal: true
+
+module EE
+  module Terraform
+    module State
+      extend ActiveSupport::Concern
+    end
+  end
+end

ee/app/models/ee/terraform/state_version.rb

@@ -7,6 +7,8 @@ module EE
 
       prepended do
         include ::Gitlab::Geo::ReplicableModel
+        include ::Gitlab::Geo::VerificationState
+
         with_replicator Geo::TerraformStateVersionReplicator
 
         scope :project_id_in, ->(ids) { joins(:terraform_state).where('terraform_states.project_id': ids) }

ee/app/models/geo/terraform_state_version_registry.rb

@@ -2,6 +2,7 @@
 
 class Geo::TerraformStateVersionRegistry < Geo::BaseRegistry
   include Geo::ReplicableRegistry
+  include ::Geo::VerifiableRegistry
 
   MODEL_CLASS = ::Terraform::StateVersion
   MODEL_FOREIGN_KEY = :terraform_state_version_id

ee/app/replicators/geo/terraform_state_version_replicator.rb

@@ -3,6 +3,7 @@
 module Geo
   class TerraformStateVersionReplicator < Gitlab::Geo::Replicator
     include ::Geo::BlobReplicatorStrategy
+    extend ::Gitlab::Utils::Override
 
     def carrierwave_uploader
       model_record.file
@@ -11,5 +12,10 @@ module Geo
     def self.model
       ::Terraform::StateVersion
     end
+
+    override :verification_feature_flag_enabled?
+    def self.verification_feature_flag_enabled?
+      Feature.enabled?(:geo_terraform_state_version_verification, default_enabled: :yaml)
+    end
   end
 end

ee/config/feature_flags/development/geo_terraform_state_version_verification.yml

+---
+name: geo_terraform_state_version_verification
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/58800
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/329963
+milestone: '13.12'
+type: development
+group: group::geo
+default_enabled: false

ee/db/geo/migrate/20210407140527_add_verification_to_terraform_state_version_registry.rb

+# frozen_string_literal: true
+
+class AddVerificationToTerraformStateVersionRegistry < ActiveRecord::Migration[6.0]
+  def change
+    add_column :terraform_state_version_registry, :verification_started_at, :datetime_with_timezone
+    add_column :terraform_state_version_registry, :verified_at, :datetime_with_timezone
+    add_column :terraform_state_version_registry, :verification_retry_at, :datetime_with_timezone
+    add_column :terraform_state_version_registry, :verification_retry_count, :integer, default: 0
+    add_column :terraform_state_version_registry, :verification_state, :integer, limit: 2, default: 0, null: false
+    add_column :terraform_state_version_registry, :checksum_mismatch, :boolean, default: false, null: false
+    add_column :terraform_state_version_registry, :verification_checksum, :binary
+    add_column :terraform_state_version_registry, :verification_checksum_mismatched, :binary
+    add_column :terraform_state_version_registry, :verification_failure, :string, limit: 255 # rubocop:disable Migration/PreventStrings because https://gitlab.com/gitlab-org/gitlab/-/issues/323806
+  end
+end

ee/db/geo/migrate/20210420180119_add_indexes_to_terraform_state_version_registry.rb

+# frozen_string_literal: true
+
+class AddIndexesToTerraformStateVersionRegistry < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  TERRAFORM_STATE_VERSION_ID_INDEX_NAME = "index_terraform_state_version_registry_on_t_state_version_id"
+  FAILED_VERIFICATION_INDEX_NAME = "terraform_state_version_registry_failed_verification"
+  NEEDS_VERIFICATION_INDEX_NAME = "terraform_state_version_registry_needs_verification"
+  PENDING_VERIFICATION_INDEX_NAME = "terraform_state_version_registry_pending_verification"
+
+  REGISTRY_TABLE = :terraform_state_version_registry
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index REGISTRY_TABLE, :terraform_state_version_id, name: TERRAFORM_STATE_VERSION_ID_INDEX_NAME, unique: true
+    add_concurrent_index REGISTRY_TABLE, :retry_at
+    add_concurrent_index REGISTRY_TABLE, :state
+    add_concurrent_index REGISTRY_TABLE, :verification_retry_at, name: FAILED_VERIFICATION_INDEX_NAME, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 3))"
+    add_concurrent_index REGISTRY_TABLE, :verification_state, name: NEEDS_VERIFICATION_INDEX_NAME, where: "((state = 2)  AND (verification_state = ANY (ARRAY[0, 3])))"
+    add_concurrent_index REGISTRY_TABLE, :verified_at, name: PENDING_VERIFICATION_INDEX_NAME, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 0))"
+  end
+
+  def down
+    remove_concurrent_index_by_name REGISTRY_TABLE, name: TERRAFORM_STATE_VERSION_ID_INDEX_NAME
+    remove_concurrent_index_by_name REGISTRY_TABLE, name: :index_terraform_state_version_registry_on_retry_at
+    remove_concurrent_index_by_name REGISTRY_TABLE, name: :index_terraform_state_version_registry_on_state
+    remove_concurrent_index_by_name REGISTRY_TABLE, name: FAILED_VERIFICATION_INDEX_NAME
+    remove_concurrent_index_by_name REGISTRY_TABLE, name: NEEDS_VERIFICATION_INDEX_NAME
+    remove_concurrent_index_by_name REGISTRY_TABLE, name: PENDING_VERIFICATION_INDEX_NAME
+  end
+end

ee/db/geo/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2021_03_25_150435) do
+ActiveRecord::Schema.define(version: 2021_04_20_180119) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -118,6 +118,15 @@ ActiveRecord::Schema.define(version: 2021_03_25_150435) do
     t.integer "state", limit: 2, default: 0, null: false
     t.integer "retry_count", limit: 2, default: 0
     t.text "last_sync_failure"
+    t.datetime_with_timezone "verification_started_at"
+    t.datetime_with_timezone "verified_at"
+    t.datetime_with_timezone "verification_retry_at"
+    t.integer "verification_retry_count"
+    t.integer "verification_state", limit: 2, default: 0, null: false
+    t.boolean "checksum_mismatch"
+    t.binary "verification_checksum"
+    t.binary "verification_checksum_mismatched"
+    t.string "verification_failure", limit: 255
     t.index ["merge_request_diff_id"], name: "index_merge_request_diff_registry_on_mr_diff_id"
     t.index ["retry_at"], name: "index_merge_request_diff_registry_on_retry_at"
     t.index ["state"], name: "index_merge_request_diff_registry_on_state"
@@ -271,9 +280,22 @@ ActiveRecord::Schema.define(version: 2021_03_25_150435) do
     t.datetime_with_timezone "last_synced_at"
     t.datetime_with_timezone "created_at", null: false
     t.text "last_sync_failure"
+    t.datetime_with_timezone "verification_started_at"
+    t.datetime_with_timezone "verified_at"
+    t.datetime_with_timezone "verification_retry_at"
+    t.integer "verification_retry_count", default: 0
+    t.integer "verification_state", limit: 2, default: 0, null: false
+    t.boolean "checksum_mismatch", default: false, null: false
+    t.binary "verification_checksum"
+    t.binary "verification_checksum_mismatched"
+    t.string "verification_failure", limit: 255
     t.index ["retry_at"], name: "index_terraform_state_version_registry_on_retry_at"
     t.index ["state"], name: "index_terraform_state_version_registry_on_state"
+    t.index ["terraform_state_version_id"], name: "index_terraform_state_version_registry_on_t_state_version_id", unique: true
     t.index ["terraform_state_version_id"], name: "index_tf_state_versions_registry_tf_state_versions_id_unique", unique: true
+    t.index ["verification_retry_at"], name: "terraform_state_version_registry_failed_verification", order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 3))"
+    t.index ["verification_state"], name: "terraform_state_version_registry_needs_verification", where: "((state = 2) AND (verification_state = ANY (ARRAY[0, 3])))"
+    t.index ["verified_at"], name: "terraform_state_version_registry_pending_verification", order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 0))"
   end
 
 end

ee/spec/factories/terraform/state_version.rb

+# frozen_string_literal: true
+
+FactoryBot.modify do
+  factory :terraform_state_version, class: '::Terraform::StateVersion' do
+    trait(:verification_succeeded) do
+      with_file
+      verification_checksum { 'abc' }
+      verification_state { ::Terraform::StateVersion.verification_state_value(:verification_succeeded) }
+    end
+
+    trait(:verification_failed) do
+      with_file
+      verification_failure { 'Could not calculate the checksum' }
+      verification_state { ::Terraform::StateVersion.verification_state_value(:verification_failed) }
+    end
+  end
+end

ee/spec/models/geo/terraform_state_version_registry_spec.rb

@@ -10,4 +10,5 @@ RSpec.describe Geo::TerraformStateVersionRegistry, :geo, type: :model do
   end
 
   include_examples 'a Geo framework registry'
+  include_examples 'a Geo verifiable registry'
 end

ee/spec/replicators/geo/terraform_state_version_replicator_spec.rb

@@ -5,5 +5,6 @@ require 'spec_helper'
 RSpec.describe Geo::TerraformStateVersionReplicator do
   let(:model_record) { build(:terraform_state_version, terraform_state: create(:terraform_state)) }
 
-  it_behaves_like 'a blob replicator'
+  include_examples 'a blob replicator'
+  include_examples 'a verifiable replicator'
 end