Add replication for versioned tf state

- Add replicator for terraform state versions
- Add registry for terraform state versions
- Add terraform state versions ee model extension
- Add metrics for terraform state replication

Relates to https://gitlab.com/gitlab-org/gitlab/issues/237926
Relates to https://gitlab.com/gitlab-org/gitlab/issues/237927

app/models/terraform/state_version.rb

@@ -14,5 +14,11 @@ module Terraform
     mount_file_store_uploader VersionedStateUploader
 
     delegate :project_id, :uuid, to: :terraform_state, allow_nil: true
+
+    def local?
+      file_store == ObjectStorage::Store::LOCAL
+    end
   end
 end
+
+Terraform::StateVersion.prepend_if_ee('EE::Terraform::StateVersion')

config/initializers_before_autoloader/000_inflections.rb

@@ -24,11 +24,12 @@ ActiveSupport::Inflector.inflections do |inflect|
     project_auto_devops
     project_registry
     project_statistics
+    snippet_repository_registry
     system_note_metadata
     terraform_state_registry
+    terraform_state_version_registry
     vulnerabilities_feedback
     vulnerability_feedback
-    snippet_repository_registry
   )
   inflect.acronym 'EE'
   inflect.acronym 'CSP'

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -190,6 +190,12 @@ configuration option in `gitlab.yml`. These metrics are served from the
 | `geo_terraform_states_synced`                  | Gauge   | 13.3  | Number of syncable terraform states synced on secondary | `url` |
 | `geo_terraform_states_failed`                  | Gauge   | 13.3  | Number of syncable terraform states failed to sync on secondary | `url` |
 | `geo_terraform_states_registry`                | Gauge   | 13.3  | Number of terraform states in the registry | `url` |
+| `geo_terraform_state_versions`                 | Gauge   | 13.5  | Number of terraform state versions on primary | `url` |
+| `geo_terraform_state_versions_checksummed`     | Gauge   | 13.5  | Number of terraform state versions checksummed on primary | `url` |
+| `geo_terraform_state_versions_checksum_failed` | Gauge   | 13.5  | Number of terraform state versions failed to calculate the checksum on primary | `url` |
+| `geo_terraform_state_versions_synced`          | Gauge   | 13.5  | Number of syncable terraform state versions synced on secondary | `url` |
+| `geo_terraform_state_versions_failed`          | Gauge   | 13.5  | Number of syncable terraform state versions failed to sync on secondary | `url` |
+| `geo_terraform_state_versions_registry`        | Gauge   | 13.5  | Number of terraform state versions in the registry | `url` |
 | `global_search_bulk_cron_queue_size`           | Gauge   | 12.10 | Number of database records waiting to be synchronized to Elasticsearch | |
 | `global_search_awaiting_indexing_queue_size`   | Gauge   | 13.2  | Number of database updates waiting to be synchronized to Elasticsearch while indexing is paused | |
 | `geo_merge_request_diffs`                      | Gauge   | 13.4  | Number of merge request diffs on primary | `url` |

doc/api/geo_nodes.md

@@ -465,7 +465,13 @@ Example response:
     "terraform_states_checksum_failed_count": 0,
     "terraform_states_registry_count": 10,
     "terraform_states_synced_count": 6,
-    "terraform_states_failed_count": 3
+    "terraform_states_failed_count": 3,
+    "terraform_state_versions_count": 10,
+    "terraform_state_versions_checksummed_count": 10,
+    "terraform_state_versions_checksum_failed_count": 0,
+    "terraform_state_versions_registry_count": 10,
+    "terraform_state_versions_synced_count": 6,
+    "terraform_state_versions_failed_count": 3,
     "snippet_repositories_count": 10,
     "snippet_repositories_checksummed_count": 10,
     "snippet_repositories_checksum_failed_count": 0,

ee/app/models/ee/terraform/state_version.rb

+# frozen_string_literal: true
+
+module EE
+  module Terraform
+    module StateVersion
+      extend ActiveSupport::Concern
+
+      prepended do
+        include ::Gitlab::Geo::ReplicableModel
+        with_replicator Geo::TerraformStateVersionReplicator
+
+        scope :with_files_stored_locally, -> { where(file_store: ::ObjectStorage::Store::LOCAL) }
+        scope :project_id_in, ->(ids) { joins(:terraform_state).where('terraform_states.project_id': ids) }
+      end
+
+      class_methods do
+        def replicables_for_geo_node(node = ::Gitlab::Geo.current_node)
+          selective_sync_scope(node).merge(object_storage_scope(node))
+        end
+
+        private
+
+        def object_storage_scope(node)
+          return all if node.sync_object_storage?
+
+          with_files_stored_locally
+        end
+
+        def selective_sync_scope(node)
+          return all unless node.selective_sync?
+
+          project_id_in(node.projects)
+        end
+      end
+
+      def log_geo_deleted_event
+        # Keep empty for now. Should be addressed in future
+        # by https://gitlab.com/gitlab-org/gitlab/-/issues/232917
+      end
+    end
+  end
+end

ee/app/models/geo/terraform_state_version_registry.rb

+# frozen_string_literal: true
+
+class Geo::TerraformStateVersionRegistry < Geo::BaseRegistry
+  include Geo::ReplicableRegistry
+
+  MODEL_CLASS = ::Terraform::StateVersion
+  MODEL_FOREIGN_KEY = :terraform_state_version_id
+
+  belongs_to :terraform_state_version, class_name: 'Terraform::StateVersion'
+end

ee/app/replicators/geo/terraform_state_version_replicator.rb

+# frozen_string_literal: true
+
+module Geo
+  class TerraformStateVersionReplicator < Gitlab::Geo::Replicator
+    include ::Geo::BlobReplicatorStrategy
+
+    def carrierwave_uploader
+      model_record.file
+    end
+
+    def self.model
+      ::Terraform::StateVersion
+    end
+
+    # Remove with https://gitlab.com/gitlab-org/gitlab/-/issues/249176
+    def self.replication_enabled_by_default?
+      false
+    end
+  end
+end

ee/app/workers/geo/secondary/registry_consistency_worker.rb

@@ -24,6 +24,7 @@ module Geo
         Geo::PackageFileRegistry,
         Geo::ProjectRegistry,
         Geo::TerraformStateRegistry,
+        Geo::TerraformStateVersionRegistry,
         Geo::UploadRegistry
       ].freeze
 

ee/config/feature_flags/development/geo_terraform_state_replication.yml

 ---
 name: geo_terraform_state_replication
-introduced_by_url: 
-rollout_issue_url: 
-group: 
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36594
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/254623
+group: group::geo
 type: development
 default_enabled: false

ee/config/feature_flags/development/geo_terraform_state_version_replication.yml

+---
+name: geo_terraform_state_version_replication
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/42529
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/254622
+group: group::geo
+type: development
+default_enabled: false

ee/lib/gitlab/geo.rb

@@ -23,6 +23,7 @@ module Gitlab
       ::Geo::MergeRequestDiffReplicator,
       ::Geo::PackageFileReplicator,
       ::Geo::TerraformStateReplicator,
+      ::Geo::TerraformStateVersionReplicator,
       ::Geo::SnippetRepositoryReplicator
     ].freeze
 

ee/spec/factories/geo/terraform_state_version_registry.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :geo_terraform_state_version_registry, class: 'Geo::TerraformStateVersionRegistry' do
+    association :terraform_state_version, factory: :terraform_state_version
+    state { Geo::TerraformStateVersionRegistry.state_value(:pending) }
+
+    trait :synced do
+      state { Geo::TerraformStateVersionRegistry.state_value(:synced) }
+      last_synced_at { 5.days.ago }
+    end
+
+    trait :failed do
+      state { Geo::TerraformStateVersionRegistry.state_value(:failed) }
+      last_synced_at { 1.day.ago }
+      retry_count { 2 }
+      last_sync_failure { 'Random error' }
+    end
+
+    trait :started do
+      state { Geo::TerraformStateVersionRegistry.state_value(:started) }
+      last_synced_at { 1.day.ago }
+      retry_count { 0 }
+    end
+  end
+end

ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json

@@ -70,6 +70,14 @@
     "terraform_states_failed_count",
     "terraform_states_synced_count",
     "terraform_states_synced_in_percentage",
+    "terraform_state_versions_count",
+    "terraform_state_versions_checksum_failed_count",
+    "terraform_state_versions_checksummed_in_percentage",
+    "terraform_state_versions_checksummed_count",
+    "terraform_state_versions_registry_count",
+    "terraform_state_versions_failed_count",
+    "terraform_state_versions_synced_count",
+    "terraform_state_versions_synced_in_percentage",
     "snippet_repositories_count",
     "snippet_repositories_checksum_failed_count",
     "snippet_repositories_checksummed_in_percentage",
@@ -183,6 +191,14 @@
     "terraform_states_synced_count": { "type": ["integer", "null"] },
     "terraform_states_synced_in_percentage": { "type": "string" },
     "terraform_states_checksummed_in_percentage": { "type": "string" },
+    "terraform_state_versions_count": { "type": ["integer", "null"] },
+    "terraform_state_versions_checksummed_count": { "type": ["integer", "null"] },
+    "terraform_state_versions_checksum_failed_count": { "type": ["integer", "null"] },
+    "terraform_state_versions_registry_count": { "type": ["integer", "null"] },
+    "terraform_state_versions_failed_count": { "type": ["integer", "null"] },
+    "terraform_state_versions_synced_count": { "type": ["integer", "null"] },
+    "terraform_state_versions_synced_in_percentage": { "type": "string" },
+    "terraform_state_versions_checksummed_in_percentage": { "type": "string" },
     "snippet_repositories_count": { "type": ["integer", "null"] },
     "snippet_repositories_checksummed_count": { "type": ["integer", "null"] },
     "snippet_repositories_checksum_failed_count": { "type": ["integer", "null"] },

ee/spec/models/ee/terraform/state_spec.rb

@@ -17,8 +17,8 @@ RSpec.describe Terraform::State do
       expect(described_class.with_files_stored_locally).to have_attributes(count: 5)
     end
 
-    it 'excludes states with local storage' do
-      stub_terraform_state_object_storage(Terraform::StateUploader)
+    it 'excludes states without local storage' do
+      stub_terraform_state_object_storage
 
       create_list(:terraform_state, 5, :with_file)
 
@@ -54,7 +54,7 @@ RSpec.describe Terraform::State do
 
       before do
         stub_current_geo_node(secondary)
-        stub_terraform_state_object_storage(Terraform::StateUploader) if terraform_object_storage_enabled
+        stub_terraform_state_object_storage if terraform_object_storage_enabled
 
         create_list(:terraform_state, 5, :with_file, project: project)
         create_list(:terraform_state, 5, :with_file, project: create(:project))
@@ -70,7 +70,7 @@ RSpec.describe Terraform::State do
 
       before do
         stub_current_geo_node(secondary)
-        stub_terraform_state_object_storage(Terraform::StateUploader)
+        stub_terraform_state_object_storage
 
         create_list(:terraform_state, 5, project: project)
       end

ee/spec/models/ee/terraform/state_version_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Terraform::StateVersion do
+  using RSpec::Parameterized::TableSyntax
+  include EE::GeoHelpers
+
+  let_it_be(:group) { create(:group) }
+  let_it_be(:project) { create(:project, group: group) }
+
+  describe '.with_files_stored_locally' do
+    it 'includes states with local storage' do
+      create_list(:terraform_state_version, 5)
+
+      expect(described_class.with_files_stored_locally).to have_attributes(count: 5)
+    end
+
+    it 'excludes states without local storage' do
+      stub_terraform_state_version_object_storage
+
+      create_list(:terraform_state_version, 5)
+
+      expect(described_class.with_files_stored_locally).to have_attributes(count: 0)
+    end
+  end
+
+  describe '.replicables_for_geo_node' do
+    where(:selective_sync_enabled, :object_storage_sync_enabled, :terraform_object_storage_enabled, :synced_states) do
+      true  | true  | true  | 5
+      true  | true  | false | 5
+      true  | false | true  | 0
+      true  | false | false | 5
+      false | false | false | 10
+      false | false | true  | 0
+      false | true  | true  | 10
+      false | true  | false | 10
+      true  | true  | false | 5
+    end
+
+    with_them do
+      let(:secondary) do
+        node = build(:geo_node, sync_object_storage: object_storage_sync_enabled)
+
+        if selective_sync_enabled
+          node.selective_sync_type = 'namespaces'
+          node.namespaces = [group]
+        end
+
+        node.save!
+        node
+      end
+
+      before do
+        stub_current_geo_node(secondary)
+        stub_terraform_state_version_object_storage if terraform_object_storage_enabled
+
+        create_list(:terraform_state_version, 5, terraform_state: create(:terraform_state, project: project))
+        create_list(:terraform_state_version, 5, terraform_state: create(:terraform_state, project: create(:project)))
+      end
+
+      it 'returns the proper number of terraform states' do
+        expect(Terraform::StateVersion.replicables_for_geo_node.count).to eq(synced_states)
+      end
+    end
+  end
+end

ee/spec/models/geo/terraform_state_version_registry_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Geo::TerraformStateVersionRegistry, :geo, type: :model do
+  let_it_be(:registry) { create(:geo_terraform_state_version_registry) }
+
+  specify 'factory is valid' do
+    expect(registry).to be_valid
+  end
+
+  include_examples 'a Geo framework registry'
+end

ee/spec/models/geo_node_status_spec.rb

@@ -1147,10 +1147,11 @@ RSpec.describe GeoNodeStatus, :geo do
   end
 
   where(:replicator, :model_factory, :registry_factory) do
-    Geo::MergeRequestDiffReplicator | :external_merge_request_diff | :geo_merge_request_diff_registry
-    Geo::PackageFileReplicator | :package_file | :geo_package_file_registry
-    Geo::TerraformStateReplicator | :terraform_state | :geo_terraform_state_registry
-    Geo::SnippetRepositoryReplicator | :snippet_repository | :geo_snippet_repository_registry
+    Geo::MergeRequestDiffReplicator      | :external_merge_request_diff | :geo_merge_request_diff_registry
+    Geo::PackageFileReplicator           | :package_file                | :geo_package_file_registry
+    Geo::TerraformStateReplicator        | :terraform_state             | :geo_terraform_state_registry
+    Geo::TerraformStateVersionReplicator | :terraform_state_version     | :geo_terraform_state_version_registry
+    Geo::SnippetRepositoryReplicator     | :snippet_repository          | :geo_snippet_repository_registry
   end
 
   with_them do

ee/spec/replicators/geo/terraform_state_version_replicator_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Geo::TerraformStateVersionReplicator do
+  let(:model_record) { build(:terraform_state_version) }
+
+  it_behaves_like 'a blob replicator'
+end

ee/spec/services/geo/registry_consistency_service_spec.rb

@@ -19,7 +19,8 @@ RSpec.describe Geo::RegistryConsistencyService, :geo, :use_clean_rails_memory_st
     { Geo::DesignRegistry => :project_with_design,
       Geo::MergeRequestDiffRegistry => :external_merge_request_diff,
       Geo::PackageFileRegistry => :package_file_with_file,
-      Geo::TerraformStateRegistry => :legacy_terraform_state }
+      Geo::TerraformStateRegistry => :legacy_terraform_state,
+      Geo::TerraformStateVersionRegistry => :terraform_state_version }
       .fetch(registry_class, default_factory_name)
   end
 

ee/spec/workers/geo/secondary/registry_consistency_worker_spec.rb

@@ -85,7 +85,8 @@ RSpec.describe Geo::Secondary::RegistryConsistencyWorker, :geo do
       lfs_object = create(:lfs_object)
       merge_request_diff = create(:merge_request_diff, :external)
       package_file = create(:conan_package_file, :conan_package)
-      terraform_state = create(:terraform_state, :with_file, project: project)
+      terraform_state = create(:legacy_terraform_state, project: project)
+      terraform_state_version = create(:terraform_state_version)
       upload = create(:upload)
 
       expect(Geo::ContainerRepositoryRegistry.where(container_repository_id: container_repository.id).count).to eq(0)
@@ -96,6 +97,7 @@ RSpec.describe Geo::Secondary::RegistryConsistencyWorker, :geo do
       expect(Geo::PackageFileRegistry.where(package_file_id: package_file.id).count).to eq(0)
       expect(Geo::ProjectRegistry.where(project_id: project.id).count).to eq(0)
       expect(Geo::TerraformStateRegistry.where(terraform_state_id: terraform_state.id).count).to eq(0)
+      expect(Geo::TerraformStateVersionRegistry.where(terraform_state_version_id: terraform_state_version.id).count).to eq(0)
       expect(Geo::UploadRegistry.where(file_id: upload.id).count).to eq(0)
 
       subject.perform
@@ -108,6 +110,7 @@ RSpec.describe Geo::Secondary::RegistryConsistencyWorker, :geo do
       expect(Geo::PackageFileRegistry.where(package_file_id: package_file.id).count).to eq(1)
       expect(Geo::ProjectRegistry.where(project_id: project.id).count).to eq(1)
       expect(Geo::TerraformStateRegistry.where(terraform_state_id: terraform_state.id).count).to eq(1)
+      expect(Geo::TerraformStateVersionRegistry.where(terraform_state_version_id: terraform_state_version.id).count).to eq(1)
       expect(Geo::UploadRegistry.where(file_id: upload.id).count).to eq(1)
     end
 

spec/factories/terraform/state_version.rb

@@ -7,5 +7,13 @@ FactoryBot.define do
 
     sequence(:version)
     file { fixture_file_upload('spec/fixtures/terraform/terraform.tfstate', 'application/json') }
+
+    trait(:checksummed) do
+      verification_checksum { 'abc' }
+    end
+
+    trait(:checksum_failure) do
+      verification_failure { 'Could not calculate the checksum' }
+    end
   end
 end

spec/models/terraform/state_spec.rb

@@ -15,7 +15,7 @@ RSpec.describe Terraform::State do
   it { is_expected.to validate_presence_of(:project_id) }
 
   before do
-    stub_terraform_state_object_storage(Terraform::StateUploader)
+    stub_terraform_state_object_storage
   end
 
   describe '#file' do
@@ -43,7 +43,7 @@ RSpec.describe Terraform::State do
 
     context 'when file is stored locally' do
       before do
-        stub_terraform_state_object_storage(Terraform::StateUploader, enabled: false)
+        stub_terraform_state_object_storage(enabled: false)
       end
 
       it_behaves_like 'mounted file in local store'

spec/models/terraform/state_version_spec.rb

@@ -29,7 +29,7 @@ RSpec.describe Terraform::StateVersion do
     subject { create(:terraform_state_version) }
 
     before do
-      stub_terraform_state_object_storage(Terraform::StateUploader)
+      stub_terraform_state_object_storage
     end
 
     describe '#file' do

spec/requests/api/terraform/state_spec.rb

@@ -18,7 +18,7 @@ RSpec.describe API::Terraform::State do
   let(:state_path) { "/projects/#{project_id}/terraform/state/#{state_name}" }
 
   before do
-    stub_terraform_state_object_storage(Terraform::StateUploader)
+    stub_terraform_state_object_storage
   end
 
   describe 'GET /projects/:id/terraform/state/:name' do

spec/support/helpers/stub_object_storage.rb

@@ -82,9 +82,16 @@ module StubObjectStorage
                                  **params)
   end
 
-  def stub_terraform_state_object_storage(uploader = described_class, **params)
+  def stub_terraform_state_object_storage(**params)
     stub_object_storage_uploader(config: Gitlab.config.terraform_state.object_store,
-                                 uploader: uploader,
+                                 uploader: Terraform::VersionedStateUploader,
+                                 remote_directory: 'terraform',
+                                 **params)
+  end
+
+  def stub_terraform_state_version_object_storage(**params)
+    stub_object_storage_uploader(config: Gitlab.config.terraform_state.object_store,
+                                 uploader: Terraform::StateUploader,
                                  remote_directory: 'terraform',
                                  **params)
   end