Commit 1eb18749 authored by Michael Kozono's avatar Michael Kozono

Merge branch 'nhxnguyen-geo-pages-replication' into 'master'

Add Geo replication for Pages (no verification) [RUN ALL RSPEC] [RUN AS-IF-FOSS]

See merge request gitlab-org/gitlab!68662
parents e34004a8 74a6db2a
......@@ -27,10 +27,6 @@ class PagesDeployment < ApplicationRecord
mount_file_store_uploader ::Pages::DeploymentUploader
def log_geo_deleted_event
# this is to be adressed in https://gitlab.com/groups/gitlab-org/-/epics/589
end
def migrated?
file.filename == MIGRATED_FILE_NAME
end
......@@ -41,3 +37,5 @@ class PagesDeployment < ApplicationRecord
self.size = file.size
end
end
PagesDeployment.prepend_mod
......@@ -25,6 +25,7 @@ ActiveSupport::Inflector.inflections do |inflect|
lfs_object_registry
merge_request_diff_registry
package_file_registry
pages_deployment_registry
pipeline_artifact_registry
project_auto_devops
project_registry
......
......@@ -252,6 +252,16 @@ configuration option in `gitlab.yml`. These metrics are served from the
| `geo_group_wiki_repositories_synced` | Gauge | 13.10 | Number of syncable group wikis synced on secondary | `url` |
| `geo_group_wiki_repositories_failed` | Gauge | 13.10 | Number of syncable group wikis failed on secondary | `url` |
| `geo_group_wiki_repositories_registry` | Gauge | 13.10 | Number of syncable group wikis in the registry | `url` |
| `geo_pages_deployments` | Gauge | 14.3 | Number of pages deployments on primary | `url` |
| `geo_pages_deployments_checksum_total` | Gauge | 14.3 | Number of pages deployments tried to checksum on primary | `url` |
| `geo_pages_deployments_checksummed` | Gauge | 14.3 | Number of pages deployments successfully checksummed on primary | `url` |
| `geo_pages_deployments_checksum_failed` | Gauge | 14.3 | Number of pages deployments failed to calculate the checksum on primary | `url` |
| `geo_pages_deployments_synced` | Gauge | 14.3 | Number of syncable pages deployments synced on secondary | `url` |
| `geo_pages_deployments_failed` | Gauge | 14.3 | Number of syncable pages deployments failed to sync on secondary | `url` |
| `geo_pages_deployments_registry` | Gauge | 14.3 | Number of pages deployments in the registry | `url` |
| `geo_pages_deployments_verification_total` | Gauge | 14.3 | Number of pages deployments verifications tried on secondary | `url` |
| `geo_pages_deployments_verified` | Gauge | 14.3 | Number of pages deployments verified on secondary | `url` |
| `geo_pages_deployments_verification_failed` | Gauge | 14.3 | Number of pages deployments verifications failed on secondary | `url` |
| `limited_capacity_worker_running_jobs` | Gauge | 13.5 | Number of running jobs | `worker` |
| `limited_capacity_worker_max_running_jobs` | Gauge | 13.5 | Maximum number of running jobs | `worker` |
| `limited_capacity_worker_remaining_work_count` | Gauge | 13.5 | Number of jobs waiting to be enqueued | `worker` |
......
......@@ -393,6 +393,18 @@ Example response:
"package_files_verification_failed_count": null,
"package_files_synced_in_percentage": "0.00%",
"package_files_verified_in_percentage": "0.00%",
"pages_deployments_count": 5,
"pages_deployments_checksum_total_count": 5,
"pages_deployments_checksummed_count": 5,
"pages_deployments_checksum_failed_count": 0,
"pages_deployments_synced_count": null,
"pages_deployments_failed_count": null,
"pages_deployments_registry_count": null,
"pages_deployments_verification_total_count": null,
"pages_deployments_verified_count": null,
"pages_deployments_verification_failed_count": null,
"pages_deployments_synced_in_percentage": "0.00%",
"pages_deployments_verified_in_percentage": "0.00%",
"terraform_state_versions_count": 5,
"terraform_state_versions_checksum_total_count": 5,
"terraform_state_versions_checksummed_count": 5,
......
......@@ -6450,6 +6450,29 @@ The edge type for [`PackageTag`](#packagetag).
| <a id="packagetagedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. |
| <a id="packagetagedgenode"></a>`node` | [`PackageTag`](#packagetag) | The item at the end of the edge. |
#### `PagesDeploymentRegistryConnection`
The connection type for [`PagesDeploymentRegistry`](#pagesdeploymentregistry).
##### Fields
| Name | Type | Description |
| ---- | ---- | ----------- |
| <a id="pagesdeploymentregistryconnectionedges"></a>`edges` | [`[PagesDeploymentRegistryEdge]`](#pagesdeploymentregistryedge) | A list of edges. |
| <a id="pagesdeploymentregistryconnectionnodes"></a>`nodes` | [`[PagesDeploymentRegistry]`](#pagesdeploymentregistry) | A list of nodes. |
| <a id="pagesdeploymentregistryconnectionpageinfo"></a>`pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. |
#### `PagesDeploymentRegistryEdge`
The edge type for [`PagesDeploymentRegistry`](#pagesdeploymentregistry).
##### Fields
| Name | Type | Description |
| ---- | ---- | ----------- |
| <a id="pagesdeploymentregistryedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. |
| <a id="pagesdeploymentregistryedgenode"></a>`node` | [`PagesDeploymentRegistry`](#pagesdeploymentregistry) | The item at the end of the edge. |
#### `PathLockConnection`
The connection type for [`PathLock`](#pathlock).
......@@ -9606,6 +9629,22 @@ four standard [pagination arguments](#connection-pagination-arguments):
| ---- | ---- | ----------- |
| <a id="geonodepackagefileregistriesids"></a>`ids` | [`[ID!]`](#id) | Filters registries by their ID. |
##### `GeoNode.pagesDeploymentRegistries`
Find Pages Deployment registries on this Geo node Available only when feature flag `geo_pages_deployment_replication` is enabled. This flag is disabled by default, because the feature is experimental and is subject to change without notice.
Returns [`PagesDeploymentRegistryConnection`](#pagesdeploymentregistryconnection).
This field returns a [connection](#connections). It accepts the
four standard [pagination arguments](#connection-pagination-arguments):
`before: String`, `after: String`, `first: Int`, `last: Int`.
###### Arguments
| Name | Type | Description |
| ---- | ---- | ----------- |
| <a id="geonodepagesdeploymentregistriesids"></a>`ids` | [`[ID!]`](#id) | Filters registries by their ID. |
##### `GeoNode.pipelineArtifactRegistries`
Find pipeline artifact registries on this Geo node.
......@@ -11791,6 +11830,23 @@ Information about pagination in a connection.
| <a id="pageinfohaspreviouspage"></a>`hasPreviousPage` | [`Boolean!`](#boolean) | When paginating backwards, are there more items?. |
| <a id="pageinfostartcursor"></a>`startCursor` | [`String`](#string) | When paginating backwards, the cursor to continue. |
### `PagesDeploymentRegistry`
Represents the Geo replication and verification state of a pages_deployment.
#### Fields
| Name | Type | Description |
| ---- | ---- | ----------- |
| <a id="pagesdeploymentregistrycreatedat"></a>`createdAt` | [`Time`](#time) | Timestamp when the PagesDeploymentRegistry was created. |
| <a id="pagesdeploymentregistryid"></a>`id` | [`ID!`](#id) | ID of the PagesDeploymentRegistry. |
| <a id="pagesdeploymentregistrylastsyncfailure"></a>`lastSyncFailure` | [`String`](#string) | Error message during sync of the PagesDeploymentRegistry. |
| <a id="pagesdeploymentregistrylastsyncedat"></a>`lastSyncedAt` | [`Time`](#time) | Timestamp of the most recent successful sync of the PagesDeploymentRegistry. |
| <a id="pagesdeploymentregistrypagesdeploymentid"></a>`pagesDeploymentId` | [`ID!`](#id) | ID of the Pages Deployment. |
| <a id="pagesdeploymentregistryretryat"></a>`retryAt` | [`Time`](#time) | Timestamp after which the PagesDeploymentRegistry should be resynced. |
| <a id="pagesdeploymentregistryretrycount"></a>`retryCount` | [`Int`](#int) | Number of consecutive failed sync attempts of the PagesDeploymentRegistry. |
| <a id="pagesdeploymentregistrystate"></a>`state` | [`RegistryState`](#registrystate) | Sync state of the PagesDeploymentRegistry. |
### `PathLock`
Represents a file or directory in the project repository that has been locked.
......
# frozen_string_literal: true
module Geo
class PagesDeploymentRegistryFinder
include FrameworkRegistryFinder
end
end
# frozen_string_literal: true
module Resolvers
module Geo
class PagesDeploymentRegistriesResolver < BaseResolver
type ::Types::Geo::GeoNodeType.connection_type, null: true
include RegistriesResolver
end
end
end
......@@ -50,6 +50,11 @@ module Types
null: true,
resolver: ::Resolvers::Geo::PipelineArtifactRegistriesResolver,
description: 'Find pipeline artifact registries on this Geo node.'
field :pages_deployment_registries, ::Types::Geo::PagesDeploymentRegistryType.connection_type,
null: true,
resolver: ::Resolvers::Geo::PagesDeploymentRegistriesResolver,
description: 'Find Pages Deployment registries on this Geo node',
feature_flag: :geo_pages_deployment_replication
end
end
end
# frozen_string_literal: true
module Types
module Geo
# rubocop:disable Graphql/AuthorizeTypes because it is included
class PagesDeploymentRegistryType < BaseObject
include ::Types::Geo::RegistryType
graphql_name 'PagesDeploymentRegistry'
description 'Represents the Geo replication and verification state of a pages_deployment'
field :pages_deployment_id, GraphQL::Types::ID, null: false, description: 'ID of the Pages Deployment.'
end
end
end
# frozen_string_literal: true
module EE
module PagesDeployment
extend ActiveSupport::Concern
prepended do
include ::Gitlab::Geo::ReplicableModel
with_replicator Geo::PagesDeploymentReplicator
end
class_methods do
def replicables_for_current_secondary(primary_key_in)
node = ::Gitlab::Geo.current_node
primary_key_in(primary_key_in)
.merge(selective_sync_scope(node))
.merge(object_storage_scope(node))
end
private
def object_storage_scope(node)
return all if node.sync_object_storage?
with_files_stored_locally
end
def selective_sync_scope(node)
return all unless node.selective_sync?
project_id_in(node.projects)
end
end
def log_geo_deleted_event
# Keep empty for now. Should be addressed in future
# by https://gitlab.com/gitlab-org/gitlab/-/issues/232917
end
end
end
# frozen_string_literal: true
class Geo::PagesDeploymentRegistry < Geo::BaseRegistry
include ::Geo::ReplicableRegistry
MODEL_CLASS = ::PagesDeployment
MODEL_FOREIGN_KEY = :pages_deployment_id
belongs_to :pages_deployment, class_name: 'PagesDeployment'
end
# frozen_string_literal: true
module Geo
class PagesDeploymentReplicator < Gitlab::Geo::Replicator
include ::Geo::BlobReplicatorStrategy
extend ::Gitlab::Utils::Override
def self.model
::PagesDeployment
end
def carrierwave_uploader
model_record.file
end
# The feature flag follows the format `geo_#{replicable_name}_replication`,
# so here it would be `geo_pages_deployment_replication`
def self.replication_enabled_by_default?
false
end
override :verification_feature_flag_enabled?
def self.verification_feature_flag_enabled?
false
end
end
end
......@@ -30,7 +30,8 @@ module Geo
Geo::TerraformStateVersionRegistry,
Geo::UploadRegistry,
Geo::SnippetRepositoryRegistry,
Geo::GroupWikiRepositoryRegistry
Geo::GroupWikiRepositoryRegistry,
Geo::PagesDeploymentRegistry
].freeze
BATCH_SIZE = 10000
......
---
name: geo_pages_deployment_replication
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68662
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/337676
milestone: '14.3'
type: development
group: group::geo
default_enabled: false
# frozen_string_literal: true
class CreatePagesDeploymentRegistry < ActiveRecord::Migration[6.1]
include Gitlab::Database::MigrationHelpers
def change
create_table :pages_deployment_registry, id: :bigserial, force: :cascade do |t|
t.bigint :pages_deployment_id, null: false
t.datetime_with_timezone :created_at, null: false
t.datetime_with_timezone :last_synced_at
t.datetime_with_timezone :retry_at
t.integer :state, default: 0, null: false, limit: 2
t.integer :retry_count, default: 0, limit: 2, null: false
t.string :last_sync_failure, limit: 255 # rubocop:disable Migration/PreventStrings see https://gitlab.com/gitlab-org/gitlab/-/issues/323806
t.index :pages_deployment_id, name: :index_pages_deployment_registry_on_pages_deployment_id, unique: true
t.index :retry_at
t.index :state
end
end
end
......@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 2021_06_24_160455) do
ActiveRecord::Schema.define(version: 2021_08_20_152707) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
......@@ -160,6 +160,19 @@ ActiveRecord::Schema.define(version: 2021_06_24_160455) do
t.index ["verified_at"], name: "package_file_registry_pending_verification", order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 0))"
end
create_table "pages_deployment_registry", force: :cascade do |t|
t.bigint "pages_deployment_id", null: false
t.datetime_with_timezone "created_at", null: false
t.datetime_with_timezone "last_synced_at"
t.datetime_with_timezone "retry_at"
t.integer "state", limit: 2, default: 0, null: false
t.integer "retry_count", limit: 2, default: 0, null: false
t.string "last_sync_failure", limit: 255
t.index ["pages_deployment_id"], name: "index_pages_deployment_registry_on_pages_deployment_id", unique: true
t.index ["retry_at"], name: "index_pages_deployment_registry_on_retry_at"
t.index ["state"], name: "index_pages_deployment_registry_on_state"
end
create_table "pipeline_artifact_registry", force: :cascade do |t|
t.bigint "pipeline_artifact_id", null: false
t.datetime_with_timezone "created_at", null: false
......
......@@ -26,7 +26,8 @@ module Gitlab
::Geo::TerraformStateVersionReplicator,
::Geo::SnippetRepositoryReplicator,
::Geo::GroupWikiRepositoryReplicator,
::Geo::PipelineArtifactReplicator
::Geo::PipelineArtifactReplicator,
::Geo::PagesDeploymentReplicator
].freeze
def self.current_node
......
# frozen_string_literal: true
FactoryBot.define do
factory :geo_pages_deployment_registry, class: 'Geo::PagesDeploymentRegistry' do
pages_deployment # This association should have data, like a file or repository
state { Geo::PagesDeploymentRegistry.state_value(:pending) }
trait :synced do
state { Geo::PagesDeploymentRegistry.state_value(:synced) }
last_synced_at { 5.days.ago }
end
trait :failed do
state { Geo::PagesDeploymentRegistry.state_value(:failed) }
last_synced_at { 1.day.ago }
retry_count { 2 }
last_sync_failure { 'Random error' }
end
trait :started do
state { Geo::PagesDeploymentRegistry.state_value(:started) }
last_synced_at { 1.day.ago }
retry_count { 0 }
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Geo::PagesDeploymentRegistryFinder do
it_behaves_like 'a framework registry finder', :geo_pages_deployment_registry
end
......@@ -78,6 +78,18 @@
"package_files_verification_total_count",
"package_files_verified_count",
"package_files_verified_in_percentage",
"pages_deployments_count",
"pages_deployments_checksum_failed_count",
"pages_deployments_checksum_total_count",
"pages_deployments_checksummed_count",
"pages_deployments_registry_count",
"pages_deployments_failed_count",
"pages_deployments_synced_count",
"pages_deployments_synced_in_percentage",
"pages_deployments_verification_failed_count",
"pages_deployments_verification_total_count",
"pages_deployments_verified_count",
"pages_deployments_verified_in_percentage",
"terraform_state_versions_count",
"terraform_state_versions_checksum_failed_count",
"terraform_state_versions_checksum_total_count",
......@@ -244,6 +256,18 @@
"package_files_verification_total_count": { "type": ["integer", "null"] },
"package_files_verified_count": { "type": ["integer", "null"] },
"package_files_verified_in_percentage": { "type": "string" },
"pages_deployments_count": { "type": ["integer", "null"] },
"pages_deployments_checksummed_count": { "type": ["integer", "null"] },
"pages_deployments_checksum_failed_count": { "type": ["integer", "null"] },
"pages_deployments_checksum_total_count": { "type": ["integer", "null"] },
"pages_deployments_registry_count": { "type": ["integer", "null"] },
"pages_deployments_failed_count": { "type": ["integer", "null"] },
"pages_deployments_synced_count": { "type": ["integer", "null"] },
"pages_deployments_synced_in_percentage": { "type": "string" },
"pages_deployments_verification_failed_count": { "type": ["integer", "null"] },
"pages_deployments_verification_total_count": { "type": ["integer", "null"] },
"pages_deployments_verified_count": { "type": ["integer", "null"] },
"pages_deployments_verified_in_percentage": { "type": "string" },
"terraform_state_versions_count": { "type": ["integer", "null"] },
"terraform_state_versions_checksummed_count": { "type": ["integer", "null"] },
"terraform_state_versions_checksum_failed_count": { "type": ["integer", "null"] },
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Resolvers::Geo::PagesDeploymentRegistriesResolver do
it_behaves_like 'a Geo registries resolver', :geo_pages_deployment_registry
end
......@@ -16,6 +16,7 @@ RSpec.describe GitlabSchema.types['GeoNode'] do
terraform_state_version_registries group_wiki_repository_registries
lfs_object_registries
pipeline_artifact_registries
pages_deployment_registries
]
expect(described_class).to have_graphql_fields(*expected_fields)
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe GitlabSchema.types['PagesDeploymentRegistry'] do
it_behaves_like 'a Geo registry type'
it 'has the expected fields (other than those included in RegistryType)' do
expected_fields = %i[pages_deployment_id]
expect(described_class).to have_graphql_fields(*expected_fields).at_least
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Geo::PagesDeploymentRegistry, :geo, type: :model do
let_it_be(:registry) { create(:geo_pages_deployment_registry) }
specify 'factory is valid' do
expect(registry).to be_valid
end
include_examples 'a Geo framework registry'
end
......@@ -1129,6 +1129,7 @@ RSpec.describe GeoNodeStatus, :geo do
Geo::TerraformStateVersionReplicator | :terraform_state_version | :geo_terraform_state_version_registry
Geo::SnippetRepositoryReplicator | :snippet_repository | :geo_snippet_repository_registry
Geo::GroupWikiRepositoryReplicator | :group_wiki_repository | :geo_group_wiki_repository_registry
Geo::PagesDeploymentReplicator | :pages_deployment | :geo_pages_deployment_registry
end
with_them do
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Geo::PagesDeploymentReplicator do
let(:model_record) { build(:pages_deployment) }
include_examples 'a blob replicator'
end
......@@ -44,4 +44,11 @@ RSpec.describe 'Gets registries' do
registry_factory: :geo_pipeline_artifact_registry,
registry_foreign_key_field_name: 'pipelineArtifactId'
}
it_behaves_like 'gets registries for', {
field_name: 'pagesDeploymentRegistries',
registry_class_name: 'PagesDeploymentRegistry',
registry_factory: :geo_pages_deployment_registry,
registry_foreign_key_field_name: 'pagesDeploymentId'
}
end
......@@ -87,6 +87,7 @@ RSpec.describe Geo::Secondary::RegistryConsistencyWorker, :geo do
terraform_state_version = create(:terraform_state_version)
pipeline_artifact = create(:ci_pipeline_artifact)
upload = create(:upload)
pages_deployment = create(:pages_deployment)
expect(Geo::ContainerRepositoryRegistry.where(container_repository_id: container_repository.id).count).to eq(0)
expect(Geo::DesignRegistry.where(project_id: project.id).count).to eq(0)
......@@ -98,6 +99,7 @@ RSpec.describe Geo::Secondary::RegistryConsistencyWorker, :geo do
expect(Geo::ProjectRegistry.where(project_id: project.id).count).to eq(0)
expect(Geo::TerraformStateVersionRegistry.where(terraform_state_version_id: terraform_state_version.id).count).to eq(0)
expect(Geo::UploadRegistry.where(file_id: upload.id).count).to eq(0)
expect(Geo::PagesDeploymentRegistry.where(pages_deployment: pages_deployment.id).count).to eq(0)
subject.perform
......@@ -111,6 +113,7 @@ RSpec.describe Geo::Secondary::RegistryConsistencyWorker, :geo do
expect(Geo::ProjectRegistry.where(project_id: project.id).count).to eq(1)
expect(Geo::TerraformStateVersionRegistry.where(terraform_state_version_id: terraform_state_version.id).count).to eq(1)
expect(Geo::UploadRegistry.where(file_id: upload.id).count).to eq(1)
expect(Geo::PagesDeploymentRegistry.where(pages_deployment: pages_deployment.id).count).to eq(1)
end
context 'when the current Geo node is disabled or primary' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment