Configure Pipeline validation Service with ApplicationSetting

We don't have any other secrets (the token) sourced from environment
vars currently so it seems to me we shouldn't start doing that now.
It also complicates or confuses the helmcharts configuration (and
omnibus too), and having all this configuration in the DB simplifies
deployment immensely

app/helpers/application_settings_helper.rb

@@ -229,6 +229,9 @@ module ApplicationSettingsHelper
       :email_author_in_body,
       :enabled_git_access_protocol,
       :enforce_terms,
+      :external_pipeline_validation_service_timeout,
+      :external_pipeline_validation_service_token,
+      :external_pipeline_validation_service_url,
       :first_day_of_week,
       :force_pages_access_control,
       :gitaly_timeout_default,

app/models/application_setting.rb

@@ -468,6 +468,13 @@ class ApplicationSetting < ApplicationRecord
   validates :admin_mode,
             inclusion: { in: [true, false], message: _('must be a boolean value') }
 
+  validates :external_pipeline_validation_service_url,
+            addressable_url: true, allow_blank: true
+
+  validates :external_pipeline_validation_service_timeout,
+            allow_nil: true,
+            numericality: { only_integer: true, greater_than: 0 }
+
   attr_encrypted :asset_proxy_secret_key,
                  mode: :per_attribute_iv,
                  key: Settings.attr_encrypted_db_key_base_truncated,
@@ -496,6 +503,7 @@ class ApplicationSetting < ApplicationRecord
   attr_encrypted :ci_jwt_signing_key, encryption_options_base_truncated_aes_256_gcm
   attr_encrypted :secret_detection_token_revocation_token, encryption_options_base_truncated_aes_256_gcm
   attr_encrypted :cloud_license_auth_token, encryption_options_base_truncated_aes_256_gcm
+  attr_encrypted :external_pipeline_validation_service_token, encryption_options_base_truncated_aes_256_gcm
 
   validates :disable_feed_token,
             inclusion: { in: [true, false], message: _('must be a boolean value') }

app/models/application_setting_implementation.rb

@@ -72,6 +72,9 @@ module ApplicationSettingImplementation
         eks_secret_access_key: nil,
         email_restrictions_enabled: false,
         email_restrictions: nil,
+        external_pipeline_validation_service_timeout: nil,
+        external_pipeline_validation_service_token: nil,
+        external_pipeline_validation_service_url: nil,
         first_day_of_week: 0,
         gitaly_timeout_default: 55,
         gitaly_timeout_fast: 10,

changelogs/unreleased/validation-service-token-to-config.yml

+---
+title: Obtain pipeline validation service token from config not ENV.
+merge_request: 59101
+author:
+type: other

db/migrate/20210414131600_add_external_pipeline_validation_to_application_setting.rb

+# frozen_string_literal: true
+
+class AddExternalPipelineValidationToApplicationSetting < ActiveRecord::Migration[6.0]
+  def up
+    add_column :application_settings, :external_pipeline_validation_service_timeout, :integer
+    # rubocop:disable Migration/AddLimitToTextColumns
+    add_column :application_settings, :encrypted_external_pipeline_validation_service_token, :text
+    add_column :application_settings, :encrypted_external_pipeline_validation_service_token_iv, :text
+    add_column :application_settings, :external_pipeline_validation_service_url, :text
+    # rubocop:enable Migration/AddLimitToTextColumns
+  end
+
+  def down
+    remove_column :application_settings, :external_pipeline_validation_service_timeout
+    remove_column :application_settings, :encrypted_external_pipeline_validation_service_token
+    remove_column :application_settings, :encrypted_external_pipeline_validation_service_token_iv
+    remove_column :application_settings, :external_pipeline_validation_service_url
+  end
+end

db/migrate/20210415142700_add_url_limit_to_pipeline_validation.rb

+# frozen_string_literal: true
+
+class AddUrlLimitToPipelineValidation < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  CONSTRAINT_NAME = 'app_settings_ext_pipeline_validation_service_url_text_limit'
+
+  def up
+    add_text_limit :application_settings, :external_pipeline_validation_service_url, 255, constraint_name: CONSTRAINT_NAME
+  end
+
+  def down
+    remove_check_constraint(:application_settings, CONSTRAINT_NAME)
+  end
+end

db/schema_migrations/20210414131600

+199c8a540cb4a0dd30a86a81f993798afb3e7384f1176b71a780d5950a52eb5f
\ No newline at end of file

db/schema_migrations/20210415142700

+2d6d62b036c937136dfbb11becfd3c2c705f0db1e3a38fdcefe676106168ab29
\ No newline at end of file

db/structure.sql

@@ -9441,7 +9441,12 @@ CREATE TABLE application_settings (
     admin_mode boolean DEFAULT false NOT NULL,
     delayed_project_removal boolean DEFAULT false NOT NULL,
     lock_delayed_project_removal boolean DEFAULT false NOT NULL,
+    external_pipeline_validation_service_timeout integer,
+    encrypted_external_pipeline_validation_service_token text,
+    encrypted_external_pipeline_validation_service_token_iv text,
+    external_pipeline_validation_service_url text,
     CONSTRAINT app_settings_container_reg_cleanup_tags_max_list_size_positive CHECK ((container_registry_cleanup_tags_service_max_list_size >= 0)),
+    CONSTRAINT app_settings_ext_pipeline_validation_service_url_text_limit CHECK ((char_length(external_pipeline_validation_service_url) <= 255)),
     CONSTRAINT app_settings_registry_exp_policies_worker_capacity_positive CHECK ((container_registry_expiration_policies_worker_capacity >= 0)),
     CONSTRAINT check_17d9558205 CHECK ((char_length((kroki_url)::text) <= 1024)),
     CONSTRAINT check_2dba05b802 CHECK ((char_length(gitpod_url) <= 255)),

doc/api/settings.md

@@ -87,7 +87,10 @@ Example response:
   "personal_access_token_prefix": "GL-",
   "rate_limiting_response_text": null,
   "keep_latest_artifact": true,
-  "admin_mode": false
+  "admin_mode": false,
+  "external_pipeline_validation_service_timeout": null,
+  "external_pipeline_validation_service_token": null,
+  "external_pipeline_validation_service_url": null
 }
 ```
 
@@ -183,7 +186,10 @@ Example response:
   "personal_access_token_prefix": "GL-",
   "rate_limiting_response_text": null,
   "keep_latest_artifact": true,
-  "admin_mode": false
+  "admin_mode": false,
+  "external_pipeline_validation_service_timeout": null,
+  "external_pipeline_validation_service_token": null,
+  "external_pipeline_validation_service_url": null
 }
 ```
 
@@ -283,6 +289,9 @@ listed in the descriptions of the relevant settings.
 | `external_authorization_service_enabled` | boolean          | no                                   | (**If enabled, requires:** `external_authorization_service_default_label`, `external_authorization_service_timeout` and `external_authorization_service_url`) Enable using an external authorization service for accessing projects |
 | `external_authorization_service_timeout` | float            | required by:<br>`external_authorization_service_enabled` | The timeout after which an authorization request is aborted, in seconds. When a request times out, access is denied to the user. (min: 0.001, max: 10, step: 0.001). |
 | `external_authorization_service_url`     | string           | required by:<br>`external_authorization_service_enabled` | URL to which authorization requests are directed. |
+| `external_pipeline_validation_service_url`  | string           | no | URL to which pipeline validation requests are directed. |
+| `external_pipeline_validation_service_token` | string           | no | An optional token to include as the `X-Gitlab-Token` header in requests to the URL in external_pipeline_validation_service_url. |
+| `external_pipeline_validation_service_timeout`  | integer           | no | How long to wait for a response from the pipeline validation service before giving up and assuming 'OK'. |
 | `file_template_project_id`               | integer          | no                                   | **(PREMIUM)** The ID of a project to load custom file templates from |
 | `first_day_of_week`                      | integer          | no                                   | Start day of the week for calendar views and date pickers. Valid values are `0` (default) for Sunday, `1` for Monday, and `6` for Saturday. |
 | `geo_node_allowed_ips`                   | string           | yes                                  | **(PREMIUM)** Comma-separated list of IPs and CIDRs of allowed secondary nodes. For example, `1.1.1.1, 2.2.2.0/24`. |

lib/gitlab/ci/pipeline/chain/validate/external.rb

@@ -82,18 +82,18 @@ module Gitlab
             end
 
             def validation_service_timeout
-              timeout = ENV['EXTERNAL_VALIDATION_SERVICE_TIMEOUT'].to_i
+              timeout = Gitlab::CurrentSettings.external_pipeline_validation_service_timeout || ENV['EXTERNAL_VALIDATION_SERVICE_TIMEOUT'].to_i
               return timeout if timeout > 0
 
               DEFAULT_VALIDATION_REQUEST_TIMEOUT
             end
 
             def validation_service_url
-              ENV['EXTERNAL_VALIDATION_SERVICE_URL']
+              Gitlab::CurrentSettings.external_pipeline_validation_service_url || ENV['EXTERNAL_VALIDATION_SERVICE_URL']
             end
 
             def validation_service_token
-              ENV['EXTERNAL_VALIDATION_SERVICE_TOKEN']
+              Gitlab::CurrentSettings.external_pipeline_validation_service_token || ENV['EXTERNAL_VALIDATION_SERVICE_TOKEN']
             end
 
             def validation_service_payload

spec/lib/gitlab/ci/pipeline/chain/validate/external_spec.rb

@@ -60,6 +60,30 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Validate::External do
       allow(Labkit::Correlation::CorrelationId).to receive(:current_id).and_return('correlation-id')
     end
 
+    context 'with configuration values in ApplicationSetting' do
+      let(:alternate_validation_service_url) { 'https://alternate-validation-service.external/' }
+      let(:validation_service_token) { 'SECURE_TOKEN' }
+      let(:shorter_timeout) { described_class::DEFAULT_VALIDATION_REQUEST_TIMEOUT - 1 }
+
+      before do
+        stub_env('EXTERNAL_VALIDATION_SERVICE_TOKEN', 'TOKEN_IN_ENV')
+        allow(Gitlab::CurrentSettings.current_application_settings).to receive(:external_pipeline_validation_service_timeout).and_return(shorter_timeout)
+        allow(Gitlab::CurrentSettings.current_application_settings).to receive(:external_pipeline_validation_service_token).and_return(validation_service_token)
+        allow(Gitlab::CurrentSettings.current_application_settings).to receive(:external_pipeline_validation_service_url).and_return(alternate_validation_service_url)
+      end
+
+      it 'uses those values rather than env vars or defaults' do
+        expect(::Gitlab::HTTP).to receive(:post) do |url, params|
+          expect(url).to eq(alternate_validation_service_url)
+          expect(params[:timeout]).to eq(shorter_timeout)
+          expect(params[:headers]).to include('X-Gitlab-Token' => validation_service_token)
+          expect(params[:timeout]).to eq(shorter_timeout)
+        end
+
+        perform!
+      end
+    end
+
     it 'respects the defined payload schema' do
       expect(::Gitlab::HTTP).to receive(:post) do |_url, params|
         expect(params[:body]).to match_schema('/external_validation')