Use CI instance variables on jobs

Adds support for using instance level CI variables for builds.
These variables can be overriden by group/project variables.

app/models/ci/instance_variable.rb

@@ -13,5 +13,64 @@ module Ci
     }
 
     scope :unprotected, -> { where(protected: false) }
+    after_commit { self.class.touch_redis_cache_timestamp }
+
+    class << self
+      def all_cached
+        cached_data[:all]
+      end
+
+      def unprotected_cached
+        cached_data[:unprotected]
+      end
+
+      def touch_redis_cache_timestamp(time = Time.current.to_f)
+        shared_backend.write(:ci_instance_variable_changed_at, time)
+      end
+
+      private
+
+      def cached_data
+        fetch_memory_cache(:ci_instance_variable_data) do
+          all_records = unscoped.all.to_a
+
+          { all: all_records, unprotected: all_records.reject(&:protected?) }
+        end
+      end
+
+      def fetch_memory_cache(key, &payload)
+        cache = process_backend.read(key)
+
+        if cache && !stale_cache?(cache)
+          cache[:data]
+        else
+          store_cache(key, &payload)
+        end
+      end
+
+      def stale_cache?(cache_info)
+        shared_timestamp = shared_backend.read(:ci_instance_variable_changed_at)
+        return true unless shared_timestamp
+
+        shared_timestamp.to_f > cache_info[:cached_at].to_f
+      end
+
+      def store_cache(key)
+        data = yield
+        time = Time.current.to_f
+
+        process_backend.write(key, data: data, cached_at: time)
+        touch_redis_cache_timestamp(time)
+        data
+      end
+
+      def shared_backend
+        Rails.cache
+      end
+
+      def process_backend
+        Gitlab::ProcessMemoryCache.cache_backend
+      end
+    end
   end
 end

app/models/concerns/ci/contextable.rb

@@ -19,6 +19,7 @@ module Ci
         variables.concat(yaml_variables)
         variables.concat(user_variables)
         variables.concat(dependency_variables) if Feature.enabled?(:ci_dependency_variables, project)
+        variables.concat(secret_instance_variables)
         variables.concat(secret_group_variables)
         variables.concat(secret_project_variables(environment: environment))
         variables.concat(trigger_request.user_variables) if trigger_request
@@ -82,6 +83,12 @@ module Ci
       )
     end
 
+    def secret_instance_variables
+      return [] unless ::Feature.enabled?(:ci_instance_level_variables, project, default_enabled: true)
+
+      project.ci_instance_variables_for(ref: git_ref)
+    end
+
     def secret_group_variables
       return [] unless project.group
 

app/models/project.rb

@@ -2018,6 +2018,14 @@ class Project < ApplicationRecord
     end
   end
 
+  def ci_instance_variables_for(ref:)
+    if protected_for?(ref)
+      Ci::InstanceVariable.all_cached
+    else
+      Ci::InstanceVariable.unprotected_cached
+    end
+  end
+
   def protected_for?(ref)
     raise Repository::AmbiguousRefError if repository.ambiguous_ref?(ref)
 

changelogs/unreleased/14108-instance-level-ci-variables-logic.yml

+---
+title: Integrate CI instance variables in the build process
+merge_request: 30186
+author:
+type: added

spec/models/ci/build_spec.rb

@@ -3117,11 +3117,7 @@ describe Ci::Build do
     end
   end
 
-  describe '#secret_group_variables' do
-    subject { build.secret_group_variables }
-
-    let!(:variable) { create(:ci_group_variable, protected: true, group: group) }
-
+  shared_examples "secret CI variables" do
     context 'when ref is branch' do
       let(:build) { create(:ci_build, ref: 'master', tag: false, project: project) }
 
@@ -3175,62 +3171,28 @@ describe Ci::Build do
     end
   end
 
-  describe '#secret_project_variables' do
-    subject { build.secret_project_variables }
-
-    let!(:variable) { create(:ci_variable, protected: true, project: project) }
-
-    context 'when ref is branch' do
-      let(:build) { create(:ci_build, ref: 'master', tag: false, project: project) }
+  describe '#secret_instance_variables' do
+    subject { build.secret_instance_variables }
 
-      context 'when ref is protected' do
-        before do
-          create(:protected_branch, :developers_can_merge, name: 'master', project: project)
-        end
+    let_it_be(:variable) { create(:ci_instance_variable, protected: true) }
 
-        it { is_expected.to include(variable) }
+    include_examples "secret CI variables"
   end
 
-      context 'when ref is not protected' do
-        it { is_expected.not_to include(variable) }
-      end
-    end
-
-    context 'when ref is tag' do
-      let(:build) { create(:ci_build, ref: 'v1.1.0', tag: true, project: project) }
-
-      context 'when ref is protected' do
-        before do
-          create(:protected_tag, project: project, name: 'v*')
-        end
+  describe '#secret_group_variables' do
+    subject { build.secret_group_variables }
 
-        it { is_expected.to include(variable) }
-      end
+    let_it_be(:variable) { create(:ci_group_variable, protected: true, group: group) }
 
-      context 'when ref is not protected' do
-        it { is_expected.not_to include(variable) }
-      end
+    include_examples "secret CI variables"
   end
 
-    context 'when ref is merge request' do
-      let(:merge_request) { create(:merge_request, :with_detached_merge_request_pipeline) }
-      let(:pipeline) { merge_request.pipelines_for_merge_request.first }
-      let(:build) { create(:ci_build, ref: merge_request.source_branch, tag: false, pipeline: pipeline, project: project) }
-
-      context 'when ref is protected' do
-        before do
-          create(:protected_branch, :developers_can_merge, name: merge_request.source_branch, project: project)
-        end
+  describe '#secret_project_variables' do
+    subject { build.secret_project_variables }
 
-        it 'does not return protected variables as it is not supported for merge request pipelines' do
-          is_expected.not_to include(variable)
-        end
-      end
+    let_it_be(:variable) { create(:ci_variable, protected: true, project: project) }
 
-      context 'when ref is not protected' do
-        it { is_expected.not_to include(variable) }
-      end
-    end
+    include_examples "secret CI variables"
   end
 
   describe '#deployment_variables' do
@@ -3283,6 +3245,29 @@ describe Ci::Build do
         expect(build.scoped_variables_hash).not_to include('MY_VAR': 'myvar')
       end
     end
+
+    context 'when overriding CI instance variables' do
+      before do
+        create(:ci_instance_variable, key: 'MY_VAR', value: 'my value 1')
+        group.variables.create!(key: 'MY_VAR', value: 'my value 2')
+      end
+
+      it 'returns a regular hash created using valid ordering' do
+        expect(build.scoped_variables_hash).to include('MY_VAR': 'my value 2')
+        expect(build.scoped_variables_hash).not_to include('MY_VAR': 'my value 1')
+      end
+    end
+
+    context 'when CI instance variables are disabled' do
+      before do
+        create(:ci_instance_variable, key: 'MY_VAR', value: 'my value 1')
+        stub_feature_flags(ci_instance_level_variables: false)
+      end
+
+      it 'does not include instance level variables' do
+        expect(build.scoped_variables_hash).not_to include('MY_VAR': 'my value 1')
+      end
+    end
   end
 
   describe '#any_unmet_prerequisites?' do

spec/models/ci/instance_variable_spec.rb

@@ -31,4 +31,63 @@ describe Ci::InstanceVariable do
       end
     end
   end
+
+  describe '.all_cached', :use_clean_rails_memory_store_caching do
+    let_it_be(:unprotected_variable) { create(:ci_instance_variable, protected: false) }
+    let_it_be(:protected_variable) { create(:ci_instance_variable, protected: true) }
+
+    it { expect(described_class.all_cached).to contain_exactly(protected_variable, unprotected_variable) }
+
+    it 'memoizes the result' do
+      expect(described_class).to receive(:store_cache).with(:ci_instance_variable_data).once.and_call_original
+
+      2.times do
+        expect(described_class.all_cached).to contain_exactly(protected_variable, unprotected_variable)
+      end
+    end
+
+    it 'removes scopes' do
+      expect(described_class.unprotected.all_cached).to contain_exactly(protected_variable, unprotected_variable)
+    end
+
+    it 'resets the cache when records are deleted' do
+      expect(described_class.all_cached).to contain_exactly(protected_variable, unprotected_variable)
+
+      protected_variable.destroy
+
+      expect(described_class.all_cached).to contain_exactly(unprotected_variable)
+    end
+
+    it 'resets the cache when records are inserted' do
+      expect(described_class.all_cached).to contain_exactly(protected_variable, unprotected_variable)
+
+      variable = create(:ci_instance_variable, protected: true)
+
+      expect(described_class.all_cached).to contain_exactly(protected_variable, unprotected_variable, variable)
+    end
+
+    it 'resets the cache when the shared key is missing' do
+      expect(Rails.cache).to receive(:read).with(:ci_instance_variable_changed_at).twice.and_return(nil)
+      expect(described_class).to receive(:store_cache).with(:ci_instance_variable_data).thrice.and_call_original
+
+      3.times do
+        expect(described_class.all_cached).to contain_exactly(protected_variable, unprotected_variable)
+      end
+    end
+  end
+
+  describe '.unprotected_cached', :use_clean_rails_memory_store_caching do
+    let_it_be(:unprotected_variable) { create(:ci_instance_variable, protected: false) }
+    let_it_be(:protected_variable) { create(:ci_instance_variable, protected: true) }
+
+    it { expect(described_class.unprotected_cached).to contain_exactly(unprotected_variable) }
+
+    it 'memoizes the result' do
+      expect(described_class).to receive(:store_cache).with(:ci_instance_variable_data).once.and_call_original
+
+      2.times do
+        expect(described_class.unprotected_cached).to contain_exactly(unprotected_variable)
+      end
+    end
+  end
 end

spec/models/project_spec.rb

@@ -3138,6 +3138,45 @@ describe Project do
     end
   end
 
+  describe '#ci_instance_variables_for' do
+    let(:project) { create(:project) }
+
+    let!(:instance_variable) do
+      create(:ci_instance_variable, value: 'secret')
+    end
+
+    let!(:protected_instance_variable) do
+      create(:ci_instance_variable, :protected, value: 'protected')
+    end
+
+    subject { project.ci_instance_variables_for(ref: 'ref') }
+
+    before do
+      stub_application_setting(
+        default_branch_protection: Gitlab::Access::PROTECTION_NONE)
+    end
+
+    context 'when the ref is not protected' do
+      before do
+        allow(project).to receive(:protected_for?).with('ref').and_return(false)
+      end
+
+      it 'contains only the CI variables' do
+        is_expected.to contain_exactly(instance_variable)
+      end
+    end
+
+    context 'when the ref is protected' do
+      before do
+        allow(project).to receive(:protected_for?).with('ref').and_return(true)
+      end
+
+      it 'contains all the variables' do
+        is_expected.to contain_exactly(instance_variable, protected_instance_variable)
+      end
+    end
+  end
+
   describe '#any_lfs_file_locks?', :request_store do
     let_it_be(:project) { create(:project) }