Add variable expansion to cross pipeline artifacts

Variables can be specified when adding cross project dependencies with
needs:

```
build_job:
  stage: build
  script:
    - ls -lhR
  needs:
    - project: $CI_PROJECT_PATH
      job: $SOME_JOB_NAME_IN_OTHER_PROJECT
      ref: $CI_COMMIT_BRANCH
      artifacts: true
```

app/models/concerns/ci/contextable.rb

@@ -9,7 +9,7 @@ module Ci
     ##
     # Variables in the environment name scope.
     #
-    def scoped_variables(environment: expanded_environment_name)
+    def scoped_variables(environment: expanded_environment_name, dependencies: true)
       Gitlab::Ci::Variables::Collection.new.tap do |variables|
         variables.concat(predefined_variables)
         variables.concat(project.predefined_variables)
@@ -18,7 +18,7 @@ module Ci
         variables.concat(deployment_variables(environment: environment))
         variables.concat(yaml_variables)
         variables.concat(user_variables)
-        variables.concat(dependency_variables)
+        variables.concat(dependency_variables) if dependencies
         variables.concat(secret_instance_variables)
         variables.concat(secret_group_variables)
         variables.concat(secret_project_variables(environment: environment))
@@ -45,6 +45,12 @@ module Ci
       end
     end
 
+    def simple_variables_without_dependencies
+      strong_memoize(:variables_without_dependencies) do
+        scoped_variables(environment: nil, dependencies: false).to_runner_variables
+      end
+    end
+
     def user_variables
       Gitlab::Ci::Variables::Collection.new.tap do |variables|
         break variables if user.blank?

doc/ci/yaml/README.md

@@ -2134,6 +2134,26 @@ build_job:
       artifacts: true
 ```
 
+Environment variables support for `project:`, `job:`, and `ref` was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/202093)
+in GitLab 13.3. This is under development, but it is ready for production use. It is deployed
+behind the `ci_expand_names_for_cross_pipeline_artifacts` feature flag, which is **disabled by default**.
+[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md)
+can enable it for your instance.
+
+For example:
+
+```yaml
+build_job:
+  stage: build
+  script:
+    - ls -lhR
+  needs:
+    - project: $CI_PROJECT_PATH
+      job: $DEPENDENCY_JOB_NAME
+      ref: $CI_COMMIT_BRANCH
+      artifacts: true
+```
+
 NOTE: **Note:**
 Downloading artifacts from jobs that are run in [`parallel:`](#parallel) is not supported.
 

ee/app/models/ee/ci/build_dependencies.rb

@@ -52,8 +52,11 @@ module EE
 
       def build_cross_dependency_relationship_fragment(dependency, search_scope)
         args = dependency.values_at(:job, :ref, :project)
-        dep_id = search_scope.max_build_id_by(*args)
+        if ::Gitlab::Ci::Features.expand_names_for_cross_pipeline_artifacts?(project)
+          args = args.map { |value| ExpandVariables.expand(value, processable_variables) }
+        end
 
+        dep_id = search_scope.max_build_id_by(*args)
         model_class.id_in(dep_id)
       end
 
@@ -61,6 +64,10 @@ module EE
         processable.user
       end
 
+      def processable_variables
+        -> { processable.simple_variables_without_dependencies }
+      end
+
       def specified_cross_pipeline_dependencies
         Array(processable.options[:cross_dependencies])
       end

ee/changelogs/unreleased/add-variable-expansion-to-cross-pipeline-artifacts.yml

+---
+title: Add variable expansion to cross pipeline artifacts
+merge_request: 36578
+author:
+type: added

ee/spec/models/ee/ci/build_dependencies_spec.rb

@@ -86,6 +86,34 @@ RSpec.describe Ci::BuildDependencies do
 
         it { is_expected.to be_empty }
       end
+
+      context 'with dependency names from environment variables' do
+        before do
+          job.yaml_variables.push(key: 'DEPENDENCY_NAME', value: 'dependency', public: true)
+          job.save!
+        end
+
+        let(:dependencies) do
+          [
+            {
+              project: '$CI_PROJECT_PATH',
+              job: '$DEPENDENCY_NAME',
+              ref: '$CI_COMMIT_BRANCH',
+              artifacts: true
+            }
+          ]
+        end
+
+        it { is_expected.to contain_exactly(dependency) }
+
+        context 'with the feature flag disabled' do
+          before do
+            stub_feature_flags(ci_expand_names_for_cross_pipeline_artifacts: false)
+          end
+
+          it { is_expected.to be_empty }
+        end
+      end
     end
 
     context 'with cross_dependencies to another pipeline in same project' do

ee/spec/services/ci/create_pipeline_service/needs_spec.rb

@@ -80,4 +80,40 @@ RSpec.describe Ci::CreatePipelineService do
       expect(bridge_dag_job.status).to eq('pending')
     end
   end
+
+  context 'with cross pipeline artifacts' do
+    let!(:dependency) { create(:ci_build, :success, name: 'dependency', project: downstream_project) }
+    let!(:dependency_variable) { create(:ci_job_variable, :dotenv_source, job: dependency) }
+
+    let(:config) do
+      <<~EOY
+      regular_job:
+        stage: build
+        variables:
+          DEPENDENCY_PROJECT: #{downstream_project.full_path}
+          DEPENDENCY_REF: #{dependency.ref}
+          DEPENDENCY_NAME: #{dependency.name}
+        script:
+          - echo 'hello'
+        needs:
+          - project: ${DEPENDENCY_PROJECT}
+            ref: ${DEPENDENCY_REF}
+            job: ${DEPENDENCY_NAME}
+            artifacts: true
+      EOY
+    end
+
+    before do
+      stub_ci_pipeline_yaml_file(config)
+      stub_licensed_features(cross_project_pipelines: true)
+    end
+
+    it 'has dependencies and variables', :aggregate_failures do
+      job = execute.builds.first
+
+      expect(job).to be_present
+      expect(job.all_dependencies).to include(dependency)
+      expect(job.scoped_variables_hash).to include(dependency_variable.key => dependency_variable.value)
+    end
+  end
 end

lib/gitlab/ci/features.rb

@@ -79,6 +79,10 @@ module Gitlab
       def self.reset_ci_minutes_for_all_namespaces?
         ::Feature.enabled?(:reset_ci_minutes_for_all_namespaces, default_enabled: false)
       end
+
+      def self.expand_names_for_cross_pipeline_artifacts?(project)
+        ::Feature.enabled?(:ci_expand_names_for_cross_pipeline_artifacts, project)
+      end
     end
   end
 end

spec/models/ci/build_spec.rb

@@ -3111,6 +3111,14 @@ RSpec.describe Ci::Build do
     end
   end
 
+  describe '#simple_variables_without_dependencies' do
+    it 'does not load dependencies' do
+      expect(build).not_to receive(:dependency_variables)
+
+      build.simple_variables_without_dependencies
+    end
+  end
+
   shared_examples "secret CI variables" do
     context 'when ref is branch' do
       let(:build) { create(:ci_build, ref: 'master', tag: false, project: project) }