Merge branch 'ci-predefined-variables' into 'master'

Add predefined CI variables to GitLab

## What does this MR do?
This adds predefined CI variables to GitLab for container registry, pipelines, project name, etc. It also makes sure that all currently documented variables are send from GitLab. This is added to follow up on this proposal: https://gitlab.com/gitlab-org/gitlab-ci-multi-runner/issues/185#note_11844286. To migrate almost all variables out of Runner to GitLab to simplify adding a new of variables without the need for changing the GitLab Runner.

## Why was this MR needed?
Our CI variables miss a lot of crucial information that should be easily accessible. This tries to fill this gap.

## What are the relevant issue numbers?
Fixes https://gitlab.com/gitlab-org/gitlab-ci-multi-runner/issues/185, https://gitlab.com/gitlab-org/gitlab-ce/issues/18164, https://gitlab.com/gitlab-org/gitlab-ce/issues/18075.

## Does this MR meet the acceptance criteria?

- [ ] [CHANGELOG](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CHANGELOG) entry added
- [ ] [Documentation created/updated](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/development/doc_styleguide.md)
- [ ] API support added
- Tests
  - [ ] Added for this feature/bug
  - [ ] All builds are passing
- [ ] Conform by the [style guides](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CONTRIBUTING.md#style-guides)
- [ ] Branch has no merge conflicts with `master` (if you do - rebase it please)
- [ ] [Squashed related commits together](https://git-scm.com/book/en/Git-Tools-Rewriting-History#Squashing-Commits)


See merge request !4826

CHANGELOG

@@ -27,6 +27,7 @@ v 8.10.0 (unreleased)
   - Escape file extension when parsing search results !5141 (winniehell)
   - Apply the trusted_proxies config to the rack request object for use with rack_attack
   - Upgrade to Rails 4.2.7. !5236
+  - Extend exposed environment variables for CI builds
   - Allow to pull code with deploy key from public projects
   - Add Sidekiq queue duration to transaction metrics.
   - Add a new column `artifacts_size` to table `ci_builds` !4964

app/models/ci/build.rb

@@ -145,7 +145,15 @@ module Ci
     end
 
     def variables
-      predefined_variables + yaml_variables + project_variables + trigger_variables
+      variables = predefined_variables
+      variables += project.predefined_variables
+      variables += pipeline.predefined_variables
+      variables += runner.predefined_variables if runner
+      variables += project.container_registry_variables
+      variables += yaml_variables
+      variables += project.secret_variables
+      variables += trigger_request.user_variables if trigger_request
+      variables
     end
 
     def merge_request
@@ -430,28 +438,23 @@ module Ci
       self.update(erased_by: user, erased_at: Time.now, artifacts_expire_at: nil)
     end
 
-    def project_variables
-      project.variables.map do |variable|
-        { key: variable.key, value: variable.value, public: false }
-      end
-    end
-
-    def trigger_variables
-      if trigger_request && trigger_request.variables
-        trigger_request.variables.map do |key, value|
-          { key: key, value: value, public: false }
-        end
-      else
-        []
-      end
-    end
-
     def predefined_variables
-      variables = []
-      variables << { key: :CI_BUILD_TAG, value: ref, public: true } if tag?
-      variables << { key: :CI_BUILD_NAME, value: name, public: true }
-      variables << { key: :CI_BUILD_STAGE, value: stage, public: true }
-      variables << { key: :CI_BUILD_TRIGGERED, value: 'true', public: true } if trigger_request
+      variables = [
+        { key: 'CI', value: 'true', public: true },
+        { key: 'GITLAB_CI', value: 'true', public: true },
+        { key: 'CI_BUILD_ID', value: id.to_s, public: true },
+        { key: 'CI_BUILD_TOKEN', value: token, public: false },
+        { key: 'CI_BUILD_REF', value: sha, public: true },
+        { key: 'CI_BUILD_BEFORE_SHA', value: before_sha, public: true },
+        { key: 'CI_BUILD_REF_NAME', value: ref, public: true },
+        { key: 'CI_BUILD_NAME', value: name, public: true },
+        { key: 'CI_BUILD_STAGE', value: stage, public: true },
+        { key: 'CI_SERVER_NAME', value: 'GitLab', public: true },
+        { key: 'CI_SERVER_VERSION', value: Gitlab::VERSION, public: true },
+        { key: 'CI_SERVER_REVISION', value: Gitlab::REVISION, public: true }
+      ]
+      variables << { key: 'CI_BUILD_TAG', value: ref, public: true } if tag?
+      variables << { key: 'CI_BUILD_TRIGGERED', value: 'true', public: true } if trigger_request
       variables
     end
 

app/models/ci/pipeline.rb

@@ -198,6 +198,12 @@ module Ci
       Note.for_commit_id(sha)
     end
 
+    def predefined_variables
+      [
+        { key: 'CI_PIPELINE_ID', value: id.to_s, public: true }
+      ]
+    end
+
     private
 
     def build_builds_for_stages(stages, user, status, trigger_request)

app/models/ci/runner.rb

@@ -114,6 +114,14 @@ module Ci
       tag_list.any?
     end
 
+    def predefined_variables
+      [
+        { key: 'CI_RUNNER_ID', value: id.to_s, public: true },
+        { key: 'CI_RUNNER_DESCRIPTION', value: description, public: true },
+        { key: 'CI_RUNNER_TAGS', value: tag_list.to_s, public: true }
+      ]
+    end
+
     private
 
     def tag_constraints

app/models/ci/trigger_request.rb

@@ -7,5 +7,13 @@ module Ci
     has_many :builds, class_name: 'Ci::Build'
 
     serialize :variables
+
+    def user_variables
+      return [] unless variables
+
+      variables.map do |key, value|
+        { key: key, value: value, public: false }
+      end
+    end
   end
 end

app/models/project.rb

@@ -1180,4 +1180,34 @@ class Project < ActiveRecord::Base
   def ensure_dir_exist
     gitlab_shell.add_namespace(repository_storage_path, namespace.path)
   end
+
+  def predefined_variables
+    [
+      { key: 'CI_PROJECT_ID', value: id.to_s, public: true },
+      { key: 'CI_PROJECT_NAME', value: path, public: true },
+      { key: 'CI_PROJECT_PATH', value: path_with_namespace, public: true },
+      { key: 'CI_PROJECT_NAMESPACE', value: namespace.path, public: true },
+      { key: 'CI_PROJECT_URL', value: web_url, public: true }
+    ]
+  end
+
+  def container_registry_variables
+    return [] unless Gitlab.config.registry.enabled
+
+    variables = [
+      { key: 'CI_REGISTRY', value: Gitlab.config.registry.host_port, public: true }
+    ]
+
+    if container_registry_enabled?
+      variables << { key: 'CI_REGISTRY_IMAGE', value: container_registry_repository_url, public: true }
+    end
+
+    variables
+  end
+
+  def secret_variables
+    variables.map do |variable|
+      { key: variable.key, value: variable.value, public: false }
+    end
+  end
 end

doc/ci/variables/README.md

@@ -18,25 +18,35 @@ The `API_TOKEN` will take the Secure Variable value: `SECURE`.
 
 ### Predefined variables (Environment Variables)
 
-| Variable                | Runner | Description |
-|-------------------------|-----|--------|
-| **CI**                  | 0.4 | Mark that build is executed in CI environment |
-| **GITLAB_CI**           | all | Mark that build is executed in GitLab CI environment |
-| **CI_SERVER**           | all | Mark that build is executed in CI environment |
-| **CI_SERVER_NAME**      | all | CI server that is used to coordinate builds |
-| **CI_SERVER_VERSION**   | all | Not yet defined |
-| **CI_SERVER_REVISION**  | all | Not yet defined |
-| **CI_BUILD_REF**        | all | The commit revision for which project is built |
-| **CI_BUILD_TAG**        | 0.5 | The commit tag name. Present only when building tags. |
-| **CI_BUILD_NAME**       | 0.5 | The name of the build as defined in `.gitlab-ci.yml` |
-| **CI_BUILD_STAGE**      | 0.5 | The name of the stage as defined in `.gitlab-ci.yml` |
-| **CI_BUILD_REF_NAME**   | all | The branch or tag name for which project is built |
-| **CI_BUILD_ID**         | all | The unique id of the current build that GitLab CI uses internally |
-| **CI_BUILD_REPO**       | all | The URL to clone the Git repository |
-| **CI_BUILD_TRIGGERED**  | 0.5 | The flag to indicate that build was [triggered] |
-| **CI_BUILD_TOKEN**      | 1.2 | Token used for authenticating with the GitLab Container Registry |
-| **CI_PROJECT_ID**       | all | The unique id of the current project that GitLab CI uses internally |
-| **CI_PROJECT_DIR**      | all | The full path where the repository is cloned and where the build is ran |
+| Variable                | GitLab | Runner | Description |
+|-------------------------|--------|--------|-------------|
+| **CI**                  | all    | 0.4    | Mark that build is executed in CI environment |
+| **GITLAB_CI**           | all    | all    | Mark that build is executed in GitLab CI environment |
+| **CI_SERVER**           | all    | all    | Mark that build is executed in CI environment |
+| **CI_SERVER_NAME**      | all    | all    | The name of CI server that is used to coordinate builds |
+| **CI_SERVER_VERSION**   | all    | all    | GitLab version that is used to schedule builds |
+| **CI_SERVER_REVISION**  | all    | all    | GitLab revision that is used to schedule builds |
+| **CI_BUILD_ID**         | all    | all    | The unique id of the current build that GitLab CI uses internally |
+| **CI_BUILD_REF**        | all    | all    | The commit revision for which project is built |
+| **CI_BUILD_TAG**        | all    | 0.5    | The commit tag name. Present only when building tags. |
+| **CI_BUILD_NAME**       | all    | 0.5    | The name of the build as defined in `.gitlab-ci.yml` |
+| **CI_BUILD_STAGE**      | all    | 0.5    | The name of the stage as defined in `.gitlab-ci.yml` |
+| **CI_BUILD_REF_NAME**   | all    | all    | The branch or tag name for which project is built |
+| **CI_BUILD_REPO**       | all    | all    | The URL to clone the Git repository |
+| **CI_BUILD_TRIGGERED**  | all    | 0.5    | The flag to indicate that build was [triggered] |
+| **CI_BUILD_TOKEN**      | all    | 1.2    | Token used for authenticating with the GitLab Container Registry |
+| **CI_PIPELINE_ID**      | 8.10   | 0.5    | The unique id of the current pipeline that GitLab CI uses internally |
+| **CI_PROJECT_ID**       | all    | all    | The unique id of the current project that GitLab CI uses internally |
+| **CI_PROJECT_NAME**     | 8.10   | 0.5    | The project name that is currently being built |
+| **CI_PROJECT_NAMESPACE**| 8.10   | 0.5    | The project namespace that is currently being built |
+| **CI_PROJECT_PATH**     | 8.10   | 0.5    | The namespace with project name |
+| **CI_PROJECT_URL**      | 8.10   | 0.5    | The HTTP address to access project |
+| **CI_PROJECT_DIR**      | all    | all    | The full path where the repository is cloned and where the build is run |
+| **CI_REGISTRY**         | 8.10   | 0.5    | If the Container Registry is enabled it returns the address of GitLab's Container Registry |
+| **CI_REGISTRY_IMAGE**   | 8.10   | 0.5    | If the Container Registry is enabled for the project it returnes the address of the registry tied to the specific project |
+| **CI_RUNNER_ID**        | 8.10   | 0.5    | The unique id of runner being used |
+| **CI_RUNNER_DESCRIPTION** | 8.10 | 0.5    | The description of the runner as saved in GitLab |
+| **CI_RUNNER_TAGS**      | 8.10   | 0.5    | The defined runner tags |
 
 **Some of the variables are only available when using runner with at least defined version.**
 
@@ -46,18 +56,28 @@ Example values:
 export CI_BUILD_ID="50"
 export CI_BUILD_REF="1ecfd275763eff1d6b4844ea3168962458c9f27a"
 export CI_BUILD_REF_NAME="master"
-export CI_BUILD_REPO="https://gitlab.com/gitlab-org/gitlab-ce.git"
+export CI_BUILD_REPO="https://gitab-ci-token:abcde-1234ABCD5678ef@gitlab.com/gitlab-org/gitlab-ce.git"
 export CI_BUILD_TAG="1.0.0"
 export CI_BUILD_NAME="spec:other"
 export CI_BUILD_STAGE="test"
 export CI_BUILD_TRIGGERED="true"
 export CI_BUILD_TOKEN="abcde-1234ABCD5678ef"
-export CI_PROJECT_DIR="/builds/gitlab-org/gitlab-ce"
+export CI_PIPELINE_ID="1000"
 export CI_PROJECT_ID="34"
+export CI_PROJECT_DIR="/builds/gitlab-org/gitlab-ce"
+export CI_PROJECT_NAME="gitlab-ce"
+export CI_PROJECT_NAMESPACE="gitlab-org"
+export CI_PROJECT_PATH="gitlab-org/gitlab-ce"
+export CI_PROJECT_URL="https://gitlab.com/gitlab-org/gitlab-ce"
+export CI_REGISTRY="registry.gitlab.com"
+export CI_REGISTRY_IMAGE="registry.gitlab.com/gitlab-org/gitlab-ce"
+export CI_RUNNER_ID="10"
+export CI_RUNNER_DESCRIPTION="my runner"
+export CI_RUNNER_TAGS="docker, linux"
 export CI_SERVER="yes"
-export CI_SERVER_NAME="GitLab CI"
-export CI_SERVER_REVISION=""
-export CI_SERVER_VERSION=""
+export CI_SERVER_NAME="GitLab"
+export CI_SERVER_REVISION="8.9.0"
+export CI_SERVER_VERSION="70606bf"
 ```
 
 ### YAML-defined variables

spec/models/build_spec.rb

@@ -191,79 +191,87 @@ describe Ci::Build, models: true do
   end
 
   describe '#variables' do
+    let(:container_registry_enabled) { false }
     let(:predefined_variables) do
       [
-        { key: :CI_BUILD_NAME, value: 'test', public: true },
-        { key: :CI_BUILD_STAGE, value: 'test', public: true },
+        { key: 'CI', value: 'true', public: true },
+        { key: 'GITLAB_CI', value: 'true', public: true },
+        { key: 'CI_BUILD_ID', value: build.id.to_s, public: true },
+        { key: 'CI_BUILD_TOKEN', value: build.token, public: false },
+        { key: 'CI_BUILD_REF', value: build.sha, public: true },
+        { key: 'CI_BUILD_BEFORE_SHA', value: build.before_sha, public: true },
+        { key: 'CI_BUILD_REF_NAME', value: 'master', public: true },
+        { key: 'CI_BUILD_NAME', value: 'test', public: true },
+        { key: 'CI_BUILD_STAGE', value: 'test', public: true },
+        { key: 'CI_SERVER_NAME', value: 'GitLab', public: true },
+        { key: 'CI_SERVER_VERSION', value: Gitlab::VERSION, public: true },
+        { key: 'CI_SERVER_REVISION', value: Gitlab::REVISION, public: true },
+        { key: 'CI_PROJECT_ID', value: project.id.to_s, public: true },
+        { key: 'CI_PROJECT_NAME', value: project.path, public: true },
+        { key: 'CI_PROJECT_PATH', value: project.path_with_namespace, public: true },
+        { key: 'CI_PROJECT_NAMESPACE', value: project.namespace.path, public: true },
+        { key: 'CI_PROJECT_URL', value: project.web_url, public: true },
+        { key: 'CI_PIPELINE_ID', value: pipeline.id.to_s, public: true }
       ]
     end
 
+    before do
+      stub_container_registry_config(enabled: container_registry_enabled, host_port: 'registry.example.com')
+    end
+
     subject { build.variables }
 
     context 'returns variables' do
-      let(:yaml_variables) do
-        [
-          { key: :DB_NAME, value: 'postgres', public: true }
-        ]
-      end
-
       before do
-        build.yaml_variables = yaml_variables
+        build.yaml_variables = []
       end
 
-      it { is_expected.to eq(predefined_variables + yaml_variables) }
-
-      context 'for tag' do
-        let(:tag_variable) do
-          [
-            { key: :CI_BUILD_TAG, value: 'master', public: true }
-          ]
-        end
+      it { is_expected.to eq(predefined_variables) }
+    end
 
-        before do
-          build.update_attributes(tag: true)
-        end
+    context 'when build is for tag' do
+      let(:tag_variable) do
+        { key: 'CI_BUILD_TAG', value: 'master', public: true }
+      end
 
-        it { is_expected.to eq(tag_variable + predefined_variables + yaml_variables) }
+      before do
+        build.update_attributes(tag: true)
       end
 
-      context 'and secure variables' do
-        let(:secure_variables) do
-          [
-            { key: 'SECRET_KEY', value: 'secret_value', public: false }
-          ]
-        end
+      it { is_expected.to include(tag_variable) }
+    end
 
-        before do
-          build.project.variables << Ci::Variable.new(key: 'SECRET_KEY', value: 'secret_value')
-        end
+    context 'when secure variable is defined' do
+      let(:secure_variable) do
+        { key: 'SECRET_KEY', value: 'secret_value', public: false }
+      end
 
-        it { is_expected.to eq(predefined_variables + yaml_variables + secure_variables) }
+      before do
+        build.project.variables << Ci::Variable.new(key: 'SECRET_KEY', value: 'secret_value')
+      end
 
-        context 'and trigger variables' do
-          let(:trigger) { create(:ci_trigger, project: project) }
-          let(:trigger_request) { create(:ci_trigger_request_with_variables, pipeline: pipeline, trigger: trigger) }
-          let(:trigger_variables) do
-            [
-              { key: :TRIGGER_KEY, value: 'TRIGGER_VALUE', public: false }
-            ]
-          end
-          let(:predefined_trigger_variable) do
-            [
-              { key: :CI_BUILD_TRIGGERED, value: 'true', public: true }
-            ]
-          end
+      it { is_expected.to include(secure_variable) }
+    end
 
-          before do
-            build.trigger_request = trigger_request
-          end
+    context 'when build is for triggers' do
+      let(:trigger) { create(:ci_trigger, project: project) }
+      let(:trigger_request) { create(:ci_trigger_request_with_variables, pipeline: pipeline, trigger: trigger) }
+      let(:user_trigger_variable) do
+        { key: :TRIGGER_KEY, value: 'TRIGGER_VALUE', public: false }
+      end
+      let(:predefined_trigger_variable) do
+        { key: 'CI_BUILD_TRIGGERED', value: 'true', public: true }
+      end
 
-          it { is_expected.to eq(predefined_variables + predefined_trigger_variable + yaml_variables + secure_variables + trigger_variables) }
-        end
+      before do
+        build.trigger_request = trigger_request
       end
+
+      it { is_expected.to include(user_trigger_variable) }
+      it { is_expected.to include(predefined_trigger_variable) }
     end
 
-    context 'when yaml_variables is undefined' do
+    context 'when yaml_variables are undefined' do
       before do
         build.yaml_variables = nil
       end
@@ -282,10 +290,10 @@ describe Ci::Build, models: true do
         context 'if config does not have a questioned job' do
           let(:config) do
             YAML.dump({
-                        test_other: {
-                          script: 'Hello World'
-                        }
-                      })
+              test_other: {
+                script: 'Hello World'
+              }
+            })
           end
 
           it { is_expected.to eq(predefined_variables) }
@@ -294,13 +302,13 @@ describe Ci::Build, models: true do
         context 'if config has variables' do
           let(:config) do
             YAML.dump({
-                        test: {
-                          script: 'Hello World',
-                          variables: {
-                            KEY: 'value'
-                          }
-                        }
-                      })
+              test: {
+                script: 'Hello World',
+                variables: {
+                  KEY: 'value'
+                }
+              }
+            })
           end
           let(:variables) do
             [{ key: :KEY, value: 'value', public: true }]
@@ -310,6 +318,58 @@ describe Ci::Build, models: true do
         end
       end
     end
+
+    context 'when container registry is enabled' do
+      let(:container_registry_enabled) { true }
+      let(:ci_registry) do
+        { key: 'CI_REGISTRY',  value: 'registry.example.com',  public: true }
+      end
+      let(:ci_registry_image) do
+        { key: 'CI_REGISTRY_IMAGE',  value: project.container_registry_repository_url, public: true }
+      end
+
+      context 'and is disabled for project' do
+        before do
+          project.update(container_registry_enabled: false)
+        end
+
+        it { is_expected.to include(ci_registry) }
+        it { is_expected.not_to include(ci_registry_image) }
+      end
+
+      context 'and is enabled for project' do
+        before do
+          project.update(container_registry_enabled: true)
+        end
+
+        it { is_expected.to include(ci_registry) }
+        it { is_expected.to include(ci_registry_image) }
+      end
+    end
+
+    context 'when runner is assigned to build' do
+      let(:runner) { create(:ci_runner, description: 'description', tag_list: ['docker', 'linux']) }
+
+      before do
+        build.update(runner: runner)
+      end
+
+      it { is_expected.to include({ key: 'CI_RUNNER_ID', value: runner.id.to_s, public: true }) }
+      it { is_expected.to include({ key: 'CI_RUNNER_DESCRIPTION', value: 'description', public: true }) }
+      it { is_expected.to include({ key: 'CI_RUNNER_TAGS', value: 'docker, linux', public: true }) }
+    end
+
+    context 'returns variables in valid order' do
+      before do
+        allow(build).to receive(:predefined_variables) { ['predefined'] }
+        allow(project).to receive(:predefined_variables) { ['project'] }
+        allow(pipeline).to receive(:predefined_variables) { ['pipeline'] }
+        allow(build).to receive(:yaml_variables) { ['yaml'] }
+        allow(project).to receive(:secret_variables) { ['secret'] }
+      end
+
+      it { is_expected.to eq(%w[predefined project pipeline yaml secret]) }
+    end
   end
 
   describe '#has_tags?' do

spec/requests/ci/api/builds_spec.rb

@@ -73,12 +73,12 @@ describe Ci::API::API do
         post ci_api("/builds/register"), token: runner.token, info: { platform: :darwin }
 
         expect(response).to have_http_status(201)
-        expect(json_response["variables"]).to eq([
+        expect(json_response["variables"]).to include(
           { "key" => "CI_BUILD_NAME", "value" => "spinach", "public" => true },
           { "key" => "CI_BUILD_STAGE", "value" => "test", "public" => true },
           { "key" => "DB_NAME", "value" => "postgres", "public" => true },
           { "key" => "SECRET_KEY", "value" => "secret_value", "public" => false }
-        ])
+        )
       end
 
       it "returns variables for triggers" do
@@ -92,14 +92,14 @@ describe Ci::API::API do
         post ci_api("/builds/register"), token: runner.token, info: { platform: :darwin }
 
         expect(response).to have_http_status(201)
-        expect(json_response["variables"]).to eq([
+        expect(json_response["variables"]).to include(
           { "key" => "CI_BUILD_NAME", "value" => "spinach", "public" => true },
           { "key" => "CI_BUILD_STAGE", "value" => "test", "public" => true },
           { "key" => "CI_BUILD_TRIGGERED", "value" => "true", "public" => true },
           { "key" => "DB_NAME", "value" => "postgres", "public" => true },
           { "key" => "SECRET_KEY", "value" => "secret_value", "public" => false },
-          { "key" => "TRIGGER_KEY", "value" => "TRIGGER_VALUE", "public" => false },
-        ])
+          { "key" => "TRIGGER_KEY", "value" => "TRIGGER_VALUE", "public" => false }
+        )
       end
 
       it "returns dependent builds" do