Merge branch 'feature/add-support-for-services-configuration' into 'master'

Add support for services configuration in .gitlab-ci.yml

See merge request !8578

changelogs/unreleased/feature-add-support-for-services-configuration.yml

+---
+title: Add support for image and services configuration in .gitlab-ci.yml
+merge_request: 8578
+author:

lib/api/entities.rb

@@ -804,7 +804,11 @@ module API
       end
 
       class Image < Grape::Entity
-        expose :name
+        expose :name, :entrypoint
+      end
+
+      class Service < Image
+        expose :alias, :command
       end
 
       class Artifacts < Grape::Entity
@@ -848,7 +852,7 @@ module API
         expose :variables
         expose :steps, using: Step
         expose :image, using: Image
-        expose :services, using: Image
+        expose :services, using: Service
         expose :artifacts, using: Artifacts
         expose :cache, using: Cache
         expose :credentials, using: Credentials

lib/ci/api/entities.rb

@@ -45,7 +45,21 @@ module Ci
         expose :artifacts_expire_at, if: ->(build, _) { build.artifacts? }
 
         expose :options do |model|
-          model.options
+          # This part ensures that output of old API is still the same after adding support
+          # for extended docker configuration options, used by new API
+          #
+          # I'm leaving this here, not in the model, because it should be removed at the same time
+          # when old API will be removed (planned for August 2017).
+          model.options.dup.tap do |options|
+            options[:image] = options[:image][:name] if options[:image].is_a?(Hash)
+            options[:services].map! do |service|
+              if service.is_a?(Hash)
+                service[:name]
+              else
+                service
+              end
+            end
+          end
         end
 
         expose :timeout do |model|

lib/gitlab/ci/build/image.rb

@@ -2,7 +2,7 @@ module Gitlab
   module Ci
     module Build
       class Image
-        attr_reader :name
+        attr_reader :alias, :command, :entrypoint, :name
 
         class << self
           def from_image(job)
@@ -21,7 +21,14 @@ module Gitlab
         end
 
         def initialize(image)
-          @name = image
+          if image.is_a?(String)
+            @name = image
+          elsif image.is_a?(Hash)
+            @alias = image[:alias]
+            @command = image[:command]
+            @entrypoint = image[:entrypoint]
+            @name = image[:name]
+          end
         end
 
         def valid?

lib/gitlab/ci/config/entry/image.rb

@@ -8,8 +8,36 @@ module Gitlab
         class Image < Node
           include Validatable
 
+          ALLOWED_KEYS = %i[name entrypoint].freeze
+
           validations do
-            validates :config, type: String
+            validates :config, hash_or_string: true
+            validates :config, allowed_keys: ALLOWED_KEYS
+
+            validates :name, type: String, presence: true
+            validates :entrypoint, type: String, allow_nil: true
+          end
+
+          def hash?
+            @config.is_a?(Hash)
+          end
+
+          def string?
+            @config.is_a?(String)
+          end
+
+          def name
+            value[:name]
+          end
+
+          def entrypoint
+            value[:entrypoint]
+          end
+
+          def value
+            return { name: @config } if string?
+            return @config if hash?
+            {}
           end
         end
       end

lib/gitlab/ci/config/entry/service.rb

+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents a configuration of Docker service.
+        #
+        class Service < Image
+          include Validatable
+
+          ALLOWED_KEYS = %i[name entrypoint command alias].freeze
+
+          validations do
+            validates :config, hash_or_string: true
+            validates :config, allowed_keys: ALLOWED_KEYS
+
+            validates :name, type: String, presence: true
+            validates :entrypoint, type: String, allow_nil: true
+            validates :command, type: String, allow_nil: true
+            validates :alias, type: String, allow_nil: true
+          end
+
+          def alias
+            value[:alias]
+          end
+
+          def command
+            value[:command]
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config/entry/services.rb

@@ -9,7 +9,30 @@ module Gitlab
           include Validatable
 
           validations do
-            validates :config, array_of_strings: true
+            validates :config, type: Array
+          end
+
+          def compose!(deps = nil)
+            super do
+              @entries = []
+              @config.each do |config|
+                @entries << Entry::Factory.new(Entry::Service)
+                  .value(config || {})
+                  .create!
+              end
+
+              @entries.each do |entry|
+                entry.compose!(deps)
+              end
+            end
+          end
+
+          def value
+            @entries.map(&:value)
+          end
+
+          def descendants
+            @entries
           end
         end
       end

lib/gitlab/ci/config/entry/validators.rb

@@ -44,6 +44,14 @@ module Gitlab
             end
           end
 
+          class HashOrStringValidator < ActiveModel::EachValidator
+            def validate_each(record, attribute, value)
+              unless value.is_a?(Hash) || value.is_a?(String)
+                record.errors.add(attribute, 'should be a hash or a string')
+              end
+            end
+          end
+
           class KeyValidator < ActiveModel::EachValidator
             include LegacyValidationHelpers
 

spec/factories/ci/builds.rb

@@ -194,8 +194,8 @@ FactoryGirl.define do
     trait :extended_options do
       options do
         {
-            image: 'ruby:2.1',
-            services: ['postgres'],
+            image: { name: 'ruby:2.1', entrypoint: '/bin/sh' },
+            services: ['postgres', { name: 'docker:dind', entrypoint: '/bin/sh', command: 'sleep 30', alias: 'docker' }],
             after_script: %w(ls date),
             artifacts: {
                 name: 'artifacts_file',

spec/lib/gitlab/ci/build/image_spec.rb

@@ -10,12 +10,28 @@ describe Gitlab::Ci::Build::Image do
       let(:image_name) { 'ruby:2.1' }
       let(:job) { create(:ci_build, options: { image: image_name } ) }
 
-      it 'fabricates an object of the proper class' do
-        is_expected.to be_kind_of(described_class)
+      context 'when image is defined as string' do
+        it 'fabricates an object of the proper class' do
+          is_expected.to be_kind_of(described_class)
+        end
+
+        it 'populates fabricated object with the proper name attribute' do
+          expect(subject.name).to eq(image_name)
+        end
       end
 
-      it 'populates fabricated object with the proper name attribute' do
-        expect(subject.name).to eq(image_name)
+      context 'when image is defined as hash' do
+        let(:entrypoint) { '/bin/sh' }
+        let(:job) { create(:ci_build, options: { image: { name: image_name, entrypoint: entrypoint } } ) }
+
+        it 'fabricates an object of the proper class' do
+          is_expected.to be_kind_of(described_class)
+        end
+
+        it 'populates fabricated object with the proper attributes' do
+          expect(subject.name).to eq(image_name)
+          expect(subject.entrypoint).to eq(entrypoint)
+        end
       end
 
       context 'when image name is empty' do
@@ -41,10 +57,39 @@ describe Gitlab::Ci::Build::Image do
       let(:service_image_name) { 'postgres' }
       let(:job) { create(:ci_build, options: { services: [service_image_name] }) }
 
-      it 'fabricates an non-empty array of objects' do
-        is_expected.to be_kind_of(Array)
-        is_expected.not_to be_empty
-        expect(subject.first.name).to eq(service_image_name)
+      context 'when service is defined as string' do
+        it 'fabricates an non-empty array of objects' do
+          is_expected.to be_kind_of(Array)
+          is_expected.not_to be_empty
+        end
+
+        it 'populates fabricated objects with the proper name attributes' do
+          expect(subject.first).to be_kind_of(described_class)
+          expect(subject.first.name).to eq(service_image_name)
+        end
+      end
+
+      context 'when service is defined as hash' do
+        let(:service_entrypoint) { '/bin/sh' }
+        let(:service_alias) { 'db' }
+        let(:service_command) { 'sleep 30' }
+        let(:job) do
+          create(:ci_build, options: { services: [{ name: service_image_name, entrypoint: service_entrypoint,
+                                                    alias: service_alias, command: service_command }] })
+        end
+
+        it 'fabricates an non-empty array of objects' do
+          is_expected.to be_kind_of(Array)
+          is_expected.not_to be_empty
+          expect(subject.first).to be_kind_of(described_class)
+        end
+
+        it 'populates fabricated objects with the proper attributes' do
+          expect(subject.first.name).to eq(service_image_name)
+          expect(subject.first.entrypoint).to eq(service_entrypoint)
+          expect(subject.first.alias).to eq(service_alias)
+          expect(subject.first.command).to eq(service_command)
+        end
       end
 
       context 'when service image name is empty' do

spec/lib/gitlab/ci/config/entry/global_spec.rb

@@ -95,13 +95,13 @@ describe Gitlab::Ci::Config::Entry::Global do
 
         describe '#image_value' do
           it 'returns valid image' do
-            expect(global.image_value).to eq 'ruby:2.2'
+            expect(global.image_value).to eq(name: 'ruby:2.2')
           end
         end
 
         describe '#services_value' do
           it 'returns array of services' do
-            expect(global.services_value).to eq ['postgres:9.1', 'mysql:5.5']
+            expect(global.services_value).to eq [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }]
           end
         end
 
@@ -150,8 +150,8 @@ describe Gitlab::Ci::Config::Entry::Global do
                        script: %w[rspec ls],
                        before_script: %w(ls pwd),
                        commands: "ls\npwd\nrspec\nls",
-                       image: 'ruby:2.2',
-                       services: ['postgres:9.1', 'mysql:5.5'],
+                       image: { name: 'ruby:2.2' },
+                       services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }],
                        stage: 'test',
                        cache: { key: 'k', untracked: true, paths: ['public/'] },
                        variables: { 'VAR' => 'value' },
@@ -161,8 +161,8 @@ describe Gitlab::Ci::Config::Entry::Global do
                          before_script: [],
                          script: %w[spinach],
                          commands: 'spinach',
-                         image: 'ruby:2.2',
-                         services: ['postgres:9.1', 'mysql:5.5'],
+                         image: { name: 'ruby:2.2' },
+                         services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }],
                          stage: 'test',
                          cache: { key: 'k', untracked: true, paths: ['public/'] },
                          variables: {},

spec/lib/gitlab/ci/config/entry/image_spec.rb

@@ -3,43 +3,104 @@ require 'spec_helper'
 describe Gitlab::Ci::Config::Entry::Image do
   let(:entry) { described_class.new(config) }
 
-  describe 'validation' do
-    context 'when entry config value is correct' do
-      let(:config) { 'ruby:2.2' }
+  context 'when configuration is a string' do
+    let(:config) { 'ruby:2.2' }
 
-      describe '#value' do
-        it 'returns image string' do
-          expect(entry.value).to eq 'ruby:2.2'
-        end
+    describe '#value' do
+      it 'returns image hash' do
+        expect(entry.value).to eq({ name: 'ruby:2.2' })
       end
+    end
+
+    describe '#errors' do
+      it 'does not append errors' do
+        expect(entry.errors).to be_empty
+      end
+    end
+
+    describe '#valid?' do
+      it 'is valid' do
+        expect(entry).to be_valid
+      end
+    end
+
+    describe '#image' do
+      it "returns image's name" do
+        expect(entry.name).to eq 'ruby:2.2'
+      end
+    end
 
-      describe '#errors' do
-        it 'does not append errors' do
-          expect(entry.errors).to be_empty
-        end
+    describe '#entrypoint' do
+      it "returns image's entrypoint" do
+        expect(entry.entrypoint).to be_nil
       end
+    end
+  end
 
-      describe '#valid?' do
-        it 'is valid' do
-          expect(entry).to be_valid
-        end
+  context 'when configuration is a hash' do
+    let(:config) { { name: 'ruby:2.2', entrypoint: '/bin/sh' } }
+
+    describe '#value' do
+      it 'returns image hash' do
+        expect(entry.value).to eq(config)
       end
     end
 
-    context 'when entry value is not correct' do
-      let(:config) { ['ruby:2.2'] }
+    describe '#errors' do
+      it 'does not append errors' do
+        expect(entry.errors).to be_empty
+      end
+    end
+
+    describe '#valid?' do
+      it 'is valid' do
+        expect(entry).to be_valid
+      end
+    end
 
-      describe '#errors' do
-        it 'saves errors' do
-          expect(entry.errors)
-            .to include 'image config should be a string'
-        end
+    describe '#image' do
+      it "returns image's name" do
+        expect(entry.name).to eq 'ruby:2.2'
       end
+    end
+
+    describe '#entrypoint' do
+      it "returns image's entrypoint" do
+        expect(entry.entrypoint).to eq '/bin/sh'
+      end
+    end
+  end
+
+  context 'when entry value is not correct' do
+    let(:config) { ['ruby:2.2'] }
+
+    describe '#errors' do
+      it 'saves errors' do
+        expect(entry.errors)
+          .to include 'image config should be a hash or a string'
+      end
+    end
+
+    describe '#valid?' do
+      it 'is not valid' do
+        expect(entry).not_to be_valid
+      end
+    end
+  end
+
+  context 'when unexpected key is specified' do
+    let(:config) { { name: 'ruby:2.2', non_existing: 'test' } }
+
+    describe '#errors' do
+      it 'saves errors' do
+        expect(entry.errors)
+            .to include 'image config contains unknown keys: non_existing'
+      end
+    end
 
-      describe '#valid?' do
-        it 'is not valid' do
-          expect(entry).not_to be_valid
-        end
+    describe '#valid?' do
+      it 'is not valid' do
+        expect(entry).not_to be_valid
       end
     end
   end

spec/lib/gitlab/ci/config/entry/job_spec.rb

@@ -104,7 +104,7 @@ describe Gitlab::Ci::Config::Entry::Job do
       end
 
       it 'overrides global config' do
-        expect(entry[:image].value).to eq 'some_image'
+        expect(entry[:image].value).to eq(name: 'some_image')
         expect(entry[:cache].value).to eq(key: 'test')
       end
     end

spec/lib/gitlab/ci/config/entry/service_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Config::Entry::Service do
+  let(:entry) { described_class.new(config) }
+
+  before { entry.compose! }
+
+  context 'when configuration is a string' do
+    let(:config) { 'postgresql:9.5' }
+
+    describe '#valid?' do
+      it 'is valid' do
+        expect(entry).to be_valid
+      end
+    end
+
+    describe '#value' do
+      it 'returns valid hash' do
+        expect(entry.value).to include(name: 'postgresql:9.5')
+      end
+    end
+
+    describe '#image' do
+      it "returns service's image name" do
+        expect(entry.name).to eq 'postgresql:9.5'
+      end
+    end
+
+    describe '#alias' do
+      it "returns service's alias" do
+        expect(entry.alias).to be_nil
+      end
+    end
+
+    describe '#command' do
+      it "returns service's command" do
+        expect(entry.command).to be_nil
+      end
+    end
+  end
+
+  context 'when configuration is a hash' do
+    let(:config) do
+      { name: 'postgresql:9.5', alias: 'db', command: 'cmd', entrypoint: '/bin/sh' }
+    end
+
+    describe '#valid?' do
+      it 'is valid' do
+        expect(entry).to be_valid
+      end
+    end
+
+    describe '#value' do
+      it 'returns valid hash' do
+        expect(entry.value).to eq config
+      end
+    end
+
+    describe '#image' do
+      it "returns service's image name" do
+        expect(entry.name).to eq 'postgresql:9.5'
+      end
+    end
+
+    describe '#alias' do
+      it "returns service's alias" do
+        expect(entry.alias).to eq 'db'
+      end
+    end
+
+    describe '#command' do
+      it "returns service's command" do
+        expect(entry.command).to eq 'cmd'
+      end
+    end
+
+    describe '#entrypoint' do
+      it "returns service's entrypoint" do
+        expect(entry.entrypoint).to eq '/bin/sh'
+      end
+    end
+  end
+
+  context 'when entry value is not correct' do
+    let(:config) { ['postgresql:9.5'] }
+
+    describe '#errors' do
+      it 'saves errors' do
+        expect(entry.errors)
+            .to include 'service config should be a hash or a string'
+      end
+    end
+
+    describe '#valid?' do
+      it 'is not valid' do
+        expect(entry).not_to be_valid
+      end
+    end
+  end
+
+  context 'when unexpected key is specified' do
+    let(:config) { { name: 'postgresql:9.5', non_existing: 'test' } }
+
+    describe '#errors' do
+      it 'saves errors' do
+        expect(entry.errors)
+            .to include 'service config contains unknown keys: non_existing'
+      end
+    end
+
+    describe '#valid?' do
+      it 'is not valid' do
+        expect(entry).not_to be_valid
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/config/entry/services_spec.rb

@@ -3,37 +3,30 @@ require 'spec_helper'
 describe Gitlab::Ci::Config::Entry::Services do
   let(:entry) { described_class.new(config) }
 
-  describe 'validations' do
-    context 'when entry config value is correct' do
-      let(:config) { ['postgres:9.1', 'mysql:5.5'] }
+  before { entry.compose! }
 
-      describe '#value' do
-        it 'returns array of services as is' do
-          expect(entry.value).to eq config
-        end
-      end
+  context 'when configuration is valid' do
+    let(:config) { ['postgresql:9.5', { name: 'postgresql:9.1', alias: 'postgres_old' }] }
 
-      describe '#valid?' do
-        it 'is valid' do
-          expect(entry).to be_valid
-        end
+    describe '#valid?' do
+      it 'is valid' do
+        expect(entry).to be_valid
       end
     end
 
-    context 'when entry value is not correct' do
-      let(:config) { 'ls' }
-
-      describe '#errors' do
-        it 'saves errors' do
-          expect(entry.errors)
-            .to include 'services config should be an array of strings'
-        end
+    describe '#value' do
+      it 'returns valid array' do
+        expect(entry.value).to eq([{ name: 'postgresql:9.5' }, { name: 'postgresql:9.1', alias: 'postgres_old' }])
       end
+    end
+  end
+
+  context 'when configuration is invalid' do
+    let(:config) { 'postgresql:9.5' }
 
-      describe '#valid?' do
-        it 'is not valid' do
-          expect(entry).not_to be_valid
-        end
+    describe '#valid?' do
+      it 'is invalid' do
+        expect(entry).not_to be_valid
       end
     end
   end

spec/requests/api/runner_spec.rb

@@ -356,8 +356,11 @@ describe API::Runner do
             expect(json_response['token']).to eq(job.token)
             expect(json_response['job_info']).to eq(expected_job_info)
             expect(json_response['git_info']).to eq(expected_git_info)
-            expect(json_response['image']).to eq({ 'name' => 'ruby:2.1' })
-            expect(json_response['services']).to eq([{ 'name' => 'postgres' }])
+            expect(json_response['image']).to eq({ 'name' => 'ruby:2.1', 'entrypoint' => '/bin/sh' })
+            expect(json_response['services']).to eq([{ 'name' => 'postgres', 'entrypoint' => nil,
+                                                       'alias' => nil, 'command' => nil },
+                                                     { 'name' => 'docker:dind', 'entrypoint' => '/bin/sh',
+                                                       'alias' => 'docker', 'command' => 'sleep 30' }])
             expect(json_response['steps']).to eq(expected_steps)
             expect(json_response['artifacts']).to eq(expected_artifacts)
             expect(json_response['cache']).to eq(expected_cache)

spec/requests/ci/api/builds_spec.rb

@@ -137,6 +137,18 @@ describe Ci::API::Builds do
             end
           end
         end
+
+        context 'when docker configuration options are used' do
+          let!(:build) { create(:ci_build, :extended_options, pipeline: pipeline, name: 'spinach', stage: 'test', stage_idx: 0) }
+
+          it 'starts a build' do
+            register_builds info: { platform: :darwin }
+
+            expect(response).to have_http_status(201)
+            expect(json_response['options']['image']).to eq('ruby:2.1')
+            expect(json_response['options']['services']).to eq(['postgres', 'docker:dind'])
+          end
+        end
       end
 
       context 'when builds are finished' do