Merge branch 'refactor/ci-config-move-job-entries' into 'master'

Move CI job config entries from legacy to new config

## What does this MR do?

This MR extracts jobs configuration logic from legacy CI config processor to the new code.

## What are the relevant issue numbers?

#15060

## Does this MR meet the acceptance criteria?

- Tests
  - [x] Added for this feature/bug
  - [x] All builds are passing
- [x] Conform by the [style guides](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CONTRIBUTING.md#style-guides)
- [x] Branch has no merge conflicts with `master` (if you do - rebase it please)

See merge request !5087

lib/ci/gitlab_ci_yaml_processor.rb

@@ -4,21 +4,11 @@ module Ci
 
     include Gitlab::Ci::Config::Node::LegacyValidationHelpers
 
-    DEFAULT_STAGE = 'test'
-    ALLOWED_YAML_KEYS = [:before_script, :after_script, :image, :services, :types, :stages, :variables, :cache]
-    ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services,
-                        :allow_failure, :type, :stage, :when, :artifacts, :cache,
-                        :dependencies, :before_script, :after_script, :variables,
-                        :environment]
-    ALLOWED_CACHE_KEYS = [:key, :untracked, :paths]
-    ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when, :expire_in]
-
     attr_reader :path, :cache, :stages
 
     def initialize(config, path = nil)
       @ci_config = Gitlab::Ci::Config.new(config)
       @config = @ci_config.to_hash
-
       @path = path
 
       unless @ci_config.valid?
@@ -26,7 +16,6 @@ module Ci
       end
 
       initial_parsing
-      validate!
     rescue Gitlab::Ci::Config::Loader::FormatError => e
       raise ValidationError, e.message
     end
@@ -73,7 +62,7 @@ module Ci
         #  - before script should be a concatenated command
         commands: [job[:before_script] || @before_script, job[:script]].flatten.compact.join("\n"),
         tag_list: job[:tags] || [],
-        name: name,
+        name: job[:name],
         allow_failure: job[:allow_failure] || false,
         when: job[:when] || 'on_success',
         environment: job[:environment],
@@ -92,6 +81,9 @@ module Ci
     private
 
     def initial_parsing
+      ##
+      # Global config
+      #
       @before_script = @ci_config.before_script
       @image = @ci_config.image
       @after_script = @ci_config.after_script
@@ -100,34 +92,28 @@ module Ci
       @stages = @ci_config.stages
       @cache = @ci_config.cache
 
-      @jobs = {}
-
-      @config.except!(*ALLOWED_YAML_KEYS)
-      @config.each { |name, param| add_job(name, param) }
-
-      raise ValidationError, "Please define at least one job" if @jobs.none?
-    end
-
-    def add_job(name, job)
-      return if name.to_s.start_with?('.')
+      ##
+      # Jobs
+      #
+      @jobs = @ci_config.jobs
 
-      raise ValidationError, "Unknown parameter: #{name}" unless job.is_a?(Hash) && job.has_key?(:script)
+      @jobs.each do |name, job|
+        # logical validation for job
 
-      stage = job[:stage] || job[:type] || DEFAULT_STAGE
-      @jobs[name] = { stage: stage }.merge(job)
+        validate_job_stage!(name, job)
+        validate_job_dependencies!(name, job)
+      end
     end
 
     def yaml_variables(name)
-      variables = global_variables.merge(job_variables(name))
+      variables = (@variables || {})
+        .merge(job_variables(name))
+
       variables.map do |key, value|
         { key: key, value: value, public: true }
       end
     end
 
-    def global_variables
-      @variables || {}
-    end
-
     def job_variables(name)
       job = @jobs[name.to_sym]
       return {} unless job
@@ -135,154 +121,16 @@ module Ci
       job[:variables] || {}
     end
 
-    def validate!
-      @jobs.each do |name, job|
-        validate_job!(name, job)
-      end
-
-      true
-    end
-
-    def validate_job!(name, job)
-      validate_job_name!(name)
-      validate_job_keys!(name, job)
-      validate_job_types!(name, job)
-      validate_job_script!(name, job)
-
-      validate_job_stage!(name, job) if job[:stage]
-      validate_job_variables!(name, job) if job[:variables]
-      validate_job_cache!(name, job) if job[:cache]
-      validate_job_artifacts!(name, job) if job[:artifacts]
-      validate_job_dependencies!(name, job) if job[:dependencies]
-    end
-
-    def validate_job_name!(name)
-      if name.blank? || !validate_string(name)
-        raise ValidationError, "job name should be non-empty string"
-      end
-    end
-
-    def validate_job_keys!(name, job)
-      job.keys.each do |key|
-        unless ALLOWED_JOB_KEYS.include? key
-          raise ValidationError, "#{name} job: unknown parameter #{key}"
-        end
-      end
-    end
-
-    def validate_job_types!(name, job)
-      if job[:image] && !validate_string(job[:image])
-        raise ValidationError, "#{name} job: image should be a string"
-      end
-
-      if job[:services] && !validate_array_of_strings(job[:services])
-        raise ValidationError, "#{name} job: services should be an array of strings"
-      end
-
-      if job[:tags] && !validate_array_of_strings(job[:tags])
-        raise ValidationError, "#{name} job: tags parameter should be an array of strings"
-      end
-
-      if job[:only] && !validate_array_of_strings_or_regexps(job[:only])
-        raise ValidationError, "#{name} job: only parameter should be an array of strings or regexps"
-      end
-
-      if job[:except] && !validate_array_of_strings_or_regexps(job[:except])
-        raise ValidationError, "#{name} job: except parameter should be an array of strings or regexps"
-      end
-
-      if job[:allow_failure] && !validate_boolean(job[:allow_failure])
-        raise ValidationError, "#{name} job: allow_failure parameter should be an boolean"
-      end
-
-      if job[:when] && !job[:when].in?(%w[on_success on_failure always manual])
-        raise ValidationError, "#{name} job: when parameter should be on_success, on_failure, always or manual"
-      end
-
-      if job[:environment] && !validate_environment(job[:environment])
-        raise ValidationError, "#{name} job: environment parameter #{Gitlab::Regex.environment_name_regex_message}"
-      end
-    end
-
-    def validate_job_script!(name, job)
-      if !validate_string(job[:script]) && !validate_array_of_strings(job[:script])
-        raise ValidationError, "#{name} job: script should be a string or an array of a strings"
-      end
-
-      if job[:before_script] && !validate_array_of_strings(job[:before_script])
-        raise ValidationError, "#{name} job: before_script should be an array of strings"
-      end
-
-      if job[:after_script] && !validate_array_of_strings(job[:after_script])
-        raise ValidationError, "#{name} job: after_script should be an array of strings"
-      end
-    end
-
     def validate_job_stage!(name, job)
+      return unless job[:stage]
+
       unless job[:stage].is_a?(String) && job[:stage].in?(@stages)
         raise ValidationError, "#{name} job: stage parameter should be #{@stages.join(", ")}"
       end
     end
 
-    def validate_job_variables!(name, job)
-      unless validate_variables(job[:variables])
-        raise ValidationError,
-          "#{name} job: variables should be a map of key-value strings"
-      end
-    end
-
-    def validate_job_cache!(name, job)
-      job[:cache].keys.each do |key|
-        unless ALLOWED_CACHE_KEYS.include? key
-          raise ValidationError, "#{name} job: cache unknown parameter #{key}"
-        end
-      end
-
-      if job[:cache][:key] && !validate_string(job[:cache][:key])
-        raise ValidationError, "#{name} job: cache:key parameter should be a string"
-      end
-
-      if job[:cache][:untracked] && !validate_boolean(job[:cache][:untracked])
-        raise ValidationError, "#{name} job: cache:untracked parameter should be an boolean"
-      end
-
-      if job[:cache][:paths] && !validate_array_of_strings(job[:cache][:paths])
-        raise ValidationError, "#{name} job: cache:paths parameter should be an array of strings"
-      end
-    end
-
-    def validate_job_artifacts!(name, job)
-      job[:artifacts].keys.each do |key|
-        unless ALLOWED_ARTIFACTS_KEYS.include? key
-          raise ValidationError, "#{name} job: artifacts unknown parameter #{key}"
-        end
-      end
-
-      if job[:artifacts][:name] && !validate_string(job[:artifacts][:name])
-        raise ValidationError, "#{name} job: artifacts:name parameter should be a string"
-      end
-
-      if job[:artifacts][:untracked] && !validate_boolean(job[:artifacts][:untracked])
-        raise ValidationError, "#{name} job: artifacts:untracked parameter should be an boolean"
-      end
-
-      if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths])
-        raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings"
-      end
-
-      if job[:artifacts][:when] && !job[:artifacts][:when].in?(%w[on_success on_failure always])
-        raise ValidationError, "#{name} job: artifacts:when parameter should be on_success, on_failure or always"
-      end
-
-      if job[:artifacts][:expire_in] && !validate_duration(job[:artifacts][:expire_in])
-        raise ValidationError, "#{name} job: artifacts:expire_in parameter should be a duration"
-      end
-    end
-
     def validate_job_dependencies!(name, job)
-      unless validate_array_of_strings(job[:dependencies])
-        raise ValidationError, "#{name} job: dependencies parameter should be an array of strings"
-      end
+      return unless job[:dependencies]
 
       stage_index = @stages.index(job[:stage])
 

lib/gitlab/ci/config.rb

@@ -8,7 +8,7 @@ module Gitlab
       # Temporary delegations that should be removed after refactoring
       #
       delegate :before_script, :image, :services, :after_script, :variables,
-               :stages, :cache, to: :@global
+               :stages, :cache, :jobs, to: :@global
 
       def initialize(config)
         @config = Loader.new(config).load!

lib/gitlab/ci/config/node/artifacts.rb

+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # Entry that represents a configuration of job artifacts.
+        #
+        class Artifacts < Entry
+          include Validatable
+          include Attributable
+
+          ALLOWED_KEYS = %i[name untracked paths when expire_in]
+
+          attributes ALLOWED_KEYS
+
+          validations do
+            validates :config, type: Hash
+            validates :config, allowed_keys: ALLOWED_KEYS
+
+            with_options allow_nil: true do
+              validates :name, type: String
+              validates :untracked, boolean: true
+              validates :paths, array_of_strings: true
+              validates :when,
+                inclusion: { in: %w[on_success on_failure always],
+                             message: 'should be on_success, on_failure ' \
+                                      'or always' }
+              validates :expire_in, duration: true
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config/node/attributable.rb

+module Gitlab
+  module Ci
+    class Config
+      module Node
+        module Attributable
+          extend ActiveSupport::Concern
+
+          class_methods do
+            def attributes(*attributes)
+              attributes.flatten.each do |attribute|
+                define_method(attribute) do
+                  return unless config.is_a?(Hash)
+
+                  config[attribute]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config/node/cache.rb

@@ -8,6 +8,12 @@ module Gitlab
         class Cache < Entry
           include Configurable
 
+          ALLOWED_KEYS = %i[key untracked paths]
+
+          validations do
+            validates :config, allowed_keys: ALLOWED_KEYS
+          end
+
           node :key, Node::Key,
             description: 'Cache key used to define a cache affinity.'
 
@@ -16,10 +22,6 @@ module Gitlab
 
           node :paths, Node::Paths,
             description: 'Specify which paths should be cached across builds.'
-
-          validations do
-            validates :config, allowed_keys: true
-          end
         end
       end
     end

lib/gitlab/ci/config/node/commands.rb

+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # Entry that represents a job script.
+        #
+        class Commands < Entry
+          include Validatable
+
+          validations do
+            include LegacyValidationHelpers
+
+            validate do
+              unless string_or_array_of_strings?(config)
+                errors.add(:config,
+                           'should be a string or an array of strings')
+              end
+            end
+
+            def string_or_array_of_strings?(field)
+              validate_string(field) || validate_array_of_strings(field)
+            end
+          end
+
+          def value
+            Array(@config)
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config/node/configurable.rb

@@ -25,10 +25,14 @@ module Gitlab
 
           private
 
-          def create_node(key, factory)
-            factory.with(value: @config[key], key: key, parent: self)
+          def compose!
+            self.class.nodes.each do |key, factory|
+              factory
+                .value(@config[key])
+                .with(key: key, parent: self)
 
-            factory.create!
+              @entries[key] = factory.create!
+            end
           end
 
           class_methods do
@@ -38,22 +42,23 @@ module Gitlab
 
             private
 
-            def node(symbol, entry_class, metadata)
-              factory = Node::Factory.new(entry_class)
+            def node(key, node, metadata)
+              factory = Node::Factory.new(node)
                 .with(description: metadata[:description])
 
-              (@nodes ||= {}).merge!(symbol.to_sym => factory)
+              (@nodes ||= {}).merge!(key.to_sym => factory)
             end
 
             def helpers(*nodes)
               nodes.each do |symbol|
                 define_method("#{symbol}_defined?") do
-                  @nodes[symbol].try(:defined?)
+                  @entries[symbol].specified? if @entries[symbol]
                 end
 
                 define_method("#{symbol}_value") do
-                  raise Entry::InvalidError unless valid?
-                  @nodes[symbol].try(:value)
+                  return unless @entries[symbol] && @entries[symbol].valid?
+
+                  @entries[symbol].value
                 end
 
                 alias_method symbol.to_sym, "#{symbol}_value".to_sym

lib/gitlab/ci/config/node/entry.rb

@@ -8,30 +8,31 @@ module Gitlab
         class Entry
           class InvalidError < StandardError; end
 
-          attr_reader :config
+          attr_reader :config, :metadata
           attr_accessor :key, :parent, :description
 
-          def initialize(config)
+          def initialize(config, **metadata)
             @config = config
-            @nodes = {}
+            @metadata = metadata
+            @entries = {}
+
             @validator = self.class.validator.new(self)
-            @validator.validate
+            @validator.validate(:new)
           end
 
           def process!
-            return if leaf?
             return unless valid?
 
             compose!
-            process_nodes!
+            descendants.each(&:process!)
           end
 
-          def nodes
-            @nodes.values
+          def leaf?
+            @entries.none?
           end
 
-          def leaf?
-            self.class.nodes.none?
+          def descendants
+            @entries.values
           end
 
           def ancestors
@@ -43,27 +44,30 @@ module Gitlab
           end
 
           def errors
-            @validator.messages + nodes.flat_map(&:errors)
+            @validator.messages + descendants.flat_map(&:errors)
           end
 
           def value
             if leaf?
               @config
             else
-              defined = @nodes.select { |_key, value| value.defined? }
-              Hash[defined.map { |key, node| [key, node.value] }]
+              meaningful = @entries.select do |_key, value|
+                value.specified? && value.relevant?
+              end
+
+              Hash[meaningful.map { |key, entry| [key, entry.value] }]
             end
           end
 
-          def defined?
+          def specified?
             true
           end
 
-          def self.default
+          def relevant?
+            true
           end
 
-          def self.nodes
-            {}
+          def self.default
           end
 
           def self.validator
@@ -73,17 +77,6 @@ module Gitlab
           private
 
           def compose!
-            self.class.nodes.each do |key, essence|
-              @nodes[key] = create_node(key, essence)
-            end
-          end
-
-          def process_nodes!
-            nodes.each(&:process!)
-          end
-
-          def create_node(key, essence)
-            raise NotImplementedError
           end
         end
       end

lib/gitlab/ci/config/node/factory.rb

@@ -10,35 +10,60 @@ module Gitlab
 
           def initialize(node)
             @node = node
+            @metadata = {}
             @attributes = {}
           end
 
+          def value(value)
+            @value = value
+            self
+          end
+
+          def metadata(metadata)
+            @metadata.merge!(metadata)
+            self
+          end
+
           def with(attributes)
             @attributes.merge!(attributes)
             self
           end
 
           def create!
-            raise InvalidFactory unless @attributes.has_key?(:value)
+            raise InvalidFactory unless defined?(@value)
 
-            fabricate.tap do |entry|
-              entry.key = @attributes[:key]
-              entry.parent = @attributes[:parent]
-              entry.description = @attributes[:description]
+            ##
+            # We assume that unspecified entry is undefined.
+            # See issue #18775.
+            #
+            if @value.nil?
+              Node::Undefined.new(
+                fabricate_undefined
+              )
+            else
+              fabricate(@node, @value)
             end
           end
 
           private
 
-          def fabricate
+          def fabricate_undefined
             ##
-            # We assume that unspecified entry is undefined.
-            # See issue #18775.
+            # If node has a default value we fabricate concrete node
+            # with default value.
             #
-            if @attributes[:value].nil?
-              Node::Undefined.new(@node)
+            if @node.default.nil?
+              fabricate(Node::Null)
             else
-              @node.new(@attributes[:value])
+              fabricate(@node, @node.default)
+            end
+          end
+
+          def fabricate(node, value = nil)
+            node.new(value, @metadata).tap do |entry|
+              entry.key = @attributes[:key]
+              entry.parent = @attributes[:parent]
+              entry.description = @attributes[:description]
             end
           end
         end

lib/gitlab/ci/config/node/global.rb

@@ -34,10 +34,36 @@ module Gitlab
             description: 'Configure caching between build jobs.'
 
           helpers :before_script, :image, :services, :after_script,
-                  :variables, :stages, :types, :cache
+                  :variables, :stages, :types, :cache, :jobs
 
-          def stages
-            stages_defined? ? stages_value : types_value
+          private
+
+          def compose!
+            super
+
+            compose_jobs!
+            compose_deprecated_entries!
+          end
+
+          def compose_jobs!
+            factory = Node::Factory.new(Node::Jobs)
+              .value(@config.except(*self.class.nodes.keys))
+              .with(key: :jobs, parent: self,
+                    description: 'Jobs definition for this pipeline')
+
+            @entries[:jobs] = factory.create!
+          end
+
+          def compose_deprecated_entries!
+            ##
+            # Deprecated `:types` key workaround - if types are defined and
+            # stages are not defined we use types definition as stages.
+            #
+            if types_defined? && !stages_defined?
+              @entries[:stages] = @entries[:types]
+            end
+
+            @entries.delete(:types)
           end
         end
       end

lib/gitlab/ci/config/node/hidden_job.rb

+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # Entry that represents a hidden CI/CD job.
+        #
+        class HiddenJob < Entry
+          include Validatable
+
+          validations do
+            validates :config, type: Hash
+            validates :config, presence: true
+          end
+
+          def relevant?
+            false
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config/node/job.rb

+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # Entry that represents a concrete CI/CD job.
+        #
+        class Job < Entry
+          include Configurable
+          include Attributable
+
+          ALLOWED_KEYS = %i[tags script only except type image services allow_failure
+                            type stage when artifacts cache dependencies before_script
+                            after_script variables environment]
+
+          attributes :tags, :allow_failure, :when, :environment, :dependencies
+
+          validations do
+            validates :config, allowed_keys: ALLOWED_KEYS
+
+            validates :config, presence: true
+            validates :name, presence: true
+            validates :name, type: Symbol
+
+            with_options allow_nil: true do
+              validates :tags, array_of_strings: true
+              validates :allow_failure, boolean: true
+              validates :when,
+                inclusion: { in: %w[on_success on_failure always manual],
+                             message: 'should be on_success, on_failure, ' \
+                                      'always or manual' }
+              validates :environment,
+                type: {
+                  with: String,
+                  message: Gitlab::Regex.environment_name_regex_message }
+              validates :environment,
+                format: {
+                  with: Gitlab::Regex.environment_name_regex,
+                  message: Gitlab::Regex.environment_name_regex_message }
+
+              validates :dependencies, array_of_strings: true
+            end
+          end
+
+          node :before_script, Script,
+            description: 'Global before script overridden in this job.'
+
+          node :script, Commands,
+            description: 'Commands that will be executed in this job.'
+
+          node :stage, Stage,
+            description: 'Pipeline stage this job will be executed into.'
+
+          node :type, Stage,
+            description: 'Deprecated: stage this job will be executed into.'
+
+          node :after_script, Script,
+            description: 'Commands that will be executed when finishing job.'
+
+          node :cache, Cache,
+            description: 'Cache definition for this job.'
+
+          node :image, Image,
+            description: 'Image that will be used to execute this job.'
+
+          node :services, Services,
+            description: 'Services that will be used to execute this job.'
+
+          node :only, Trigger,
+            description: 'Refs policy this job will be executed for.'
+
+          node :except, Trigger,
+            description: 'Refs policy this job will be executed for.'
+
+          node :variables, Variables,
+            description: 'Environment variables available for this job.'
+
+          node :artifacts, Artifacts,
+            description: 'Artifacts configuration for this job.'
+
+          helpers :before_script, :script, :stage, :type, :after_script,
+                  :cache, :image, :services, :only, :except, :variables,
+                  :artifacts
+
+          def name
+            @metadata[:name]
+          end
+
+          def value
+            @config.merge(to_hash.compact)
+          end
+
+          private
+
+          def to_hash
+            { name: name,
+              before_script: before_script,
+              script: script,
+              image: image,
+              services: services,
+              stage: stage,
+              cache: cache,
+              only: only,
+              except: except,
+              variables: variables_defined? ? variables : nil,
+              artifacts: artifacts,
+              after_script: after_script }
+          end
+
+          def compose!
+            super
+
+            if type_defined? && !stage_defined?
+              @entries[:stage] = @entries[:type]
+            end
+
+            @entries.delete(:type)
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config/node/jobs.rb

+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # Entry that represents a set of jobs.
+        #
+        class Jobs < Entry
+          include Validatable
+
+          validations do
+            validates :config, type: Hash
+
+            validate do
+              unless has_visible_job?
+                errors.add(:config, 'should contain at least one visible job')
+              end
+            end
+
+            def has_visible_job?
+              config.any? { |name, _| !hidden?(name) }
+            end
+          end
+
+          def hidden?(name)
+            name.to_s.start_with?('.')
+          end
+
+          private
+
+          def compose!
+            @config.each do |name, config|
+              node = hidden?(name) ? Node::HiddenJob : Node::Job
+
+              factory = Node::Factory.new(node)
+                .value(config || {})
+                .metadata(name: name)
+                .with(key: name, parent: self,
+                      description: "#{name} job definition.")
+
+              @entries[name] = factory.create!
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config/node/legacy_validation_helpers.rb

@@ -41,10 +41,6 @@ module Gitlab
             false
           end
 
-          def validate_environment(value)
-            value.is_a?(String) && value =~ Gitlab::Regex.environment_name_regex
-          end
-
           def validate_boolean(value)
             value.in?([true, false])
           end

lib/gitlab/ci/config/node/null.rb

+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # This class represents an undefined node.
+        #
+        # Implements the Null Object pattern.
+        #
+        class Null < Entry
+          def value
+            nil
+          end
+
+          def valid?
+            true
+          end
+
+          def errors
+            []
+          end
+
+          def specified?
+            false
+          end
+
+          def relevant?
+            false
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config/node/stage.rb

+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # Entry that represents a stage for a job.
+        #
+        class Stage < Entry
+          include Validatable
+
+          validations do
+            validates :config, type: String
+          end
+
+          def self.default
+            'test'
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config/node/trigger.rb

+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # Entry that represents a trigger policy for the job.
+        #
+        class Trigger < Entry
+          include Validatable
+
+          validations do
+            include LegacyValidationHelpers
+
+            validate :array_of_strings_or_regexps
+
+            def array_of_strings_or_regexps
+              unless validate_array_of_strings_or_regexps(config)
+                errors.add(:config, 'should be an array of strings or regexps')
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config/node/undefined.rb

@@ -3,24 +3,13 @@ module Gitlab
     class Config
       module Node
         ##
-        # This class represents an undefined entry node.
+        # This class represents an unspecified entry node.
         #
-        # It takes original entry class as configuration and returns default
-        # value of original entry as self value.
+        # It decorates original entry adding method that indicates it is
+        # unspecified.
         #
-        #
-        class Undefined < Entry
-          include Validatable
-
-          validations do
-            validates :config, type: Class
-          end
-
-          def value
-            @config.default
-          end
-
-          def defined?
+        class Undefined < SimpleDelegator
+          def specified?
             false
           end
         end

lib/gitlab/ci/config/node/validator.rb

@@ -21,18 +21,19 @@ module Gitlab
             'Validator'
           end
 
-          def unknown_keys
-            return [] unless config.is_a?(Hash)
-
-            config.keys - @node.class.nodes.keys
-          end
-
           private
 
           def location
             predecessors = ancestors.map(&:key).compact
-            current = key || @node.class.name.demodulize.underscore
-            predecessors.append(current).join(':')
+            predecessors.append(key_name).join(':')
+          end
+
+          def key_name
+            if key.blank?
+              @node.class.name.demodulize.underscore.humanize
+            else
+              key
+            end
           end
         end
       end

lib/gitlab/ci/config/node/validators.rb

@@ -5,10 +5,11 @@ module Gitlab
         module Validators
           class AllowedKeysValidator < ActiveModel::EachValidator
             def validate_each(record, attribute, value)
-              if record.unknown_keys.any?
-                unknown_list = record.unknown_keys.join(', ')
-                record.errors.add(:config,
-                                  "contains unknown keys: #{unknown_list}")
+              unknown_keys = record.config.try(:keys).to_a - options[:in]
+
+              if unknown_keys.any?
+                record.errors.add(:config, 'contains unknown keys: ' +
+                                            unknown_keys.join(', '))
               end
             end
           end
@@ -33,6 +34,16 @@ module Gitlab
             end
           end
 
+          class DurationValidator < ActiveModel::EachValidator
+            include LegacyValidationHelpers
+
+            def validate_each(record, attribute, value)
+              unless validate_duration(value)
+                record.errors.add(attribute, 'should be a duration')
+              end
+            end
+          end
+
           class KeyValidator < ActiveModel::EachValidator
             include LegacyValidationHelpers
 
@@ -49,7 +60,8 @@ module Gitlab
               raise unless type.is_a?(Class)
 
               unless value.is_a?(type)
-                record.errors.add(attribute, "should be a #{type.name}")
+                message = options[:message] || "should be a #{type.name}"
+                record.errors.add(attribute, message)
               end
             end
           end

spec/lib/gitlab/ci/config/node/artifacts_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Config::Node::Artifacts do
+  let(:entry) { described_class.new(config) }
+
+  describe 'validation' do
+    context 'when entry config value is correct' do
+      let(:config) { { paths: %w[public/] } }
+
+      describe '#value' do
+        it 'returns artifacs configuration' do
+          expect(entry.value).to eq config
+        end
+      end
+
+      describe '#valid?' do
+        it 'is valid' do
+          expect(entry).to be_valid
+        end
+      end
+    end
+
+    context 'when entry value is not correct' do
+      describe '#errors' do
+        context 'when value of attribute is invalid' do
+          let(:config) { { name: 10 } }
+
+          it 'reports error' do
+            expect(entry.errors)
+              .to include 'artifacts name should be a string'
+          end
+        end
+
+        context 'when there is an unknown key present' do
+          let(:config) { { test: 100 } }
+
+          it 'reports error' do
+            expect(entry.errors)
+              .to include 'artifacts config contains unknown keys: test'
+          end
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/config/node/attributable_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Config::Node::Attributable do
+  let(:node) { Class.new }
+  let(:instance) { node.new }
+
+  before do
+    node.include(described_class)
+
+    node.class_eval do
+      attributes :name, :test
+    end
+  end
+
+  context 'config is a hash' do
+    before do
+      allow(instance)
+        .to receive(:config)
+        .and_return({ name: 'some name', test: 'some test' })
+    end
+
+    it 'returns the value of config' do
+      expect(instance.name).to eq 'some name'
+      expect(instance.test).to eq 'some test'
+    end
+
+    it 'returns no method error for unknown attributes' do
+      expect { instance.unknown }.to raise_error(NoMethodError)
+    end
+  end
+
+  context 'config is not a hash' do
+    before do
+      allow(instance)
+        .to receive(:config)
+        .and_return('some test')
+    end
+
+    it 'returns nil' do
+      expect(instance.test).to be_nil
+    end
+  end
+end

spec/lib/gitlab/ci/config/node/commands_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Config::Node::Commands do
+  let(:entry) { described_class.new(config) }
+
+  context 'when entry config value is an array' do
+    let(:config) { ['ls', 'pwd'] }
+
+    describe '#value' do
+      it 'returns array of strings' do
+        expect(entry.value).to eq config
+      end
+    end
+
+    describe '#errors' do
+      it 'does not append errors' do
+        expect(entry.errors).to be_empty
+      end
+    end
+  end
+
+  context 'when entry config value is a string' do
+    let(:config) { 'ls' }
+
+    describe '#value' do
+      it 'returns array with single element' do
+        expect(entry.value).to eq ['ls']
+      end
+    end
+
+    describe '#valid?' do
+      it 'is valid' do
+        expect(entry).to be_valid
+      end
+    end
+  end
+
+  context 'when entry value is not valid' do
+    let(:config) { 1 }
+
+    describe '#errors' do
+      it 'saves errors' do
+        expect(entry.errors)
+          .to include 'commands config should be a ' \
+                      'string or an array of strings'
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/config/node/factory_spec.rb

@@ -2,13 +2,13 @@ require 'spec_helper'
 
 describe Gitlab::Ci::Config::Node::Factory do
   describe '#create!' do
-    let(:factory) { described_class.new(entry_class) }
-    let(:entry_class) { Gitlab::Ci::Config::Node::Script }
+    let(:factory) { described_class.new(node) }
+    let(:node) { Gitlab::Ci::Config::Node::Script }
 
-    context 'when setting up a value' do
+    context 'when setting a concrete value' do
       it 'creates entry with valid value' do
         entry = factory
-          .with(value: ['ls', 'pwd'])
+          .value(['ls', 'pwd'])
           .create!
 
         expect(entry.value).to eq ['ls', 'pwd']
@@ -17,7 +17,7 @@ describe Gitlab::Ci::Config::Node::Factory do
       context 'when setting description' do
         it 'creates entry with description' do
           entry = factory
-            .with(value: ['ls', 'pwd'])
+            .value(['ls', 'pwd'])
             .with(description: 'test description')
             .create!
 
@@ -29,7 +29,8 @@ describe Gitlab::Ci::Config::Node::Factory do
       context 'when setting key' do
         it 'creates entry with custom key' do
           entry = factory
-            .with(value: ['ls', 'pwd'], key: 'test key')
+            .value(['ls', 'pwd'])
+            .with(key: 'test key')
             .create!
 
           expect(entry.key).to eq 'test key'
@@ -37,19 +38,20 @@ describe Gitlab::Ci::Config::Node::Factory do
       end
 
       context 'when setting a parent' do
-        let(:parent) { Object.new }
+        let(:object) { Object.new }
 
         it 'creates entry with valid parent' do
           entry = factory
-            .with(value: 'ls', parent: parent)
+            .value('ls')
+            .with(parent: object)
             .create!
 
-          expect(entry.parent).to eq parent
+          expect(entry.parent).to eq object
         end
       end
     end
 
-    context 'when not setting up a value' do
+    context 'when not setting a value' do
       it 'raises error' do
         expect { factory.create! }.to raise_error(
           Gitlab::Ci::Config::Node::Factory::InvalidFactory
@@ -60,11 +62,25 @@ describe Gitlab::Ci::Config::Node::Factory do
     context 'when creating entry with nil value' do
       it 'creates an undefined entry' do
         entry = factory
-          .with(value: nil)
+          .value(nil)
           .create!
 
         expect(entry).to be_an_instance_of Gitlab::Ci::Config::Node::Undefined
       end
     end
+
+    context 'when passing metadata' do
+      let(:node) { spy('node') }
+
+      it 'passes metadata as a parameter' do
+        factory
+          .value('some value')
+          .metadata(some: 'hash')
+          .create!
+
+        expect(node).to have_received(:new)
+          .with('some value', { some: 'hash' })
+      end
+    end
   end
 end

spec/lib/gitlab/ci/config/node/global_spec.rb

@@ -22,38 +22,40 @@ describe Gitlab::Ci::Config::Node::Global do
           variables: { VAR: 'value' },
           after_script: ['make clean'],
           stages: ['build', 'pages'],
-          cache: { key: 'k', untracked: true, paths: ['public/'] } }
+          cache: { key: 'k', untracked: true, paths: ['public/'] },
+          rspec: { script: %w[rspec ls] },
+          spinach: { script: 'spinach' } }
       end
 
       describe '#process!' do
         before { global.process! }
 
         it 'creates nodes hash' do
-          expect(global.nodes).to be_an Array
+          expect(global.descendants).to be_an Array
         end
 
         it 'creates node object for each entry' do
-          expect(global.nodes.count).to eq 8
+          expect(global.descendants.count).to eq 8
         end
 
         it 'creates node object using valid class' do
-          expect(global.nodes.first)
+          expect(global.descendants.first)
             .to be_an_instance_of Gitlab::Ci::Config::Node::Script
-          expect(global.nodes.second)
+          expect(global.descendants.second)
             .to be_an_instance_of Gitlab::Ci::Config::Node::Image
         end
 
         it 'sets correct description for nodes' do
-          expect(global.nodes.first.description)
+          expect(global.descendants.first.description)
             .to eq 'Script that will be executed before each job.'
-          expect(global.nodes.second.description)
+          expect(global.descendants.second.description)
             .to eq 'Docker image that will be used to execute jobs.'
         end
-      end
 
-      describe '#leaf?' do
-        it 'is not leaf' do
-          expect(global).not_to be_leaf
+        describe '#leaf?' do
+          it 'is not leaf' do
+            expect(global).not_to be_leaf
+          end
         end
       end
 
@@ -63,6 +65,12 @@ describe Gitlab::Ci::Config::Node::Global do
             expect(global.before_script).to be nil
           end
         end
+
+        describe '#leaf?' do
+          it 'is leaf' do
+            expect(global).to be_leaf
+          end
+        end
       end
 
       context 'when processed' do
@@ -106,7 +114,10 @@ describe Gitlab::Ci::Config::Node::Global do
           end
 
           context 'when deprecated types key defined' do
-            let(:hash) { { types: ['test', 'deploy'] } }
+            let(:hash) do
+              { types: ['test', 'deploy'],
+                rspec: { script: 'rspec' } }
+            end
 
             it 'returns array of types as stages' do
               expect(global.stages).to eq %w[test deploy]
@@ -120,20 +131,33 @@ describe Gitlab::Ci::Config::Node::Global do
               .to eq(key: 'k', untracked: true, paths: ['public/'])
           end
         end
+
+        describe '#jobs' do
+          it 'returns jobs configuration' do
+            expect(global.jobs).to eq(
+              rspec: { name: :rspec,
+                       script: %w[rspec ls],
+                       stage: 'test' },
+              spinach: { name: :spinach,
+                         script: %w[spinach],
+                         stage: 'test' }
+            )
+          end
+        end
       end
     end
 
     context 'when most of entires not defined' do
-      let(:hash) { { cache: { key: 'a' }, rspec: {} } }
+      let(:hash) { { cache: { key: 'a' }, rspec: { script: %w[ls] } } }
       before { global.process! }
 
       describe '#nodes' do
         it 'instantizes all nodes' do
-          expect(global.nodes.count).to eq 8
+          expect(global.descendants.count).to eq 8
         end
 
         it 'contains undefined nodes' do
-          expect(global.nodes.first)
+          expect(global.descendants.first)
             .to be_an_instance_of Gitlab::Ci::Config::Node::Undefined
         end
       end
@@ -164,7 +188,7 @@ describe Gitlab::Ci::Config::Node::Global do
     # details.
     #
     context 'when entires specified but not defined' do
-      let(:hash) { { variables: nil } }
+      let(:hash) { { variables: nil, rspec: { script: 'rspec' } } }
       before { global.process! }
 
       describe '#variables' do
@@ -196,10 +220,8 @@ describe Gitlab::Ci::Config::Node::Global do
     end
 
     describe '#before_script' do
-      it 'raises error' do
-        expect { global.before_script }.to raise_error(
-          Gitlab::Ci::Config::Node::Entry::InvalidError
-        )
+      it 'returns nil' do
+        expect(global.before_script).to be_nil
       end
     end
   end
@@ -220,9 +242,9 @@ describe Gitlab::Ci::Config::Node::Global do
     end
   end
 
-  describe '#defined?' do
+  describe '#specified?' do
     it 'is concrete entry that is defined' do
-      expect(global.defined?).to be true
+      expect(global.specified?).to be true
     end
   end
 end

spec/lib/gitlab/ci/config/node/hidden_job_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Config::Node::HiddenJob do
+  let(:entry) { described_class.new(config) }
+
+  describe 'validations' do
+    context 'when entry config value is correct' do
+      let(:config) { { image: 'ruby:2.2' } }
+
+      describe '#value' do
+        it 'returns key value' do
+          expect(entry.value).to eq(image: 'ruby:2.2')
+        end
+      end
+
+      describe '#valid?' do
+        it 'is valid' do
+          expect(entry).to be_valid
+        end
+      end
+    end
+
+    context 'when entry value is not correct' do
+      context 'incorrect config value type' do
+        let(:config) { ['incorrect'] }
+
+        describe '#errors' do
+          it 'saves errors' do
+            expect(entry.errors)
+              .to include 'hidden job config should be a hash'
+          end
+        end
+      end
+
+      context 'when config is empty' do
+        let(:config) { {} }
+
+        describe '#valid' do
+          it 'is invalid' do
+            expect(entry).not_to be_valid
+          end
+        end
+      end
+    end
+  end
+
+  describe '#leaf?' do
+    it 'is a leaf' do
+      expect(entry).to be_leaf
+    end
+  end
+
+  describe '#relevant?' do
+    it 'is not a relevant entry' do
+      expect(entry).not_to be_relevant
+    end
+  end
+end

spec/lib/gitlab/ci/config/node/job_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Config::Node::Job do
+  let(:entry) { described_class.new(config, name: :rspec) }
+
+  before { entry.process! }
+
+  describe 'validations' do
+    context 'when entry config value is correct' do
+      let(:config) { { script: 'rspec' } }
+
+      describe '#valid?' do
+        it 'is valid' do
+          expect(entry).to be_valid
+        end
+      end
+
+      context 'when job name is empty' do
+        let(:entry) { described_class.new(config, name: ''.to_sym) }
+
+        it 'reports error' do
+          expect(entry.errors)
+            .to include "job name can't be blank"
+        end
+      end
+    end
+
+    context 'when entry value is not correct' do
+      context 'incorrect config value type' do
+        let(:config) { ['incorrect'] }
+
+        describe '#errors' do
+          it 'reports error about a config type' do
+            expect(entry.errors)
+              .to include 'job config should be a hash'
+          end
+        end
+      end
+
+      context 'when config is empty' do
+        let(:config) { {} }
+
+        describe '#valid' do
+          it 'is invalid' do
+            expect(entry).not_to be_valid
+          end
+        end
+      end
+
+      context 'when unknown keys detected' do
+        let(:config) { { unknown: true } }
+
+        describe '#valid' do
+          it 'is not valid' do
+            expect(entry).not_to be_valid
+          end
+        end
+      end
+    end
+  end
+
+  describe '#value' do
+    context 'when entry is correct' do
+      let(:config) do
+        { before_script: %w[ls pwd],
+          script: 'rspec',
+          after_script: %w[cleanup] }
+      end
+
+      it 'returns correct value' do
+        expect(entry.value)
+          .to eq(name: :rspec,
+                 before_script: %w[ls pwd],
+                 script: %w[rspec],
+                 stage: 'test',
+                 after_script: %w[cleanup])
+      end
+    end
+  end
+
+  describe '#relevant?' do
+    it 'is a relevant entry' do
+      expect(entry).to be_relevant
+    end
+  end
+end

spec/lib/gitlab/ci/config/node/jobs_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Config::Node::Jobs do
+  let(:entry) { described_class.new(config) }
+
+  describe 'validations' do
+    before { entry.process! }
+
+    context 'when entry config value is correct' do
+      let(:config) { { rspec: { script: 'rspec' } } }
+
+      describe '#valid?' do
+        it 'is valid' do
+          expect(entry).to be_valid
+        end
+      end
+    end
+
+    context 'when entry value is not correct' do
+      describe '#errors' do
+        context 'incorrect config value type' do
+          let(:config) { ['incorrect'] }
+
+          it 'returns error about incorrect type' do
+            expect(entry.errors)
+              .to include 'jobs config should be a hash'
+          end
+        end
+
+        context 'when job is unspecified' do
+          let(:config) { { rspec: nil } }
+
+          it 'reports error' do
+            expect(entry.errors).to include "rspec config can't be blank"
+          end
+        end
+
+        context 'when no visible jobs present' do
+          let(:config) { { '.hidden'.to_sym => { script: [] } } }
+
+          it 'returns error about no visible jobs defined' do
+            expect(entry.errors)
+              .to include 'jobs config should contain at least one visible job'
+          end
+        end
+      end
+    end
+  end
+
+  context 'when valid job entries processed' do
+    before { entry.process! }
+
+    let(:config) do
+      { rspec: { script: 'rspec' },
+        spinach: { script: 'spinach' },
+        '.hidden'.to_sym => {} }
+    end
+
+    describe '#value' do
+      it 'returns key value' do
+        expect(entry.value).to eq(
+          rspec: { name: :rspec,
+                   script: %w[rspec],
+                   stage: 'test' },
+          spinach: { name: :spinach,
+                     script: %w[spinach],
+                     stage: 'test' })
+      end
+    end
+
+    describe '#descendants' do
+      it 'creates valid descendant nodes' do
+        expect(entry.descendants.count).to eq 3
+        expect(entry.descendants.first(2))
+          .to all(be_an_instance_of(Gitlab::Ci::Config::Node::Job))
+        expect(entry.descendants.last)
+          .to be_an_instance_of(Gitlab::Ci::Config::Node::HiddenJob)
+      end
+    end
+
+    describe '#value' do
+      it 'returns value of visible jobs only' do
+        expect(entry.value.keys).to eq [:rspec, :spinach]
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/config/node/null_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Config::Node::Null do
+  let(:null) { described_class.new(nil) }
+
+  describe '#leaf?' do
+    it 'is leaf node' do
+      expect(null).to be_leaf
+    end
+  end
+
+  describe '#valid?' do
+    it 'is always valid' do
+      expect(null).to be_valid
+    end
+  end
+
+  describe '#errors' do
+    it 'is does not contain errors' do
+      expect(null.errors).to be_empty
+    end
+  end
+
+  describe '#value' do
+    it 'returns nil' do
+      expect(null.value).to eq nil
+    end
+  end
+
+  describe '#relevant?' do
+    it 'is not relevant' do
+      expect(null.relevant?).to eq false
+    end
+  end
+
+  describe '#specified?' do
+    it 'is not defined' do
+      expect(null.specified?).to eq false
+    end
+  end
+end

spec/lib/gitlab/ci/config/node/stage_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Config::Node::Stage do
+  let(:stage) { described_class.new(config) }
+
+  describe 'validations' do
+    context 'when stage config value is correct' do
+      let(:config) { 'build' }
+
+      describe '#value' do
+        it 'returns a stage key' do
+          expect(stage.value).to eq config
+        end
+      end
+
+      describe '#valid?' do
+        it 'is valid' do
+          expect(stage).to be_valid
+        end
+      end
+    end
+
+    context 'when value has a wrong type' do
+      let(:config) { { test: true } }
+
+      it 'reports errors about wrong type' do
+        expect(stage.errors)
+          .to include 'stage config should be a string'
+      end
+    end
+  end
+
+  describe '.default' do
+    it 'returns default stage' do
+      expect(described_class.default).to eq 'test'
+    end
+  end
+end

spec/lib/gitlab/ci/config/node/trigger_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Config::Node::Trigger do
+  let(:entry) { described_class.new(config) }
+
+  describe 'validations' do
+    context 'when entry config value is valid' do
+      context 'when config is a branch or tag name' do
+        let(:config) { %w[master feature/branch] }
+
+        describe '#valid?' do
+          it 'is valid' do
+            expect(entry).to be_valid
+          end
+        end
+
+        describe '#value' do
+          it 'returns key value' do
+            expect(entry.value).to eq config
+          end
+        end
+      end
+
+      context 'when config is a regexp' do
+        let(:config) { ['/^issue-.*$/'] }
+
+        describe '#valid?' do
+          it 'is valid' do
+            expect(entry).to be_valid
+          end
+        end
+      end
+
+      context 'when config is a special keyword' do
+        let(:config) { %w[tags triggers branches] }
+
+        describe '#valid?' do
+          it 'is valid' do
+            expect(entry).to be_valid
+          end
+        end
+      end
+    end
+
+    context 'when entry value is not valid' do
+      let(:config) { [1] }
+
+      describe '#errors' do
+        it 'saves errors' do
+          expect(entry.errors)
+            .to include 'trigger config should be an array of strings or regexps'
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/config/node/undefined_spec.rb

@@ -2,39 +2,31 @@ require 'spec_helper'
 
 describe Gitlab::Ci::Config::Node::Undefined do
   let(:undefined) { described_class.new(entry) }
-  let(:entry) { Class.new }
-
-  describe '#leaf?' do
-    it 'is leaf node' do
-      expect(undefined).to be_leaf
-    end
-  end
+  let(:entry) { spy('Entry') }
 
   describe '#valid?' do
-    it 'is always valid' do
-      expect(undefined).to be_valid
+    it 'delegates method to entry' do
+      expect(undefined.valid).to eq entry
     end
   end
 
   describe '#errors' do
-    it 'is does not contain errors' do
-      expect(undefined.errors).to be_empty
+    it 'delegates method to entry' do
+      expect(undefined.errors).to eq entry
     end
   end
 
   describe '#value' do
-    before do
-      allow(entry).to receive(:default).and_return('some value')
-    end
-
-    it 'returns default value for entry' do
-      expect(undefined.value).to eq 'some value'
+    it 'delegates method to entry' do
+      expect(undefined.value).to eq entry
     end
   end
 
-  describe '#undefined?' do
-    it 'is not a defined entry' do
-      expect(undefined.defined?).to be false
+  describe '#specified?' do
+    it 'is always false' do
+      allow(entry).to receive(:specified?).and_return(true)
+
+      expect(undefined.specified?).to be false
     end
   end
 end