Merge branch...

Merge branch 'ee-42861-move-include-external-files-in-gitlab-ci-yml-from-starter-to-libre' into 'master'

EE Resolve "Move "include external files in .gitlab-ci.yml" from Starter to Libre"

See merge request gitlab-org/gitlab-ee!7292

app/models/ci/pipeline.rb

@@ -477,7 +477,7 @@ module Ci
       return @config_processor if defined?(@config_processor)
 
       @config_processor ||= begin
-        initialize_yaml_processor
+        ::Gitlab::Ci::YamlProcessor.new(ci_yaml_file, { project: project, sha: sha })
       rescue Gitlab::Ci::YamlProcessor::ValidationError => e
         self.yaml_errors = e.message
         nil
@@ -487,10 +487,6 @@ module Ci
       end
     end
 
-    def initialize_yaml_processor
-      Gitlab::Ci::YamlProcessor.new(ci_yaml_file)
-    end
-
     def ci_yaml_file_path
       if project.ci_config_path.blank?
         '.gitlab-ci.yml'

app/models/repository.rb

@@ -1005,14 +1005,6 @@ class Repository
                                        remote_branch: merge_request.target_branch)
   end
 
-  def blob_data_at(sha, path)
-    blob = blob_at(sha, path)
-    return unless blob
-
-    blob.load_all_data!
-    blob.data
-  end
-
   def squash(user, merge_request)
     raw.squash(user, merge_request.id, branch: merge_request.target_branch,
                                        start_sha: merge_request.diff_start_sha,
@@ -1021,6 +1013,14 @@ class Repository
                                        message: merge_request.title)
   end
 
+  def blob_data_at(sha, path)
+    blob = blob_at(sha, path)
+    return unless blob
+
+    blob.load_all_data!
+    blob.data
+  end
+
   private
 
   # TODO Generice finder, later split this on finders by Ref or Oid

doc/ci/yaml/README.md

@@ -1354,11 +1354,12 @@ test:
   retry: 2
 ```
 
-## `include` **[STARTER]**
+## `include`
 
 > Introduced in [GitLab Edition Premium][ee] 10.5.
 > Available for Starter, Premium and Ultimate [versions][gitlab-versions] since 10.6.
-> Behaviour expanded in GitLab 10.8 to allow more flexible overriding
+> Behaviour expanded in GitLab 10.8 to allow more flexible overriding.
+> Available for Libre since [11.4](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/21603)
 
 Using the `include` keyword, you can allow the inclusion of external YAML files.
 

ee/app/models/ee/ci/pipeline.rb

@@ -54,10 +54,6 @@ module EE
         @dast_artifact ||= artifacts_with_files.find(&:has_dast_json?)
       end
 
-      def initialize_yaml_processor
-        ::Gitlab::Ci::YamlProcessor.new(ci_yaml_file, { project: project, sha: sha })
-      end
-
       def has_sast_data?
         sast_artifact&.success?
       end

ee/app/models/license.rb

@@ -13,7 +13,6 @@ class License < ActiveRecord::Base
     contribution_analytics
     elastic_search
     export_issues
-    external_files_in_gitlab_ci
     group_burndown_charts
     group_webhooks
     issuable_default_templates

ee/lib/ee/gitlab/ci/config.rb

-module EE
-  module Gitlab
-    module Ci
-      #
-      # EE Base GitLab CI configuration facade
-      #
-      module Config
-        def initialize(config, opts = {})
-          super
-        rescue ::Gitlab::Ci::External::Processor::FileError => e
-          raise ::Gitlab::Ci::YamlProcessor::ValidationError, e.message
-        end
-
-        private
-
-        def build_config(config, opts = {})
-          initial_config = ::Gitlab::Ci::Config::Loader.new(config).load!
-          project = opts.fetch(:project, nil)
-
-          if project&.feature_available?(:external_files_in_gitlab_ci)
-            process_external_files(initial_config, project, opts)
-          elsif initial_config.include?(:include)
-            raise ::Gitlab::Ci::YamlProcessor::ValidationError, "Your license does not allow to use 'include' keyword in CI/CD configuration file"
-          else
-            initial_config
-          end
-        end
-
-        def process_external_files(config, project, opts)
-          sha = opts.fetch(:sha) { project.repository.root_ref_sha }
-          ::Gitlab::Ci::External::Processor.new(config, project, sha).perform
-        end
-      end
-    end
-  end
-end

ee/spec/lib/ee/gitlab/ci/config_spec.rb

-require 'spec_helper'
-
-describe EE::Gitlab::Ci::Config do
-  let(:config_class) { ::Gitlab::Ci::Config }
-  let(:project) { create(:project, :repository) }
-  let(:remote_location) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' }
-  let(:local_location) { 'ee/spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml' }
-
-  let(:remote_file_content) do
-    <<~HEREDOC
-      variables:
-        AUTO_DEVOPS_DOMAIN: domain.example.com
-        POSTGRES_USER: user
-        POSTGRES_PASSWORD: testing-password
-        POSTGRES_ENABLED: "true"
-        POSTGRES_DB: $CI_ENVIRONMENT_SLUG
-    HEREDOC
-  end
-
-  let(:local_file_content) do
-    File.read(Rails.root.join(local_location))
-  end
-
-  let(:gitlab_ci_yml) do
-    <<~HEREDOC
-      include:
-        - #{local_location}
-        - #{remote_location}
-
-      image: ruby:2.2
-    HEREDOC
-  end
-
-  let(:config) do
-    config_class.new(gitlab_ci_yml, project: project, sha: '12345')
-  end
-
-  before do
-    WebMock.stub_request(:get, remote_location)
-      .to_return(body: remote_file_content)
-
-    allow(project.repository)
-      .to receive(:blob_data_at).and_return(local_file_content)
-  end
-
-  context 'when the project does not have a valid license' do
-    before do
-      allow(project).to receive(:feature_available?)
-        .with(:external_files_in_gitlab_ci).and_return(false)
-    end
-
-    it "should raise a ValidationError" do
-      expect { config }.to raise_error(
-        ::Gitlab::Ci::YamlProcessor::ValidationError,
-        "Your license does not allow to use 'include' keyword in CI/CD configuration file"
-      )
-    end
-  end
-
-  context 'when the project has a valid license' do
-    before do
-      allow(project).to receive(:feature_available?)
-        .with(:external_files_in_gitlab_ci).and_return(true)
-    end
-
-    context "when gitlab_ci_yml has valid 'include' defined" do
-      before do
-      end
-
-      it 'should return a composed hash' do
-        before_script_values = [
-          "apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs", "ruby -v",
-          "which ruby",
-          "gem install bundler --no-ri --no-rdoc",
-          "bundle install --jobs $(nproc)  \"${FLAGS[@]}\""
-        ]
-        variables = {
-          AUTO_DEVOPS_DOMAIN: "domain.example.com",
-          POSTGRES_USER: "user",
-          POSTGRES_PASSWORD: "testing-password",
-          POSTGRES_ENABLED: "true",
-          POSTGRES_DB: "$CI_ENVIRONMENT_SLUG"
-        }
-        composed_hash = {
-          before_script: before_script_values,
-          image: "ruby:2.2",
-          rspec: { script: ["bundle exec rspec"] },
-          variables: variables
-        }
-
-        expect(config.to_hash).to eq(composed_hash)
-      end
-    end
-
-    context "when gitlab_ci.yml has invalid 'include' defined"  do
-      let(:gitlab_ci_yml) do
-        <<~HEREDOC
-          include: invalid
-        HEREDOC
-      end
-
-      it 'raises error YamlProcessor validationError' do
-        expect { config }.to raise_error(
-          ::Gitlab::Ci::YamlProcessor::ValidationError,
-          "Local file 'invalid' is not valid."
-        )
-      end
-    end
-
-    describe 'external file version' do
-      context 'when external local file SHA is defined' do
-        it 'is using a defined value' do
-          expect(project.repository).to receive(:blob_data_at)
-            .with('eeff1122', local_location)
-
-          config_class.new(gitlab_ci_yml, project: project, sha: 'eeff1122')
-        end
-      end
-
-      context 'when external local file SHA is not defined' do
-        it 'is using latest SHA on the default branch' do
-          expect(project.repository).to receive(:root_ref_sha)
-
-          config_class.new(gitlab_ci_yml, project: project)
-        end
-      end
-    end
-
-    context "when both external files and gitlab_ci.yml defined the same key" do
-      let(:gitlab_ci_yml) do
-        <<~HEREDOC
-        include:
-          - #{remote_location}
-
-        image: ruby:2.2
-        HEREDOC
-      end
-
-      let(:remote_file_content) do
-        <<~HEREDOC
-        image: php:5-fpm-alpine
-        HEREDOC
-      end
-
-      it 'should take precedence' do
-        expect(config.to_hash).to eq({ image: 'ruby:2.2' })
-      end
-    end
-
-    context "when both external files and gitlab_ci.yml define a dictionary of distinct variables" do
-      let(:remote_file_content) do
-        <<~HEREDOC
-        variables:
-          A: 'alpha'
-          B: 'beta'
-        HEREDOC
-      end
-
-      let(:gitlab_ci_yml) do
-        <<~HEREDOC
-        include:
-          - #{remote_location}
-
-        variables:
-          C: 'gamma'
-          D: 'delta'
-        HEREDOC
-      end
-
-      it 'should merge the variables dictionaries' do
-        expect(config.to_hash).to eq({ variables: { A: 'alpha', B: 'beta', C: 'gamma', D: 'delta' } })
-      end
-    end
-
-    context "when both external files and gitlab_ci.yml define a dictionary of overlapping variables" do
-      let(:remote_file_content) do
-        <<~HEREDOC
-        variables:
-          A: 'alpha'
-          B: 'beta'
-          C: 'omnicron'
-        HEREDOC
-      end
-
-      let(:gitlab_ci_yml) do
-        <<~HEREDOC
-        include:
-          - #{remote_location}
-
-        variables:
-          C: 'gamma'
-          D: 'delta'
-        HEREDOC
-      end
-
-      it 'later declarations should take precedence' do
-        expect(config.to_hash).to eq({ variables: { A: 'alpha', B: 'beta', C: 'gamma', D: 'delta' } })
-      end
-    end
-
-    context 'when both external files and gitlab_ci.yml define a job' do
-      let(:remote_file_content) do
-        <<~HEREDOC
-        job1:
-          script:
-          - echo 'hello from remote file'
-        HEREDOC
-      end
-
-      let(:gitlab_ci_yml) do
-        <<~HEREDOC
-        include:
-          - #{remote_location}
-
-        job1:
-          variables:
-            VARIABLE_DEFINED_IN_MAIN_FILE: 'some value'
-        HEREDOC
-      end
-
-      it 'merges the jobs' do
-        expect(config.to_hash).to eq({
-          job1: {
-            script: ["echo 'hello from remote file'"],
-            variables: {
-              VARIABLE_DEFINED_IN_MAIN_FILE: 'some value'
-            }
-          }
-        })
-      end
-
-      context 'when the script key is in both' do
-        let(:gitlab_ci_yml) do
-          <<~HEREDOC
-          include:
-            - #{remote_location}
-
-          job1:
-            script:
-            - echo 'hello from main file'
-            variables:
-              VARIABLE_DEFINED_IN_MAIN_FILE: 'some value'
-          HEREDOC
-        end
-
-        it 'uses the script from the gitlab_ci.yml' do
-          expect(config.to_hash).to eq({
-            job1: {
-              script: ["echo 'hello from main file'"],
-              variables: {
-                VARIABLE_DEFINED_IN_MAIN_FILE: 'some value'
-              }
-            }
-          })
-        end
-      end
-    end
-  end
-end

lib/gitlab/ci/config.rb

@@ -4,8 +4,6 @@ module Gitlab
     # Base GitLab CI Configuration facade
     #
     class Config
-      prepend EE::Gitlab::Ci::Config
-
       ConfigError = Class.new(StandardError)
 
       def initialize(config, opts = {})
@@ -17,6 +15,8 @@ module Gitlab
         @global.compose!
       rescue Loader::FormatError, Extendable::ExtensionError => e
         raise Config::ConfigError, e.message
+      rescue ::Gitlab::Ci::External::Processor::FileError => e
+        raise ::Gitlab::Ci::YamlProcessor::ValidationError, e.message
       end
 
       def valid?
@@ -68,9 +68,20 @@ module Gitlab
 
       private
 
-      # 'opts' argument is used in EE see /ee/lib/ee/gitlab/ci/config.rb
       def build_config(config, opts = {})
-        Loader.new(config).load!
+        initial_config = Loader.new(config).load!
+        project = opts.fetch(:project, nil)
+
+        if project
+          process_external_files(initial_config, project, opts)
+        else
+          initial_config
+        end
+      end
+
+      def process_external_files(config, project, opts)
+        sha = opts.fetch(:sha) { project.repository.root_ref_sha }
+        ::Gitlab::Ci::External::Processor.new(config, project, sha).perform
       end
     end
   end

ee/lib/gitlab/ci/external/file/base.rb → lib/gitlab/ci/external/file/base.rb

+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module External

ee/lib/gitlab/ci/external/file/local.rb → lib/gitlab/ci/external/file/local.rb

+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module External

ee/lib/gitlab/ci/external/file/remote.rb → lib/gitlab/ci/external/file/remote.rb

+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module External

ee/lib/gitlab/ci/external/mapper.rb → lib/gitlab/ci/external/mapper.rb

+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module External

ee/lib/gitlab/ci/external/processor.rb → lib/gitlab/ci/external/processor.rb

+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module External

ee/spec/lib/gitlab/ci/external/file/local_spec.rb → spec/lib/gitlab/ci/external/file/local_spec.rb

+# frozen_string_literal: true
+
 require 'spec_helper'
 
 describe Gitlab::Ci::External::File::Local do

ee/spec/lib/gitlab/ci/external/file/remote_spec.rb → spec/lib/gitlab/ci/external/file/remote_spec.rb

+# frozen_string_literal: true
+
 require 'spec_helper'
 
 describe Gitlab::Ci::External::File::Remote do

ee/spec/lib/gitlab/ci/external/mapper_spec.rb → spec/lib/gitlab/ci/external/mapper_spec.rb

+# frozen_string_literal: true
+
 require 'spec_helper'
 
 describe Gitlab::Ci::External::Mapper do

ee/spec/lib/gitlab/ci/external/processor_spec.rb → spec/lib/gitlab/ci/external/processor_spec.rb

+# frozen_string_literal: true
+
 require 'spec_helper'
 
 describe Gitlab::Ci::External::Processor do
@@ -107,7 +109,7 @@ describe Gitlab::Ci::External::Processor do
       let(:remote_file) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' }
       let(:external_files) do
         [
-          '/ee/spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml',
+          '/spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml',
           remote_file
         ]
       end
@@ -128,7 +130,7 @@ describe Gitlab::Ci::External::Processor do
       end
 
       before do
-        local_file_content = File.read(Rails.root.join('ee/spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml'))
+        local_file_content = File.read(Rails.root.join('spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml'))
         allow_any_instance_of(Gitlab::Ci::External::File::Local).to receive(:fetch_local_content).and_return(local_file_content)
         WebMock.stub_request(:get, remote_file).to_return(body: remote_file_content)
       end