Merge branch 'ce-to-ee-2018-02-14' into 'master'

CE upstream - 2018-02-14 15:38 UTC

Closes #4914

See merge request gitlab-org/gitlab-ee!4540

PROCESS.md

@@ -71,11 +71,15 @@ star, smile, etc.). Some good tips about code reviews can be found in our
 
 ## Feature freeze on the 7th for the release on the 22nd
 
-After the 7th (Pacific Standard Time Zone) of each month, RC1 of the upcoming release (to be shipped on the 22nd) is created and deployed to GitLab.com and the stable branch for this release is frozen, which means master is no longer merged into it.
+After 7th at 23:59 (Pacific Standard Time Zone) of each month, RC1 of the upcoming release (to be shipped on the 22nd) is created and deployed to GitLab.com and the stable branch for this release is frozen, which means master is no longer merged into it.
 Merge requests may still be merged into master during this period,
 but they will go into the _next_ release, unless they are manually cherry-picked into the stable branch.
+
 By freezing the stable branches 2 weeks prior to a release, we reduce the risk of a last minute merge request potentially breaking things.
 
+Any release candidate that gets created after this date can become a final release,
+hence the name release candidate.
+
 ### Between the 1st and the 7th
 
 These types of merge requests for the upcoming release need special consideration:
@@ -193,11 +197,10 @@ to be backported down to the `9.5` release, you will need to assign it the
 ### Asking for an exception
 
 If you think a merge request should go into an RC or patch even though it does not meet these requirements,
-you can ask for an exception to be made. Exceptions require sign-off from 3 people besides the developer:
+you can ask for an exception to be made.
 
-1. a Release Manager
-2. an Engineering Lead
-3. an Engineering Director, the VP of Engineering, or the CTO
+Go to [Release tasks issue tracker](https://gitlab.com/gitlab-org/release/tasks/issues/new) and create an issue
+using the `Exception-request` issue template.
 
 You can find who is who on the [team page](https://about.gitlab.com/team/).
 

app/models/ci/build.rb

@@ -44,12 +44,41 @@ module Ci
 
     scope :unstarted, ->() { where(runner_id: nil) }
     scope :ignore_failures, ->() { where(allow_failure: false) }
+
+    # This convoluted mess is because we need to handle two cases of
+    # artifact files during the migration. And a simple OR clause
+    # makes it impossible to optimize.
+
+    # Instead we want to use UNION ALL and do two carefully
+    # constructed disjoint queries. But Rails cannot handle UNION or
+    # UNION ALL queries so we do the query in a subquery and wrap it
+    # in an otherwise redundant WHERE IN query (IN is fine for
+    # non-null columns).
+
+    # This should all be ripped out when the migration is finished and
+    # replaced with just the new storage to avoid the extra work.
+
     scope :with_artifacts, ->() do
-      where('(artifacts_file IS NOT NULL AND artifacts_file <> ?) OR EXISTS (?)',
-        '', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id'))
+      old = Ci::Build.select(:id).where(%q[artifacts_file <> ''])
+      new = Ci::Build.select(:id).where(%q[(artifacts_file IS NULL OR artifacts_file = '') AND EXISTS (?)],
+                                        Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id'))
+      where('ci_builds.id IN (? UNION ALL ?)', old, new)
     end
-    scope :with_artifacts_not_expired, ->() { with_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) }
-    scope :with_expired_artifacts, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) }
+
+    scope :with_artifacts_not_expired, ->() do
+      old = Ci::Build.select(:id).where(%q[artifacts_file <> '' AND (artifacts_expire_at IS NULL OR artifacts_expire_at > ?)], Time.now)
+      new = Ci::Build.select(:id).where(%q[(artifacts_file IS NULL OR artifacts_file = '') AND EXISTS (?)],
+                                        Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id AND (expire_at IS NULL OR expire_at > ?)', Time.now))
+      where('ci_builds.id IN (? UNION ALL ?)', old, new)
+    end
+
+    scope :with_expired_artifacts, ->() do
+      old = Ci::Build.select(:id).where(%q[artifacts_file <> '' AND artifacts_expire_at < ?], Time.now)
+      new = Ci::Build.select(:id).where(%q[(artifacts_file IS NULL OR artifacts_file = '') AND EXISTS (?)],
+                                        Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id AND expire_at < ?', Time.now))
+      where('ci_builds.id IN (? UNION ALL ?)', old, new)
+    end
+
     scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) }
     scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + [:manual]) }
     scope :ref_protected, -> { where(protected: true) }

app/models/event.rb

@@ -102,10 +102,6 @@ class Event < ActiveRecord::Base
 
   self.inheritance_column = 'action'
 
-  # "data" will be removed in 10.0 but it may be possible that JOINs happen that
-  # include this column, hence we're ignoring it as well.
-  ignore_column :data
-
   class << self
     def model_name
       ActiveModel::Name.new(self, nil, 'event')

app/services/projects/create_from_template_service.rb

@@ -5,11 +5,15 @@ module Projects
     end
 
     def execute
-      params[:file] = Gitlab::ProjectTemplate.find(params[:template_name]).file
+      template_name = params.delete(:template_name)
+      file = Gitlab::ProjectTemplate.find(template_name).file
+
+      params[:file] = file
+
+      GitlabProjectsImportService.new(current_user, params).execute
 
-      GitlabProjectsImportService.new(@current_user, @params).execute
     ensure
-      params[:file]&.close
+      file&.close
     end
   end
 end

app/services/projects/gitlab_projects_import_service.rb

@@ -11,12 +11,14 @@ module Projects
 
     def execute
       FileUtils.mkdir_p(File.dirname(import_upload_path))
+
+      file = params.delete(:file)
       FileUtils.copy_entry(file.path, import_upload_path)
 
-      Gitlab::ImportExport::ProjectCreator.new(params[:namespace_id],
-                                               current_user,
-                                               import_upload_path,
-                                               params[:path]).execute
+      params[:import_type] = 'gitlab_project'
+      params[:import_source] = import_upload_path
+
+      ::Projects::CreateService.new(current_user, params).execute
     end
 
     private
@@ -28,9 +30,5 @@ module Projects
     def tmp_filename
       SecureRandom.hex
     end
-
-    def file
-      params[:file]
-    end
   end
 end

changelogs/unreleased/expired-ci-artifacts.yml

+---
+title: Change SQL for expired artifacts to use new ci_job_artifacts.expire_at
+merge_request: 16578
+author:
+type: performance

changelogs/unreleased/fix-template-project-visibility.yml

+---
+title: Respect description and visibility when creating project from template
+merge_request: 16820
+author: George Tsiolis
+type: fixed

db/migrate/20180119160751_optimize_ci_job_artifacts.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class OptimizeCiJobArtifacts < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    # job_id is just here to be a covering index for index only scans
+    # since we'll almost always be joining against ci_builds on job_id
+    add_concurrent_index(:ci_job_artifacts, [:expire_at, :job_id])
+    add_concurrent_index(:ci_builds, [:artifacts_expire_at], where: "artifacts_file <> ''")
+  end
+
+  def down
+    remove_concurrent_index(:ci_job_artifacts, [:expire_at, :job_id])
+    remove_concurrent_index(:ci_builds, [:artifacts_expire_at], where: "artifacts_file <> ''")
+  end
+end

db/schema.rb

@@ -370,6 +370,7 @@ ActiveRecord::Schema.define(version: 20180208183958) do
     t.integer "failure_reason"
   end
 
+  add_index "ci_builds", ["artifacts_expire_at"], name: "index_ci_builds_on_artifacts_expire_at", where: "(artifacts_file <> ''::text)", using: :btree
   add_index "ci_builds", ["auto_canceled_by_id"], name: "index_ci_builds_on_auto_canceled_by_id", using: :btree
   add_index "ci_builds", ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree
   add_index "ci_builds", ["commit_id", "status", "type"], name: "index_ci_builds_on_commit_id_and_status_and_type", using: :btree
@@ -411,6 +412,7 @@ ActiveRecord::Schema.define(version: 20180208183958) do
     t.integer "file_store"
   end
 
+  add_index "ci_job_artifacts", ["expire_at", "job_id"], name: "index_ci_job_artifacts_on_expire_at_and_job_id", using: :btree
   add_index "ci_job_artifacts", ["job_id", "file_type"], name: "index_ci_job_artifacts_on_job_id_and_file_type", unique: true, using: :btree
   add_index "ci_job_artifacts", ["project_id"], name: "index_ci_job_artifacts_on_project_id", using: :btree
 

doc/development/feature_flags.md

 # Manage feature flags
 
-Starting from GitLab 9.3 we support feature flags via
+Starting from GitLab 9.3 we support feature flags for features in GitLab via
 [Flipper](https://github.com/jnunemaker/flipper/). You should use the `Feature`
 class (defined in `lib/feature.rb`) in your code to get, set and list feature
 flags.
@@ -19,3 +19,8 @@ dynamic (querying the DB etc.).
 
 Once defined in `lib/feature.rb`, you will be able to activate a
 feature for a given feature group via the [`feature_group` param of the features API](../api/features.md#set-or-create-a-feature)
+
+## Feature flags for user applications
+
+GitLab does not yet support the use of feature flags in deployed user applications.
+You can follow the progress on that [in the issue on our issue tracker](https://gitlab.com/gitlab-org/gitlab-ee/issues/779).
\ No newline at end of file

doc/development/i18n/index.md

@@ -40,37 +40,12 @@ See [Translation guidelines](translation.md).
 
 ### Proof reading
 
-Proof reading helps ensure the accuracy and consistency of translations.
-All translations are proof read before being accepted.
-If a translations requires changes, you will be notified with a comment explaining why.
-
-Community assistance proof reading translations is encouraged and appreciated.
-Requests to become a proof reader will be considered on the merits of previous translations.
-
-- Bulgarian
-- Chinese Simplified
-  - [Huang Tao](https://crowdin.com/profile/htve)
-- Chinese Traditional
-  - [Huang Tao](https://crowdin.com/profile/htve)
-- Chinese Traditional, Hong Kong
-  - [Huang Tao](https://crowdin.com/profile/htve)
-- Dutch
-- Esperanto
-- French
-- German
-- Italian
-  - [Paolo Falomo](https://crowdin.com/profile/paolo.falomo)
-- Japanese
-- Korean
-  - [Huang Tao](https://crowdin.com/profile/htve)
-- Portuguese, Brazilian
-- Russian
-  - [Alexy Lustin](https://crowdin.com/profile/lustin)
-  - [Nikita Grylov](https://crowdin.com/profile/nixel2007)
-- Spanish
-- Ukrainian
-
-If you would like to be added as a proof reader, please [open an issue](https://gitlab.com/gitlab-org/gitlab-ce/issues).
+Proof reading helps ensure the accuracy and consistency of translations. All
+translations are proof read before being accepted. If a translations requires
+changes, you will be notified with a comment explaining why.
+
+See [Proofreading Translations](proofreader.md) for more information on who's
+able to proofread and instructions on becoming a proofreader yourself.
 
 ## Release
 

doc/development/i18n/proofreader.md

+# Proofread Translations
+
+Most translations are contributed, reviewed, and accepted by the community. We
+are very appreciative of the work done by translators and proofreaders!
+
+## Proofreaders
+
+- Bulgarian
+- Chinese Simplified
+  - Huang Tao - [GitLab](https://gitlab.com/htve), [Crowdin](https://crowdin.com/profile/htve)
+- Chinese Traditional
+  - Huang Tao - [GitLab](https://gitlab.com/htve), [Crowdin](https://crowdin.com/profile/htve)
+- Chinese Traditional, Hong Kong
+  - Huang Tao - [GitLab](https://gitlab.com/htve), [Crowdin](https://crowdin.com/profile/htve)
+- Dutch
+- Esperanto
+- French
+- German
+- Italian
+  - Paolo Falomo - [GitLab](https://gitlab.com/paolofalomo), [Crowdin](https://crowdin.com/profile/paolo.falomo)
+- Japanese
+- Korean
+  - Huang Tao - [GitLab](https://gitlab.com/htve), [Crowdin](https://crowdin.com/profile/htve)
+- Portuguese, Brazilian
+  - Paulo George Gomes Bezerra - [GitLab](https://gitlab.com/paulobezerra), [Crowdin](https://crowdin.com/profile/paulogomes.rep)
+- Russian
+  - Nikita Grylov - [GitLab](https://gitlab.com/nixel2007), [Crowdin](https://crowdin.com/profile/nixel2007)
+  - Alexy Lustin - [GitLab](https://gitlab.com/allustin), [Crowdin](https://crowdin.com/profile/lustin)
+- Spanish
+- Ukrainian
+  - Volodymyr Sobotovych - [GitLab](https://gitlab.com/wheleph), [Crowdin](https://crowdin.com/profile/wheleph)
+  - Andrew Vityuk - [GitLab](https://gitlab.com/3_1_3_u), [Crowdin](https://crowdin.com/profile/andruwa13)
+
+## Become a proofreader
+
+> **Note:** Before requesting Proofreader permissions in Crowdin please make
+> sure that you have a history of contributing translations to the GitLab
+> project.
+
+1. Once your translations have been accepted,
+   [open a merge request](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/new)
+   to request Proofreader permissions and add yourself to the list above.
+
+    In the merge request description, please include links to any projects you
+    have previously translated.
+
+1. Your request to become a proofreader will be considered on the merits of
+   your previous translations.

lib/gitlab/import_export/project_creator.rb

-module Gitlab
-  module ImportExport
-    class ProjectCreator
-      def initialize(namespace_id, current_user, file, project_path)
-        @namespace_id = namespace_id
-        @current_user = current_user
-        @file = file
-        @project_path = project_path
-      end
-
-      def execute
-        ::Projects::CreateService.new(
-          @current_user,
-          name: @project_path,
-          path: @project_path,
-          namespace_id: @namespace_id,
-          import_type: "gitlab_project",
-          import_source: @file
-        ).execute
-      end
-    end
-  end
-end

spec/factories/ci/builds.rb

@@ -180,8 +180,8 @@ FactoryBot.define do
 
     trait :artifacts do
       after(:create) do |build|
-        create(:ci_job_artifact, :archive, job: build)
-        create(:ci_job_artifact, :metadata, job: build)
+        create(:ci_job_artifact, :archive, job: build, expire_at: build.artifacts_expire_at)
+        create(:ci_job_artifact, :metadata, job: build, expire_at: build.artifacts_expire_at)
         build.reload
       end
     end

spec/services/projects/create_from_template_service_spec.rb

@@ -4,8 +4,10 @@ describe Projects::CreateFromTemplateService do
   let(:user) { create(:user) }
   let(:project_params) do
     {
-      path: user.to_param,
-      template_name: 'rails'
+        path: user.to_param,
+        template_name: 'rails',
+        description: 'project description',
+        visibility_level: Gitlab::VisibilityLevel::PRIVATE
     }
   end
 
@@ -22,5 +24,7 @@ describe Projects::CreateFromTemplateService do
 
     expect(project).to be_saved
     expect(project.scheduled?).to be(true)
+    expect(project.description).to match('project description')
+    expect(project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE)
   end
 end