Merge branch 'ce-to-ee-2017-07-19' into 'master'

CE upstream - Wednesday

Closes gitaly#350, gitlab-ce#32561, gitaly#379, #2942, and #2950

See merge request !2475

.gitlab-ci.yml

@@ -166,6 +166,9 @@ build-package:
   when: manual
   script:
     - scripts/trigger-build
+  only:
+    - //@gitlab-org/gitlab-ce
+    - //@gitlab-org/gitlab-ee
 
 # Prepare and merge knapsack tests
 knapsack:
@@ -186,6 +189,7 @@ update-knapsack:
   <<: *only-canonical-masters
   stage: post-test
   script:
+    - retry gem install fog-aws mime-types
     - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec-pg_node_*.json
     - scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach-pg_node_*.json
     - '[[ -z ${KNAPSACK_S3_BUCKET} ]] || scripts/sync-reports put $KNAPSACK_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH'

GITALY_SERVER_VERSION

-0.21.1
+0.21.2

Gemfile

@@ -93,7 +93,7 @@ gem 'carrierwave', '~> 1.1'
 gem 'dropzonejs-rails', '~> 0.7.1'
 
 # for backups
-gem 'fog-aws', '~> 0.9'
+gem 'fog-aws', '~> 1.4'
 gem 'fog-core', '~> 1.44'
 gem 'fog-google', '~> 0.5'
 gem 'fog-local', '~> 0.3'
@@ -249,7 +249,6 @@ gem 'webpack-rails', '~> 0.9.10'
 gem 'rack-proxy', '~> 0.6.0'
 
 gem 'sass-rails', '~> 5.0.6'
-gem 'coffee-rails', '~> 4.1.0'
 gem 'uglifier', '~> 2.7.2'
 
 gem 'addressable', '~> 2.3.8'
@@ -283,7 +282,7 @@ gem 'peek', '~> 1.0.1'
 gem 'peek-gc', '~> 0.0.2'
 gem 'peek-host', '~> 1.0.0'
 gem 'peek-mysql2', '~> 1.1.0', group: :mysql
-gem 'peek-performance_bar', '~> 1.2.1'
+gem 'peek-performance_bar', '~> 1.3.0'
 gem 'peek-pg', '~> 1.3.0', group: :postgres
 gem 'peek-rblineprof', '~> 0.2.0'
 gem 'peek-redis', '~> 1.2.0'
@@ -296,7 +295,7 @@ group :metrics do
   gem 'influxdb', '~> 0.2', require: false
 
   # Prometheus
-  gem 'prometheus-client-mmap', '~>0.7.0.beta5'
+  gem 'prometheus-client-mmap', '~>0.7.0.beta9'
   gem 'raindrops', '~> 0.18'
 end
 
@@ -408,3 +407,6 @@ gem 'toml-rb', '~> 0.3.15', require: false
 # Feature toggles
 gem 'flipper', '~> 0.10.2'
 gem 'flipper-active_record', '~> 0.10.2'
+
+# Structured logging
+gem 'lograge', '~> 0.5'

Gemfile.lock

@@ -130,13 +130,6 @@ GEM
     coderay (1.1.1)
     coercible (1.0.0)
       descendants_tracker (~> 0.0.1)
-    coffee-rails (4.1.1)
-      coffee-script (>= 2.2.0)
-      railties (>= 4.0.0, < 5.1.x)
-    coffee-script (2.4.1)
-      coffee-script-source
-      execjs
-    coffee-script-source (1.10.0)
     colorize (0.7.7)
     concurrent-ruby (1.0.5)
     concurrent-ruby-ext (1.0.5)
@@ -207,7 +200,7 @@ GEM
     et-orbi (1.0.3)
       tzinfo
     eventmachine (1.0.8)
-    excon (0.55.0)
+    excon (0.57.1)
     execjs (2.6.0)
     expression_parser (0.9.0)
     extlib (0.9.16)
@@ -246,26 +239,26 @@ GEM
       fog-json (~> 1.0)
       ipaddress (~> 0.8)
       xml-simple (~> 1.1)
-    fog-aws (0.13.0)
+    fog-aws (1.4.0)
       fog-core (~> 1.38)
       fog-json (~> 1.0)
       fog-xml (~> 0.1)
       ipaddress (~> 0.8)
-    fog-core (1.44.1)
+    fog-core (1.44.3)
       builder
       excon (~> 0.49)
       formatador (~> 0.2)
-    fog-google (0.5.0)
+    fog-google (0.5.3)
       fog-core
       fog-json
       fog-xml
     fog-json (1.0.2)
       fog-core (~> 1.0)
       multi_json (~> 1.10)
-    fog-local (0.3.0)
+    fog-local (0.3.1)
       fog-core (~> 1.27)
-    fog-openstack (0.1.6)
-      fog-core (>= 1.39)
+    fog-openstack (0.1.21)
+      fog-core (>= 1.40)
       fog-json (>= 1.0)
       ipaddress (>= 0.8)
     fog-rackspace (0.1.1)
@@ -478,6 +471,10 @@ GEM
     logging (2.2.2)
       little-plugger (~> 1.1)
       multi_json (~> 1.10)
+    lograge (0.5.1)
+      actionpack (>= 4, < 5.2)
+      activesupport (>= 4, < 5.2)
+      railties (>= 4, < 5.2)
     loofah (2.0.3)
       nokogiri (>= 1.5.9)
     mail (2.6.5)
@@ -589,7 +586,7 @@ GEM
       atomic (>= 1.0.0)
       mysql2
       peek
-    peek-performance_bar (1.2.1)
+    peek-performance_bar (1.3.0)
       peek (>= 0.1.0)
     peek-pg (1.3.0)
       concurrent-ruby
@@ -624,7 +621,7 @@ GEM
     premailer-rails (1.9.7)
       actionmailer (>= 3, < 6)
       premailer (~> 1.7, >= 1.7.9)
-    prometheus-client-mmap (0.7.0.beta8)
+    prometheus-client-mmap (0.7.0.beta9)
       mmap2 (~> 2.2, >= 2.2.7)
     pry (0.10.4)
       coderay (~> 1.1.0)
@@ -969,7 +966,6 @@ DEPENDENCIES
   charlock_holmes (~> 0.7.3)
   chronic (~> 0.10.2)
   chronic_duration (~> 0.10.6)
-  coffee-rails (~> 4.1.0)
   concurrent-ruby (~> 1.0.5)
   connection_pool (~> 2.0)
   creole (~> 0.5.0)
@@ -996,7 +992,7 @@ DEPENDENCIES
   flipper (~> 0.10.2)
   flipper-active_record (~> 0.10.2)
   fog-aliyun (~> 0.1.0)
-  fog-aws (~> 0.9)
+  fog-aws (~> 1.4)
   fog-core (~> 1.44)
   fog-google (~> 0.5)
   fog-local (~> 0.3)
@@ -1043,6 +1039,7 @@ DEPENDENCIES
   letter_opener_web (~> 1.3.0)
   license_finder (~> 2.1.0)
   licensee (~> 8.7.0)
+  lograge (~> 0.5)
   loofah (~> 2.0.3)
   mail_room (~> 0.9.1)
   method_source (~> 0.8)
@@ -1076,7 +1073,7 @@ DEPENDENCIES
   peek-gc (~> 0.0.2)
   peek-host (~> 1.0.0)
   peek-mysql2 (~> 1.1.0)
-  peek-performance_bar (~> 1.2.1)
+  peek-performance_bar (~> 1.3.0)
   peek-pg (~> 1.3.0)
   peek-rblineprof (~> 0.2.0)
   peek-redis (~> 1.2.0)
@@ -1084,7 +1081,7 @@ DEPENDENCIES
   pg (~> 0.18.2)
   poltergeist (~> 1.9.0)
   premailer-rails (~> 1.9.7)
-  prometheus-client-mmap (~> 0.7.0.beta5)
+  prometheus-client-mmap (~> 0.7.0.beta9)
   pry-byebug (~> 3.4.1)
   pry-rails (~> 0.3.4)
   rack-attack (~> 4.4.1)

app/assets/javascripts/dropzone_input.js

@@ -5,21 +5,28 @@ import './preview_markdown';
 
 window.DropzoneInput = (function() {
   function DropzoneInput(form) {
-    var updateAttachingMessage, $attachingFileMessage, $mdArea, $attachButton, $cancelButton, $retryLink, $uploadingErrorContainer, $uploadingErrorMessage, $uploadProgress, $uploadingProgressContainer, appendToTextArea, btnAlert, child, closeAlertMessage, closeSpinner, divHover, divSpinner, dropzone, $formDropzone, formTextarea, getFilename, handlePaste, iconPaperclip, iconSpinner, insertToTextArea, isImage, maxFileSize, pasteText, uploadsPath, showError, showSpinner, uploadFile, addFileToForm;
     Dropzone.autoDiscover = false;
-    divHover = '<div class="div-dropzone-hover"></div>';
-    iconPaperclip = '<i class="fa fa-paperclip div-dropzone-icon"></i>';
-    $attachButton = form.find('.button-attach-file');
-    $attachingFileMessage = form.find('.attaching-file-message');
-    $cancelButton = form.find('.button-cancel-uploading-files');
-    $retryLink = form.find('.retry-uploading-link');
-    $uploadProgress = form.find('.uploading-progress');
-    $uploadingErrorContainer = form.find('.uploading-error-container');
-    $uploadingErrorMessage = form.find('.uploading-error-message');
-    $uploadingProgressContainer = form.find('.uploading-progress-container');
-    uploadsPath = window.uploads_path || null;
-    maxFileSize = gon.max_file_size || 10;
-    formTextarea = form.find('.js-gfm-input');
+    const divHover = '<div class="div-dropzone-hover"></div>';
+    const iconPaperclip = '<i class="fa fa-paperclip div-dropzone-icon"></i>';
+    const $attachButton = form.find('.button-attach-file');
+    const $attachingFileMessage = form.find('.attaching-file-message');
+    const $cancelButton = form.find('.button-cancel-uploading-files');
+    const $retryLink = form.find('.retry-uploading-link');
+    const $uploadProgress = form.find('.uploading-progress');
+    const $uploadingErrorContainer = form.find('.uploading-error-container');
+    const $uploadingErrorMessage = form.find('.uploading-error-message');
+    const $uploadingProgressContainer = form.find('.uploading-progress-container');
+    const uploadsPath = window.uploads_path || null;
+    const maxFileSize = gon.max_file_size || 10;
+    const formTextarea = form.find('.js-gfm-input');
+    let handlePaste;
+    let pasteText;
+    let addFileToForm;
+    let updateAttachingMessage;
+    let isImage;
+    let getFilename;
+    let uploadFile;
+
     formTextarea.wrap('<div class="div-dropzone"></div>');
     formTextarea.on('paste', (function(_this) {
       return function(event) {
@@ -28,16 +35,16 @@ window.DropzoneInput = (function() {
     })(this));
 
     // Add dropzone area to the form.
-    $mdArea = formTextarea.closest('.md-area');
+    const $mdArea = formTextarea.closest('.md-area');
     form.setupMarkdownPreview();
-    $formDropzone = form.find('.div-dropzone');
+    const $formDropzone = form.find('.div-dropzone');
     $formDropzone.parent().addClass('div-dropzone-wrapper');
     $formDropzone.append(divHover);
     $formDropzone.find('.div-dropzone-hover').append(iconPaperclip);
 
     if (!uploadsPath) return;
 
-    dropzone = $formDropzone.dropzone({
+    const dropzone = $formDropzone.dropzone({
       url: uploadsPath,
       dictDefaultMessage: '',
       clickable: true,
@@ -117,7 +124,7 @@ window.DropzoneInput = (function() {
       }
     });
 
-    child = $(dropzone[0]).children('textarea');
+    const child = $(dropzone[0]).children('textarea');
 
     // removeAllFiles(true) stops uploading files (if any)
     // and remove them from dropzone files queue.
@@ -214,6 +221,35 @@ window.DropzoneInput = (function() {
       return value.first();
     };
 
+    const showSpinner = function(e) {
+      return $uploadingProgressContainer.removeClass('hide');
+    };
+
+    const closeSpinner = function() {
+      return $uploadingProgressContainer.addClass('hide');
+    };
+
+    const showError = function(message) {
+      $uploadingErrorContainer.removeClass('hide');
+      $uploadingErrorMessage.html(message);
+    };
+
+    const closeAlertMessage = function() {
+      return form.find('.div-dropzone-alert').alert('close');
+    };
+
+    const insertToTextArea = function(filename, url) {
+      return $(child).val(function(index, val) {
+        return val.replace(`{{${filename}}}`, url);
+      });
+    };
+
+    const appendToTextArea = function(url) {
+      return $(child).val(function(index, val) {
+        return val + url + "\n";
+      });
+    };
+
     uploadFile = function(item, filename) {
       var formData;
       formData = new FormData();
@@ -262,35 +298,6 @@ window.DropzoneInput = (function() {
       messageContainer.text(attachingMessage);
     };
 
-    insertToTextArea = function(filename, url) {
-      return $(child).val(function(index, val) {
-        return val.replace(`{{${filename}}}`, url);
-      });
-    };
-
-    appendToTextArea = function(url) {
-      return $(child).val(function(index, val) {
-        return val + url + "\n";
-      });
-    };
-
-    showSpinner = function(e) {
-      return $uploadingProgressContainer.removeClass('hide');
-    };
-
-    closeSpinner = function() {
-      return $uploadingProgressContainer.addClass('hide');
-    };
-
-    showError = function(message) {
-      $uploadingErrorContainer.removeClass('hide');
-      $uploadingErrorMessage.html(message);
-    };
-
-    closeAlertMessage = function() {
-      return form.find('.div-dropzone-alert').alert('close');
-    };
-
     form.find('.markdown-selector').click(function(e) {
       e.preventDefault();
       $(this).closest('.gfm-form').find('.div-dropzone').click();

app/assets/javascripts/lib/utils/http_status.js

@@ -3,6 +3,7 @@
  */
 
 export default {
+  ABORTED: 0,
   NO_CONTENT: 204,
   OK: 200,
 };

app/assets/javascripts/lib/utils/poll.js

@@ -81,6 +81,9 @@ export default class Poll {
       })
       .catch((error) => {
         notificationCallback(false);
+        if (error.status === httpStatusCodes.ABORTED) {
+          return;
+        }
         errorCallback(error);
       });
   }

app/assets/stylesheets/framework/dropdowns.scss

@@ -205,6 +205,7 @@
 
   @media (max-width: $screen-sm-min) {
     width: 100%;
+    min-width: 180px;
   }
 
   &.dropdown-open-left {
@@ -288,27 +289,15 @@
     padding: 5px 8px;
     color: $gl-text-color-secondary;
   }
-
-  .badge {
-    position: absolute;
-    right: 8px;
-    top: 5px;
-  }
 }
 
 .droplab-dropdown {
-  .description {
-    display: inline-block;
-    white-space: normal;
-    margin-left: 5px;
-  }
-
   .dropdown-toggle > i {
     pointer-events: none;
   }
 
-  li {
-    padding: $gl-btn-padding $gl-btn-padding 2px;
+  .dropdown-menu li {
+    padding: $gl-btn-padding;
     cursor: pointer;
 
     > a,
@@ -344,9 +333,25 @@
       visibility: visible;
     }
 
+    &.divider {
+      margin: 0 8px;
+      padding: 0;
+      border-top: $gray-darkest;
+    }
+
     .icon {
       visibility: hidden;
     }
+
+    .description {
+      display: inline-block;
+      white-space: normal;
+      margin-left: 5px;
+
+      p {
+        margin-bottom: 0;
+      }
+    }
   }
 
   .icon {
@@ -354,12 +359,6 @@
     vertical-align: top;
     padding-top: 2px;
   }
-
-  .divider {
-    margin: 0 8px;
-    padding: 0;
-    border-top: $gray-darkest;
-  }
 }
 
 .droplab-dropdown .dropdown-menu,
@@ -462,10 +461,6 @@
   left: auto;
   right: 0;
   margin-top: -5px;
-
-  @media (max-width: $screen-xs-max) {
-    left: 0;
-  }
 }
 
 .dropdown-menu-selectable {

app/assets/stylesheets/framework/filters.scss

@@ -275,7 +275,7 @@
 }
 
 .filtered-search-input-dropdown-menu {
-  max-height: 215px;
+  max-height: 225px;
   max-width: 280px;
   overflow: auto;
 
@@ -382,10 +382,6 @@
   }
 }
 
-.dropdown-menu .filter-dropdown-item {
-  padding: 0;
-}
-
 @media (max-width: $screen-xs-max) {
   .issues-details-filters {
     padding: 0 0 10px;
@@ -435,6 +431,7 @@
 
     .fa {
       width: 15px;
+      line-height: $line-height-base;
     }
 
     .dropdown-label-box {

app/assets/stylesheets/framework/lists.scss

@@ -242,6 +242,8 @@ ul.content-list {
 ul.controls {
   float: right;
   list-style: none;
+  display: flex;
+  align-items: center;
 
   .btn {
     padding: 10px 14px;
@@ -265,6 +267,12 @@ ul.controls {
       }
     }
   }
+
+  .issuable-pipeline-broken a,
+  .issuable-pipeline-status a,
+  .author_link {
+    display: flex;
+  }
 }
 
 ul.indent-list {

app/assets/stylesheets/framework/nav.scss

@@ -352,7 +352,7 @@
       position: absolute;
       top: 7px;
       right: 15px;
-      z-index: 2;
+      z-index: 300;
 
       li.active {
         font-weight: bold;

app/assets/stylesheets/framework/typography.scss

@@ -2,6 +2,10 @@
   color: $gl-text-color;
   word-wrap: break-word;
 
+  [dir="auto"] {
+    text-align: initial;
+  }
+
   a {
     color: $md-link-color;
   }

app/assets/stylesheets/new_sidebar.scss

@@ -65,7 +65,6 @@ $new-sidebar-width: 220px;
 
 .settings-avatar {
   background-color: $white-light;
-  transition: background-color 100ms linear;
 
   i {
     font-size: 20px;
@@ -73,7 +72,6 @@ $new-sidebar-width: 220px;
     color: $gl-text-color-secondary;
     text-align: center;
     align-self: center;
-    transition: color 100ms linear;
   }
 }
 
@@ -90,6 +88,7 @@ $new-sidebar-width: 220px;
   box-shadow: inset -2px 0 0 $border-color;
 
   a {
+    transition: none;
     text-decoration: none;
   }
 
@@ -177,7 +176,6 @@ $new-sidebar-width: 220px;
       color: $hover-color;
 
       .badge {
-        transition: background-color 100ms linear, color 100ms linear;
         background-color: $indigo-500;
         color: $hover-color;
       }

app/assets/stylesheets/pages/issuable.scss

@@ -809,8 +809,6 @@
   }
 
   .description {
-    margin-bottom: 10px;
-
     .text {
       margin: 0;
     }

app/controllers/projects/issues_controller.rb

@@ -269,7 +269,7 @@ class Projects::IssuesController < Projects::ApplicationController
     if action_name == 'new'
       redirect_to external.new_issue_path
     else
-      redirect_to external.project_path
+      redirect_to external.issue_tracker_path
     end
   end
 

app/helpers/projects_helper.rb

@@ -195,7 +195,7 @@ module ProjectsHelper
       controller.controller_name,
       controller.action_name,
       current_application_settings.cache_key,
-      'v2.4'
+      'v2.5'
     ]
 
     key << pipeline_status_cache_key(project.pipeline_status) if project.pipeline_status.has_status?

app/models/commit.rb

 class Commit
   extend ActiveModel::Naming
+  extend Gitlab::Cache::RequestCache
 
   include ActiveModel::Conversion
   include Noteable
@@ -169,19 +170,9 @@ class Commit
   end
 
   def author
-    if RequestStore.active?
-      key = "commit_author:#{author_email.downcase}"
-      # nil is a valid value since no author may exist in the system
-      if RequestStore.store.key?(key)
-        @author = RequestStore.store[key]
-      else
-        @author = find_author_by_any_email
-        RequestStore.store[key] = @author
-      end
-    else
-      @author ||= find_author_by_any_email
-    end
+    User.find_by_any_email(author_email.downcase)
   end
+  request_cache(:author) { author_email.downcase }
 
   def committer
     @committer ||= User.find_by_any_email(committer_email.downcase)
@@ -322,7 +313,7 @@ class Commit
 
   def raw_diffs(*args)
     if Gitlab::GitalyClient.feature_enabled?(:commit_raw_diffs)
-      Gitlab::GitalyClient::Commit.new(project.repository).diff_from_parent(self, *args)
+      Gitlab::GitalyClient::CommitService.new(project.repository).diff_from_parent(self, *args)
     else
       raw.diffs(*args)
     end
@@ -331,7 +322,7 @@ class Commit
   def raw_deltas
     @deltas ||= Gitlab::GitalyClient.migrate(:commit_deltas) do |is_enabled|
       if is_enabled
-        Gitlab::GitalyClient::Commit.new(project.repository).commit_deltas(self)
+        Gitlab::GitalyClient::CommitService.new(project.repository).commit_deltas(self)
       else
         raw.deltas
       end
@@ -368,10 +359,6 @@ class Commit
     end
   end
 
-  def find_author_by_any_email
-    User.find_by_any_email(author_email.downcase)
-  end
-
   def repo_changes
     changes = { added: [], modified: [], removed: [] }
 

app/models/concerns/repository_mirroring.rb

@@ -29,7 +29,8 @@ module RepositoryMirroring
 
   def remote_tags(remote)
     gitlab_shell.list_remote_tags(storage_path, path_with_namespace, remote).map do |name, target|
-      Gitlab::Git::Tag.new(raw_repository, name, target)
+      target_commit = Gitlab::Git::Commit.find(raw_repository, target)
+      Gitlab::Git::Tag.new(raw_repository, name, target, target_commit)
     end
   end
 
@@ -40,7 +41,8 @@ module RepositoryMirroring
       name = ref.name.sub(/\Arefs\/remotes\/#{remote_name}\//, '')
 
       begin
-        branches << Gitlab::Git::Branch.new(raw_repository, name, ref.target)
+        target_commit = Gitlab::Git::Commit.find(raw_repository, ref.target)
+        branches << Gitlab::Git::Branch.new(raw_repository, name, ref.target, target_commit)
       rescue Rugged::ReferenceError
         # Omit invalid branch
       end

app/models/group.rb

@@ -5,7 +5,6 @@ require 'carrierwave/orm/activerecord'
 class Group < Namespace
   include EE::Group
   include Gitlab::ConfigHelper
-  include Gitlab::VisibilityLevel
   include AccessRequestable
   include Avatarable
   include Referable
@@ -121,10 +120,6 @@ class Group < Namespace
     full_name
   end
 
-  def visibility_level_field
-    :visibility_level
-  end
-
   def visibility_level_allowed_by_projects
     allowed_by_projects = self.projects.where('visibility_level > ?', self.visibility_level).none?
 

app/models/namespace.rb

@@ -6,6 +6,7 @@ class Namespace < ActiveRecord::Base
   include Sortable
   include Gitlab::ShellAdapter
   include Gitlab::CurrentSettings
+  include Gitlab::VisibilityLevel
   include Routable
   include AfterCommitQueue
 
@@ -106,6 +107,10 @@ class Namespace < ActiveRecord::Base
     end
   end
 
+  def visibility_level_field
+    :visibility_level
+  end
+
   def to_param
     full_path
   end

app/models/note.rb

@@ -196,7 +196,7 @@ class Note < ActiveRecord::Base
   # override to return commits, which are not active record
   def noteable
     if for_commit?
-      project.commit(commit_id)
+      @commit ||= project.commit(commit_id)
     else
       super
     end

app/models/project_services/gitlab_issue_tracker_service.rb

@@ -23,7 +23,7 @@ class GitlabIssueTrackerService < IssueTrackerService
     project_issue_url(project, id: iid)
   end
 
-  def project_path
+  def issue_tracker_path
     project_issues_path(project)
   end
 

app/models/project_services/issue_tracker_service.rb

@@ -24,8 +24,8 @@ class IssueTrackerService < Service
     self.issues_url.gsub(':id', iid.to_s)
   end
 
-  def project_path
-    read_attribute(:project_url)
+  def issue_tracker_path
+    project_url
   end
 
   def new_issue_path

app/models/project_services/kubernetes_service.rb

@@ -60,22 +60,22 @@ class KubernetesService < DeploymentService
 
   def fields
     [
-        { type: 'text',
-          name: 'namespace',
-          title: 'Kubernetes namespace',
-          placeholder: namespace_placeholder },
         { type: 'text',
           name: 'api_url',
           title: 'API URL',
           placeholder: 'Kubernetes API URL, like https://kube.example.com/' },
-        { type: 'text',
-          name: 'token',
-          title: 'Service token',
-          placeholder: 'Service token' },
         { type: 'textarea',
           name: 'ca_pem',
-          title: 'Custom CA bundle',
-          placeholder: 'Certificate Authority bundle (PEM format)' }
+          title: 'CA Certificate',
+          placeholder: 'Certificate Authority bundle (PEM format)' },
+        { type: 'text',
+          name: 'namespace',
+          title: 'Project namespace (optional/unique)',
+          placeholder: namespace_placeholder },
+        { type: 'text',
+          name: 'token',
+          title: 'Token',
+          placeholder: 'Service token' }
     ]
   end
 

app/policies/ci/build_policy.rb

@@ -3,9 +3,13 @@ module Ci
     condition(:protected_action) do
       next false unless @subject.action?
 
-      !::Gitlab::UserAccess
-        .new(@user, project: @subject.project)
-        .can_merge_to_branch?(@subject.ref)
+      access = ::Gitlab::UserAccess.new(@user, project: @subject.project)
+
+      if @subject.tag?
+        !access.can_create_tag?(@subject.ref)
+      else
+        !access.can_merge_to_branch?(@subject.ref)
+      end
     end
 
     rule { protected_action }.prevent :update_build

app/services/metrics_service.rb

@@ -31,6 +31,6 @@ class MetricsService
   end
 
   def multiprocess_metrics_path
-    @multiprocess_metrics_path ||= Rails.root.join(ENV['prometheus_multiproc_dir']).freeze
+    ::Prometheus::Client.configuration.multiprocess_files_dir
   end
 end

app/uploaders/gitlab_uploader.rb

@@ -16,7 +16,7 @@ class GitlabUploader < CarrierWave::Uploader::Base
   def self.base_dir
     return root_dir unless file_storage?
 
-    File.join(root_dir, 'system')
+    File.join(root_dir, '-', 'system')
   end
 
   def self.file_storage?

app/views/layouts/nav/_dashboard.html.haml

@@ -42,18 +42,18 @@
         .key
           = icon('arrow-up', 'aria-label' => 'hidden')
           I
+      %span.badge.pull-right= number_with_delimiter(assigned_issuables_count(:issues))
       %span
         Issues
-      .badge= number_with_delimiter(assigned_issuables_count(:issues))
   = nav_link(path: 'dashboard#merge_requests') do
     = link_to assigned_mrs_dashboard_path, title: 'Merge Requests', class: 'dashboard-shortcuts-merge_requests' do
       .shortcut-mappings
         .key
           = icon('arrow-up', 'aria-label' => 'hidden')
           M
+      %span.badge.pull-right= number_with_delimiter(assigned_issuables_count(:merge_requests))
       %span
         Merge Requests
-      .badge= number_with_delimiter(assigned_issuables_count(:merge_requests))
   = nav_link(controller: 'dashboard/snippets') do
     = link_to dashboard_snippets_path, class: 'dashboard-shortcuts-snippets', title: 'Snippets' do
       .shortcut-mappings

app/views/layouts/nav/_new_group_sidebar.html.haml

@@ -6,15 +6,15 @@
       = @group.name
   %ul.sidebar-top-level-items
     = nav_link(path: ['groups#show', 'groups#activity', 'groups#subgroups'], html_options: { class: 'home' }) do
-      = link_to group_path(@group), title: 'Home' do
+      = link_to group_path(@group), title: 'About group' do
         %span
-          Group
+          About
 
       %ul.sidebar-sub-level-items
         = nav_link(path: ['groups#show', 'groups#subgroups'], html_options: { class: 'home' }) do
-          = link_to group_path(@group), title: 'Group Home' do
+          = link_to group_path(@group), title: 'Group details' do
             %span
-              Home
+              Details
 
         = nav_link(path: 'groups#activity') do
           = link_to activity_group_path(@group), title: 'Activity' do

app/views/layouts/nav/_new_project_sidebar.html.haml

@@ -7,14 +7,14 @@
       = @project.name
   %ul.sidebar-top-level-items
     = nav_link(path: ['projects#show', 'projects#activity', 'cycle_analytics#show'], html_options: { class: 'home' }) do
-      = link_to project_path(@project), title: 'Project', class: 'shortcuts-project' do
+      = link_to project_path(@project), title: 'About project', class: 'shortcuts-project' do
         %span
-          Project
+          About
 
       %ul.sidebar-sub-level-items
         = nav_link(path: 'projects#show') do
-          = link_to project_path(@project), title: _('Project home'), class: 'shortcuts-project' do
-            %span= _('Home')
+          = link_to project_path(@project), title: _('Project details'), class: 'shortcuts-project' do
+            %span= _('Details')
 
         = nav_link(path: 'projects#activity') do
           = link_to activity_project_path(@project), title: _('Activity'), class: 'shortcuts-project-activity' do
@@ -165,7 +165,7 @@
             Snippets
 
     - if project_nav_tab? :settings
-      = nav_link(path: %w[projects#edit members#show integrations#show services#edit repository#show ci_cd#show pages#show]) do
+      = nav_link(path: %w[projects#edit project_members#index integrations#show services#edit repository#show ci_cd#show pages#show]) do
         = link_to edit_project_path(@project), title: 'Settings', class: 'shortcuts-tree' do
           %span
             Settings
@@ -177,8 +177,8 @@
               = link_to edit_project_path(@project), title: 'General' do
                 %span
                   General
-          = nav_link(controller: :members) do
-            = link_to project_settings_members_path(@project), title: 'Members' do
+          = nav_link(controller: :project_members) do
+            = link_to project_project_members_path(@project), title: 'Members' do
               %span
                 Members
           - if can_edit

app/views/projects/artifacts/browse.html.haml

@@ -5,12 +5,6 @@
 
 .tree-holder
   .nav-block
-    .tree-controls
-      = link_to download_project_job_artifacts_path(@project, @build),
-        rel: 'nofollow', download: '', class: 'btn btn-default download' do
-        = icon('download')
-        Download artifacts archive
-
     %ul.breadcrumb.repo-breadcrumb
       %li
         = link_to 'Artifacts', browse_project_job_artifacts_path(@project, @build)
@@ -18,6 +12,12 @@
         %li
           = link_to truncate(title, length: 40), browse_project_job_artifacts_path(@project, @build, path)
 
+    .tree-controls
+      = link_to download_project_job_artifacts_path(@project, @build),
+        rel: 'nofollow', download: '', class: 'btn btn-default download' do
+        = icon('download')
+        Download artifacts archive
+
   .tree-content-holder
     %table.table.tree-table
       %thead

app/views/projects/cycle_analytics/show.html.haml

@@ -11,8 +11,8 @@
 #cycle-analytics{ class: container_class, "v-cloak" => "true", data: { request_path: project_cycle_analytics_path(@project) } }
   - if @cycle_analytics_no_data
     .landing.content-block{ "v-if" => "!isOverviewDialogDismissed" }
-      %button.dismiss-button{ type: 'button', 'aria-label': 'Dismiss Cycle Analytics introduction box' }
-        = icon("times", "@click" => "dismissOverviewDialog()")
+      %button.dismiss-button{ type: 'button', 'aria-label': 'Dismiss Cycle Analytics introduction box', "@click" => "dismissOverviewDialog()" }
+        = icon("times")
       .svg-container
         = custom_icon('icon_cycle_analytics_splash')
       .inner-content

app/views/shared/issuable/_search_bar.html.haml

@@ -118,7 +118,7 @@
           - if type == :boards
             - if can?(current_user, :admin_list, @project)
               .dropdown.prepend-left-10#js-add-list
-                %button.btn.btn-create.btn-inverted.js-new-board-list{ type: "button", data: { toggle: "dropdown", labels: labels_filter_path, namespace_path: @project.try(:namespace).try(:path), project_path: @project.try(:path) } }
+                %button.btn.btn-create.btn-inverted.js-new-board-list{ type: "button", data: { toggle: "dropdown", labels: labels_filter_path, namespace_path: @project.try(:namespace).try(:full_path), project_path: @project.try(:path) } }
                   Add list
                 .dropdown-menu.dropdown-menu-paging.dropdown-menu-align-right.dropdown-menu-issues-board-new.dropdown-menu-selectable
                   = render partial: "shared/issuable/label_page_default", locals: { show_footer: true, show_create: true, show_boards_content: true, title: "Add list" }

app/workers/project_service_worker.rb

@@ -2,6 +2,8 @@ class ProjectServiceWorker
   include Sidekiq::Worker
   include DedicatedSidekiqQueue
 
+  sidekiq_options dead: false
+
   def perform(hook_id, data)
     data = data.with_indifferent_access
     Service.find(hook_id).execute(data)

app/workers/web_hook_worker.rb

@@ -2,7 +2,7 @@ class WebHookWorker
   include Sidekiq::Worker
   include DedicatedSidekiqQueue
 
-  sidekiq_options retry: 4
+  sidekiq_options retry: 4, dead: false
 
   def perform(hook_id, data, hook_name)
     hook = WebHook.find(hook_id)

changelogs/unreleased/12892-reset-css-text-align-to-initial-for-rtl.md

+---
+title: "reset text-align to initial to let elements with dir="auto" align texts to right in RTL languages ( default is left )"
+merge_request: 12892
+author: goshhob

changelogs/unreleased/29901-refactor-initialization-dropzone_input-js.yml

+---
+title: refactor initializations in dropzone_input.js
+merge_request: 12768
+author: Brandon Everett

changelogs/unreleased/31571-don-t-let-webhooks-jobs-go-to-the-dead-jobs-queue.yml

+---
+title: Prevent web hook and project service background jobs from going to the dead
+  jobs queue
+merge_request:
+author:

changelogs/unreleased/33741-clarify-k8s-service-keys.yml

+---
+title: Clarifies and rearranges the input variables on the kubernetes integration
+  page and adjusts the docs slightly to meet the same order
+merge_request: !12188
+author:

changelogs/unreleased/34173-add-portuguese-brazil-translations-of-commits-page.yml

+---
+title: Add Portuguese Brazil translations of Commits Page
+merge_request: 12408
+author: Huang Tao

changelogs/unreleased/34468-remove-extra-blank-on-admin-on-mobile.yml

+---
+title: Use smaller min-width for dropdown-menu-nav only on mobile
+merge_request: 12528
+author: Takuya Noguchi

changelogs/unreleased/34789-add-japanese-translations-of-commits-page.yml

+---
+title: Add Japanese translations for Cycle Analytics & Project pages & Repository pages & Commits pages & Pipeline Charts.
+merge_request: 12693
+author: Huang Tao

changelogs/unreleased/34831-remove-coffee-rails-gem.yml

+---
+title: Remove coffee-rails gem
+merge_request:
+author: Takuya Noguchi

changelogs/unreleased/34880-add-ukrainian-translations-to-i18n.yml

+---
+title: Add Ukrainian translations for Cycle Analytics & Project pages & Repository pages & Commits pages & Pipeline Charts.
+merge_request: 12744
+author: Huang Tao

changelogs/unreleased/34881-add-russian-translations-to-i18n.yml

+---
+title: Add Russian translations for Cycle Analytics & Project pages & Repository pages & Commits pages & Pipeline Charts.
+merge_request: 12743
+author: Huang Tao

changelogs/unreleased/34927-protect-manual-actions-on-tags.yml

+---
+title: Protect manual actions against protected tag too
+merge_request: 12908
+author:

changelogs/unreleased/35087-mr-status-misaligned.yml

+---
+title: Fix alignment of controls in mr issuable list
+merge_request:
+author:

changelogs/unreleased/35155-upgrade-fog-core-to-1-44-3-and-its-providers-to-the-latest.yml

+---
+title: Bump fog-core to 1.44.3 and fog providers' plugins to latest
+merge_request: 12897
+author: Takuya Noguchi

changelogs/unreleased/35164-cycle-analytics-firefox.yml

+---
+title: allow closing Cycle Analytics intro box in firefox
+merge_request:
+author:

changelogs/unreleased/35181-cannot-create-label-from-board-page.yml

+---
+title: Fix label creation from new list for subgroup projects
+merge_request:
+author:

changelogs/unreleased/35225-transient-poll.yml

+---
+title: fix transient js error in rspec tests
+merge_request:
+author:

changelogs/unreleased/bvl-free-system-namespace.yml

+---
+title: "Move uploads from `uploads/system` to `uploads/-/system` to free up `system` as a group name"
+merge_request: 11713
+author:

changelogs/unreleased/request-store-wrap.yml

+---
+title: Add RequestCache which makes caching with RequestStore easier
+merge_request: 12920
+author:

changelogs/unreleased/sh-structured-logging.yml

+---
+title: Add structured logging for Rails processes
+merge_request:
+author:

config/boot.rb

@@ -4,8 +4,3 @@ require 'rubygems'
 ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
 
 require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
-
-# set default directory for multiproces metrics gathering
-if ENV['RAILS_ENV'] == 'development' || ENV['RAILS_ENV'] == 'test'
-  ENV['prometheus_multiproc_dir'] ||= 'tmp/prometheus_multiproc_dir'
-end

config/environments/test.rb

@@ -43,4 +43,9 @@ Rails.application.configure do
   config.cache_store = :null_store
 
   config.active_job.queue_adapter = :test
+
+  if ENV['CI'] && !ENV['RAILS_ENABLE_TEST_LOG']
+    config.logger = ActiveSupport::TaggedLogging.new(Logger.new(nil))
+    config.log_level = :fatal
+  end
 end

config/initializers/7_prometheus_metrics.rb

+require 'prometheus/client'
+
+Prometheus::Client.configure do |config|
+  config.logger = Rails.logger
+
+  config.initial_mmap_file_size = 4 * 1024
+  config.multiprocess_files_dir = ENV['prometheus_multiproc_dir']
+
+  if Rails.env.development? && Rails.env.test?
+    config.multiprocess_files_dir ||= Rails.root.join('tmp/prometheus_multiproc_dir')
+  end
+end

config/initializers/gettext_rails_i18n_patch.rb

@@ -33,7 +33,6 @@ module GettextI18nRailsJs
         [
           ".js",
           ".jsx",
-          ".coffee",
           ".vue"
         ].include? ::File.extname(file)
       end

config/initializers/lograge.rb

+# Only use Lograge for Rails
+unless Sidekiq.server?
+  filename = File.join(Rails.root, 'log', "#{Rails.env}_json.log")
+
+  Rails.application.configure do
+    config.lograge.enabled = true
+    # Store the lograge JSON files in a separate file
+    config.lograge.keep_original_rails_log = true
+    # Don't use the Logstash formatter since this requires logstash-event, an
+    # unmaintained gem that monkey patches `Time`
+    config.lograge.formatter = Lograge::Formatters::Json.new
+    config.lograge.logger = ActiveSupport::Logger.new(filename)
+    # Add request parameters to log output
+    config.lograge.custom_options = lambda do |event|
+      {
+        time: event.time,
+        params: event.payload[:params].except(%w(controller action format))
+      }
+    end
+  end
+end

config/initializers/peek.rb

@@ -26,7 +26,3 @@ class PEEK_DB_CLIENT
 end
 
 PEEK_DB_VIEW.prepend ::Gitlab::PerformanceBar::PeekQueryTracker
-
-class Peek::Views::PerformanceBar::ProcessUtilization
-  prepend ::Gitlab::PerformanceBar::PeekPerformanceBarWithRackBody
-end

config/routes/uploads.rb

 scope path: :uploads do
   # Note attachments and User/Group/Project avatars
-  get "system/:model/:mounted_as/:id/:filename",
+  get "-/system/:model/:mounted_as/:id/:filename",
       to:           "uploads#show",
       constraints:  { model: /note|user|group|project/, mounted_as: /avatar|attachment/, filename: /[^\/]+/ }
 
@@ -15,7 +15,7 @@ scope path: :uploads do
     constraints: { filename: /[^\/]+/ }
 
   # Appearance
-  get "system/:model/:mounted_as/:id/:filename",
+  get "-/system/:model/:mounted_as/:id/:filename",
       to:           "uploads#show",
       constraints:  { model: /appearance/, mounted_as: /logo|header_logo/, filename: /.+/ }
 

db/migrate/20170713104829_add_foreign_key_to_merge_requests.rb

+class AddForeignKeyToMergeRequests < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class MergeRequest < ActiveRecord::Base
+    self.table_name = 'merge_requests'
+    include ::EachBatch
+  end
+
+  def up
+    scope = <<-SQL.strip_heredoc
+      head_pipeline_id IS NOT NULL
+        AND NOT EXISTS (
+          SELECT 1 FROM ci_pipelines
+            WHERE ci_pipelines.id = merge_requests.head_pipeline_id
+        )
+    SQL
+
+    MergeRequest.where(scope).each_batch(of: 1000) do |merge_requests|
+      merge_requests.update_all(head_pipeline_id: nil)
+    end
+
+    unless foreign_key_exists?(:merge_requests, :head_pipeline_id)
+      add_concurrent_foreign_key(:merge_requests, :ci_pipelines,
+                                 column: :head_pipeline_id, on_delete: :nullify)
+    end
+  end
+
+  def down
+    if foreign_key_exists?(:merge_requests, :head_pipeline_id)
+      remove_foreign_key(:merge_requests, column: :head_pipeline_id)
+    end
+  end
+
+  private
+
+  def foreign_key_exists?(table, column)
+    foreign_keys(table).any? do |key|
+      key.options[:column] == column.to_s
+    end
+  end
+end

db/migrate/20170717074009_move_system_upload_folder.rb

+class MoveSystemUploadFolder < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+
+  def up
+    unless file_storage?
+      say 'Using object storage, no need to move.'
+      return
+    end
+
+    unless File.directory?(old_directory)
+      say "#{old_directory} doesn't exist, no need to move it."
+      return
+    end
+
+    FileUtils.mkdir_p(File.join(base_directory, '-'))
+
+    say "Moving #{old_directory} -> #{new_directory}"
+    FileUtils.mv(old_directory, new_directory)
+    FileUtils.ln_s(new_directory, old_directory)
+  end
+
+  def down
+    unless file_storage?
+      say 'Using object storage, no need to move.'
+      return
+    end
+
+    unless File.directory?(new_directory)
+      say "#{new_directory} doesn't exist, no need to move it."
+      return
+    end
+
+    if File.symlink?(old_directory)
+      say "Removing #{old_directory} -> #{new_directory} symlink"
+      FileUtils.rm(old_directory)
+    end
+
+    say "Moving #{new_directory} -> #{old_directory}"
+    FileUtils.mv(new_directory, old_directory)
+  end
+
+  def new_directory
+    File.join(base_directory, '-', 'system')
+  end
+
+  def old_directory
+    File.join(base_directory, 'system')
+  end
+
+  def base_directory
+    File.join(Rails.root, 'public', 'uploads')
+  end
+
+  def file_storage?
+    CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File
+  end
+end

db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CleanupMoveSystemUploadFolderSymlink < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    if File.symlink?(old_directory)
+      say "Removing #{old_directory} -> #{new_directory} symlink"
+      FileUtils.rm(old_directory)
+    else
+      say "Symlink #{old_directory} non existant, nothing to do."
+    end
+  end
+
+  def down
+    if File.directory?(new_directory)
+      say "Symlinking #{old_directory} -> #{new_directory}"
+      FileUtils.ln_s(new_directory, old_directory)
+    else
+      say "#{new_directory} doesn't exist, skipping."
+    end
+  end
+
+  def new_directory
+    File.join(base_directory, '-', 'system')
+  end
+
+  def old_directory
+    File.join(base_directory, 'system')
+  end
+
+  def base_directory
+    File.join(Rails.root, 'public', 'uploads')
+  end
+end

db/post_migrate/20170717150329_enqueue_migrate_system_uploads_to_new_folder.rb

+class EnqueueMigrateSystemUploadsToNewFolder < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  OLD_FOLDER = 'uploads/system/'
+  NEW_FOLDER = 'uploads/-/system/'
+
+  disable_ddl_transaction!
+
+  def up
+    BackgroundMigrationWorker.perform_async('MigrateSystemUploadsToNewFolder',
+                                            [OLD_FOLDER, NEW_FOLDER])
+  end
+
+  def down
+    BackgroundMigrationWorker.perform_async('MigrateSystemUploadsToNewFolder',
+                                            [NEW_FOLDER, OLD_FOLDER])
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20170707184244) do
+ActiveRecord::Schema.define(version: 20170717150329) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -1948,6 +1948,7 @@ ActiveRecord::Schema.define(version: 20170707184244) do
   add_foreign_key "merge_request_diffs", "merge_requests", name: "fk_8483f3258f", on_delete: :cascade
   add_foreign_key "merge_request_metrics", "ci_pipelines", column: "pipeline_id", on_delete: :cascade
   add_foreign_key "merge_request_metrics", "merge_requests", on_delete: :cascade
+  add_foreign_key "merge_requests", "ci_pipelines", column: "head_pipeline_id", name: "fk_fd82eae0b9", on_delete: :nullify
   add_foreign_key "merge_requests", "projects", column: "target_project_id", name: "fk_a6963e8447", on_delete: :cascade
   add_foreign_key "merge_requests_closing_issues", "issues", on_delete: :cascade
   add_foreign_key "merge_requests_closing_issues", "merge_requests", on_delete: :cascade

doc/administration/monitoring/prometheus/index.md

@@ -95,8 +95,9 @@ Sample Prometheus queries:
 ## Configuring Prometheus to monitor Kubernetes
 
 > Introduced in GitLab 9.0.
+> Pod monitoring introduced in GitLab 9.4.
 
-If your GitLab server is running within Kubernetes, Prometheus will collect metrics from the Nodes in the cluster including performance data on each container. This is particularly helpful if your CI/CD environments run in the same cluster, as you can use the [Prometheus project integration][] to monitor them.
+If your GitLab server is running within Kubernetes, Prometheus will collect metrics from the Nodes and [annotated Pods](https://prometheus.io/docs/operating/configuration/#<kubernetes_sd_config>) in the cluster, including performance data on each container. This is particularly helpful if your CI/CD environments run in the same cluster, as you can use the [Prometheus project integration][] to monitor them.
 
 To disable the monitoring of Kubernetes:
 

doc/ci/environments.md

@@ -602,9 +602,8 @@ exist, you should see something like:
 >**Notes:**
 >
 - For the monitor dashboard to appear, you need to:
-  - Have enabled the [Kubernetes integration][kube]
-  - Have your app deployed on Kubernetes
   - Have enabled the [Prometheus integration][prom]
+  - Configured Prometheus to collect at least one [supported metric](../user/project/integrations/prometheus_library/metrics.md)
 - With GitLab 9.2, all deployments to an environment are shown directly on the
   monitoring dashboard
 

doc/development/gotchas.md

@@ -3,35 +3,6 @@
 The purpose of this guide is to document potential "gotchas" that contributors
 might encounter or should avoid during development of GitLab CE and EE.
 
-## Do not `describe` symbols
-
-Consider the following model spec:
-
-```ruby
-require 'rails_helper'
-
-describe User do
-  describe :to_param do
-    it 'converts the username to a param' do
-      user = described_class.new(username: 'John Smith')
-
-      expect(user.to_param).to eq 'john-smith'
-    end
-  end
-end
-```
-
-When run, this spec doesn't do what we might expect:
-
-```sh
-spec/models/user_spec.rb|6 error|  Failure/Error: u = described_class.new NoMethodError: undefined method `new' for :to_param:Symbol
-```
-
-### Solution
-
-Except for the top-level `describe` block, always provide a String argument to
-`describe`.
-
 ## Do not assert against the absolute value of a sequence-generated attribute
 
 Consider the following factory:

doc/development/testing.md

@@ -195,7 +195,6 @@ Please consult the [dedicated "Frontend testing" guide](./fe_guide/testing.md).
 - Use `context` to test branching logic.
 - Use multi-line `do...end` blocks for `before` and `after`, even when it would
   fit on a single line.
-- Don't `describe` symbols (see [Gotchas](gotchas.md#dont-describe-symbols)).
 - Don't assert against the absolute value of a sequence-generated attribute (see [Gotchas](gotchas.md#dont-assert-against-the-absolute-value-of-a-sequence-generated-attribute)).
 - Don't supply the `:each` argument to hooks since it's the default.
 - Prefer `not_to` to `to_not` (_this is enforced by RuboCop_).
@@ -479,6 +478,11 @@ slowest test files and try to improve them.
   run the suite against MySQL for tags, `master`, and any branch that includes
   `mysql` in the name.
 - On EE, the test suite always runs both PostgreSQL and MySQL.
+- Rails logging to `log/test.log` is disabled by default in CI [for
+  performance reasons][logging]. To override this setting, provide the
+  `RAILS_ENABLE_TEST_LOG` environment variable.
+
+[logging]: https://jtway.co/speed-up-your-rails-test-suite-by-6-in-1-line-13fedb869ec4
 
 ## Spinach (feature) tests
 

doc/update/9.3-to-9.4.md

@@ -157,8 +157,7 @@ configuration file may contain syntax errors. The block name
 file, should be `[[storage]]` instead.
 
 ```shell
-cd /home/git/gitaly
-sudo -u git -H editor config.toml
+sudo -u git -H sed -i.pre-9.4 's/\[\[storages\]\]/[[storage]]/' /home/git/gitaly/config.toml
 ```
 
 #### Compile Gitaly

doc/user/project/integrations/kubernetes.md

@@ -19,10 +19,10 @@ of your project and select the **Kubernetes** service to configure it.
 
 The Kubernetes service takes the following arguments:
 
-1. Kubernetes namespace
 1. API URL
-1. Service token
 1. Custom CA bundle
+1. Kubernetes namespace
+1. Service token
 
 The API URL is the URL that GitLab uses to access the Kubernetes API. Kubernetes
 exposes several APIs - we want the "base" URL that is common to all of them,

doc/user/project/integrations/prometheus.md

@@ -17,35 +17,30 @@ the settings page with a default template. To configure the template, see the
 Integration with Prometheus requires the following:
 
 1. GitLab 9.0 or higher
-1. The [Kubernetes integration must be enabled][kube] on your project
-1. Your app must be deployed on [Kubernetes][]
-1. Prometheus must be configured to collect Kubernetes metrics
+1. Prometheus must be configured to collect one of the [supported metrics](prometheus_library/metrics.md)
 1. Each metric must be have a label to indicate the environment
-1. GitLab must have network connectivity to the Prometheus sever
+1. GitLab must have network connectivity to the Prometheus server
 
-There are a few steps necessary to set up integration between Prometheus and
-GitLab.
+## Getting started with Prometheus monitoring
 
-## Configuring Prometheus to collect Kubernetes metrics
+Depending on your deployment and where you have located your GitLab server, there are a few options to get started with Prometheus monitoring.
 
-In order for Prometheus to collect Kubernetes metrics, you first must have a
-Prometheus server up and running. You have two options here:
+* If both GitLab and your applications are installed in the same Kubernetes cluster, you can leverage the [bundled Prometheus server within GitLab](#configuring-omnibus-gitlab-prometheus-to-monitor-kubernetes).
+* If your applications are deployed on Kubernetes, but GitLab is not in the same cluster, then you can [configure a Prometheus server in your Kubernetes cluster](#configuring-your-own-prometheus-server-within-kubernetes).
+* If your applications are not running in Kubernetes, [get started with Prometheus](#getting-started-with-prometheus-outside-of-kubernetes).
 
-- If you installed Omnibus GitLab inside of Kubernetes, you can simply use the
-  [bundled version of Prometheus][promgldocs]. In that case, follow the info in the
-  [Omnibus GitLab section](#configuring-omnibus-gitlab-prometheus-to-monitor-kubernetes)
-  below.
-- If you are using GitLab.com or installed GitLab outside of Kubernetes, you
-  will likely need to run a Prometheus server within the Kubernetes cluster.
-  Once installed, the easiest way to monitor Kubernetes is to simply use
-  Prometheus' support for [Kubernetes Service Discovery][prometheus-k8s-sd].
-  In that case, follow the instructions on
-  [configuring your own Prometheus server within Kubernetes](#configuring-your-own-prometheus-server-within-kubernetes).
+### Getting started with Prometheus outside of Kubernetes
 
-### Configuring Omnibus GitLab Prometheus to monitor Kubernetes
+Installing and configuring Prometheus to monitor applications is fairly straight forward.
+
+1. [Install Prometheus](https://prometheus.io/docs/introduction/install/)
+1. Set up one of the [supported monitoring targets](prometheus_library/metrics.md)
+1. Configure the Prometheus server to [collect their metrics](https://prometheus.io/docs/operating/configuration/#scrape_config)
+
+### Configuring Omnibus GitLab Prometheus to monitor Kubernetes deployments
 
 With Omnibus GitLab running inside of Kubernetes, you can leverage the bundled
-version of Prometheus to collect the required metrics.
+version of Prometheus to collect the supported metrics. Once enabled, Prometheus will automatically begin monitoring Kubernetes Nodes and any [annotated Pods](https://prometheus.io/docs/operating/configuration/#<kubernetes_sd_config>). 
 
 1. Read how to configure the bundled Prometheus server in the
    [Administration guide][gitlab-prometheus-k8s-monitor].
@@ -74,7 +69,7 @@ kubectl apply -f path/to/prometheus.yml
 Once deployed, you should see the Prometheus service, deployment, and
 pod start within the `prometheus` namespace. The server will begin to collect
 metrics from each Kubernetes Node in the cluster, based on the configuration
-provided in the template.
+provided in the template. It will also attempt to collect metrics from any Kubernetes Pods that have been [annotated for Prometheus](https://prometheus.io/docs/operating/configuration/#pod).
 
 Since GitLab is not running within Kubernetes, the template provides external
 network access via a `NodePort` running on `30090`. This method allows access
@@ -133,30 +128,6 @@ to integrate with.
 
 ![Configure Prometheus Service](img/prometheus_service_configuration.png)
 
-## Metrics and Labels
-
-GitLab retrieves performance data from two metrics, `container_cpu_usage_seconds_total`
-and `container_memory_usage_bytes`. These metrics are collected from the
-Kubernetes pods via Prometheus, and report CPU and Memory utilization of each
-container or Pod running in the cluster.
-
-In order to isolate and only display relevant metrics for a given environment
-however, GitLab needs a method to detect which pods are associated. To do that,
-GitLab will specifically request metrics that have an `environment` tag that
-matches the [$CI_ENVIRONMENT_SLUG][ci-environment-slug].
-
-If you are using [GitLab Auto-Deploy][autodeploy] and one of the methods of
-configuring Prometheus above, the `environment` will be automatically added.
-
-### GitLab Prometheus queries
-
-The queries utilized by GitLab are shown in the following table.
-
-| Metric | Query |
-| ------ | ----- |
-| Average Memory (MB) | `(sum(container_memory_usage_bytes{container_name!="POD",environment="$CI_ENVIRONMENT_SLUG"}) / count(container_memory_usage_bytes{container_name!="POD",environment="$CI_ENVIRONMENT_SLUG"})) /1024/1024` |
-| Average CPU Utilization (%) | `sum(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="$CI_ENVIRONMENT_SLUG"}[2m])) / count(container_cpu_usage_seconds_total{container_name!="POD",environment="$CI_ENVIRONMENT_SLUG"}) * 100` |
-
 ## Monitoring CI/CD Environments
 
 Once configured, GitLab will attempt to retrieve performance metrics for any
@@ -168,8 +139,9 @@ environment which has had a successful deployment.
 
 > [Introduced][ce-10408] in GitLab 9.2.
 > GitLab 9.3 added the [numeric comparison](https://gitlab.com/gitlab-org/gitlab-ce/issues/27439) of the 30 minute averages.
+> Requires [Kubernetes](prometheus_library/kubernetes.md) metrics
 
-Developers can view the performance impact of their changes within the merge
+Developers can view theperformance impact of their changes within the merge
 request workflow. When a source branch has been deployed to an environment, a sparkline and numeric comparison of the average memory consumption will appear. On the sparkline, a dot
 indicates when the current changes were deployed, with up to 30 minutes of
 performance data displayed before and after. The comparison shows the difference between the 30 minute average before and after the deployment. This information is updated after

doc/user/project/integrations/prometheus_library/cloudwatch.md

+# Monitoring AWS Resources
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/12621) in GitLab 9.4
+
+GitLab has support for automatically detecting and monitoring AWS resources, starting with the [Elastic Load Balancer](https://aws.amazon.com/elasticloadbalancing/). This is provided by leveraging the official [Cloudwatch exporter](https://github.com/prometheus/cloudwatch_exporter), which translates [Cloudwatch metrics](https://aws.amazon.com/cloudwatch/) into a Prometheus readable form.
+
+## Metrics supported
+
+| Name | Query |
+| ---- | ----- |
+| Throughput (req/sec) | sum(aws_elb_request_count_sum{%{environment_filter}}) / 60 |
+| Latency (ms) | avg(aws_elb_latency_average{%{environment_filter}}) * 1000 |
+| HTTP Error Rate (%) | sum(aws_elb_httpcode_backend_5_xx_sum{%{environment_filter}}) / sum(aws_elb_request_count_sum{%{environment_filter}}) |
+
+## Configuring Prometheus to monitor for Cloudwatch metrics
+
+To get started with Cloudwatch monitoring, you should install and configure the [Cloudwatch exporter](https://github.com/hnlq715/nginx-vts-exporter) which retrieves and parses the specified Cloudwatch metrics and translates them into a Prometheus monitoring endpoint.
+
+Right now, the only AWS resource supported is the Elastic Load Balancer, whose Cloudwatch metrics can be found [here](http://docs.aws.amazon.com/elasticloadbalancing/latest/classic/elb-cloudwatch-metrics.html).
+
+A sample Cloudwatch Exporter configuration file, configured for basic AWS ELB monitoring, is [available for download](../samples/cloudwatch.yml).
+
+## Specifying the Environment label
+
+In order to isolate and only display relevant metrics for a given environment
+however, GitLab needs a method to detect which labels are associated. To do this, GitLab will [look for an `environment` label](metrics.md#identifying-environments).

doc/user/project/integrations/prometheus_library/kubernetes.md

+# Monitoring Kubernetes
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/8935) in GitLab 9.0
+
+GitLab has support for automatically detecting and monitoring Kubernetes metrics. Kubernetes exposes Node level metrics out of the box via the built-in [Prometheus metrics support in cAdvisor](https://github.com/google/cadvisor). No additional services or exporters are needed.
+
+## Metrics supported
+
+| Name | Query |
+| ---- | ----- |
+| Average Memory Usage (MB) | (sum(container_memory_usage_bytes{container_name!="POD",%{environment_filter}}) / count(container_memory_usage_bytes{container_name!="POD",%{environment_filter}})) /1024/1024 |
+| Average CPU Utilization (%) | sum(rate(container_cpu_usage_seconds_total{container_name!="POD",%{environment_filter}}[2m])) / count(container_cpu_usage_seconds_total{container_name!="POD",%{environment_filter}}) * 100 |
+
+## Configuring Prometheus to monitor for Kubernetes node metrics
+
+In order for Prometheus to collect Kubernetes metrics, you first must have a
+Prometheus server up and running. You have two options here:
+
+- If you have an Omnibus based GitLab installation within your Kubernetes cluster, you can leverage the bundled Prometheus server to [monitor Kubernetes](../../../../administration/monitoring/prometheus/index.md#configuring-prometheus-to-monitor-kubernetes).
+- To configure your own Prometheus server, you can follow the [Prometheus documentation](https://prometheus.io/docs/introduction/overview/) or [our guide](../../../../administration/monitoring/prometheus/index.md#configuring-your-own-prometheus-server-within-kubernetes).
+
+## Specifying the Environment label
+
+In order to isolate and only display relevant metrics for a given environment
+however, GitLab needs a method to detect which labels are associated. To do this, GitLab will [look for an `environment` label](metrics.md#identifying-environments).
+
+If you are using [GitLab Auto-Deploy][autodeploy] and one of the two [provided Kubernetes monitoring solutions](../prometheus.md#getting-started-with-prometheus-monitoring), the `environment` label will be automatically added.

doc/user/project/integrations/prometheus_library/metrics.md

+# Prometheus Metrics library
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/8935) in GitLab 9.0
+
+GitLab offers automatic detection of select [Prometheus exporters](https://prometheus.io/docs/instrumenting/exporters/). Currently supported exporters are:
+* [Kubernetes](kubernetes.md)
+* [NGINX](nginx.md)
+* [Amazon Cloud Watch](cloudwatch.md)
+
+We have tried to surface the most important metrics for each exporter, and will be continuing to add support for additional exporters in future releases. If you would like to add support for other official exporters, [contributions](#adding-to-the-library) are welcome.
+
+## Identifying Environments
+
+GitLab retrieves performance data from the configured Prometheus server, and attempts to identifying the presence of known metrics. Once identified, GitLab then needs to be able to map the data to a particular environment.
+
+In order to isolate and only display relevant metrics for a given environment, GitLab needs a method to detect which labels are associated. To do that,
+GitLab will look for the required metrics which have a label that
+matches the [$CI_ENVIRONMENT_SLUG][ci-environment-slug].
+
+For example if you are deploying to an environment named `production`, there must be a label for the metric with the value of `production`.
+
+## Adding to the library
+
+We strive to support the 2-4 most important metrics for each common system service that supports Prometheus. If you are looking for support for a particular exporter which has not yet been added to the library, additions can be made [to the `additional_metrics.yml`](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/prometheus/additional_metrics.yml) file.
+
+> Note: The library is only for monitoring public, common, system services which all customers can benefit from. Support for monitoring [customer proprietary metrics](https://gitlab.com/gitlab-org/gitlab-ee/issues/2273) will be added in a subsequent release.

doc/user/project/integrations/prometheus_library/nginx.md

+# Monitoring NGINX
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/12621) in GitLab 9.4
+
+GitLab has support for automatically detecting and monitoring NGINX. This is provided by leveraging the [NGINX VTS exporter](https://github.com/hnlq715/nginx-vts-exporter), which translates [VTS statistics](https://github.com/vozlt/nginx-module-vts) into a Prometheus readable form.
+
+## Metrics supported
+
+| Name | Query |
+| ---- | ----- |
+| Throughput (req/sec) | sum(rate(nginx_requests_total{server_zone!="*", server_zone!="_", %{environment_filter}}[2m])) |
+| Latency (ms) | avg(nginx_upstream_response_msecs_avg{%{environment_filter}}) * 1000 |
+| HTTP Error Rate (%) | sum(nginx_responses_total{status_code="5xx", %{environment_filter}}) / sum(nginx_responses_total{server_zone!="*", server_zone!="_", %{environment_filter}}) |
+
+## Configuring Prometheus to monitor for NGINX metrics
+
+To get started with NGINX monitoring, you should first enable the [VTS statistics](https://github.com/vozlt/nginx-module-vts)) module for your NGINX server. This will capture and display statistics in an HTML readable form. Next, you should install and configure the [NGINX VTS exporter](https://github.com/hnlq715/nginx-vts-exporter) which parses these statistics and translates them into a Prometheus monitoring endpoint.
+
+If you are using NGINX as your Kubernetes ingress, there is [upcoming direct support](https://github.com/kubernetes/ingress/pull/423) for enabling Prometheus monitoring in the 0.9.0 release.
+
+## Specifying the Environment label
+
+In order to isolate and only display relevant metrics for a given environment
+however, GitLab needs a method to detect which labels are associated. To do this, GitLab will [look for an `environment` label](metrics.md#identifying-environments).

doc/user/project/integrations/samples/cloudwatch.yml

+region: us-east-1
+  metrics:
+    - aws_namespace: AWS/ELB
+      aws_metric_name: RequestCount
+      aws_dimensions: [AvailabilityZone, LoadBalancerName]
+      aws_dimension_select:
+        LoadBalancerName: [gitlab-ha-lb]
+      aws_statistics: [Sum]
+    - aws_namespace: AWS/ELB
+      aws_metric_name: Latency
+      aws_dimensions: [AvailabilityZone, LoadBalancerName]
+      aws_dimension_select:
+        LoadBalancerName: [gitlab-ha-lb]
+      aws_statistics: [Average]
+    - aws_namespace: AWS/ELB
+      aws_metric_name: HTTPCode_Backend_2XX
+      aws_dimensions: [AvailabilityZone, LoadBalancerName]
+      aws_dimension_select:
+        LoadBalancerName: [gitlab-ha-lb]
+      aws_statistics: [Sum]
+    - aws_namespace: AWS/ELB
+      aws_metric_name: HTTPCode_Backend_5XX
+      aws_dimensions: [AvailabilityZone, LoadBalancerName]
+      aws_dimension_select:
+        LoadBalancerName: [gitlab-ha-lb]
+      aws_statistics: [Sum]

doc/user/project/integrations/samples/prometheus.yml

@@ -24,6 +24,44 @@ data:
           target_label: environment
           regex: (.+)-.+-.+
           replacement: $1
+      - job_name: kubernetes-pods
+        tls_config:
+          ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+          insecure_skip_verify: true
+        bearer_token_file: "/var/run/secrets/kubernetes.io/serviceaccount/token"
+        kubernetes_sd_configs:
+        - role: pod
+          api_server: https://kubernetes.default.svc:443
+          tls_config:
+            ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+          bearer_token_file: "/var/run/secrets/kubernetes.io/serviceaccount/token"
+        relabel_configs:
+        - source_labels:
+          - __meta_kubernetes_pod_annotation_prometheus_io_scrape
+          action: keep
+          regex: 'true'
+        - source_labels:
+          - __meta_kubernetes_pod_annotation_prometheus_io_path
+          action: replace
+          target_label: __metrics_path__
+          regex: "(.+)"
+        - source_labels:
+          - __address__
+          - __meta_kubernetes_pod_annotation_prometheus_io_port
+          action: replace
+          regex: "([^:]+)(?::[0-9]+)?;([0-9]+)"
+          replacement: "$1:$2"
+          target_label: __address__
+        - action: labelmap
+          regex: __meta_kubernetes_pod_label_(.+)
+        - source_labels:
+          - __meta_kubernetes_namespace
+          action: replace
+          target_label: kubernetes_namespace
+        - source_labels:
+          - __meta_kubernetes_pod_name
+          action: replace
+          target_label: kubernetes_pod_name
 ---
 apiVersion: v1
 kind: Service

features/steps/groups.rb

@@ -104,7 +104,7 @@ class Spinach::Features::Groups < Spinach::FeatureSteps
 
   step 'I should see new group "Owned" avatar' do
     expect(owned_group.avatar).to be_instance_of AvatarUploader
-    expect(owned_group.avatar.url).to eq "/uploads/system/group/avatar/#{Group.find_by(name: "Owned").id}/banana_sample.gif"
+    expect(owned_group.avatar.url).to eq "/uploads/-/system/group/avatar/#{Group.find_by(name: "Owned").id}/banana_sample.gif"
   end
 
   step 'I should see the "Remove avatar" button' do

features/steps/profile/profile.rb

@@ -36,7 +36,7 @@ class Spinach::Features::Profile < Spinach::FeatureSteps
 
   step 'I should see new avatar' do
     expect(@user.avatar).to be_instance_of AvatarUploader
-    expect(@user.avatar.url).to eq "/uploads/system/user/avatar/#{@user.id}/banana_sample.gif"
+    expect(@user.avatar.url).to eq "/uploads/-/system/user/avatar/#{@user.id}/banana_sample.gif"
   end
 
   step 'I should see the "Remove avatar" button' do

features/steps/project/project.rb

@@ -39,7 +39,7 @@ class Spinach::Features::Project < Spinach::FeatureSteps
   step 'I should see new project avatar' do
     expect(@project.avatar).to be_instance_of AvatarUploader
     url = @project.avatar.url
-    expect(url).to eq "/uploads/system/project/avatar/#{@project.id}/banana_sample.gif"
+    expect(url).to eq "/uploads/-/system/project/avatar/#{@project.id}/banana_sample.gif"
   end
 
   step 'I should see the "Remove avatar" button' do

lib/api/internal.rb

@@ -162,7 +162,7 @@ module API
         #
         # begin
         #   repository = wiki? ? project.wiki.repository : project.repository
-        #   Gitlab::GitalyClient::Notifications.new(repository.raw_repository).post_receive
+        #   Gitlab::GitalyClient::NotificationService.new(repository.raw_repository).post_receive
         # rescue GRPC::Unavailable => e
         #   render_api_error!(e, 500)
         # end

lib/declarative_policy.rb

@@ -8,7 +8,12 @@ require_dependency 'declarative_policy/step'
 
 require_dependency 'declarative_policy/base'
 
+require 'thread'
+
 module DeclarativePolicy
+  CLASS_CACHE_MUTEX = Mutex.new
+  CLASS_CACHE_IVAR = :@__DeclarativePolicy_CLASS_CACHE
+
   class << self
     def policy_for(user, subject, opts = {})
       cache = opts[:cache] || {}
@@ -23,7 +28,36 @@ module DeclarativePolicy
 
       subject = find_delegate(subject)
 
-      subject.class.ancestors.each do |klass|
+      class_for_class(subject.class)
+    end
+
+    private
+
+    # This method is heavily cached because there are a lot of anonymous
+    # modules in play in a typical rails app, and #name performs quite
+    # slowly for anonymous classes and modules.
+    #
+    # See https://bugs.ruby-lang.org/issues/11119
+    #
+    # if the above bug is resolved, this caching could likely be removed.
+    def class_for_class(subject_class)
+      unless subject_class.instance_variable_defined?(CLASS_CACHE_IVAR)
+        CLASS_CACHE_MUTEX.synchronize do
+          # re-check in case of a race
+          break if subject_class.instance_variable_defined?(CLASS_CACHE_IVAR)
+
+          policy_class = compute_class_for_class(subject_class)
+          subject_class.instance_variable_set(CLASS_CACHE_IVAR, policy_class)
+        end
+      end
+
+      policy_class = subject_class.instance_variable_get(CLASS_CACHE_IVAR)
+      raise "no policy for #{subject.class.name}" if policy_class.nil?
+      policy_class
+    end
+
+    def compute_class_for_class(subject_class)
+      subject_class.ancestors.each do |klass|
         next unless klass.name
 
         begin
@@ -37,12 +71,8 @@ module DeclarativePolicy
           nil
         end
       end
-
-      raise "no policy for #{subject.class.name}"
     end
 
-    private
-
     def find_delegate(subject)
       seen = Set.new
 

lib/declarative_policy/cache.rb

@@ -21,11 +21,14 @@ module DeclarativePolicy
       private
 
       def id_for(obj)
-        if obj.respond_to?(:id) && obj.id
-          obj.id.to_s
-        else
-          "##{obj.object_id}"
-        end
+        id =
+          begin
+            obj.id
+          rescue NoMethodError
+            nil
+          end
+
+        id || "##{obj.object_id}"
       end
     end
   end

lib/declarative_policy/condition.rb

@@ -82,13 +82,14 @@ module DeclarativePolicy
     # depending on the scope, we may cache only by the user or only by
     # the subject, resulting in sharing across different policy objects.
     def cache_key
-      case @condition.scope
-      when :normal  then "/dp/condition/#{@condition.key}/#{user_key},#{subject_key}"
-      when :user    then "/dp/condition/#{@condition.key}/#{user_key}"
-      when :subject then "/dp/condition/#{@condition.key}/#{subject_key}"
-      when :global  then "/dp/condition/#{@condition.key}"
-      else raise 'invalid scope'
-      end
+      @cache_key ||=
+        case @condition.scope
+        when :normal  then "/dp/condition/#{@condition.key}/#{user_key},#{subject_key}"
+        when :user    then "/dp/condition/#{@condition.key}/#{user_key}"
+        when :subject then "/dp/condition/#{@condition.key}/#{subject_key}"
+        when :global  then "/dp/condition/#{@condition.key}"
+        else raise 'invalid scope'
+        end
     end
 
     def user_key

lib/gitlab/background_migration.rb

 module Gitlab
   module BackgroundMigration
+    def self.queue
+      @queue ||= BackgroundMigrationWorker.sidekiq_options['queue']
+    end
+
     # Begins stealing jobs from the background migrations queue, blocking the
     # caller until all jobs have been completed.
     #
+    # When a migration raises a StandardError is is going to be retries up to
+    # three times, for example, to recover from a deadlock.
+    #
+    # When Exception is being raised, it enqueues the migration again, and
+    # re-raises the exception.
+    #
     # steal_class - The name of the class for which to steal jobs.
     def self.steal(steal_class)
-      queue = Sidekiq::Queue
-        .new(BackgroundMigrationWorker.sidekiq_options['queue'])
+      enqueued = Sidekiq::Queue.new(self.queue)
+      scheduled = Sidekiq::ScheduledSet.new
 
-      queue.each do |job|
-        migration_class, migration_args = job.args
+      [scheduled, enqueued].each do |queue|
+        queue.each do |job|
+          migration_class, migration_args = job.args
 
-        next unless migration_class == steal_class
+          next unless job.queue == self.queue
+          next unless migration_class == steal_class
 
-        perform(migration_class, migration_args)
+          begin
+            perform(migration_class, migration_args, retries: 3) if job.delete
+          rescue Exception # rubocop:disable Lint/RescueException
+            BackgroundMigrationWorker # enqueue this migration again
+              .perform_async(migration_class, migration_args)
 
-        job.delete
+            raise
+          end
+        end
       end
     end
 
+    ##
+    # Performs a background migration.
+    #
     # class_name - The name of the background migration class as defined in the
     #              Gitlab::BackgroundMigration namespace.
     #

lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb

+module Gitlab
+  module BackgroundMigration
+    class MigrateSystemUploadsToNewFolder
+      include Gitlab::Database::MigrationHelpers
+      attr_reader :old_folder, :new_folder
+
+      class Upload < ActiveRecord::Base
+        self.table_name = 'uploads'
+        include EachBatch
+      end
+
+      def perform(old_folder, new_folder)
+        replace_sql = replace_sql(uploads[:path], old_folder, new_folder)
+        affected_uploads = Upload.where(uploads[:path].matches("#{old_folder}%"))
+
+        affected_uploads.each_batch do |batch|
+          batch.update_all("path = #{replace_sql}")
+        end
+      end
+
+      def uploads
+        Arel::Table.new('uploads')
+      end
+    end
+  end
+end

lib/gitlab/cache/request_cache.rb

+module Gitlab
+  module Cache
+    # This module provides a simple way to cache values in RequestStore,
+    # and the cache key would be based on the class name, method name,
+    # optionally customized instance level values, optionally customized
+    # method level values, and optional method arguments.
+    #
+    # A simple example:
+    #
+    # class UserAccess
+    #   extend Gitlab::Cache::RequestCache
+    #
+    #   request_cache_key do
+    #     [user&.id, project&.id]
+    #   end
+    #
+    #   request_cache def can_push_to_branch?(ref)
+    #     # ...
+    #   end
+    # end
+    #
+    # This way, the result of `can_push_to_branch?` would be cached in
+    # `RequestStore.store` based on the cache key. If RequestStore is not
+    # currently active, then it would be stored in a hash saved in an
+    # instance variable, so the cache logic would be the same.
+    # Here's another example using customized method level values:
+    #
+    # class Commit
+    #   extend Gitlab::Cache::RequestCache
+    #
+    #   def author
+    #     User.find_by_any_email(author_email.downcase)
+    #   end
+    #   request_cache(:author) { author_email.downcase }
+    # end
+    #
+    # So that we could have different strategies for different methods
+    #
+    module RequestCache
+      def self.extended(klass)
+        return if klass < self
+
+        extension = Module.new
+        klass.const_set(:RequestCacheExtension, extension)
+        klass.prepend(extension)
+      end
+
+      def request_cache_key(&block)
+        if block_given?
+          @request_cache_key = block
+        else
+          @request_cache_key
+        end
+      end
+
+      def request_cache(method_name, &method_key_block)
+        const_get(:RequestCacheExtension).module_eval do
+          cache_key_method_name = "#{method_name}_cache_key"
+
+          define_method(method_name) do |*args|
+            store =
+              if RequestStore.active?
+                RequestStore.store
+              else
+                ivar_name = # ! and ? cannot be used as ivar name
+                  "@cache_#{method_name.to_s.tr('!?', "\u2605\u2606")}"
+
+                instance_variable_get(ivar_name) ||
+                  instance_variable_set(ivar_name, {})
+              end
+
+            key = __send__(cache_key_method_name, args)
+
+            store.fetch(key) { store[key] = super(*args) }
+          end
+
+          define_method(cache_key_method_name) do |args|
+            klass = self.class
+
+            instance_key = instance_exec(&klass.request_cache_key) if
+              klass.request_cache_key
+
+            method_key = instance_exec(&method_key_block) if method_key_block
+
+            [klass.name, method_name, *instance_key, *method_key, *args]
+              .join(':')
+          end
+
+          private cache_key_method_name
+        end
+      end
+    end
+  end
+end

lib/gitlab/database/migration_helpers.rb

@@ -140,6 +140,8 @@ module Gitlab
           return add_foreign_key(source, target,
                                  column: column,
                                  on_delete: on_delete)
+        else
+          on_delete = 'SET NULL' if on_delete == :nullify
         end
 
         disable_statement_timeout
@@ -155,7 +157,7 @@ module Gitlab
         ADD CONSTRAINT #{key_name}
         FOREIGN KEY (#{column})
         REFERENCES #{target} (id)
-        #{on_delete ? "ON DELETE #{on_delete}" : ''}
+        #{on_delete ? "ON DELETE #{on_delete.upcase}" : ''}
         NOT VALID;
         EOF
 

lib/gitlab/git/blob.rb

@@ -29,7 +29,7 @@ module Gitlab
           path = path.sub(/\A\/*/, '')
           path = '/' if path.empty?
           name = File.basename(path)
-          entry = Gitlab::GitalyClient::Commit.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE)
+          entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE)
           return unless entry
 
           case entry.type
@@ -87,10 +87,10 @@ module Gitlab
         def raw(repository, sha)
           Gitlab::GitalyClient.migrate(:git_blob_raw) do |is_enabled|
             if is_enabled
-              Gitlab::GitalyClient::Blob.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE)
+              Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE)
             else
               blob = repository.lookup(sha)
-    
+
               new(
                 id: blob.oid,
                 size: blob.size,
@@ -182,7 +182,7 @@ module Gitlab
         Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled|
           @data = begin
             if is_enabled
-              Gitlab::GitalyClient::Blob.new(repository).get_blob(oid: id, limit: -1).data
+              Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: id, limit: -1).data
             else
               repository.lookup(id).content
             end

lib/gitlab/git/branch.rb

@@ -3,39 +3,8 @@
 module Gitlab
   module Git
     class Branch < Ref
-      def initialize(repository, name, target)
-        if target.is_a?(Gitaly::FindLocalBranchResponse)
-          target = target_from_gitaly_local_branches_response(target)
-        end
-
-        super(repository, name, target)
-      end
-
-      def target_from_gitaly_local_branches_response(response)
-        # Git messages have no encoding enforcements. However, in the UI we only
-        # handle UTF-8, so basically we cross our fingers that the message force
-        # encoded to UTF-8 is readable.
-        message = response.commit_subject.dup.force_encoding('UTF-8')
-
-        # NOTE: For ease of parsing in Gitaly, we have only the subject of
-        # the commit and not the full message. This is ok, since all the
-        # code that uses `local_branches` only cares at most about the
-        # commit message.
-        # TODO: Once gitaly "takes over" Rugged consider separating the
-        # subject from the message to make it clearer when there's one
-        # available but not the other.
-        hash = {
-          id: response.commit_id,
-          message: message,
-          authored_date: Time.at(response.commit_author.date.seconds),
-          author_name: response.commit_author.name.dup,
-          author_email: response.commit_author.email.dup,
-          committed_date: Time.at(response.commit_committer.date.seconds),
-          committer_name: response.commit_committer.name.dup,
-          committer_email: response.commit_committer.email.dup
-        }
-
-        Gitlab::Git::Commit.decorate(hash)
+      def initialize(repository, name, target, target_commit)
+        super(repository, name, target, target_commit)
       end
     end
   end

lib/gitlab/git/ref.rb

@@ -33,10 +33,9 @@ module Gitlab
         object
       end
 
-      def initialize(repository, name, target)
-        encode! name
-        @name = name.gsub(/\Arefs\/(tags|heads)\//, '')
-        @dereferenced_target = Gitlab::Git::Commit.find(repository, target)
+      def initialize(repository, name, target, derefenced_target)
+        @name = Gitlab::Git.ref_name(name)
+        @dereferenced_target = derefenced_target
         @target = if target.respond_to?(:oid)
                     target.oid
                   elsif target.respond_to?(:name)

lib/gitlab/git/repository.rb

@@ -80,16 +80,10 @@ module Gitlab
       end
 
       # Returns an Array of Branches
-      def branches(filter: nil, sort_by: nil)
-        branches = rugged.branches.each(filter).map do |rugged_ref|
-          begin
-            Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target)
-          rescue Rugged::ReferenceError
-            # Omit invalid branch
-          end
-        end.compact
-
-        sort_branches(branches, sort_by)
+      #
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/389
+      def branches(sort_by: nil)
+        branches_filter(sort_by: sort_by)
       end
 
       def reload_rugged
@@ -107,7 +101,10 @@ module Gitlab
         reload_rugged if force_reload
 
         rugged_ref = rugged.branches[name]
-        Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) if rugged_ref
+        if rugged_ref
+          target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
+          Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
+        end
       end
 
       def local_branches(sort_by: nil)
@@ -115,7 +112,7 @@ module Gitlab
           if is_enabled
             gitaly_ref_client.local_branches(sort_by: sort_by)
           else
-            branches(filter: :local, sort_by: sort_by)
+            branches_filter(filter: :local, sort_by: sort_by)
           end
         end
       end
@@ -162,6 +159,8 @@ module Gitlab
       end
 
       # Returns an Array of Tags
+      #
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/390
       def tags
         rugged.references.each("refs/tags/*").map do |ref|
           message = nil
@@ -174,7 +173,8 @@ module Gitlab
             end
           end
 
-          Gitlab::Git::Tag.new(self, ref.name, ref.target, message)
+          target_commit = Gitlab::Git::Commit.find(self, ref.target)
+          Gitlab::Git::Tag.new(self, ref.name, ref.target, target_commit, message)
         end.sort_by(&:name)
       end
 
@@ -204,13 +204,6 @@ module Gitlab
         branch_names + tag_names
       end
 
-      # Deprecated. Will be removed in 5.2
-      def heads
-        rugged.references.each("refs/heads/*").map do |head|
-          Gitlab::Git::Ref.new(self, head.name, head.target)
-        end.sort_by(&:name)
-      end
-
       def has_commits?
         !empty?
       end
@@ -297,28 +290,6 @@ module Gitlab
         (size.to_f / 1024).round(2)
       end
 
-      # Returns an array of BlobSnippets for files at the specified +ref+ that
-      # contain the +query+ string.
-      def search_files(query, ref = nil)
-        greps = []
-        ref ||= root_ref
-
-        populated_index(ref).each do |entry|
-          # Discard submodules
-          next if submodule?(entry)
-
-          blob = Gitlab::Git::Blob.raw(self, entry[:oid])
-
-          # Skip binary files
-          next if blob.data.encoding == Encoding::ASCII_8BIT
-
-          blob.load_all_data!(self)
-          greps += build_greps(blob.data, query, ref, entry[:path])
-        end
-
-        greps
-      end
-
       # Use the Rugged Walker API to build an array of commits.
       #
       # Usage.
@@ -707,7 +678,8 @@ module Gitlab
       #   create_branch("other-feature", "master")
       def create_branch(ref, start_point = "HEAD")
         rugged_ref = rugged.branches.create(ref, start_point)
-        Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target)
+        target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
+        Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
       rescue Rugged::ReferenceError => e
         raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/
         raise InvalidRef.new("Invalid reference #{start_point}")
@@ -836,15 +808,29 @@ module Gitlab
       end
 
       def gitaly_ref_client
-        @gitaly_ref_client ||= Gitlab::GitalyClient::Ref.new(self)
+        @gitaly_ref_client ||= Gitlab::GitalyClient::RefService.new(self)
       end
 
       def gitaly_commit_client
-        @gitaly_commit_client ||= Gitlab::GitalyClient::Commit.new(self)
+        @gitaly_commit_client ||= Gitlab::GitalyClient::CommitService.new(self)
       end
 
       private
 
+      # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'.
+      def branches_filter(filter: nil, sort_by: nil)
+        branches = rugged.branches.each(filter).map do |rugged_ref|
+          begin
+            target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
+            Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
+          rescue Rugged::ReferenceError
+            # Omit invalid branch
+          end
+        end.compact
+
+        sort_branches(branches, sort_by)
+      end
+
       def raw_log(options)
         default_options = {
           limit: 10,
@@ -1099,73 +1085,6 @@ module Gitlab
         index
       end
 
-      # Return an array of BlobSnippets for lines in +file_contents+ that match
-      # +query+
-      def build_greps(file_contents, query, ref, filename)
-        # The file_contents string is potentially huge so we make sure to loop
-        # through it one line at a time. This gives Ruby the chance to GC lines
-        # we are not interested in.
-        #
-        # We need to do a little extra work because we are not looking for just
-        # the lines that matches the query, but also for the context
-        # (surrounding lines). We will use Enumerable#each_cons to efficiently
-        # loop through the lines while keeping surrounding lines on hand.
-        #
-        # First, we turn "foo\nbar\nbaz" into
-        # [
-        #  [nil, -3], [nil, -2], [nil, -1],
-        #  ['foo', 0], ['bar', 1], ['baz', 3],
-        #  [nil, 4], [nil, 5], [nil, 6]
-        # ]
-        lines_with_index = Enumerator.new do |yielder|
-          # Yield fake 'before' lines for the first line of file_contents
-          (-SEARCH_CONTEXT_LINES..-1).each do |i|
-            yielder.yield [nil, i]
-          end
-
-          # Yield the actual file contents
-          count = 0
-          file_contents.each_line do |line|
-            line.chomp!
-            yielder.yield [line, count]
-            count += 1
-          end
-
-          # Yield fake 'after' lines for the last line of file_contents
-          (count + 1..count + SEARCH_CONTEXT_LINES).each do |i|
-            yielder.yield [nil, i]
-          end
-        end
-
-        greps = []
-
-        # Loop through consecutive blocks of lines with indexes
-        lines_with_index.each_cons(2 * SEARCH_CONTEXT_LINES + 1) do |line_block|
-          # Get the 'middle' line and index from the block
-          line, _ = line_block[SEARCH_CONTEXT_LINES]
-
-          next unless line && line.match(/#{Regexp.escape(query)}/i)
-
-          # Yay, 'line' contains a match!
-          # Get an array with just the context lines (no indexes)
-          match_with_context = line_block.map(&:first)
-          # Remove 'nil' lines in case we are close to the first or last line
-          match_with_context.compact!
-
-          # Get the line number (1-indexed) of the first context line
-          first_context_line_number = line_block[0][1] + 1
-
-          greps << Gitlab::Git::BlobSnippet.new(
-            ref,
-            match_with_context,
-            first_context_line_number,
-            filename
-          )
-        end
-
-        greps
-      end
-
       # Return the Rugged patches for the diff between +from+ and +to+.
       def diff_patches(from, to, options = {}, *paths)
         options ||= {}

lib/gitlab/git/tag.rb

@@ -5,8 +5,8 @@ module Gitlab
     class Tag < Ref
       attr_reader :object_sha
 
-      def initialize(repository, name, target, message = nil)
-        super(repository, name, target)
+      def initialize(repository, name, target, target_commit, message = nil)
+        super(repository, name, target, target_commit)
 
         @message = message
       end

lib/gitlab/gitaly_client/blob.rb → lib/gitlab/gitaly_client/blob_service.rb

 module Gitlab
   module GitalyClient
-    class Blob
+    class BlobService
       def initialize(repository)
         @gitaly_repo = repository.gitaly_repository
       end
-      
+
       def get_blob(oid:, limit:)
         request = Gitaly::GetBlobRequest.new(
           repository: @gitaly_repo,

lib/gitlab/gitaly_client/commit.rb → lib/gitlab/gitaly_client/commit_service.rb

 module Gitlab
   module GitalyClient
-    class Commit
+    class CommitService
       # The ID of empty tree.
       # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012
       EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze
@@ -17,20 +17,20 @@ module Gitlab
           child_id: child_id
         )
 
-        GitalyClient.call(@repository.storage, :commit, :commit_is_ancestor, request).value
+        GitalyClient.call(@repository.storage, :commit_service, :commit_is_ancestor, request).value
       end
 
       def diff_from_parent(commit, options = {})
         request_params = commit_diff_request_params(commit, options)
         request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false)
         request = Gitaly::CommitDiffRequest.new(request_params)
-        response = GitalyClient.call(@repository.storage, :diff, :commit_diff, request)
+        response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request)
         Gitlab::Git::DiffCollection.new(GitalyClient::DiffStitcher.new(response), options)
       end
 
       def commit_deltas(commit)
         request = Gitaly::CommitDeltaRequest.new(commit_diff_request_params(commit))
-        response = GitalyClient.call(@repository.storage, :diff, :commit_delta, request)
+        response = GitalyClient.call(@repository.storage, :diff_service, :commit_delta, request)
         response.flat_map do |msg|
           msg.deltas.map { |d| Gitlab::Git::Diff.new(d) }
         end
@@ -44,7 +44,7 @@ module Gitlab
           limit: limit.to_i
         )
 
-        response = GitalyClient.call(@repository.storage, :commit, :tree_entry, request)
+        response = GitalyClient.call(@repository.storage, :commit_service, :tree_entry, request)
         entry = response.first
         return unless entry.oid.present?
 

lib/gitlab/gitaly_client/notifications.rb → lib/gitlab/gitaly_client/notification_service.rb

 module Gitlab
   module GitalyClient
-    class Notifications
+    class NotificationService
       # 'repository' is a Gitlab::Git::Repository
       def initialize(repository)
         @gitaly_repo = repository.gitaly_repository
@@ -10,7 +10,7 @@ module Gitlab
       def post_receive
         GitalyClient.call(
           @storage,
-          :notifications,
+          :notification_service,
           :post_receive,
           Gitaly::PostReceiveRequest.new(repository: @gitaly_repo)
         )

lib/gitlab/gitaly_client/ref.rb → lib/gitlab/gitaly_client/ref_service.rb

 module Gitlab
   module GitalyClient
-    class Ref
+    class RefService
       include Gitlab::EncodingHelper
 
       # 'repository' is a Gitlab::Git::Repository
@@ -12,19 +12,19 @@ module Gitlab
 
       def default_branch_name
         request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo)
-        response = GitalyClient.call(@storage, :ref, :find_default_branch_name, request)
+        response = GitalyClient.call(@storage, :ref_service, :find_default_branch_name, request)
         Gitlab::Git.branch_name(response.name)
       end
 
       def branch_names
         request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo)
-        response = GitalyClient.call(@storage, :ref, :find_all_branch_names, request)
+        response = GitalyClient.call(@storage, :ref_service, :find_all_branch_names, request)
         consume_refs_response(response) { |name| Gitlab::Git.branch_name(name) }
       end
 
       def tag_names
         request = Gitaly::FindAllTagNamesRequest.new(repository: @gitaly_repo)
-        response = GitalyClient.call(@storage, :ref, :find_all_tag_names, request)
+        response = GitalyClient.call(@storage, :ref_service, :find_all_tag_names, request)
         consume_refs_response(response) { |name| Gitlab::Git.tag_name(name) }
       end
 
@@ -34,7 +34,7 @@ module Gitlab
           commit_id: commit_id,
           prefix: ref_prefix
         )
-        encode!(GitalyClient.call(@storage, :ref, :find_ref_name, request).name.dup)
+        encode!(GitalyClient.call(@storage, :ref_service, :find_ref_name, request).name.dup)
       end
 
       def count_tag_names
@@ -48,7 +48,7 @@ module Gitlab
       def local_branches(sort_by: nil)
         request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo)
         request.sort_by = sort_by_param(sort_by) if sort_by
-        response = GitalyClient.call(@storage, :ref, :find_local_branches, request)
+        response = GitalyClient.call(@storage, :ref_service, :find_local_branches, request)
         consume_branches_response(response)
       end
 
@@ -72,11 +72,39 @@ module Gitlab
             Gitlab::Git::Branch.new(
               @repository,
               encode!(gitaly_branch.name.dup),
-              gitaly_branch
+              gitaly_branch.commit_id,
+              commit_from_local_branches_response(gitaly_branch)
             )
           end
         end
       end
+
+      def commit_from_local_branches_response(response)
+        # Git messages have no encoding enforcements. However, in the UI we only
+        # handle UTF-8, so basically we cross our fingers that the message force
+        # encoded to UTF-8 is readable.
+        message = response.commit_subject.dup.force_encoding('UTF-8')
+
+        # NOTE: For ease of parsing in Gitaly, we have only the subject of
+        # the commit and not the full message. This is ok, since all the
+        # code that uses `local_branches` only cares at most about the
+        # commit message.
+        # TODO: Once gitaly "takes over" Rugged consider separating the
+        # subject from the message to make it clearer when there's one
+        # available but not the other.
+        hash = {
+          id: response.commit_id,
+          message: message,
+          authored_date: Time.at(response.commit_author.date.seconds),
+          author_name: response.commit_author.name.dup,
+          author_email: response.commit_author.email.dup,
+          committed_date: Time.at(response.commit_committer.date.seconds),
+          committer_name: response.commit_committer.name.dup,
+          committer_email: response.commit_committer.email.dup
+        }
+
+        Gitlab::Git::Commit.decorate(hash)
+      end
     end
   end
 end