Merge branch 'master' into 'dz-rename-reserved-project-names'

# Conflicts:
#   db/schema.rb

.gitlab-ci.yml

@@ -425,7 +425,7 @@ notify:slack:
     SETUP_DB: "false"
     USE_BUNDLE_INSTALL: "false"
   script:
-    - ./scripts/notify_slack.sh "#development" "Build on \`$CI_BUILD_REF_NAME\` failed! Commit \`$(git log -1 --oneline)\` See <https://gitlab.com/gitlab-org/$(basename "$PWD")/commit/"$CI_BUILD_REF"/builds>"
+    - ./scripts/notify_slack.sh "#development" "Build on \`$CI_BUILD_REF_NAME\` failed! Commit \`$(git log -1 --oneline)\` See <https://gitlab.com/gitlab-org/$(basename "$PWD")/commit/"$CI_BUILD_REF"/pipelines>"
   when: on_failure
   only:
     - master@gitlab-org/gitlab-ce

CONTRIBUTING.md

@@ -300,6 +300,7 @@ you start with a very simple UI? Can you do part of the refactor? The increased
 reviewability of small MRs that leads to higher code quality is more important
 to us than having a minimal commit log. The smaller an MR is the more likely it
 is it will be merged (quickly). After that you can send more MRs to enhance it.
+The ['How to get faster PR reviews' document of Kubernetes](https://github.com/kubernetes/community/blob/master/contributors/devel/faster_reviews.md) also has some great points regarding this.
 
 For examples of feedback on merge requests please look at already
 [closed merge requests][closed-merge-requests]. If you would like quick feedback

Gemfile

@@ -332,7 +332,7 @@ gem 'octokit', '~> 4.3.0'
 
 gem 'mail_room', '~> 0.9.0'
 
-gem 'email_reply_parser', '~> 0.5.8'
+gem 'email_reply_trimmer', '~> 0.1'
 gem 'html2text'
 
 gem 'ruby-prof', '~> 0.16.2'

Gemfile.lock

@@ -167,7 +167,7 @@ GEM
       railties (>= 4.2)
     dropzonejs-rails (0.7.2)
       rails (> 3.1)
-    email_reply_parser (0.5.8)
+    email_reply_trimmer (0.1.6)
     email_spec (1.6.0)
       launchy (~> 2.1)
       mail (~> 2.2)
@@ -839,7 +839,7 @@ DEPENDENCIES
   diffy (~> 3.1.0)
   doorkeeper (~> 4.2.0)
   dropzonejs-rails (~> 0.7.1)
-  email_reply_parser (~> 0.5.8)
+  email_reply_trimmer (~> 0.1)
   email_spec (~> 1.6.0)
   factory_girl_rails (~> 4.7.0)
   ffaker (~> 2.0.0)

README.md

@@ -15,10 +15,10 @@ To see how GitLab looks please see the [features page on our website](https://ab
 
 - Manage Git repositories with fine grained access controls that keep your code secure
 - Perform code reviews and enhance collaboration with merge requests
-- Each project can also have an issue tracker and a wiki
+- Complete continuous integration (CI) and CD pipelines to builds, test, and deploy your applications
+- Each project can also have an issue tracker, issue board, and a wiki
 - Used by more than 100,000 organizations, GitLab is the most popular solution to manage Git repositories on-premises
 - Completely free and open source (MIT Expat license)
-- Powered by [Ruby on Rails](https://github.com/rails/rails)
 
 ## Hiring
 
@@ -74,11 +74,11 @@ Instructions on how to start GitLab and how to run the tests can be found in the
 
 GitLab is a Ruby on Rails application that runs on the following software:
 
-- Ubuntu/Debian/CentOS/RHEL
+- Ubuntu/Debian/CentOS/RHEL/OpenSUSE
 - Ruby (MRI) 2.3
 - Git 2.8.4+
 - Redis 2.8+
-- MySQL or PostgreSQL
+- PostgreSQL (preferred) or MySQL
 
 For more information please see the [architecture documentation](https://docs.gitlab.com/ce/development/architecture.html).
 

app/assets/javascripts/build.js

@@ -92,8 +92,8 @@
         success: function(buildData) {
           $('.js-build-output').html(buildData.trace_html);
           if (removeRefreshStatuses.indexOf(buildData.status) >= 0) {
-            this.initScrollMonitor();
-            return this.$buildRefreshAnimation.remove();
+            this.$buildRefreshAnimation.remove();
+            return this.initScrollMonitor();
           }
         }.bind(this)
       });

app/assets/javascripts/gfm_auto_complete.js.es6

@@ -367,7 +367,7 @@
       return $input.trigger('keyup');
     },
     isLoading(data) {
-      if (!data) return false;
+      if (!data || !data.length) return false;
       if (Array.isArray(data)) data = data[0];
       return data === this.defaultLoadingData[0] || data.name === this.defaultLoadingData[0];
     },

app/assets/javascripts/issue.js

@@ -139,15 +139,12 @@
         return;
       }
       return $.getJSON($container.data('path')).error(function() {
-        $container.find('.checking').hide();
         $container.find('.unavailable').show();
         return new Flash('Failed to check if a new branch can be created.', 'alert');
       }).success(function(data) {
         if (data.can_create_branch) {
-          $container.find('.checking').hide();
           $container.find('.available').show();
         } else {
-          $container.find('.checking').hide();
           return $container.find('.unavailable').show();
         }
       });

app/assets/stylesheets/framework/lists.scss

@@ -239,7 +239,6 @@ ul.content-list {
 }
 
 ul.controls {
-  padding-top: 1px;
   float: right;
   list-style: none;
 

app/assets/stylesheets/pages/deploy_keys.scss

+.deploy-keys-list {
+  width: 100%;
+  overflow: auto;
+
+  table {
+    border: 1px solid $table-border-color;
+  }
+}
+
+.deploy-keys-title {
+  padding-bottom: 2px;
+  line-height: 2;
+}

app/assets/stylesheets/pages/note_form.scss

@@ -109,7 +109,7 @@
   margin: auto;
   margin-top: 0;
   text-align: center;
-  font-size: 13px;
+  font-size: 12px;
 
   @media (max-width: $screen-sm-max) {
     // On smaller devices the warning becomes the fourth item in the list,

app/assets/stylesheets/pages/notes.scss

@@ -557,19 +557,15 @@ ul.notes {
   &.is-active {
     color: $gl-text-green;
 
-    svg path {
+    svg {
       fill: $gl-text-green;
     }
   }
 
   svg {
     position: relative;
-    color: $gray-darkest;
-
-    path {
     fill: $gray-darkest;
   }
-  }
 }
 
 .discussion-next-btn {

app/assets/stylesheets/pages/pipelines.scss

@@ -335,7 +335,6 @@
   width: 100%;
   background-color: $gray-light;
   padding: $gl-padding;
-  overflow: auto;
   white-space: nowrap;
   transition: max-height 0.3s, padding 0.3s;
 
@@ -621,14 +620,14 @@
 }
 
 .dropdown-counter-badge {
-  float: right;
   color: $border-color;
   font-weight: 100;
   font-size: 15px;
-  margin-right: 2px;
+  position: absolute;
+  right: 5px;
+  top: 8px;
 }
 
-
 .grouped-pipeline-dropdown {
   padding: 0;
   width: 191px;

app/controllers/admin/deploy_keys_controller.rb

@@ -10,7 +10,7 @@ class Admin::DeployKeysController < Admin::ApplicationController
   end
 
   def create
-    @deploy_key = deploy_keys.new(deploy_key_params)
+    @deploy_key = deploy_keys.new(deploy_key_params.merge(user: current_user))
 
     if @deploy_key.save
       redirect_to admin_deploy_keys_path
@@ -39,6 +39,6 @@ class Admin::DeployKeysController < Admin::ApplicationController
   end
 
   def deploy_key_params
-    params.require(:deploy_key).permit(:key, :title)
+    params.require(:deploy_key).permit(:key, :title, :can_push)
   end
 end

app/controllers/admin/groups_controller.rb

 class Admin::GroupsController < Admin::ApplicationController
-  before_action :group, only: [:edit, :show, :update, :destroy, :project_update, :members_update]
+  before_action :group, only: [:edit, :update, :destroy, :project_update, :members_update]
 
   def index
-    @groups = Group.all
+    @groups = Group.with_statistics
     @groups = @groups.sort(@sort = params[:sort])
     @groups = @groups.search(params[:name]) if params[:name].present?
     @groups = @groups.page(params[:page])
   end
 
   def show
+    @group = Group.with_statistics.find_by_full_path(params[:id])
     @members = @group.members.order("access_level DESC").page(params[:members_page])
     @requesters = AccessRequestsFinder.new(@group).execute(current_user)
-    @projects = @group.projects.page(params[:projects_page])
+    @projects = @group.projects.with_statistics.page(params[:projects_page])
   end
 
   def new

app/controllers/admin/projects_controller.rb

@@ -3,7 +3,7 @@ class Admin::ProjectsController < Admin::ApplicationController
   before_action :group, only: [:show, :transfer]
 
   def index
-    @projects = Project.all
+    @projects = Project.with_statistics
     @projects = @projects.in_namespace(params[:namespace_id]) if params[:namespace_id].present?
     @projects = @projects.where(visibility_level: params[:visibility_level]) if params[:visibility_level].present?
     @projects = @projects.with_push if params[:with_push].present?

app/controllers/groups_controller.rb

@@ -75,7 +75,7 @@ class GroupsController < Groups::ApplicationController
   end
 
   def projects
-    @projects = @group.projects.page(params[:page])
+    @projects = @group.projects.with_statistics.page(params[:page])
   end
 
   def update

app/controllers/projects/deploy_keys_controller.rb

@@ -16,7 +16,7 @@ class Projects::DeployKeysController < Projects::ApplicationController
   end
 
   def create
-    @key = DeployKey.new(deploy_key_params)
+    @key = DeployKey.new(deploy_key_params.merge(user: current_user))
     set_index_vars
 
     if @key.valid? && @project.deploy_keys << @key
@@ -53,6 +53,6 @@ class Projects::DeployKeysController < Projects::ApplicationController
   end
 
   def deploy_key_params
-    params.require(:deploy_key).permit(:key, :title)
+    params.require(:deploy_key).permit(:key, :title, :can_push)
   end
 end

app/helpers/projects_helper.rb

@@ -90,10 +90,12 @@ module ProjectsHelper
   end
 
   def project_for_deploy_key(deploy_key)
-    if deploy_key.projects.include?(@project)
+    if deploy_key.has_access_to?(@project)
       @project
     else
-      deploy_key.projects.find { |project| can?(current_user, :read_project, project) }
+      deploy_key.projects.find do |project|
+        can?(current_user, :read_project, project)
+      end
     end
   end
 
@@ -246,11 +248,6 @@ module ProjectsHelper
     end
   end
 
-  def repository_size(project = @project)
-    size_in_bytes = project.repository_size * 1.megabyte
-    number_to_human_size(size_in_bytes, delimiter: ',', precision: 2)
-  end
-
   def default_url_to_repo(project = @project)
     case default_clone_protocol
     when 'ssh'
@@ -398,20 +395,6 @@ module ProjectsHelper
     [@project.path_with_namespace, sha, "readme"].join('-')
   end
 
-  def round_commit_count(project)
-    count = project.commit_count
-
-    if count > 10000
-      '10000+'
-    elsif count > 5000
-      '5000+'
-    elsif count > 1000
-      '1000+'
-    else
-      count
-    end
-  end
-
   def current_ref
     @ref || @repository.try(:root_ref)
   end

app/helpers/sorting_helper.rb

@@ -11,6 +11,7 @@ module SortingHelper
       sort_value_due_date_soon => sort_title_due_date_soon,
       sort_value_due_date_later => sort_title_due_date_later,
       sort_value_largest_repo => sort_title_largest_repo,
+      sort_value_largest_group => sort_title_largest_group,
       sort_value_recently_signin => sort_title_recently_signin,
       sort_value_oldest_signin => sort_title_oldest_signin,
       sort_value_downvotes => sort_title_downvotes,
@@ -92,6 +93,10 @@ module SortingHelper
     'Largest repository'
   end
 
+  def sort_title_largest_group
+    'Largest group'
+  end
+
   def sort_title_recently_signin
     'Recent sign in'
   end
@@ -193,7 +198,11 @@ module SortingHelper
   end
 
   def sort_value_largest_repo
-    'repository_size_desc'
+    'storage_size_desc'
+  end
+
+  def sort_value_largest_group
+    'storage_size_desc'
   end
 
   def sort_value_recently_signin

app/helpers/storage_helper.rb

+module StorageHelper
+  def storage_counter(size_in_bytes)
+    precision = size_in_bytes < 1.megabyte ? 0 : 1
+
+    number_to_human_size(size_in_bytes, delimiter: ',', precision: precision, significant: false)
+  end
+end

app/models/ci/build.rb

@@ -43,6 +43,8 @@ module Ci
     before_destroy { project }
 
     after_create :execute_hooks
+    after_save :update_project_statistics, if: :artifacts_size_changed?
+    after_destroy :update_project_statistics
 
     class << self
       def first_pending
@@ -584,5 +586,9 @@ module Ci
       Ci::MaskSecret.mask!(trace, token)
       trace
     end
+
+    def update_project_statistics
+      ProjectCacheWorker.perform_async(project_id, [], [:build_artifacts_size])
+    end
   end
 end

app/models/ci/pipeline.rb

@@ -93,11 +93,8 @@ module Ci
         .select("max(#{quoted_table_name}.id)")
         .group(:ref, :sha)
 
-      if ref
-        where(id: max_id, ref: ref)
-      else
-        where(id: max_id)
-      end
+      relation = ref ? where(ref: ref) : self
+      relation.where(id: max_id)
     end
 
     def self.latest_status(ref = nil)
@@ -105,7 +102,7 @@ module Ci
     end
 
     def self.latest_successful_for(ref)
-      success.latest(ref).first
+      success.latest(ref).order(id: :desc).first
     end
 
     def self.truncate_sha(sha)

app/models/deploy_key.rb

@@ -20,4 +20,18 @@ class DeployKey < Key
   def destroyed_when_orphaned?
     self.private?
   end
+
+  def has_access_to?(project)
+    projects.include?(project)
+  end
+
+  def can_push_to?(project)
+    can_push? && has_access_to?(project)
+  end
+
+  private
+
+  # we don't want to notify the user for deploy keys
+  def notify_user
+  end
 end

app/models/group.rb

@@ -48,8 +48,14 @@ class Group < Namespace
     end
 
     def sort(method)
+      if method == 'storage_size_desc'
+        # storage_size is a virtual column so we need to
+        # pass a string to avoid AR adding the table name
+        reorder('storage_size DESC, namespaces.id DESC')
+      else
         order_by(method)
       end
+    end
 
     def reference_prefix
       User.reference_prefix

app/models/key.rb

@@ -57,10 +57,6 @@ class Key < ActiveRecord::Base
     )
   end
 
-  def notify_user
-    run_after_commit { NotificationService.new.new_key(self) }
-  end
-
   def post_create_hook
     SystemHooksService.new.execute_hooks_for(self, :create)
   end
@@ -86,4 +82,8 @@ class Key < ActiveRecord::Base
 
     self.fingerprint = Gitlab::KeyFingerprint.new(self.key).fingerprint
   end
+
+  def notify_user
+    run_after_commit { NotificationService.new.new_key(self) }
+  end
 end

app/models/lfs_objects_project.rb

@@ -5,4 +5,13 @@ class LfsObjectsProject < ActiveRecord::Base
   validates :lfs_object_id, presence: true
   validates :lfs_object_id, uniqueness: { scope: [:project_id], message: "already exists in project" }
   validates :project_id, presence: true
+
+  after_create :update_project_statistics
+  after_destroy :update_project_statistics
+
+  private
+
+  def update_project_statistics
+    ProjectCacheWorker.perform_async(project_id, [], [:lfs_objects_size])
+  end
 end

app/models/merge_request.rb

@@ -198,7 +198,9 @@ class MergeRequest < ActiveRecord::Base
   end
 
   def diff_size
-    diffs(diff_options).size
+    opts = diff_options || {}
+
+    raw_diffs(opts).size
   end
 
   def diff_base_commit

app/models/namespace.rb

@@ -9,6 +9,7 @@ class Namespace < ActiveRecord::Base
   cache_markdown_field :description, pipeline: :description
 
   has_many :projects, dependent: :destroy
+  has_many :project_statistics
   belongs_to :owner, class_name: "User"
 
   belongs_to :parent, class_name: "Namespace"
@@ -38,6 +39,18 @@ class Namespace < ActiveRecord::Base
 
   scope :root, -> { where('type IS NULL') }
 
+  scope :with_statistics, -> do
+    joins('LEFT JOIN project_statistics ps ON ps.namespace_id = namespaces.id')
+      .group('namespaces.id')
+      .select(
+        'namespaces.*',
+        'COALESCE(SUM(ps.storage_size), 0) AS storage_size',
+        'COALESCE(SUM(ps.repository_size), 0) AS repository_size',
+        'COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size',
+        'COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size',
+      )
+  end
+
   class << self
     def by_path(path)
       find_by('lower(path) = :value', value: path.downcase)

app/models/project.rb

@@ -44,6 +44,7 @@ class Project < ActiveRecord::Base
   after_create :ensure_dir_exist
   after_create :create_project_feature, unless: :project_feature
   after_save :ensure_dir_exist, if: :namespace_id_changed?
+  after_save :update_project_statistics, if: :namespace_id_changed?
 
   # set last_activity_at to the same as created_at
   after_create :set_last_activity_at
@@ -151,6 +152,7 @@ class Project < ActiveRecord::Base
 
   has_one :import_data, dependent: :destroy, class_name: "ProjectImportData"
   has_one :project_feature, dependent: :destroy
+  has_one :statistics, class_name: 'ProjectStatistics', dependent: :delete
 
   has_many :commit_statuses, dependent: :destroy, foreign_key: :gl_project_id
   has_many :pipelines, dependent: :destroy, class_name: 'Ci::Pipeline', foreign_key: :gl_project_id
@@ -220,6 +222,7 @@ class Project < ActiveRecord::Base
   scope :with_push, -> { joins(:events).where('events.action = ?', Event::PUSHED) }
 
   scope :with_project_feature, -> { joins('LEFT JOIN project_features ON projects.id = project_features.project_id') }
+  scope :with_statistics, -> { includes(:statistics) }
 
   # "enabled" here means "not disabled". It includes private features!
   scope :with_feature_enabled, ->(feature) {
@@ -332,8 +335,10 @@ class Project < ActiveRecord::Base
     end
 
     def sort(method)
-      if method == 'repository_size_desc'
-        reorder(repository_size: :desc, id: :desc)
+      if method == 'storage_size_desc'
+        # storage_size is a joined column so we need to
+        # pass a string to avoid AR adding the table name
+        reorder('project_statistics.storage_size DESC, projects.id DESC')
       else
         order_by(method)
       end
@@ -1036,14 +1041,6 @@ class Project < ActiveRecord::Base
     forked? && project == forked_from_project
   end
 
-  def update_repository_size
-    update_attribute(:repository_size, repository.size)
-  end
-
-  def update_commit_count
-    update_attribute(:commit_count, repository.commit_count)
-  end
-
   def forks_count
     forks.count
   end
@@ -1322,4 +1319,9 @@ class Project < ActiveRecord::Base
   def full_path_changed?
     path_changed? || namespace_id_changed?
   end
+
+  def update_project_statistics
+    stats = statistics || build_statistics
+    stats.update(namespace_id: namespace_id)
+  end
 end

app/models/project_statistics.rb

+class ProjectStatistics < ActiveRecord::Base
+  belongs_to :project
+  belongs_to :namespace
+
+  before_save :update_storage_size
+
+  STORAGE_COLUMNS = [:repository_size, :lfs_objects_size, :build_artifacts_size]
+  STATISTICS_COLUMNS = [:commit_count] + STORAGE_COLUMNS
+
+  def total_repository_size
+    repository_size + lfs_objects_size
+  end
+
+  def refresh!(only: nil)
+    STATISTICS_COLUMNS.each do |column, generator|
+      if only.blank? || only.include?(column)
+        public_send("update_#{column}")
+      end
+    end
+
+    save!
+  end
+
+  def update_commit_count
+    self.commit_count = project.repository.commit_count
+  end
+
+  def update_repository_size
+    self.repository_size = project.repository.size
+  end
+
+  def update_lfs_objects_size
+    self.lfs_objects_size = project.lfs_objects.sum(:size)
+  end
+
+  def update_build_artifacts_size
+    self.build_artifacts_size = project.builds.sum(:artifacts_size)
+  end
+
+  def update_storage_size
+    self.storage_size = STORAGE_COLUMNS.sum(&method(:read_attribute))
+  end
+end

app/services/git_push_service.rb

@@ -77,7 +77,7 @@ class GitPushService < BaseService
       types = []
     end
 
-    ProjectCacheWorker.perform_async(@project.id, types)
+    ProjectCacheWorker.perform_async(@project.id, types, [:commit_count, :repository_size])
   end
 
   # Schedules processing of commit messages.

app/services/git_tag_push_service.rb

@@ -12,7 +12,7 @@ class GitTagPushService < BaseService
     project.execute_hooks(@push_data.dup, :tag_push_hooks)
     project.execute_services(@push_data.dup, :tag_push_hooks)
     Ci::CreatePipelineService.new(project, current_user, @push_data).execute
-    ProjectCacheWorker.perform_async(project.id)
+    ProjectCacheWorker.perform_async(project.id, [], [:commit_count, :repository_size])
 
     true
   end

app/services/notes/create_service.rb

@@ -41,7 +41,7 @@ module Notes
         # We must add the error after we call #save because errors are reset
         # when #save is called
         if only_commands
-          note.errors.add(:commands_only, 'Your commands have been executed!')
+          note.errors.add(:commands_only, 'Commands applied')
         end
 
         note.commands_changes = command_params.keys

app/views/admin/deploy_keys/index.html.haml

 - page_title "Deploy Keys"
-.panel.panel-default.prepend-top-default
-  .panel-heading
+
+%h3.page-title.deploy-keys-title
   Public deploy keys (#{@deploy_keys.count})
-    .controls
-      = link_to 'New Deploy Key', new_admin_deploy_key_path, class: "btn btn-new btn-sm"
-  - if @deploy_keys.any?
-    .table-holder
+  .pull-right
+    = link_to 'New Deploy Key', new_admin_deploy_key_path, class: 'btn btn-new btn-sm btn-inverted'
+
+- if @deploy_keys.any?
+  .table-holder.deploy-keys-list
     %table.table
-        %thead.panel-heading
+      %thead
         %tr
-            %th Title
-            %th Fingerprint
-            %th Added at
-            %th
+          %th.col-sm-2 Title
+          %th.col-sm-4 Fingerprint
+          %th.col-sm-2 Write access allowed
+          %th.col-sm-2 Added at
+          %th.col-sm-2
       %tbody
         - @deploy_keys.each do |deploy_key|
           %tr
@@ -20,8 +22,13 @@
               %strong= deploy_key.title
             %td
               %code.key-fingerprint= deploy_key.fingerprint
+            %td
+              - if deploy_key.can_push?
+                Yes
+              - else
+                No
             %td
               %span.cgray
                 added #{time_ago_with_tooltip(deploy_key.created_at)}
             %td
-                = link_to 'Remove', admin_deploy_key_path(deploy_key), data: { confirm: 'Are you sure?'}, method: :delete, class: "btn btn-sm btn-remove delete-key pull-right"
+              = link_to 'Remove', admin_deploy_key_path(deploy_key), data: { confirm: 'Are you sure?'}, method: :delete, class: 'btn btn-sm btn-remove delete-key pull-right'

app/views/admin/deploy_keys/new.html.haml

@@ -16,6 +16,14 @@
           Paste a machine public key here. Read more about how to generate it
           = link_to "here", help_page_path("ssh/README")
         = f.text_area :key, class: "form-control thin_area", rows: 5
+    .form-group
+      .control-label
+      .col-sm-10
+        = f.label :can_push do
+          = f.check_box :can_push
+          %strong Write access allowed
+        %p.light.append-bottom-0
+          Allow this key to push to repository as well? (Default only allows pull access.)
 
     .form-actions
       = f.submit 'Create', class: "btn-create btn"

app/views/admin/groups/_group.html.haml

@@ -5,6 +5,9 @@
     = link_to 'Edit', admin_group_edit_path(group), id: "edit_#{dom_id(group)}", class: 'btn'
     = link_to 'Delete', [:admin, group], data: { confirm: "Are you sure you want to remove #{group.name}?" }, method: :delete, class: 'btn btn-remove'
   .stats
+    %span.badge
+      = storage_counter(group.storage_size)
+
     %span
       = icon('bookmark')
       = number_with_delimiter(group.projects.count)

app/views/admin/groups/index.html.haml

@@ -27,6 +27,8 @@
                   = sort_title_recently_updated
                 = link_to admin_groups_path(sort: sort_value_oldest_updated, name: project_name) do
                   = sort_title_oldest_updated
+                = link_to admin_groups_path(sort: sort_value_largest_group, name: project_name) do
+                  = sort_title_largest_group
           = link_to new_admin_group_path, class: "btn btn-new" do
             New Group
   %ul.content-list

app/views/admin/groups/show.html.haml

@@ -38,6 +38,18 @@
           %strong
             = @group.created_at.to_s(:medium)
 
+        %li
+          %span.light Storage:
+          %strong= storage_counter(@group.storage_size)
+          (
+          = storage_counter(@group.repository_size)
+          repositories,
+          = storage_counter(@group.build_artifacts_size)
+          build artifacts,
+          = storage_counter(@group.lfs_objects_size)
+          LFS
+          )
+
         %li
           %span.light Group Git LFS status:
           %strong
@@ -55,8 +67,8 @@
           %li
             %strong
               = link_to project.name_with_namespace, [:admin, project.namespace.becomes(Namespace), project]
-              %span.label.label-gray
-                = repository_size(project)
+              %span.badge
+                = storage_counter(project.statistics.storage_size)
             %span.pull-right.light
               %span.monospace= project.path_with_namespace + ".git"
       .panel-footer
@@ -73,8 +85,8 @@
             %li
               %strong
                 = link_to project.name_with_namespace, [:admin, project.namespace.becomes(Namespace), project]
-                %span.label.label-gray
-                  = repository_size(project)
+                %span.badge
+                  = storage_counter(project.statistics.storage_size)
               %span.pull-right.light
                 %span.monospace= project.path_with_namespace + ".git"
 

app/views/admin/projects/index.html.haml

@@ -69,8 +69,8 @@
             .controls
               - if project.archived
                 %span.label.label-warning archived
-              %span.label.label-gray
-                = repository_size(project)
+              %span.badge
+                = storage_counter(project.statistics.storage_size)
               = link_to 'Edit', edit_namespace_project_path(project.namespace, project), id: "edit_#{dom_id(project)}", class: "btn"
               = link_to 'Delete', [project.namespace.becomes(Namespace), project], data: { confirm: remove_project_message(project) }, method: :delete, class: "btn btn-remove"
             .title

app/views/admin/projects/show.html.haml

@@ -65,9 +65,16 @@
               = @project.repository.path_to_repo
 
           %li
-            %span.light Size
-            %strong
-              = repository_size(@project)
+            %span.light Storage:
+            %strong= storage_counter(@project.statistics.storage_size)
+            (
+            = storage_counter(@project.statistics.repository_size)
+            repository,
+            = storage_counter(@project.statistics.build_artifacts_size)
+            build artifacts,
+            = storage_counter(@project.statistics.lfs_objects_size)
+            LFS
+            )
 
           %li
             %span.light last commit:

app/views/groups/projects.html.haml

@@ -18,8 +18,8 @@
         .pull-right
           - if project.archived
             %span.label.label-warning archived
-          %span.label.label-gray
-            = repository_size(project)
+          %span.badge
+            = storage_counter(project.statistics.storage_size)
           = link_to 'Members', namespace_project_project_members_path(project.namespace, project), id: "edit_#{dom_id(project)}", class: "btn btn-sm"
           = link_to 'Edit', edit_namespace_project_path(project.namespace, project), id: "edit_#{dom_id(project)}", class: "btn btn-sm"
           = link_to 'Remove', project, data: { confirm: remove_project_message(project)}, method: :delete, class: "btn btn-sm btn-remove"

app/views/projects/deploy_keys/_deploy_key.html.haml

@@ -6,6 +6,9 @@
       = deploy_key.title
     .description
       = deploy_key.fingerprint
+    - if deploy_key.can_push?
+      .write-access-allowed
+        Write access allowed
   .deploy-key-content.prepend-left-default.deploy-key-projects
     - deploy_key.projects.each do |project|
       - if can?(current_user, :read_project, project)

app/views/projects/deploy_keys/_form.html.haml

@@ -10,4 +10,13 @@
     %p.light.append-bottom-0
       Paste a machine public key here. Read more about how to generate it
       = link_to "here", help_page_path("ssh/README")
+  .form-group
+    .checkbox
+      = f.label :can_push do
+        = f.check_box :can_push
+        %strong Write access allowed
+  .form-group
+    %p.light.append-bottom-0
+      Allow this key to push to repository as well? (Default only allows pull access.)
+
   = f.submit "Add key", class: "btn-create btn"

app/views/projects/diffs/_file_header.html.haml

@@ -21,7 +21,7 @@
       - if diff_file.deleted_file
         deleted
 
-  = clipboard_button(clipboard_text: diff_file.new_path, class: 'btn-clipboard btn-transparent prepend-left-5', title: 'Copy filename to clipboard')
+  = clipboard_button(clipboard_text: diff_file.new_path, class: 'btn-clipboard btn-transparent prepend-left-5', title: 'Copy file path to clipboard')
 
   - if diff_file.mode_changed?
     %small

app/views/projects/issues/_new_branch.html.haml

 - if can?(current_user, :push_code, @project)
   .pull-right
     #new-branch.new-branch{'data-path' => can_create_branch_namespace_project_issue_path(@project.namespace, @project, @issue)}
-      = link_to '#', class: 'checking btn btn-grouped', disabled: 'disabled' do
-        = icon('spinner spin')
-        Checking branches
       = link_to namespace_project_branches_path(@project.namespace, @project, branch_name: @issue.to_branch_name, issue_iid: @issue.iid),
         method: :post, class: 'btn btn-new btn-inverted btn-grouped has-tooltip available hide', title: @issue.to_branch_name do
         New branch

app/views/projects/mattermosts/_team_selection.html.haml

@@ -8,7 +8,9 @@
     = @teams.one? ? 'The team' : 'Select the team'
     where the slash commands will be used in
   - selected_id = @teams.keys.first if @teams.one?
-  = f.select(:team_id, mattermost_teams_options(@teams), {}, { class: 'form-control', selected: "#{selected_id}", disabled: @teams.one? })
+  - options = mattermost_teams_options(@teams)
+  - options = options_for_select(options, selected_id)
+  = f.select(:team_id, options, {}, { class: 'form-control', selected: "#{selected_id}" })
   .help-block
     - if @teams.one?
       This is the only team where you are an administrator.

app/views/projects/show.html.haml

@@ -17,10 +17,10 @@
     %ul.nav
       %li
         = link_to project_files_path(@project) do
-          Files (#{repository_size})
+          Files (#{storage_counter(@project.statistics.total_repository_size)})
       %li
         = link_to namespace_project_commits_path(@project.namespace, @project, current_ref) do
-          #{'Commit'.pluralize(@project.commit_count)} (#{number_with_delimiter(@project.commit_count)})
+          #{'Commit'.pluralize(@project.statistics.commit_count)} (#{number_with_delimiter(@project.statistics.commit_count)})
       %li
         = link_to namespace_project_branches_path(@project.namespace, @project) do
           #{'Branch'.pluralize(@repository.branch_count)} (#{number_with_delimiter(@repository.branch_count)})
@@ -70,8 +70,8 @@
             = link_to 'Set up Koding', add_koding_stack_path(@project)
         - if @repository.gitlab_ci_yml.blank? && @project.deployment_service.present?
           %li.missing
-            = link_to add_special_file_path(@project, file_name: '.gitlab-ci.yml', commit_message: 'Set up autodeploy', target_branch: 'autodeploy', context: 'autodeploy') do
-              Set up autodeploy
+            = link_to add_special_file_path(@project, file_name: '.gitlab-ci.yml', commit_message: 'Set up auto deploy', target_branch: 'auto-deploy', context: 'autodeploy') do
+              Set up auto deploy
 
   - if @repository.commit
     .project-last-commit{ class: container_class }

app/workers/project_cache_worker.rb

@@ -6,26 +6,27 @@ class ProjectCacheWorker
   LEASE_TIMEOUT = 15.minutes.to_i
 
   # project_id - The ID of the project for which to flush the cache.
-  # refresh - An Array containing extra types of data to refresh such as
+  # files - An Array containing extra types of files to refresh such as
   #         `:readme` to flush the README and `:changelog` to flush the
   #         CHANGELOG.
-  def perform(project_id, refresh = [])
+  # statistics - An Array containing columns from ProjectStatistics to
+  #              refresh, if empty all columns will be refreshed
+  def perform(project_id, files = [], statistics = [])
     project = Project.find_by(id: project_id)
 
     return unless project && project.repository.exists?
 
-    update_repository_size(project)
-    project.update_commit_count
+    update_statistics(project, statistics.map(&:to_sym))
 
-    project.repository.refresh_method_caches(refresh.map(&:to_sym))
+    project.repository.refresh_method_caches(files.map(&:to_sym))
   end
 
-  def update_repository_size(project)
-    return unless try_obtain_lease_for(project.id, :update_repository_size)
+  def update_statistics(project, statistics = [])
+    return unless try_obtain_lease_for(project.id, :update_statistics)
 
-    Rails.logger.info("Updating repository size for project #{project.id}")
+    Rails.logger.info("Updating statistics for project #{project.id}")
 
-    project.update_repository_size
+    project.statistics.refresh!(only: statistics)
   end
 
   private

changelogs/unreleased/25705-your-commands-have-been-executed-is-overkill.yml

+---
+title: Replace wording for slash command confirmation message
+merge_request: 8123

changelogs/unreleased/25909-fix-mr-list-timestamp-alignment.yml

+---
+title: Fix mr list timestamp alignment
+merge_request: 8271
+author:

changelogs/unreleased/25931-gitlab-merge-request-view-crash-when-commiting-a-js-sourcemap-file.yml

+---
+title: Fix timeout when MR contains large files marked as binary by .gitattributes
+merge_request:
+author:

changelogs/unreleased/26026-pipeline-overflow-firefox.yml

+---
+title: Fix line breaking in nodes of the pipeline graph in firefox
+merge_request: 8292
+author:

changelogs/unreleased/26038-fix-confedential-warning-text-alignment.yml

+---
+title: Fixes confendential warning text alignment
+merge_request: 8293
+author:

changelogs/unreleased/26040-hide-scroll-top.yml

+---
+title: Hide Scroll Top button for failed build page
+merge_request: 8295
+author:

changelogs/unreleased/auto-deploy-with-space.yml

+---
+title: Rename "autodeploy" to "auto deploy"
+merge_request:
+author:

changelogs/unreleased/changelog-mr8322.yml

+---
+title: Disable PostgreSQL statement timeouts when removing unneeded services
+merge_request: 8322
+author:

changelogs/unreleased/dz-rename-invalid-users.yml

+---
+title: Rename users with namespace ending with .git
+merge_request: 8309
+author:

changelogs/unreleased/feature-1376-allow-write-access-deploy-keys.yml

+---
+title: Allow to add deploy keys with write-access
+merge_request: 5807
+author: Ali Ibrahim

changelogs/unreleased/feature-more-storage-statistics.yml

+---
+title: Add more storage statistics
+merge_request: 7754
+author: Markus Koller

changelogs/unreleased/filename-to-file-path.yml

+---
+title: Rename filename to file path in tooltip of file header in merge request diff
+merge_request: 8314
\ No newline at end of file

changelogs/unreleased/fix-latest-pipeine-ordering.yml

+---
+title: Fix finding the latest pipeline
+merge_request: 8301
+author:

changelogs/unreleased/label-gfm-error-fix.yml

+---
+title: Fixed GFM autocomplete error when no data exists
+merge_request:
+author:

changelogs/unreleased/re-style-issue-new-branch.yml

+---
+title: Remove checking branches state in issue new branch button
+merge_request: 8023

changelogs/unreleased/resolve-note-svg-color.yml

+---
+title: Fixed resolve discussion note button color
+merge_request:
+author:

config/initializers/inflections.rb

@@ -10,5 +10,5 @@
 # end
 #
 ActiveSupport::Inflector.inflections do |inflect|
-  inflect.uncountable %w(award_emoji)
+  inflect.uncountable %w(award_emoji project_statistics)
 end

db/migrate/20160811172945_add_can_push_to_keys.rb

+class AddCanPushToKeys < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+
+  def up
+    add_column_with_default(:keys, :can_push, :boolean, default: false, allow_null: false)
+  end
+
+  def down
+    remove_column(:keys, :can_push)
+  end
+end

db/migrate/20161201155511_create_project_statistics.rb

+class CreateProjectStatistics < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    # use bigint columns to support values >2GB
+    counter_column = { limit: 8, null: false, default: 0 }
+
+    create_table :project_statistics do |t|
+      t.references :project, null: false, index: { unique: true }, foreign_key: { on_delete: :cascade }
+      t.references :namespace, null: false, index: true
+      t.integer :commit_count, counter_column
+      t.integer :storage_size, counter_column
+      t.integer :repository_size, counter_column
+      t.integer :lfs_objects_size, counter_column
+      t.integer :build_artifacts_size, counter_column
+    end
+  end
+end

db/migrate/20161201160452_migrate_project_statistics.rb

+class MigrateProjectStatistics < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = true
+  DOWNTIME_REASON = 'Removes two columns from the projects table'
+
+  def up
+    # convert repository_size in float (megabytes) to integer (bytes),
+    # initialize total storage_size with repository_size
+    execute <<-EOF
+      INSERT INTO project_statistics (project_id, namespace_id, commit_count, storage_size, repository_size)
+        SELECT id, namespace_id, commit_count, (repository_size * 1024 * 1024), (repository_size * 1024 * 1024) FROM projects
+    EOF
+
+    remove_column :projects, :repository_size
+    remove_column :projects, :commit_count
+  end
+
+  def down
+    add_column_with_default :projects, :repository_size, :float, default: 0.0
+    add_column_with_default :projects, :commit_count, :integer, default: 0
+  end
+end

db/migrate/20161226122833_remove_dot_git_from_usernames.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveDotGitFromUsernames < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+  include Gitlab::ShellAdapter
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def up
+    invalid_users.each do |user|
+      id = user['id']
+      namespace_id = user['namespace_id']
+      path_was = user['username']
+      path_was_wildcard = quote_string("#{path_was}/%")
+      path = quote_string(rename_path(path_was))
+
+      move_namespace(namespace_id, path_was, path)
+
+      execute "UPDATE routes SET path = '#{path}' WHERE source_type = 'Namespace' AND source_id = #{namespace_id}"
+      execute "UPDATE namespaces SET path = '#{path}' WHERE id = #{namespace_id}"
+      execute "UPDATE users SET username = '#{path}' WHERE id = #{id}"
+
+      select_all("SELECT id, path FROM routes WHERE path LIKE '#{path_was_wildcard}'").each do |route|
+        new_path = "#{path}/#{route['path'].split('/').last}"
+        execute "UPDATE routes SET path = '#{new_path}' WHERE id = #{route['id']}"
+      end
+    end
+  end
+
+  def down
+    # nothing to do here
+  end
+
+  private
+
+  def invalid_users
+    select_all("SELECT u.id, u.username, n.path AS namespace_path, n.id AS namespace_id FROM users u
+                INNER JOIN namespaces n ON n.owner_id = u.id
+                WHERE n.type is NULL AND n.path LIKE '%.git'")
+  end
+
+  def route_exists?(path)
+    select_all("SELECT id, path FROM routes WHERE path = '#{quote_string(path)}'").present?
+  end
+
+  # Accepts invalid path like test.git and returns test_git or
+  # test_git1 if test_git already taken
+  def rename_path(path)
+    # To stay closer with original name and reduce risk of duplicates
+    # we rename suffix instead of removing it
+    path = path.sub(/\.git\z/, '_git')
+
+    counter = 0
+    base = path
+
+    while route_exists?(path)
+      counter += 1
+      path = "#{base}#{counter}"
+    end
+
+    path
+  end
+
+  def move_namespace(namespace_id, path_was, path)
+    repository_storage_paths = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{namespace_id}").map do |row|
+      Gitlab.config.repositories.storages[row['repository_storage']]
+    end.compact
+
+    # Move the namespace directory in all storages paths used by member projects
+    repository_storage_paths.each do |repository_storage_path|
+      # Ensure old directory exists before moving it
+      gitlab_shell.add_namespace(repository_storage_path, path_was)
+
+      unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path)
+        Rails.logger.error "Exception moving path #{repository_storage_path} from #{path_was} to #{path}"
+
+        # if we cannot move namespace directory we should rollback
+        # db changes in order to prevent out of sync between db and fs
+        raise Exception.new('namespace directory cannot be moved')
+      end
+    end
+
+    Gitlab::UploadsTransfer.new.rename_namespace(path_was, path)
+  end
+end

db/post_migrate/20161221140236_remove_unneeded_services.rb

@@ -4,6 +4,8 @@ class RemoveUnneededServices < ActiveRecord::Migration
   DOWNTIME = false
 
   def up
+    disable_statement_timeout
+
     execute("DELETE FROM services WHERE active = false AND properties = '{}';")
   end
 

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20161221153951) do
+ActiveRecord::Schema.define(version: 20161226122833) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -527,6 +527,7 @@ ActiveRecord::Schema.define(version: 20161221153951) do
     t.string "type"
     t.string "fingerprint"
     t.boolean "public", default: false, null: false
+    t.boolean "can_push", default: false, null: false
   end
 
   add_index "keys", ["fingerprint"], name: "index_keys_on_fingerprint", unique: true, using: :btree
@@ -901,6 +902,19 @@ ActiveRecord::Schema.define(version: 20161221153951) do
 
   add_index "project_import_data", ["project_id"], name: "index_project_import_data_on_project_id", using: :btree
 
+  create_table "project_statistics", force: :cascade do |t|
+    t.integer "project_id", null: false
+    t.integer "namespace_id", null: false
+    t.integer "commit_count", limit: 8, default: 0, null: false
+    t.integer "storage_size", limit: 8, default: 0, null: false
+    t.integer "repository_size", limit: 8, default: 0, null: false
+    t.integer "lfs_objects_size", limit: 8, default: 0, null: false
+    t.integer "build_artifacts_size", limit: 8, default: 0, null: false
+  end
+
+  add_index "project_statistics", ["namespace_id"], name: "index_project_statistics_on_namespace_id", using: :btree
+  add_index "project_statistics", ["project_id"], name: "index_project_statistics_on_project_id", unique: true, using: :btree
+
   create_table "projects", force: :cascade do |t|
     t.string "name"
     t.string "path"
@@ -915,11 +929,9 @@ ActiveRecord::Schema.define(version: 20161221153951) do
     t.boolean "archived", default: false, null: false
     t.string "avatar"
     t.string "import_status"
-    t.float "repository_size", default: 0.0
     t.integer "star_count", default: 0, null: false
     t.string "import_type"
     t.string "import_source"
-    t.integer "commit_count", default: 0
     t.text "import_error"
     t.integer "ci_id"
     t.boolean "shared_runners_enabled", default: true, null: false
@@ -1288,6 +1300,7 @@ ActiveRecord::Schema.define(version: 20161221153951) do
   add_foreign_key "personal_access_tokens", "users"
   add_foreign_key "project_authorizations", "projects", on_delete: :cascade
   add_foreign_key "project_authorizations", "users", on_delete: :cascade
+  add_foreign_key "project_statistics", "projects", on_delete: :cascade
   add_foreign_key "protected_branch_merge_access_levels", "protected_branches"
   add_foreign_key "protected_branch_push_access_levels", "protected_branches"
   add_foreign_key "subscriptions", "projects", on_delete: :cascade

doc/administration/build_artifacts.md

@@ -88,3 +88,9 @@ artifacts through the [Admin area settings](../user/admin_area/settings/continuo
 
 [reconfigure gitlab]: restart_gitlab.md "How to restart GitLab"
 [restart gitlab]: restart_gitlab.md "How to restart GitLab"
+
+## Storage statistics
+
+You can see the total storage used for build artifacts on groups and projects
+in the administration area, as well as through the [groups](../api/groups.md)
+and [projects APIs](../api/projects.md).

doc/api/deploy_keys.md

@@ -20,12 +20,14 @@ Example response:
     "id": 1,
     "title": "Public key",
     "key": "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAIEAiPWx6WM4lhHNedGfBpPJNPpZ7yKu+dnn1SJejgt4596k6YjzGGphH2TUxwKzxcKDKKezwkpfnxPkSMkuEspGRt/aZZ9wa++Oi7Qkr8prgHc4soW6NUlfDzpvZK2H5E7eQaSeP3SAwGmQKUFHCddNaP0L+hM7zhFNzjFvpaMgJw0=",
+    "can_push": false,
     "created_at": "2013-10-02T10:12:29Z"
   },
   {
     "id": 3,
     "title": "Another Public key",
     "key": "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAIEAiPWx6WM4lhHNedGfBpPJNPpZ7yKu+dnn1SJejgt4596k6YjzGGphH2TUxwKzxcKDKKezwkpfnxPkSMkuEspGRt/aZZ9wa++Oi7Qkr8prgHc4soW6NUlfDzpvZK2H5E7eQaSeP3SAwGmQKUFHCddNaP0L+hM7zhFNzjFvpaMgJw0=",
+    "can_push": true,
     "created_at": "2013-10-02T11:12:29Z"
   }
 ]
@@ -55,12 +57,14 @@ Example response:
     "id": 1,
     "title": "Public key",
     "key": "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAIEAiPWx6WM4lhHNedGfBpPJNPpZ7yKu+dnn1SJejgt4596k6YjzGGphH2TUxwKzxcKDKKezwkpfnxPkSMkuEspGRt/aZZ9wa++Oi7Qkr8prgHc4soW6NUlfDzpvZK2H5E7eQaSeP3SAwGmQKUFHCddNaP0L+hM7zhFNzjFvpaMgJw0=",
+    "can_push": false,
     "created_at": "2013-10-02T10:12:29Z"
   },
   {
     "id": 3,
     "title": "Another Public key",
     "key": "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAIEAiPWx6WM4lhHNedGfBpPJNPpZ7yKu+dnn1SJejgt4596k6YjzGGphH2TUxwKzxcKDKKezwkpfnxPkSMkuEspGRt/aZZ9wa++Oi7Qkr8prgHc4soW6NUlfDzpvZK2H5E7eQaSeP3SAwGmQKUFHCddNaP0L+hM7zhFNzjFvpaMgJw0=",
+    "can_push": false,
     "created_at": "2013-10-02T11:12:29Z"
   }
 ]
@@ -92,6 +96,7 @@ Example response:
   "id": 1,
   "title": "Public key",
   "key": "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAIEAiPWx6WM4lhHNedGfBpPJNPpZ7yKu+dnn1SJejgt4596k6YjzGGphH2TUxwKzxcKDKKezwkpfnxPkSMkuEspGRt/aZZ9wa++Oi7Qkr8prgHc4soW6NUlfDzpvZK2H5E7eQaSeP3SAwGmQKUFHCddNaP0L+hM7zhFNzjFvpaMgJw0=",
+  "can_push": false,
   "created_at": "2013-10-02T10:12:29Z"
 }
 ```
@@ -112,9 +117,10 @@ POST /projects/:id/deploy_keys
 | `id`       | integer | yes | The ID of the project |
 | `title`    | string  | yes | New deploy key's title |
 | `key`      | string  | yes | New deploy key |
+| `can_push` | boolean | no  | Can deploy key push to the project's repository |
 
 ```bash
-curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --header "Content-Type: application/json" --data '{"title": "My deploy key", "key": "ssh-rsa AAAA..."}' "https://gitlab.example.com/api/v3/projects/5/deploy_keys/"
+curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --header "Content-Type: application/json" --data '{"title": "My deploy key", "key": "ssh-rsa AAAA...", "can_push": "true"}' "https://gitlab.example.com/api/v3/projects/5/deploy_keys/"
 ```
 
 Example response:
@@ -124,6 +130,7 @@ Example response:
    "key" : "ssh-rsa AAAA...",
    "id" : 12,
    "title" : "My deploy key",
+   "can_push": true,
    "created_at" : "2015-08-29T12:44:31.550Z"
 }
 ```

doc/api/groups.md

@@ -13,6 +13,7 @@ Parameters:
 | `search` | string | no | Return list of authorized groups matching the search criteria |
 | `order_by` | string | no | Order groups by `name` or `path`. Default is `name` |
 | `sort` | string | no | Order groups in `asc` or `desc` order. Default is `asc` |
+| `statistics` | boolean | no | Include group statistics (admins only) |
 
 ```
 GET /groups
@@ -31,7 +32,6 @@ GET /groups
 
 You can search for groups by name or path, see below.
 
-=======
 ## List owned groups
 
 Get a list of groups which are owned by the authenticated user.
@@ -40,6 +40,12 @@ Get a list of groups which are owned by the authenticated user.
 GET /groups/owned
 ```
 
+Parameters:
+
+| Attribute | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `statistics` | boolean | no | Include group statistics |
+
 ## List a group's projects
 
 Get a list of projects in this group.

doc/api/projects.md

@@ -307,6 +307,8 @@ Parameters:
 | `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` |
 | `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` |
 | `search` | string | no | Return list of authorized projects matching the search criteria |
+| `simple` | boolean | no | Return only the ID, URL, name, and path of each project |
+| `statistics` | boolean | no | Include project statistics |
 
 ### List starred projects
 
@@ -325,6 +327,7 @@ Parameters:
 | `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` |
 | `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` |
 | `search` | string | no | Return list of authorized projects matching the search criteria |
+| `simple` | boolean | no | Return only the ID, URL, name, and path of each project |
 
 ### List ALL projects
 
@@ -343,6 +346,7 @@ Parameters:
 | `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` |
 | `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` |
 | `search` | string | no | Return list of authorized projects matching the search criteria |
+| `statistics` | boolean | no | Include project statistics |
 
 ### Get single project
 

doc/ci/README.md

@@ -23,7 +23,7 @@
 - [CI/CD pipelines settings](../user/project/pipelines/settings.md)
 - [Review Apps](review_apps/index.md)
 - [Git submodules](git_submodules.md) Using Git submodules in your CI jobs
-- [Autodeploy](autodeploy/index.md)
+- [Auto deploy](autodeploy/index.md)
 
 ## Breaking changes
 

doc/ci/autodeploy/index.md

-# Autodeploy
+# Auto deploy
 
 > [Introduced][mr-8135] in GitLab 8.15.
 
-Autodeploy is an easy way to configure GitLab CI for the deployment of your
+Auto deploy is an easy way to configure GitLab CI for the deployment of your
 application. GitLab Community maintains a list of `.gitlab-ci.yml`
 templates for various infrastructure providers and deployment scripts
 powering them. These scripts are responsible for packaging your application,
@@ -15,7 +15,7 @@ deployment.
 
 ## Supported templates
 
-The list of supported autodeploy templates is available [here][autodeploy-templates].
+The list of supported auto deploy templates is available [here][auto-deploy-templates].
 
 ## Configuration
 
@@ -24,17 +24,17 @@ credentials. For example, if you want to deploy to OpenShift you have to
 enable [Kubernetes service][kubernetes-service].
 1. Configure GitLab Runner to use Docker or Kubernetes executor with
 [privileged mode enabled][docker-in-docker].
-1. Navigate to the "Project" tab and click "Set up autodeploy" button.
-   ![Autodeploy button](img/autodeploy_button.png)
+1. Navigate to the "Project" tab and click "Set up auto deploy" button.
+   ![Auto deploy button](img/auto_deploy_button.png)
 1. Select a template.
-  ![Dropdown with autodeploy templates](img/autodeploy_dropdown.png)
+  ![Dropdown with auto deploy templates](img/auto_deploy_dropdown.png)
 1. Commit your changes and create a merge request.
 1. Test your deployment configuration using a [Review App][review-app] that was
 created automatically for you.
 
 [mr-8135]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/8135
 [project-services]: ../../project_services/project_services.md
-[autodeploy-templates]: https://gitlab.com/gitlab-org/gitlab-ci-yml/tree/master/autodeploy
+[auto-deploy-templates]: https://gitlab.com/gitlab-org/gitlab-ci-yml/tree/master/autodeploy
 [kubernetes-service]: ../../project_services/kubernetes.md
 [docker-in-docker]: ../docker/using_docker_build.md#use-docker-in-docker-executor
 [review-app]: ../review_apps/index.md

doc/development/ux_guide/copy.md

@@ -101,3 +101,11 @@ The form should be titled `Edit issue`. The submit button should be labeled `Sav
 | Approve | Approve an open merge request ||
 | Remove approval, unapproved | Remove approval of an open merge request | Do not use `unapprove` as that is not an English word|
 | Merge | Merge an open merge request ||
+
+### Comments & Discussions
+
+#### Comment
+A **comment** is a written piece of text that users of GitLab can create. Comments have the meta data of author and time stamp. Comments can be added in a variety of contexts, such as issues, merge requests, and discussions.
+
+#### Dicussion
+A **discussion** is a group of 1 or more comments. A discussion can include sub discussions. Some discussions have the special capability of being able to be **resolved**. Both the comments in the discussion and the discussion itself can be resolved.
\ No newline at end of file

doc/workflow/lfs/lfs_administration.md

@@ -40,6 +40,12 @@ In `config/gitlab.yml`:
     storage_path: /mnt/storage/lfs-objects
 ```
 
+## Storage statistics
+
+You can see the total storage used for LFS objects on groups and projects
+in the administration area, as well as through the [groups](../api/groups.md)
+and [projects APIs](../api/projects.md).
+
 ## Known limitations
 
 * Currently, storing GitLab Git LFS objects on a non-local storage (like S3 buckets)
@@ -47,3 +53,5 @@ In `config/gitlab.yml`:
 * Currently, removing LFS objects from GitLab Git LFS storage is not supported
 * LFS authentications via SSH was added with GitLab 8.12
 * Only compatible with the GitLFS client versions 1.1.0 and up, or 1.0.2.
+* The storage statistics currently count each LFS object multiple times for
+  every project linking to it

features/admin/deploy_keys.feature

+@admin
+Feature: Admin Deploy Keys
+  Background:
+    Given I sign in as an admin
+    And there are public deploy keys in system
+
+  Scenario: Deploy Keys list
+    When I visit admin deploy keys page
+    Then I should see all public deploy keys
+
+  Scenario: Deploy Keys new
+    When I visit admin deploy keys page
+    And I click 'New Deploy Key'
+    And I submit new deploy key
+    Then I should be on admin deploy keys page
+    And I should see newly created deploy key without write access
+
+  Scenario: Deploy Keys new with write access
+    When I visit admin deploy keys page
+    And I click 'New Deploy Key'
+    And I submit new deploy key with write access
+    Then I should be on admin deploy keys page
+    And I should see newly created deploy key with write access

features/steps/admin/deploy_keys.rb

+class Spinach::Features::AdminDeployKeys < Spinach::FeatureSteps
+  include SharedAuthentication
+  include SharedPaths
+  include SharedAdmin
+
+  step 'there are public deploy keys in system' do
+    create(:deploy_key, public: true)
+    create(:another_deploy_key, public: true)
+  end
+
+  step 'I should see all public deploy keys' do
+    DeployKey.are_public.each do |p|
+      expect(page).to have_content p.title
+    end
+  end
+
+  step 'I visit admin deploy key page' do
+    visit admin_deploy_key_path(deploy_key)
+  end
+
+  step 'I visit admin deploy keys page' do
+    visit admin_deploy_keys_path
+  end
+
+  step 'I click \'New Deploy Key\'' do
+    click_link 'New Deploy Key'
+  end
+
+  step 'I submit new deploy key' do
+    fill_in "deploy_key_title", with: "laptop"
+    fill_in "deploy_key_key", with: "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAzrEJUIR6Y03TCE9rIJ+GqTBvgb8t1jI9h5UBzCLuK4VawOmkLornPqLDrGbm6tcwM/wBrrLvVOqi2HwmkKEIecVO0a64A4rIYScVsXIniHRS6w5twyn1MD3sIbN+socBDcaldECQa2u1dI3tnNVcs8wi77fiRe7RSxePsJceGoheRQgC8AZ510UdIlO+9rjIHUdVN7LLyz512auAfYsgx1OfablkQ/XJcdEwDNgi9imI6nAXhmoKUm1IPLT2yKajTIC64AjLOnE0YyCh6+7RFMpiMyu1qiOCpdjYwTgBRiciNRZCH8xIedyCoAmiUgkUT40XYHwLuwiPJICpkAzp7Q== user@laptop"
+    click_button "Create"
+  end
+
+  step 'I submit new deploy key with write access' do
+    fill_in "deploy_key_title", with: "server"
+    fill_in "deploy_key_key", with: "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAzrEJUIR6Y03TCE9rIJ+GqTBvgb8t1jI9h5UBzCLuK4VawOmkLornPqLDrGbm6tcwM/wBrrLvVOqi2HwmkKEIecVO0a64A4rIYScVsXIniHRS6w5twyn1MD3sIbN+socBDcaldECQa2u1dI3tnNVcs8wi77fiRe7RSxePsJceGoheRQgC8AZ510UdIlO+9rjIHUdVN7LLyz512auAfYsgx1OfablkQ/XJcdEwDNgi9imI6nAXhmoKUm1IPLT2yKajTIC64AjLOnE0YyCh6+7RFMpiMyu1qiOCpdjYwTgBRiciNRZCH8xIedyCoAmiUgkUT40XYHwLuwiPJICpkAzp7Q== user@laptop"
+    check "deploy_key_can_push"
+    click_button "Create"
+  end
+
+  step 'I should be on admin deploy keys page' do
+    expect(current_path).to eq admin_deploy_keys_path
+  end
+
+  step 'I should see newly created deploy key without write access' do
+    expect(page).to have_content(deploy_key.title)
+    expect(page).to have_content('No')
+  end
+
+  step 'I should see newly created deploy key with write access' do
+    expect(page).to have_content(deploy_key.title)
+    expect(page).to have_content('Yes')
+  end
+
+  def deploy_key
+    @deploy_key ||= DeployKey.are_public.first
+  end
+end

lib/api/entities.rb

@@ -78,21 +78,21 @@ module API
       expose :container_registry_enabled
 
       # Expose old field names with the new permissions methods to keep API compatible
-      expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:user]) }
-      expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:user]) }
-      expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:user]) }
-      expose(:builds_enabled) { |project, options| project.feature_available?(:builds, options[:user]) }
-      expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:user]) }
+      expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:current_user]) }
+      expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:current_user]) }
+      expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:current_user]) }
+      expose(:builds_enabled) { |project, options| project.feature_available?(:builds, options[:current_user]) }
+      expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:current_user]) }
 
       expose :created_at, :last_activity_at
       expose :shared_runners_enabled
       expose :lfs_enabled?, as: :lfs_enabled
       expose :creator_id
-      expose :namespace
+      expose :namespace, using: 'API::Entities::Namespace'
       expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? }
       expose :avatar_url
       expose :star_count, :forks_count
-      expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:user]) && project.default_issues_tracker? }
+      expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? }
       expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] }
       expose :public_builds
       expose :shared_with_groups do |project, options|
@@ -101,6 +101,16 @@ module API
       expose :only_allow_merge_if_build_succeeds
       expose :request_access_enabled
       expose :only_allow_merge_if_all_discussions_are_resolved
+
+      expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics
+    end
+
+    class ProjectStatistics < Grape::Entity
+      expose :commit_count
+      expose :storage_size
+      expose :repository_size
+      expose :lfs_objects_size
+      expose :build_artifacts_size
     end
 
     class Member < UserBasic
@@ -127,6 +137,15 @@ module API
       expose :avatar_url
       expose :web_url
       expose :request_access_enabled
+
+      expose :statistics, if: :statistics do
+        with_options format_with: -> (value) { value.to_i } do
+          expose :storage_size
+          expose :repository_size
+          expose :lfs_objects_size
+          expose :build_artifacts_size
+        end
+      end
     end
 
     class GroupDetail < Group
@@ -298,7 +317,7 @@ module API
     end
 
     class SSHKey < Grape::Entity
-      expose :id, :title, :key, :created_at
+      expose :id, :title, :key, :created_at, :can_push
     end
 
     class SSHKeyWithUser < SSHKey
@@ -391,7 +410,7 @@ module API
     end
 
     class Namespace < Grape::Entity
-      expose :id, :path, :kind
+      expose :id, :name, :path, :kind
     end
 
     class MemberAccess < Grape::Entity
@@ -440,12 +459,12 @@ module API
     class ProjectWithAccess < Project
       expose :permissions do
         expose :project_access, using: Entities::ProjectAccess do |project, options|
-          project.project_members.find_by(user_id: options[:user].id)
+          project.project_members.find_by(user_id: options[:current_user].id)
         end
 
         expose :group_access, using: Entities::GroupAccess do |project, options|
           if project.group
-            project.group.group_members.find_by(user_id: options[:user].id)
+            project.group.group_members.find_by(user_id: options[:current_user].id)
           end
         end
       end

lib/api/groups.rb

@@ -11,6 +11,20 @@ module API
         optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group'
         optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access'
       end
+
+      params :statistics_params do
+        optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
+      end
+
+      def present_groups(groups, options = {})
+        options = options.reverse_merge(
+          with: Entities::Group,
+          current_user: current_user,
+        )
+
+        groups = groups.with_statistics if options[:statistics]
+        present paginate(groups), options
+      end
     end
 
     resource :groups do
@@ -18,6 +32,7 @@ module API
         success Entities::Group
       end
       params do
+        use :statistics_params
         optional :skip_groups, type: Array[Integer], desc: 'Array of group ids to exclude from list'
         optional :all_available, type: Boolean, desc: 'Show all group that you have access to'
         optional :search, type: String, desc: 'Search for a specific group'
@@ -38,7 +53,7 @@ module API
         groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present?
         groups = groups.reorder(params[:order_by] => params[:sort])
 
-        present paginate(groups), with: Entities::Group
+        present_groups groups, statistics: params[:statistics] && current_user.is_admin?
       end
 
       desc 'Get list of owned groups for authenticated user' do
@@ -46,10 +61,10 @@ module API
       end
       params do
         use :pagination
+        use :statistics_params
       end
       get '/owned' do
-        groups = current_user.owned_groups
-        present paginate(groups), with: Entities::Group, user: current_user
+        present_groups current_user.owned_groups, statistics: params[:statistics]
       end
 
       desc 'Create a group. Available only for users who can create groups.' do
@@ -66,7 +81,7 @@ module API
         group = ::Groups::CreateService.new(current_user, declared_params(include_missing: false)).execute
 
         if group.persisted?
-          present group, with: Entities::Group
+          present group, with: Entities::Group, current_user: current_user
         else
           render_api_error!("Failed to save group #{group.errors.messages}", 400)
         end
@@ -92,7 +107,7 @@ module API
         authorize! :admin_group, group
 
         if ::Groups::UpdateService.new(group, current_user, declared_params(include_missing: false)).execute
-          present group, with: Entities::GroupDetail
+          present group, with: Entities::GroupDetail, current_user: current_user
         else
           render_validation_error!(group)
         end
@@ -103,7 +118,7 @@ module API
       end
       get ":id" do
         group = find_group!(params[:id])
-        present group, with: Entities::GroupDetail
+        present group, with: Entities::GroupDetail, current_user: current_user
       end
 
       desc 'Remove a group.'
@@ -134,7 +149,7 @@ module API
         projects = GroupProjectsFinder.new(group).execute(current_user)
         projects = filter_projects(projects)
         entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project
-        present paginate(projects), with: entity, user: current_user
+        present paginate(projects), with: entity, current_user: current_user
       end
 
       desc 'Transfer a project to the group namespace. Available only for admin.' do
@@ -150,7 +165,7 @@ module API
         result = ::Projects::TransferService.new(project, current_user).execute(group)
 
         if result
-          present group, with: Entities::GroupDetail
+          present group, with: Entities::GroupDetail, current_user: current_user
         else
           render_api_error!("Failed to transfer project #{project.errors.messages}", 400)
         end

lib/api/helpers.rb

@@ -248,7 +248,7 @@ module API
       rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500)
     end
 
-    # Projects helpers
+    # project helpers
 
     def filter_projects(projects)
       if params[:search].present?

lib/api/projects.rb

@@ -40,6 +40,15 @@ module API
 
     resource :projects do
       helpers do
+        params :collection_params do
+          use :sort_params
+          use :filter_params
+          use :pagination
+
+          optional :simple, type: Boolean, default: false,
+                            desc: 'Return only the ID, URL, name, and path of each project'
+        end
+
         params :sort_params do
           optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at],
                               default: 'created_at', desc: 'Return projects ordered by field'
@@ -52,97 +61,94 @@ module API
           optional :visibility, type: String, values: %w[public internal private],
                                 desc: 'Limit by visibility'
           optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria'
-          use :sort_params
+        end
+
+        params :statistics_params do
+          optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
         end
 
         params :create_params do
           optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.'
           optional :import_url, type: String, desc: 'URL from which the project is imported'
         end
+
+        def present_projects(projects, options = {})
+          options = options.reverse_merge(
+            with: Entities::Project,
+            current_user: current_user,
+            simple: params[:simple],
+          )
+
+          projects = filter_projects(projects)
+          projects = projects.with_statistics if options[:statistics]
+          options[:with] = Entities::BasicProjectDetails if options[:simple]
+
+          present paginate(projects), options
+        end
       end
 
       desc 'Get a list of visible projects for authenticated user' do
         success Entities::BasicProjectDetails
       end
       params do
-        optional :simple, type: Boolean, default: false,
-                          desc: 'Return only the ID, URL, name, and path of each project'
-        use :filter_params
-        use :pagination
+        use :collection_params
       end
       get '/visible' do
-        projects = ProjectsFinder.new.execute(current_user)
-        projects = filter_projects(projects)
-        entity = params[:simple] || !current_user ? Entities::BasicProjectDetails : Entities::ProjectWithAccess
-
-        present paginate(projects), with: entity, user: current_user
+        entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails
+        present_projects ProjectsFinder.new.execute(current_user), with: entity
       end
 
       desc 'Get a projects list for authenticated user' do
         success Entities::BasicProjectDetails
       end
       params do
-        optional :simple, type: Boolean, default: false,
-                          desc: 'Return only the ID, URL, name, and path of each project'
-        use :filter_params
-        use :pagination
+        use :collection_params
       end
       get do
         authenticate!
 
-        projects = current_user.authorized_projects
-        projects = filter_projects(projects)
-        entity = params[:simple] ? Entities::BasicProjectDetails : Entities::ProjectWithAccess
-
-        present paginate(projects), with: entity, user: current_user
+        present_projects current_user.authorized_projects,
+          with: Entities::ProjectWithAccess
       end
 
       desc 'Get an owned projects list for authenticated user' do
         success Entities::BasicProjectDetails
       end
       params do
-        use :filter_params
-        use :pagination
+        use :collection_params
+        use :statistics_params
       end
       get '/owned' do
         authenticate!
 
-        projects = current_user.owned_projects
-        projects = filter_projects(projects)
-
-        present paginate(projects), with: Entities::ProjectWithAccess, user: current_user
+        present_projects current_user.owned_projects,
+          with: Entities::ProjectWithAccess,
+          statistics: params[:statistics]
       end
 
       desc 'Gets starred project for the authenticated user' do
         success Entities::BasicProjectDetails
       end
       params do
-        use :filter_params
-        use :pagination
+        use :collection_params
       end
       get '/starred' do
         authenticate!
 
-        projects = current_user.viewable_starred_projects
-        projects = filter_projects(projects)
-
-        present paginate(projects), with: Entities::Project, user: current_user
+        present_projects current_user.viewable_starred_projects
       end
 
       desc 'Get all projects for admin user' do
         success Entities::BasicProjectDetails
       end
       params do
-        use :filter_params
-        use :pagination
+        use :collection_params
+        use :statistics_params
       end
       get '/all' do
         authenticated_as_admin!
 
-        projects = Project.all
-        projects = filter_projects(projects)
-
-        present paginate(projects), with: Entities::ProjectWithAccess, user: current_user
+        present_projects Project.all, with: Entities::ProjectWithAccess, statistics: params[:statistics]
       end
 
       desc 'Search for projects the current user has access to' do
@@ -221,7 +227,7 @@ module API
       end
       get ":id" do
         entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails
-        present user_project, with: entity, user: current_user,
+        present user_project, with: entity, current_user: current_user,
                               user_can_admin_project: can?(current_user, :admin_project, user_project)
       end
 

lib/gitlab/auth/result.rb

@@ -9,8 +9,7 @@ module Gitlab
 
       def lfs_deploy_token?(for_project)
         type == :lfs_deploy_token &&
-          actor &&
-          actor.projects.include?(for_project)
+          actor.try(:has_access_to?, for_project)
       end
 
       def success?

lib/gitlab/checks/change_access.rb

 module Gitlab
   module Checks
     class ChangeAccess
-      attr_reader :user_access, :project
+      attr_reader :user_access, :project, :skip_authorization
 
-      def initialize(change, user_access:, project:, env: {})
+      def initialize(
+        change, user_access:, project:, env: {}, skip_authorization: false)
         @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref)
         @branch_name = Gitlab::Git.branch_name(@ref)
         @user_access = user_access
         @project = project
         @env = env
+        @skip_authorization = skip_authorization
       end
 
       def exec
@@ -24,6 +26,7 @@ module Gitlab
       protected
 
       def protected_branch_checks
+        return if skip_authorization
         return unless @branch_name
         return unless project.protected_branch?(@branch_name)
 
@@ -49,6 +52,8 @@ module Gitlab
       end
 
       def tag_checks
+        return if skip_authorization
+
         tag_ref = Gitlab::Git.tag_name(@ref)
 
         if tag_ref && protected_tag?(tag_ref) && user_access.cannot_do_action?(:admin_project)
@@ -57,6 +62,8 @@ module Gitlab
       end
 
       def push_checks
+        return if skip_authorization
+
         if user_access.cannot_do_action?(:push_code)
           "You are not allowed to push code to this project."
         end

lib/gitlab/diff/file_collection/merge_request_diff.rb

@@ -61,7 +61,10 @@ module Gitlab
         end
 
         def cacheable?(diff_file)
-          @merge_request_diff.present? && diff_file.blob && diff_file.blob.text?
+          @merge_request_diff.present? &&
+            diff_file.blob &&
+            diff_file.blob.text? &&
+            @project.repository.diffable?(diff_file.blob)
         end
 
         def cache_key

lib/gitlab/email/reply_parser.rb

@@ -13,9 +13,17 @@ module Gitlab
 
         encoding = body.encoding
 
-        body = discourse_email_trimmer(body)
+        body = EmailReplyTrimmer.trim(body)
 
-        body = EmailReplyParser.parse_reply(body)
+        return '' unless body
+
+        # not using /\s+$/ here because that deletes empty lines
+        body = body.gsub(/[ \t]$/, '')
+
+        # NOTE: We currently don't support empty quotes.
+        # EmailReplyTrimmer allows this as a special case,
+        # so we detect it manually here.
+        return "" if body.lines.all? { |l| l.strip.empty? || l.start_with?('>') }
 
         body.force_encoding(encoding).encode("UTF-8")
       end
@@ -57,30 +65,6 @@ module Gitlab
       rescue
         nil
       end
-
-      REPLYING_HEADER_LABELS = %w(From Sent To Subject Reply To Cc Bcc Date)
-      REPLYING_HEADER_REGEX = Regexp.union(REPLYING_HEADER_LABELS.map { |label| "#{label}:" })
-
-      def discourse_email_trimmer(body)
-        lines = body.scrub.lines.to_a
-        range_end = 0
-
-        lines.each_with_index do |l, idx|
-          # This one might be controversial but so many reply lines have years, times and end with a colon.
-          # Let's try it and see how well it works.
-          break if (l =~ /\d{4}/ && l =~ /\d:\d\d/ && l =~ /\:$/) ||
-              (l =~ /On \w+ \d+,? \d+,?.*wrote:/)
-
-          # Headers on subsequent lines
-          break if (0..2).all? { |off| lines[idx + off] =~ REPLYING_HEADER_REGEX }
-          # Headers on the same line
-          break if REPLYING_HEADER_LABELS.count { |label| l.include?(label) } >= 3
-
-          range_end = idx
-        end
-
-        lines[0..range_end].join.strip
-      end
     end
   end
 end

lib/gitlab/git_access.rb

@@ -7,7 +7,8 @@ module Gitlab
     ERROR_MESSAGES = {
       upload: 'You are not allowed to upload code for this project.',
       download: 'You are not allowed to download code from this project.',
-      deploy_key: 'Deploy keys are not allowed to push code.',
+      deploy_key_upload:
+        'This deploy key does not have write access to this project.',
       no_repo: 'A repository for this project does not exist yet.'
     }
 
@@ -31,12 +32,13 @@ module Gitlab
       check_active_user!
       check_project_accessibility!
       check_command_existence!(cmd)
+      check_repository_existence!
 
       case cmd
       when *DOWNLOAD_COMMANDS
-        download_access_check
+        check_download_access!
       when *PUSH_COMMANDS
-        push_access_check(changes)
+        check_push_access!(changes)
       end
 
       build_status_object(true)
@@ -44,32 +46,10 @@ module Gitlab
       build_status_object(false, ex.message)
     end
 
-    def download_access_check
-      if user
-        user_download_access_check
-      elsif deploy_key.nil? && !guest_can_downlod_code?
-        raise UnauthorizedError, ERROR_MESSAGES[:download]
-      end
-    end
-
-    def push_access_check(changes)
-      if user
-        user_push_access_check(changes)
-      else
-        raise UnauthorizedError, ERROR_MESSAGES[deploy_key ? :deploy_key : :upload]
-      end
-    end
-
-    def guest_can_downlod_code?
+    def guest_can_download_code?
       Guest.can?(:download_code, project)
     end
 
-    def user_download_access_check
-      unless user_can_download_code? || build_can_download_code?
-        raise UnauthorizedError, ERROR_MESSAGES[:download]
-      end
-    end
-
     def user_can_download_code?
       authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_code)
     end
@@ -78,35 +58,6 @@ module Gitlab
       authentication_abilities.include?(:build_download_code) && user_access.can_do_action?(:build_download_code)
     end
 
-    def user_push_access_check(changes)
-      unless authentication_abilities.include?(:push_code)
-        raise UnauthorizedError, ERROR_MESSAGES[:upload]
-      end
-
-      if changes.blank?
-        return # Allow access.
-      end
-
-      unless project.repository.exists?
-        raise UnauthorizedError, ERROR_MESSAGES[:no_repo]
-      end
-
-      changes_list = Gitlab::ChangesList.new(changes)
-
-      # Iterate over all changes to find if user allowed all of them to be applied
-      changes_list.each do |change|
-        status = change_access_check(change)
-        unless status.allowed?
-          # If user does not have access to make at least one change - cancel all push
-          raise UnauthorizedError, status.message
-        end
-      end
-    end
-
-    def change_access_check(change)
-      Checks::ChangeAccess.new(change, user_access: user_access, project: project, env: @env).exec
-    end
-
     def protocol_allowed?
       Gitlab::ProtocolAccess.allowed?(protocol)
     end
@@ -120,6 +71,8 @@ module Gitlab
     end
 
     def check_active_user!
+      return if deploy_key?
+
       if user && !user_access.allowed?
         raise UnauthorizedError, "Your account has been blocked."
       end
@@ -137,31 +90,90 @@ module Gitlab
       end
     end
 
-    def matching_merge_request?(newrev, branch_name)
-      Checks::MatchingMergeRequest.new(newrev, branch_name, project).match?
+    def check_repository_existence!
+      unless project.repository.exists?
+        raise UnauthorizedError, ERROR_MESSAGES[:no_repo]
+      end
     end
 
-    def deploy_key
-      actor if actor.is_a?(DeployKey)
+    def check_download_access!
+      return if deploy_key?
+
+      passed = user_can_download_code? ||
+        build_can_download_code? ||
+        guest_can_download_code?
+
+      unless passed
+        raise UnauthorizedError, ERROR_MESSAGES[:download]
+      end
     end
 
-    def deploy_key_can_read_project?
+    def check_push_access!(changes)
       if deploy_key
-        return true if project.public?
-        deploy_key.projects.include?(project)
+        check_deploy_key_push_access!
+      elsif user
+        check_user_push_access!
       else
-        false
+        raise UnauthorizedError, ERROR_MESSAGES[:upload]
       end
+
+      return if changes.blank? # Allow access.
+
+      check_change_access!(changes)
     end
 
-    def can_read_project?
-      if user
-        user_access.can_read_project?
-      elsif deploy_key
-        deploy_key_can_read_project?
-      else
-        Guest.can?(:read_project, project)
+    def check_user_push_access!
+      unless authentication_abilities.include?(:push_code)
+        raise UnauthorizedError, ERROR_MESSAGES[:upload]
+      end
+    end
+
+    def check_deploy_key_push_access!
+      unless deploy_key.can_push_to?(project)
+        raise UnauthorizedError, ERROR_MESSAGES[:deploy_key_upload]
+      end
     end
+
+    def check_change_access!(changes)
+      changes_list = Gitlab::ChangesList.new(changes)
+
+      # Iterate over all changes to find if user allowed all of them to be applied
+      changes_list.each do |change|
+        status = check_single_change_access(change)
+        unless status.allowed?
+          # If user does not have access to make at least one change - cancel all push
+          raise UnauthorizedError, status.message
+        end
+      end
+    end
+
+    def check_single_change_access(change)
+      Checks::ChangeAccess.new(
+        change,
+        user_access: user_access,
+        project: project,
+        env: @env,
+        skip_authorization: deploy_key?).exec
+    end
+
+    def matching_merge_request?(newrev, branch_name)
+      Checks::MatchingMergeRequest.new(newrev, branch_name, project).match?
+    end
+
+    def deploy_key
+      actor if deploy_key?
+    end
+
+    def deploy_key?
+      actor.is_a?(DeployKey)
+    end
+
+    def can_read_project?
+      if deploy_key
+        deploy_key.has_access_to?(project)
+      elsif user
+        user.can?(:read_project, project)
+      end || Guest.can?(:read_project, project)
     end
 
     protected

lib/gitlab/git_access_wiki.rb

 module Gitlab
   class GitAccessWiki < GitAccess
-    def guest_can_downlod_code?
+    def guest_can_download_code?
       Guest.can?(:download_wiki_code, project)
     end
 
@@ -8,7 +8,7 @@ module Gitlab
       authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_wiki_code)
     end
 
-    def change_access_check(change)
+    def check_single_change_access(change)
       if user_access.can_do_action?(:create_wiki)
         build_status_object(true)
       else

lib/gitlab/template/gitlab_ci_yml_template.rb

@@ -15,7 +15,7 @@ module Gitlab
           {
             'General' => '',
             'Pages' => 'Pages',
-            'Autodeploy' => 'autodeploy'
+            'Auto deploy' => 'autodeploy'
           }
         end
 
@@ -28,7 +28,7 @@ module Gitlab
         end
 
         def dropdown_names(context)
-          categories = context == 'autodeploy' ? ['Autodeploy'] : ['General', 'Pages']
+          categories = context == 'autodeploy' ? ['Auto deploy'] : ['General', 'Pages']
           super().slice(*categories)
         end
       end

lib/gitlab/user_access.rb

@@ -8,6 +8,8 @@ module Gitlab
     end
 
     def can_do_action?(action)
+      return false if no_user_or_blocked?
+
       @permission_cache ||= {}
       @permission_cache[action] ||= user.can?(action, project)
     end
@@ -17,7 +19,7 @@ module Gitlab
     end
 
     def allowed?
-      return false if user.blank? || user.blocked?
+      return false if no_user_or_blocked?
 
       if user.requires_ldap_check? && user.try_obtain_ldap_lease
         return false unless Gitlab::LDAP::Access.allowed?(user)
@@ -27,7 +29,7 @@ module Gitlab
     end
 
     def can_push_to_branch?(ref)
-      return false unless user
+      return false if no_user_or_blocked?
 
       if project.protected_branch?(ref)
         return true if project.empty_repo? && project.user_can_push_to_empty_repo?(user)
@@ -40,7 +42,7 @@ module Gitlab
     end
 
     def can_merge_to_branch?(ref)
-      return false unless user
+      return false if no_user_or_blocked?
 
       if project.protected_branch?(ref)
         access_levels = project.protected_branches.matching(ref).map(&:merge_access_levels).flatten
@@ -51,9 +53,15 @@ module Gitlab
     end
 
     def can_read_project?
-      return false unless user
+      return false if no_user_or_blocked?
 
       user.can?(:read_project, project)
     end
+
+    private
+
+    def no_user_or_blocked?
+      user.nil? || user.blocked?
+    end
   end
 end

lib/tasks/gitlab/import.rake

@@ -63,8 +63,7 @@ namespace :gitlab do
 
             if project.persisted?
               puts " * Created #{project.name} (#{repo_path})".color(:green)
-              project.update_repository_size
-              project.update_commit_count
+              ProjectCacheWorker.perform(project.id)
             else
               puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red)
               puts "   Errors: #{project.errors.messages}".color(:red)

lib/tasks/gitlab/update_commit_count.rake

-namespace :gitlab do
-  desc "GitLab | Update commit count for projects"
-  task update_commit_count: :environment do
-    projects = Project.where(commit_count: 0)
-    puts "#{projects.size} projects need to be updated. This might take a while."
-    ask_to_continue unless ENV['force'] == 'yes'
-
-    projects.find_each(batch_size: 100) do |project|
-      print "#{project.name_with_namespace.color(:yellow)} ... "
-
-      unless project.repo_exists?
-        puts "skipping, because the repo is empty".color(:magenta)
-        next
-      end
-
-      project.update_commit_count
-      puts project.commit_count.to_s.color(:green)
-    end
-  end
-end