Merge branch 'ce-to-ee-2018-05-30' into 'master'

CE upstream - 2018-05-30 15:02 UTC

Closes #6045, #2678, and omnibus-gitlab#2138

See merge request gitlab-org/gitlab-ee!5906

.gitlab-ci.yml

@@ -6,7 +6,7 @@ image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.7-golang-1.9-git
     - gitlab-org
 
 .default-cache: &default-cache
-  key: "ruby-2.3.7-with-yarn"
+  key: "ruby-2.3.7-debian-stretch-with-yarn"
   paths:
     - vendor/ruby
     - .yarn-cache/
@@ -701,7 +701,7 @@ static-analysis:
   script:
     - scripts/static-analysis
   cache:
-    key: "ruby-2.3.7-with-yarn-and-rubocop"
+    key: "ruby-2.3.7-debian-stretch-with-yarn-and-rubocop"
     paths:
       - vendor/ruby
       - .yarn-cache/
@@ -742,7 +742,7 @@ ee_compat_check:
   except:
     - master
     - tags
-    - /^[\d-]+-stable(-ee)?/
+    - /[\d-]+-stable(-ee)?/
     - /^security-/
     - branches@gitlab-org/gitlab-ee
     - branches@gitlab/gitlab-ee

Gemfile

@@ -143,7 +143,7 @@ gem 'gitlab-markup', '~> 1.6.2'
 gem 'redcarpet', '~> 3.4'
 gem 'commonmarker', '~> 0.17'
 gem 'RedCloth', '~> 4.3.2'
-gem 'rdoc', '~> 4.2'
+gem 'rdoc', '~> 6.0'
 gem 'org-ruby', '~> 0.9.12'
 gem 'creole', '~> 0.5.0'
 gem 'wikicloth', '0.8.1'
@@ -229,7 +229,7 @@ gem 'asana', '~> 0.6.0'
 gem 'ruby-fogbugz', '~> 0.2.1'
 
 # Kubernetes integration
-gem 'kubeclient', '~> 3.0'
+gem 'kubeclient', '~> 3.1.0'
 
 # Sanitize user input
 gem 'sanitize', '~> 2.0'
@@ -332,7 +332,7 @@ group :development, :test do
   gem 'pry-byebug', '~> 3.4.1', platform: :mri
   gem 'pry-rails', '~> 0.3.4'
 
-  gem 'awesome_print', '~> 1.2.0', require: false
+  gem 'awesome_print', require: false
   gem 'fuubar', '~> 2.2.0'
 
   gem 'database_cleaner', '~> 1.5.0'

Gemfile.lock

@@ -69,7 +69,7 @@ GEM
     attr_encrypted (3.1.0)
       encryptor (~> 3.0.0)
     attr_required (1.0.0)
-    awesome_print (1.2.0)
+    awesome_print (1.8.0)
     aws-sdk (2.9.32)
       aws-sdk-resources (= 2.9.32)
     aws-sdk-core (2.9.32)
@@ -176,7 +176,7 @@ GEM
     diff-lcs (1.3)
     diffy (3.1.0)
     docile (1.1.5)
-    domain_name (0.5.20170404)
+    domain_name (0.5.20180417)
       unf (>= 0.0.5, < 1.0.0)
     doorkeeper (4.3.2)
       railties (>= 4.2)
@@ -474,9 +474,9 @@ GEM
     knapsack (1.16.0)
       rake
       timecop (>= 0.1.0)
-    kubeclient (3.0.0)
+    kubeclient (3.1.0)
       http (~> 2.2.2)
-      recursive-open-struct (~> 1.0.4)
+      recursive-open-struct (~> 1.0, >= 1.0.4)
       rest-client (~> 2.0)
     launchy (2.4.3)
       addressable (~> 2.3)
@@ -723,12 +723,11 @@ GEM
       ffi
     rbnacl-libsodium (1.0.11)
       rbnacl (>= 3.0.1)
-    rdoc (4.2.2)
-      json (~> 1.4)
+    rdoc (6.0.4)
     re2 (1.1.1)
     recaptcha (3.0.0)
       json
-    recursive-open-struct (1.0.5)
+    recursive-open-struct (1.1.0)
     redcarpet (3.4.0)
     redis (3.3.5)
     redis-actionpack (5.0.2)
@@ -830,7 +829,7 @@ GEM
     rubyzip (1.2.1)
     rufus-scheduler (3.4.0)
       et-orbi (~> 1.0)
-    rugged (0.27.0)
+    rugged (0.27.1)
     safe_yaml (1.0.4)
     sanitize (2.1.0)
       nokogiri (>= 1.4.4)
@@ -1007,7 +1006,7 @@ DEPENDENCIES
   asciidoctor-plantuml (= 0.0.8)
   asset_sync (~> 2.4)
   attr_encrypted (~> 3.1.0)
-  awesome_print (~> 1.2.0)
+  awesome_print
   aws-sdk
   babosa (~> 1.0.2)
   base32 (~> 0.3.0)
@@ -1104,7 +1103,7 @@ DEPENDENCIES
   jwt (~> 1.5.6)
   kaminari (~> 1.0)
   knapsack (~> 1.16)
-  kubeclient (~> 3.0)
+  kubeclient (~> 3.1.0)
   letter_opener_web (~> 1.3.0)
   license_finder (~> 3.1)
   licensee (~> 8.9)
@@ -1161,7 +1160,7 @@ DEPENDENCIES
   rblineprof (~> 0.3.6)
   rbnacl (~> 4.0)
   rbnacl-libsodium
-  rdoc (~> 4.2)
+  rdoc (~> 6.0)
   re2 (~> 1.1.1)
   recaptcha (~> 3.0)
   redcarpet (~> 3.4)

app/assets/javascripts/ide/components/commit_sidebar/form.vue

@@ -144,14 +144,14 @@ export default {
           <loading-button
             :loading="submitCommitLoading"
             :disabled="commitButtonDisabled"
-            container-class="btn btn-success btn-sm pull-left"
+            container-class="btn btn-success btn-sm float-left"
             :label="__('Commit')"
             @click="commitChanges"
           />
           <button
             v-if="!discardDraftButtonDisabled"
             type="button"
-            class="btn btn-default btn-sm pull-right"
+            class="btn btn-default btn-sm float-right"
             @click="discardDraft"
           >
             {{ __('Discard draft') }}
@@ -159,7 +159,7 @@ export default {
           <button
             v-else
             type="button"
-            class="btn btn-default btn-sm pull-right"
+            class="btn btn-default btn-sm float-right"
             @click="toggleIsSmall"
           >
             {{ __('Collapse') }}

app/assets/javascripts/ide/components/new_dropdown/modal.vue

@@ -73,10 +73,9 @@ export default {
     <form
       slot="body"
       @submit.prevent="createEntryInStore"
-      class="form-group row append-bottom-0"
+      class="form-group row"
     >
-      <fieldset class="form-group append-bottom-0">
-        <label class="label-light col-form-label col-sm-3 ide-new-modal-label">
+      <label class="label-light col-form-label col-sm-3">
         {{ __('Name') }}
       </label>
       <div class="col-sm-9">
@@ -87,7 +86,6 @@ export default {
           ref="fieldName"
         />
       </div>
-      </fieldset>
     </form>
   </deprecated-modal>
 </template>

app/assets/javascripts/issuable_form.js

@@ -32,7 +32,7 @@ export default class IssuableForm {
     }
 
     this.initAutosave();
-    this.form.on('submit:success', this.handleSubmit);
+    this.form.on('submit', this.handleSubmit);
     this.form.on('click', '.btn-cancel', this.resetAutosave);
     this.initWip();
 

app/assets/javascripts/merge_request_tabs.js

@@ -362,7 +362,7 @@ export default class MergeRequestTabs {
   //
   // status - Boolean, true to show, false to hide
   toggleLoading(status) {
-    $('.mr-loading-status .loading').toggleClass('hidden', status);
+    $('.mr-loading-status .loading').toggleClass('hidden', !status);
   }
 
   diffViewType() {

app/assets/javascripts/pages/users/user_tabs.js

@@ -77,8 +77,7 @@ export default class UserTabs {
     this.action = action || this.defaultAction;
     this.$parentEl = $(parentEl) || $(document);
     this.windowLocation = window.location;
-    this.$parentEl.find('.nav-links a')
-      .each((i, navLink) => {
+    this.$parentEl.find('.nav-links a').each((i, navLink) => {
       this.loaded[$(navLink).attr('data-action')] = false;
     });
     this.actions = Object.keys(this.loaded);
@@ -116,8 +115,7 @@ export default class UserTabs {
   }
 
   activateTab(action) {
-    return this.$parentEl.find(`.nav-links .js-${action}-tab a`)
-      .tab('show');
+    return this.$parentEl.find(`.nav-links .js-${action}-tab a`).tab('show');
   }
 
   setTab(action, endpoint) {
@@ -137,7 +135,8 @@ export default class UserTabs {
   loadTab(action, endpoint) {
     this.toggleLoading(true);
 
-    return axios.get(endpoint)
+    return axios
+      .get(endpoint)
       .then(({ data }) => {
         const tabSelector = `div#${action}`;
         this.$parentEl.find(tabSelector).html(data.html);
@@ -161,10 +160,11 @@ export default class UserTabs {
     const utcOffset = $calendarWrap.data('utcOffset');
     let utcFormatted = 'UTC';
     if (utcOffset !== 0) {
-      utcFormatted = `UTC${utcOffset > 0 ? '+' : ''}${(utcOffset / 3600)}`;
+      utcFormatted = `UTC${utcOffset > 0 ? '+' : ''}${utcOffset / 3600}`;
     }
 
-    axios.get(calendarPath)
+    axios
+      .get(calendarPath)
       .then(({ data }) => {
         $calendarWrap.html(CALENDAR_TEMPLATE);
         $calendarWrap.find('.calendar-hint').append(`(Timezone: ${utcFormatted})`);
@@ -180,17 +180,20 @@ export default class UserTabs {
   }
 
   toggleLoading(status) {
-    return this.$parentEl.find('.loading-status .loading')
-      .toggleClass('hidden', status);
+    return this.$parentEl.find('.loading-status .loading').toggleClass('hidden', !status);
   }
 
   setCurrentAction(source) {
     let newState = source;
     newState = newState.replace(/\/+$/, '');
     newState += this.windowLocation.search + this.windowLocation.hash;
-    history.replaceState({
+    history.replaceState(
+      {
         url: newState,
-    }, document.title, newState);
+      },
+      document.title,
+      newState,
+    );
     return newState;
   }
 

app/assets/javascripts/projects/gke_cluster_dropdowns/components/gke_machine_type_dropdown.vue

@@ -132,7 +132,7 @@ export default {
       </div>
     </div>
     <span
-      class="help-block"
+      class="form-text text-muted"
       :class="{ 'gl-field-error': hasErrors }"
       v-if="hasErrors"
     >

app/assets/javascripts/projects/gke_cluster_dropdowns/components/gke_project_id_dropdown.vue

@@ -193,7 +193,7 @@ export default {
       </div>
     </div>
     <span
-      class="help-block"
+      class="form-text text-muted"
       :class="{ 'gl-field-error': hasErrors }"
       v-html="helpText"
     ></span>

app/assets/javascripts/projects/gke_cluster_dropdowns/components/gke_zone_dropdown.vue

@@ -106,7 +106,7 @@ export default {
       </div>
     </div>
     <span
-      class="help-block"
+      class="form-text text-muted"
       :class="{ 'gl-field-error': hasErrors }"
       v-if="hasErrors"
     >

app/assets/stylesheets/bootstrap_migration.scss

@@ -131,6 +131,10 @@ table {
 }
 
 .card {
+  .card-title {
+    margin-bottom: 0;
+  }
+
   &.card-without-border {
     @extend .border-0;
   }
@@ -147,3 +151,7 @@ table {
 .nav-tabs .nav-link {
   border: 0;
 }
+
+pre code {
+  white-space: pre-wrap;
+}

app/assets/stylesheets/pages/repo.scss

@@ -1088,10 +1088,6 @@
   font-size: 12px;
 }
 
-.ide-new-modal-label {
-  line-height: 34px;
-}
-
 .multi-file-commit-panel-success-message {
   position: absolute;
   top: 61px;

app/controllers/groups/shared_projects_controller.rb

+module Groups
+  class SharedProjectsController < Groups::ApplicationController
+    respond_to :json
+    before_action :group
+    skip_cross_project_access_check :index
+
+    def index
+      shared_projects = GroupProjectsFinder.new(
+        group: group,
+        current_user: current_user,
+        params: finder_params,
+        options: { only_shared: true }
+      ).execute
+      serializer = GroupChildSerializer.new(current_user: current_user)
+                     .with_pagination(request, response)
+
+      render json: serializer.represent(shared_projects)
+    end
+
+    private
+
+    def finder_params
+      @finder_params ||= begin
+                           # Make the `search` param consistent for the frontend,
+                           # which will be using `filter`.
+                           params[:search] ||= params[:filter] if params[:filter]
+                           params.permit(:sort, :search)
+                         end
+    end
+  end
+end

app/controllers/profiles_controller.rb

@@ -93,8 +93,6 @@ class ProfilesController < Profiles::ApplicationController
       :linkedin,
       :location,
       :name,
-      :password,
-      :password_confirmation,
       :public_email,
       :skype,
       :twitter,

app/models/concerns/batch_destroy_dependent_associations.rb

+# Provides a way to work around Rails issue where dependent objects are all
+# loaded into memory before destroyed: https://github.com/rails/rails/issues/22510.
+#
+# This concern allows an ActiveRecord module to destroy all its dependent
+# associations in batches. The idea is borrowed from https://github.com/thisismydesign/batch_dependent_associations.
+#
+# The differences here with that gem:
+#
+# 1. We allow excluding certain associations.
+# 2. We don't need to support delete_all since we can use the EachBatch concern.
+module BatchDestroyDependentAssociations
+  extend ActiveSupport::Concern
+
+  DEPENDENT_ASSOCIATIONS_BATCH_SIZE = 1000
+
+  def dependent_associations_to_destroy
+    self.class.reflect_on_all_associations(:has_many).select { |assoc| assoc.options[:dependent] == :destroy }
+  end
+
+  def destroy_dependent_associations_in_batches(exclude: [])
+    dependent_associations_to_destroy.each do |association|
+      next if exclude.include?(association.name)
+
+      # rubocop:disable GitlabSecurity/PublicSend
+      public_send(association.name).find_each(batch_size: DEPENDENT_ASSOCIATIONS_BATCH_SIZE, &:destroy)
+    end
+  end
+end

app/models/event.rb

@@ -40,6 +40,7 @@ class Event < ActiveRecord::Base
   ).freeze
 
   RESET_PROJECT_ACTIVITY_INTERVAL = 1.hour
+  REPOSITORY_UPDATED_AT_INTERVAL = 5.minutes
 
   delegate :name, :email, :public_email, :username, to: :author, prefix: true, allow_nil: true
   delegate :title, to: :issue, prefix: true, allow_nil: true
@@ -397,6 +398,7 @@ class Event < ActiveRecord::Base
 
   def set_last_repository_updated_at
     Project.unscoped.where(id: project_id)
+      .where("last_repository_updated_at < ? OR last_repository_updated_at IS NULL", REPOSITORY_UPDATED_AT_INTERVAL.ago)
       .update_all(last_repository_updated_at: created_at)
   end
 

app/models/project.rb

@@ -24,6 +24,7 @@ class Project < ActiveRecord::Base
   include ChronicDurationAttribute
   include FastDestroyAll::Helpers
   include WithUploads
+  include BatchDestroyDependentAssociations
 
   # EE specific modules
   prepend EE::Project

app/models/service.rb

@@ -207,10 +207,11 @@ class Service < ActiveRecord::Base
     args.each do |arg|
       class_eval %{
         def #{arg}?
+          # '!!' is used because nil or empty string is converted to nil
           if Gitlab.rails5?
-            !ActiveModel::Type::Boolean::FALSE_VALUES.include?(#{arg})
+            !!ActiveRecord::Type::Boolean.new.cast(#{arg})
           else
-            ActiveRecord::ConnectionAdapters::Column::TRUE_VALUES.include?(#{arg})
+            !!ActiveRecord::Type::Boolean.new.type_cast_from_database(#{arg})
           end
         end
       }

app/serializers/group_child_entity.rb

@@ -31,7 +31,7 @@ class GroupChildEntity < Grape::Entity
   end
 
   # Project only attributes
-  expose :star_count,
+  expose :star_count, :archived,
          if: lambda { |_instance, _options| project? }
 
   # Group only attributes

app/services/projects/destroy_service.rb

@@ -138,7 +138,13 @@ module Projects
 
         trash_repositories!
 
-        project.team.truncate
+        # Rails attempts to load all related records into memory before
+        # destroying: https://github.com/rails/rails/issues/22510
+        # This ensures we delete records in batches.
+        #
+        # Exclude container repositories because its before_destroy would be
+        # called multiple times, and it doesn't destroy any database records.
+        project.destroy_dependent_associations_in_batches(exclude: [:container_repositories])
         project.destroy!
       end
     end

app/uploaders/object_storage.rb

@@ -11,6 +11,7 @@ module ObjectStorage
   ObjectStorageUnavailable = Class.new(StandardError)
 
   DIRECT_UPLOAD_TIMEOUT = 4.hours
+  DIRECT_UPLOAD_EXPIRE_OFFSET = 15.minutes
   TMP_UPLOAD_PATH = 'tmp/uploads'.freeze
 
   module Store
@@ -174,11 +175,12 @@ module ObjectStorage
         id = [CarrierWave.generate_cache_id, SecureRandom.hex].join('-')
         upload_path = File.join(TMP_UPLOAD_PATH, id)
         connection = ::Fog::Storage.new(self.object_store_credentials)
-        expire_at = Time.now + DIRECT_UPLOAD_TIMEOUT
+        expire_at = Time.now + DIRECT_UPLOAD_TIMEOUT + DIRECT_UPLOAD_EXPIRE_OFFSET
         options = { 'Content-Type' => 'application/octet-stream' }
 
         {
           ID: id,
+          Timeout: DIRECT_UPLOAD_TIMEOUT,
           GetURL: connection.get_object_url(remote_store_path, upload_path, expire_at),
           DeleteURL: connection.delete_object_url(remote_store_path, upload_path, expire_at),
           StoreURL: connection.put_object_url(remote_store_path, upload_path, expire_at, options)

app/views/groups/runners/_runner.html.haml

@@ -8,13 +8,13 @@
       = link_to edit_group_runner_path(@group, runner) do
         = icon('edit')
 
-    .pull-right
+    .float-right
       - if runner.active?
         = link_to _('Pause'), pause_group_runner_path(@group, runner), method: :post, class: 'btn btn-sm btn-danger', data: { confirm: _("Are you sure?") }
       - else
         = link_to _('Resume'), resume_group_runner_path(@group, runner), method: :post, class: 'btn btn-success btn-sm'
       = link_to _('Remove Runner'), group_runner_path(@group, runner), data: { confirm: _("Are you sure?") }, method: :delete, class: 'btn btn-danger btn-sm'
-  .pull-right
+  .float-right
     %small.light
       \##{runner.id}
   - if runner.description.present?

app/views/profiles/active_sessions/_active_session.html.haml

 - is_current_session = active_session.current?(session)
 
 %li.list-group-item
-  .pull-left.append-right-10{ data: { toggle: 'tooltip' }, title: active_session.human_device_type }
+  .float-left.append-right-10{ data: { toggle: 'tooltip' }, title: active_session.human_device_type }
     = active_session_device_type_icon(active_session)
 
-  .description.pull-left
+  .description.float-left
     %div
       %strong= active_session.ip_address
     - if is_current_session
@@ -25,7 +25,7 @@
       = l(active_session.created_at, format: :short)
 
   - unless is_current_session
-    .pull-right
+    .float-right
       = link_to profile_active_session_path(active_session.session_id), data: { confirm: 'Are you sure? The device will be signed out of GitLab.' }, method: :delete, class: "btn btn-danger prepend-left-10" do
         %span.sr-only Revoke
         Revoke

app/views/projects/branches/_branch.html.haml

@@ -33,7 +33,7 @@
         = s_('Branches|Cant find HEAD commit for this branch')
 
   - if branch.name != @repository.root_ref
-    .divergence-graph.d-none.d-sm-block{ title: s_('%{number_commits_behind} commits behind %{default_branch}, %{number_commits_ahead} commits ahead') % { number_commits_behind: diverging_count_label(number_commits_behind),
+    .divergence-graph.d-none.d-md-block{ title: s_('%{number_commits_behind} commits behind %{default_branch}, %{number_commits_ahead} commits ahead') % { number_commits_behind: diverging_count_label(number_commits_behind),
       default_branch: @repository.root_ref,
       number_commits_ahead: diverging_count_label(number_commits_ahead) } }
       .graph-side
@@ -44,7 +44,7 @@
         .bar.bar-ahead{ style: "width: #{number_commits_ahead * bar_graph_width_factor}%" }
         %span.count.count-ahead= diverging_count_label(number_commits_ahead)
 
-  .controls.d-none.d-sm-block<
+  .controls.d-none.d-md-block<
     - if merge_project && create_mr_button?(@repository.root_ref, branch.name)
       = link_to create_mr_path(@repository.root_ref, branch.name), class: 'btn btn-default' do
         = _('Merge request')

app/views/projects/clusters/gcp/_form.html.haml

@@ -23,7 +23,7 @@
             %span.dropdown-toggle-text
               = _('Select project')
             = icon('chevron-down')
-        %span.help-block  
+        %span.form-text.text-muted  
 
     .form-group
       = provider_gcp_field.label :zone, s_('ClusterIntegration|Zone')

app/views/projects/mattermosts/_team_selection.html.haml

@@ -13,9 +13,9 @@
   = f.hidden_field(:team_id, value: selected_id, required: true) if @teams.one?
   .form-text.text-muted
     - if @teams.one?
-      This is the only available team.
+      This is the only available team that you are a member of.
     - else
-      The list shows all available teams.
+      The list shows all available teams that you are a member of.
     To create a team,
     = link_to "#{Gitlab.config.mattermost.host}/create_team" do
       use Mattermost's interface

app/views/shared/snippets/_snippet.html.haml

@@ -28,7 +28,7 @@
     = link_to user_snippets_path(snippet.author) do
       = snippet.author_name
     - if link_project && snippet.project_id?
-      %span.d-none.d-sm-block
+      %span.d-none.d-sm-inline-block
         in
         = link_to project_path(snippet.project) do
           = snippet.project.full_name

app/views/users/terms/index.html.haml

@@ -4,10 +4,10 @@
   = markdown_field(@term, :terms)
 .row-content-block.footer-block.clearfix
   - if can?(current_user, :accept_terms, @term)
-    .pull-right
+    .float-right
       = button_to accept_term_path(@term, redirect_params), class: 'btn btn-success prepend-left-8' do
         = _('Accept terms')
   - if can?(current_user, :decline_terms, @term)
-    .pull-right
+    .float-right
       = button_to decline_term_path(@term, redirect_params), class: 'btn btn-default prepend-left-8' do
         =  _('Decline and sign out')

changelogs/unreleased/46844-update-awesome_print-to-1-8-0.yml

+---
+title: Update awesome_print to 1.8.0
+merge_request: 19163
+author: Takuya Noguchi
+type: other

changelogs/unreleased/46849-update-rdoc-to-6-0-4.yml

+---
+title: Update rdoc to 6.0.4
+merge_request: 19167
+author: Takuya Noguchi
+type: other

changelogs/unreleased/ab-35364-throttle-updates-last-repository-at.yml

+---
+title: Throttle updates to Project#last_repository_updated_at.
+merge_request: 19183
+author:
+type: performance

changelogs/unreleased/bump-kubeclient-version-3-1-0.yml

+---
+title: Updates the version of kubeclient from 3.0 to 3.1.0
+merge_request: 19199
+author:
+type: other

changelogs/unreleased/dm-api-projects-members-preload.yml

+---
+title: Only preload member records for the relevant projects/groups/user in projects
+  API
+merge_request:
+author:
+type: performance

changelogs/unreleased/fix-bitbucket_import_anonymous.yml

+---
+title: Import bitbucket issues that are reported by an anonymous user
+merge_request: 18199
+author: bartl
+type: fixed

changelogs/unreleased/fix-missing-timeout.yml

+---
+title: Missing timeout value in object storage pre-authorization
+merge_request: 19201
+author:
+type: fixed

changelogs/unreleased/mattermost-api-v4.yml

+---
+title: Updated Mattermost integration to use API v4 and only allow creation of Mattermost slash commands in the current user's teams
+merge_request: 19043
+author: Harrison Healey
+type: changed

changelogs/unreleased/security-dm-delete-deploy-key.yml

+---
+title: Fix API to remove deploy key from project instead of deleting it entirely
+merge_request:
+author:
+type: security

changelogs/unreleased/security-fj-import-export-assignment.yml

+---
+title: Fixed bug that allowed importing arbitrary project attributes
+merge_request:
+author:
+type: security

changelogs/unreleased/security-users-can-update-their-password-without-entering-current-password.yml

+---
+title: Prevent user passwords from being changed without providing the previous password
+merge_request:
+author:
+type: security

changelogs/unreleased/sh-batch-dependent-destroys.yml

+---
+title: Fix project destruction failing due to idle in transaction timeouts
+merge_request:
+author:
+type: fixed

changelogs/unreleased/sh-fix-submit-not-clearing-local-storage.yml

+---
+title: Fix local storage not being cleared after creating a new issue
+merge_request:
+author:
+type: fixed

config/routes/group.rb

@@ -31,6 +31,7 @@ constraints(::Constraints::GroupUrlConstrainer.new) do
     resource :variables, only: [:show, :update]
 
     resources :children, only: [:index]
+    resources :shared_projects, only: [:index]
 
     resources :labels, except: [:show] do
       post :toggle_subscription, on: :member

doc/administration/custom_hooks.md

@@ -3,7 +3,7 @@
 >
 **Note:** Custom Git hooks must be configured on the filesystem of the GitLab
 server. Only GitLab server administrators will be able to complete these tasks.
-Please explore [webhooks] as an option if you do not
+Please explore [webhooks] and [CI] as an option if you do not
 have filesystem access. For a user configurable Git hook interface, see
 [Push Rules](https://docs.gitlab.com/ee/push_rules/push_rules.html),
 available in GitLab Enterprise Edition.
@@ -83,6 +83,7 @@ STDERR takes precedence over STDOUT.
 
 ![Custom message from custom Git hook](img/custom_hooks_error_msg.png)
 
+[CI]: ../ci/README.md
 [hooks]: https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks#Server-Side-Hooks
 [webhooks]: ../user/project/integrations/webhooks.md
 [gitlab-geo]: ../administration/geo/replication/index.md

doc/administration/high_availability/gitlab.md

@@ -47,7 +47,8 @@ for each GitLab application server in your environment.
    URL. Depending your the NFS configuration, you may need to change some GitLab
    data locations. See [NFS documentation](nfs.md) for `/etc/gitlab/gitlab.rb`
    configuration values for various scenarios. The example below assumes you've
-   added NFS mounts in the default data locations.
+   added NFS mounts in the default data locations. Additionally the UID and GIDs
+   given are just examples and you should configure with your preferred values.
 
     ```ruby
     external_url 'https://gitlab.example.com'
@@ -69,6 +70,14 @@ for each GitLab application server in your environment.
     gitlab_rails['redis_port'] = '6379'
     gitlab_rails['redis_host'] = '10.1.0.6' # IP/hostname of Redis server
     gitlab_rails['redis_password'] = 'Redis Password'
+    
+    # Ensure UIDs and GIDs match between servers for permissions via NFS
+    user['uid'] = 9000
+    user['gid'] = 9000
+    web_server['uid'] = 9001
+    web_server['gid'] = 9001
+    registry['uid'] = 9002
+    registry['gid'] = 9002
     ```
 
     > **Note:** To maintain uniformity of links across HA clusters, the `external_url`

doc/administration/high_availability/nfs.md

@@ -25,7 +25,9 @@ options:
   errors when the Omnibus package tries to alter permissions. Note that GitLab
   and other bundled components do **not** run as `root` but as non-privileged
   users. The recommendation for `no_root_squash` is to allow the Omnibus package
-  to set ownership and permissions on files, as needed.
+  to set ownership and permissions on files, as needed. In some cases where the
+  `no_root_squash` option is not available, the `root` flag can achieve the same
+  result.
 - `sync` - Force synchronous behavior. Default is asynchronous and under certain
   circumstances it could lead to data loss if a failure occurs before data has
   synced.

doc/administration/raketasks/storage.md

@@ -24,7 +24,6 @@ gitlab-rake gitlab:storage:migrate_to_hashed
 
 ```bash
 rake gitlab:storage:migrate_to_hashed
-
 ```
 
 You can monitor the progress in the _Admin > Monitoring > Background jobs_ screen.
@@ -52,7 +51,6 @@ gitlab-rake gitlab:storage:legacy_projects
 
 ```bash
 rake gitlab:storage:legacy_projects
-
 ```
 
 ------
@@ -86,7 +84,6 @@ gitlab-rake gitlab:storage:hashed_projects
 
 ```bash
 rake gitlab:storage:hashed_projects
-
 ```
 
 ------
@@ -120,7 +117,6 @@ gitlab-rake gitlab:storage:legacy_attachments
 
 ```bash
 rake gitlab:storage:legacy_attachments
-
 ```
 
 ------
@@ -137,7 +133,6 @@ gitlab-rake gitlab:storage:list_legacy_attachments
 
 ```bash
 rake gitlab:storage:list_legacy_attachments
-
 ```
 
 ## List attachments on Hashed storage
@@ -154,7 +149,6 @@ gitlab-rake gitlab:storage:hashed_attachments
 
 ```bash
 rake gitlab:storage:hashed_attachments
-
 ```
 
 ------
@@ -171,7 +165,6 @@ gitlab-rake gitlab:storage:list_hashed_attachments
 
 ```bash
 rake gitlab:storage:list_hashed_attachments
-
 ```
 
 [storage-types]: ../repository_storage_types.md

doc/development/code_review.md

@@ -22,7 +22,7 @@ There are a few rules to get your merge request accepted:
   1. If your merge request includes UX, frontend and backend changes [^1], it must
     be **approved by a [UX team member, a frontend and a backend maintainer][team]**.
   1. If your merge request includes a new dependency or a filesystem change, it must
-    be **approved by a [Build team member][team]**. See [how to work with the Build team][build handbook] for more details.
+    be *approved by a [Distribution team member][team]*. See how to work with the [Distribution team for more details.](https://about.gitlab.com/handbook/engineering/dev-backend/distribution/)
 1. To lower the amount of merge requests maintainers need to review, you can
   ask or assign any [reviewers][projects] for a first review.
   1. If you need some guidance (e.g. it's your first merge request), feel free

doc/development/ee_features.md

@@ -368,27 +368,17 @@ resolve when you add the indentation to the equation.
 EE-specific views should be placed in `ee/app/views/`, using extra
 sub-directories if appropriate.
 
+#### Using `render_if_exists`
+
 Instead of using regular `render`, we should use `render_if_exists`, which
 will not render anything if it cannot find the specific partial. We use this
 so that we could put `render_if_exists` in CE, keeping code the same between
 CE and EE.
 
-Also, it should search for the EE partial first, and then CE partial, and
-then if nothing found, render nothing.
-
-This has two uses:
-
-- CE renders nothing, and EE renders its EE partial.
-- CE renders its CE partial, and EE renders its EE partial, while the view
-  file stays the same.
-
 The advantages of this:
 
 - Minimal code difference between CE and EE.
 - Very clear hints about where we're extending EE views while reading CE codes.
-- Whenever we want to show something different in CE, we could just add CE
-  partials. Same applies the other way around. If we just use
-  `render_if_exists`, it would be very easy to change the content in EE.
 
 The disadvantage of this:
 
@@ -396,6 +386,42 @@ The disadvantage of this:
   port `render_if_exists` to CE.
 - If we have typos in the partial name, it would be silently ignored.
 
+#### Using `render_ce`
+
+For `render` and `render_if_exists`, they search for the EE partial first,
+and then CE partial. They would only render a particular partial, not all
+partials with the same name. We could take the advantage of this, so that
+the same partial path (e.g. `shared/issuable/form/default_templates`) could
+be referring to the CE partial in CE (i.e.
+`app/views/shared/issuable/form/_default_templates.html.haml`), while EE
+partial in EE (i.e.
+`ee/app/views/shared/issuable/form/_default_templates.html.haml`). This way,
+we could show different things between CE and EE.
+
+However sometimes we would also want to reuse the CE partial in EE partial
+because we might just want to add something to the existing CE partial. We
+could workaround this by adding another partial with a different name, but it
+would be tedious to do so.
+
+In this case, we could as well just use `render_ce` which would ignore any EE
+partials. One example would be
+`ee/app/views/shared/issuable/form/_default_templates.html.haml`:
+
+``` haml
+- if @project.feature_available?(:issuable_default_templates)
+  = render_ce 'shared/issuable/form/default_templates'
+- elsif show_promotions?
+  = render 'shared/promotions/promote_issue_templates'
+```
+
+In the above example, we can't use
+`render 'shared/issuable/form/default_templates'` because it would find the
+same EE partial, causing infinite recursion. Instead, we could use `render_ce`
+so it ignores any partials in `ee/` and then it would render the CE partial
+(i.e. `app/views/shared/issuable/form/_default_templates.html.haml`)
+for the same path (i.e. `shared/issuable/form/default_templates`). This way
+we could easily wrap around the CE partial.
+
 ### Code in `lib/`
 
 Place EE-specific logic in the top-level `EE` module namespace. Namespace the

doc/user/project/merge_requests/index.md

@@ -35,7 +35,6 @@ With **[GitLab Enterprise Edition][ee]**, you can also:
 
 - View the deployment process across projects with [Multi-Project Pipeline Graphs](../../../ci/multi_project_pipeline_graphs.md#multi-project-pipeline-graphs) **[PREMIUM]**
 - Request [approvals](merge_request_approvals.md) from your managers **[STARTER]**
-- [Squash and merge](squash_and_merge.md) for a cleaner commit history **[STARTER]**
 - Analyze the impact of your changes with [Code Quality reports](code_quality_diff.md) **[STARTER]**
 - Manage the licenses of your dependencies with [License Management](#license-management) **[ULTIMATE]**
 - Analyze your source code for vulnerabilities with [Static Application Security Testing](sast.md) **[ULTIMATE]**
@@ -64,7 +63,7 @@ B. Consider you're a web developer writing a webpage for your company's website:
 1. Your changes are previewed with [Review Apps](../../../ci/review_apps/index.md)
 1. You request your web designers for their implementation
 1. You request the [approval](merge_request_approvals.md) from your manager **[STARTER]**
-1. Once approved, your merge request is [squashed and merged](squash_and_merge.md), and [deployed to staging with GitLab Pages](https://about.gitlab.com/2016/08/26/ci-deployment-and-environments/) (Squash and Merge is available in GitLab Starter)
+1. Once approved, your merge request is [squashed and merged](squash_and_merge.md), and [deployed to staging with GitLab Pages](https://about.gitlab.com/2016/08/26/ci-deployment-and-environments/)
 1. Your production team [cherry picks](#cherry-pick-changes) the merge commit into production
 
 ## Merge requests per project

lib/api/deploy_keys.rb

@@ -148,10 +148,10 @@ module API
         requires :key_id, type: Integer, desc: 'The ID of the deploy key'
       end
       delete ":id/deploy_keys/:key_id" do
-        key = user_project.deploy_keys.find(params[:key_id])
-        not_found!('Deploy Key') unless key
+        deploy_key_project = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id])
+        not_found!('Deploy Key') unless deploy_key_project
 
-        destroy_conditionally!(key)
+        destroy_conditionally!(deploy_key_project)
       end
     end
   end

lib/api/entities.rb

@@ -860,8 +860,8 @@ module API
     class ProjectWithAccess < Project
       expose :permissions do
         expose :project_access, using: Entities::ProjectAccess do |project, options|
-          if options.key?(:project_members)
-            (options[:project_members] || []).find { |member| member.source_id == project.id }
+          if options[:project_members]
+            options[:project_members].find { |member| member.source_id == project.id }
           else
             project.project_member(options[:current_user])
           end
@@ -869,8 +869,8 @@ module API
 
         expose :group_access, using: Entities::GroupAccess do |project, options|
           if project.group
-            if options.key?(:group_members)
-              (options[:group_members] || []).find { |member| member.source_id == project.namespace_id }
+            if options[:group_members]
+              options[:group_members].find { |member| member.source_id == project.namespace_id }
             else
               project.group.group_member(options[:current_user])
             end
@@ -881,13 +881,24 @@ module API
       def self.preload_relation(projects_relation, options = {})
         relation = super(projects_relation, options)
 
-        unless options.key?(:group_members)
-          relation = relation.preload(group: [group_members: [:source, user: [notification_settings: :source]]])
+        # MySQL doesn't support LIMIT inside an IN subquery
+        if Gitlab::Database.mysql?
+          project_ids = relation.pluck('projects.id')
+          namespace_ids = relation.pluck(:namespace_id)
+        else
+          project_ids = relation.select('projects.id')
+          namespace_ids = relation.select(:namespace_id)
         end
 
-        unless options.key?(:project_members)
-          relation = relation.preload(project_members: [:source, user: [notification_settings: :source]])
-        end
+        options[:project_members] = options[:current_user]
+          .project_members
+          .where(source_id: project_ids)
+          .preload(:source, user: [notification_settings: :source])
+
+        options[:group_members] = options[:current_user]
+          .group_members
+          .where(source_id: namespace_ids)
+          .preload(:source, user: [notification_settings: :source])
 
         relation
       end

lib/api/projects.rb

@@ -58,16 +58,9 @@ module API
         projects = paginate(projects)
         projects, options = with_custom_attributes(projects, options)
 
-        if current_user
-          project_members = current_user.project_members.preload(:source, user: [notification_settings: :source])
-          group_members = current_user.group_members.preload(:source, user: [notification_settings: :source])
-        end
-
         options = options.reverse_merge(
           with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails,
           statistics: params[:statistics],
-          project_members: project_members,
-          group_members: group_members,
           current_user: current_user
         )
         options[:with] = Entities::BasicProjectDetails if params[:simple]

lib/api/settings.rb

@@ -117,7 +117,6 @@ module API
       optional :version_check_enabled, type: Boolean, desc: 'Let GitLab inform you when an update is available.'
       optional :email_author_in_body, type: Boolean, desc: 'Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead.'
       optional :html_emails_enabled, type: Boolean, desc: 'By default GitLab sends emails in HTML and plain text formats so mail clients can choose what format to use. Disable this option if you only want to send emails in plain text format.'
-      optional :email_additional_text, type: String, desc: 'Additional text added to the bottom of every email for legal/auditing/compliance reasons'
       optional :housekeeping_enabled, type: Boolean, desc: 'Enable automatic repository housekeeping (git repack, git gc)'
       given housekeeping_enabled: ->(val) { val } do
         requires :housekeeping_bitmaps_enabled, type: Boolean, desc: "Creating pack file bitmaps makes housekeeping take a little longer but bitmaps should accelerate 'git clone' performance."
@@ -138,6 +137,8 @@ module API
                  desc: "Restrictions on the complexity of uploaded #{type.upcase} keys. A value of #{ApplicationSetting::FORBIDDEN_KEY_VALUE} disables all #{type.upcase} keys."
       end
 
+      ## EE-only START
+      optional :email_additional_text, type: String, desc: 'Additional text added to the bottom of every email for legal/auditing/compliance reasons'
       optional :help_text, type: String, desc: 'GitLab server administrator information'
       optional :elasticsearch_indexing, type: Boolean, desc: 'Enable Elasticsearch indexing'
       given elasticsearch_indexing: ->(val) { val } do
@@ -153,6 +154,7 @@ module API
       optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.'
       optional :repository_storages, type: Array[String], desc: 'A list of names of enabled storage paths, taken from `gitlab.yml`. New projects will be created in one of these stores, chosen at random.'
       optional :repository_size_limit, type: Integer, desc: 'Size limit per repository (MB)'
+      ## EE-only END
 
       optional_attributes = ::ApplicationSettingsHelper.visible_attributes << :performance_bar_allowed_group_id
 

lib/bitbucket/representation/issue.rb

@@ -12,7 +12,7 @@ module Bitbucket
       end
 
       def author
-        raw.fetch('reporter', {}).fetch('username', nil)
+        raw.dig('reporter', 'username')
       end
 
       def description

lib/gitlab/hashed_storage/rake_helper.rb

+module Gitlab
+  module HashedStorage
+    module RakeHelper
+      def self.batch_size
+        ENV.fetch('BATCH', 200).to_i
+      end
+
+      def self.listing_limit
+        ENV.fetch('LIMIT', 500).to_i
+      end
+
+      def self.project_id_batches(&block)
+        Project.with_unmigrated_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches
+          ids = relation.pluck(:id)
+
+          yield ids.min, ids.max
+        end
+      end
+
+      def self.legacy_attachments_relation
+        Upload.joins(<<~SQL).where('projects.storage_version < :version OR projects.storage_version IS NULL', version: Project::HASHED_STORAGE_FEATURES[:attachments])
+          JOIN projects
+            ON (uploads.model_type='Project' AND uploads.model_id=projects.id)
+        SQL
+      end
+
+      def self.hashed_attachments_relation
+        Upload.joins(<<~SQL).where('projects.storage_version >= :version', version: Project::HASHED_STORAGE_FEATURES[:attachments])
+          JOIN projects
+          ON (uploads.model_type='Project' AND uploads.model_id=projects.id)
+        SQL
+      end
+
+      def self.relation_summary(relation_name, relation)
+        relation_count = relation.count
+        $stdout.puts "* Found #{relation_count} #{relation_name}".color(:green)
+
+        relation_count
+      end
+
+      def self.projects_list(relation_name, relation)
+        listing(relation_name, relation.with_route) do |project|
+          $stdout.puts "  - #{project.full_path} (id: #{project.id})".color(:red)
+        end
+      end
+
+      def self.attachments_list(relation_name, relation)
+        listing(relation_name, relation) do |upload|
+          $stdout.puts "  - #{upload.path} (id: #{upload.id})".color(:red)
+        end
+      end
+
+      def self.listing(relation_name, relation)
+        relation_count = relation_summary(relation_name, relation)
+        return unless relation_count > 0
+
+        limit = listing_limit
+
+        if relation_count > limit
+          $stdout.puts "  ! Displaying first #{limit} #{relation_name}..."
+        end
+
+        relation.find_each(batch_size: batch_size).with_index do |element, index|
+          yield element
+
+          break if index + 1 >= limit
+        end
+      end
+    end
+  end
+end

lib/gitlab/import_export/attribute_cleaner.rb

@@ -7,14 +7,15 @@ module Gitlab
         new(*args).clean
       end
 
-      def initialize(relation_hash:, relation_class:)
+      def initialize(relation_hash:, relation_class:, excluded_keys: [])
         @relation_hash = relation_hash
         @relation_class = relation_class
+        @excluded_keys = excluded_keys
       end
 
       def clean
         @relation_hash.reject do |key, _value|
-          prohibited_key?(key) || !@relation_class.attribute_method?(key)
+          prohibited_key?(key) || !@relation_class.attribute_method?(key) || excluded_key?(key)
         end.except('id')
       end
 
@@ -23,6 +24,12 @@ module Gitlab
       def prohibited_key?(key)
         key.end_with?('_id') && !ALLOWED_REFERENCES.include?(key)
       end
+
+      def excluded_key?(key)
+        return false if @excluded_keys.empty?
+
+        @excluded_keys.include?(key)
+      end
     end
   end
 end

lib/gitlab/import_export/attributes_finder.rb

@@ -32,6 +32,10 @@ module Gitlab
         @methods[key].nil? ? {} : { methods: @methods[key] }
       end
 
+      def find_excluded_keys(klass_name)
+        @excluded_attributes[klass_name.to_sym]&.map(&:to_s) || []
+      end
+
       private
 
       def find_attributes_only(value)

lib/gitlab/import_export/import_export.yml

@@ -100,8 +100,6 @@ excluded_attributes:
     - :import_jid
     - :created_at
     - :updated_at
-    - :import_jid
-    - :import_jid
     - :id
     - :star_count
     - :last_activity_at

lib/gitlab/import_export/project_tree_restorer.rb

@@ -88,16 +88,18 @@ module Gitlab
       end
 
       def project_params
-        @project_params ||= json_params.merge(override_params)
+        @project_params ||= begin
+          attrs = json_params.merge(override_params)
+
+          # Cleaning all imported and overridden params
+          Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: attrs,
+                                                       relation_class: Project,
+                                                       excluded_keys: excluded_keys_for_relation(:project))
+        end
       end
 
       def override_params
-        return {} unless params = @project.import_data&.data&.fetch('override_params', nil)
-
-        @override_params ||= params.select do |key, _value|
-          Project.column_names.include?(key.to_s) &&
-            !reader.project_tree[:except].include?(key.to_sym)
-        end
+        @override_params ||= @project.import_data&.data&.fetch('override_params', nil) || {}
       end
 
       def json_params
@@ -171,7 +173,8 @@ module Gitlab
                                                        relation_hash: parsed_relation_hash(relation_hash, relation.to_sym),
                                                        members_mapper: members_mapper,
                                                        user: @user,
-                                                       project: @restored_project)
+                                                       project: @restored_project,
+                                                       excluded_keys: excluded_keys_for_relation(relation))
         end.compact
 
         relation_hash_list.is_a?(Array) ? relation_array : relation_array.first
@@ -192,6 +195,10 @@ module Gitlab
       def reader
         @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
       end
+
+      def excluded_keys_for_relation(relation)
+        @reader.attributes_finder.find_excluded_keys(relation)
+      end
     end
   end
 end

lib/gitlab/import_export/reader.rb

 module Gitlab
   module ImportExport
     class Reader
-      attr_reader :tree
+      attr_reader :tree, :attributes_finder
 
       def initialize(shared:)
         @shared = shared

lib/gitlab/import_export/relation_factory.rb

@@ -37,13 +37,21 @@ module Gitlab
         new(*args).create
       end
 
-      def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:)
+      def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:, excluded_keys: [])
         @relation_name = OVERRIDES[relation_sym] || relation_sym
         @relation_hash = relation_hash.except('noteable_id')
         @members_mapper = members_mapper
         @user = user
         @project = project
         @imported_object_retries = 0
+
+        # Remove excluded keys from relation_hash
+        # We don't do this in the parsed_relation_hash because of the 'transformed attributes'
+        # For example, MergeRequestDiffFiles exports its diff attribute as utf8_diff. Then,
+        # in the create method that attribute is renamed to diff. And because diff is an excluded key,
+        # if we clean the excluded keys in the parsed_relation_hash, it will be removed
+        # from the object attributes and the export will fail.
+        @relation_hash.except!(*excluded_keys)
       end
 
       # Creates an object from an actual model with name "relation_sym" with params from

lib/gitlab/import_formatter.rb

@@ -9,6 +9,7 @@ module Gitlab
     end
 
     def author_line(author)
+      author ||= "Anonymous"
       "*Created by: #{author}*\n\n"
     end
   end

lib/mattermost/command.rb

 module Mattermost
   class Command < Client
     def create(params)
-      response = session_post("/api/v3/teams/#{params[:team_id]}/commands/create",
+      response = session_post('/api/v4/commands',
         body: params.to_json)
 
       response['token']

lib/mattermost/session.rb

@@ -112,7 +112,7 @@ module Mattermost
     end
 
     def destroy
-      post('/api/v3/users/logout')
+      post('/api/v4/users/logout')
     end
 
     def oauth_uri
@@ -120,7 +120,7 @@ module Mattermost
 
       @oauth_uri = nil
 
-      response = get("/api/v3/oauth/gitlab/login", follow_redirects: false)
+      response = get('/oauth/gitlab/login', follow_redirects: false, format: 'text/html')
       return unless (300...400) === response.code
 
       redirect_uri = response.headers['location']

lib/mattermost/team.rb

 module Mattermost
   class Team < Client
-    # Returns **all** teams for an admin
+    # Returns all teams that the current user is a member of
     def all
-      session_get('/api/v3/teams/all').values
+      session_get("/api/v4/users/me/teams")
     end
 
     # Creates a team on the linked Mattermost instance, the team admin will be the
     # `current_user` passed to the Mattermost::Client instance
     def create(name:, display_name:, type:)
-      session_post('/api/v3/teams/create', body: {
+      session_post('/api/v4/teams', body: {
         name: name,
         display_name: display_name,
         type: type

lib/tasks/gitlab/storage.rake

@@ -3,6 +3,7 @@ namespace :gitlab do
     desc 'GitLab | Storage | Migrate existing projects to Hashed Storage'
     task migrate_to_hashed: :environment do
       legacy_projects_count = Project.with_unmigrated_storage.count
+      helper = Gitlab::HashedStorage::RakeHelper
 
       if legacy_projects_count == 0
         puts 'There are no projects requiring storage migration. Nothing to do!'
@@ -10,9 +11,9 @@ namespace :gitlab do
         next
       end
 
-      print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{batch_size}"
+      print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{helper.batch_size}"
 
-      project_id_batches do |start, finish|
+      helper.project_id_batches do |start, finish|
         StorageMigratorWorker.perform_async(start, finish)
 
         print '.'
@@ -23,118 +24,50 @@ namespace :gitlab do
 
     desc 'Gitlab | Storage | Summary of existing projects using Legacy Storage'
     task legacy_projects: :environment do
-      relation_summary('projects', Project.without_storage_feature(:repository))
+      helper = Gitlab::HashedStorage::RakeHelper
+      helper.relation_summary('projects using Legacy Storage', Project.without_storage_feature(:repository))
     end
 
     desc 'Gitlab | Storage | List existing projects using Legacy Storage'
     task list_legacy_projects: :environment do
-      projects_list('projects using Legacy Storage', Project.without_storage_feature(:repository))
+      helper = Gitlab::HashedStorage::RakeHelper
+      helper.projects_list('projects using Legacy Storage', Project.without_storage_feature(:repository))
     end
 
     desc 'Gitlab | Storage | Summary of existing projects using Hashed Storage'
     task hashed_projects: :environment do
-      relation_summary('projects using Hashed Storage', Project.with_storage_feature(:repository))
+      helper = Gitlab::HashedStorage::RakeHelper
+      helper.relation_summary('projects using Hashed Storage', Project.with_storage_feature(:repository))
     end
 
     desc 'Gitlab | Storage | List existing projects using Hashed Storage'
     task list_hashed_projects: :environment do
-      projects_list('projects using Hashed Storage', Project.with_storage_feature(:repository))
+      helper = Gitlab::HashedStorage::RakeHelper
+      helper.projects_list('projects using Hashed Storage', Project.with_storage_feature(:repository))
     end
 
     desc 'Gitlab | Storage | Summary of project attachments using Legacy Storage'
     task legacy_attachments: :environment do
-      relation_summary('attachments using Legacy Storage', legacy_attachments_relation)
+      helper = Gitlab::HashedStorage::RakeHelper
+      helper.relation_summary('attachments using Legacy Storage', helper.legacy_attachments_relation)
     end
 
     desc 'Gitlab | Storage | List existing project attachments using Legacy Storage'
     task list_legacy_attachments: :environment do
-      attachments_list('attachments using Legacy Storage', legacy_attachments_relation)
+      helper = Gitlab::HashedStorage::RakeHelper
+      helper.attachments_list('attachments using Legacy Storage', helper.legacy_attachments_relation)
     end
 
     desc 'Gitlab | Storage | Summary of project attachments using Hashed Storage'
     task hashed_attachments: :environment do
-      relation_summary('attachments using Hashed Storage', hashed_attachments_relation)
+      helper = Gitlab::HashedStorage::RakeHelper
+      helper.relation_summary('attachments using Hashed Storage', helper.hashed_attachments_relation)
     end
 
     desc 'Gitlab | Storage | List existing project attachments using Hashed Storage'
     task list_hashed_attachments: :environment do
-      attachments_list('attachments using Hashed Storage', hashed_attachments_relation)
-    end
-
-    def batch_size
-      ENV.fetch('BATCH', 200).to_i
-    end
-
-    def project_id_batches(&block)
-      Project.with_unmigrated_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches
-        ids = relation.pluck(:id)
-
-        yield ids.min, ids.max
-      end
-    end
-
-    def legacy_attachments_relation
-      Upload.joins(<<~SQL).where('projects.storage_version < :version OR projects.storage_version IS NULL', version: Project::HASHED_STORAGE_FEATURES[:attachments])
-        JOIN projects
-          ON (uploads.model_type='Project' AND uploads.model_id=projects.id)
-      SQL
-    end
-
-    def hashed_attachments_relation
-      Upload.joins(<<~SQL).where('projects.storage_version >= :version', version: Project::HASHED_STORAGE_FEATURES[:attachments])
-        JOIN projects
-        ON (uploads.model_type='Project' AND uploads.model_id=projects.id)
-      SQL
-    end
-
-    def relation_summary(relation_name, relation)
-      relation_count = relation.count
-      puts "* Found #{relation_count} #{relation_name}".color(:green)
-
-      relation_count
-    end
-
-    def projects_list(relation_name, relation)
-      relation_count = relation_summary(relation_name, relation)
-
-      projects = relation.with_route
-      limit = ENV.fetch('LIMIT', 500).to_i
-
-      return unless relation_count > 0
-
-      puts "  ! Displaying first #{limit} #{relation_name}..." if relation_count > limit
-
-      counter = 0
-      projects.find_in_batches(batch_size: batch_size) do |batch|
-        batch.each do |project|
-          counter += 1
-
-          puts "  - #{project.full_path} (id: #{project.id})".color(:red)
-
-          return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator, Cop/AvoidReturnFromBlocks
-        end
-      end
-    end
-
-    def attachments_list(relation_name, relation)
-      relation_count = relation_summary(relation_name, relation)
-
-      limit = ENV.fetch('LIMIT', 500).to_i
-
-      return unless relation_count > 0
-
-      puts "  ! Displaying first #{limit} #{relation_name}..." if relation_count > limit
-
-      counter = 0
-      relation.find_in_batches(batch_size: batch_size) do |batch|
-        batch.each do |upload|
-          counter += 1
-
-          puts "  - #{upload.path} (id: #{upload.id})".color(:red)
-
-          return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator, Cop/AvoidReturnFromBlocks
-        end
-      end
+      helper = Gitlab::HashedStorage::RakeHelper
+      helper.attachments_list('attachments using Hashed Storage', helper.hashed_attachments_relation)
     end
   end
 end

spec/controllers/groups/shared_projects_controller_spec.rb

+require 'spec_helper'
+
+describe Groups::SharedProjectsController do
+  def get_shared_projects(params = {})
+    get :index, params.reverse_merge(format: :json, group_id: group.full_path)
+  end
+
+  def share_project(project)
+    Projects::GroupLinks::CreateService.new(
+      project,
+      user,
+      link_group_access: ProjectGroupLink::DEVELOPER
+    ).execute(group)
+  end
+
+  set(:group) { create(:group) }
+  set(:user) { create(:user) }
+  set(:shared_project) do
+    shared_project = create(:project, namespace: user.namespace)
+    share_project(shared_project)
+
+    shared_project
+  end
+
+  let(:json_project_ids) { json_response.map { |project_info| project_info['id'] } }
+
+  before do
+    sign_in(user)
+  end
+
+  describe 'GET #index' do
+    it 'returns only projects shared with the group' do
+      create(:project, namespace: group)
+
+      get_shared_projects
+
+      expect(json_project_ids).to contain_exactly(shared_project.id)
+    end
+
+    it 'allows filtering shared projects' do
+      project = create(:project, :archived, namespace: user.namespace, name: "Searching for")
+      share_project(project)
+
+      get_shared_projects(filter: 'search')
+
+      expect(json_project_ids).to contain_exactly(project.id)
+    end
+
+    it 'allows sorting projects' do
+      shared_project.update!(name: 'bbb')
+      second_project = create(:project, namespace: user.namespace, name: 'aaaa')
+      share_project(second_project)
+
+      get_shared_projects(sort: 'name_asc')
+
+      expect(json_project_ids).to eq([second_project.id, shared_project.id])
+    end
+  end
+end

spec/controllers/profiles_controller_spec.rb

@@ -3,6 +3,19 @@ require('spec_helper')
 describe ProfilesController, :request_store do
   let(:user) { create(:user) }
 
+  describe 'POST update' do
+    it 'does not update password' do
+      sign_in(user)
+
+      expect do
+        post :update,
+             user: { password: 'hello12345', password_confirmation: 'hello12345' }
+      end.not_to change { user.reload.encrypted_password }
+
+      expect(response.status).to eq(302)
+    end
+  end
+
   describe 'PUT update' do
     it 'allows an email update from a user without an external email address' do
       sign_in(user)

spec/controllers/projects/merge_requests_controller_spec.rb

@@ -275,6 +275,7 @@ describe Projects::MergeRequestsController do
         namespace_id: project.namespace,
         project_id: project,
         id: merge_request.iid,
+        squash: false,
         format: 'json'
       }
     end

spec/features/issues_spec.rb

@@ -627,6 +627,20 @@ describe 'Issues' do
         end
       end
 
+      it 'clears local storage after creating a new issue', :js do
+        2.times do
+          visit new_project_issue_path(project)
+          wait_for_requests
+
+          expect(page).to have_field('Title', with: '')
+
+          fill_in 'issue_title', with: 'bug 345'
+          fill_in 'issue_description', with: 'bug description'
+
+          click_button 'Submit issue'
+        end
+      end
+
       context 'dropzone upload file', :js do
         before do
           visit new_project_issue_path(project)

spec/features/projects/merge_requests/user_views_diffs_spec.rb

@@ -26,6 +26,10 @@ describe 'User views diffs', :js do
     expect(page).to have_css('#inline-diff-btn', count: 1)
   end
 
+  it 'hides loading spinner after load' do
+    expect(page).not_to have_selector('.mr-loading-status .loading', visible: true)
+  end
+
   context 'when in the inline view' do
     include_examples 'unfold diffs'
   end

spec/features/projects/services/user_activates_mattermost_slash_command_spec.rb

@@ -64,7 +64,7 @@ feature 'Setup Mattermost slash commands', :js do
       click_link 'Add to Mattermost'
 
       expect(page).to have_content('The team where the slash commands will be used in')
-      expect(page).to have_content('This is the only available team.')
+      expect(page).to have_content('This is the only available team that you are a member of.')
     end
 
     it 'shows a disabled prefilled select if user is a member of 1 team' do
@@ -94,7 +94,7 @@ feature 'Setup Mattermost slash commands', :js do
       click_link 'Add to Mattermost'
 
       expect(page).to have_content('Select the team where the slash commands will be used in')
-      expect(page).to have_content('The list shows all available teams.')
+      expect(page).to have_content('The list shows all available teams that you are a member of.')
     end
 
     it 'shows a select with team options user is a member of multiple teams' do

spec/features/users/terms_spec.rb

@@ -62,7 +62,8 @@ describe 'Users > Terms' do
       expect(current_path).to eq(project_issues_path(project))
     end
 
-    it 'redirects back to the page the user was trying to save' do
+    # Disabled until https://gitlab.com/gitlab-org/gitlab-ce/issues/37162 is solved properly
+    xit 'redirects back to the page the user was trying to save' do
       visit new_project_issue_path(project)
 
       fill_in :issue_title, with: 'Hello world, a new issue'

spec/features/users/user_browses_projects_on_user_page_spec.rb

@@ -26,18 +26,23 @@ describe 'Users > User browses projects on user page', :js do
     end
   end
 
+  it 'hides loading spinner after load', :js do
+    visit user_path(user)
+    click_nav_link('Personal projects')
+
+    wait_for_requests
+
+    expect(page).not_to have_selector('.loading-status .loading', visible: true)
+  end
+
   it 'paginates projects', :js do
     project = create(:project, namespace: user.namespace, updated_at: 2.minutes.since)
     project2 = create(:project, namespace: user.namespace, updated_at: 1.minute.since)
     allow(Project).to receive(:default_per_page).and_return(1)
 
     sign_in(user)
-
     visit user_path(user)
-
-    page.within('.user-profile-nav') do
-      click_link('Personal projects')
-    end
+    click_nav_link('Personal projects')
 
     wait_for_requests
 
@@ -92,7 +97,6 @@ describe 'Users > User browses projects on user page', :js do
         click_nav_link('Personal projects')
 
         expect(title).to start_with(user.name)
-
         expect(page).to have_content(private_project.name)
         expect(page).to have_content(public_project.name)
         expect(page).to have_content(internal_project.name)

spec/lib/gitlab/bitbucket_import/importer_spec.rb

@@ -19,6 +19,18 @@ describe Gitlab::BitbucketImport::Importer do
     ]
   end
 
+  let(:reporters) do
+    [
+      nil,
+      { "username" => "reporter1" },
+      nil,
+      { "username" => "reporter2" },
+      { "username" => "reporter1" },
+      nil,
+      { "username" => "reporter3" }
+    ]
+  end
+
   let(:sample_issues_statuses) do
     issues = []
 
@@ -36,6 +48,10 @@ describe Gitlab::BitbucketImport::Importer do
       }
     end
 
+    reporters.map.with_index do |reporter, index|
+      issues[index]['reporter'] = reporter
+    end
+
     issues
   end
 
@@ -147,5 +163,19 @@ describe Gitlab::BitbucketImport::Importer do
         expect(importer.errors).to be_empty
       end
     end
+
+    describe 'issue import' do
+      it 'maps reporters to anonymous if bitbucket reporter is nil' do
+        allow(importer).to receive(:import_wiki)
+        importer.execute
+
+        expect(project.issues.size).to eq(7)
+        expect(project.issues.where("description LIKE ?", '%Anonymous%').size).to eq(3)
+        expect(project.issues.where("description LIKE ?", '%reporter1%').size).to eq(2)
+        expect(project.issues.where("description LIKE ?", '%reporter2%').size).to eq(1)
+        expect(project.issues.where("description LIKE ?", '%reporter3%').size).to eq(1)
+        expect(importer.errors).to be_empty
+      end
+    end
   end
 end

spec/lib/gitlab/import_export/attribute_cleaner_spec.rb

@@ -15,7 +15,10 @@ describe Gitlab::ImportExport::AttributeCleaner do
       'project_id' => 99,
       'user_id' => 99,
       'random_id_in_the_middle' => 99,
-      'notid' => 99
+      'notid' => 99,
+      'import_source' => 'whatever',
+      'import_type' => 'whatever',
+      'non_existent_attr' => 'whatever'
     }
   end
 
@@ -28,10 +31,30 @@ describe Gitlab::ImportExport::AttributeCleaner do
     }
   end
 
+  let(:excluded_keys) { %w[import_source import_type] }
+
+  subject { described_class.clean(relation_hash: unsafe_hash, relation_class: relation_class, excluded_keys: excluded_keys) }
+
+  before do
+    allow(relation_class).to receive(:attribute_method?).and_return(true)
+    allow(relation_class).to receive(:attribute_method?).with('non_existent_attr').and_return(false)
+  end
+
   it 'removes unwanted attributes from the hash' do
-    # allow(relation_class).to receive(:attribute_method?).and_return(true)
+    expect(subject).to eq(post_safe_hash)
+  end
+
+  it 'removes attributes not present in relation_class' do
+    expect(subject.keys).not_to include 'non_existent_attr'
+  end
+
+  it 'removes excluded keys from the hash' do
+    expect(subject.keys).not_to include excluded_keys
+  end
+
+  it 'does not remove excluded key if not listed' do
     parsed_hash = described_class.clean(relation_hash: unsafe_hash, relation_class: relation_class)
 
-    expect(parsed_hash).to eq(post_safe_hash)
+    expect(parsed_hash.keys).to eq post_safe_hash.keys + excluded_keys
   end
 end

spec/lib/gitlab/import_export/project.json

 {
   "description": "Nisi et repellendus ut enim quo accusamus vel magnam.",
+  "import_type": "gitlab_project",
+  "creator_id": 123,
   "visibility_level": 10,
   "archived": false,
   "labels": [

spec/lib/gitlab/import_export/project.light.json

 {
   "description": "Nisi et repellendus ut enim quo accusamus vel magnam.",
+  "import_type": "gitlab_project",
+  "creator_id": 123,
   "visibility_level": 10,
   "archived": false,
   "milestones": [

spec/lib/gitlab/import_export/project_tree_restorer_spec.rb

@@ -23,6 +23,10 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
         allow_any_instance_of(Gitlab::Git::Repository).to receive(:create_branch)
 
         project_tree_restorer = described_class.new(user: @user, shared: @shared, project: @project)
+
+        expect(Gitlab::ImportExport::RelationFactory).to receive(:create).with(hash_including(excluded_keys: ['whatever'])).and_call_original.at_least(:once)
+        allow(project_tree_restorer).to receive(:excluded_keys_for_relation).and_return(['whatever'])
+
         @restored_project_json = project_tree_restorer.restore
       end
     end
@@ -248,6 +252,11 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
       expect(labels.where(type: "ProjectLabel").count).to eq(results.fetch(:first_issue_labels, 0))
       expect(labels.where(type: "ProjectLabel").where.not(group_id: nil).count).to eq(0)
     end
+
+    it 'does not set params that are excluded from import_export settings' do
+      expect(project.import_type).to be_nil
+      expect(project.creator_id).not_to eq 123
+    end
   end
 
   shared_examples 'restores group correctly' do |**results|

spec/lib/gitlab/import_export/relation_factory_spec.rb

@@ -4,12 +4,14 @@ describe Gitlab::ImportExport::RelationFactory do
   let(:project) { create(:project) }
   let(:members_mapper) { double('members_mapper').as_null_object }
   let(:user) { create(:admin) }
+  let(:excluded_keys) { [] }
   let(:created_object) do
     described_class.create(relation_sym: relation_sym,
                            relation_hash: relation_hash,
                            members_mapper: members_mapper,
                            user: user,
-                           project: project)
+                           project: project,
+                           excluded_keys: excluded_keys)
   end
 
   context 'hook object' do
@@ -67,6 +69,14 @@ describe Gitlab::ImportExport::RelationFactory do
         expect(created_object.service_id).not_to eq(service_id)
       end
     end
+
+    context 'excluded attributes' do
+      let(:excluded_keys) { %w[url] }
+
+      it 'are removed from the imported object' do
+        expect(created_object.url).to be_nil
+      end
+    end
   end
 
   # Mocks an ActiveRecordish object with the dodgy columns

spec/lib/mattermost/command_spec.rb

@@ -21,13 +21,13 @@ describe Mattermost::Command do
 
     context 'for valid trigger word' do
       before do
-        stub_request(:post, 'http://mattermost.example.com/api/v3/teams/abc/commands/create')
+        stub_request(:post, 'http://mattermost.example.com/api/v4/commands')
           .with(body: {
             team_id: 'abc',
             trigger: 'gitlab'
           }.to_json)
           .to_return(
-            status: 200,
+            status: 201,
             headers: { 'Content-Type' => 'application/json' },
             body: { token: 'token' }.to_json
           )
@@ -40,16 +40,16 @@ describe Mattermost::Command do
 
     context 'for error message' do
       before do
-        stub_request(:post, 'http://mattermost.example.com/api/v3/teams/abc/commands/create')
+        stub_request(:post, 'http://mattermost.example.com/api/v4/commands')
           .to_return(
-            status: 500,
+            status: 400,
             headers: { 'Content-Type' => 'application/json' },
             body: {
               id: 'api.command.duplicate_trigger.app_error',
               message: 'This trigger word is already in use. Please choose another word.',
               detailed_error: '',
               request_id: 'obc374man7bx5r3dbc1q5qhf3r',
-              status_code: 500
+              status_code: 400
             }.to_json
           )
       end

spec/lib/mattermost/session_spec.rb

@@ -22,8 +22,8 @@ describe Mattermost::Session, type: :request do
     let(:location) { 'http://location.tld' }
     let(:cookie_header) {'MMOAUTH=taskik8az7rq8k6rkpuas7htia; Path=/;'}
     let!(:stub) do
-      WebMock.stub_request(:get, "#{mattermost_url}/api/v3/oauth/gitlab/login")
-        .to_return(headers: { 'location' => location, 'Set-Cookie' => cookie_header }, status: 307)
+      WebMock.stub_request(:get, "#{mattermost_url}/oauth/gitlab/login")
+        .to_return(headers: { 'location' => location, 'Set-Cookie' => cookie_header }, status: 302)
     end
 
     context 'without oauth uri' do
@@ -76,7 +76,7 @@ describe Mattermost::Session, type: :request do
               end
             end
 
-          WebMock.stub_request(:post, "#{mattermost_url}/api/v3/users/logout")
+          WebMock.stub_request(:post, "#{mattermost_url}/api/v4/users/logout")
             .to_return(headers: { Authorization: 'token thisworksnow' }, status: 200)
         end
 

spec/lib/mattermost/team_spec.rb

@@ -12,9 +12,8 @@ describe Mattermost::Team do
   describe '#all' do
     subject { described_class.new(nil).all }
 
-    context 'for valid request' do
-      let(:response) do
-        { "xiyro8huptfhdndadpz8r3wnbo" => {
+    let(:test_team) do
+      {
         "id" => "xiyro8huptfhdndadpz8r3wnbo",
         "create_at" => 1482174222155,
         "update_at" => 1482174222155,
@@ -27,11 +26,14 @@ describe Mattermost::Team do
         "allowed_domains" => "",
         "invite_id" => "o4utakb9jtb7imctdfzbf9r5ro",
         "allow_open_invite" => false
-        } }
+      }
     end
 
+    context 'for valid request' do
+      let(:response) { [test_team] }
+
       before do
-        stub_request(:get, 'http://mattermost.example.com/api/v3/teams/all')
+        stub_request(:get, 'http://mattermost.example.com/api/v4/users/me/teams')
           .to_return(
             status: 200,
             headers: { 'Content-Type' => 'application/json' },
@@ -39,14 +41,14 @@ describe Mattermost::Team do
           )
       end
 
-      it 'returns a token' do
-        is_expected.to eq(response.values)
+      it 'returns teams' do
+        is_expected.to eq(response)
       end
     end
 
     context 'for error message' do
       before do
-        stub_request(:get, 'http://mattermost.example.com/api/v3/teams/all')
+        stub_request(:get, 'http://mattermost.example.com/api/v4/users/me/teams')
           .to_return(
             status: 500,
             headers: { 'Content-Type' => 'application/json' },
@@ -89,9 +91,9 @@ describe Mattermost::Team do
       end
 
       before do
-        stub_request(:post, "http://mattermost.example.com/api/v3/teams/create")
+        stub_request(:post, "http://mattermost.example.com/api/v4/teams")
           .to_return(
-            status: 200,
+            status: 201,
             body: response.to_json,
             headers: { 'Content-Type' => 'application/json' }
           )
@@ -104,7 +106,7 @@ describe Mattermost::Team do
 
     context 'for existing team' do
       before do
-        stub_request(:post, 'http://mattermost.example.com/api/v3/teams/create')
+        stub_request(:post, 'http://mattermost.example.com/api/v4/teams')
           .to_return(
             status: 400,
             headers: { 'Content-Type' => 'application/json' },

spec/models/concerns/batch_destroy_dependent_associations_spec.rb

+require 'spec_helper'
+
+describe BatchDestroyDependentAssociations do
+  class TestProject < ActiveRecord::Base
+    self.table_name = 'projects'
+
+    has_many :builds, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
+    has_many :notification_settings, as: :source, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent
+    has_many :pages_domains
+    has_many :todos
+
+    include BatchDestroyDependentAssociations
+  end
+
+  describe '#dependent_associations_to_destroy' do
+    set(:project) { TestProject.new }
+
+    it 'returns the right associations' do
+      expect(project.dependent_associations_to_destroy.map(&:name)).to match_array([:builds])
+    end
+  end
+
+  describe '#destroy_dependent_associations_in_batches' do
+    set(:project) { create(:project) }
+    set(:build) { create(:ci_build, project: project) }
+    set(:notification_setting) { create(:notification_setting, project: project) }
+    let!(:todos) { create(:todo, project: project) }
+
+    it 'destroys multiple builds' do
+      create(:ci_build, project: project)
+
+      expect(Ci::Build.count).to eq(2)
+
+      project.destroy_dependent_associations_in_batches
+
+      expect(Ci::Build.count).to eq(0)
+    end
+
+    it 'destroys builds in batches' do
+      expect(project).to receive_message_chain(:builds, :find_each).and_yield(build)
+      expect(build).to receive(:destroy).and_call_original
+
+      project.destroy_dependent_associations_in_batches
+
+      expect(Ci::Build.count).to eq(0)
+      expect(Todo.count).to eq(1)
+      expect(User.count).to be > 0
+      expect(NotificationSetting.count).to eq(User.count)
+    end
+
+    it 'excludes associations' do
+      project.destroy_dependent_associations_in_batches(exclude: [:builds])
+
+      expect(Ci::Build.count).to eq(1)
+      expect(Todo.count).to eq(1)
+      expect(User.count).to be > 0
+      expect(NotificationSetting.count).to eq(User.count)
+    end
+  end
+end

spec/models/event_spec.rb

@@ -50,6 +50,19 @@ describe Event do
       end
     end
 
+    describe '#set_last_repository_updated_at' do
+      it 'only updates once every Event::REPOSITORY_UPDATED_AT_INTERVAL minutes' do
+        last_known_timestamp = (Event::REPOSITORY_UPDATED_AT_INTERVAL - 1.minute).ago
+        project.update(last_repository_updated_at: last_known_timestamp)
+        project.reload # a reload removes fractions of seconds
+
+        expect do
+          create_push_event(project, project.owner)
+          project.reload
+        end.not_to change { project.last_repository_updated_at }
+      end
+    end
+
     describe 'after_create :track_user_interacted_projects' do
       let(:event) { build(:push_event, project: project, author: project.owner) }
 

spec/models/project_services/mattermost_slash_commands_service_spec.rb

@@ -25,7 +25,7 @@ describe MattermostSlashCommandsService do
 
       context 'the requests succeeds' do
         before do
-          stub_request(:post, 'http://mattermost.example.com/api/v3/teams/abc/commands/create')
+          stub_request(:post, 'http://mattermost.example.com/api/v4/commands')
             .with(body: {
               team_id: 'abc',
               trigger: 'gitlab',
@@ -59,7 +59,7 @@ describe MattermostSlashCommandsService do
 
       context 'an error is received' do
         before do
-          stub_request(:post, 'http://mattermost.example.com/api/v3/teams/abc/commands/create')
+          stub_request(:post, 'http://mattermost.example.com/api/v4/commands')
             .to_return(
               status: 500,
               headers: { 'Content-Type' => 'application/json' },
@@ -89,11 +89,11 @@ describe MattermostSlashCommandsService do
 
       context 'the requests succeeds' do
         before do
-          stub_request(:get, 'http://mattermost.example.com/api/v3/teams/all')
+          stub_request(:get, 'http://mattermost.example.com/api/v4/users/me/teams')
             .to_return(
               status: 200,
               headers: { 'Content-Type' => 'application/json' },
-              body: { 'list' => true }.to_json
+              body: [{ id: 'test_team_id' }].to_json
             )
         end
 
@@ -104,7 +104,7 @@ describe MattermostSlashCommandsService do
 
       context 'an error is received' do
         before do
-          stub_request(:get, 'http://mattermost.example.com/api/v3/teams/all')
+          stub_request(:get, 'http://mattermost.example.com/api/v4/users/me/teams')
             .to_return(
               status: 500,
               headers: { 'Content-Type' => 'application/json' },

spec/requests/api/deploy_keys_spec.rb

@@ -171,7 +171,7 @@ describe API::DeployKeys do
       deploy_key
     end
 
-    it 'deletes existing key' do
+    it 'removes existing key from project' do
       expect do
         delete api("/projects/#{project.id}/deploy_keys/#{deploy_key.id}", admin)
 
@@ -179,6 +179,44 @@ describe API::DeployKeys do
       end.to change { project.deploy_keys.count }.by(-1)
     end
 
+    context 'when the deploy key is public' do
+      it 'does not delete the deploy key' do
+        expect do
+          delete api("/projects/#{project.id}/deploy_keys/#{deploy_key.id}", admin)
+
+          expect(response).to have_gitlab_http_status(204)
+        end.not_to change { DeployKey.count }
+      end
+    end
+
+    context 'when the deploy key is not public' do
+      let!(:deploy_key) { create(:deploy_key, public: false) }
+
+      context 'when the deploy key is only used by this project' do
+        it 'deletes the deploy key' do
+          expect do
+            delete api("/projects/#{project.id}/deploy_keys/#{deploy_key.id}", admin)
+
+            expect(response).to have_gitlab_http_status(204)
+          end.to change { DeployKey.count }.by(-1)
+        end
+      end
+
+      context 'when the deploy key is used by other projects' do
+        before do
+          create(:deploy_keys_project, project: project2, deploy_key: deploy_key)
+        end
+
+        it 'does not delete the deploy key' do
+          expect do
+            delete api("/projects/#{project.id}/deploy_keys/#{deploy_key.id}", admin)
+
+            expect(response).to have_gitlab_http_status(204)
+          end.not_to change { DeployKey.count }
+        end
+      end
+    end
+
     it 'returns 404 Not Found with invalid ID' do
       delete api("/projects/#{project.id}/deploy_keys/404", admin)
 

spec/services/merge_requests/squash_service_spec.rb

+require 'spec_helper'
+
+describe MergeRequests::SquashService do
+  let(:service) { described_class.new(project, user, {}) }
+  let(:user) { project.owner }
+  let(:project) { create(:project, :repository) }
+  let(:repository) { project.repository.raw }
+  let(:log_error) { "Failed to squash merge request #{merge_request.to_reference(full: true)}:" }
+  let(:squash_dir_path) do
+    File.join(Gitlab.config.shared.path, 'tmp/squash', repository.gl_repository, merge_request.id.to_s)
+  end
+  let(:merge_request_with_one_commit) do
+    create(:merge_request,
+           source_branch: 'feature', source_project: project,
+           target_branch: 'master', target_project: project)
+  end
+
+  let(:merge_request_with_only_new_files) do
+    create(:merge_request,
+           source_branch: 'video', source_project: project,
+           target_branch: 'master', target_project: project)
+  end
+
+  let(:merge_request_with_large_files) do
+    create(:merge_request,
+           source_branch: 'squash-large-files', source_project: project,
+           target_branch: 'master', target_project: project)
+  end
+
+  shared_examples 'the squash succeeds' do
+    it 'returns the squashed commit SHA' do
+      result = service.execute(merge_request)
+
+      expect(result).to match(status: :success, squash_sha: a_string_matching(/\h{40}/))
+      expect(result[:squash_sha]).not_to eq(merge_request.diff_head_sha)
+    end
+
+    it 'cleans up the temporary directory' do
+      service.execute(merge_request)
+
+      expect(File.exist?(squash_dir_path)).to be(false)
+    end
+
+    it 'does not keep the branch push event' do
+      expect { service.execute(merge_request) }.not_to change { Event.count }
+    end
+
+    context 'the squashed commit' do
+      let(:squash_sha) { service.execute(merge_request)[:squash_sha] }
+      let(:squash_commit) { project.repository.commit(squash_sha) }
+
+      it 'copies the author info and message from the merge request' do
+        expect(squash_commit.author_name).to eq(merge_request.author.name)
+        expect(squash_commit.author_email).to eq(merge_request.author.email)
+
+        # Commit messages have a trailing newline, but titles don't.
+        expect(squash_commit.message.chomp).to eq(merge_request.title)
+      end
+
+      it 'sets the current user as the committer' do
+        expect(squash_commit.committer_name).to eq(user.name.chomp('.'))
+        expect(squash_commit.committer_email).to eq(user.email)
+      end
+
+      it 'has the same diff as the merge request, but a different SHA' do
+        rugged = project.repository.rugged
+        mr_diff = rugged.diff(merge_request.diff_base_sha, merge_request.diff_head_sha)
+        squash_diff = rugged.diff(merge_request.diff_start_sha, squash_sha)
+
+        expect(squash_diff.patch.length).to eq(mr_diff.patch.length)
+        expect(squash_commit.sha).not_to eq(merge_request.diff_head_sha)
+      end
+    end
+  end
+
+  describe '#execute' do
+    context 'when there is only one commit in the merge request' do
+      it 'returns that commit SHA' do
+        result = service.execute(merge_request_with_one_commit)
+
+        expect(result).to match(status: :success, squash_sha: merge_request_with_one_commit.diff_head_sha)
+      end
+
+      it 'does not perform any git actions' do
+        expect(repository).not_to receive(:popen)
+
+        service.execute(merge_request_with_one_commit)
+      end
+    end
+
+    context 'when squashing only new files' do
+      let(:merge_request) { merge_request_with_only_new_files }
+
+      include_examples 'the squash succeeds'
+    end
+
+    context 'when squashing with files too large to display' do
+      let(:merge_request) { merge_request_with_large_files }
+
+      include_examples 'the squash succeeds'
+    end
+
+    context 'git errors' do
+      let(:merge_request) { merge_request_with_only_new_files }
+      let(:error) { 'A test error' }
+
+      context 'with gitaly enabled' do
+        before do
+          allow(repository.gitaly_operation_client).to receive(:user_squash)
+            .and_raise(Gitlab::Git::Repository::GitError, error)
+        end
+
+        it 'logs the stage and output' do
+          expect(service).to receive(:log_error).with(log_error)
+          expect(service).to receive(:log_error).with(error)
+
+          service.execute(merge_request)
+        end
+
+        it 'returns an error' do
+          expect(service.execute(merge_request)).to match(status: :error,
+                                                          message: a_string_including('squash'))
+        end
+      end
+
+      context 'with Gitaly disabled', :skip_gitaly_mock do
+        stages = {
+          'add worktree for squash' => 'worktree',
+          'configure sparse checkout' => 'config',
+          'get files in diff' => 'diff --name-only',
+          'check out target branch' => 'checkout',
+          'apply patch' => 'diff --binary',
+          'commit squashed changes' => 'commit',
+          'get SHA of squashed commit' => 'rev-parse'
+        }
+
+        stages.each do |stage, command|
+          context "when the #{stage} stage fails" do
+            before do
+              git_command = a_collection_containing_exactly(
+                a_string_starting_with("#{Gitlab.config.git.bin_path} #{command}")
+              ).or(
+                a_collection_starting_with([Gitlab.config.git.bin_path] + command.split)
+              )
+
+              allow(repository).to receive(:popen).and_return(['', 0])
+              allow(repository).to receive(:popen).with(git_command, anything, anything, anything).and_return([error, 1])
+            end
+
+            it 'logs the stage and output' do
+              expect(service).to receive(:log_error).with(log_error)
+              expect(service).to receive(:log_error).with(error)
+
+              service.execute(merge_request)
+            end
+
+            it 'returns an error' do
+              expect(service.execute(merge_request)).to match(status: :error,
+                                                              message: a_string_including('squash'))
+            end
+
+            it 'cleans up the temporary directory' do
+              expect(File.exist?(squash_dir_path)).to be(false)
+
+              service.execute(merge_request)
+            end
+          end
+        end
+      end
+    end
+
+    context 'when any other exception is thrown' do
+      let(:merge_request) { merge_request_with_only_new_files }
+      let(:error) { 'A test error' }
+
+      before do
+        allow(merge_request).to receive(:commits_count).and_raise(error)
+      end
+
+      it 'logs the MR reference and exception' do
+        expect(service).to receive(:log_error).with(a_string_including("#{project.full_path}#{merge_request.to_reference}"))
+        expect(service).to receive(:log_error).with(error)
+
+        service.execute(merge_request)
+      end
+
+      it 'returns an error' do
+        expect(service.execute(merge_request)).to match(status: :error,
+                                                        message: a_string_including('squash'))
+      end
+
+      it 'cleans up the temporary directory' do
+        service.execute(merge_request)
+
+        expect(File.exist?(squash_dir_path)).to be(false)
+      end
+    end
+  end
+end

spec/tasks/gitlab/storage_rake_spec.rb

 require 'rake_helper'
 
-describe 'gitlab:storage rake tasks' do
+describe 'gitlab:storage:*' do
   before do
     Rake.application.rake_require 'tasks/gitlab/storage'
 
     stub_warn_user_is_not_gitlab
   end
 
-  describe 'migrate_to_hashed rake task' do
+  shared_examples "rake listing entities" do |entity_name, storage_type|
+    context 'limiting to 2' do
+      before do
+        stub_env('LIMIT' => 2)
+      end
+
+      it "lists 2 out of 3 #{storage_type.downcase} #{entity_name}" do
+        create_collection
+
+        expect { run_rake_task(task) }.to output(/Found 3 #{entity_name} using #{storage_type} Storage.*Displaying first 2 #{entity_name}/m).to_stdout
+      end
+    end
+
+    context "without any #{storage_type.downcase} #{entity_name.singularize}" do
+      it 'displays message for empty results' do
+        expect { run_rake_task(task) }.to output(/Found 0 #{entity_name} using #{storage_type} Storage/).to_stdout
+      end
+    end
+  end
+
+  shared_examples "rake entities summary" do |entity_name, storage_type|
+    context "with existing 3 #{storage_type.downcase} #{entity_name}" do
+      it "reports 3 #{storage_type.downcase} #{entity_name}" do
+        create_collection
+
+        expect { run_rake_task(task) }.to output(/Found 3 #{entity_name} using #{storage_type} Storage/).to_stdout
+      end
+    end
+
+    context "without any #{storage_type.downcase} #{entity_name.singularize}" do
+      it 'displays message for empty results' do
+        expect { run_rake_task(task) }.to output(/Found 0 #{entity_name} using #{storage_type} Storage/).to_stdout
+      end
+    end
+  end
+
+  describe 'gitlab:storage:migrate_to_hashed' do
     context '0 legacy projects' do
       it 'does nothing' do
         expect(StorageMigratorWorker).not_to receive(:perform_async)
@@ -16,8 +52,8 @@ describe 'gitlab:storage rake tasks' do
       end
     end
 
-    context '5 legacy projects' do
-      let(:projects) { create_list(:project, 5, storage_version: 0) }
+    context '3 legacy projects' do
+      let(:projects) { create_list(:project, 3, storage_version: 0) }
 
       context 'in batches of 1' do
         before do
@@ -49,4 +85,64 @@ describe 'gitlab:storage rake tasks' do
       end
     end
   end
+
+  describe 'gitlab:storage:legacy_projects' do
+    it_behaves_like 'rake entities summary', 'projects', 'Legacy' do
+      let(:task) { 'gitlab:storage:legacy_projects' }
+      let(:create_collection) { create_list(:project, 3, storage_version: 0) }
+    end
+  end
+
+  describe 'gitlab:storage:list_legacy_projects' do
+    it_behaves_like 'rake listing entities', 'projects', 'Legacy' do
+      let(:task) { 'gitlab:storage:list_legacy_projects' }
+      let(:create_collection) { create_list(:project, 3, storage_version: 0) }
+    end
+  end
+
+  describe 'gitlab:storage:hashed_projects' do
+    it_behaves_like 'rake entities summary', 'projects', 'Hashed' do
+      let(:task) { 'gitlab:storage:hashed_projects' }
+      let(:create_collection) { create_list(:project, 3, storage_version: 1) }
+    end
+  end
+
+  describe 'gitlab:storage:list_hashed_projects' do
+    it_behaves_like 'rake listing entities', 'projects', 'Hashed' do
+      let(:task) { 'gitlab:storage:list_hashed_projects' }
+      let(:create_collection) { create_list(:project, 3, storage_version: 1) }
+    end
+  end
+
+  describe 'gitlab:storage:legacy_attachments' do
+    it_behaves_like 'rake entities summary', 'attachments', 'Legacy' do
+      let(:task) { 'gitlab:storage:legacy_attachments' }
+      let(:project) { create(:project, storage_version: 1) }
+      let(:create_collection) { create_list(:upload, 3, model: project) }
+    end
+  end
+
+  describe 'gitlab:storage:list_legacy_attachments' do
+    it_behaves_like 'rake listing entities', 'attachments', 'Legacy' do
+      let(:task) { 'gitlab:storage:list_legacy_attachments' }
+      let(:project) { create(:project, storage_version: 1) }
+      let(:create_collection) { create_list(:upload, 3, model: project) }
+    end
+  end
+
+  describe 'gitlab:storage:hashed_attachments' do
+    it_behaves_like 'rake entities summary', 'attachments', 'Hashed' do
+      let(:task) { 'gitlab:storage:hashed_attachments' }
+      let(:project) { create(:project, storage_version: 2) }
+      let(:create_collection) { create_list(:upload, 3, model: project) }
+    end
+  end
+
+  describe 'gitlab:storage:list_hashed_attachments' do
+    it_behaves_like 'rake listing entities', 'attachments', 'Hashed' do
+      let(:task) { 'gitlab:storage:list_hashed_attachments' }
+      let(:project) { create(:project, storage_version: 2) }
+      let(:create_collection) { create_list(:upload, 3, model: project) }
+    end
+  end
 end

spec/uploaders/object_storage_spec.rb

@@ -394,6 +394,8 @@ describe ObjectStorage do
         is_expected.to have_key(:RemoteObject)
 
         expect(subject[:RemoteObject]).to have_key(:ID)
+        expect(subject[:RemoteObject]).to include(Timeout: a_kind_of(Integer))
+        expect(subject[:RemoteObject][:Timeout]).to be(ObjectStorage::DIRECT_UPLOAD_TIMEOUT)
         expect(subject[:RemoteObject]).to have_key(:GetURL)
         expect(subject[:RemoteObject]).to have_key(:DeleteURL)
         expect(subject[:RemoteObject]).to have_key(:StoreURL)