Merge branch 'master' into '41416-making-instance-wide-data-tools-more-accessible'

# Conflicts:
#   app/models/application_setting.rb
#   lib/api/settings.rb

CHANGELOG.md

@@ -2,6 +2,10 @@
 documentation](doc/development/changelog.md) for instructions on adding your own
 entry.
 
+## 11.1.4 (2018-07-30)
+
+- No changes.
+
 ## 11.1.3 (2018-07-27)
 
 ### Fixed (8 changes, 1 of them is from the community)

GITALY_SERVER_VERSION

-0.113.0
+0.114.0

GITLAB_WORKHORSE_VERSION

-5.0.0
+5.1.0

Gemfile

@@ -306,7 +306,7 @@ group :metrics do
   gem 'influxdb', '~> 0.2', require: false
 
   # Prometheus
-  gem 'prometheus-client-mmap', '~> 0.9.3'
+  gem 'prometheus-client-mmap', '~> 0.9.4'
   gem 'raindrops', '~> 0.18'
 end
 

Gemfile.lock

@@ -635,7 +635,7 @@ GEM
       parser
       unparser
     procto (0.0.3)
-    prometheus-client-mmap (0.9.3)
+    prometheus-client-mmap (0.9.4)
     pry (0.10.4)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
@@ -1126,7 +1126,7 @@ DEPENDENCIES
   peek-sidekiq (~> 1.0.3)
   pg (~> 0.18.2)
   premailer-rails (~> 1.9.7)
-  prometheus-client-mmap (~> 0.9.3)
+  prometheus-client-mmap (~> 0.9.4)
   pry-byebug (~> 3.4.1)
   pry-rails (~> 0.3.4)
   rack-attack (~> 4.4.1)

Gemfile.rails5.lock

@@ -639,7 +639,7 @@ GEM
       parser
       unparser
     procto (0.0.3)
-    prometheus-client-mmap (0.9.3)
+    prometheus-client-mmap (0.9.4)
     pry (0.10.4)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
@@ -1136,7 +1136,7 @@ DEPENDENCIES
   peek-sidekiq (~> 1.0.3)
   pg (~> 0.18.2)
   premailer-rails (~> 1.9.7)
-  prometheus-client-mmap (~> 0.9.3)
+  prometheus-client-mmap (~> 0.9.4)
   pry-byebug (~> 3.4.1)
   pry-rails (~> 0.3.4)
   rack-attack (~> 4.4.1)

app/assets/javascripts/boards/components/board.js

@@ -2,6 +2,9 @@
 
 import Sortable from 'sortablejs';
 import Vue from 'vue';
+import { n__ } from '~/locale';
+import Icon from '~/vue_shared/components/icon.vue';
+import Tooltip from '~/vue_shared/directives/tooltip';
 import AccessorUtilities from '../../lib/utils/accessor';
 import boardList from './board_list.vue';
 import BoardBlankState from './board_blank_state.vue';
@@ -17,6 +20,10 @@ gl.issueBoards.Board = Vue.extend({
     boardList,
     'board-delete': gl.issueBoards.BoardDelete,
     BoardBlankState,
+    Icon,
+  },
+  directives: {
+    Tooltip,
   },
   props: {
     list: {
@@ -46,6 +53,12 @@ gl.issueBoards.Board = Vue.extend({
       filter: Store.filter,
     };
   },
+  computed: {
+    counterTooltip() {
+      const { issuesSize } = this.list;
+      return `${n__('%d issue', '%d issues', issuesSize)}`;
+    },
+  },
   watch: {
     filter: {
       handler() {

app/assets/javascripts/boards/models/list.js

@@ -136,6 +136,8 @@ class List {
         }
 
         this.createIssues(data.issues);
+
+        return data;
       });
   }
 

app/assets/javascripts/boards/stores/boards_store.js

@@ -125,11 +125,17 @@ gl.issueBoards.BoardsStore = {
     } else if (listTo.type === 'backlog' && listFrom.type === 'assignee') {
       issue.removeAssignee(listFrom.assignee);
       listFrom.removeIssue(issue);
-    } else if ((listTo.type !== 'label' && listFrom.type === 'assignee') ||
-               (listTo.type !== 'assignee' && listFrom.type === 'label')) {
+    } else if (this.shouldRemoveIssue(listFrom, listTo)) {
       listFrom.removeIssue(issue);
     }
   },
+  shouldRemoveIssue(listFrom, listTo) {
+    return (
+      (listTo.type !== 'label' && listFrom.type === 'assignee') ||
+      (listTo.type !== 'assignee' && listFrom.type === 'label') ||
+      (listFrom.type === 'backlog')
+    );
+  },
   moveIssueInList (list, issue, oldIndex, newIndex, idArray) {
     const beforeId = parseInt(idArray[newIndex - 1], 10) || null;
     const afterId = parseInt(idArray[newIndex + 1], 10) || null;

app/assets/javascripts/diffs/components/diff_file_header.vue

@@ -50,7 +50,7 @@ export default {
     };
   },
   computed: {
-    ...mapGetters('diffs', ['diffHasExpandedDiscussions']),
+    ...mapGetters('diffs', ['diffHasExpandedDiscussions', 'diffHasDiscussions']),
     hasExpandedDiscussions() {
       return this.diffHasExpandedDiscussions(this.diffFile);
     },
@@ -108,6 +108,9 @@ export default {
         false,
       );
     },
+    gfmCopyText() {
+      return `\`${this.diffFile.filePath}\``;
+    },
   },
   methods: {
     ...mapActions('diffs', ['toggleFileDiscussions']),
@@ -191,6 +194,7 @@ export default {
       <clipboard-button
         :title="__('Copy file path to clipboard')"
         :text="diffFile.filePath"
+        :gfm="gfmCopyText"
         css-class="btn-default btn-transparent btn-clipboard"
       />
 
@@ -217,6 +221,7 @@ export default {
         v-if="diffFile.blob && diffFile.blob.readableText"
       >
         <button
+          :disabled="!diffHasDiscussions(diffFile)"
           :class="{ active: hasExpandedDiscussions }"
           :title="s__('MergeRequests|Toggle comments for this file')"
           class="js-btn-vue-toggle-comments btn"

app/assets/javascripts/diffs/store/getters.js

@@ -47,6 +47,14 @@ export const diffHasExpandedDiscussions = (state, getters) => diff => {
   );
 };
 
+/**
+ * Checks if the diff has any discussion
+ * @param {Boolean} diff
+ * @returns {Boolean}
+ */
+export const diffHasDiscussions = (state, getters) => diff =>
+  getters.getDiffFileDiscussions(diff).length > 0;
+
 /**
  * Returns an array with the discussions of the given diff
  * @param {Object} diff

app/assets/javascripts/ide/components/ide.vue

 <script>
 import Mousetrap from 'mousetrap';
 import { mapActions, mapState, mapGetters } from 'vuex';
+import { __ } from '~/locale';
 import NewModal from './new_dropdown/modal.vue';
 import IdeSidebar from './ide_side_bar.vue';
 import RepoTabs from './repo_tabs.vue';
@@ -25,7 +26,6 @@ export default {
   },
   computed: {
     ...mapState([
-      'changedFiles',
       'openFiles',
       'viewer',
       'currentMergeRequestId',
@@ -34,18 +34,10 @@ export default {
       'currentProjectId',
       'errorMessage',
     ]),
-    ...mapGetters(['activeFile', 'hasChanges']),
+    ...mapGetters(['activeFile', 'hasChanges', 'someUncommitedChanges']),
   },
   mounted() {
-    const returnValue = 'Are you sure you want to lose unsaved changes?';
-    window.onbeforeunload = e => {
-      if (!this.changedFiles.length) return undefined;
-
-      Object.assign(e, {
-        returnValue,
-      });
-      return returnValue;
-    };
+    window.onbeforeunload = e => this.onBeforeUnload(e);
 
     Mousetrap.bind(['t', 'command+p', 'ctrl+p'], e => {
       if (e.preventDefault) {
@@ -59,6 +51,16 @@ export default {
   },
   methods: {
     ...mapActions(['toggleFileFinder']),
+    onBeforeUnload(e = {}) {
+      const returnValue = __('Are you sure you want to lose unsaved changes?');
+
+      if (!this.someUncommitedChanges) return undefined;
+
+      Object.assign(e, {
+        returnValue,
+      });
+      return returnValue;
+    },
     mousetrapStopCallback(e, el, combo) {
       if (
         (combo === 't' && el.classList.contains('dropdown-input-field')) ||

app/assets/javascripts/lib/utils/http_status.js

@@ -2,11 +2,34 @@
  * exports HTTP status codes
  */
 
-export default {
+const httpStatusCodes = {
   ABORTED: 0,
-  NO_CONTENT: 204,
   OK: 200,
+  CREATED: 201,
+  ACCEPTED: 202,
+  NON_AUTHORITATIVE_INFORMATION: 203,
+  NO_CONTENT: 204,
+  RESET_CONTENT: 205,
+  PARTIAL_CONTENT: 206,
+  MULTI_STATUS: 207,
+  ALREADY_REPORTED: 208,
+  IM_USED: 226,
   MULTIPLE_CHOICES: 300,
   BAD_REQUEST: 400,
   NOT_FOUND: 404,
 };
+
+export const successCodes = [
+  httpStatusCodes.OK,
+  httpStatusCodes.CREATED,
+  httpStatusCodes.ACCEPTED,
+  httpStatusCodes.NON_AUTHORITATIVE_INFORMATION,
+  httpStatusCodes.NO_CONTENT,
+  httpStatusCodes.RESET_CONTENT,
+  httpStatusCodes.PARTIAL_CONTENT,
+  httpStatusCodes.MULTI_STATUS,
+  httpStatusCodes.ALREADY_REPORTED,
+  httpStatusCodes.IM_USED,
+];
+
+export default httpStatusCodes;

app/assets/javascripts/lib/utils/poll.js

-import httpStatusCodes from './http_status';
+import httpStatusCodes, { successCodes } from './http_status';
 import { normalizeHeaders } from './common_utils';
 
 /**
@@ -62,7 +62,7 @@ export default class Poll {
   checkConditions(response) {
     const headers = normalizeHeaders(response.headers);
     const pollInterval = parseInt(headers[this.intervalHeader], 10);
-    if (pollInterval > 0 && response.status === httpStatusCodes.OK && this.canPoll) {
+    if (pollInterval > 0 && successCodes.indexOf(response.status) !== -1 && this.canPoll) {
       clearTimeout(this.timeoutID);
       this.timeoutID = setTimeout(() => {
         this.makeRequest();

app/assets/javascripts/vue_shared/components/clipboard_button.vue

@@ -31,6 +31,11 @@ export default {
       type: String,
       required: true,
     },
+    gfm: {
+      type: String,
+      required: false,
+      default: null,
+    },
     title: {
       type: String,
       required: true,
@@ -51,6 +56,14 @@ export default {
       default: 'btn-default',
     },
   },
+  computed: {
+    clipboardText() {
+      if (this.gfm !== null) {
+        return JSON.stringify({ text: this.text, gfm: this.gfm });
+      }
+      return this.text;
+    },
+  },
 };
 </script>
 
@@ -59,7 +72,7 @@ export default {
     v-tooltip
     :class="cssClass"
     :title="title"
-    :data-clipboard-text="text"
+    :data-clipboard-text="clipboardText"
     :data-container="tooltipContainer"
     :data-placement="tooltipPlacement"
     type="button"

app/assets/stylesheets/framework/typography.scss

@@ -444,3 +444,5 @@ textarea {
     color: $placeholder-text-color;
   }
 }
+
+.lh-100 { line-height: 1; }

app/assets/stylesheets/pages/boards.scss

@@ -205,7 +205,7 @@
 
 .board-title {
   margin: 0;
-  padding: 12px $gl-padding;
+  padding: $gl-padding-8 $gl-padding;
   font-size: 1em;
   border-bottom: 1px solid $border-color;
   display: flex;

app/assets/stylesheets/pages/issues/issue_count_badge.scss

 .issue-count-badge {
   display: inline-flex;
-  align-items: stretch;
-  height: 24px;
-}
-
-.issue-count-badge-count {
-  display: flex;
-  align-items: center;
-  padding-right: 10px;
-  padding-left: 10px;
-  border: 1px solid $border-color;
   border-radius: $border-radius-base;
-  line-height: 1;
-
-  &.has-btn {
-    border-right: 0;
-    border-top-right-radius: 0;
-    border-bottom-right-radius: 0;
-  }
+  border: 1px solid $border-color;
+  padding: 5px $gl-padding-8;
 }
 
-.issue-count-badge-add-button {
-  display: flex;
+.issue-count-badge-count {
+  display: inline-flex;
   align-items: center;
-  border: 1px solid $border-color;
-  border-radius: 0 $border-radius-base $border-radius-base 0;
-  line-height: 1;
 }

app/controllers/admin/jobs_controller.rb

@@ -2,7 +2,7 @@ class Admin::JobsController < Admin::ApplicationController
   def index
     @scope = params[:scope]
     @all_builds = Ci::Build
-    @builds = @all_builds.order('created_at DESC')
+    @builds = @all_builds.order('id DESC')
     @builds =
       case @scope
       when 'pending'

app/controllers/application_controller.rb

@@ -397,7 +397,7 @@ class ApplicationController < ActionController::Base
       # actually stored in the session and a token is needed
       # for every request. If you want the token to work as a
       # sign in token, you can simply remove store: false.
-      sign_in user, store: false
+      sign_in(user, store: false, message: :sessionless_sign_in)
     end
   end
 

app/controllers/boards/issues_controller.rb

@@ -12,8 +12,9 @@ module Boards
     skip_before_action :authenticate_user!, only: [:index]
 
     def index
-      issues = Boards::Issues::ListService.new(board_parent, current_user, filter_params).execute
-      issues = issues.page(params[:page]).per(params[:per] || 20)
+      list_service = Boards::Issues::ListService.new(board_parent, current_user, filter_params)
+      issues = list_service.execute
+      issues = issues.page(params[:page]).per(params[:per] || 20).without_count
       make_sure_position_is_set(issues) if Gitlab::Database.read_write?
       issues = issues.preload(:project,
                               :milestone,
@@ -22,10 +23,7 @@ module Boards
                               notes: [:award_emoji, :author]
                              )
 
-      render json: {
-        issues: serialize_as_json(issues),
-        size: issues.total_count
-      }
+      render_issues(issues, list_service.metadata)
     end
 
     def create
@@ -51,6 +49,13 @@ module Boards
 
     private
 
+    def render_issues(issues, metadata)
+      data = { issues: serialize_as_json(issues) }
+      data.merge!(metadata)
+
+      render json: data
+    end
+
     def make_sure_position_is_set(issues)
       issues.each do |issue|
         issue.move_to_end && issue.save unless issue.relative_position

app/controllers/concerns/authenticates_with_two_factor.rb

@@ -60,7 +60,7 @@ module AuthenticatesWithTwoFactor
 
       remember_me(user) if user_params[:remember_me] == '1'
       user.save!
-      sign_in(user)
+      sign_in(user, message: :two_factor_authenticated)
     else
       user.increment_failed_attempts!
       Gitlab::AppLogger.info("Failed Login: user=#{user.username} ip=#{request.remote_ip} method=OTP")
@@ -77,7 +77,7 @@ module AuthenticatesWithTwoFactor
       session.delete(:challenge)
 
       remember_me(user) if user_params[:remember_me] == '1'
-      sign_in(user)
+      sign_in(user, message: :two_factor_authenticated)
     else
       user.increment_failed_attempts!
       Gitlab::AppLogger.info("Failed Login: user=#{user.username} ip=#{request.remote_ip} method=U2F")

app/controllers/projects/mirrors_controller.rb

@@ -13,7 +13,9 @@ class Projects::MirrorsController < Projects::ApplicationController
   end
 
   def update
-    if project.update(mirror_params)
+    result = ::Projects::UpdateService.new(project, current_user, mirror_params).execute
+
+    if result[:status] == :success
       flash[:notice] = 'Mirroring settings were successfully updated.'
     else
       flash[:alert] = project.errors.full_messages.join(', ').html_safe

app/controllers/projects/wikis_controller.rb

 class Projects::WikisController < Projects::ApplicationController
   include PreviewMarkdown
+  include Gitlab::Utils::StrongMemoize
 
   before_action :authorize_read_wiki!
   before_action :authorize_create_wiki!, only: [:edit, :create, :history]
   before_action :authorize_admin_wiki!, only: :destroy
   before_action :load_project_wiki
+  before_action :load_page, only: [:show, :edit, :update, :history, :destroy]
+  before_action :valid_encoding?, only: [:show, :edit, :update], if: :load_page
+  before_action only: [:edit, :update], unless: :valid_encoding? do
+    redirect_to(project_wiki_path(@project, @page))
+  end
 
   def pages
     @wiki_pages = Kaminari.paginate_array(@project_wiki.pages).page(params[:page])
@@ -12,11 +18,11 @@ class Projects::WikisController < Projects::ApplicationController
   end
 
   def show
-    @page = @project_wiki.find_page(params[:id], params[:version_id])
-
     view_param = @project_wiki.empty? ? params[:view] : 'create'
 
     if @page
+      set_encoding_error unless valid_encoding?
+
       render 'show'
     elsif file = @project_wiki.find_file(params[:id], params[:version_id])
       response.headers['Content-Security-Policy'] = "default-src 'none'"
@@ -38,13 +44,11 @@ class Projects::WikisController < Projects::ApplicationController
   end
 
   def edit
-    @page = @project_wiki.find_page(params[:id])
   end
 
   def update
     return render('empty') unless can?(current_user, :create_wiki, @project)
 
-    @page = @project_wiki.find_page(params[:id])
     @page = WikiPages::UpdateService.new(@project, current_user, wiki_params).execute(@page)
 
     if @page.valid?
@@ -79,8 +83,6 @@ class Projects::WikisController < Projects::ApplicationController
   end
 
   def history
-    @page = @project_wiki.find_page(params[:id])
-
     if @page
       @page_versions = Kaminari.paginate_array(@page.versions(page: params[:page].to_i),
                                                total_count: @page.count_versions)
@@ -94,8 +96,6 @@ class Projects::WikisController < Projects::ApplicationController
   end
 
   def destroy
-    @page = @project_wiki.find_page(params[:id])
-
     WikiPages::DestroyService.new(@project, current_user).execute(@page)
 
     redirect_to project_wiki_path(@project, :home),
@@ -141,4 +141,25 @@ class Projects::WikisController < Projects::ApplicationController
       page.update_attributes(args) # rubocop:disable Rails/ActiveRecordAliases
     end
   end
+
+  def load_page
+    @page ||= @project_wiki.find_page(*page_params)
+  end
+
+  def page_params
+    keys = [:id]
+    keys << :version_id if params[:action] == 'show'
+
+    params.values_at(*keys)
+  end
+
+  def valid_encoding?
+    strong_memoize(:valid_encoding) do
+      @page.content.encoding == Encoding::UTF_8
+    end
+  end
+
+  def set_encoding_error
+    flash.now[:notice] = "The content of this page is not encoded in UTF-8. Edits can only be made via the Git repository."
+  end
 end

app/controllers/sessions_controller.rb

@@ -89,6 +89,14 @@ class SessionsController < Devise::SessionsController
     ).increment
   end
 
+  ##
+  # We do have some duplication between lib/gitlab/auth/activity.rb here, but
+  # leaving this method here because of backwards compatibility.
+  #
+  def login_counter
+    @login_counter ||= Gitlab::Metrics.counter(:user_session_logins_total, 'User sign in count')
+  end
+
   def log_failed_login
     Gitlab::AppLogger.info("Failed Login: username=#{user_params[:login]} ip=#{request.remote_ip}")
   end
@@ -97,10 +105,6 @@ class SessionsController < Devise::SessionsController
     (options = env["warden.options"]) && options[:action] == "unauthenticated"
   end
 
-  def login_counter
-    @login_counter ||= Gitlab::Metrics.counter(:user_session_logins_total, 'User sign in count')
-  end
-
   # Handle an "initial setup" state, where there's only one user, it's an admin,
   # and they require a password change.
   def check_initial_setup

app/finders/issuable_finder.rb

@@ -130,7 +130,7 @@ class IssuableFinder
 
     counts[:all] = counts.values.sum
 
-    counts
+    counts.with_indifferent_access
   end
 
   def group

app/helpers/application_settings_helper.rb

@@ -148,6 +148,7 @@ module ApplicationSettingsHelper
       :after_sign_up_text,
       :akismet_api_key,
       :akismet_enabled,
+      :allow_local_requests_from_hooks_and_services,
       :authorized_keys_enabled,
       :auto_devops_enabled,
       :auto_devops_domain,
@@ -174,6 +175,7 @@ module ApplicationSettingsHelper
       :ed25519_key_restriction,
       :email_author_in_body,
       :enabled_git_access_protocol,
+      :enforce_terms,
       :gitaly_timeout_default,
       :gitaly_timeout_medium,
       :gitaly_timeout_fast,
@@ -182,6 +184,7 @@ module ApplicationSettingsHelper
       :help_page_hide_commercial_content,
       :help_page_support_url,
       :help_page_text,
+      :hide_third_party_offers,
       :home_page_url,
       :housekeeping_bitmaps_enabled,
       :housekeeping_enabled,
@@ -203,6 +206,7 @@ module ApplicationSettingsHelper
       :metrics_port,
       :metrics_sample_interval,
       :metrics_timeout,
+      :mirror_available,
       :pages_domain_verification_enabled,
       :password_authentication_enabled_for_web,
       :password_authentication_enabled_for_git,
@@ -233,6 +237,7 @@ module ApplicationSettingsHelper
       :sign_in_text,
       :signup_enabled,
       :terminal_max_session_time,
+      :terms,
       :throttle_unauthenticated_enabled,
       :throttle_unauthenticated_requests_per_period,
       :throttle_unauthenticated_period_in_seconds,
@@ -250,12 +255,7 @@ module ApplicationSettingsHelper
       :instance_statistics_visibility_private,
       :user_default_external,
       :user_oauth_applications,
-      :version_check_enabled,
-      :allow_local_requests_from_hooks_and_services,
-      :hide_third_party_offers,
-      :enforce_terms,
-      :terms,
-      :mirror_available
+      :version_check_enabled
     ]
   end
 end

app/models/application_setting.rb

@@ -227,26 +227,28 @@ class ApplicationSetting < ActiveRecord::Base
   def self.defaults
     {
       after_sign_up_text: nil,
+      allow_local_requests_from_hooks_and_services: false,
       akismet_enabled: false,
       authorized_keys_enabled: true, # TODO default to false if the instance is configured to use AuthorizedKeysCommand
       container_registry_token_expire_delay: 5,
       default_artifacts_expire_in: '30 days',
       default_branch_protection: Settings.gitlab['default_branch_protection'],
+      default_group_visibility: Settings.gitlab.default_projects_features['visibility_level'],
       default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'],
       default_projects_limit: Settings.gitlab['default_projects_limit'],
       default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'],
-      default_group_visibility: Settings.gitlab.default_projects_features['visibility_level'],
       disabled_oauth_sign_in_sources: [],
       domain_whitelist: Settings.gitlab['domain_whitelist'],
       dsa_key_restriction: 0,
       ecdsa_key_restriction: 0,
       ed25519_key_restriction: 0,
+      gitaly_timeout_default: 55,
+      gitaly_timeout_fast: 10,
+      gitaly_timeout_medium: 30,
       gravatar_enabled: Settings.gravatar['enabled'],
-      help_page_text: nil,
       help_page_hide_commercial_content: false,
-      unique_ips_limit_per_user: 10,
-      unique_ips_limit_time_window: 3600,
-      unique_ips_limit_enabled: false,
+      help_page_text: nil,
+      hide_third_party_offers: false,
       housekeeping_bitmaps_enabled: true,
       housekeeping_enabled: true,
       housekeeping_full_repack_period: 50,
@@ -257,12 +259,14 @@ class ApplicationSetting < ActiveRecord::Base
       koding_url: nil,
       max_artifacts_size: Settings.artifacts['max_size'],
       max_attachment_size: Settings.gitlab['max_attachment_size'],
-      password_authentication_enabled_for_web: Settings.gitlab['signin_enabled'],
+      mirror_available: true,
       password_authentication_enabled_for_git: true,
+      password_authentication_enabled_for_web: Settings.gitlab['signin_enabled'],
       performance_bar_allowed_group_id: nil,
       rsa_key_restriction: 0,
       plantuml_enabled: false,
       plantuml_url: nil,
+      polling_interval_multiplier: 1,
       project_export_enabled: true,
       recaptcha_enabled: false,
       repository_checks_enabled: true,
@@ -277,18 +281,19 @@ class ApplicationSetting < ActiveRecord::Base
       sign_in_text: nil,
       signup_enabled: Settings.gitlab['signup_enabled'],
       terminal_max_session_time: 0,
-      throttle_unauthenticated_enabled: false,
-      throttle_unauthenticated_requests_per_period: 3600,
-      throttle_unauthenticated_period_in_seconds: 3600,
-      throttle_authenticated_web_enabled: false,
-      throttle_authenticated_web_requests_per_period: 7200,
-      throttle_authenticated_web_period_in_seconds: 3600,
       throttle_authenticated_api_enabled: false,
-      throttle_authenticated_api_requests_per_period: 7200,
       throttle_authenticated_api_period_in_seconds: 3600,
+      throttle_authenticated_api_requests_per_period: 7200,
+      throttle_authenticated_web_enabled: false,
+      throttle_authenticated_web_period_in_seconds: 3600,
+      throttle_authenticated_web_requests_per_period: 7200,
+      throttle_unauthenticated_enabled: false,
+      throttle_unauthenticated_period_in_seconds: 3600,
+      throttle_unauthenticated_requests_per_period: 3600,
       two_factor_grace_period: 48,
-      user_default_external: false,
-      polling_interval_multiplier: 1,
+      unique_ips_limit_enabled: false,
+      unique_ips_limit_per_user: 10,
+      unique_ips_limit_time_window: 3600,
       usage_ping_enabled: Settings.gitlab['usage_ping_enabled'],
       instance_statistics_visibility_private: false,
       gitaly_timeout_fast: 10,

app/models/ci/build.rb

@@ -22,9 +22,10 @@ module Ci
     has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id
 
     has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy, inverse_of: :job # rubocop:disable Cop/ActiveRecordDependent
-    has_one :job_artifacts_archive, -> { where(file_type: Ci::JobArtifact.file_types[:archive]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
-    has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
-    has_one :job_artifacts_trace, -> { where(file_type: Ci::JobArtifact.file_types[:trace]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
+
+    Ci::JobArtifact.file_types.each do |key, value|
+      has_one :"job_artifacts_#{key}", -> { where(file_type: value) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
+    end
 
     has_one :metadata, class_name: 'Ci::BuildMetadata'
     has_one :runner_session, class_name: 'Ci::BuildRunnerSession', validate: true, inverse_of: :build
@@ -386,6 +387,10 @@ module Ci
       trace.exist?
     end
 
+    def has_test_reports?
+      job_artifacts.test_reports.any?
+    end
+
     def has_old_trace?
       old_trace.present?
     end
@@ -453,16 +458,22 @@ module Ci
       save
     end
 
+    def erase_test_reports!
+      # TODO: Use fast_destroy_all in the context of https://gitlab.com/gitlab-org/gitlab-ce/issues/35240
+      job_artifacts_junit&.destroy
+    end
+
     def erase(opts = {})
       return false unless erasable?
 
       erase_artifacts!
+      erase_test_reports!
       erase_trace!
       update_erased!(opts[:erased_by])
     end
 
     def erasable?
-      complete? && (artifacts? || has_trace?)
+      complete? && (artifacts? || has_test_reports? || has_trace?)
     end
 
     def erased?
@@ -539,10 +550,6 @@ module Ci
       Gitlab::Ci::Build::Image.from_services(self)
     end
 
-    def artifacts
-      [options[:artifacts]]
-    end
-
     def cache
       cache = options[:cache]
 

app/models/ci/build_runner_session.rb

@@ -17,7 +17,7 @@ module Ci
       {
         subprotocols: ['terminal.gitlab.com'].freeze,
         url: "#{url}/exec".sub("https://", "wss://"),
-        headers: { Authorization: authorization.presence }.compact,
+        headers: { Authorization: [authorization.presence] }.compact,
         ca_pem: certificate.presence
       }
     end

app/models/ci/job_artifact.rb

@@ -4,11 +4,17 @@ module Ci
     include ObjectStorage::BackgroundMove
     extend Gitlab::Ci::Model
 
+    TEST_REPORT_FILE_TYPES = %w[junit].freeze
+    DEFAULT_FILE_NAMES = { junit: 'junit.xml' }.freeze
+    TYPE_AND_FORMAT_PAIRS = { archive: :zip, metadata: :gzip, trace: :raw, junit: :gzip }.freeze
+
     belongs_to :project
     belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id
 
     mount_uploader :file, JobArtifactUploader
 
+    validates :file_format, presence: true, unless: :trace?, on: :create
+    validate :valid_file_format?, unless: :trace?, on: :create
     before_save :set_size, if: :file_changed?
     after_save :update_project_statistics_after_save, if: :size_changed?
     after_destroy :update_project_statistics_after_destroy, unless: :project_destroyed?
@@ -17,14 +23,33 @@ module Ci
 
     scope :with_files_stored_locally, -> { where(file_store: [nil, ::JobArtifactUploader::Store::LOCAL]) }
 
+    scope :test_reports, -> do
+      types = self.file_types.select { |file_type| TEST_REPORT_FILE_TYPES.include?(file_type) }.values
+
+      where(file_type: types)
+    end
+
     delegate :exists?, :open, to: :file
 
     enum file_type: {
       archive: 1,
       metadata: 2,
-      trace: 3
+      trace: 3,
+      junit: 4
     }
 
+    enum file_format: {
+      raw: 1,
+      zip: 2,
+      gzip: 3
+    }
+
+    def valid_file_format?
+      unless TYPE_AND_FORMAT_PAIRS[self.file_type&.to_sym] == self.file_format&.to_sym
+        errors.add(:file_format, 'Invalid file format with specified file type')
+      end
+    end
+
     def update_file_store
       # The file.object_store is set during `uploader.store!`
       # which happens after object is inserted/updated

app/models/concerns/atomic_internal_id.rb

@@ -26,6 +26,10 @@ module AtomicInternalId
 
   module ClassMethods
     def has_internal_id(column, scope:, init:, presence: true) # rubocop:disable Naming/PredicateName
+      # We require init here to retain the ability to recalculate in the absence of a
+      # InternaLId record (we may delete records in `internal_ids` for example).
+      raise "has_internal_id requires a init block, none given." unless init
+
       before_validation :"ensure_#{scope}_#{column}!", on: :create
       validates column, presence: presence
 

app/models/milestone.rb

@@ -149,6 +149,10 @@ class Milestone < ActiveRecord::Base
       reorder(Gitlab::Database.nulls_last_order('due_date', 'ASC'))
     when 'due_date_desc'
       reorder(Gitlab::Database.nulls_last_order('due_date', 'DESC'))
+    when 'name_asc'
+      reorder(Arel::Nodes::Ascending.new(arel_table[:title].lower))
+    when 'name_desc'
+      reorder(Arel::Nodes::Descending.new(arel_table[:title].lower))
     when 'start_date_asc'
       reorder(Gitlab::Database.nulls_last_order('start_date', 'ASC'))
     when 'start_date_desc'

app/models/project.rb

@@ -548,6 +548,10 @@ class Project < ActiveRecord::Base
     repository.commit_by(oid: oid)
   end
 
+  def commits_by(oids:)
+    repository.commits_by(oids: oids)
+  end
+
   # ref can't be HEAD, can only be branch/tag name or SHA
   def latest_successful_builds_for(ref = default_branch)
     latest_pipeline = pipelines.latest_successful_for(ref)

app/models/project_statistics.rb

@@ -5,7 +5,7 @@ class ProjectStatistics < ActiveRecord::Base
   before_save :update_storage_size
 
   COLUMNS_TO_REFRESH = [:repository_size, :lfs_objects_size, :commit_count].freeze
-  INCREMENTABLE_COLUMNS = [:build_artifacts_size].freeze
+  INCREMENTABLE_COLUMNS = { build_artifacts_size: %i[storage_size] }.freeze
 
   def total_repository_size
     repository_size + lfs_objects_size
@@ -38,11 +38,28 @@ class ProjectStatistics < ActiveRecord::Base
     self.storage_size = repository_size + lfs_objects_size + build_artifacts_size
   end
 
+  # Since this incremental update method does not call update_storage_size above,
+  # we have to update the storage_size here as additional column.
+  # Additional columns are updated depending on key => [columns], which allows
+  # to update statistics which are and also those which aren't included in storage_size
+  # or any other additional summary column in the future.
   def self.increment_statistic(project_id, key, amount)
-    raise ArgumentError, "Cannot increment attribute: #{key}" unless key.in?(INCREMENTABLE_COLUMNS)
+    raise ArgumentError, "Cannot increment attribute: #{key}" unless INCREMENTABLE_COLUMNS.key?(key)
     return if amount == 0
 
     where(project_id: project_id)
-      .update_all(["#{key} = COALESCE(#{key}, 0) + (?)", amount])
+      .columns_to_increment(key, amount)
+  end
+
+  def self.columns_to_increment(key, amount)
+    updates = ["#{key} = COALESCE(#{key}, 0) + (#{amount})"]
+
+    if (additional = INCREMENTABLE_COLUMNS[key])
+      additional.each do |column|
+        updates << "#{column} = COALESCE(#{column}, 0) + (#{amount})"
+      end
+    end
+
+    update_all(updates.join(', '))
   end
 end

app/models/repository.rb

@@ -312,6 +312,8 @@ class Repository
   # types - An Array of file types (e.g. `:readme`) used to refresh extra
   #         caches.
   def refresh_method_caches(types)
+    return if types.empty?
+
     to_refresh = []
 
     types.each do |type|

app/models/user.rb

@@ -248,6 +248,7 @@ class User < ActiveRecord::Base
   scope :todo_authors, ->(user_id, state) { where(id: Todo.where(user_id: user_id, state: state).select(:author_id)) }
   scope :order_recent_sign_in, -> { reorder(Gitlab::Database.nulls_last_order('current_sign_in_at', 'DESC')) }
   scope :order_oldest_sign_in, -> { reorder(Gitlab::Database.nulls_last_order('current_sign_in_at', 'ASC')) }
+  scope :confirmed, -> { where.not(confirmed_at: nil) }
 
   def self.with_two_factor_indistinct
     joins("LEFT OUTER JOIN u2f_registrations AS u2f ON u2f.user_id = users.id")
@@ -293,14 +294,17 @@ class User < ActiveRecord::Base
     end
 
     # Find a User by their primary email or any associated secondary email
-    def find_by_any_email(email)
-      by_any_email(email).take
+    def find_by_any_email(email, confirmed: false)
+      by_any_email(email, confirmed: confirmed).take
     end
 
     # Returns a relation containing all the users for the given Email address
-    def by_any_email(email)
+    def by_any_email(email, confirmed: false)
       users = where(email: email)
+      users = users.confirmed if confirmed
+
       emails = joins(:emails).where(emails: { email: email })
+      emails = emails.confirmed if confirmed
       union = Gitlab::SQL::Union.new([users, emails])
 
       from("(#{union.to_sql}) #{table_name}")

app/presenters/ci/build_runner_presenter.rb

+module Ci
+  class BuildRunnerPresenter < SimpleDelegator
+    def artifacts
+      return unless options[:artifacts]
+
+      list = []
+      list << create_archive(options[:artifacts])
+      list << create_reports(options[:artifacts][:reports], expire_in: options[:artifacts][:expire_in])
+      list.flatten.compact
+    end
+
+    private
+
+    def create_archive(artifacts)
+      return unless artifacts[:untracked] || artifacts[:paths]
+
+      {
+        artifact_type: :archive,
+        artifact_format: :zip,
+        name: artifacts[:name],
+        untracked: artifacts[:untracked],
+        paths: artifacts[:paths],
+        when: artifacts[:when],
+        expire_in: artifacts[:expire_in]
+      }
+    end
+
+    def create_reports(reports, expire_in:)
+      return unless reports&.any?
+
+      reports.map do |k, v|
+        {
+          artifact_type: k.to_sym,
+          artifact_format: :gzip,
+          name: ::Ci::JobArtifact::DEFAULT_FILE_NAMES[k.to_sym],
+          paths: v,
+          when: 'always',
+          expire_in: expire_in
+        }
+      end
+    end
+  end
+end

app/serializers/merge_request_widget_entity.rb

@@ -132,6 +132,10 @@ class MergeRequestWidgetEntity < IssuableEntity
       can?(request.current_user, :create_note, merge_request)
     end
 
+    expose :can_create_issue do |merge_request|
+      can?(current_user, :create_issue, merge_request.project)
+    end
+
     expose :can_update do |merge_request|
       can?(request.current_user, :update_merge_request, merge_request)
     end

app/services/boards/issues/list_service.rb

@@ -3,14 +3,35 @@
 module Boards
   module Issues
     class ListService < Boards::BaseService
+      include Gitlab::Utils::StrongMemoize
+
       def execute
-        issues = IssuesFinder.new(current_user, filter_params).execute
-        issues = filter(issues)
-        issues.order_by_position_and_priority
+        fetch_issues.order_by_position_and_priority
+      end
+
+      def metadata
+        keys = metadata_fields.keys
+        columns = metadata_fields.values_at(*keys).join(', ')
+        results = Issue.where(id: fetch_issues.select('issues.id')).pluck(columns)
+
+        Hash[keys.zip(results.flatten)]
       end
 
       private
 
+      def metadata_fields
+        { size: 'COUNT(*)' }
+      end
+
+      # We memoize the query here since the finder methods we use are quite complex. This does not memoize the result of the query.
+      def fetch_issues
+        strong_memoize(:fetch_issues) do
+          issues = IssuesFinder.new(current_user, filter_params).execute
+
+          filter(issues).reorder(nil)
+        end
+      end
+
       def filter(issues)
         issues = without_board_labels(issues) unless list&.movable? || list&.closed?
         issues = with_list_label(issues) if list&.label?

app/services/git_push_service.rb

@@ -3,6 +3,7 @@
 class GitPushService < BaseService
   attr_accessor :push_data, :push_commits
   include Gitlab::Access
+  include Gitlab::Utils::StrongMemoize
 
   # The N most recent commits to process in a single push payload.
   PROCESS_COMMIT_LIMIT = 100
@@ -21,14 +22,14 @@ class GitPushService < BaseService
   #  6. Checks if the project's main language has changed
   #
   def execute
-    @project.repository.after_create if @project.empty_repo?
-    @project.repository.after_push_commit(branch_name)
+    project.repository.after_create if project.empty_repo?
+    project.repository.after_push_commit(branch_name)
 
     if push_remove_branch?
-      @project.repository.after_remove_branch
+      project.repository.after_remove_branch
       @push_commits = []
     elsif push_to_new_branch?
-      @project.repository.after_create_branch
+      project.repository.after_create_branch
 
       # Re-find the pushed commits.
       if default_branch?
@@ -38,14 +39,14 @@ class GitPushService < BaseService
         # Use the pushed commits that aren't reachable by the default branch
         # as a heuristic. This may include more commits than are actually pushed, but
         # that shouldn't matter because we check for existing cross-references later.
-        @push_commits = @project.repository.commits_between(@project.default_branch, params[:newrev])
+        @push_commits = project.repository.commits_between(project.default_branch, params[:newrev])
 
         # don't process commits for the initial push to the default branch
         process_commit_messages
       end
     elsif push_to_existing_branch?
       # Collect data for this git push
-      @push_commits = @project.repository.commits_between(params[:oldrev], params[:newrev])
+      @push_commits = project.repository.commits_between(params[:oldrev], params[:newrev])
 
       process_commit_messages
 
@@ -64,7 +65,7 @@ class GitPushService < BaseService
   end
 
   def update_gitattributes
-    @project.repository.copy_gitattributes(params[:ref])
+    project.repository.copy_gitattributes(params[:ref])
   end
 
   def update_caches
@@ -76,7 +77,7 @@ class GitPushService < BaseService
       else
         paths = Set.new
 
-        @push_commits.last(PROCESS_COMMIT_LIMIT).each do |commit|
+        last_pushed_commits.each do |commit|
           commit.raw_deltas.each do |diff|
             paths << diff.new_path
           end
@@ -88,11 +89,11 @@ class GitPushService < BaseService
       types = []
     end
 
-    ProjectCacheWorker.perform_async(@project.id, types, [:commit_count, :repository_size])
+    ProjectCacheWorker.perform_async(project.id, types, [:commit_count, :repository_size])
   end
 
   def update_signatures
-    commit_shas = @push_commits.last(PROCESS_COMMIT_LIMIT).map(&:sha)
+    commit_shas = last_pushed_commits.map(&:sha)
 
     return if commit_shas.empty?
 
@@ -103,16 +104,14 @@ class GitPushService < BaseService
 
     commit_shas = Gitlab::Git::Commit.shas_with_signatures(project.repository, commit_shas)
 
-    commit_shas.each do |sha|
-      CreateGpgSignatureWorker.perform_async(sha, project.id)
-    end
+    CreateGpgSignatureWorker.perform_async(commit_shas, project.id)
   end
 
   # Schedules processing of commit messages.
   def process_commit_messages
     default = default_branch?
 
-    @push_commits.last(PROCESS_COMMIT_LIMIT).each do |commit|
+    last_pushed_commits.each do |commit|
       if commit.matches_cross_reference_regex?
         ProcessCommitWorker
           .perform_async(project.id, current_user.id, commit.to_hash, default)
@@ -123,10 +122,10 @@ class GitPushService < BaseService
   protected
 
   def update_remote_mirrors
-    return unless @project.has_remote_mirror?
+    return unless project.has_remote_mirror?
 
-    @project.mark_stuck_remote_mirrors_as_failed!
-    @project.update_remote_mirrors
+    project.mark_stuck_remote_mirrors_as_failed!
+    project.update_remote_mirrors
   end
 
   def execute_related_hooks
@@ -134,14 +133,14 @@ class GitPushService < BaseService
     # could cause the last commit of a merge request to change.
     #
     UpdateMergeRequestsWorker
-      .perform_async(@project.id, current_user.id, params[:oldrev], params[:newrev], params[:ref])
+      .perform_async(project.id, current_user.id, params[:oldrev], params[:newrev], params[:ref])
 
-    EventCreateService.new.push(@project, current_user, build_push_data)
-    Ci::CreatePipelineService.new(@project, current_user, build_push_data).execute(:push)
+    EventCreateService.new.push(project, current_user, build_push_data)
+    Ci::CreatePipelineService.new(project, current_user, build_push_data).execute(:push)
 
     SystemHookPushWorker.perform_async(build_push_data.dup, :push_hooks)
-    @project.execute_hooks(build_push_data.dup, :push_hooks)
-    @project.execute_services(build_push_data.dup, :push_hooks)
+    project.execute_hooks(build_push_data.dup, :push_hooks)
+    project.execute_services(build_push_data.dup, :push_hooks)
 
     if push_remove_branch?
       AfterBranchDeleteService
@@ -151,52 +150,50 @@ class GitPushService < BaseService
   end
 
   def perform_housekeeping
-    housekeeping = Projects::HousekeepingService.new(@project)
+    housekeeping = Projects::HousekeepingService.new(project)
     housekeeping.increment!
     housekeeping.execute if housekeeping.needed?
   rescue Projects::HousekeepingService::LeaseTaken
   end
 
   def process_default_branch
-    @push_commits_count = project.repository.commit_count_for_ref(params[:ref])
-
-    offset = [@push_commits_count - PROCESS_COMMIT_LIMIT, 0].max
+    offset = [push_commits_count - PROCESS_COMMIT_LIMIT, 0].max
     @push_commits = project.repository.commits(params[:newrev], offset: offset, limit: PROCESS_COMMIT_LIMIT)
 
-    @project.after_create_default_branch
+    project.after_create_default_branch
   end
 
   def build_push_data
     @push_data ||= Gitlab::DataBuilder::Push.build(
-      @project,
+      project,
       current_user,
       params[:oldrev],
       params[:newrev],
       params[:ref],
       @push_commits,
-      commits_count: @push_commits_count)
+      commits_count: push_commits_count)
   end
 
   def push_to_existing_branch?
     # Return if this is not a push to a branch (e.g. new commits)
-    Gitlab::Git.branch_ref?(params[:ref]) && !Gitlab::Git.blank_ref?(params[:oldrev])
+    branch_ref? && !Gitlab::Git.blank_ref?(params[:oldrev])
   end
 
   def push_to_new_branch?
-    Gitlab::Git.branch_ref?(params[:ref]) && Gitlab::Git.blank_ref?(params[:oldrev])
+    strong_memoize(:push_to_new_branch) do
+      branch_ref? && Gitlab::Git.blank_ref?(params[:oldrev])
+    end
   end
 
   def push_remove_branch?
-    Gitlab::Git.branch_ref?(params[:ref]) && Gitlab::Git.blank_ref?(params[:newrev])
-  end
-
-  def push_to_branch?
-    Gitlab::Git.branch_ref?(params[:ref])
+    strong_memoize(:push_remove_branch) do
+      branch_ref? && Gitlab::Git.blank_ref?(params[:newrev])
+    end
   end
 
   def default_branch?
-    Gitlab::Git.branch_ref?(params[:ref]) &&
-      (Gitlab::Git.ref_name(params[:ref]) == project.default_branch || project.default_branch.nil?)
+    branch_ref? &&
+      (branch_name == project.default_branch || project.default_branch.nil?)
   end
 
   def commit_user(commit)
@@ -204,6 +201,24 @@ class GitPushService < BaseService
   end
 
   def branch_name
-    @branch_name ||= Gitlab::Git.ref_name(params[:ref])
+    strong_memoize(:branch_name) do
+      Gitlab::Git.ref_name(params[:ref])
+    end
+  end
+
+  def branch_ref?
+    strong_memoize(:branch_ref) do
+      Gitlab::Git.branch_ref?(params[:ref])
+    end
+  end
+
+  def push_commits_count
+    strong_memoize(:push_commits_count) do
+      project.repository.commit_count_for_ref(params[:ref])
+    end
+  end
+
+  def last_pushed_commits
+    @last_pushed_commits ||= @push_commits.last(PROCESS_COMMIT_LIMIT)
   end
 end

app/services/git_tag_push_service.rb

@@ -9,12 +9,12 @@ class GitTagPushService < BaseService
 
     @push_data = build_push_data
 
-    EventCreateService.new.push(project, current_user, @push_data)
-    Ci::CreatePipelineService.new(project, current_user, @push_data).execute(:push)
+    EventCreateService.new.push(project, current_user, push_data)
+    Ci::CreatePipelineService.new(project, current_user, push_data).execute(:push)
 
-    SystemHooksService.new.execute_hooks(build_system_push_data.dup, :tag_push_hooks)
-    project.execute_hooks(@push_data.dup, :tag_push_hooks)
-    project.execute_services(@push_data.dup, :tag_push_hooks)
+    SystemHooksService.new.execute_hooks(build_system_push_data, :tag_push_hooks)
+    project.execute_hooks(push_data.dup, :tag_push_hooks)
+    project.execute_services(push_data.dup, :tag_push_hooks)
 
     ProjectCacheWorker.perform_async(project.id, [], [:commit_count, :repository_size])
 

app/services/issues/update_service.rb

@@ -37,6 +37,8 @@ module Issues
       end
 
       if issue.previous_changes.include?('confidential')
+        # don't enqueue immediately to prevent todos removal in case of a mistake
+        TodosDestroyer::ConfidentialIssueWorker.perform_in(1.hour, issue.id) if issue.confidential?
         create_confidentiality_note(issue)
       end
 

app/services/members/destroy_service.rb

@@ -15,6 +15,8 @@ module Members
         notification_service.decline_access_request(member)
       end
 
+      enqeue_delete_todos(member)
+
       after_execute(member: member)
 
       member
@@ -22,6 +24,12 @@ module Members
 
     private
 
+    def enqeue_delete_todos(member)
+      type = member.is_a?(GroupMember) ? 'Group' : 'Project'
+      # don't enqueue immediately to prevent todos removal in case of a mistake
+      TodosDestroyer::EntityLeaveWorker.perform_in(1.hour, member.user_id, member.source_id, type)
+    end
+
     def can_destroy_member?(member)
       can?(current_user, destroy_member_permission(member), member)
     end

app/services/projects/update_service.rb

@@ -25,13 +25,7 @@ module Projects
       return validation_failed! if project.errors.any?
 
       if project.update(params.except(:default_branch))
-        if project.previous_changes.include?('path')
-          project.rename_repo
-        else
-          system_hook_service.execute_hooks_for(project, :update)
-        end
-
-        update_pages_config if changing_pages_https_only?
+        after_update
 
         success
       else
@@ -47,6 +41,30 @@ module Projects
 
     private
 
+    def after_update
+      todos_features_changes = %w(
+        issues_access_level
+        merge_requests_access_level
+        repository_access_level
+      )
+      project_changed_feature_keys = project.project_feature.previous_changes.keys
+
+      if project.previous_changes.include?(:visibility_level) && project.private?
+        # don't enqueue immediately to prevent todos removal in case of a mistake
+        TodosDestroyer::ProjectPrivateWorker.perform_in(1.hour, project.id)
+      elsif (project_changed_feature_keys & todos_features_changes).present?
+        TodosDestroyer::PrivateFeaturesWorker.perform_in(1.hour, project.id)
+      end
+
+      if project.previous_changes.include?('path')
+        project.rename_repo
+      else
+        system_hook_service.execute_hooks_for(project, :update)
+      end
+
+      update_pages_config if changing_pages_https_only?
+    end
+
     def validation_failed!
       model_errors = project.errors.full_messages.to_sentence
       error_message = model_errors.presence || 'Project could not be updated!'

app/services/todos/destroy/base_service.rb

+module Todos
+  module Destroy
+    class BaseService
+      def execute
+        return unless todos_to_remove?
+
+        without_authorized(todos).delete_all
+      end
+
+      private
+
+      def without_authorized(items)
+        items.where('user_id NOT IN (?)', authorized_users)
+      end
+
+      def authorized_users
+        ProjectAuthorization.select(:user_id).where(project_id: project_ids)
+      end
+
+      def todos
+        raise NotImplementedError
+      end
+
+      def project_ids
+        raise NotImplementedError
+      end
+
+      def todos_to_remove?
+        raise NotImplementedError
+      end
+    end
+  end
+end

app/services/todos/destroy/confidential_issue_service.rb

+module Todos
+  module Destroy
+    class ConfidentialIssueService < ::Todos::Destroy::BaseService
+      extend ::Gitlab::Utils::Override
+
+      attr_reader :issue
+
+      def initialize(issue_id)
+        @issue = Issue.find_by(id: issue_id)
+      end
+
+      private
+
+      override :todos
+      def todos
+        Todo.where(target: issue)
+          .where('user_id != ?', issue.author_id)
+          .where('user_id NOT IN (?)', issue.assignees.select(:id))
+      end
+
+      override :todos_to_remove?
+      def todos_to_remove?
+        issue&.confidential?
+      end
+
+      override :project_ids
+      def project_ids
+        issue.project_id
+      end
+
+      override :authorized_users
+      def authorized_users
+        ProjectAuthorization.select(:user_id)
+          .where(project_id: project_ids)
+          .where('access_level >= ?', Gitlab::Access::REPORTER)
+      end
+    end
+  end
+end

app/services/todos/destroy/entity_leave_service.rb

+module Todos
+  module Destroy
+    class EntityLeaveService < ::Todos::Destroy::BaseService
+      extend ::Gitlab::Utils::Override
+
+      attr_reader :user_id, :entity
+
+      def initialize(user_id, entity_id, entity_type)
+        unless %w(Group Project).include?(entity_type)
+          raise ArgumentError.new("#{entity_type} is not an entity user can leave")
+        end
+
+        @user_id = user_id
+        @entity = entity_type.constantize.find_by(id: entity_id)
+      end
+
+      private
+
+      override :todos
+      def todos
+        if entity.private?
+          Todo.where(project_id: project_ids, user_id: user_id)
+        else
+          project_ids.each do |project_id|
+            TodosDestroyer::PrivateFeaturesWorker.perform_async(project_id, user_id)
+          end
+
+          Todo.where(
+            target_id: confidential_issues.select(:id), target_type: Issue, user_id: user_id
+          )
+        end
+      end
+
+      override :project_ids
+      def project_ids
+        case entity
+        when Project
+          [entity.id]
+        when Namespace
+          Project.select(:id).where(namespace_id: entity.self_and_descendants.select(:id))
+        end
+      end
+
+      override :todos_to_remove?
+      def todos_to_remove?
+        # if an entity is provided we want to check always at least private features
+        !!entity
+      end
+
+      def confidential_issues
+        assigned_ids = IssueAssignee.select(:issue_id).where(user_id: user_id)
+
+        Issue.where(project_id: project_ids, confidential: true)
+          .where('author_id != ?', user_id)
+          .where('id NOT IN (?)', assigned_ids)
+      end
+    end
+  end
+end

app/services/todos/destroy/private_features_service.rb

+module Todos
+  module Destroy
+    class PrivateFeaturesService < ::Todos::Destroy::BaseService
+      attr_reader :project_ids, :user_id
+
+      def initialize(project_ids, user_id = nil)
+        @project_ids = project_ids
+        @user_id = user_id
+      end
+
+      def execute
+        ProjectFeature.where(project_id: project_ids).each do |project_features|
+          target_types = []
+          target_types << Issue if private?(project_features.issues_access_level)
+          target_types << MergeRequest if private?(project_features.merge_requests_access_level)
+          target_types << Commit if private?(project_features.repository_access_level)
+
+          next if target_types.empty?
+
+          remove_todos(project_features.project_id, target_types)
+        end
+      end
+
+      private
+
+      def private?(feature_level)
+        feature_level == ProjectFeature::PRIVATE
+      end
+
+      def remove_todos(project_id, target_types)
+        items = Todo.where(project_id: project_id)
+        items = items.where(user_id: user_id) if user_id
+
+        items.where('user_id NOT IN (?)', authorized_users)
+          .where(target_type: target_types)
+          .delete_all
+      end
+    end
+  end
+end

app/services/todos/destroy/project_private_service.rb

+module Todos
+  module Destroy
+    class ProjectPrivateService < ::Todos::Destroy::BaseService
+      extend ::Gitlab::Utils::Override
+
+      attr_reader :project
+
+      def initialize(project_id)
+        @project = Project.find_by(id: project_id)
+      end
+
+      private
+
+      override :todos
+      def todos
+        Todo.where(project_id: project_ids)
+      end
+
+      override :project_ids
+      def project_ids
+        project.id
+      end
+
+      override :todos_to_remove?
+      def todos_to_remove?
+        project&.private?
+      end
+    end
+  end
+end

app/views/projects/wikis/_main_links.html.haml

@@ -4,6 +4,6 @@
       = s_("Wiki|New page")
   = link_to project_wiki_history_path(@project, @page), class: "btn" do
     = s_("Wiki|Page history")
-  - if can?(current_user, :create_wiki, @project) && @page.latest?
+  - if can?(current_user, :create_wiki, @project) && @page.latest? && @valid_encoding
     = link_to project_wiki_edit_path(@project, @page), class: "btn js-wiki-edit" do
       = _("Edit")

app/views/shared/boards/components/_board.html.haml

@@ -32,17 +32,21 @@
             "v-if" => "!list.preset && list.id" }
             %button.board-delete.has-tooltip.float-right{ type: "button", title: _("Delete list"), "aria-label" => _("Delete list"), data: { placement: "bottom" }, "@click.stop" => "deleteBoard" }
               = icon("trash")
-        .issue-count-badge.clearfix{ "v-if" => 'list.type !== "blank" && list.type !== "promotion"' }
-          %span.issue-count-badge-count.float-left{ ":class" => '{ "has-btn": list.type !== "closed" && !disabled }' }
+        .issue-count-badge.text-secondary{ "v-if" => 'list.type !== "blank" && list.type !== "promotion"', ":title": "counterTooltip", "v-tooltip": true, data: { placement: "top" } }
+          %span.issue-count-badge-count
+            %icon.mr-1{ name: "issues" }
             {{ list.issuesSize }}
-          - if can?(current_user, :admin_list, current_board_parent)
-            %button.issue-count-badge-add-button.btn.btn-sm.btn-default.has-tooltip.js-no-trigger-collapse{ type: "button",
-              "@click" => "showNewIssueForm",
-              "v-if" => 'list.type !== "closed"',
-              "aria-label" => _("New issue"),
-              "title" => _("New issue"),
-              data: { placement: "top", container: "body" } }
-              = icon("plus", class: "js-no-trigger-collapse")
+          = render_if_exists "shared/boards/components/list_weight"
+
+        - if can?(current_user, :admin_list, current_board_parent)
+          %button.issue-count-badge-add-button.btn.btn-sm.btn-default.ml-1.has-tooltip.js-no-trigger-collapse{ type: "button",
+            "@click" => "showNewIssueForm",
+            "v-if" => 'list.type !== "closed"',
+            "aria-label" => _("New issue"),
+            "title" => _("New issue"),
+            data: { placement: "top", container: "body" } }
+            = icon("plus", class: "js-no-trigger-collapse")
+
     %board-list{ "v-if" => 'list.type !== "blank" && list.type !== "promotion"',
       ":list" => "list",
       ":issues" => "list.issues",

app/workers/all_queues.yml

@@ -73,6 +73,11 @@
 - repository_check:repository_check_batch
 - repository_check:repository_check_single_repository
 
+- todos_destroyer:todos_destroyer_confidential_issue
+- todos_destroyer:todos_destroyer_entity_leave
+- todos_destroyer:todos_destroyer_project_private
+- todos_destroyer:todos_destroyer_private_features
+
 - default
 - mailers # ActionMailer::DeliveryJob.queue_name
 

app/workers/concerns/todos_destroyer_queue.rb

+# frozen_string_literal: true
+
+##
+# Concern for setting Sidekiq settings for the various Todos Destroyers.
+#
+module TodosDestroyerQueue
+  extend ActiveSupport::Concern
+
+  included do
+    queue_namespace :todos_destroyer
+  end
+end

app/workers/create_gpg_signature_worker.rb

@@ -3,15 +3,23 @@
 class CreateGpgSignatureWorker
   include ApplicationWorker
 
-  def perform(commit_sha, project_id)
+  def perform(commit_shas, project_id)
+    return if commit_shas.empty?
+
     project = Project.find_by(id: project_id)
     return unless project
 
-    commit = project.commit(commit_sha)
+    commits = project.commits_by(oids: commit_shas)
 
-    return unless commit
+    return if commits.empty?
 
     # This calculates and caches the signature in the database
-    Gitlab::Gpg::Commit.new(commit).signature
+    commits.each do |commit|
+      begin
+        Gitlab::Gpg::Commit.new(commit).signature
+      rescue => e
+        Rails.logger.error("Failed to create signature for commit #{commit.id}. Error: #{e.message}")
+      end
+    end
   end
 end

app/workers/todos_destroyer/confidential_issue_worker.rb

+module TodosDestroyer
+  class ConfidentialIssueWorker
+    include ApplicationWorker
+    include TodosDestroyerQueue
+
+    def perform(issue_id)
+      ::Todos::Destroy::ConfidentialIssueService.new(issue_id).execute
+    end
+  end
+end

app/workers/todos_destroyer/entity_leave_worker.rb

+module TodosDestroyer
+  class EntityLeaveWorker
+    include ApplicationWorker
+    include TodosDestroyerQueue
+
+    def perform(user_id, entity_id, entity_type)
+      ::Todos::Destroy::EntityLeaveService.new(user_id, entity_id, entity_type).execute
+    end
+  end
+end

app/workers/todos_destroyer/private_features_worker.rb

+module TodosDestroyer
+  class PrivateFeaturesWorker
+    include ApplicationWorker
+    include TodosDestroyerQueue
+
+    def perform(project_id, user_id = nil)
+      ::Todos::Destroy::PrivateFeaturesService.new(project_id, user_id).execute
+    end
+  end
+end

app/workers/todos_destroyer/project_private_worker.rb

+module TodosDestroyer
+  class ProjectPrivateWorker
+    include ApplicationWorker
+    include TodosDestroyerQueue
+
+    def perform(project_id)
+      ::Todos::Destroy::ProjectPrivateService.new(project_id).execute
+    end
+  end
+end

changelogs/unreleased/25990-interactive-web-terminals-authorization.yml

+---
+title: Fix authorization for interactive web terminals
+merge_request: 20811
+author:
+type: fixed

changelogs/unreleased/32821-better-error-message-add-invalid-user-to-project.yml

+---
+title: Improve error message when adding invalid user to a project
+merge_request: 20885
+author: Jacopo Beschi @jacopo-beschi
+type: added

changelogs/unreleased/48823-copy-gfm.yml

+---
+title: Resolve Copy diff file path as GFM is broken
+merge_request: 20725
+author:
+type: fixed

changelogs/unreleased/49161-disable-toggle-comments.yml

+---
+title: Disables toggle comments button if diff has no discussions
+merge_request:
+author:
+type: other

changelogs/unreleased/49701-sorting-by-name-on-milestones-page-error.yml

+---
+title: Fix sorting by name on milestones page
+merge_request: 20881
+author:
+type: fixed

changelogs/unreleased/49747-update-poll-2xx.yml

+---
+title: Changes poll.js to keep polling on any 2xx http status code
+merge_request: 20904
+author:
+type: other

changelogs/unreleased/artifact-format-v2.yml

+---
+title: Extend gitlab-ci.yml to request junit.xml test reports
+merge_request: 20390
+author:
+type: added

changelogs/unreleased/feature-gb-login-activity-metrics.yml

+---
+title: Add more comprehensive metrics tracking authentication activity
+merge_request: 20668
+author:
+type: added

changelogs/unreleased/fix-storage-size-for-artifacts-change.yml

+---
+title: Update total storage size when changing size of artifacts
+merge_request: 20697
+author: Peter Marko
+type: fixed

changelogs/unreleased/fj-37736-improve-performance-post-receive-create-gpg-siganture-worker.yml

+---
+title: Performing Commit GPG signature calculation in bulk
+merge_request: 20870
+author:
+type: performance

changelogs/unreleased/fj-49512-fix-gitlab-git-pages-encoding.yml

+---
+title: Prevent editing and updating wiki pages with non UTF-8 encoding via web interface
+merge_request: 20906
+author:
+type: fixed

changelogs/unreleased/ide-warn-staged-files.yml

+---
+title: Warn user when reload IDE with staged changes
+merge_request: 20857
+author:
+type: added

changelogs/unreleased/rails5-gpg-permit-concurrent.yml

+---
+title: Permit concurrent loads in gpg keychain mutex
+merge_request: 20894
+author: Jasper Maes
+type: fixed

changelogs/unreleased/sh-fix-admin-jobs-controller-timing-out.yml

+---
+title: Fix /admin/jobs failing to load due to statement timeout
+merge_request: 20909
+author:
+type: performance

changelogs/unreleased/sh-simplify-liveness-check.yml

+---
+title: Add /-/health basic health check endpoint
+merge_request: 20456
+author:
+type: added

changelogs/unreleased/sh-support-users-find-by-confirmed-emails.yml

+---
+title: Add support for searching users by confirmed e-mails
+merge_request: 20893
+author:
+type: other

changelogs/unreleased/todos-visibility-change.yml

+---
+title: Delete todos when user loses access to read the target
+merge_request: 20665
+author:
+type: other

changelogs/unreleased/zj-remove-git-rake-tasks.yml

+---
+title: Remove gitlab:user:check_repos, gitlab:check_repo, gitlab:git:prune, gitlab:git:gc, and gitlab:git:repack
+merge_request: 20806
+author:
+type: removed

config/application.rb

@@ -154,6 +154,10 @@ module Gitlab
 
     config.action_view.sanitized_allowed_protocols = %w(smb)
 
+    # This middleware needs to precede ActiveRecord::QueryCache and other middlewares that
+    # connect to the database.
+    config.middleware.insert_after "Rails::Rack::Logger", "Gitlab::Middleware::BasicHealthCheck"
+
     config.middleware.insert_after Warden::Manager, Rack::Attack
 
     # Allow access to GitLab API from other domains

config/initializers/rbtrace.rb

-# frozen_string_literal: true
-
-require 'rbtrace' if ENV['ENABLE_RBTRACE']

config/initializers/sidekiq.rb

@@ -8,6 +8,8 @@ Sidekiq.default_worker_options = { retry: 3 }
 enable_json_logs = Gitlab.config.sidekiq.log_format == 'json'
 
 Sidekiq.configure_server do |config|
+  require 'rbtrace' if ENV['ENABLE_RBTRACE']
+
   config.redis = queues_config_hash
 
   config.server_middleware do |chain|

config/initializers/warden.rb

 Rails.application.configure do |config|
   Warden::Manager.after_set_user(scope: :user) do |user, auth, opts|
     Gitlab::Auth::UniqueIpsLimiter.limit_user!(user)
-  end
 
-  Warden::Manager.before_failure(scope: :user) do |env, opts|
-    Gitlab::Auth::BlockedUserTracker.log_if_user_blocked(env)
+    activity = Gitlab::Auth::Activity.new(user, opts)
+
+    case opts[:event]
+    when :authentication
+      activity.user_authenticated!
+    when :set_user
+      activity.user_authenticated!
+      activity.user_session_override!
+    when :fetch # rubocop:disable Lint/EmptyWhen
+      # We ignore session fetch events
+    else
+      activity.user_session_override!
+    end
   end
 
   Warden::Manager.after_authentication(scope: :user) do |user, auth, opts|
@@ -15,7 +25,17 @@ Rails.application.configure do |config|
     ActiveSession.set(user, auth.request)
   end
 
-  Warden::Manager.before_logout(scope: :user) do |user, auth, opts|
-    ActiveSession.destroy(user || auth.user, auth.request.session.id)
+  Warden::Manager.before_failure(scope: :user) do |env, opts|
+    tracker = Gitlab::Auth::BlockedUserTracker.new(env)
+    tracker.log_blocked_user_activity! if tracker.user_blocked?
+
+    Gitlab::Auth::Activity.new(tracker.user, opts).user_authentication_failed!
+  end
+
+  Warden::Manager.before_logout(scope: :user) do |user_warden, auth, opts|
+    user = user_warden || auth.user
+
+    ActiveSession.destroy(user, auth.request.session.id)
+    Gitlab::Auth::Activity.new(user, opts).user_session_destroyed!
   end
 end

config/routes.rb

@@ -46,6 +46,7 @@ Rails.application.routes.draw do
   get 'health_check(/:checks)' => 'health_check#index', as: :health_check
 
   scope path: '-' do
+    # '/-/health' implemented by BasicHealthMiddleware
     get 'liveness' => 'health#liveness'
     get 'readiness' => 'health#readiness'
     post 'storage_check' => 'health#storage_check'

config/sidekiq_queues.yml

@@ -45,6 +45,7 @@
   - [github_import_advance_stage, 1]
   - [project_service, 1]
   - [delete_user, 1]
+  - [todos_destroyer, 1]
   - [delete_merged_branches, 1]
   - [authorized_projects, 1]
   - [expire_build_instance_artifacts, 1]

config/unicorn.rb.example

@@ -124,6 +124,10 @@ before_fork do |server, worker|
 end
 
 after_fork do |server, worker|
+  # Unicorn clears out signals before it forks, so rbtrace won't work
+  # unless it is enabled after the fork.
+  require 'rbtrace' if ENV['ENABLE_RBTRACE']
+
   # per-process listener ports for debugging/admin/migrations
   # addr = "127.0.0.1:#{9293 + worker.nr}"
   # server.listen(addr, :tries => -1, :delay => 5, :tcp_nopush => true)

db/migrate/20180705160945_add_file_format_to_ci_job_artifacts.rb

+class AddFileFormatToCiJobArtifacts < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    add_column :ci_job_artifacts, :file_format, :integer, limit: 2
+  end
+end

db/schema.rb

@@ -394,6 +394,7 @@ ActiveRecord::Schema.define(version: 20180722103201) do
     t.datetime_with_timezone "expire_at"
     t.string "file"
     t.binary "file_sha256"
+    t.integer "file_format", limit: 2
   end
 
   add_index "ci_job_artifacts", ["expire_at", "job_id"], name: "index_ci_job_artifacts_on_expire_at_and_job_id", using: :btree

doc/README.md

@@ -177,7 +177,8 @@ instant how code changes impact your production environment.
 - [Prometheus metrics](user/project/integrations/prometheus_library/metrics.md): Let Prometheus collect metrics from various services, like Kubernetes, NGINX, NGINX ingress controller, HAProxy, and Amazon Cloud Watch.
 - [GitLab Performance Monitoring](administration/monitoring/performance/index.md): Use InfluxDB and Grafana to monitor the performance of your GitLab instance (will be eventually replaced by Prometheus).
 - [Health check](user/admin_area/monitoring/health_check.md): GitLab provides liveness and readiness probes to indicate service health and reachability to required services.
-- [GitLab Cycle Analytics](user/project/cycle_analytics.md): Cycle Analytics measures the time it takes to go from an [idea to production](https://about.gitlab.com/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/#from-idea-to-production-with-gitlab) for each project you have.
+- [GitLab Cycle Analytics](user/project/cycle_analytics.md): Cycle Analytics measures the time it takes to go from an
+  [idea to production](https://about.gitlab.com/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/#from-idea-to-production-with-gitlab) for each project you have.
 
 ## Getting started with GitLab
 

doc/administration/high_availability/nfs.md

@@ -39,23 +39,11 @@ Our support team will not be able to assist on performance issues related to
 file system access.
 
 Customers and users have reported that AWS EFS does not perform well for GitLab's
-use-case. There are several issues that can cause problems. For these reasons
-GitLab does not recommend using EFS with GitLab.
-
-- EFS bases allowed IOPS on volume size. The larger the volume, the more IOPS
-  are allocated. For smaller volumes, users may experience decent performance
-  for a period of time due to 'Burst Credits'. Over a period of weeks to months
-  credits may run out and performance will bottom out.
-- To keep "Burst Credits" available, it may be necessary to provision more space
-  with 'dummy data'.  However, this may get expensive.
-- Another option to maintain "Burst Credits" is to use FS Cache on the server so
-  that AWS doesn't always have to go into EFS to access files.
-- For larger volumes, allocated IOPS may not be the problem. Workloads where
-  many small files are written in a serialized manner are not well-suited for EFS.
-  EBS with an NFS server on top will perform much better.
-
-In addition, avoid storing GitLab log files (e.g. those in `/var/log/gitlab`)
-because this will also affect performance. We recommend that the log files be
+use-case. Workloads where many small files are written in a serialized manner, like `git`,
+are not well-suited for EFS. EBS with an NFS server on top will perform much better.
+
+If you do choose to use EFS, avoid storing GitLab log files (e.g. those in `/var/log/gitlab`)
+there because this will also affect performance. We recommend that the log files be
 stored on a local volume.
 
 For more details on another person's experience with EFS, see

doc/administration/raketasks/check.md

-# Check Rake Tasks
+# Integrity Check Rake Task
 
 ## Repository Integrity
 
@@ -28,14 +28,8 @@ exactly which repositories are causing the trouble.
 
 ### Check all GitLab repositories
 
->**Note:**
->
->  - `gitlab:repo:check` has been deprecated in favor of `gitlab:git:fsck`
->  - [Deprecated][ce-15931] in GitLab 10.4.
->  - `gitlab:repo:check` will be removed in the future. [Removal issue][ce-41699]
-
 This task loops through all repositories on the GitLab server and runs the
-3 integrity checks described previously.
+integrity check described previously.
 
 **Omnibus Installation**
 
@@ -49,33 +43,6 @@ sudo gitlab-rake gitlab:git:fsck
 sudo -u git -H bundle exec rake gitlab:git:fsck RAILS_ENV=production
 ```
 
-### Check repositories for a specific user
-
-This task checks all repositories that a specific user has access to. This is important
-because sometimes you know which user is experiencing trouble but you don't know
-which project might be the cause.
-
-If the rake task is executed without brackets at the end, you will be prompted
-to enter a username.
-
-**Omnibus Installation**
-
-```bash
-sudo gitlab-rake gitlab:user:check_repos
-sudo gitlab-rake gitlab:user:check_repos[<username>]
-```
-
-**Source Installation**
-
-```bash
-sudo -u git -H bundle exec rake gitlab:user:check_repos RAILS_ENV=production
-sudo -u git -H bundle exec rake gitlab:user:check_repos[<username>] RAILS_ENV=production
-```
-
-Example output:
-
-![gitlab:user:check_repos output](../img/raketasks/check_repos_output.png)
-
 ## Uploaded Files Integrity
 
 Various types of files can be uploaded to a GitLab installation by users.
@@ -167,5 +134,4 @@ The LDAP check Rake task will test the bind_dn and password credentials
 executed as part of the `gitlab:check` task, but can run independently.
 See [LDAP Rake Tasks - LDAP Check](ldap.md#check) for details.
 
-[ce-15931]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/15931
-[ce-41699]: https://gitlab.com/gitlab-org/gitlab-ce/issues/41699
+[git-fsck]: https://git-scm.com/docs/git-fsck

doc/administration/troubleshooting/debug.md

@@ -77,7 +77,12 @@ and more. However, this is not enabled by default. To enable it, define the
 gitlab_rails['env'] = {"ENABLE_RBTRACE" => "1"}
 ```
 
-Then reconfigure the system and restart Unicorn and Sidekiq.
+Then reconfigure the system and restart Unicorn and Sidekiq. To run this
+in Omnibus, run as root:
+
+```ruby
+/opt/gitlab/embedded/bin/ruby /opt/gitlab/embedded/bin/rbtrace
+```
 
 ## Common Problems
 

doc/development/fe_guide/performance.md

@@ -15,7 +15,7 @@ Use the following rules when creating realtime solutions.
 Use that as your polling interval. This way it is [easy for system administrators to change the
 polling rate](../../administration/polling.md).
 A `Poll-Interval: -1` means you should disable polling, and this must be implemented.
-1. A response with HTTP status `4XX` or `5XX` should disable polling as well.
+1. A response with HTTP status different from 2XX should disable polling as well.
 1. Use a common library for polling.
 1. Poll on active tabs only. Please use [Visibility](https://github.com/ai/visibilityjs).
 1. Use regular polling intervals, do not use backoff polling, or jitter, as the interval will be
@@ -25,15 +25,15 @@ controlled by the server.
 
 ### Lazy Loading Images
 
-To improve the time to first render we are using lazy loading for images. This works by setting 
-the actual image source on the `data-src` attribute. After the HTML is rendered and JavaScript is loaded, 
+To improve the time to first render we are using lazy loading for images. This works by setting
+the actual image source on the `data-src` attribute. After the HTML is rendered and JavaScript is loaded,
 the value of `data-src` will be moved to `src` automatically if the image is in the current viewport.
 
 *  Prepare images in HTML for lazy loading by renaming the `src` attribute to `data-src` AND adding the class `lazy`
 *  If you are using the Rails `image_tag` helper, all images will be lazy-loaded by default unless `lazy: false` is provided.
 
 If you are asynchronously adding content which contains lazy images then you need to call the function
-`gl.lazyLoader.searchLazyImages()` which will search for lazy images and load them if needed. 
+`gl.lazyLoader.searchLazyImages()` which will search for lazy images and load them if needed.
 But in general it should be handled automatically through a `MutationObserver` in the lazy loading function.
 
 ### Animations
@@ -97,19 +97,19 @@ bundle and included on the page.
 
     ```javascript
     import initMyWidget from './my_widget';
-  
+
     document.addEventListener('DOMContentLoaded', () => {
       initMyWidget();
     });
     ```
 
-- **Supporting Module Placement:**  
+- **Supporting Module Placement:**
     - If a class or a module is _specific to a particular route_, try to locate
       it close to the entry point it will be used. For instance, if
       `my_widget.js` is only imported within `pages/widget/show/index.js`, you
       should place the module at `pages/widget/show/my_widget.js` and import it
       with a relative path (e.g. `import initMyWidget from './my_widget';`).
-      
+
     - If a class or module is _used by multiple routes_, place it within a
       shared directory at the closest common parent directory for the entry
       points that import it.  For example, if `my_widget.js` is imported within

doc/university/high-availability/aws/README.md

@@ -2,10 +2,8 @@
 comments: false
 ---
 
-DANGER: This guide exists for reference of how an AWS deployment could work.
-We are currently seeing very slow EFS access performance which causes GitLab to
-be 5-10x slower than using NFS or Local disk. We _do not_ recommend follow this
-guide at this time.
+> **Note**: We **do not** recommend using the AWS Elastic File System (EFS), as it can result
+in [significantly degraded performance](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/doc/administration/high_availability/nfs.md#aws-elastic-file-system).
 
 # High Availability on AWS
 

doc/user/admin_area/monitoring/health_check.md

@@ -20,14 +20,24 @@ To access monitoring resources, the client IP needs to be included in a whitelis
 
 [Read how to add IPs to a whitelist for the monitoring endpoints][admin].
 
-## Using the endpoint
+## Using the endpoints
 
 With default whitelist settings, the probes can be accessed from localhost:
 
+- `http://localhost/-/health`
 - `http://localhost/-/readiness`
 - `http://localhost/-/liveness`
 
-which will then provide a report of system health in JSON format.
+
+The first endpoint, `/-/health/`, only checks whether the application server is running. It does
+-not verify the database or other services are running. A successful response will return
+a 200 status code with the following message:
+
+```
+GitLab OK
+```
+
+The readiness and liveness probes will provide a report of system health in JSON format.
 
 Readiness example output:
 
@@ -42,12 +52,6 @@ Readiness example output:
    "shared_state_check" : {
       "status" : "ok"
    },
-   "fs_shards_check" : {
-      "labels" : {
-         "shard" : "default"
-      },
-      "status" : "ok"
-   },
    "db_check" : {
       "status" : "ok"
    },
@@ -61,9 +65,6 @@ Liveness example output:
 
 ```
 {
-   "fs_shards_check" : {
-      "status" : "ok"
-   },
    "cache_check" : {
       "status" : "ok"
    },

lib/api/entities.rb

@@ -1236,7 +1236,13 @@ module API
       end
 
       class Artifacts < Grape::Entity
-        expose :name, :untracked, :paths, :when, :expire_in
+        expose :name
+        expose :untracked
+        expose :paths
+        expose :when
+        expose :expire_in
+        expose :artifact_type
+        expose :artifact_format
       end
 
       class Cache < Grape::Entity

lib/api/members.rb

@@ -75,7 +75,10 @@ module API
           member = source.members.find_by(user_id: params[:user_id])
           conflict!('Member already exists') if member
 
-          member = source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at])
+          user = User.find_by_id(params[:user_id])
+          not_found!('User') unless user
+
+          member = source.add_user(user, params[:access_level], current_user: current_user, expires_at: params[:expires_at])
 
           if !member
             not_allowed! # This currently can only be reached in EE

lib/api/projects.rb

@@ -13,6 +13,10 @@ module API
         # EE::API::Projects would override this helper
       end
 
+      params :optional_update_params_ee do
+        # EE::API::Projects would override this helper
+      end
+
       # EE::API::Projects would override this method
       def apply_filters(projects)
         projects = projects.with_issues_available_for_user(current_user) if params[:with_issues_enabled]
@@ -21,6 +25,37 @@ module API
 
         projects
       end
+
+      def verify_update_project_attrs!(project, attrs)
+      end
+    end
+
+    def self.update_params_at_least_one_of
+      [
+        :jobs_enabled,
+        :resolve_outdated_diff_discussions,
+        :ci_config_path,
+        :container_registry_enabled,
+        :default_branch,
+        :description,
+        :issues_enabled,
+        :lfs_enabled,
+        :merge_requests_enabled,
+        :merge_method,
+        :name,
+        :only_allow_merge_if_all_discussions_are_resolved,
+        :only_allow_merge_if_pipeline_succeeds,
+        :path,
+        :printing_merge_request_link_enabled,
+        :public_builds,
+        :request_access_enabled,
+        :shared_runners_enabled,
+        :snippets_enabled,
+        :tag_list,
+        :visibility,
+        :wiki_enabled,
+        :avatar
+      ]
     end
 
     helpers do
@@ -252,39 +287,13 @@ module API
         success Entities::Project
       end
       params do
-        # CE
-        at_least_one_of_ce =
-          [
-            :jobs_enabled,
-            :resolve_outdated_diff_discussions,
-            :ci_config_path,
-            :container_registry_enabled,
-            :default_branch,
-            :description,
-            :issues_enabled,
-            :lfs_enabled,
-            :merge_requests_enabled,
-            :merge_method,
-            :name,
-            :only_allow_merge_if_all_discussions_are_resolved,
-            :only_allow_merge_if_pipeline_succeeds,
-            :path,
-            :printing_merge_request_link_enabled,
-            :public_builds,
-            :request_access_enabled,
-            :shared_runners_enabled,
-            :snippets_enabled,
-            :tag_list,
-            :visibility,
-            :wiki_enabled,
-            :avatar
-          ]
         optional :name, type: String, desc: 'The name of the project'
         optional :default_branch, type: String, desc: 'The default branch of the project'
         optional :path, type: String, desc: 'The path of the repository'
 
         use :optional_project_params
-        at_least_one_of(*at_least_one_of_ce)
+
+        at_least_one_of(*::API::Projects.update_params_at_least_one_of)
       end
       put ':id' do
         authorize_admin_project
@@ -294,6 +303,8 @@ module API
 
         attrs = translate_params_for_compatibility(attrs)
 
+        verify_update_project_attrs!(user_project, attrs)
+
         result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute
 
         if result[:status] == :success

lib/api/runner.rb

@@ -109,7 +109,7 @@ module API
         if result.valid?
           if result.build
             Gitlab::Metrics.add_event(:build_found)
-            present result.build, with: Entities::JobRequest::Response
+            present Ci::BuildRunnerPresenter.new(result.build), with: Entities::JobRequest::Response
           else
             Gitlab::Metrics.add_event(:build_not_found)
             header 'X-GitLab-Last-Update', new_update
@@ -231,6 +231,10 @@ module API
         requires :id, type: Integer, desc: %q(Job's ID)
         optional :token, type: String, desc: %q(Job's authentication token)
         optional :expire_in, type: String, desc: %q(Specify when artifacts should expire)
+        optional :artifact_type, type: String, desc: %q(The type of artifact),
+                                 default: 'archive', values: Ci::JobArtifact.file_types.keys
+        optional :artifact_format, type: String, desc: %q(The format of artifact),
+                                   default: 'zip', values: Ci::JobArtifact.file_formats.keys
         optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse))
         optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse))
         optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse))
@@ -254,29 +258,29 @@ module API
         bad_request!('Missing artifacts file!') unless artifacts
         file_to_large! unless artifacts.size < max_artifacts_size
 
-        bad_request!("Already uploaded") if job.job_artifacts_archive
-
         expire_in = params['expire_in'] ||
           Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in
 
-        job.build_job_artifacts_archive(
+        job.job_artifacts.build(
           project: job.project,
           file: artifacts,
-          file_type: :archive,
+          file_type: params['artifact_type'],
+          file_format: params['artifact_format'],
           file_sha256: artifacts.sha256,
           expire_in: expire_in)
 
         if metadata
-          job.build_job_artifacts_metadata(
+          job.job_artifacts.build(
             project: job.project,
             file: metadata,
             file_type: :metadata,
+            file_format: :gzip,
             file_sha256: metadata.sha256,
             expire_in: expire_in)
         end
 
         if job.update(artifacts_expire_in: expire_in)
-          present job, with: Entities::JobRequest::Response
+          present Ci::BuildRunnerPresenter.new(job), with: Entities::JobRequest::Response
         else
           render_validation_error!(job)
         end

lib/gitlab/auth/activity.rb

+module Gitlab
+  module Auth
+    ##
+    # Metrics and logging for user authentication activity.
+    #
+    class Activity
+      extend Gitlab::Utils::StrongMemoize
+
+      COUNTERS = {
+        user_authenticated: 'Counter of successful authentication events',
+        user_unauthenticated: 'Counter of authentication failures',
+        user_not_found: 'Counter of failed log-ins when user is unknown',
+        user_password_invalid: 'Counter of failed log-ins with invalid password',
+        user_session_override: 'Counter of manual log-ins and sessions overrides',
+        user_session_destroyed: 'Counter of user sessions being destroyed',
+        user_two_factor_authenticated: 'Counter of two factor authentications',
+        user_sessionless_authentication: 'Counter of sessionless authentications',
+        user_blocked: 'Counter of sign in attempts when user is blocked'
+      }.freeze
+
+      def initialize(user, opts)
+        @user = user
+        @opts = opts
+      end
+
+      def user_authentication_failed!
+        self.class.user_unauthenticated_counter_increment!
+
+        case @opts[:message]
+        when :not_found_in_database
+          self.class.user_not_found_counter_increment!
+        when :invalid
+          self.class.user_password_invalid_counter_increment!
+        end
+
+        self.class.user_blocked_counter_increment! if @user&.blocked?
+      end
+
+      def user_authenticated!
+        self.class.user_authenticated_counter_increment!
+      end
+
+      def user_session_override!
+        self.class.user_session_override_counter_increment!
+
+        case @opts[:message]
+        when :two_factor_authenticated
+          self.class.user_two_factor_authenticated_counter_increment!
+        when :sessionless_sign_in
+          self.class.user_sessionless_authentication_counter_increment!
+        end
+      end
+
+      def user_session_destroyed!
+        self.class.user_session_destroyed_counter_increment!
+      end
+
+      def self.each_counter
+        COUNTERS.each_pair do |metric, description|
+          yield "#{metric}_counter", metric, description
+        end
+      end
+
+      each_counter do |counter, metric, description|
+        define_singleton_method(counter) do
+          strong_memoize(counter) do
+            Gitlab::Metrics.counter("gitlab_auth_#{metric}_total".to_sym, description)
+          end
+        end
+
+        define_singleton_method("#{counter}_increment!") do
+          public_send(counter).increment # rubocop:disable GitlabSecurity/PublicSend
+        end
+      end
+    end
+  end
+end