Merge branch 'ce-to-ee-2017-07-20' into 'master'

CE upstream: Thursday

Closes gitlab-com/support-forum#2246

See merge request !2486

.gitlab-ci.yml

 image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6"
 
-cache:
+.default-cache: &default-cache
   key: "ruby-233-with-yarn"
   paths:
     - vendor/ruby
     - .yarn-cache/
 
+.push-cache: &push-cache
+  cache:
+    <<: *default-cache
+    policy: push
+
+.pull-cache: &pull-cache
+  cache:
+    <<: *default-cache
+    policy: pull
+
 variables:
   MYSQL_ALLOW_EMPTY_PASSWORD: "1"
   ELASTIC_URL: "http://elasticsearch:9200"
@@ -27,11 +37,11 @@ before_script:
   - source scripts/prepare_build.sh
 
 stages:
-- build
-- prepare
-- test
-- post-test
-- pages
+  - build
+  - prepare
+  - test
+  - post-test
+  - pages
 
 # Predefined scopes
 .dedicated-runner: &dedicated-runner
@@ -44,10 +54,6 @@ stages:
     SETUP_DB: "false"
     USE_BUNDLE_INSTALL: "false"
     KNAPSACK_S3_BUCKET: "gitlab-ce-cache"
-  cache:
-    key: "knapsack"
-    paths:
-      - knapsack/
   artifacts:
     expire_in: 31d
     paths:
@@ -84,8 +90,9 @@ stages:
     - /(^docs[\/-].*|.*-docs$)/
 
 .rspec-knapsack: &rspec-knapsack
-  stage: test
   <<: *dedicated-runner
+  <<: *pull-cache
+  stage: test
   script:
     - JOB_NAME=( $CI_JOB_NAME )
     - export CI_NODE_INDEX=${JOB_NAME[-2]}
@@ -115,8 +122,9 @@ stages:
   <<: *except-docs
 
 .spinach-knapsack: &spinach-knapsack
-  stage: test
   <<: *dedicated-runner
+  <<: *pull-cache
+  stage: test
   script:
     - JOB_NAME=( $CI_JOB_NAME )
     - export CI_NODE_INDEX=${JOB_NAME[-2]}
@@ -163,6 +171,7 @@ build-package:
     USE_BUNDLE_INSTALL: "false"
     EE_PACKAGE: "true"
   stage: build
+  cache: {}
   when: manual
   script:
     - scripts/trigger-build
@@ -176,6 +185,11 @@ knapsack:
   <<: *dedicated-runner
   <<: *except-docs
   stage: prepare
+  cache:
+    key: knapsack
+    paths:
+      - knapsack/
+    policy: pull
   script:
     - mkdir -p knapsack/${CI_PROJECT_NAME}/
     - wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $KNAPSACK_RSPEC_SUITE_REPORT_PATH
@@ -188,6 +202,11 @@ update-knapsack:
   <<: *dedicated-runner
   <<: *only-canonical-masters
   stage: post-test
+  cache:
+    key: knapsack
+    paths:
+      - knapsack/
+    policy: push
   script:
     - retry gem install fog-aws mime-types
     - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec-pg_node_*.json
@@ -200,6 +219,8 @@ setup-test-env:
   <<: *dedicated-runner
   <<: *except-docs
   stage: prepare
+  cache:
+    <<: *default-cache
   script:
     - node --version
     - yarn install --pure-lockfile --cache-folder .yarn-cache
@@ -279,6 +300,7 @@ spinach-mysql 4 5: *spinach-knapsack-mysql
 
 # Static analysis jobs
 .ruby-static-analysis: &ruby-static-analysis
+  <<: *pull-cache
   variables:
     SIMPLECOV: "false"
     SETUP_DB: "false"
@@ -287,6 +309,7 @@ spinach-mysql 4 5: *spinach-knapsack-mysql
   <<: *ruby-static-analysis
   <<: *dedicated-runner
   <<: *except-docs
+  <<: *pull-cache
   stage: test
   script:
     - bundle exec rake $CI_JOB_NAME
@@ -303,9 +326,9 @@ static-analysis:
 # - Check validity of relative links
 # - Make sure cURL examples in API docs use the full switches
 docs lint:
+  <<: *dedicated-runner
   image: "registry.gitlab.com/gitlab-org/gitlab-build-images:nanoc-bootstrap-ruby-2.4-alpine"
   stage: test
-  <<: *dedicated-runner
   cache: {}
   dependencies: []
   before_script: []
@@ -328,9 +351,10 @@ downtime_check:
 
 # DB migration, rollback, and seed jobs
 .db-migrate-reset: &db-migrate-reset
-  stage: test
   <<: *dedicated-runner
   <<: *except-docs
+  <<: *pull-cache
+  stage: test
   script:
     - bundle exec rake db:migrate:reset
 
@@ -343,11 +367,12 @@ db:migrate:reset-mysql:
   <<: *use-mysql
 
 .migration-paths: &migration-paths
-  stage: test
   <<: *dedicated-runner
+  <<: *only-canonical-masters
+  <<: *pull-cache
+  stage: test
   variables:
     SETUP_DB: "false"
-  <<: *only-canonical-masters
   script:
     - git fetch origin v8.14.10-ee
     - git checkout -f FETCH_HEAD
@@ -368,9 +393,10 @@ migration:path-mysql:
   <<: *use-mysql
 
 .db-rollback: &db-rollback
-  stage: test
   <<: *dedicated-runner
   <<: *except-docs
+  <<: *pull-cache
+  stage: test
   script:
     - bundle exec rake db:rollback STEP=120
     - bundle exec rake db:migrate
@@ -384,9 +410,10 @@ db:rollback-mysql:
   <<: *use-mysql
 
 .db-seed_fu: &db-seed_fu
-  stage: test
   <<: *dedicated-runner
   <<: *except-docs
+  <<: *pull-cache
+  stage: test
   variables:
     SIZE: "1"
     SETUP_DB: "false"
@@ -411,9 +438,10 @@ db:seed_fu-mysql:
 
 # Frontend-related jobs
 gitlab:assets:compile:
-  stage: test
   <<: *dedicated-runner
   <<: *except-docs
+  <<: *pull-cache
+  stage: test
   dependencies: []
   variables:
     NODE_ENV: "production"
@@ -434,11 +462,12 @@ gitlab:assets:compile:
       - webpack-report/
 
 karma:
-  image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.3-golang-1.8-git-2.7-chrome-59.0-node-7.1-postgresql-9.6"
-  stage: test
   <<: *use-pg
   <<: *dedicated-runner
   <<: *except-docs
+  <<: *pull-cache
+  image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.3-golang-1.8-git-2.7-chrome-59.0-node-7.1-postgresql-9.6"
+  stage: test
   variables:
     BABEL_ENV: "coverage"
     CHROME_LOG_FILE: "chrome_debug.log"
@@ -456,6 +485,7 @@ karma:
 
 codeclimate:
   <<: *except-docs
+  <<: *pull-cache
   before_script: []
   image: docker:latest
   stage: test
@@ -471,10 +501,11 @@ codeclimate:
     paths: [codeclimate.json]
 
 coverage:
-  stage: post-test
-  services: []
   <<: *dedicated-runner
   <<: *except-docs
+  <<: *pull-cache
+  stage: post-test
+  services: []
   variables:
     SETUP_DB: "false"
     USE_BUNDLE_INSTALL: "true"
@@ -491,6 +522,7 @@ coverage:
 lint:javascript:report:
   <<: *dedicated-runner
   <<: *except-docs
+  <<: *pull-cache
   stage: post-test
   before_script: []
   script:
@@ -503,9 +535,10 @@ lint:javascript:report:
     - eslint-report.html
 
 pages:
+  <<: *dedicated-runner
+  <<: *pull-cache
   before_script: []
   stage: pages
-  <<: *dedicated-runner
   dependencies:
     - coverage
     - karma
@@ -529,6 +562,7 @@ pages:
 # rubygems.org in the future.
 cache gems:
   <<: *dedicated-runner
+  <<: *pull-cache
   only:
     - tags
   variables:
@@ -543,8 +577,9 @@ cache gems:
     - master@gitlab-org/gitlab-ee
 
 gitlab_git_test:
+  <<: *pull-cache
+  <<: *except-docs
   variables:
     SETUP_DB: "false"
   script:
     - spec/support/prepare-gitlab-git-test-for-commit --check-for-changes
-  <<: *except-docs

GITALY_SERVER_VERSION

-0.21.2
+0.22.0

Gemfile

@@ -37,11 +37,13 @@ gem 'omniauth-saml',          '~> 1.7.0'
 gem 'omniauth-shibboleth', '~> 1.2.0'
 gem 'omniauth-twitter', '~> 1.2.0'
 gem 'omniauth_crowd', '~> 2.2.0'
-gem 'gssapi', group: :kerberos
-gem 'omniauth-authentiq',     '~> 0.3.0'
+gem 'omniauth-authentiq', '~> 0.3.1'
 gem 'rack-oauth2', '~> 1.2.1'
 gem 'jwt', '~> 1.5.6'
 
+# Kerberos authentication. EE-only
+gem 'gssapi', group: :kerberos
+
 # Spam and anti-bot protection
 gem 'recaptcha', '~> 3.0', require: 'recaptcha/rails'
 gem 'akismet', '~> 2.0'
@@ -400,7 +402,7 @@ gem 'sys-filesystem', '~> 1.1.6'
 gem 'net-ntp'
 
 # Gitaly GRPC client
-gem 'gitaly', '~> 0.14.0'
+gem 'gitaly', '~> 0.17.0'
 
 gem 'toml-rb', '~> 0.3.15', require: false
 

Gemfile.lock

@@ -293,7 +293,7 @@ GEM
       po_to_json (>= 1.0.0)
       rails (>= 3.2.0)
     gherkin-ruby (0.3.2)
-    gitaly (0.14.0)
+    gitaly (0.17.0)
       google-protobuf (~> 3.1)
       grpc (~> 1.0)
     github-linguist (4.7.6)
@@ -517,7 +517,7 @@ GEM
       rack (>= 1.0, < 3)
     omniauth-auth0 (1.4.1)
       omniauth-oauth2 (~> 1.1)
-    omniauth-authentiq (0.3.0)
+    omniauth-authentiq (0.3.1)
       omniauth-oauth2 (~> 1.3, >= 1.3.1)
     omniauth-azure-oauth2 (0.0.6)
       jwt (~> 1.0)
@@ -1006,7 +1006,7 @@ DEPENDENCIES
   gettext (~> 3.2.2)
   gettext_i18n_rails (~> 1.8.0)
   gettext_i18n_rails_js (~> 1.2.0)
-  gitaly (~> 0.14.0)
+  gitaly (~> 0.17.0)
   github-linguist (~> 4.7.0)
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-license (~> 1.0)
@@ -1054,7 +1054,7 @@ DEPENDENCIES
   oj (~> 2.17.4)
   omniauth (~> 1.4.2)
   omniauth-auth0 (~> 1.4.1)
-  omniauth-authentiq (~> 0.3.0)
+  omniauth-authentiq (~> 0.3.1)
   omniauth-azure-oauth2 (~> 0.0.6)
   omniauth-cas3 (~> 1.1.2)
   omniauth-facebook (~> 4.0.0)

app/assets/javascripts/due_date_select.js

@@ -2,6 +2,8 @@
 /* global dateFormat */
 /* global Pikaday */
 
+import DateFix from './lib/utils/datefix';
+
 class DueDateSelect {
   constructor({ $dropdown, $loading } = {}) {
     const $dropdownParent = $dropdown.closest('.dropdown');
@@ -43,14 +45,13 @@ class DueDateSelect {
 
   initDatePicker() {
     const $dueDateInput = $(`input[name='${this.fieldName}']`);
-
+    const dateFix = DateFix.dashedFix($dueDateInput.val());
     const calendar = new Pikaday({
       field: $dueDateInput.get(0),
       theme: 'gitlab-theme',
       format: 'yyyy-mm-dd',
       onSelect: (dateText) => {
         const formattedDate = dateFormat(new Date(dateText), 'yyyy-mm-dd');
-
         $dueDateInput.val(formattedDate);
 
         if (this.$dropdown.hasClass('js-issue-boards-due-date')) {
@@ -62,7 +63,7 @@ class DueDateSelect {
       }
     });
 
-    calendar.setDate(new Date($dueDateInput.val()));
+    calendar.setDate(dateFix);
     this.$datePicker.append(calendar.el);
     this.$datePicker.data('pikaday', calendar);
   }
@@ -168,6 +169,7 @@ class DueDateSelectors {
   initMilestoneDatePicker() {
     $('.datepicker').each(function() {
       const $datePicker = $(this);
+      const dateFix = DateFix.dashedFix($datePicker.val());
       const calendar = new Pikaday({
         field: $datePicker.get(0),
         theme: 'gitlab-theme animate-picker',
@@ -177,7 +179,8 @@ class DueDateSelectors {
           $datePicker.val(dateFormat(new Date(dateText), 'yyyy-mm-dd'));
         }
       });
-      calendar.setDate(new Date($datePicker.val()));
+
+      calendar.setDate(dateFix);
 
       $datePicker.data('pikaday', calendar);
     });

app/assets/javascripts/layout_nav.js

 /* eslint-disable func-names, space-before-function-paren, no-var, prefer-arrow-callback, no-unused-vars, one-var, one-var-declaration-per-line, vars-on-top, max-len */
 import _ from 'underscore';
+import Cookies from 'js-cookie';
+import NewNavSidebar from './new_sidebar';
 
 (function() {
   var hideEndFade;
@@ -53,6 +55,11 @@ import _ from 'underscore';
   }
 
   $(() => {
+    if (Cookies.get('new_nav') === 'true') {
+      const newNavSidebar = new NewNavSidebar();
+      newNavSidebar.bindEvents();
+    }
+
     $(window).on('scroll', _.throttle(applyScrollNavClass, 100));
   });
 }).call(window);

app/assets/javascripts/lib/utils/datefix.js

+const DateFix = {
+  dashedFix(val) {
+    const [y, m, d] = val.split('-');
+    return new Date(y, m - 1, d);
+  },
+};
+
+export default DateFix;

app/assets/javascripts/new_sidebar.js

+export default class NewNavSidebar {
+  constructor() {
+    this.initDomElements();
+  }
+
+  initDomElements() {
+    this.$sidebar = $('.nav-sidebar');
+    this.$overlay = $('.mobile-overlay');
+    this.$openSidebar = $('.toggle-mobile-nav');
+    this.$closeSidebar = $('.close-nav-button');
+  }
+
+  bindEvents() {
+    this.$openSidebar.on('click', () => this.toggleSidebarNav(true));
+    this.$closeSidebar.on('click', () => this.toggleSidebarNav(false));
+    this.$overlay.on('click', () => this.toggleSidebarNav(false));
+  }
+
+  toggleSidebarNav(show) {
+    this.$sidebar.toggleClass('nav-sidebar-expanded', show);
+    this.$overlay.toggleClass('mobile-nav-open', show);
+  }
+}

app/assets/stylesheets/framework/typography.scss

@@ -116,9 +116,12 @@
 
   blockquote p {
     color: $gl-grayish-blue !important;
-    margin: 0;
     font-size: inherit;
     line-height: 1.5;
+
+    &:last-child {
+      margin: 0;
+    }
   }
 
   p {

app/assets/stylesheets/new_nav.scss

@@ -275,8 +275,6 @@ header.navbar-gitlab-new {
 .breadcrumbs {
   display: flex;
   min-height: 60px;
-  padding-top: $gl-padding-top;
-  padding-bottom: $gl-padding-top;
   color: $gl-text-color;
   border-bottom: 1px solid $border-color;
 
@@ -300,6 +298,7 @@ header.navbar-gitlab-new {
   display: flex;
   width: 100%;
   position: relative;
+  align-items: center;
 
   .dropdown-menu-projects {
     margin-top: -$gl-padding;

app/assets/stylesheets/new_sidebar.scss

@@ -26,6 +26,9 @@ $new-sidebar-width: 220px;
 }
 
 .context-header {
+  position: relative;
+
+  a {
     border-bottom: 1px solid $border-color;
     font-weight: 600;
     display: flex;
@@ -33,9 +36,8 @@ $new-sidebar-width: 220px;
     padding: 10px 16px 10px 10px;
     color: $gl-text-color;
 
-  .avatar-container {
-    flex: 0 0 40px;
-    background-color: $white-light;
+    @media (max-width: $screen-xs-max) {
+      padding-right: 30px;
     }
 
     &:hover {
@@ -55,12 +57,44 @@ $new-sidebar-width: 220px;
         }
       }
     }
+  }
+
+  .avatar-container {
+    flex: 0 0 40px;
+    background-color: $white-light;
+  }
 
   .project-title,
   .group-title {
     overflow: hidden;
     text-overflow: ellipsis;
   }
+
+
+  &:hover {
+    .close-nav-button {
+      color: $white-light;
+    }
+  }
+
+  .close-nav-button {
+    display: none;
+    position: absolute;
+    top: 0;
+    right: 0;
+    height: 100%;
+    background-color: transparent;
+    border: 0;
+    padding: 0 10px;
+
+    @media (max-width: $screen-xs-max) {
+      display: block;
+    }
+
+    &:hover {
+      color: $gl-text-color;
+    }
+  }
 }
 
 .settings-avatar {
@@ -79,7 +113,7 @@ $new-sidebar-width: 220px;
   position: fixed;
   z-index: 400;
   width: $new-sidebar-width;
-  transition: width $sidebar-transition-duration;
+  transition: left $sidebar-transition-duration;
   top: 50px;
   bottom: 0;
   left: 0;
@@ -87,6 +121,10 @@ $new-sidebar-width: 220px;
   background-color: $gray-normal;
   box-shadow: inset -2px 0 0 $border-color;
 
+  &.nav-sidebar-expanded {
+    left: 0;
+  }
+
   a {
     transition: none;
     text-decoration: none;
@@ -117,7 +155,7 @@ $new-sidebar-width: 220px;
   }
 
   @media (max-width: $screen-xs-max) {
-    width: 0;
+    left: (-$new-sidebar-width);
   }
 }
 
@@ -183,6 +221,38 @@ $new-sidebar-width: 220px;
   }
 }
 
+.toggle-mobile-nav {
+  display: none;
+  background-color: transparent;
+  border: 0;
+  padding: 6px 16px;
+  margin: 0 16px 0 -15px;
+  height: 46px;
+  border-right: 1px solid $gl-text-color-quaternary;
+
+  i {
+    font-size: 20px;
+    color: $gl-text-color-secondary;
+  }
+
+  @media (max-width: $screen-xs-max) {
+    display: inline-block;
+  }
+}
+
+.mobile-overlay {
+  display: none;
+
+  &.mobile-nav-open {
+    display: block;
+    position: fixed;
+    background-color: $black-transparent;
+    height: 100%;
+    width: 100%;
+    z-index: 300;
+  }
+}
+
 
 // Make issue boards full-height now that sub-nav is gone
 

app/assets/stylesheets/pages/profile.scss

@@ -376,3 +376,18 @@ table.u2f-registrations {
     }
   }
 }
+
+.nav-wip {
+  border: 1px solid $blue-500;
+  background: $blue-25;
+  padding: $gl-padding;
+  margin-bottom: $gl-padding;
+
+  a {
+    color: $blue-500;
+  }
+
+  p:last-child {
+    margin-bottom: 0;
+  }
+}

app/controllers/admin/hook_logs_controller.rb

@@ -10,9 +10,9 @@ class Admin::HookLogsController < Admin::ApplicationController
   end
 
   def retry
-    status, message = hook.execute(hook_log.request_data, hook_log.trigger)
+    result = hook.execute(hook_log.request_data, hook_log.trigger)
 
-    set_hook_execution_notice(status, message)
+    set_hook_execution_notice(result)
 
     redirect_to edit_admin_hook_path(@hook)
   end

app/controllers/admin/hooks_controller.rb

@@ -38,9 +38,9 @@ class Admin::HooksController < Admin::ApplicationController
   end
 
   def test
-    status, message = hook.execute(sample_hook_data, 'system_hooks')
+    result = TestHooks::SystemService.new(hook, current_user, params[:trigger]).execute
 
-    set_hook_execution_notice(status, message)
+    set_hook_execution_notice(result)
 
     redirect_back_or_default
   end
@@ -66,15 +66,4 @@ class Admin::HooksController < Admin::ApplicationController
       :url
     )
   end
-
-  def sample_hook_data
-    {
-      event_name: "project_create",
-      name: "Ruby",
-      path: "ruby",
-      project_id: 1,
-      owner_name: "Someone",
-      owner_email: "example@gitlabhq.com"
-    }
-  end
 end

app/controllers/concerns/hooks_execution.rb

@@ -3,11 +3,14 @@ module HooksExecution
 
   private
 
-  def set_hook_execution_notice(status, message)
-    if status && status >= 200 && status < 400
-      flash[:notice] = "Hook executed successfully: HTTP #{status}"
-    elsif status
-      flash[:alert] = "Hook executed successfully but returned HTTP #{status} #{message}"
+  def set_hook_execution_notice(result)
+    http_status = result[:http_status]
+    message = result[:message]
+
+    if http_status && http_status >= 200 && http_status < 400
+      flash[:notice] = "Hook executed successfully: HTTP #{http_status}"
+    elsif http_status
+      flash[:alert] = "Hook executed successfully but returned HTTP #{http_status} #{message}"
     else
       flash[:alert] = "Hook execution failed: #{message}"
     end

app/controllers/groups/hooks_controller.rb

 class Groups::HooksController < Groups::ApplicationController
+  include HooksExecution
+
   # Authorize
   before_action :group
   before_action :authorize_admin_group!
@@ -27,13 +29,11 @@ class Groups::HooksController < Groups::ApplicationController
 
   def test
     if @group.first_non_empty_project
-      status, message = TestHookService.new.execute(hook, current_user)
+      service = TestHooks::ProjectService.new(hook, current_user, 'push_events')
+      service.project = @group.first_non_empty_project
+      result = service.execute
 
-      if status
-        flash[:notice] = 'Hook successfully executed.'
-      else
-        flash[:alert] = "Hook execution failed: #{message}"
-      end
+      set_hook_execution_notice(result)
     else
       flash[:alert] = 'Hook execution failed. Ensure the group has a project with commits.'
     end

app/controllers/projects/hook_logs_controller.rb

@@ -14,9 +14,9 @@ class Projects::HookLogsController < Projects::ApplicationController
   end
 
   def retry
-    status, message = hook.execute(hook_log.request_data, hook_log.trigger)
+    result = hook.execute(hook_log.request_data, hook_log.trigger)
 
-    set_hook_execution_notice(status, message)
+    set_hook_execution_notice(result)
 
     redirect_to edit_project_hook_path(@project, @hook)
   end

app/controllers/projects/hooks_controller.rb

@@ -9,6 +9,10 @@ class Projects::HooksController < Projects::ApplicationController
 
   layout "project_settings"
 
+  def index
+    redirect_to project_settings_integrations_path(@project)
+  end
+
   def create
     @hook = @project.hooks.new(hook_params)
     @hook.save
@@ -33,13 +37,9 @@ class Projects::HooksController < Projects::ApplicationController
   end
 
   def test
-    if !@project.empty_repo?
-      status, message = TestHookService.new.execute(hook, current_user)
+    result = TestHooks::ProjectService.new(hook, current_user, params[:trigger]).execute
 
-      set_hook_execution_notice(status, message)
-    else
-      flash[:alert] = 'Hook execution failed. Ensure the project has commits.'
-    end
+    set_hook_execution_notice(result)
 
     redirect_back_or_default(default: { action: 'index' })
   end

app/helpers/hooks_helper.rb

+module HooksHelper
+  def link_to_test_hook(hook, trigger)
+    path = case hook
+           when ProjectHook
+             project = hook.project
+             test_project_hook_path(project, hook, trigger: trigger)
+           when SystemHook
+             test_admin_hook_path(hook, trigger: trigger)
+           end
+
+    trigger_human_name = trigger.to_s.tr('_', ' ').camelize
+
+    link_to path, rel: 'nofollow' do
+      content_tag(:span, trigger_human_name)
+    end
+  end
+end

app/models/ci/build.rb

@@ -100,6 +100,14 @@ module Ci
           BuildSuccessWorker.perform_async(id)
         end
       end
+
+      before_transition any => [:failed] do |build|
+        next if build.retries_max.zero?
+
+        if build.retries_count < build.retries_max
+          Ci::Build.retry(build, build.user)
+        end
+      end
     end
 
     def detailed_status(current_user)
@@ -134,6 +142,14 @@ module Ci
       success? || failed? || canceled?
     end
 
+    def retries_count
+      pipeline.builds.retried.where(name: self.name).count
+    end
+
+    def retries_max
+      self.options.fetch(:retry, 0).to_i
+    end
+
     def latest?
       !retried?
     end

app/models/concerns/editable.rb

@@ -4,4 +4,8 @@ module Editable
   def is_edited?
     last_edited_at.present? && last_edited_at != created_at
   end
+
+  def last_edited_by
+    super || User.ghost
+  end
 end

app/models/hooks/group_hook.rb

@@ -4,4 +4,7 @@ class GroupHook < ProjectHook
   self.singular_route_key = :hook
 
   belongs_to :group
+
+  clear_validators!
+  validates :url, presence: true, url: true
 end

app/models/hooks/project_hook.rb

@@ -3,13 +3,22 @@ class ProjectHook < WebHook
 
   self.singular_route_key = :hook
 
-  belongs_to :project
+  TRIGGERS = {
+    push_hooks:               :push_events,
+    tag_push_hooks:           :tag_push_events,
+    issue_hooks:              :issues_events,
+    confidential_issue_hooks: :confidential_issues_events,
+    note_hooks:               :note_events,
+    merge_request_hooks:      :merge_requests_events,
+    job_hooks:                :job_events,
+    pipeline_hooks:           :pipeline_events,
+    wiki_page_hooks:          :wiki_page_events
+  }.freeze
+
+  TRIGGERS.each do |trigger, event|
+    scope trigger, -> { where(event => true) }
+  end
 
-  scope :issue_hooks, -> { where(issues_events: true) }
-  scope :confidential_issue_hooks, -> { where(confidential_issues_events: true) }
-  scope :note_hooks, -> { where(note_events: true) }
-  scope :merge_request_hooks, -> { where(merge_requests_events: true) }
-  scope :job_hooks, -> { where(job_events: true) }
-  scope :pipeline_hooks, -> { where(pipeline_events: true) }
-  scope :wiki_page_hooks, ->  { where(wiki_page_events: true) }
+  belongs_to :project
+  validates :project, presence: true
 end

app/models/hooks/service_hook.rb

 class ServiceHook < WebHook
   belongs_to :service
+  validates :service, presence: true
 
   def execute(data, hook_name = 'service_hook')
     WebHookService.new(self, data, hook_name).execute

app/models/hooks/system_hook.rb

 class SystemHook < WebHook
-  scope :repository_update_hooks, ->  { where(repository_update_events: true) }
+  TRIGGERS = {
+    repository_update_hooks: :repository_update_events,
+    push_hooks:              :push_events,
+    tag_push_hooks:          :tag_push_events
+  }.freeze
+
+  TRIGGERS.each do |trigger, event|
+    scope trigger, -> { where(event => true) }
+  end
 
   default_value_for :push_events, false
   default_value_for :repository_update_events, true

app/models/hooks/web_hook.rb

 class WebHook < ActiveRecord::Base
   include Sortable
 
-  default_value_for :push_events, true
-  default_value_for :issues_events, false
-  default_value_for :confidential_issues_events, false
-  default_value_for :note_events, false
-  default_value_for :merge_requests_events, false
-  default_value_for :tag_push_events, false
-  default_value_for :job_events, false
-  default_value_for :pipeline_events, false
-  default_value_for :repository_update_events, false
-  default_value_for :enable_ssl_verification, true
-
   has_many :web_hook_logs, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
 
-  scope :push_hooks, -> { where(push_events: true) }
-  scope :tag_push_hooks, -> { where(tag_push_events: true) }
-
   validates :url, presence: true, url: true
 
   def execute(data, hook_name)

app/models/project_services/jenkins_service.rb

@@ -35,13 +35,13 @@ class JenkinsService < CiService
 
   def test(data)
     begin
-      code, message = execute(data)
-      return { success: false, result: message } if code != 200
+      result = execute(data)
+      return { success: false, result: result[:message] } if result[:http_status] != 200
     rescue StandardError => error
       return { success: false, result: error }
     end
 
-    { success: true, result: message }
+    { success: true, result: result[:message] }
   end
 
   def hook_url

app/models/user.rb

@@ -404,9 +404,11 @@ class User < ActiveRecord::Base
     # Return (create if necessary) the ghost user. The ghost user
     # owns records previously belonging to deleted users.
     def ghost
-      unique_internal(where(ghost: true), 'ghost', 'ghost%s@example.com') do |u|
+      email = 'ghost%s@example.com'
+      unique_internal(where(ghost: true), 'ghost', email) do |u|
         u.bio = 'This is a "Ghost User", created to hold all issues authored by users that have since been deleted. This user cannot be removed.'
         u.name = 'Ghost User'
+        u.notification_email = email
       end
     end
   end

app/services/ci/create_pipeline_service.rb

@@ -145,7 +145,7 @@ module Ci
     end
 
     def pipeline_created_counter
-      @pipeline_created_counter ||= Gitlab::Metrics.counter(:pipelines_created_count, "Pipelines created count")
+      @pipeline_created_counter ||= Gitlab::Metrics.counter(:pipelines_created_total, "Counter of pipelines created")
     end
   end
 end

app/services/system_hooks_service.rb

@@ -4,7 +4,7 @@ class SystemHooksService
   end
 
   def execute_hooks(data, hooks_scope = :all)
-    SystemHook.send(hooks_scope).each do |hook|
+    SystemHook.public_send(hooks_scope).find_each do |hook|
       hook.async_execute(data, 'system_hooks')
     end
   end

app/services/test_hook_service.rb

-class TestHookService
-  def execute(hook, current_user)
-    data = Gitlab::DataBuilder::Push.build_sample(project(hook), current_user)
-    hook.execute(data, 'push_hooks')
-  end
-
-  private
-
-  def project(hook)
-    if hook.is_a? GroupHook
-      hook.group.first_non_empty_project
-    else
-      hook.project
-    end
-  end
-end

app/services/test_hooks/base_service.rb

+module TestHooks
+  class BaseService
+    attr_accessor :hook, :current_user, :trigger
+
+    def initialize(hook, current_user, trigger)
+      @hook = hook
+      @current_user = current_user
+      @trigger = trigger
+    end
+
+    def execute
+      trigger_data_method = "#{trigger}_data"
+
+      if !self.respond_to?(trigger_data_method, true) ||
+          !hook.class::TRIGGERS.value?(trigger.to_sym)
+
+        return error('Testing not available for this hook')
+      end
+
+      error_message = catch(:validation_error) do
+        sample_data = self.__send__(trigger_data_method)
+
+        return hook.execute(sample_data, trigger)
+      end
+
+      error(error_message)
+    end
+
+    private
+
+    def error(message, http_status = nil)
+      result = {
+        message: message,
+        status: :error
+      }
+
+      result[:http_status] = http_status if http_status
+      result
+    end
+  end
+end

app/services/test_hooks/project_service.rb

+module TestHooks
+  class ProjectService < TestHooks::BaseService
+    attr_writer :project
+
+    def project
+      @project ||= hook.project
+    end
+
+    private
+
+    def push_events_data
+      throw(:validation_error, 'Ensure the project has at least one commit.') if project.empty_repo?
+
+      Gitlab::DataBuilder::Push.build_sample(project, current_user)
+    end
+
+    alias_method :tag_push_events_data, :push_events_data
+
+    def note_events_data
+      note = project.notes.first
+      throw(:validation_error, 'Ensure the project has notes.') unless note.present?
+
+      Gitlab::DataBuilder::Note.build(note, current_user)
+    end
+
+    def issues_events_data
+      issue = project.issues.first
+      throw(:validation_error, 'Ensure the project has issues.') unless issue.present?
+
+      issue.to_hook_data(current_user)
+    end
+
+    alias_method :confidential_issues_events_data, :issues_events_data
+
+    def merge_requests_events_data
+      merge_request = project.merge_requests.first
+      throw(:validation_error, 'Ensure the project has merge requests.') unless merge_request.present?
+
+      merge_request.to_hook_data(current_user)
+    end
+
+    def job_events_data
+      build = project.builds.first
+      throw(:validation_error, 'Ensure the project has CI jobs.') unless build.present?
+
+      Gitlab::DataBuilder::Build.build(build)
+    end
+
+    def pipeline_events_data
+      pipeline = project.pipelines.first
+      throw(:validation_error, 'Ensure the project has CI pipelines.') unless pipeline.present?
+
+      Gitlab::DataBuilder::Pipeline.build(pipeline)
+    end
+
+    def wiki_page_events_data
+      page = project.wiki.pages.first
+      if !project.wiki_enabled? || page.blank?
+        throw(:validation_error, 'Ensure the wiki is enabled and has pages.')
+      end
+
+      Gitlab::DataBuilder::WikiPage.build(page, current_user, 'create')
+    end
+  end
+end

app/services/test_hooks/system_service.rb

+module TestHooks
+  class SystemService < TestHooks::BaseService
+    private
+
+    def project
+      @project ||= begin
+        project = Project.first
+
+        throw(:validation_error, 'Ensure that at least one project exists.') unless project
+
+        project
+      end
+    end
+
+    def push_events_data
+      if project.empty_repo?
+        throw(:validation_error, "Ensure project \"#{project.human_name}\" has commits.")
+      end
+
+      Gitlab::DataBuilder::Push.build_sample(project, current_user)
+    end
+
+    def tag_push_events_data
+      if project.repository.tags.empty?
+        throw(:validation_error, "Ensure project \"#{project.human_name}\" has tags.")
+      end
+
+      Gitlab::DataBuilder::Push.build_sample(project, current_user)
+    end
+
+    def repository_update_events_data
+      commit = project.commit
+      ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{project.default_branch}"
+
+      unless commit
+        throw(:validation_error, "Ensure project \"#{project.human_name}\" has commits.")
+      end
+
+      change = Gitlab::DataBuilder::Repository.single_change(
+        commit.parent_id || Gitlab::Git::BLANK_SHA,
+        commit.id,
+        ref
+      )
+
+      Gitlab::DataBuilder::Repository.update(project, current_user, [change], [ref])
+    end
+  end
+end

app/services/users/migrate_to_ghost_user_service.rb

@@ -50,10 +50,12 @@ module Users
 
     def migrate_issues
       user.issues.update_all(author_id: ghost_user.id)
+      Issue.where(last_edited_by_id: user.id).update_all(last_edited_by_id: ghost_user.id)
     end
 
     def migrate_merge_requests
       user.merge_requests.update_all(author_id: ghost_user.id)
+      MergeRequest.where(merge_user_id: user.id).update_all(merge_user_id: ghost_user.id)
     end
 
     def migrate_notes

app/services/web_hook_service.rb

@@ -39,7 +39,11 @@ class WebHookService
       execution_duration: Time.now - start_time
     )
 
-    [response.code, response.to_s]
+    {
+      status: :success,
+      http_status: response.code,
+      message: response.to_s
+    }
   rescue SocketError, OpenSSL::SSL::SSLError, Errno::ECONNRESET, Errno::ECONNREFUSED, Net::OpenTimeout => e
     log_execution(
       trigger: hook_name,
@@ -52,7 +56,10 @@ class WebHookService
 
     Rails.logger.error("WebHook Error => #{e}")
 
-    [nil, e.to_s]
+    {
+      status: :error,
+      message: e.to_s
+    }
   end
 
   def async_execute

app/services/wiki_pages/base_service.rb

@@ -2,24 +2,10 @@ module WikiPages
   class BaseService < ::BaseService
     prepend EE::WikiPages::BaseService
 
-    def hook_data(page, action)
-      hook_data = {
-        object_kind: page.class.name.underscore,
-        user: current_user.hook_attrs,
-        project: @project.hook_attrs,
-        wiki: @project.wiki.hook_attrs,
-        object_attributes: page.hook_attrs
-      }
-
-      page_url = Gitlab::UrlBuilder.build(page)
-      hook_data[:object_attributes].merge!(url: page_url, action: action)
-      hook_data
-    end
-
     private
 
     def execute_hooks(page, action = 'create')
-      page_data = hook_data(page, action)
+      page_data = Gitlab::DataBuilder::WikiPage.build(page, current_user, action)
       @project.execute_hooks(page_data, :wiki_page_hooks)
       @project.execute_services(page_data, :wiki_page_hooks)
     end

app/views/admin/hooks/edit.html.haml

@@ -12,7 +12,7 @@
   = render partial: 'form', locals: { form: f, hook: @hook }
   .form-actions
     = f.submit 'Save changes', class: 'btn btn-create'
-    = link_to 'Test hook', test_admin_hook_path(@hook), class: 'btn btn-default'
+    = render 'shared/web_hooks/test_button', triggers: SystemHook::TRIGGERS, hook: @hook
     = link_to 'Remove', admin_hook_path(@hook), method: :delete, class: 'btn btn-remove pull-right', data: { confirm: 'Are you sure?' }
 
 %hr

app/views/admin/hooks/index.html.haml

@@ -22,12 +22,12 @@
       - @hooks.each do |hook|
         %li
           .controls
-            = link_to 'Test hook', test_admin_hook_path(hook), class: 'btn btn-sm'
+            = render 'shared/web_hooks/test_button', triggers: SystemHook::TRIGGERS, hook: hook, button_class: 'btn-small'
             = link_to 'Edit', edit_admin_hook_path(hook), class: 'btn btn-sm'
             = link_to 'Remove', admin_hook_path(hook), data: { confirm: 'Are you sure?' }, method: :delete, class: 'btn btn-remove btn-sm'
           .monospace= hook.url
           %div
-            - %w(repository_update_events push_events tag_push_events issues_events note_events merge_requests_events job_events).each do |trigger|
-              - if hook.send(trigger)
-                %span.label.label-gray= trigger.titleize
+            - SystemHook::TRIGGERS.each_value do |event|
+              - if hook.public_send(event)
+                %span.label.label-gray= event.to_s.titleize
             %span.label.label-gray SSL Verification: #{hook.enable_ssl_verification ? 'enabled' : 'disabled'}

app/views/layouts/_page.html.haml

@@ -10,6 +10,8 @@
     - if content_for?(:sub_nav)
       = yield :sub_nav
   .content-wrapper{ class: "#{(layout_nav_class unless show_new_nav?)}" }
+    - if show_new_nav?
+      .mobile-overlay
     .alert-wrapper
       = render "layouts/broadcast"
       - if show_new_nav?

app/views/layouts/nav/_breadcrumbs.html.haml

@@ -3,6 +3,10 @@
 
 %nav.breadcrumbs{ role: "navigation" }
   .breadcrumbs-container{ class: [container_class, @content_class] }
+    - if defined?(@new_sidebar)
+      = button_tag class: 'toggle-mobile-nav', type: 'button' do
+        %span.sr-only Open sidebar
+        = icon ('bars')
     .breadcrumbs-links.js-title-container
       - unless hide_top_links
         .title

app/views/layouts/nav/_new_admin_sidebar.html.haml

 .nav-sidebar
-  = link_to admin_root_path, title: 'Admin Overview', class: 'context-header' do
+  .context-header
+    = link_to admin_root_path, title: 'Admin Overview' do
       .avatar-container.s40.settings-avatar
         = icon('wrench')
       .project-title Admin Area
+    = button_tag class: 'close-nav-button', type: 'button' do
+      %span.sr-only Close sidebar
+      = icon ('times')
   %ul.sidebar-top-level-items
     = nav_link(controller: %w(dashboard admin projects users groups jobs runners cohorts), html_options: {class: 'home'}) do
       = link_to admin_root_path, title: 'Overview', class: 'shortcuts-tree' do

app/views/layouts/nav/_new_group_sidebar.html.haml

 .nav-sidebar
-  = link_to group_path(@group), title: @group.name, class: 'context-header' do
+  .context-header
+    = link_to group_path(@group), title: @group.name do
       .avatar-container.s40.group-avatar
         = image_tag group_icon(@group), class: "avatar s40 avatar-tile"
       .group-title
         = @group.name
+    = button_tag class: 'close-nav-button', type: 'button' do
+      %span.sr-only Close sidebar
+      = icon ('times')
   %ul.sidebar-top-level-items
     = nav_link(path: ['groups#show', 'groups#activity', 'groups#subgroups'], html_options: { class: 'home' }) do
       = link_to group_path(@group), title: 'About group' do

app/views/layouts/nav/_new_profile_sidebar.html.haml

 .nav-sidebar
-  = link_to profile_path, title: 'Profile Settings', class: 'context-header' do
+  .context-header
+    = link_to profile_path, title: 'Profile Settings' do
       .avatar-container.s40.settings-avatar
         = icon('user')
       .project-title User Settings
+    = button_tag class: 'close-nav-button', type: 'button' do
+      %span.sr-only Close sidebar
+      = icon ('times')
   %ul.sidebar-top-level-items
     = nav_link(path: 'profiles#show', html_options: {class: 'home'}) do
       = link_to profile_path, title: 'Profile Settings' do

app/views/layouts/nav/_new_project_sidebar.html.haml

 .nav-sidebar
   - can_edit = can?(current_user, :admin_project, @project)
-  = link_to project_path(@project), title: @project.name, class: 'context-header' do
+  .context-header
+    = link_to project_path(@project), title: @project.name do
       .avatar-container.s40.project-avatar
         = project_icon(@project, alt: @project.name, class: 'avatar s40 avatar-tile')
       .project-title
         = @project.name
+    = button_tag class: 'close-nav-button', type: 'button' do
+      %span.sr-only Close sidebar
+      = icon ('times')
   %ul.sidebar-top-level-items
     = nav_link(path: ['projects#show', 'projects#activity', 'cycle_analytics#show'], html_options: { class: 'home' }) do
       = link_to project_path(@project), title: 'About project', class: 'shortcuts-project' do

app/views/profiles/preferences/show.html.haml

@@ -24,6 +24,12 @@
     %p
       This setting allows you to turn on or off the new upcoming navigation concept.
   .col-lg-8.syntax-theme
+    .nav-wip
+      %p
+        The new navigation is currently a work-in-progress concept and is currently only usable on wide-screens. There are a number of improvements that we are working on in order to further refine our navigation.
+      %p
+        %a{ href: 'https://gitlab.com/gitlab-org/gitlab-ce/issues/32794', target: 'blank' } Learn more
+        about the improvements that are coming soon!
     = label_tag do
       .preview= image_tag "old_nav.png"
       %input.js-experiment-feature-toggle{ type: "radio", value: "false", name: "new_nav", checked: !show_new_nav? }

app/views/projects/branches/index.html.haml

@@ -7,6 +7,7 @@
 
 %div{ class: container_class }
   .top-area.adjust
+    - if can?(current_user, :admin_project, @project)
       .nav-text
         Protected branches can be managed in
         = link_to 'project settings', project_protected_branches_path(@project)

app/views/projects/hooks/edit.html.haml

@@ -13,9 +13,10 @@
       = render partial: 'shared/web_hooks/form', locals: { form: f, hook: @hook }
 
       = f.submit 'Save changes', class: 'btn btn-create'
-      = link_to 'Test hook', test_project_hook_path(@project, @hook), class: 'btn btn-default'
+      = render 'shared/web_hooks/test_button', triggers: ProjectHook::TRIGGERS, hook: @hook
       = link_to 'Remove', project_hook_path(@project, @hook), method: :delete, class: 'btn btn-remove pull-right', data: { confirm: 'Are you sure?' }
 
 %hr
 
 = render partial: 'projects/hook_logs/index', locals: { hook: @hook, hook_logs: @hook_logs, project: @project }
+

app/views/projects/jobs/_header.html.haml

 - show_controls = local_assigns.fetch(:show_controls, true)
 - pipeline = @build.pipeline
 
-.content-block.build-header.top-area
+.content-block.build-header.top-area.page-content-header
   .header-content
     = render 'ci/status/badge', status: @build.detailed_status(current_user), link: false, title: @build.status_title
     %strong

app/views/projects/settings/integrations/_project_hook.html.haml

@@ -3,14 +3,14 @@
     .col-md-8.col-lg-7
       %strong.light-header= hook.url
       %div
-        - %w(push_events tag_push_events issues_events confidential_issues_events note_events merge_requests_events job_events pipeline_events wiki_page_events).each do |trigger|
-          - if hook.send(trigger)
-            %span.label.label-gray.deploy-project-label= trigger.titleize
+        - ProjectHook::TRIGGERS.each_value do |event|
+          - if hook.public_send(event)
+            %span.label.label-gray.deploy-project-label= event.to_s.titleize
     .col-md-4.col-lg-5.text-right-lg.prepend-top-5
       %span.append-right-10.inline
-        SSL Verification: #{hook.enable_ssl_verification ? "enabled" : "disabled"}
-      = link_to "Edit", edit_project_hook_path(@project, hook), class: "btn btn-sm"
-      = link_to "Test", test_project_hook_path(@project, hook), class: "btn btn-sm"
-      = link_to project_hook_path(@project, hook), data: { confirm: 'Are you sure?'}, method: :delete, class: "btn btn-transparent" do
+        SSL Verification: #{hook.enable_ssl_verification ? 'enabled' : 'disabled'}
+      = link_to 'Edit', edit_project_hook_path(@project, hook), class: 'btn btn-sm'
+      = render 'shared/web_hooks/test_button', triggers: ProjectHook::TRIGGERS, hook: hook, button_class: 'btn-small'
+      = link_to project_hook_path(@project, hook), data: { confirm: 'Are you sure?'}, method: :delete, class: 'btn btn-transparent' do
         %span.sr-only Remove
         = icon('trash')

app/views/shared/empty_states/_issues.html.haml

@@ -16,6 +16,7 @@
           Also, issues are searchable and filterable.
         - if project_select_button
           = render 'shared/new_project_item_select', path: 'issues/new', label: 'New issue'
+        - else
           = link_to 'New issue', button_path, class: 'btn btn-new', title: 'New issue', id: 'new_issue_link'
       - else
         .text-center

app/views/shared/web_hooks/_test_button.html.haml

+- triggers = local_assigns.fetch(:triggers)
+- button_class = local_assigns.fetch(:button_class, '')
+- hook = local_assigns.fetch(:hook)
+
+.hook-test-button.dropdown.inline
+  %button.btn{ 'data-toggle' => 'dropdown', class: button_class }
+    Test
+    = icon('caret-down')
+  %ul.dropdown-menu.dropdown-menu-align-right{ role: 'menu' }
+    - triggers.each_value do |event|
+      %li
+        = link_to_test_hook(hook, event)

changelogs/unreleased/23036-replace-dashboard-event-filters-spinach.yml

+---
+title: Replaces dashboard/event_filters.feature spinach with rspec
+merge_request: 12651
+author: Alexander Randa (@randaalex)

changelogs/unreleased/23036-replace-dashboard-spinach.yml

+---
+title: Replaces dashboard/dashboard.feature spinach with rspec
+merge_request: 12876
+author: Alexander Randa (@randaalex)

changelogs/unreleased/33770-respect-blockquote-line-breaks.yml

+---
+title: Respect blockquote line breaks in markdown
+merge_request:
+author:

changelogs/unreleased/34174-add-french-translations-of-commits-page.yml

+---
+title: Add French translations of Commits Page
+merge_request: 12409
+author: Huang Tao

changelogs/unreleased/34563-usage-ping-github.yml

+---
+title: Add GitHub imported projects count to usage data
+merge_request:
+author:

changelogs/unreleased/34930-fix-edited-by.yml

+---
+title: Use Ghost user for last_edited_by and merge_user when original user is deleted
+merge_request: 12933
+author:

changelogs/unreleased/35209-add-wip-info-new-nav-pref.yml

+---
+title: Add wip message to new navigation preference section
+merge_request:
+author:

changelogs/unreleased/35253-desc-protected-branches-for-non-member.yml

+---
+title: Hide description about protected branches to non-member
+merge_request: 12945
+author: Takuya Noguchi

changelogs/unreleased/5971-webhook-testing.yml

+---
+title: Allow testing any events for project hooks and system hooks
+merge_request: 11728
+author: Alexander Randa (@randaalex)

changelogs/unreleased/feature-gb-auto-retry-failed-ci-job.yml

+---
+title: Allow to configure automatic retry of a failed CI/CD job
+merge_request: 12909
+author:

changelogs/unreleased/fix-gb-fix-container-registry-tag-routing.yml

+---
+title: Fix docker tag reference routing constraints
+merge_request: 12961
+author:

config/gitlab.yml.example

@@ -478,13 +478,13 @@ production: &base
       #             service_validate_url: '/cas/p3/serviceValidate',
       #             logout_url: '/cas/logout'} }
       # - { name: 'authentiq',
-      #     # for client credentials (client ID and secret), go to https://www.authentiq.com/
+      #     # for client credentials (client ID and secret), go to https://www.authentiq.com/developers
       #     app_id: 'YOUR_CLIENT_ID',
       #     app_secret: 'YOUR_CLIENT_SECRET',
       #     args: {
       #             scope: 'aq:name email~rs address aq:push'
-      #             # redirect_uri parameter is optional except when 'gitlab.host' in this file is set to 'localhost'
-      #             # redirect_uri: 'YOUR_REDIRECT_URI'
+      #             # callback_url parameter is optional except when 'gitlab.host' in this file is set to 'localhost'
+      #             # callback_url: 'YOUR_CALLBACK_URL'
       #           }
       #   }
       # - { name: 'github',

config/routes/project.rb

@@ -307,7 +307,7 @@ constraints(ProjectUrlConstrainer.new) do
       namespace :registry do
         resources :repository, only: [] do
           resources :tags, only: [:destroy],
-                           constraints: { id: Gitlab::Regex.container_registry_reference_regex }
+                           constraints: { id: Gitlab::Regex.container_registry_tag_regex }
         end
       end
 

db/migrate/20170710083355_clean_stage_id_reference_migration.rb

+class CleanStageIdReferenceMigration < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  ##
+  # `MigrateStageIdReferenceInBackground` background migration cleanup.
+  #
+  def up
+    Gitlab::BackgroundMigration.steal('MigrateBuildStageIdReference')
+  end
+
+  def down
+    # noop
+  end
+end

doc/administration/auth/authentiq.md

@@ -32,7 +32,7 @@ Authentiq will generate a Client ID and the accompanying Client Secret for you t
         "app_id" => "YOUR_CLIENT_ID",
         "app_secret" => "YOUR_CLIENT_SECRET",
         "args" => { 
-               scope: 'aq:name email~rs aq:push'
+               "scope": 'aq:name email~rs address aq:push'
          }
       }
     ]
@@ -45,21 +45,20 @@ Authentiq will generate a Client ID and the accompanying Client Secret for you t
         app_id: 'YOUR_CLIENT_ID',
         app_secret: 'YOUR_CLIENT_SECRET',
         args: {
-               scope: 'aq:name email~rs aq:push'
+               scope: 'aq:name email~rs address aq:push'
               }
       }
     ```
     
     
 5. The `scope` is set to request the user's name, email (required and signed), and permission to send push notifications to sign in on subsequent visits.
-See [OmniAuth Authentiq strategy](https://github.com/AuthentiqID/omniauth-authentiq#scopes-and-redirect-uri-configuration) for more information on scopes and modifiers.
+See [OmniAuth Authentiq strategy](https://github.com/AuthentiqID/omniauth-authentiq/wiki/Scopes,-callback-url-configuration-and-responses) for more information on scopes and modifiers.
 
 6. Change `YOUR_CLIENT_ID` and `YOUR_CLIENT_SECRET` to the Client credentials you received in step 1.
 
 7. Save the configuration file.
 
-8. [Reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart GitLab](../restart_gitlab.md#installations-from-source)
-   for the changes to take effect if you installed GitLab via Omnibus or from source respectively.
+8. [Reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect if you installed GitLab via Omnibus or from source respectively.
 
 On the sign in page there should now be an Authentiq icon below the regular sign in form. Click the icon to begin the authentication process. 
 

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -26,24 +26,25 @@ server, because the embedded server configuration is overwritten once every
 
 In this experimental phase, only a few metrics are available:
 
-| Metric                            | Type      | Description |
-| --------------------------------- | --------- | ----------- |
-| db_ping_timeout                   | Gauge     | Whether or not the last database ping timed out |
-| db_ping_success                   | Gauge     | Whether or not the last database ping succeeded |
-| db_ping_latency_seconds           | Gauge     | Round trip time of the database ping |
-| filesystem_access_latency_seconds | Gauge     | Latency in accessing a specific filesystem |
-| filesystem_accessible             | Gauge     | Whether or not a specific filesystem is accessible |
-| filesystem_write_latency_seconds  | Gauge     | Write latency of a specific filesystem |
-| filesystem_writable               | Gauge     | Whether or not the filesystem is writable |
-| filesystem_read_latency_seconds   | Gauge     | Read latency of a specific filesystem |
-| filesystem_readable               | Gauge     | Whether or not the filesystem is readable |
-| http_requests_total               | Counter   | Rack request count |
-| http_request_duration_seconds     | Histogram | HTTP response time from rack middleware |
-| rack_uncaught_errors_total        | Counter   | Rack connections handling uncaught errors count |
-| redis_ping_timeout                | Gauge     | Whether or not the last redis ping timed out |
-| redis_ping_success                | Gauge     | Whether or not the last redis ping succeeded |
-| redis_ping_latency_seconds        | Gauge     | Round trip time of the redis ping |
-| user_session_logins_total         | Counter   | Counter of how many users have logged in |
+| Metric                            | Type      | Since | Description |
+|:--------------------------------- |:--------- |:----- |:----------- |
+| db_ping_timeout                   | Gauge     | 9.4   | Whether or not the last database ping timed out |
+| db_ping_success                   | Gauge     | 9.4   | Whether or not the last database ping succeeded |
+| db_ping_latency_seconds           | Gauge     | 9.4   | Round trip time of the database ping |
+| filesystem_access_latency_seconds | Gauge     | 9.4   | Latency in accessing a specific filesystem |
+| filesystem_accessible             | Gauge     | 9.4   | Whether or not a specific filesystem is accessible |
+| filesystem_write_latency_seconds  | Gauge     | 9.4   | Write latency of a specific filesystem |
+| filesystem_writable               | Gauge     | 9.4   | Whether or not the filesystem is writable |
+| filesystem_read_latency_seconds   | Gauge     | 9.4   | Read latency of a specific filesystem |
+| filesystem_readable               | Gauge     | 9.4   | Whether or not the filesystem is readable |
+| http_requests_total               | Counter   | 9.4   | Rack request count |
+| http_request_duration_seconds     | Histogram | 9.4   | HTTP response time from rack middleware |
+| pipelines_created_total           | Counter   | 9.4   | Counter of pipelines created |
+| rack_uncaught_errors_total        | Counter   | 9.4   | Rack connections handling uncaught errors count |
+| redis_ping_timeout                | Gauge     | 9.4   | Whether or not the last redis ping timed out |
+| redis_ping_success                | Gauge     | 9.4   | Whether or not the last redis ping succeeded |
+| redis_ping_latency_seconds        | Gauge     | 9.4   | Round trip time of the redis ping |
+| user_session_logins_total         | Counter   | 9.4   | Counter of how many users have logged in |
 
 [← Back to the main Prometheus page](index.md)
 

doc/api/projects.md

@@ -1267,17 +1267,21 @@ endpoint can be accessed without authentication if the project is publicly
 accessible.
 
 ```
-GET /projects/search/:query
+GET /projects
 ```
 
 Parameters:
 
 | Attribute | Type | Required | Description |
 | --------- | ---- | -------- | ----------- |
-| `query` | string | yes | A string contained in the project name |
+| `search` | string | yes | A string contained in the project name |
 | `order_by` | string | no | Return requests ordered by `id`, `name`, `created_at` or `last_activity_at` fields |
 | `sort` | string | no | Return requests sorted in `asc` or `desc` order |
 
+```bash
+curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/projects?search=test
+```
+
 ## Start the Housekeeping task for a Project
 
 >**Note:** This feature was introduced in GitLab 9.0

doc/ci/yaml/README.md

@@ -395,6 +395,7 @@ job_name:
 | after_script  | no       | Override a set of commands that are executed after job |
 | environment   | no       | Defines a name of environment to which deployment is done by this job |
 | coverage      | no       | Define code coverage settings for a given job |
+| retry         | no       | Define how many times a job can be auto-retried in case of a failure |
 
 ### script
 
@@ -1129,9 +1130,33 @@ A simple example:
 
 ```yaml
 job1:
+  script: rspec
   coverage: '/Code coverage: \d+\.\d+/'
 ```
 
+### retry
+
+**Notes:**
+- [Introduced][ce-3442] in GitLab 9.5.
+
+`retry` allows you to configure how many times a job is going to be retried in
+case of a failure.
+
+When a job fails, and has `retry` configured it is going to be processed again
+up to the amount of times specified by the `retry` keyword.
+
+If `retry` is set to 2, and a job succeeds in a second run (first retry), it won't be retried
+again. `retry` value has to be a positive integer, equal or larger than 0, but
+lower or equal to 2 (two retries maximum, three runs in total).
+
+A simple example:
+
+```yaml
+test:
+  script: rspec
+  retry: 2
+```
+
 ## Git Strategy
 
 > Introduced in GitLab 8.9 as an experimental feature.  May change or be removed
@@ -1506,3 +1531,4 @@ CI with various languages.
 [variables]: ../variables/README.md
 [ce-7983]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7983
 [ce-7447]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7447
+[ce-3442]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3442

doc/user/project/integrations/webhooks.md

@@ -1039,6 +1039,13 @@ X-Gitlab-Event: Build Hook
 }
 ```
 
+## Testing webhooks
+
+You can trigger the webhook manually. Sample data from the project will be used.Sample data will take from the project. 
+> For example: for triggering `Push Events` your project should have at least one commit.
+
+![Webhook testing](img/webhook_testing.png)
+
 ## Troubleshoot webhooks
 
 Gitlab stores each perform of the webhook.
@@ -1081,7 +1088,7 @@ Pick an unused port (e.g. 8000) and start the script: `ruby print_http_body.rb
 8000`.  Then add your server as a webhook receiver in GitLab as
 `http://my.host:8000/`.
 
-When you press 'Test Hook' in GitLab, you should see something like this in the
+When you press 'Test' in GitLab, you should see something like this in the
 console:
 
 ```

features/dashboard/dashboard.feature

-@dashboard
-Feature: Dashboard
-  Background:
-    Given I sign in as a user
-    And I own project "Shop"
-    And project "Shop" has push event
-    And project "Shop" has CI enabled
-    And project "Shop" has CI build
-    And project "Shop" has labels: "bug", "feature", "enhancement"
-    And project "Shop" has issue: "bug report"
-    And I visit dashboard page
-
-  Scenario: I should see projects list
-    Then I should see "New Project" link
-    Then I should see "Shop" project link
-    Then I should see "Shop" project CI status
-
-  @javascript
-  Scenario: I should see activity list
-    And I visit dashboard activity page
-    Then I should see project "Shop" activity feed
-
-  Scenario: I should see groups list
-    Given I have group with projects
-    And I visit dashboard page
-    Then I should see groups list
-
-  @javascript
-  Scenario: I should see last push widget
-    Then I should see last push widget
-    And I click "Create Merge Request" link
-    Then I see prefilled new Merge Request page
-
-  @javascript
-  Scenario: Sorting Issues
-    Given I visit dashboard issues page
-    And I sort the list by "Oldest updated"
-    And I visit dashboard activity page
-    And I visit dashboard issues page
-    Then The list should be sorted by "Oldest updated"
-
-  @javascript
-  Scenario: Filtering Issues by label
-    Given project "Shop" has issue "Bugfix1" with label "feature"
-    When I visit dashboard issues page
-    And I filter the list by label "feature"
-    Then I should see "Bugfix1" in issues list
-
-  @javascript
-  Scenario: Visiting Project's issues after sorting
-    Given I visit dashboard issues page
-    And I sort the list by "Oldest updated"
-    And I visit project "Shop" issues page
-    Then The list should be sorted by "Oldest updated"
-
-  @javascript
-  Scenario: Sorting Merge Requests
-    Given I visit dashboard merge requests page
-    And I sort the list by "Oldest updated"
-    And I visit dashboard activity page
-    And I visit dashboard merge requests page
-    Then The list should be sorted by "Oldest updated"
-
-  @javascript
-  Scenario: Visiting Project's merge requests after sorting
-    Given project "Shop" has a "Bugfix MR" merge request open
-    And I visit dashboard merge requests page
-    And I sort the list by "Oldest updated"
-    And I visit project "Shop" merge requests page
-    Then The list should be sorted by "Oldest updated"

features/dashboard/event_filters.feature

-@dashboard
-Feature: Event Filters
-  Background:
-    Given I sign in as a user
-    And I own a project
-    And this project has push event
-    And this project has new member event
-    And this project has merge request event
-    And I visit dashboard activity page
-
-  @javascript
-  Scenario: I should see all events
-    Then I should see push event
-    And I should see new member event
-    And I should see merge request event
-
-  @javascript
-  Scenario: I should see only pushed events
-    When I click "push" event filter
-    Then I should see push event
-    And I should not see new member event
-    And I should not see merge request event
-
-  @javascript
-  Scenario: I should see only joined events
-    When I click "team" event filter
-    Then I should see new member event
-    And I should not see push event
-    And I should not see merge request event
-
-  @javascript
-  Scenario: I should see only merged events
-    When I click "merge" event filter
-    Then I should see merge request event
-    And I should not see push event
-    And I should not see new member event
-
-  @javascript
-  Scenario: I should see only selected events while page reloaded
-    When I click "push" event filter
-    And I visit dashboard activity page
-    Then I should see push event
-    And I should not see new member event
-    When I click "team" event filter
-    And I visit dashboard activity page
-    Then I should not see push event
-    And I should see new member event
-    And I should not see merge request event
-    When I click "push" event filter
-    And I visit dashboard activity page
-    Then I should see push event
-    And I should not see new member event
-    And I should not see merge request event
-    When I click "merge" event filter
-    And I visit dashboard activity page
-    Then I should see merge request event
-    And I should not see push event
-    And I should not see new member event

features/steps/dashboard/dashboard.rb

-class Spinach::Features::Dashboard < Spinach::FeatureSteps
-  include SharedAuthentication
-  include SharedPaths
-  include SharedProject
-  include SharedIssuable
-
-  step 'I should see "New Project" link' do
-    expect(page).to have_link "New project"
-  end
-
-  step 'I should see "Shop" project link' do
-    expect(page).to have_link "Shop"
-  end
-
-  step 'I should see "Shop" project CI status' do
-    expect(page).to have_link "Commit: skipped"
-  end
-
-  step 'I should see last push widget' do
-    expect(page).to have_content "You pushed to fix"
-    expect(page).to have_link "Create merge request"
-  end
-
-  step 'I click "Create merge request" link' do
-    find_link("Create merge request", visible: false).trigger('click')
-  end
-
-  step 'I see prefilled new Merge Request page' do
-    expect(page).to have_selector('.merge-request-form')
-    expect(current_path).to eq project_new_merge_request_path(@project)
-    expect(find("#merge_request_target_project_id").value).to eq @project.id.to_s
-    expect(find("input#merge_request_source_branch").value).to eq "fix"
-    expect(find("input#merge_request_target_branch").value).to eq "master"
-  end
-
-  step 'I have group with projects' do
-    @group   = create(:group)
-    @project = create(:empty_project, namespace: @group)
-    @event   = create(:closed_issue_event, project: @project)
-
-    @project.team << [current_user, :master]
-  end
-
-  step 'I should see projects list' do
-    @user.authorized_projects.all.each do |project|
-      expect(page).to have_link project.name_with_namespace
-    end
-  end
-
-  step 'I should see groups list' do
-    Group.all.each do |group|
-      expect(page).to have_link group.name
-    end
-  end
-
-  step 'group has a projects that does not belongs to me' do
-    @forbidden_project1 = create(:empty_project, group: @group)
-    @forbidden_project2 = create(:empty_project, group: @group)
-  end
-
-  step 'I should see 1 project at group list' do
-    expect(find('span.last_activity/span')).to have_content('1')
-  end
-
-  step 'I filter the list by label "feature"' do
-    page.within ".labels-filter" do
-      find('.dropdown').click
-      click_link "feature"
-    end
-  end
-
-  step 'I should see "Bugfix1" in issues list' do
-    page.within "ul.content-list" do
-      expect(page).to have_content "Bugfix1"
-    end
-  end
-
-  step 'project "Shop" has issue "Bugfix1" with label "feature"' do
-    project = Project.find_by(name: "Shop")
-    issue = create(:issue, title: "Bugfix1", project: project, assignees: [current_user])
-    issue.labels << project.labels.find_by(title: 'feature')
-  end
-end

features/steps/dashboard/event_filters.rb

-class Spinach::Features::EventFilters < Spinach::FeatureSteps
-  include WaitForRequests
-  include SharedAuthentication
-  include SharedPaths
-  include SharedProject
-
-  step 'I should see push event' do
-    expect(page).to have_selector('span.pushed')
-  end
-
-  step 'I should not see push event' do
-    expect(page).not_to have_selector('span.pushed')
-  end
-
-  step 'I should see new member event' do
-    expect(page).to have_selector('span.joined')
-  end
-
-  step 'I should not see new member event' do
-    expect(page).not_to have_selector('span.joined')
-  end
-
-  step 'I should see merge request event' do
-    expect(page).to have_selector('span.accepted')
-  end
-
-  step 'I should not see merge request event' do
-    expect(page).not_to have_selector('span.accepted')
-  end
-
-  step 'this project has push event' do
-    data = {
-      before: Gitlab::Git::BLANK_SHA,
-      after: "0220c11b9a3e6c69dc8fd35321254ca9a7b98f7e",
-      ref: "refs/heads/new_design",
-      user_id: @user.id,
-      user_name: @user.name,
-      repository: {
-        name: @project.name,
-        url: "localhost/rubinius",
-        description: "",
-        homepage: "localhost/rubinius",
-        private: true
-      }
-    }
-
-    @event = Event.create(
-      project: @project,
-      action: Event::PUSHED,
-      data: data,
-      author_id: @user.id
-    )
-  end
-
-  step 'this project has new member event' do
-    user = create(:user, { name: "John Doe" })
-    Event.create(
-      project: @project,
-      author_id: user.id,
-      action: Event::JOINED
-    )
-  end
-
-  step 'this project has merge request event' do
-    merge_request = create :merge_request, author: @user, source_project: @project, target_project: @project
-    Event.create(
-      project: @project,
-      action: Event::MERGED,
-      target_id: merge_request.id,
-      target_type: "MergeRequest",
-      author_id: @user.id
-    )
-  end
-
-  When 'I click "push" event filter' do
-    wait_for_requests
-    click_link("Push events")
-    wait_for_requests
-  end
-
-  When 'I click "team" event filter' do
-    wait_for_requests
-    click_link("Team")
-    wait_for_requests
-  end
-
-  When 'I click "merge" event filter' do
-    wait_for_requests
-    click_link("Merge events")
-    wait_for_requests
-  end
-end

features/steps/group_hooks.rb

@@ -55,7 +55,7 @@ class Spinach::Features::GroupHooks < Spinach::FeatureSteps
   step 'hook should be triggered' do
     expect(current_path).to eq group_hooks_path(@group)
     expect(page).to have_selector '.flash-notice',
-                              text: 'Hook successfully executed.'
+                              text: 'Hook executed successfully: HTTP 200'
   end
 
   step 'I should see hook error message' do

features/steps/shared/project.rb

@@ -239,11 +239,6 @@ module SharedProject
     create(:label, project: project, title: 'enhancement')
   end
 
-  step 'project "Shop" has issue: "bug report"' do
-    project = Project.find_by(name: "Shop")
-    create(:issue, project: project, title: "bug report")
-  end
-
   step 'project "Shop" has CI enabled' do
     project = Project.find_by(name: "Shop")
     project.enable_ci

lib/ci/gitlab_ci_yaml_processor.rb

@@ -83,7 +83,8 @@ module Ci
           before_script: job[:before_script],
           script: job[:script],
           after_script: job[:after_script],
-          environment: job[:environment]
+          environment: job[:environment],
+          retry: job[:retry]
         }.compact }
     end
 

lib/gitlab/background_migration.rb

@@ -26,7 +26,7 @@ module Gitlab
           next unless migration_class == steal_class
 
           begin
-            perform(migration_class, migration_args, retries: 3) if job.delete
+            perform(migration_class, migration_args) if job.delete
           rescue Exception # rubocop:disable Lint/RescueException
             BackgroundMigrationWorker # enqueue this migration again
               .perform_async(migration_class, migration_args)

lib/gitlab/ci/config/entry/job.rb

@@ -11,7 +11,7 @@ module Gitlab
 
           ALLOWED_KEYS = %i[tags script only except type image services allow_failure
                             type stage when artifacts cache dependencies before_script
-                            after_script variables environment coverage].freeze
+                            after_script variables environment coverage retry].freeze
 
           validations do
             validates :config, allowed_keys: ALLOWED_KEYS
@@ -23,6 +23,9 @@ module Gitlab
             with_options allow_nil: true do
               validates :tags, array_of_strings: true
               validates :allow_failure, boolean: true
+              validates :retry, numericality: { only_integer: true,
+                                                greater_than_or_equal_to: 0,
+                                                less_than_or_equal_to: 2 }
               validates :when,
                 inclusion: { in: %w[on_success on_failure always manual],
                              message: 'should be on_success, on_failure, ' \
@@ -76,9 +79,9 @@ module Gitlab
 
           helpers :before_script, :script, :stage, :type, :after_script,
                   :cache, :image, :services, :only, :except, :variables,
-                  :artifacts, :commands, :environment, :coverage
+                  :artifacts, :commands, :environment, :coverage, :retry
 
-          attributes :script, :tags, :allow_failure, :when, :dependencies
+          attributes :script, :tags, :allow_failure, :when, :dependencies, :retry
 
           def compose!(deps = nil)
             super do
@@ -142,6 +145,7 @@ module Gitlab
               environment: environment_defined? ? environment_value : nil,
               environment_name: environment_defined? ? environment_value[:name] : nil,
               coverage: coverage_defined? ? coverage_value : nil,
+              retry: retry_defined? ? retry_value.to_i : nil,
               artifacts: artifacts_value,
               after_script: after_script_value,
               ignore: ignored? }

lib/gitlab/data_builder/push.rb

@@ -74,6 +74,8 @@ module Gitlab
         build(project, user, commits.last&.id, commits.first&.id, ref, commits)
       end
 
+      private
+
       def checkout_sha(repository, newrev, ref)
         # Checkout sha is nil when we remove branch or tag
         return if Gitlab::Git.blank_ref?(newrev)

lib/gitlab/data_builder/wiki_page.rb

+module Gitlab
+  module DataBuilder
+    module WikiPage
+      extend self
+
+      def build(wiki_page, user, action)
+        wiki = wiki_page.wiki
+
+        {
+          object_kind: wiki_page.class.name.underscore,
+          user: user.hook_attrs,
+          project: wiki.project.hook_attrs,
+          wiki: wiki.hook_attrs,
+          object_attributes: wiki_page.hook_attrs.merge(
+            url: Gitlab::UrlBuilder.build(wiki_page),
+            action: action
+          )
+        }
+      end
+    end
+  end
+end

lib/gitlab/git/diff.rb

@@ -234,6 +234,8 @@ module Gitlab
         @new_file = diff.from_id == BLANK_SHA
         @renamed_file = diff.from_path != diff.to_path
         @deleted_file = diff.to_id == BLANK_SHA
+
+        collapse! if diff.respond_to?(:collapsed) && diff.collapsed
       end
 
       def prune_diff_if_eligible

lib/gitlab/git/diff_collection.rb

@@ -7,16 +7,28 @@ module Gitlab
 
       DEFAULT_LIMITS = { max_files: 100, max_lines: 5000 }.freeze
 
+      attr_reader :limits
+
+      delegate :max_files, :max_lines, :max_bytes, :safe_max_files, :safe_max_lines, :safe_max_bytes, to: :limits
+
+      def self.collection_limits(options = {})
+        limits = {}
+        limits[:max_files] = options.fetch(:max_files, DEFAULT_LIMITS[:max_files])
+        limits[:max_lines] = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines])
+        limits[:max_bytes] = limits[:max_files] * 5.kilobytes # Average 5 KB per file
+        limits[:safe_max_files] = [limits[:max_files], DEFAULT_LIMITS[:max_files]].min
+        limits[:safe_max_lines] = [limits[:max_lines], DEFAULT_LIMITS[:max_lines]].min
+        limits[:safe_max_bytes] = limits[:safe_max_files] * 5.kilobytes # Average 5 KB per file
+
+        OpenStruct.new(limits)
+      end
+
       def initialize(iterator, options = {})
         @iterator = iterator
-        @max_files = options.fetch(:max_files, DEFAULT_LIMITS[:max_files])
-        @max_lines = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines])
-        @max_bytes = @max_files * 5.kilobytes # Average 5 KB per file
-        @safe_max_files = [@max_files, DEFAULT_LIMITS[:max_files]].min
-        @safe_max_lines = [@max_lines, DEFAULT_LIMITS[:max_lines]].min
-        @safe_max_bytes = @safe_max_files * 5.kilobytes # Average 5 KB per file
+        @limits = self.class.collection_limits(options)
         @enforce_limits = !!options.fetch(:limits, true)
         @expanded = !!options.fetch(:expanded, true)
+        @from_gitaly = options.fetch(:from_gitaly, false)
 
         @line_count = 0
         @byte_count = 0
@@ -26,11 +38,25 @@ module Gitlab
       end
 
       def each(&block)
-        Gitlab::GitalyClient.migrate(:commit_raw_diffs) do
-          each_patch(&block)
+        @array.each(&block)
+
+        return if @overflow
+        return if @iterator.nil?
+
+        Gitlab::GitalyClient.migrate(:commit_raw_diffs) do |is_enabled|
+          if is_enabled && @from_gitaly
+            each_gitaly_patch(&block)
+          else
+            each_rugged_patch(&block)
           end
         end
 
+        @populated = true
+
+        # Allow iterator to be garbage-collected. It cannot be reused anyway.
+        @iterator = nil
+      end
+
       def empty?
         any? # Make sure the iterator has been exercised
         @empty
@@ -74,23 +100,32 @@ module Gitlab
       end
 
       def over_safe_limits?(files)
-        files >= @safe_max_files || @line_count > @safe_max_lines || @byte_count >= @safe_max_bytes
+        files >= safe_max_files || @line_count > safe_max_lines || @byte_count >= safe_max_bytes
       end
 
-      def each_patch
-        i = 0
-        @array.each do |diff|
-          yield diff
+      def each_gitaly_patch
+        i = @array.length
+
+        @iterator.each do |raw|
+          diff = Gitlab::Git::Diff.new(raw, expanded: !@enforce_limits || @expanded)
+
+          if raw.overflow_marker
+            @overflow = true
+            break
+          end
+
+          yield @array[i] = diff
           i += 1
         end
+      end
 
-        return if @overflow
-        return if @iterator.nil?
+      def each_rugged_patch
+        i = @array.length
 
         @iterator.each do |raw|
           @empty = false
 
-          if @enforce_limits && i >= @max_files
+          if @enforce_limits && i >= max_files
             @overflow = true
             break
           end
@@ -106,7 +141,7 @@ module Gitlab
           @line_count += diff.line_count
           @byte_count += diff.diff.bytesize
 
-          if @enforce_limits && (@line_count >= @max_lines || @byte_count >= @max_bytes)
+          if @enforce_limits && (@line_count >= max_lines || @byte_count >= max_bytes)
             # This last Diff instance pushes us over the lines limit. We stop and
             # discard it.
             @overflow = true
@@ -116,11 +151,6 @@ module Gitlab
           yield @array[i] = diff
           i += 1
         end
-
-        @populated = true
-
-        # Allow iterator to be garbage-collected. It cannot be reused anyway.
-        @iterator = nil
       end
     end
   end

lib/gitlab/gitaly_client.rb

@@ -86,8 +86,8 @@ module Gitlab
       feature.enabled?
     end
 
-    def self.migrate(feature)
-      is_enabled  = feature_enabled?(feature)
+    def self.migrate(feature, status: MigrationStatus::OPT_IN)
+      is_enabled  = feature_enabled?(feature, status: status)
       metric_name = feature.to_s
       metric_name += "_gitaly" if is_enabled
 

lib/gitlab/gitaly_client/commit_service.rb

@@ -23,9 +23,13 @@ module Gitlab
       def diff_from_parent(commit, options = {})
         request_params = commit_diff_request_params(commit, options)
         request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false)
+        request_params[:enforce_limits] = options.fetch(:limits, true)
+        request_params[:collapse_diffs] = request_params[:enforce_limits] || !options.fetch(:expanded, true)
+        request_params.merge!(Gitlab::Git::DiffCollection.collection_limits(options).to_h)
+
         request = Gitaly::CommitDiffRequest.new(request_params)
         response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request)
-        Gitlab::Git::DiffCollection.new(GitalyClient::DiffStitcher.new(response), options)
+        Gitlab::Git::DiffCollection.new(GitalyClient::DiffStitcher.new(response), options.merge(from_gitaly: true))
       end
 
       def commit_deltas(commit)

lib/gitlab/gitaly_client/diff.rb

 module Gitlab
   module GitalyClient
     class Diff
-      FIELDS = %i(from_path to_path old_mode new_mode from_id to_id patch).freeze
+      FIELDS = %i(from_path to_path old_mode new_mode from_id to_id patch overflow_marker collapsed).freeze
 
       attr_accessor(*FIELDS)
 

lib/gitlab/regex.rb

@@ -20,17 +20,23 @@ module Gitlab
       "It must start with letter, digit, emoji or '_'."
     end
 
-    def container_registry_reference_regex
-      Gitlab::PathRegex.git_reference_regex
-    end
-
     ##
-    # Docker Distribution Registry 2.4.1 repository name rules
+    # Docker Distribution Registry repository / tag name rules
+    #
+    # See https://github.com/docker/distribution/blob/master/reference/regexp.go.
     #
     def container_repository_name_regex
       @container_repository_regex ||= %r{\A[a-z0-9]+(?:[-._/][a-z0-9]+)*\Z}
     end
 
+    ##
+    # We do not use regexp anchors here because these are not allowed when
+    # used as a routing constraint.
+    #
+    def container_registry_tag_regex
+      @container_registry_tag_regex ||= /[\w][\w.-]{0,127}/
+    end
+
     def environment_name_regex_chars
       'a-zA-Z0-9_/\\$\\{\\}\\. -'
     end

lib/gitlab/usage_data.rb

@@ -43,6 +43,7 @@ module Gitlab
             notes: Note.count,
             pages_domains: PagesDomain.count,
             projects: Project.count,
+            projects_imported_from_github: Project.where(import_type: 'github').count,
             projects_prometheus_active: PrometheusService.active.count,
             protected_branches: ProtectedBranch.count,
             releases: Release.count,

lib/gitlab/visibility_level.rb

@@ -89,12 +89,12 @@ module Gitlab
       end
 
       def level_name(level)
-        level_name = 'Unknown'
+        level_name = N_('VisibilityLevel|Unknown')
         options.each do |name, lvl|
           level_name = name if lvl == level.to_i
         end
 
-        level_name
+        s_(level_name)
       end
 
       def level_value(level)

locale/bg/gitlab.po

@@ -4,11 +4,11 @@ msgid ""
 msgstr ""
 "Project-Id-Version: gitlab 1.0.0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2017-06-28 13:32+0200\n"
+"POT-Creation-Date: 2017-07-05 08:50-0500\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"PO-Revision-Date: 2017-07-05 08:18-0400\n"
+"PO-Revision-Date: 2017-07-13 08:13-0400\n"
 "Last-Translator: Lyubomir Vasilev <lyubomirv@abv.bg>\n"
 "Language-Team: Bulgarian (https://translate.zanata.org/project/view/GitLab)\n"
 "Language: bg\n"
@@ -641,6 +641,12 @@ msgstr "Всички"
 msgid "PipelineSchedules|Inactive"
 msgstr "Неактивно"
 
+msgid "PipelineSchedules|Input variable key"
+msgstr "Въведете ключ за променливата"
+
+msgid "PipelineSchedules|Input variable value"
+msgstr "Въведете стойността на променливата"
+
 msgid "PipelineSchedules|Next Run"
 msgstr "Следващо изпълнение"
 
@@ -650,12 +656,18 @@ msgstr "Нищо"
 msgid "PipelineSchedules|Provide a short description for this pipeline"
 msgstr "Въведете кратко описание за тази схема"
 
+msgid "PipelineSchedules|Remove variable row"
+msgstr "Премахване на реда за променлива"
+
 msgid "PipelineSchedules|Take ownership"
 msgstr "Поемане на собствеността"
 
 msgid "PipelineSchedules|Target"
 msgstr "Цел"
 
+msgid "PipelineSchedules|Variables"
+msgstr "Променливи"
+
 msgid "PipelineSheduleIntervalPattern|Custom"
 msgstr "собствен"
 
@@ -1148,6 +1160,15 @@ msgstr "Няма достатъчно данни за този етап."
 msgid "Withdraw Access Request"
 msgstr "Оттегляне на заявката за достъп"
 
+msgid ""
+"You are going to remove %{group_name}.\n"
+"Removed groups CANNOT be restored!\n"
+"Are you ABSOLUTELY sure?"
+msgstr ""
+"На път сте да премахнете „%{group_name}“.\n"
+"Ако я премахнете, групата НЕ може да бъде възстановена!\n"
+"НАИСТИНА ли искате това?"
+
 msgid ""
 "You are going to remove %{project_name_with_namespace}.\n"
 "Removed project CANNOT be restored!\n"

locale/eo/gitlab.po

@@ -4,11 +4,11 @@ msgid ""
 msgstr ""
 "Project-Id-Version: gitlab 1.0.0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2017-06-28 13:32+0200\n"
+"POT-Creation-Date: 2017-07-05 08:50-0500\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"PO-Revision-Date: 2017-07-05 08:18-0400\n"
+"PO-Revision-Date: 2017-07-13 08:46-0400\n"
 "Last-Translator: Lyubomir Vasilev <lyubomirv@abv.bg>\n"
 "Language-Team: Esperanto (https://translate.zanata.org/project/view/GitLab)\n"
 "Language: eo\n"
@@ -642,6 +642,12 @@ msgstr "Ĉiuj"
 msgid "PipelineSchedules|Inactive"
 msgstr "Malŝaltitaj"
 
+msgid "PipelineSchedules|Input variable key"
+msgstr "Entajpu ŝlosilon por la variablo"
+
+msgid "PipelineSchedules|Input variable value"
+msgstr "Entajpu la valoron de la variablo"
+
 msgid "PipelineSchedules|Next Run"
 msgstr "Sekvanta plenumo"
 
@@ -651,12 +657,18 @@ msgstr "Nenio"
 msgid "PipelineSchedules|Provide a short description for this pipeline"
 msgstr "Entajpu mallongan priskribon pri ĉi tiu ĉenstablo"
 
+msgid "PipelineSchedules|Remove variable row"
+msgstr "Forigi la variablan linion"
+
 msgid "PipelineSchedules|Take ownership"
 msgstr "Akiri posedon"
 
 msgid "PipelineSchedules|Target"
 msgstr "Celo"
 
+msgid "PipelineSchedules|Variables"
+msgstr "Variabloj"
+
 msgid "PipelineSheduleIntervalPattern|Custom"
 msgstr "Propra"
 
@@ -1150,6 +1162,15 @@ msgstr "Ne estas sufiĉe da datenoj por montri ĉi tiun etapon."
 msgid "Withdraw Access Request"
 msgstr "Nuligi la peton pri atingeblo"
 
+msgid ""
+"You are going to remove %{group_name}.\n"
+"Removed groups CANNOT be restored!\n"
+"Are you ABSOLUTELY sure?"
+msgstr ""
+"Vi forigos „%{group_name}“.\n"
+"Oni NE POVAS malfari la forigon de grupo!\n"
+"Ĉu vi estas ABSOLUTE certa?"
+
 msgid ""
 "You are going to remove %{project_name_with_namespace}.\n"
 "Removed project CANNOT be restored!\n"

locale/es/gitlab.po

@@ -7,7 +7,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: gitlab 1.0.0\n"
 "Report-Msgid-Bugs-To: \n"
-"PO-Revision-Date: 2017-07-12 12:35-0500\n"
+"PO-Revision-Date: 2017-07-13 12:10-0500\n"
 "Language-Team: Spanish\n"
 "Language: es\n"
 "MIME-Version: 1.0\n"
@@ -1059,6 +1059,9 @@ msgstr "Privado"
 msgid "VisibilityLevel|Public"
 msgstr "Público"
 
+msgid "VisibilityLevel|Unknown"
+msgstr "Desconocido"
+
 msgid "Want to see the data? Please ask an administrator for access."
 msgstr "¿Quieres ver los datos? Por favor pide acceso al administrador."
 

locale/gitlab.pot

@@ -8,8 +8,8 @@ msgid ""
 msgstr ""
 "Project-Id-Version: gitlab 1.0.0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2017-07-12 12:31-0500\n"
-"PO-Revision-Date: 2017-07-12 12:31-0500\n"
+"POT-Creation-Date: 2017-07-13 12:07-0500\n"
+"PO-Revision-Date: 2017-07-13 12:07-0500\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
 "Language: \n"
@@ -1060,6 +1060,9 @@ msgstr ""
 msgid "VisibilityLevel|Public"
 msgstr ""
 
+msgid "VisibilityLevel|Unknown"
+msgstr ""
+
 msgid "Want to see the data? Please ask an administrator for access."
 msgstr ""
 

locale/ja/gitlab.po

@@ -3,7 +3,7 @@
 # Kohei Ota <inductor@kela.jp>, 2017. #zanata
 # Taisuke Inoue <taisuke.inoue.jp@gmail.com>, 2017. #zanata
 # Takuya Noguchi <takninnovationresearch@gmail.com>, 2017. #zanata
-# YANO  TETTER <tetuyano+zana@gmail.com>, 2017. #zanata
+# YANO  Tethurou <tetuyano+zana@gmail.com>, 2017. #zanata
 msgid ""
 msgstr ""
 "Project-Id-Version: gitlab 1.0.0\n"
@@ -12,9 +12,9 @@ msgstr ""
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"Language-Team: Japanese (https://translate.zanata.org/project/view/GitLab)\n"
-"PO-Revision-Date: 2017-07-18 09:27-0400\n"
 "Last-Translator: YANO  TETTER <tetuyano+zana@gmail.com>\n"
+"PO-Revision-Date: 2017-07-19 09:45-0400\n"
+"Last-Translator: YANO  Tethurou <tetuyano+zana@gmail.com>\n"
 "Language: ja\n"
 "X-Generator: Zanata 3.9.6\n"
 "Plural-Forms: nplurals=1; plural=0\n"
@@ -628,6 +628,12 @@ msgstr "全件"
 msgid "PipelineSchedules|Inactive"
 msgstr "無効"
 
+msgid "PipelineSchedules|Input variable key"
+msgstr "変数の名前を入力"
+
+msgid "PipelineSchedules|Input variable value"
+msgstr "変数の値を入力"
+
 msgid "PipelineSchedules|Next Run"
 msgstr "次の実行"
 
@@ -637,12 +643,18 @@ msgstr "なし"
 msgid "PipelineSchedules|Provide a short description for this pipeline"
 msgstr "このパイプラインについて簡単に記述してください。"
 
+msgid "PipelineSchedules|Remove variable row"
+msgstr "変数を削除"
+
 msgid "PipelineSchedules|Take ownership"
 msgstr "権限を取得する"
 
 msgid "PipelineSchedules|Target"
 msgstr "ターゲット"
 
+msgid "PipelineSchedules|Variables"
+msgstr "変数"
+
 msgid "PipelineSheduleIntervalPattern|Custom"
 msgstr "カスタム"
 
@@ -1103,6 +1115,14 @@ msgstr "データ不足のため、このステージの表示はできません
 msgid "Withdraw Access Request"
 msgstr "アクセスリクエストを取り消す"
 
+msgid ""
+"You are going to remove %{group_name}.\n"
+"Removed groups CANNOT be restored!\n"
+"Are you ABSOLUTELY sure?"
+msgstr "%{group_name} グループを削除しようとしています。\n"
+"削除されたグループは絶対に元に戻せません！\n"
+"本当によろしいですか？"
+
 msgid ""
 "You are going to remove %{project_name_with_namespace}.\n"
 "Removed project CANNOT be restored!\n"

locale/zh_CN/gitlab.po

@@ -4,11 +4,11 @@ msgid ""
 msgstr ""
 "Project-Id-Version: gitlab 1.0.0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2017-06-28 13:32+0200\n"
+"POT-Creation-Date: 2017-07-05 08:50-0500\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"PO-Revision-Date: 2017-07-10 09:58-0400\n"
+"PO-Revision-Date: 2017-07-12 06:23-0400\n"
 "Last-Translator: Huang Tao <htve@outlook.com>\n"
 "Language-Team: Chinese (China) (https://translate.zanata.org/project/view/GitLab)\n"
 "Language: zh-CN\n"
@@ -621,6 +621,12 @@ msgstr "所有"
 msgid "PipelineSchedules|Inactive"
 msgstr "未启用"
 
+msgid "PipelineSchedules|Input variable key"
+msgstr "输入变量名"
+
+msgid "PipelineSchedules|Input variable value"
+msgstr "输入变量值"
+
 msgid "PipelineSchedules|Next Run"
 msgstr "下次运行时间"
 
@@ -630,12 +636,18 @@ msgstr "无"
 msgid "PipelineSchedules|Provide a short description for this pipeline"
 msgstr "为此流水线提供简短描述"
 
+msgid "PipelineSchedules|Remove variable row"
+msgstr "删除变量"
+
 msgid "PipelineSchedules|Take ownership"
-msgstr "取得所有者"
+msgstr "取得所有权"
 
 msgid "PipelineSchedules|Target"
 msgstr "目标"
 
+msgid "PipelineSchedules|Variables"
+msgstr "变量"
+
 msgid "PipelineSheduleIntervalPattern|Custom"
 msgstr "自定义"
 
@@ -1085,6 +1097,14 @@ msgstr "该阶段的数据不足，无法显示。"
 msgid "Withdraw Access Request"
 msgstr "取消权限申请"
 
+msgid ""
+"You are going to remove %{group_name}.\n"
+"Removed groups CANNOT be restored!\n"
+"Are you ABSOLUTELY sure?"
+msgstr "即将删除 %{group_name}。\n"
+"已删除的群组无法恢复！\n"
+"确定继续吗？"
+
 msgid ""
 "You are going to remove %{project_name_with_namespace}.\n"
 "Removed project CANNOT be restored!\n"

locale/zh_HK/gitlab.po

@@ -3,13 +3,13 @@ msgid ""
 msgstr ""
 "Project-Id-Version: gitlab 1.0.0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2017-06-28 13:32+0200\n"
+"POT-Creation-Date: 2017-07-05 08:50-0500\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"PO-Revision-Date: 2017-07-06 11:26-0400\n"
+"PO-Revision-Date: 2017-07-12 06:32-0400\n"
 "Last-Translator: Huang Tao <htve@outlook.com>\n"
-"Language-Team: Chinese (Hong Kong SAR China)\n"
+"Language-Team: Chinese (Hong Kong SAR China) (https://translate.zanata.org/project/view/GitLab)\n"
 "Language: zh-HK\n"
 "X-Generator: Zanata 3.9.6\n"
 "Plural-Forms: nplurals=1; plural=0\n"
@@ -620,6 +620,12 @@ msgstr "所有"
 msgid "PipelineSchedules|Inactive"
 msgstr "未啟用"
 
+msgid "PipelineSchedules|Input variable key"
+msgstr "輸入變量名"
+
+msgid "PipelineSchedules|Input variable value"
+msgstr "輸入變量值"
+
 msgid "PipelineSchedules|Next Run"
 msgstr "下次運行時間"
 
@@ -629,12 +635,18 @@ msgstr "無"
 msgid "PipelineSchedules|Provide a short description for this pipeline"
 msgstr "為此流水線提供簡短描述"
 
+msgid "PipelineSchedules|Remove variable row"
+msgstr "刪除變量"
+
 msgid "PipelineSchedules|Take ownership"
-msgstr "取得所有者"
+msgstr "取得所有權"
 
 msgid "PipelineSchedules|Target"
 msgstr "目標"
 
+msgid "PipelineSchedules|Variables"
+msgstr "變量"
+
 msgid "PipelineSheduleIntervalPattern|Custom"
 msgstr "自定義"
 
@@ -1084,6 +1096,14 @@ msgstr "該階段的數據不足，無法顯示。"
 msgid "Withdraw Access Request"
 msgstr "取消權限申请"
 
+msgid ""
+"You are going to remove %{group_name}.\n"
+"Removed groups CANNOT be restored!\n"
+"Are you ABSOLUTELY sure?"
+msgstr "即將刪除 %{group_name}。\n"
+"已刪除的群組無法恢復！\n"
+"確定繼續嗎？"
+
 msgid ""
 "You are going to remove %{project_name_with_namespace}.\n"
 "Removed project CANNOT be restored!\n"

spec/controllers/profiles/accounts_controller_spec.rb

@@ -29,7 +29,7 @@ describe Profiles::AccountsController do
       end
     end
 
-    [:twitter, :facebook, :google_oauth2, :gitlab, :github, :bitbucket, :crowd, :auth0].each do |provider|
+    [:twitter, :facebook, :google_oauth2, :gitlab, :github, :bitbucket, :crowd, :auth0, :authentiq].each do |provider|
       describe "#{provider} provider" do
         let(:user) { create(:omniauth_user, provider: provider.to_s) }