Merge branch 'master' into 8-8-stable-ee

CHANGELOG

@@ -85,6 +85,7 @@ v 8.7.4
   - Running rake gitlab:db:drop_tables now drops tables with cascade !4020
   - Running rake gitlab:db:drop_tables uses "IF EXISTS" as a precaution !4100
   - Use a case-insensitive comparison in sanitizing URI schemes
+  - Bump GitLab Pages to 0.2.4 to fix Content-Type for predefined 404
 
 v 8.7.3
   - Emails, Gitlab::Email::Message, Gitlab::Diff, and Premailer::Adapter::Nokogiri are now instrumented

CHANGELOG-EE

@@ -4,9 +4,13 @@ v 8.8.0 (unreleased)
   - [Elastic] Database indexer prints its status
   - [Elastic][Fix] Database indexer skips projects with invalid HEAD reference
   - Fix skipping pages when restoring backups
+  - Add EE license via API !400
   - [Elastic] More efficient snippets search
   - [Elastic] Add rake task for removing all indexes
   - [Elastic] Add rake task for clearing indexing status
+  - [Elastic] Improve code search
+  - [Elastic] Fix encoding issues during indexing
+  - Warn admin if current active count exceeds license
   - Set KRB5 as default clone protocol when Kerberos is enabled and user is logged in (Borja Aparicio)
   - Reduce emails-on-push HTML size by using a simple monospace font
   - API requests to /internal/authorized_keys are now tagged properly

GITLAB_PAGES_VERSION

-0.2.2
+0.2.4

Gemfile

@@ -102,7 +102,7 @@ gem "seed-fu", '~> 2.3.5'
 # Search
 gem 'elasticsearch-model'
 gem 'elasticsearch-rails'
-gem 'gitlab-elasticsearch-git', '~> 0.0.12', require: "elasticsearch/git"
+gem 'gitlab-elasticsearch-git', '~> 0.0.14', require: "elasticsearch/git"
 
 # Markdown and HTML processing
 gem 'html-pipeline', '~> 1.11.0'

Gemfile.lock

@@ -352,14 +352,14 @@ GEM
       mime-types (>= 1.19)
       rugged (>= 0.24.0b13)
     github-markup (1.3.3)
-    gitlab-elasticsearch-git (0.0.12)
+    gitlab-elasticsearch-git (0.0.14)
       activemodel (~> 4.2)
       activesupport (~> 4.2)
       charlock_holmes (~> 0.7)
       elasticsearch-api (~> 1.0)
       elasticsearch-model (~> 0.1.8)
       github-linguist (~> 4.7)
-      rugged (~> 0.24.0b13)
+      rugged (~> 0.24)
     gitlab-flowdock-git-hook (1.0.1)
       flowdock (~> 0.7)
       gitlab-grit (>= 2.4.1)
@@ -965,7 +965,7 @@ DEPENDENCIES
   gemnasium-gitlab-service (~> 0.2)
   github-linguist (~> 4.7.0)
   github-markup (~> 1.3.1)
-  gitlab-elasticsearch-git (~> 0.0.12)
+  gitlab-elasticsearch-git (~> 0.0.14)
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-license (~> 0.0.4)
   gitlab_emoji (~> 0.3.0)

app/helpers/license_helper.rb

 module LicenseHelper
+  def current_active_user_count
+    User.active.count
+  end
+
   def license_message(signed_in: signed_in?, is_admin: (current_user && current_user.is_admin?))
     @license_message ||=
       if License.current
@@ -33,13 +37,15 @@ module LicenseHelper
 
     return unless signed_in
 
-    return unless (license.notify_admins? && is_admin) || license.notify_users?
+    return unless ((license.notify_admins? || license.warn_upgrade_license_message?) && is_admin) || license.notify_users?
 
     message = []
 
-    message << "The GitLab Enterprise Edition license"
-    message << (license.expired? ? "expired" : "will expire")
-    message << "on #{license.expires_at}."
+    unless license.warn_upgrade_license_message?
+      message << "The GitLab Enterprise Edition license"
+      message << (license.expired? ? "expired" : "will expire")
+      message << "on #{license.expires_at}."
+    end
 
     if license.expired? && license.will_block_changes?
       message << "Pushing code and creation of issues and merge requests"
@@ -50,18 +56,24 @@ module LicenseHelper
         else
           "will be disabled on #{license.block_changes_at}."
         end
-    end
 
-    message <<
-      if is_admin
-        "Upload a new license in the admin area"
-      else
-        "Ask an admin to upload a new license"
-      end
+      message <<
+        if is_admin
+          "Upload a new license in the admin area"
+        else
+          "Ask an admin to upload a new license"
+        end
 
-    message << "to"
-    message << (license.block_changes? ? "restore" : "ensure uninterrupted")
-    message << "service."
+      message << "to"
+      message << (license.block_changes? ? "restore" : "ensure uninterrupted")
+      message << "service."
+    elsif license.warn_upgrade_license_message?
+      message << "Your GitLab license currently covers #{license.user_count}"
+      message << "users, but it looks like your site has grown to"
+      message << "#{current_active_user_count} users. Please contact"
+      message << "sales@gitlab.com to increase the seats on your license."
+      message << "Thank you for choosing GitLab."
+    end
 
     message.join(" ")
   end

app/mailers/emails/projects.rb

@@ -66,6 +66,7 @@ module Emails
       # used in notify layout
       @target_url = @message.target_url
       @project = Project.find(project_id)
+
       @diff_notes_disabled = true
 
       add_project_headers

app/models/license.rb

@@ -86,6 +86,24 @@ class License < ActiveRecord::Base
     add_ons[code].to_i > 0
   end
 
+  def user_count
+    return unless self.license? && self.restricted?(:active_user_count)
+
+    self.restrictions[:active_user_count]
+  end
+
+  def warn_upgrade_license_message?
+    restricted_user_count = user_count
+
+    return unless restricted_user_count
+
+    return false unless Time.now >= self.starts_at + 3.months
+
+    active_user_count = User.active.count
+
+    restricted_user_count && active_user_count > restricted_user_count
+  end
+
   private
 
   def reset_current
@@ -103,9 +121,9 @@ class License < ActiveRecord::Base
   end
 
   def active_user_count
-    return unless self.license? && self.restricted?(:active_user_count)
+    restricted_user_count = user_count
 
-    restricted_user_count = self.restrictions[:active_user_count]
+    return unless restricted_user_count
 
     date_range = (self.starts_at - 1.year)..self.starts_at
     active_user_count = HistoricalData.during(date_range).maximum(:active_user_count) || 0

app/views/projects/git_hooks/index.html.haml

 - page_title "Git Hooks"
-%h3.page-title
-  Git hooks
-%p.light
-  Rules that define what git pushes are accepted for this project. Request new rules for free by creating an issue on the <a href="https://gitlab.com/gitlab-org/gitlab-ee/issues/">GitLab EE issue tracker</a> and labeling it 'Feature proposal'. Or if you can please contribute a tested merge request.
 
-%hr.clearfix
+.row.prepend-top-default.append-bottom-default
+  .col-lg-3
+    %h4.prepend-top-0
+      = page_title
+    %p.light
+      Git Hooks outline what is accepted for this project. You can request new rules (for free) by creating an issue on our
+      = succeed '.' do
+        %a{ href: "https://gitlab.com/gitlab-org/gitlab-ee/issues/" }GitLab EE issue tracker
+      Alternatively, submit a merge request to GitLab EE.
+  .col-lg-9
+    %h5.prepend-top-0
+      Add new web hook
+    = form_for [@project.namespace.becomes(Namespace), @project, @git_hook] do |f|
+      = form_errors(@git_hook)
 
-= form_for [@project.namespace.becomes(Namespace), @project, @git_hook], html: { class: 'form-horizontal' } do |f|
-  -if @git_hook.errors.any?
-    .alert.alert-danger
-      - @git_hook.errors.full_messages.each do |msg|
-        %p= msg
-  = render "shared/git_hooks_form", f: f
+      = render "shared/git_hooks_form", f: f

app/views/shared/_git_hooks_form.html.haml

 .form-group
-  .col-sm-offset-2.col-sm-10
-    .checkbox
-      = f.label :deny_delete_tag do
-        = f.check_box :deny_delete_tag
-        %strong
-          Do not allow users to remove git tags with
-          %code git push
-        .help-block Tags can still be deleted through the web UI.
+  = f.check_box :deny_delete_tag, class: "pull-left"
+  .prepend-left-20
+    = f.label :deny_delete_tag, class: "label-light append-bottom-0" do
+      Do not allow users to remove git tags with
+      %code git push
+    %p.light.append-bottom-0
+      Tags can still be deleted through the web UI.
 
 .form-group
-  .col-sm-offset-2.col-sm-10
-    .checkbox
-      = f.label :member_check do
-        = f.check_box :member_check
-        %strong
-          Check whether author is a GitLab user
-        .help-block Restrict commits by author (email) to existing GitLab users
+  = f.check_box :member_check, class: "pull-left"
+  .prepend-left-20
+    = f.label :member_check, "Check whether author is a GitLab user", class: "label-light append-bottom-0"
+    %p.light.append-bottom-0
+      Restrict commits by author (email) to existing GitLab users
 
 .form-group
-  = f.label :commit_message_regex, "Commit message", class: 'control-label'
-  .col-sm-10
-    = f.text_field :commit_message_regex, class: "form-control", placeholder: 'Example: Fixes \d+\..*'
-    .help-block
-      All commit messages must match this
-      = link_to 'Ruby regular expression', 'http://www.ruby-doc.org/core-2.1.1/Regexp.html'
-      to be pushed.
-      If this field is empty it allows any commit message.
-      For example you can require that an issue number is always mentioned in the commit message.
+  = f.label :commit_message_regex, "Commit message", class: 'label-light'
+  = f.text_field :commit_message_regex, class: "form-control", placeholder: 'Example: Fixes \d+\..*'
+  .help-block
+    All commit messages must match this
+    = link_to 'Ruby regular expression', 'http://www.ruby-doc.org/core-2.1.1/Regexp.html'
+    to be pushed.
+    If this field is empty it allows any commit message.
+    For example you can require that an issue number is always mentioned in the commit message.
 
 .form-group
-  = f.label :author_email_regex, "Commit author's email", class: 'control-label'
-  .col-sm-10
-    = f.text_field :author_email_regex, class: "form-control", placeholder: 'Example: Fixes @my-company.com$'
-    .help-block
-      All commit author's email must match this
-      = link_to 'Ruby regular expression', 'http://www.ruby-doc.org/core-2.1.1/Regexp.html'
-      to be pushed.
-      If this field is empty it allows any email.
+  = f.label :author_email_regex, "Commit author's email", class: 'label-light'
+  = f.text_field :author_email_regex, class: "form-control", placeholder: 'Example: Fixes @my-company.com$'
+  .help-block
+    All commit author's email must match this
+    = link_to 'Ruby regular expression', 'http://www.ruby-doc.org/core-2.1.1/Regexp.html'
+    to be pushed.
+    If this field is empty it allows any email.
 
 .form-group
-  = f.label :file_name_regex, "Prohibited file names", class: 'control-label'
-  .col-sm-10
-    = f.text_field :file_name_regex, class: "form-control", placeholder: 'Example: (jar|exe)$'
-    .help-block
-      All commited filenames must not match this
-      = link_to 'Ruby regular expression', 'http://www.ruby-doc.org/core-2.1.1/Regexp.html'
-      to be pushed.
-      If this field is empty it allows any filenames.
+  = f.label :file_name_regex, "Prohibited file names", class: 'label-light'
+  = f.text_field :file_name_regex, class: "form-control", placeholder: 'Example: (jar|exe)$'
+  .help-block
+    All commited filenames must not match this
+    = link_to 'Ruby regular expression', 'http://www.ruby-doc.org/core-2.1.1/Regexp.html'
+    to be pushed.
+    If this field is empty it allows any filenames.
 
 .form-group
-  = f.label :max_file_size, "Maximum file size (MB)", class: 'control-label'
-  .col-sm-10
-    = f.number_field :max_file_size, class: "form-control", min: 0
-    .help-block
-      Pushes that contain added or updated files that exceed this file size are rejected.
-      Set to 0 to allow files of any size.
+  = f.label :max_file_size, "Maximum file size (MB)", class: 'label-light'
+  = f.number_field :max_file_size, class: "form-control", min: 0
+  .help-block
+    Pushes that contain added or updated files that exceed this file size are rejected.
+    Set to 0 to allow files of any size.
 
-.form-actions
-  = f.submit "Save Git hooks", class: "btn btn-create"
+= f.submit "Save Git hooks", class: "btn btn-create"
\ No newline at end of file

doc/update/8.7-to-8.8.md

@@ -137,6 +137,44 @@ To make sure you didn't miss anything run a more thorough check:
 
 If all items are green, then congratulations, the upgrade is complete!
 
+### 10. Elasticsearch indexes update (if you currently use Elasticsearch)
+
+When index mapping is changed the whole index should be removed and built from the scratch.
+
+1. Delete old indexes:
+
+    ```
+    # Omnibus installations
+    sudo gitlab-rake gitlab:elastic:delete_indexes
+
+    # Installations from source
+    bundle exec rake gitlab:elastic:delete_indexes
+    ```
+
+1. Create new, empty indexes:
+
+    ```
+    # Omnibus installations
+    sudo gitlab-rake gitlab:elastic:create_empty_indexes
+
+    # Installations from source
+    bundle exec rake gitlab:elastic:create_empty_indexes
+    ```
+
+1. Fill newly created indexes. The following command is acceptable for not very big GitLab instances (storage size no more than few gigabytes)
+
+    ```
+    # Omnibus installations
+    sudo gitlab-rake gitlab:elastic:index
+
+    # Installations from source
+    bundle exec rake gitlab:elastic:index
+    ```
+
+  If your instance is big we recommend to follow [Add GitLab's data to the Elasticsearch index](../integration/elasticsearch.md#add-gitlabs-data-to-the-elasticsearch-index)
+
+
+
 ## Things went south? Revert to previous version (8.6)
 
 ### 1. Revert the code to the previous version

lib/api/api.rb

@@ -56,12 +56,12 @@ module API
     mount ::API::Settings
     mount ::API::Keys
     mount ::API::Tags
-    mount ::API::LicenseInfo
+    mount ::API::License
     mount ::API::Triggers
     mount ::API::Builds
     mount ::API::Variables
     mount ::API::Runners
-    mount ::API::Licenses
+    mount ::API::LicenseTemplates
     mount ::API::Subscriptions
   end
 end

lib/api/license.rb

+module API
+  class License < Grape::API
+    before { authenticated_as_admin! }
+
+    resource :license do
+
+      # Get information on the currently active license
+      #
+      # Example request:
+      #   GET /license
+      get do
+        license = ::License.current
+
+        present license, with: Entities::License
+      end
+
+      # Add a new license
+      #
+      # Parameters:
+      #   license (required) - The license text
+      #
+      # Example request:
+      #   POST /license
+      post do
+        required_attributes! [:license]
+
+        license = ::License.new(data: params[:license])
+        if license.save
+          present license, with: Entities::License
+        else
+          render_api_error!(license.errors.full_messages.first, 400)
+        end
+      end
+    end
+  end
+end

lib/api/license_info.rb

-module API
-  class LicenseInfo < Grape::API
-    before { authenticated_as_admin! }
-
-    resource :license do
-
-      # Get information on the currently active license
-      #
-      # Example request:
-      #   GET /license
-      get do
-        @license = License.current
-
-        present @license, with: Entities::License
-      end
-    end
-  end
-end

lib/api/licenses.rb → lib/api/license_templates.rb

 module API
   # Licenses API
-  class Licenses < Grape::API
+  class LicenseTemplates < Grape::API
     PROJECT_TEMPLATE_REGEX =
     /[\<\{\[]
       (project|description|

spec/helpers/license_helper_spec.rb

+require 'spec_helper'
+
+describe LicenseHelper do
+  describe '#license_message' do
+    context 'no license installed' do
+      before do
+        expect(License).to receive(:current).and_return(nil)
+      end
+
+      it 'admin user' do
+        admin_msg = 'No GitLab Enterprise Edition license has been provided yet. Pushing code and creation of issues and merge requests has been disabled. Upload a license in the admin area to activate this functionality.'
+
+        expect(license_message(signed_in: true, is_admin: true)).to eq(admin_msg)
+      end
+
+      it 'normal user' do
+        user_msg = 'No GitLab Enterprise Edition license has been provided yet. Pushing code and creation of issues and merge requests has been disabled. Ask an admin to upload a license to activate this functionality.'
+        expect(license_message(signed_in: true, is_admin: false)).to eq(user_msg)
+      end
+    end
+
+    context 'license available' do
+      let(:license) { create(:license) }
+
+      before do
+        allow(License).to receive(:current).and_return(license)
+      end
+
+      it 'warn for overusage' do
+        allow(license).to receive(:starts_at).and_return(Time.now - 3.months)
+        allow(license).to receive(:expired?).and_return(false)
+        allow(license).to receive(:restricted?).and_return(true)
+        allow(license).to receive(:notify_admins?).and_return(true)
+        allow(license).to receive(:restrictions).and_return({ active_user_count: 50 })
+        allow(User).to receive(:active).and_return(Array.new(100))
+
+        warn_msg = 'Your GitLab license currently covers 50 users, but it looks like your site has grown to 100 users. Please contact sales@gitlab.com to increase the seats on your license. Thank you for choosing GitLab.'
+        expect(license_message(signed_in: true, is_admin: true)).to eq(warn_msg)
+      end
+    end
+  end
+end

spec/requests/api/license_spec.rb

@@ -24,4 +24,29 @@ describe API::API, api: true  do
       expect(response.status).to eq 403
     end
   end
+
+  describe 'POST /license' do
+    it 'adds a new license if admin is logged in' do
+      post api('/license', admin), license: gl_license.export
+
+      expect(response.status).to eq 201
+      expect(json_response['user_limit']).to eq 0
+      expect(Date.parse(json_response['starts_at'])).to eq Date.today - 1.month
+      expect(Date.parse(json_response['expires_at'])).to eq Date.today + 11.months
+      expect(json_response['active_users']).to eq 1
+      expect(json_response['licensee']).to_not be_empty
+    end
+
+    it 'denies access if not admin' do
+      post api('/license', user), license: license
+
+      expect(response.status).to eq 403
+    end
+
+    it 'returns 400 if the license cannot be saved' do
+      post api('/license', admin), license: 'foo'
+
+      expect(response.status).to eq(400)
+    end
+  end
 end

spec/requests/api/licenses_spec.rb → spec/requests/api/license_templates_spec.rb

 require 'spec_helper'
 
-describe API::Licenses, api: true  do
+describe API::API, api: true  do
   include ApiHelpers
 
   describe 'Entity' do