Merge branch 'ce-to-ee-2018-01-18' into 'master'

CE upstream - Thursday

Closes gitaly#923

See merge request gitlab-org/gitlab-ee!4150

GITALY_SERVER_VERSION

-0.69.0
+0.70.0

app/assets/javascripts/render_mermaid.js

@@ -19,7 +19,11 @@ export default function renderMermaid($els) {
 
   import(/* webpackChunkName: 'mermaid' */ 'blackst0ne-mermaid').then((mermaid) => {
     mermaid.initialize({
-      loadOnStart: false,
+      // mermaid core options
+      mermaid: {
+        startOnLoad: false,
+      },
+      // mermaidAPI options
       theme: 'neutral',
     });
 

app/controllers/projects/commits_controller.rb

@@ -13,31 +13,37 @@ class Projects::CommitsController < Projects::ApplicationController
     @merge_request = MergeRequestsFinder.new(current_user, project_id: @project.id).execute.opened
       .find_by(source_project: @project, source_branch: @ref, target_branch: @repository.root_ref)
 
-    respond_to do |format|
-      format.html
-      format.atom { render layout: 'xml.atom' }
-
-      format.json do
-        pager_json(
-          'projects/commits/_commits',
-          @commits.size,
-          project: @project,
-          ref: @ref)
+    # https://gitlab.com/gitlab-org/gitaly/issues/931
+    Gitlab::GitalyClient.allow_n_plus_1_calls do
+      respond_to do |format|
+        format.html
+        format.atom { render layout: 'xml.atom' }
+
+        format.json do
+          pager_json(
+            'projects/commits/_commits',
+            @commits.size,
+            project: @project,
+            ref: @ref)
+        end
       end
     end
   end
 
   def signatures
-    respond_to do |format|
-      format.json do
-        render json: {
-          signatures: @commits.select(&:has_signature?).map do |commit|
-            {
-              commit_sha: commit.sha,
-              html: view_to_html_string('projects/commit/_signature', signature: commit.signature)
-            }
-          end
-        }
+    # https://gitlab.com/gitlab-org/gitaly/issues/931
+    Gitlab::GitalyClient.allow_n_plus_1_calls do
+      respond_to do |format|
+        format.json do
+          render json: {
+            signatures: @commits.select(&:has_signature?).map do |commit|
+              {
+                commit_sha: commit.sha,
+                html: view_to_html_string('projects/commit/_signature', signature: commit.signature)
+              }
+            end
+          }
+        end
       end
     end
   end

changelogs/unreleased/sh-fix-mermaid-start-on-load-typo.yml

+---
+title: Fix Mermaid drawings not loading on some browsers
+merge_request:
+author:
+type: fixed

db/migrate/20170425112128_create_pipeline_schedules_table.rb

@@ -17,7 +17,7 @@ class CreatePipelineSchedulesTable < ActiveRecord::Migration
       t.boolean :active, default: true
       t.datetime :deleted_at
 
-      t.timestamps
+      t.timestamps null: true
     end
 
     add_index(:ci_pipeline_schedules, :project_id)

doc/README.md

@@ -10,9 +10,9 @@ platform for software development!
 
 GitLab offers the most scalable Git-based fully integrated platform for software development, with flexible products and subscription plans:
 
-- **GitLab Community Edition (CE)** is an [opensource product](https://gitlab.com/gitlab-org/gitlab-ce/),
+- **GitLab Community Edition (CE)** is an [open source product](https://gitlab.com/gitlab-org/gitlab-ce/),
 self-hosted, free to use. Every feature available in GitLab CE is also available on GitLab Enterprise Edition (Starter and Premium) and GitLab.com.
-- **GitLab Enterprise Edition (EE)** is an [opencore product](https://gitlab.com/gitlab-org/gitlab-ee/),
+- **GitLab Enterprise Edition (EE)** is an [open-core product](https://gitlab.com/gitlab-org/gitlab-ee/),
 self-hosted, fully featured solution of GitLab, available under distinct [subscriptions](https://about.gitlab.com/products/): **GitLab Enterprise Edition Starter (EES)**, **GitLab Enterprise Edition Premium (EEP)**, and **GitLab Enterprise Edition Ultimate (EEU)**.
 - **GitLab.com**: SaaS GitLab solution, with [free and paid subscriptions](https://about.gitlab.com/gitlab-com/). GitLab.com is hosted by GitLab, Inc., and administrated by GitLab (users don't have access to admin settings).
 
@@ -166,8 +166,8 @@ Regular users don't have access to GitLab administration tools and settings.
 
 ## Contributor documentation
 
-GitLab Community Edition is [opensource](https://gitlab.com/gitlab-org/gitlab-ce/)
-and Enterprise Editions are [opencore](https://gitlab.com/gitlab-org/gitlab-ee/).
+GitLab Community Edition is [open source](https://gitlab.com/gitlab-org/gitlab-ce/)
+and Enterprise Editions are [open-core](https://gitlab.com/gitlab-org/gitlab-ee/).
 Learn how to contribute to GitLab:
 
 - [Development](development/README.md): All styleguides and explanations how to contribute.

lib/gitlab/git/commit.rb

@@ -239,6 +239,24 @@ module Gitlab
             end
           end
         end
+
+        def extract_signature(repository, commit_id)
+          repository.gitaly_migrate(:extract_commit_signature) do |is_enabled|
+            if is_enabled
+              repository.gitaly_commit_client.extract_signature(commit_id)
+            else
+              rugged_extract_signature(repository, commit_id)
+            end
+          end
+        end
+
+        def rugged_extract_signature(repository, commit_id)
+          begin
+            Rugged::Commit.extract_signature(repository.rugged, commit_id)
+          rescue Rugged::OdbError
+            nil
+          end
+        end
       end
 
       def initialize(repository, raw_commit, head = nil)

lib/gitlab/git/repository.rb

@@ -490,11 +490,7 @@ module Gitlab
           return []
         end
 
-        if log_using_shell?(options)
-          log_by_shell(sha, options)
-        else
-          log_by_walk(sha, options)
-        end
+        log_by_shell(sha, options)
       end
 
       def count_commits(options)
@@ -1506,27 +1502,6 @@ module Gitlab
         end
       end
 
-      def log_using_shell?(options)
-        options[:path].present? ||
-          options[:disable_walk] ||
-          options[:skip_merges] ||
-          options[:after] ||
-          options[:before]
-      end
-
-      def log_by_walk(sha, options)
-        walk_options = {
-          show: sha,
-          sort: Rugged::SORT_NONE,
-          limit: options[:limit],
-          offset: options[:offset]
-        }
-        Rugged::Walker.walk(rugged, walk_options).to_a
-      end
-
-      # Gitaly note: JV: although #log_by_shell shells out to Git I think the
-      # complexity is such that we should migrate it as Ruby before trying to
-      # do it in Go.
       def log_by_shell(sha, options)
         limit = options[:limit].to_i
         offset = options[:offset].to_i

lib/gitlab/gitaly_client/commit_service.rb

@@ -282,6 +282,23 @@ module Gitlab
         end
       end
 
+      def extract_signature(commit_id)
+        request = Gitaly::ExtractCommitSignatureRequest.new(repository: @gitaly_repo, commit_id: commit_id)
+        response = GitalyClient.call(@repository.storage, :commit_service, :extract_commit_signature, request)
+
+        signature = ''.b
+        signed_text = ''.b
+
+        response.each do |message|
+          signature << message.signature
+          signed_text << message.signed_text
+        end
+
+        return if signature.blank? && signed_text.blank?
+
+        [signature, signed_text]
+      end
+
       private
 
       def call_commit_diff(request_params, options = {})

lib/gitlab/gpg/commit.rb

@@ -4,12 +4,8 @@ module Gitlab
       def initialize(commit)
         @commit = commit
 
-        @signature_text, @signed_text =
-          begin
-            Rugged::Commit.extract_signature(@commit.project.repository.rugged, @commit.sha)
-          rescue Rugged::OdbError
-            nil
-          end
+        repo = commit.project.repository.raw_repository
+        @signature_text, @signed_text = Gitlab::Git::Commit.extract_signature(repo, commit.sha)
       end
 
       def has_signature?

spec/lib/gitlab/git/commit_spec.rb

@@ -388,6 +388,84 @@ describe Gitlab::Git::Commit, seed_helper: true do
         end
       end
     end
+
+    describe '.extract_signature' do
+      subject { described_class.extract_signature(repository, commit_id) }
+
+      shared_examples '.extract_signature' do
+        context 'when the commit is signed' do
+          let(:commit_id) { '0b4bc9a49b562e85de7cc9e834518ea6828729b9' }
+
+          it 'returns signature and signed text' do
+            signature, signed_text = subject
+
+            expected_signature = <<~SIGNATURE
+              -----BEGIN PGP SIGNATURE-----
+              Version: GnuPG/MacGPG2 v2.0.22 (Darwin)
+              Comment: GPGTools - https://gpgtools.org
+
+              iQEcBAABCgAGBQJTDvaZAAoJEGJ8X1ifRn8XfvYIAMuB0yrbTGo1BnOSoDfyrjb0
+              Kw2EyUzvXYL72B63HMdJ+/0tlSDC6zONF3fc+bBD8z+WjQMTbwFNMRbSSy2rKEh+
+              mdRybOP3xBIMGgEph0/kmWln39nmFQBsPRbZBWoU10VfI/ieJdEOgOphszgryRar
+              TyS73dLBGE9y9NIININVaNISet9D9QeXFqc761CGjh4YIghvPpi+YihMWapGka6v
+              hgKhX+hc5rj+7IEE0CXmlbYR8OYvAbAArc5vJD7UTxAY4Z7/l9d6Ydt9GQ25khfy
+              ANFgltYzlR6evLFmDjssiP/mx/ZMN91AL0ueJ9nNGv411Mu2CUW+tDCaQf35mdc=
+              =j51i
+              -----END PGP SIGNATURE-----
+            SIGNATURE
+
+            expect(signature).to eq(expected_signature.chomp)
+            expect(signature).to be_a_binary_string
+
+            expected_signed_text = <<~SIGNED_TEXT
+              tree 22bfa2fbd217df24731f43ff43a4a0f8db759dae
+              parent ae73cb07c9eeaf35924a10f713b364d32b2dd34f
+              author Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com> 1393489561 +0200
+              committer Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com> 1393489561 +0200
+
+              Feature added
+
+              Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
+            SIGNED_TEXT
+
+            expect(signed_text).to eq(expected_signed_text)
+            expect(signed_text).to be_a_binary_string
+          end
+        end
+
+        context 'when the commit has no signature' do
+          let(:commit_id) { '4b4918a572fa86f9771e5ba40fbd48e1eb03e2c6' }
+
+          it 'returns nil' do
+            expect(subject).to be_nil
+          end
+        end
+
+        context 'when the commit cannot be found' do
+          let(:commit_id) { Gitlab::Git::BLANK_SHA }
+
+          it 'returns nil' do
+            expect(subject).to be_nil
+          end
+        end
+
+        context 'when the commit ID is invalid' do
+          let(:commit_id) { '4b4918a572fa86f9771e5ba40fbd48e' }
+
+          it 'raises ArgumentError' do
+            expect { subject }.to raise_error(ArgumentError)
+          end
+        end
+      end
+
+      context 'with gitaly' do
+        it_behaves_like '.extract_signature'
+      end
+
+      context 'without gitaly', :skip_gitaly_mock do
+        it_behaves_like '.extract_signature'
+      end
+    end
   end
 
   describe '#init_from_rugged' do

spec/lib/gitlab/git/repository_spec.rb

@@ -899,44 +899,6 @@ describe Gitlab::Git::Repository, seed_helper: true do
       end
     end
 
-    context "compare results between log_by_walk and log_by_shell" do
-      let(:options) { { ref: "master" } }
-      let(:commits_by_walk) { repository.log(options).map(&:id) }
-      let(:commits_by_shell) { repository.log(options.merge({ disable_walk: true })).map(&:id) }
-
-      it { expect(commits_by_walk).to eq(commits_by_shell) }
-
-      context "with limit" do
-        let(:options) { { ref: "master", limit: 1 } }
-
-        it { expect(commits_by_walk).to eq(commits_by_shell) }
-      end
-
-      context "with offset" do
-        let(:options) { { ref: "master", offset: 1 } }
-
-        it { expect(commits_by_walk).to eq(commits_by_shell) }
-      end
-
-      context "with skip_merges" do
-        let(:options) { { ref: "master", skip_merges: true } }
-
-        it { expect(commits_by_walk).to eq(commits_by_shell) }
-      end
-
-      context "with path" do
-        let(:options) { { ref: "master", path: "encoding" } }
-
-        it { expect(commits_by_walk).to eq(commits_by_shell) }
-
-        context "with follow" do
-          let(:options) { { ref: "master", path: "encoding", follow: true } }
-
-          it { expect(commits_by_walk).to eq(commits_by_shell) }
-        end
-      end
-    end
-
     context "where provides 'after' timestamp" do
       options = { after: Time.iso8601('2014-03-03T20:15:01+00:00') }
 

spec/lib/gitlab/gpg/commit_spec.rb

@@ -38,8 +38,8 @@ describe Gitlab::Gpg::Commit do
           end
 
           before do
-            allow(Rugged::Commit).to receive(:extract_signature)
-            .with(Rugged::Repository, commit_sha)
+            allow(Gitlab::Git::Commit).to receive(:extract_signature)
+            .with(Gitlab::Git::Repository, commit_sha)
             .and_return(
               [
                 GpgHelpers::User1.signed_commit_signature,
@@ -77,8 +77,8 @@ describe Gitlab::Gpg::Commit do
           end
 
           before do
-            allow(Rugged::Commit).to receive(:extract_signature)
-            .with(Rugged::Repository, commit_sha)
+            allow(Gitlab::Git::Commit).to receive(:extract_signature)
+            .with(Gitlab::Git::Repository, commit_sha)
             .and_return(
               [
                 GpgHelpers::User3.signed_commit_signature,
@@ -116,8 +116,8 @@ describe Gitlab::Gpg::Commit do
           end
 
           before do
-            allow(Rugged::Commit).to receive(:extract_signature)
-            .with(Rugged::Repository, commit_sha)
+            allow(Gitlab::Git::Commit).to receive(:extract_signature)
+            .with(Gitlab::Git::Repository, commit_sha)
             .and_return(
               [
                 GpgHelpers::User1.signed_commit_signature,
@@ -151,8 +151,8 @@ describe Gitlab::Gpg::Commit do
           end
 
           before do
-            allow(Rugged::Commit).to receive(:extract_signature)
-            .with(Rugged::Repository, commit_sha)
+            allow(Gitlab::Git::Commit).to receive(:extract_signature)
+            .with(Gitlab::Git::Repository, commit_sha)
             .and_return(
               [
                 GpgHelpers::User1.signed_commit_signature,
@@ -187,8 +187,8 @@ describe Gitlab::Gpg::Commit do
         end
 
         before do
-          allow(Rugged::Commit).to receive(:extract_signature)
-          .with(Rugged::Repository, commit_sha)
+          allow(Gitlab::Git::Commit).to receive(:extract_signature)
+          .with(Gitlab::Git::Repository, commit_sha)
           .and_return(
             [
               GpgHelpers::User1.signed_commit_signature,
@@ -217,8 +217,8 @@ describe Gitlab::Gpg::Commit do
       let!(:commit) { create :commit, project: project, sha: commit_sha }
 
       before do
-        allow(Rugged::Commit).to receive(:extract_signature)
-          .with(Rugged::Repository, commit_sha)
+        allow(Gitlab::Git::Commit).to receive(:extract_signature)
+          .with(Gitlab::Git::Repository, commit_sha)
           .and_return(
             [
               GpgHelpers::User1.signed_commit_signature,

spec/lib/gitlab/gpg/invalid_gpg_signature_updater_spec.rb

@@ -26,8 +26,8 @@ RSpec.describe Gitlab::Gpg::InvalidGpgSignatureUpdater do
     before do
       allow_any_instance_of(Project).to receive(:commit).and_return(commit)
 
-      allow(Rugged::Commit).to receive(:extract_signature)
-        .with(Rugged::Repository, commit_sha)
+      allow(Gitlab::Git::Commit).to receive(:extract_signature)
+        .with(Gitlab::Git::Repository, commit_sha)
         .and_return(signature)
     end
 

spec/migrations/update_upload_paths_to_system_spec.rb

-require "spec_helper"
-require Rails.root.join("db", "post_migrate", "20170317162059_update_upload_paths_to_system.rb")
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20170317162059_update_upload_paths_to_system.rb')
 
-describe UpdateUploadPathsToSystem do
+describe UpdateUploadPathsToSystem, :migration do
   let(:migration) { described_class.new }
+  let(:uploads_table) { table(:uploads) }
+  let(:base_upload_attributes) { { size: 42, uploader: 'John Doe' } }
 
   before do
     allow(migration).to receive(:say)
   end
 
-  describe "#uploads_to_switch_to_new_path" do
-    it "contains only uploads with the old path for the correct models" do
-      _upload_for_other_type = create(:upload, model: create(:ci_pipeline), path: "uploads/ci_pipeline/avatar.jpg")
-      _upload_with_system_path = create(:upload, model: create(:project), path: "uploads/-/system/project/avatar.jpg")
-      _upload_with_other_path = create(:upload, model: create(:project), path: "thelongsecretforafileupload/avatar.jpg")
-      old_upload = create(:upload, model: create(:project), path: "uploads/project/avatar.jpg")
-      group_upload = create(:upload, model: create(:group), path: "uploads/group/avatar.jpg")
+  describe '#uploads_to_switch_to_new_path' do
+    it 'contains only uploads with the old path for the correct models' do
+      _upload_for_other_type = create_upload('Pipeline', 'uploads/ci_pipeline/avatar.jpg')
+      _upload_with_system_path = create_upload('Project', 'uploads/-/system/project/avatar.jpg')
+      _upload_with_other_path = create_upload('Project', 'thelongsecretforafileupload/avatar.jpg')
+      old_upload = create_upload('Project', 'uploads/project/avatar.jpg')
+      group_upload = create_upload('Namespace', 'uploads/group/avatar.jpg')
 
-      expect(Upload.where(migration.uploads_to_switch_to_new_path)).to contain_exactly(old_upload, group_upload)
+      expect(uploads_table.where(migration.uploads_to_switch_to_new_path)).to contain_exactly(old_upload, group_upload)
     end
   end
 
-  describe "#uploads_to_switch_to_old_path" do
-    it "contains only uploads with the new path for the correct models" do
-      _upload_for_other_type = create(:upload, model: create(:ci_pipeline), path: "uploads/ci_pipeline/avatar.jpg")
-      upload_with_system_path = create(:upload, model: create(:project), path: "uploads/-/system/project/avatar.jpg")
-      _upload_with_other_path = create(:upload, model: create(:project), path: "thelongsecretforafileupload/avatar.jpg")
-      _old_upload = create(:upload, model: create(:project), path: "uploads/project/avatar.jpg")
+  describe '#uploads_to_switch_to_old_path' do
+    it 'contains only uploads with the new path for the correct models' do
+      _upload_for_other_type = create_upload('Pipeline', 'uploads/ci_pipeline/avatar.jpg')
+      upload_with_system_path = create_upload('Project', 'uploads/-/system/project/avatar.jpg')
+      _upload_with_other_path = create_upload('Project', 'thelongsecretforafileupload/avatar.jpg')
+      _old_upload = create_upload('Project', 'uploads/project/avatar.jpg')
 
-      expect(Upload.where(migration.uploads_to_switch_to_old_path)).to contain_exactly(upload_with_system_path)
+      expect(uploads_table.where(migration.uploads_to_switch_to_old_path)).to contain_exactly(upload_with_system_path)
     end
   end
 
-  describe "#up", :truncate do
-    it "updates old upload records to the new path" do
-      old_upload = create(:upload, model: create(:project), path: "uploads/project/avatar.jpg")
+  describe '#up' do
+    it 'updates old upload records to the new path' do
+      old_upload = create_upload('Project', 'uploads/project/avatar.jpg')
 
       migration.up
 
-      expect(old_upload.reload.path).to eq("uploads/-/system/project/avatar.jpg")
+      expect(old_upload.reload.path).to eq('uploads/-/system/project/avatar.jpg')
     end
   end
 
-  describe "#down", :truncate do
-    it "updates the new system patsh to the old paths" do
-      new_upload = create(:upload, model: create(:project), path: "uploads/-/system/project/avatar.jpg")
+  describe '#down' do
+    it 'updates the new system patsh to the old paths' do
+      new_upload = create_upload('Project', 'uploads/-/system/project/avatar.jpg')
 
       migration.down
 
-      expect(new_upload.reload.path).to eq("uploads/project/avatar.jpg")
+      expect(new_upload.reload.path).to eq('uploads/project/avatar.jpg')
     end
   end
+
+  def create_upload(type, path)
+    uploads_table.create(base_upload_attributes.merge(model_type: type, path: path))
+  end
 end