Merge branch 'ce-to-ee-2018-12-09' into 'master'

CE upstream - 2018-12-09 15:28 UTC

Closes gitlab-ce#55038 and csslab#1

See merge request gitlab-org/gitlab-ee!8768

.gitlab/CODEOWNERS

@@ -6,8 +6,8 @@
 /doc/ @axil @marcia
 
 # Frontend maintainers should see everything in `app/assets/`
-app/assets/ @ClemMakesApps @fatihacet @filipa @iamphill @mikegreiling @timzallmann
-*.scss @annabeldunstone @ClemMakesApps @fatihacet @filipa @iamphill @mikegreiling @timzallmann
+app/assets/ @ClemMakesApps @fatihacet @filipa @iamphill @mikegreiling @timzallmann @kushalpandya
+*.scss @annabeldunstone @ClemMakesApps @fatihacet @filipa @iamphill @mikegreiling @timzallmann @kushalpandya
 
 # Someone from the database team should review changes in `db/`
 db/ @abrandl @NikolayS

app/assets/javascripts/user_popovers.js

@@ -47,6 +47,7 @@ const handleUserPopoverMouseOver = event => {
       bio: null,
       organization: null,
       status: null,
+      loaded: false,
     };
     if (userId || username) {
       const UserPopoverComponent = Vue.extend(UserPopover);
@@ -94,14 +95,11 @@ const handleUserPopoverMouseOver = event => {
           renderedPopover = null;
         });
     }
-  }, 200);
+  }, 200); // 200ms delay so not every mouseover triggers Popover + API Call
 };
 
 export default elements => {
-  let userLinks = elements;
-  if (!elements) {
-    userLinks = [...document.querySelectorAll('.js-user-link')];
-  }
+  const userLinks = elements || [...document.querySelectorAll('.js-user-link')];
 
   userLinks.forEach(el => {
     el.addEventListener('mouseenter', handleUserPopoverMouseOver);

app/assets/javascripts/vue_shared/components/user_popover/user_popover.vue

@@ -53,10 +53,10 @@ export default {
       return !this.user.name;
     },
     jobInfoIsLoading() {
-      return !this.loaded && this.user.organization === null;
+      return !this.user.loaded && this.user.organization === null;
     },
     locationIsLoading() {
-      return !this.loaded && this.user.location === null;
+      return !this.user.loaded && this.user.location === null;
     },
   },
 };

changelogs/unreleased/52007-frontmatter-toml-json.yml

+---
+title: Changed frontmatter filtering to support YAML, JSON, TOML, and arbitrary languages
+merge_request: 23331
+author: Travis Miller
+type: changed

changelogs/unreleased/tc-backfill-hashed-project_repositories.yml

+---
+title: Fill project_repositories for hashed storage projects
+merge_request: 23482
+author:
+type: added

db/post_migrate/20181130102132_backfill_hashed_project_repositories.rb

+# frozen_string_literal: true
+
+class BackfillHashedProjectRepositories < ActiveRecord::Migration[4.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  BATCH_SIZE     = 1_000
+  DELAY_INTERVAL = 5.minutes
+  MIGRATION      = 'BackfillHashedProjectRepositories'
+
+  disable_ddl_transaction!
+
+  class Project < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'projects'
+  end
+
+  def up
+    queue_background_migration_jobs_by_range_at_intervals(Project, MIGRATION, DELAY_INTERVAL)
+  end
+
+  def down
+    # no-op: since there could have been existing rows before the migration do not remove anything
+  end
+end

lib/banzai/filter/front_matter_filter.rb

+# frozen_string_literal: true
+
+module Banzai
+  module Filter
+    class FrontMatterFilter < HTML::Pipeline::Filter
+      DELIM_LANG = {
+        '---' => 'yaml',
+        '+++' => 'toml',
+        ';;;' => 'json'
+      }.freeze
+
+      DELIM = Regexp.union(DELIM_LANG.keys)
+
+      PATTERN = %r{
+        \A(?:[^\r\n]*coding:[^\r\n]*)?         # optional encoding line
+        \s*
+        ^(?<delim>#{DELIM})[ \t]*(?<lang>\S*)  # opening front matter marker (optional language specifier)
+        \s*
+        ^(?<front_matter>.*?)                  # front matter (not greedy)
+        \s*
+        ^\k<delim>                             # closing front matter marker
+        \s*
+      }mx
+
+      def call
+        html.sub(PATTERN) do |_match|
+          lang = $~[:lang].presence || DELIM_LANG[$~[:delim]]
+
+          ["```#{lang}", $~[:front_matter], "```", "\n"].join("\n")
+        end
+      end
+    end
+  end
+end

lib/banzai/filter/yaml_front_matter_filter.rb

-# frozen_string_literal: true
-
-module Banzai
-  module Filter
-    class YamlFrontMatterFilter < HTML::Pipeline::Filter
-      DELIM = '---'.freeze
-
-      # Hat-tip to Middleman: https://git.io/v2e0z
-      PATTERN = %r{
-        \A(?:[^\r\n]*coding:[^\r\n]*\r?\n)?
-        (?<start>#{DELIM})[ ]*\r?\n
-        (?<frontmatter>.*?)[ ]*\r?\n?
-        ^(?<stop>#{DELIM})[ ]*\r?\n?
-        \r?\n?
-        (?<content>.*)
-      }mx.freeze
-
-      def call
-        match = PATTERN.match(html)
-
-        return html unless match
-
-        "```yaml\n#{match['frontmatter']}\n```\n\n#{match['content']}"
-      end
-    end
-  end
-end

lib/banzai/pipeline/pre_process_pipeline.rb

@@ -5,7 +5,7 @@ module Banzai
     class PreProcessPipeline < BasePipeline
       def self.filters
         FilterArray[
-          Filter::YamlFrontMatterFilter,
+          Filter::FrontMatterFilter,
           Filter::BlockquoteFenceFilter,
         ]
       end

lib/gitlab/background_migration/backfill_hashed_project_repositories.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Class that will create fill the project_repositories table
+    # for all projects that are on hashed storage and an entry is
+    # is missing in this table.
+    class BackfillHashedProjectRepositories
+      # Shard model
+      class Shard < ActiveRecord::Base
+        self.table_name = 'shards'
+      end
+
+      # Class that will find or create the shard by name.
+      # There is only a small set of shards, which would
+      # not change quickly, so look them up from memory
+      # instead of hitting the DB each time.
+      class ShardFinder
+        def find_shard_id(name)
+          shard_id = shards.fetch(name, nil)
+          return shard_id if shard_id.present?
+
+          Shard.transaction(requires_new: true) do
+            create!(name)
+          end
+        rescue ActiveRecord::RecordNotUnique
+          reload!
+          retry
+        end
+
+        private
+
+        def create!(name)
+          Shard.create!(name: name).tap { |shard| @shards[name] = shard.id }
+        end
+
+        def shards
+          @shards ||= reload!
+        end
+
+        def reload!
+          @shards = Hash[*Shard.all.map { |shard| [shard.name, shard.id] }.flatten]
+        end
+      end
+
+      # ProjectRegistry model
+      class ProjectRepository < ActiveRecord::Base
+        self.table_name = 'project_repositories'
+
+        belongs_to :project, inverse_of: :project_repository
+      end
+
+      # Project model
+      class Project < ActiveRecord::Base
+        self.table_name = 'projects'
+
+        HASHED_PATH_PREFIX = '@hashed'
+
+        HASHED_STORAGE_FEATURES = {
+          repository: 1,
+          attachments: 2
+        }.freeze
+
+        has_one :project_repository, inverse_of: :project
+
+        class << self
+          def on_hashed_storage
+            where(Project.arel_table[:storage_version]
+              .gteq(HASHED_STORAGE_FEATURES[:repository]))
+          end
+
+          def without_project_repository
+            joins(left_outer_join_project_repository)
+              .where(ProjectRepository.arel_table[:project_id].eq(nil))
+          end
+
+          def left_outer_join_project_repository
+            projects_table = Project.arel_table
+            repository_table = ProjectRepository.arel_table
+
+            projects_table
+              .join(repository_table, Arel::Nodes::OuterJoin)
+              .on(projects_table[:id].eq(repository_table[:project_id]))
+              .join_sources
+          end
+        end
+
+        def hashed_storage?
+          self.storage_version && self.storage_version >= 1
+        end
+
+        def hashed_disk_path
+          "#{HASHED_PATH_PREFIX}/#{disk_hash[0..1]}/#{disk_hash[2..3]}/#{disk_hash}"
+        end
+
+        def disk_hash
+          @disk_hash ||= Digest::SHA2.hexdigest(id.to_s)
+        end
+      end
+
+      def perform(start_id, stop_id)
+        Gitlab::Database.bulk_insert(:project_repositories, project_repositories(start_id, stop_id))
+      end
+
+      private
+
+      def project_repositories(start_id, stop_id)
+        Project.on_hashed_storage
+          .without_project_repository
+          .where(id: start_id..stop_id)
+          .map { |project| build_attributes_for_project(project) }
+          .compact
+      end
+
+      def build_attributes_for_project(project)
+        return unless project.hashed_storage?
+
+        {
+          project_id: project.id,
+          shard_id:   find_shard_id(project.repository_storage),
+          disk_path:  project.hashed_disk_path
+        }
+      end
+
+      def find_shard_id(repository_storage)
+        shard_finder.find_shard_id(repository_storage)
+      end
+
+      def shard_finder
+        @shard_finder ||= ShardFinder.new
+      end
+    end
+  end
+end

lib/gitlab/gitaly_client.rb

@@ -193,7 +193,7 @@ module Gitlab
       feature = feature_stack && feature_stack[0]
       metadata['call_site'] = feature.to_s if feature
       metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage
-      metadata['correlation_id'] = Gitlab::CorrelationId.current_id if Gitlab::CorrelationId.current_id
+      metadata['x-gitlab-correlation-id'] = Gitlab::CorrelationId.current_id if Gitlab::CorrelationId.current_id
 
       metadata.merge!(server_feature_flags)
 

spec/javascripts/user_popovers_spec.js

@@ -48,7 +48,7 @@ describe('User Popovers', () => {
         expect(document.querySelector('.popover')).toBeNull();
         done();
       });
-    }, 210);
+    }, 210); // We need to wait until the 200ms mouseover delay is over, only then the popover will be visible
   });
 
   it('Should Not show a popover on short mouse over', done => {

spec/lib/banzai/filter/front_matter_filter_spec.rb

+require 'rails_helper'
+
+describe Banzai::Filter::FrontMatterFilter do
+  include FilterSpecHelper
+
+  it 'allows for `encoding:` before the front matter' do
+    content = <<~MD
+      # encoding: UTF-8
+      ---
+      foo: foo
+      bar: bar
+      ---
+
+      # Header
+
+      Content
+    MD
+
+    output = filter(content)
+
+    expect(output).not_to match 'encoding'
+  end
+
+  it 'converts YAML front matter to a fenced code block' do
+    content = <<~MD
+      ---
+      foo: :foo_symbol
+      bar: :bar_symbol
+      ---
+
+      # Header
+
+      Content
+    MD
+
+    output = filter(content)
+
+    aggregate_failures do
+      expect(output).not_to include '---'
+      expect(output).to include "```yaml\nfoo: :foo_symbol\n"
+    end
+  end
+
+  it 'converts TOML frontmatter to a fenced code block' do
+    content = <<~MD
+      +++
+      foo = :foo_symbol
+      bar = :bar_symbol
+      +++
+
+      # Header
+
+      Content
+    MD
+
+    output = filter(content)
+
+    aggregate_failures do
+      expect(output).not_to include '+++'
+      expect(output).to include "```toml\nfoo = :foo_symbol\n"
+    end
+  end
+
+  it 'converts JSON front matter to a fenced code block' do
+    content = <<~MD
+      ;;;
+      {
+        "foo": ":foo_symbol",
+        "bar": ":bar_symbol"
+      }
+      ;;;
+
+      # Header
+
+      Content
+    MD
+
+    output = filter(content)
+
+    aggregate_failures do
+      expect(output).not_to include ';;;'
+      expect(output).to include "```json\n{\n  \"foo\": \":foo_symbol\",\n"
+    end
+  end
+
+  it 'converts arbitrary front matter to a fenced code block' do
+    content = <<~MD
+      ---arbitrary
+      foo = :foo_symbol
+      bar = :bar_symbol
+      ---
+
+      # Header
+
+      Content
+    MD
+
+    output = filter(content)
+
+    aggregate_failures do
+      expect(output).not_to include '---arbitrary'
+      expect(output).to include "```arbitrary\nfoo = :foo_symbol\n"
+    end
+  end
+
+  context 'on content without front matter' do
+    it 'returns the content unmodified' do
+      content = <<~MD
+        # This is some Markdown
+
+        It has no YAML front matter to parse.
+      MD
+
+      expect(filter(content)).to eq content
+    end
+  end
+
+  context 'on front matter without content' do
+    it 'converts YAML front matter to a fenced code block' do
+      content = <<~MD
+        ---
+        foo: :foo_symbol
+        bar: :bar_symbol
+        ---
+      MD
+
+      output = filter(content)
+
+      aggregate_failures do
+        expect(output).to eq <<~MD
+          ```yaml
+          foo: :foo_symbol
+          bar: :bar_symbol
+          ```
+
+        MD
+      end
+    end
+  end
+end

spec/lib/banzai/filter/yaml_front_matter_filter_spec.rb

-require 'rails_helper'
-
-describe Banzai::Filter::YamlFrontMatterFilter do
-  include FilterSpecHelper
-
-  it 'allows for `encoding:` before the frontmatter' do
-    content = <<-MD.strip_heredoc
-      # encoding: UTF-8
-      ---
-      foo: foo
-      ---
-
-      # Header
-
-      Content
-    MD
-
-    output = filter(content)
-
-    expect(output).not_to match 'encoding'
-  end
-
-  it 'converts YAML frontmatter to a fenced code block' do
-    content = <<-MD.strip_heredoc
-      ---
-      bar: :bar_symbol
-      ---
-
-      # Header
-
-      Content
-    MD
-
-    output = filter(content)
-
-    aggregate_failures do
-      expect(output).not_to include '---'
-      expect(output).to include "```yaml\nbar: :bar_symbol\n```"
-    end
-  end
-
-  context 'on content without frontmatter' do
-    it 'returns the content unmodified' do
-      content = <<-MD.strip_heredoc
-        # This is some Markdown
-
-        It has no YAML frontmatter to parse.
-      MD
-
-      expect(filter(content)).to eq content
-    end
-  end
-end

spec/lib/gitlab/background_migration/backfill_hashed_project_repositories_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::BackfillHashedProjectRepositories, :migration, schema: 20181130102132 do
+  let(:namespaces) { table(:namespaces) }
+  let(:project_repositories) { table(:project_repositories) }
+  let(:projects) { table(:projects) }
+  let(:shards) { table(:shards) }
+  let(:group) { namespaces.create!(name: 'foo', path: 'foo') }
+  let(:shard) { shards.create!(name: 'default') }
+
+  describe described_class::ShardFinder do
+    describe '#find_shard_id' do
+      it 'creates a new shard when it does not exist yet' do
+        expect { subject.find_shard_id('other') }.to change(shards, :count).by(1)
+      end
+
+      it 'returns the shard when it exists' do
+        shards.create(id: 5, name: 'other')
+
+        shard_id = subject.find_shard_id('other')
+
+        expect(shard_id).to eq(5)
+      end
+
+      it 'only queries the database once to retrieve shards' do
+        subject.find_shard_id('default')
+
+        expect { subject.find_shard_id('default') }.not_to exceed_query_limit(0)
+      end
+    end
+  end
+
+  describe described_class::Project do
+    describe '.on_hashed_storage' do
+      it 'finds projects with repository on hashed storage' do
+        projects.create!(id: 1, name: 'foo', path: 'foo', namespace_id: group.id, storage_version: 1)
+        projects.create!(id: 2, name: 'bar', path: 'bar', namespace_id: group.id, storage_version: 2)
+        projects.create!(id: 3, name: 'baz', path: 'baz', namespace_id: group.id, storage_version: 0)
+        projects.create!(id: 4, name: 'zoo', path: 'zoo', namespace_id: group.id, storage_version: nil)
+
+        expect(described_class.on_hashed_storage.pluck(:id)).to match_array([1, 2])
+      end
+    end
+
+    describe '.without_project_repository' do
+      it 'finds projects which do not have a projects_repositories entry' do
+        projects.create!(id: 1, name: 'foo', path: 'foo', namespace_id: group.id)
+        projects.create!(id: 2, name: 'bar', path: 'bar', namespace_id: group.id)
+        project_repositories.create!(project_id: 2, disk_path: '@phony/foo/bar', shard_id: shard.id)
+
+        expect(described_class.without_project_repository.pluck(:id)).to contain_exactly(1)
+      end
+    end
+  end
+
+  describe '#perform' do
+    it 'creates a project_repository row for projects on hashed storage that need one' do
+      projects.create!(id: 1, name: 'foo', path: 'foo', namespace_id: group.id, storage_version: 1)
+      projects.create!(id: 2, name: 'bar', path: 'bar', namespace_id: group.id, storage_version: 2)
+
+      expect { described_class.new.perform(1, projects.last.id) }.to change(project_repositories, :count).by(2)
+    end
+
+    it 'does nothing for projects on hashed storage that have already a project_repository row' do
+      projects.create!(id: 1, name: 'foo', path: 'foo', namespace_id: group.id, storage_version: 1)
+      project_repositories.create!(project_id: 1, disk_path: '@phony/foo/bar', shard_id: shard.id)
+
+      expect { described_class.new.perform(1, projects.last.id) }.not_to change(project_repositories, :count)
+    end
+
+    it 'does nothing for projects on legacy storage' do
+      projects.create!(name: 'foo', path: 'foo', namespace_id: group.id, storage_version: 0)
+
+      expect { described_class.new.perform(1, projects.last.id) }.not_to change(project_repositories, :count)
+    end
+
+    it 'inserts rows in a single query' do
+      projects.create!(name: 'foo', path: 'foo', namespace_id: group.id, storage_version: 1, repository_storage: shard.name)
+
+      control_count = ActiveRecord::QueryRecorder.new { described_class.new.perform(1, projects.last.id) }
+
+      projects.create!(name: 'bar', path: 'bar', namespace_id: group.id, storage_version: 1, repository_storage: shard.name)
+      projects.create!(name: 'zoo', path: 'zoo', namespace_id: group.id, storage_version: 1, repository_storage: shard.name)
+
+      expect { described_class.new.perform(1, projects.last.id) }.not_to exceed_query_limit(control_count)
+    end
+  end
+end