Automatic merge of gitlab-org/gitlab-ce master

app/models/internal_id.rb

@@ -66,6 +66,17 @@ class InternalId < ActiveRecord::Base
       InternalIdGenerator.new(subject, scope, usage, init).generate
     end
 
+    # Flushing records is generally safe in a sense that those
+    # records are going to be re-created when needed.
+    #
+    # A filter condition has to be provided to not accidentally flush
+    # records for all projects.
+    def flush_records!(filter)
+      raise ArgumentError, "filter cannot be empty" if filter.blank?
+
+      where(filter).delete_all
+    end
+
     def available?
       @available_flag ||= ActiveRecord::Migrator.current_version >= REQUIRED_SCHEMA_VERSION # rubocop:disable Gitlab/PredicateMemoization
     end
@@ -111,7 +122,7 @@ class InternalId < ActiveRecord::Base
 
     # Generates next internal id and returns it
     def generate
-      InternalId.transaction do
+      subject.transaction do
         # Create a record in internal_ids if one does not yet exist
         # and increment its last value
         #
@@ -125,7 +136,7 @@ class InternalId < ActiveRecord::Base
     #
     # Note this will acquire a ROW SHARE lock on the InternalId record
     def track_greatest(new_value)
-      InternalId.transaction do
+      subject.transaction do
         (lookup || create_record).track_greatest_and_save!(new_value)
       end
     end
@@ -148,7 +159,7 @@ class InternalId < ActiveRecord::Base
     # violation. We can safely roll-back the nested transaction and perform
     # a lookup instead to retrieve the record.
     def create_record
-      InternalId.transaction(requires_new: true) do
+      subject.transaction(requires_new: true) do
         InternalId.create!(
           **scope,
           usage: usage_value,

app/models/project.rb

@@ -1585,6 +1585,13 @@ class Project < ActiveRecord::Base
   def after_import
     repository.after_import
     wiki.repository.after_import
+
+    # The import assigns iid values on its own, e.g. by re-using GitHub ids.
+    # Flush existing InternalId records for this project for consistency reasons.
+    # Those records are going to be recreated with the next normal creation
+    # of a model instance (e.g. an Issue).
+    InternalId.flush_records!(project: self)
+
     import_state.finish
     import_state.remove_jid
     update_project_counter_caches
@@ -1689,11 +1696,19 @@ class Project < ActiveRecord::Base
       .append(key: 'CI_PROJECT_NAMESPACE', value: namespace.full_path)
       .append(key: 'CI_PROJECT_URL', value: web_url)
       .append(key: 'CI_PROJECT_VISIBILITY', value: visibility)
+      .concat(pages_variables)
       .concat(container_registry_variables)
       .concat(auto_devops_variables)
       .concat(api_variables)
   end
 
+  def pages_variables
+    Gitlab::Ci::Variables::Collection.new.tap do |variables|
+      variables.append(key: 'CI_PAGES_DOMAIN', value: Gitlab.config.pages.host)
+      variables.append(key: 'CI_PAGES_URL', value: pages_url)
+    end
+  end
+
   def api_variables
     Gitlab::Ci::Variables::Collection.new.tap do |variables|
       variables.append(key: 'CI_API_V4_URL', value: API::Helpers::Version.new('v4').root_url)

changelogs/unreleased/ab-54270-github-iid.yml

+---
+title: Improve efficiency of GitHub importer by reducing amount of locks needed.
+merge_request: 24102
+author:
+type: performance

changelogs/unreleased/adrianmoisey-GITLAB_PAGES_PREDEFINED_VARIABLES.yml

+---
+title: Add GitLab Pages predefined CI variables 'CI_PAGES_DOMAIN' and 'CI_PAGES_URL'
+merge_request: 24504
+author: Adrian Moisey
+type: added

db/post_migrate/20190102152410_delete_inconsistent_internal_id_records2.rb

+# frozen_string_literal: true
+class DeleteInconsistentInternalIdRecords2 < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  # This migration cleans up any inconsistent records in internal_ids.
+  #
+  # That is, it deletes records that track a `last_value` that is
+  # smaller than the maximum internal id (usually `iid`) found in
+  # the corresponding model records.
+
+  def up
+    disable_statement_timeout do
+      delete_internal_id_records('milestones', 'project_id')
+      delete_internal_id_records('milestones', 'namespace_id', 'group_id')
+    end
+  end
+
+  class InternalId < ActiveRecord::Base
+    self.table_name = 'internal_ids'
+    enum usage: { issues: 0, merge_requests: 1, deployments: 2, milestones: 3, epics: 4, ci_pipelines: 5 }
+  end
+
+  private
+
+  def delete_internal_id_records(base_table, scope_column_name, base_scope_column_name = scope_column_name)
+    sql = <<~SQL
+      SELECT id FROM ( -- workaround for MySQL
+       SELECT internal_ids.id FROM (
+         SELECT #{base_scope_column_name} AS #{scope_column_name}, max(iid) as maximum_iid from #{base_table} GROUP BY #{scope_column_name}
+       ) maxima JOIN internal_ids USING (#{scope_column_name})
+       WHERE internal_ids.usage=#{InternalId.usages.fetch(base_table)} AND maxima.maximum_iid > internal_ids.last_value
+      ) internal_ids
+    SQL
+
+    InternalId.where("id IN (#{sql})").tap do |ids| # rubocop:disable GitlabSecurity/SqlInjection
+      say "Deleting internal_id records for #{base_table}: #{ids.map { |i| [i.project_id, i.last_value] }}" unless ids.empty?
+    end.delete_all
+  end
+end

doc/ci/variables/README.md

@@ -87,6 +87,8 @@ future GitLab releases.**
 | **CI_NODE_INDEX**                         | 11.5   | all    | Index of the job in the job set. If the job is not parallelized, this variable is not set. |
 | **CI_NODE_TOTAL**                         | 11.5   | all    | Total number of instances of this job running in parallel. If the job is not parallelized, this variable is set to `1`. |
 | **CI_API_V4_URL**                         | 11.7   | all    | The GitLab API v4 root URL |
+| **CI_PAGES_DOMAIN**                       | 11.8   | all    | The configured domain that hosts GitLab Pages. |
+| **CI_PAGES_URL**                          | 11.8   | all    | URL to GitLab Pages-built pages. Always belongs to a subdomain of `CI_PAGES_DOMAIN`. |
 | **CI_PIPELINE_ID**                        | 8.10   | all    | The unique id of the current pipeline that GitLab CI uses internally |
 | **CI_PIPELINE_IID**                       | 11.0   | all    | The unique id of the current pipeline scoped to project |
 | **CI_PIPELINE_SOURCE**                    | 10.0   | all    | Indicates how the pipeline was triggered. Possible options are: `push`, `web`, `trigger`, `schedule`, `api`, and `pipeline`. For pipelines created before GitLab 9.5, this will show as `unknown` |
@@ -415,6 +417,10 @@ Running on runner-8a2f473d-project-1796893-concurrent-0 via runner-8a2f473d-mach
 ++ CI_SERVER_VERSION=8.14.3-ee
 ++ export CI_SERVER_REVISION=82823
 ++ CI_SERVER_REVISION=82823
+++ export CI_PAGES_DOMAIN=gitlab.io
+++ CI_PAGES_DOMAIN=gitlab.io
+++ export CI_PAGES_URL=https://gitlab-examples.gitlab.io/ci-debug-trace
+++ CI_PAGES_URL=https://gitlab-examples.gitlab.io/ci-debug-trace
 ++ export CI_PROJECT_ID=17893
 ++ CI_PROJECT_ID=17893
 ++ export CI_PROJECT_NAME=ci-debug-trace
@@ -518,6 +524,8 @@ export CI_JOB_TRIGGERED="true"
 export CI_JOB_TOKEN="abcde-1234ABCD5678ef"
 export CI_PIPELINE_ID="1000"
 export CI_PIPELINE_IID="10"
+export CI_PAGES_DOMAIN="gitlab.io"
+export CI_PAGES_URL="https://gitlab-org.gitlab.io/gitlab-ce"
 export CI_PROJECT_ID="34"
 export CI_PROJECT_DIR="/builds/gitlab-org/gitlab-ce"
 export CI_PROJECT_NAME="gitlab-ce"

lib/gitlab/github_import/bulk_importing.rb

@@ -15,12 +15,10 @@ module Gitlab
       end
 
       # Bulk inserts the given rows into the database.
-      def bulk_insert(model, rows, batch_size: 100, pre_hook: nil)
+      def bulk_insert(model, rows, batch_size: 100)
         rows.each_slice(batch_size) do |slice|
-          pre_hook.call(slice) if pre_hook
           Gitlab::Database.bulk_insert(model.table_name, slice)
         end
-        rows
       end
     end
   end

lib/gitlab/github_import/importer/issue_importer.rb

@@ -57,11 +57,7 @@ module Gitlab
             updated_at: issue.updated_at
           }
 
-          insert_and_return_id(attributes, project.issues).tap do |id|
-            # We use .insert_and_return_id which effectively disables all callbacks.
-            # Trigger iid logic here to make sure we track internal id values consistently.
-            project.issues.find(id).ensure_project_iid!
-          end
+          insert_and_return_id(attributes, project.issues)
         rescue ActiveRecord::InvalidForeignKey
           # It's possible the project has been deleted since scheduling this
           # job. In this case we'll just skip creating the issue.

lib/gitlab/github_import/importer/milestones_importer.rb

@@ -19,20 +19,10 @@ module Gitlab
         # rubocop: enable CodeReuse/ActiveRecord
 
         def execute
-          # We insert records in bulk, by-passing any standard model callbacks.
-          # The pre_hook here makes sure we track internal ids consistently.
-          # Note this has to be called before performing an insert of a batch
-          # because we're outside a transaction scope here.
-          bulk_insert(Milestone, build_milestones, pre_hook: method(:track_greatest_iid))
+          bulk_insert(Milestone, build_milestones)
           build_milestones_cache
         end
 
-        def track_greatest_iid(slice)
-          greatest_iid = slice.max { |e| e[:iid] }[:iid]
-
-          InternalId.track_greatest(nil, { project: project }, :milestones, greatest_iid, ->(_) { project.milestones.maximum(:iid) })
-        end
-
         def build_milestones
           build_database_rows(each_milestone)
         end

lib/gitlab/import/merge_request_helpers.rb

@@ -24,10 +24,6 @@ module Gitlab
 
           merge_request = project.merge_requests.reload.find(merge_request_id)
 
-          # We use .insert_and_return_id which effectively disables all callbacks.
-          # Trigger iid logic here to make sure we track internal id values consistently.
-          merge_request.ensure_target_project_iid!
-
           [merge_request, false]
         end
       rescue ActiveRecord::InvalidForeignKey

spec/lib/gitlab/github_import/bulk_importing_spec.rb

@@ -58,17 +58,5 @@ describe Gitlab::GithubImport::BulkImporting do
 
       importer.bulk_insert(model, rows, batch_size: 5)
     end
-
-    it 'calls pre_hook for each slice if given' do
-      rows = [{ title: 'Foo' }] * 10
-      model = double(:model, table_name: 'kittens')
-      pre_hook = double('pre_hook', call: nil)
-      allow(Gitlab::Database).to receive(:bulk_insert)
-
-      expect(pre_hook).to receive(:call).with(rows[0..4])
-      expect(pre_hook).to receive(:call).with(rows[5..9])
-
-      importer.bulk_insert(model, rows, batch_size: 5, pre_hook: pre_hook)
-    end
   end
 end

spec/lib/gitlab/github_import/importer/issue_importer_spec.rb

@@ -78,11 +78,6 @@ describe Gitlab::GithubImport::Importer::IssueImporter, :clean_gitlab_redis_cach
         .to receive(:id_for)
         .with(issue)
         .and_return(milestone.id)
-
-      allow(importer.user_finder)
-        .to receive(:author_id_for)
-        .with(issue)
-        .and_return([user.id, true])
     end
 
     context 'when the issue author could be found' do
@@ -177,23 +172,6 @@ describe Gitlab::GithubImport::Importer::IssueImporter, :clean_gitlab_redis_cach
 
       expect(importer.create_issue).to be_a_kind_of(Numeric)
     end
-
-    it 'triggers internal_id functionality to track greatest iids' do
-      allow(importer.user_finder)
-        .to receive(:author_id_for)
-        .with(issue)
-        .and_return([user.id, true])
-
-      issue = build_stubbed(:issue, project: project)
-      allow(importer)
-        .to receive(:insert_and_return_id)
-        .and_return(issue.id)
-      allow(project.issues).to receive(:find).with(issue.id).and_return(issue)
-
-      expect(issue).to receive(:ensure_project_iid!)
-
-      importer.create_issue
-    end
   end
 
   describe '#create_assignees' do

spec/lib/gitlab/github_import/importer/milestones_importer_spec.rb

@@ -29,25 +29,13 @@ describe Gitlab::GithubImport::Importer::MilestonesImporter, :clean_gitlab_redis
 
       expect(importer)
         .to receive(:bulk_insert)
-        .with(Milestone, [milestone_hash], any_args)
+        .with(Milestone, [milestone_hash])
 
       expect(importer)
         .to receive(:build_milestones_cache)
 
       importer.execute
     end
-
-    it 'tracks internal ids' do
-      milestone_hash = { iid: 1, title: '1.0', project_id: project.id }
-      allow(importer)
-        .to receive(:build_milestones)
-        .and_return([milestone_hash])
-
-      expect(InternalId).to receive(:track_greatest)
-        .with(nil, { project: project }, :milestones, 1, any_args)
-
-      importer.execute
-    end
   end
 
   describe '#build_milestones' do

spec/lib/gitlab/github_import/importer/pull_request_importer_spec.rb

@@ -111,16 +111,6 @@ describe Gitlab::GithubImport::Importer::PullRequestImporter, :clean_gitlab_redi
         expect(mr).to be_instance_of(MergeRequest)
         expect(exists).to eq(false)
       end
-
-      it 'triggers internal_id functionality to track greatest iids' do
-        mr = build_stubbed(:merge_request, source_project: project, target_project: project)
-        allow(importer).to receive(:insert_and_return_id).and_return(mr.id)
-        allow(project.merge_requests).to receive(:find).with(mr.id).and_return(mr)
-
-        expect(mr).to receive(:ensure_target_project_iid!)
-
-        importer.create_merge_request
-      end
     end
 
     context 'when the author could not be found' do

spec/models/ci/build_spec.rb

@@ -2134,6 +2134,8 @@ describe Ci::Build do
           { key: 'CI_PROJECT_NAMESPACE', value: project.namespace.full_path, public: true },
           { key: 'CI_PROJECT_URL', value: project.web_url, public: true },
           { key: 'CI_PROJECT_VISIBILITY', value: 'private', public: true },
+          { key: 'CI_PAGES_DOMAIN', value: Gitlab.config.pages.host, public: true },
+          { key: 'CI_PAGES_URL', value: project.pages_url, public: true },
           { key: 'CI_API_V4_URL', value: 'http://localhost/api/v4', public: true },
           { key: 'CI_PIPELINE_IID', value: pipeline.iid.to_s, public: true },
           { key: 'CI_CONFIG_PATH', value: pipeline.ci_yaml_file_path, public: true },

spec/models/internal_id_spec.rb

@@ -13,6 +13,29 @@ describe InternalId do
     it { is_expected.to validate_presence_of(:usage) }
   end
 
+  describe '.flush_records!' do
+    subject { described_class.flush_records!(project: project) }
+
+    let(:another_project) { create(:project) }
+
+    before do
+      create_list(:issue, 2, project: project)
+      create_list(:issue, 2, project: another_project)
+    end
+
+    it 'deletes all records for the given project' do
+      expect { subject }.to change { described_class.where(project: project).count }.from(1).to(0)
+    end
+
+    it 'retains records for other projects' do
+      expect { subject }.not_to change { described_class.where(project: another_project).count }
+    end
+
+    it 'does not allow an empty filter' do
+      expect { described_class.flush_records!({}) }.to raise_error(/filter cannot be empty/)
+    end
+  end
+
   describe '.generate_next' do
     subject { described_class.generate_next(issue, scope, usage, init) }
 

spec/models/project_spec.rb

@@ -4048,6 +4048,7 @@ describe Project do
       expect(import_state).to receive(:remove_jid)
       expect(project).to receive(:after_create_default_branch)
       expect(project).to receive(:refresh_markdown_cache!)
+      expect(InternalId).to receive(:flush_records!).with(project: project)
 
       project.after_import
     end