Merge branch 'add_measurement_to_import_rake_task' into 'master'

Add measurement to import_rake task

See merge request gitlab-org/gitlab!24475

doc/development/import_project.md

@@ -53,8 +53,18 @@ As part of this script we also disable direct and background upload to avoid sit
 
 We can simply run this script from the terminal:
 
+Parameters:
+
+| Attribute | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `username`      | string | yes | User name |
+| `namespace_path` | string | yes | Namespace path |
+| `project_path` | string | yes | Project name |
+| `archive_path` | string | yes | Path to the exported project tarball you want to import |
+| `measurement_enabled` | boolean | no | Measure execution time, number of SQL calls and GC count |
+
 ```shell
-bundle exec rake "gitlab:import_export:import[root, root, testingprojectimport, /path/to/file.tar.gz]"
+bundle exec rake "gitlab:import_export:import[root, root, testingprojectimport, /path/to/file.tar.gz, true]"
 ```
 
 ### Importing via the Rails console

lib/tasks/gitlab/import_export/import.rake

@@ -7,12 +7,12 @@
 #   2. Performs Sidekiq job synchronously
 #
 # @example
-#   bundle exec rake "gitlab:import_export:import[root, root, imported_project, /path/to/file.tar.gz]"
+#   bundle exec rake "gitlab:import_export:import[root, root, imported_project, /path/to/file.tar.gz, true]"
 #
 namespace :gitlab do
   namespace :import_export do
     desc 'GitLab | Import/Export | EXPERIMENTAL | Import large project archives'
-    task :import, [:username, :namespace_path, :project_path, :archive_path] => :gitlab_environment do |_t, args|
+    task :import, [:username, :namespace_path, :project_path, :archive_path, :measurement_enabled] => :gitlab_environment do |_t, args|
       # Load it here to avoid polluting Rake tasks with Sidekiq test warnings
       require 'sidekiq/testing'
 
@@ -26,7 +26,8 @@ namespace :gitlab do
         namespace_path: args.namespace_path,
         project_path:   args.project_path,
         username:       args.username,
-        file_path:      args.archive_path
+        file_path:      args.archive_path,
+        measurement_enabled: args.measurement_enabled == 'true'
       ).import
     end
   end
@@ -38,6 +39,7 @@ class GitlabProjectImport
     @file_path    = opts.fetch(:file_path)
     @namespace    = Namespace.find_by_full_path(opts.fetch(:namespace_path))
     @current_user = User.find_by_username(opts.fetch(:username))
+    @measurement_enabled = opts.fetch(:measurement_enabled)
   end
 
   def import
@@ -72,6 +74,54 @@ class GitlabProjectImport
     RequestStore.clear!
   end
 
+  def with_count_queries(&block)
+    count = 0
+
+    counter_f = ->(name, started, finished, unique_id, payload) {
+      unless payload[:name].in? %w[CACHE SCHEMA]
+        count += 1
+      end
+    }
+
+    ActiveSupport::Notifications.subscribed(counter_f, "sql.active_record", &block)
+
+    puts "Number of sql calls: #{count}"
+  end
+
+  def with_gc_counter
+    gc_counts_before = GC.stat.select { |k, v| k =~ /count/ }
+    yield
+    gc_counts_after = GC.stat.select { |k, v| k =~ /count/ }
+    stats = gc_counts_before.merge(gc_counts_after) { |k, vb, va| va - vb }
+    puts "Total GC count: #{stats[:count]}"
+    puts "Minor GC count: #{stats[:minor_gc_count]}"
+    puts "Major GC count: #{stats[:major_gc_count]}"
+  end
+
+  def with_measure_time
+    timing = Benchmark.realtime do
+      yield
+    end
+
+    time = Time.at(timing).utc.strftime("%H:%M:%S")
+    puts "Time to finish: #{time}"
+  end
+
+  def with_measuring
+    puts "Measuring enabled..."
+    with_gc_counter do
+      with_count_queries do
+        with_measure_time do
+          yield
+        end
+      end
+    end
+  end
+
+  def measurement_enabled?
+    @measurement_enabled != false
+  end
+
   # We want to ensure that all Sidekiq jobs are executed
   # synchronously as part of that process.
   # This ensures that all expensive operations do not escape
@@ -79,8 +129,13 @@ class GitlabProjectImport
   def with_isolated_sidekiq_job
     Sidekiq::Testing.fake! do
       with_request_store do
+        # If you are attempting to import a large project into a development environment,
+        # you may see Gitaly throw an error about too many calls or invocations.
+        # This is due to a n+1 calls limit being set for development setups (not enforced in production)
+        # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24475#note_283090635
+        # For development setups, this code-path will be excluded from n+1 detection.
         ::Gitlab::GitalyClient.allow_n_plus_1_calls do
-          yield
+          measurement_enabled? ? with_measuring { yield } : yield
         end
       end
 

spec/support/shared_examples/tasks/gitlab/import_export/import_measurement_shared_examples.rb

+# frozen_string_literal: true
+
+RSpec.shared_examples 'import measurement' do
+  context 'when measurement is enabled' do
+    let(:measurement_enabled) { true }
+
+    it 'prints measurement results' do
+      expect { subject }.to output(including('Measuring enabled...', 'Number of sql calls:', 'Total GC count:', 'Total GC count:')).to_stdout
+    end
+  end
+
+  context 'when measurement is not enabled' do
+    let(:measurement_enabled) { false }
+
+    it 'does not output measurement results' do
+      expect { subject }.not_to output(/Measuring enabled.../).to_stdout
+    end
+  end
+
+  context 'when measurement is not provided' do
+    let(:task_params) { [username, namespace_path, project_name, archive_path] }
+
+    it 'does not output measurement results' do
+      expect { subject }.not_to output(/Measuring enabled.../).to_stdout
+    end
+
+    it 'does not raise any exception' do
+      expect { subject }.not_to raise_error
+    end
+  end
+end

spec/tasks/gitlab/import_export/import_rake_spec.rb

@@ -6,7 +6,8 @@ describe 'gitlab:import_export:import rake task' do
   let(:username) { 'root' }
   let(:namespace_path) { username }
   let!(:user) { create(:user, username: username) }
-  let(:task_params) { [username, namespace_path, project_name, archive_path] }
+  let(:measurement_enabled) { false }
+  let(:task_params) { [username, namespace_path, project_name, archive_path, measurement_enabled] }
   let(:project) { Project.find_by_full_path("#{namespace_path}/#{project_name}") }
 
   before do
@@ -68,6 +69,8 @@ describe 'gitlab:import_export:import rake task' do
 
       subject
     end
+
+    it_behaves_like 'import measurement'
   end
 
   context 'when project import is invalid' do