BulkImports: Add fields to track concurrent work

Add a way to track the status of each BulkImport's pipeline
individually. The end goal is to enable BulkImports to run in smaller
and more concurrent jobs. The pipeline status/progress will be tracked
on the `bulk_import_trackers` table, using `BulkImports::Tracker` model.

More information on:
- https://gitlab.com/gitlab-org/gitlab/-/issues/323382

The next steps
- https://gitlab.com/gitlab-org/gitlab/-/issues/323384
  will be to run each pipeline on its own background job.

app/models/bulk_imports/tracker.rb

 # frozen_string_literal: true
 
-# This model is responsible for keeping track of the requests/pagination
-# happening during a Group Migration (BulkImport).
 class BulkImports::Tracker < ApplicationRecord
   self.table_name = 'bulk_import_trackers'
 
@@ -15,4 +13,32 @@ class BulkImports::Tracker < ApplicationRecord
     uniqueness: { scope: :bulk_import_entity_id }
 
   validates :next_page, presence: { if: :has_next_page? }
+
+  validates :stage, presence: true
+
+  state_machine :status, initial: :created do
+    state :created, value: 0
+    state :started, value: 1
+    state :finished, value: 2
+    state :failed, value: -1
+    state :skipped, value: -2
+
+    event :start do
+      transition created: :started
+    end
+
+    event :finish do
+      transition started: :finished
+      transition failed: :failed
+      transition skipped: :skipped
+    end
+
+    event :skip do
+      transition any => :skipped
+    end
+
+    event :fail_op do
+      transition any => :failed
+    end
+  end
 end

changelogs/unreleased/kassio-bulkimports-add-new-fields-to-trackers.yml

+---
+title: Add fields to BulkImports::Tracker for concurrent work
+merge_request: 55686
+author:
+type: changed

db/migrate/20210303193544_add_concurrent_fields_to_bulk_imports_trackers.rb

+# frozen_string_literal: true
+
+class AddConcurrentFieldsToBulkImportsTrackers < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  # rubocop:disable Migration/AddLimitToTextColumns
+  # rubocop:disable Rails/NotNullColumn
+  def up
+    add_column :bulk_import_trackers, :jid, :text
+    add_column :bulk_import_trackers, :stage, :smallint, default: 0, null: false
+    add_column :bulk_import_trackers, :status, :smallint, default: 0, null: false
+  end
+  # rubocop:enable Migration/AddLimitToTextColumns
+  # rubocop:enable Rails/NotNullColumn
+
+  def down
+    remove_column :bulk_import_trackers, :jid, :text
+    remove_column :bulk_import_trackers, :stage, :smallint
+    remove_column :bulk_import_trackers, :status, :smallint
+  end
+end

db/migrate/20210303193605_add_text_limits_to_bulk_imports_trackers_jid_and_pipeline_name.rb

+# frozen_string_literal: true
+
+class AddTextLimitsToBulkImportsTrackersJidAndPipelineName < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_text_limit :bulk_import_trackers, :jid, 255
+  end
+
+  def down
+    remove_text_limit :bulk_import_trackers, :jid
+  end
+end

db/schema_migrations/20210303193544

+4d88c8ff507a985d90ed13dc16eba068e440efd12e5ec72782cef2a4bc3cf9e7
\ No newline at end of file

db/schema_migrations/20210303193605

+d79c7059c9d79b3e437ed649f19e587e713ba8dbd0af9f58ace4f4f522b4ac11
\ No newline at end of file

db/structure.sql

@@ -10219,8 +10219,12 @@ CREATE TABLE bulk_import_trackers (
     relation text NOT NULL,
     next_page text,
     has_next_page boolean DEFAULT false NOT NULL,
+    jid text,
+    stage smallint DEFAULT 0 NOT NULL,
+    status smallint DEFAULT 0 NOT NULL,
     CONSTRAINT check_2d45cae629 CHECK ((char_length(relation) <= 255)),
     CONSTRAINT check_40aeaa600b CHECK ((char_length(next_page) <= 255)),
+    CONSTRAINT check_603f91cb06 CHECK ((char_length(jid) <= 255)),
     CONSTRAINT check_next_page_requirement CHECK (((has_next_page IS FALSE) OR (next_page IS NOT NULL)))
 );
 

spec/factories/bulk_import/trackers.rb

@@ -4,6 +4,7 @@ FactoryBot.define do
   factory :bulk_import_tracker, class: 'BulkImports::Tracker' do
     association :entity, factory: :bulk_import_entity
 
+    stage { 0 }
     relation { :relation }
     has_next_page { false }
   end

spec/models/bulk_imports/tracker_spec.rb

@@ -15,6 +15,8 @@ RSpec.describe BulkImports::Tracker, type: :model do
     it { is_expected.to validate_presence_of(:relation) }
     it { is_expected.to validate_uniqueness_of(:relation).scoped_to(:bulk_import_entity_id) }
 
+    it { is_expected.to validate_presence_of(:stage) }
+
     context 'when has_next_page is true' do
       it "validates presence of `next_page`" do
         tracker = build(:bulk_import_tracker, has_next_page: true)