Merge branch 'georgekoltsov/group-mig-epic-events' into 'master'

Migrate Epic Events when using Group Migration

See merge request gitlab-org/gitlab!54475

doc/user/group/import/index.md

@@ -48,6 +48,7 @@ The following resources are migrated to the target instance:
   - author ([Introduced in 13.9](https://gitlab.com/gitlab-org/gitlab/-/issues/298745))
   - parent epic ([Introduced in 13.9](https://gitlab.com/gitlab-org/gitlab/-/issues/297459))
   - emoji award ([Introduced in 13.9](https://gitlab.com/gitlab-org/gitlab/-/issues/297466))
+  - events ([Introduced in 13.10](https://gitlab.com/gitlab-org/gitlab/-/issues/297465))
 
 Any other items are **not** migrated.
 

ee/changelogs/unreleased/georgekoltsov-group-mig-epic-events.yml

+---
+title: Migrate Epic Events when using Group Migration
+merge_request: 54475
+author:
+type: changed

ee/lib/ee/bulk_imports/groups/graphql/get_epic_events_query.rb

+# frozen_string_literal: true
+
+module EE
+  module BulkImports
+    module Groups
+      module Graphql
+        module GetEpicEventsQuery
+          extend self
+
+          def to_s
+            <<-'GRAPHQL'
+            query($full_path: ID!, $epic_iid: ID!, $cursor: String) {
+              group(fullPath: $full_path) {
+                epic(iid: $epic_iid) {
+                  events(first: 100, after: $cursor) {
+                    page_info: pageInfo {
+                      end_cursor: endCursor
+                      has_next_page: hasNextPage
+                    }
+                    nodes {
+                      action
+                      created_at: createdAt
+                      updated_at: updatedAt
+                      author {
+                        public_email: publicEmail
+                      }
+                    }
+                  }
+                }
+              }
+            }
+            GRAPHQL
+          end
+
+          def variables(context)
+            iid = context.extra[:epic_iid]
+            tracker = "epic_#{iid}_events"
+
+            {
+              full_path: context.entity.source_full_path,
+              cursor: context.entity.next_page_for(tracker),
+              epic_iid: iid
+            }
+          end
+
+          def data_path
+            base_path << 'nodes'
+          end
+
+          def page_info_path
+            base_path << 'page_info'
+          end
+
+          private
+
+          def base_path
+            %w[data group epic events]
+          end
+        end
+      end
+    end
+  end
+end

ee/lib/ee/bulk_imports/groups/pipelines/epic_award_emoji_pipeline.rb

@@ -11,7 +11,7 @@ module EE
             query: EE::BulkImports::Groups::Graphql::GetEpicAwardEmojiQuery
 
           transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer
-          transformer ::BulkImports::Common::Transformers::AwardEmojiTransformer
+          transformer ::BulkImports::Common::Transformers::UserReferenceTransformer
 
           loader EE::BulkImports::Groups::Loaders::EpicAwardEmojiLoader
 

ee/lib/ee/bulk_imports/groups/pipelines/epic_events_pipeline.rb

+# frozen_string_literal: true
+
+module EE
+  module BulkImports
+    module Groups
+      module Pipelines
+        class EpicEventsPipeline
+          include ::BulkImports::Pipeline
+
+          extractor ::BulkImports::Common::Extractors::GraphqlExtractor,
+                    query: EE::BulkImports::Groups::Graphql::GetEpicEventsQuery
+
+          transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer
+          transformer ::BulkImports::Common::Transformers::UserReferenceTransformer, reference: 'author'
+
+          def initialize(context)
+            @context = context
+            @group = context.group
+            @epic_iids = @group.epics.order(iid: :desc).pluck(:iid) # rubocop: disable CodeReuse/ActiveRecord
+
+            set_next_epic
+          end
+
+          def transform(context, data)
+            # Only create 'reopened' & 'closed' events.
+            # 'created' event get created when epic is persisted.
+            # Avoid creating duplicates & protect from additional
+            # potential undesired events.
+            return unless data['action'] == 'REOPENED' || data['action'] == 'CLOSED'
+
+            data.merge!(
+              'group_id' => context.group.id,
+              'action' => data['action'].downcase
+            )
+          end
+
+          def load(context, data)
+            return unless data
+
+            epic = context.group.epics.find_by_iid(context.extra[:epic_iid])
+
+            return unless epic
+
+            ::Event.transaction do
+              create_event!(epic, data)
+              create_resource_state_event!(epic, data)
+            end
+          end
+
+          def after_run(extracted_data)
+            iid = context.extra[:epic_iid]
+            tracker = "epic_#{iid}_events"
+
+            context.entity.update_tracker_for(
+              relation: tracker,
+              has_next_page: extracted_data.has_next_page?,
+              next_page: extracted_data.next_page
+            )
+
+            set_next_epic unless extracted_data.has_next_page?
+
+            if extracted_data.has_next_page? || context.extra[:epic_iid]
+              run
+            end
+          end
+
+          private
+
+          def set_next_epic
+            context.extra[:epic_iid] = @epic_iids.pop
+          end
+
+          def create_event!(epic, data)
+            epic.events.create!(data)
+          end
+
+          # In order for events to be shown in the UI we need to create
+          # `ResourceStateEvent` record
+          def create_resource_state_event!(epic, data)
+            state_event_data = {
+              user_id: data['author_id'],
+              state: data['action'],
+              created_at: data['created_at']
+            }
+
+            epic.resource_state_events.create!(state_event_data)
+          end
+        end
+      end
+    end
+  end
+end

ee/lib/ee/bulk_imports/importers/group_importer.rb

@@ -12,7 +12,8 @@ module EE
         def pipelines
           super + [
             EE::BulkImports::Groups::Pipelines::EpicsPipeline,
-            EE::BulkImports::Groups::Pipelines::EpicAwardEmojiPipeline
+            EE::BulkImports::Groups::Pipelines::EpicAwardEmojiPipeline,
+            EE::BulkImports::Groups::Pipelines::EpicEventsPipeline
           ]
         end
       end

ee/spec/lib/ee/bulk_imports/groups/graphql/get_epic_events_query_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe EE::BulkImports::Groups::Graphql::GetEpicEventsQuery do
+  it 'has a valid query' do
+    context = BulkImports::Pipeline::Context.new(create(:bulk_import_entity), epic_iid: 1)
+
+    result = GitlabSchema.execute(
+      described_class.to_s,
+      variables: described_class.variables(context)
+    ).to_h
+
+    expect(result['errors']).to be_blank
+  end
+
+  describe '#data_path' do
+    it 'returns data path' do
+      expected = %w[data group epic events nodes]
+
+      expect(described_class.data_path).to eq(expected)
+    end
+  end
+
+  describe '#page_info_path' do
+    it 'returns pagination information path' do
+      expected = %w[data group epic events page_info]
+
+      expect(described_class.page_info_path).to eq(expected)
+    end
+  end
+end

ee/spec/lib/ee/bulk_imports/groups/pipelines/epic_award_emoji_pipeline_spec.rb

@@ -113,7 +113,7 @@ RSpec.describe EE::BulkImports::Groups::Pipelines::EpicAwardEmojiPipeline do
       expect(described_class.transformers)
         .to contain_exactly(
           { klass: BulkImports::Common::Transformers::ProhibitedAttributesTransformer, options: nil },
-          { klass: BulkImports::Common::Transformers::AwardEmojiTransformer, options: nil }
+          { klass: BulkImports::Common::Transformers::UserReferenceTransformer, options: nil }
         )
     end
 

ee/spec/lib/ee/bulk_imports/groups/pipelines/epic_events_pipeline_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe EE::BulkImports::Groups::Pipelines::EpicEventsPipeline do
+  let_it_be(:cursor) { 'cursor' }
+  let_it_be(:user) { create(:user) }
+  let_it_be(:group) { create(:group) }
+  let_it_be(:epic) { create(:epic, group: group) }
+  let_it_be(:tracker) { "epic_#{epic.iid}_events" }
+  let_it_be(:bulk_import) { create(:bulk_import, user: user) }
+  let_it_be(:entity) do
+    create(
+      :bulk_import_entity,
+      group: group,
+      bulk_import: bulk_import,
+      source_full_path: 'source/full/path',
+      destination_name: 'My Destination Group',
+      destination_namespace: group.full_path
+    )
+  end
+
+  let_it_be(:context) { BulkImports::Pipeline::Context.new(entity) }
+
+  before do
+    stub_licensed_features(epics: true)
+    group.add_owner(user)
+  end
+
+  subject { described_class.new(context) }
+
+  describe '#initialize' do
+    it 'update context with next epic iid' do
+      subject
+
+      expect(context.extra[:epic_iid]).to eq(epic.iid)
+    end
+  end
+
+  describe '#run' do
+    it 'imports epic events and resource state events' do
+      data = extractor_data(has_next_page: false, cursor: cursor)
+
+      allow_next_instance_of(BulkImports::Common::Extractors::GraphqlExtractor) do |extractor|
+        allow(extractor)
+          .to receive(:extract)
+          .and_return(data)
+      end
+
+      subject.run
+      expect(epic.events.first.action).to eq('closed')
+      expect(epic.resource_state_events.first.state).to eq('closed')
+    end
+  end
+
+  describe '#transform' do
+    it 'downcases action & adds group_id' do
+      data = { 'action' => 'CLOSED' }
+      result = subject.transform(context, data)
+
+      expect(result['group_id']).to eq(group.id)
+      expect(result['action']).to eq(data['action'].downcase)
+    end
+
+    context 'when action is not listed as permitted' do
+      it 'returns' do
+        data = { 'action' => 'created' }
+
+        expect(subject.transform(nil, data)).to eq(nil)
+      end
+    end
+  end
+
+  describe '#load' do
+    context 'when exception occurs during resource state event creation' do
+      it 'reverts created event' do
+        allow(subject).to receive(:create_resource_state_event!).and_raise(StandardError)
+
+        data = { 'action' => 'reopened', 'author_id' => user.id }
+
+        expect { subject.load(context, data) }.to raise_error(StandardError)
+        expect(epic.events.count).to eq(0)
+        expect(epic.resource_state_events.count).to eq(0)
+      end
+    end
+
+    context 'when epic could not be found' do
+      it 'does not create new event' do
+        context.extra[:epic_iid] = 'not_iid'
+
+        expect { subject.load(context, nil) }.to not_change { Event.count }.and not_change { ResourceStateEvent.count }
+      end
+    end
+  end
+
+  describe '#after_run' do
+    context 'when extracted data has next page' do
+      it 'updates tracker information and runs pipeline again' do
+        data = extractor_data(has_next_page: true, cursor: cursor)
+
+        expect(subject).to receive(:run)
+
+        subject.after_run(data)
+
+        page_tracker = entity.trackers.find_by(relation: tracker)
+
+        expect(page_tracker.has_next_page).to eq(true)
+        expect(page_tracker.next_page).to eq(cursor)
+      end
+    end
+
+    context 'when extracted data has no next page' do
+      it 'updates tracker information and does not run pipeline' do
+        data = extractor_data(has_next_page: false)
+
+        expect(subject).not_to receive(:run)
+
+        subject.after_run(data)
+
+        page_tracker = entity.trackers.find_by(relation: tracker)
+
+        expect(page_tracker.has_next_page).to eq(false)
+        expect(page_tracker.next_page).to be_nil
+      end
+
+      it 'updates context with next epic iid' do
+        epic2 = create(:epic, group: group)
+        data = extractor_data(has_next_page: false)
+
+        expect(subject).to receive(:run)
+
+        subject.after_run(data)
+
+        expect(context.extra[:epic_iid]).to eq(epic2.iid)
+      end
+    end
+  end
+
+  describe 'pipeline parts' do
+    it { expect(described_class).to include_module(BulkImports::Pipeline) }
+    it { expect(described_class).to include_module(BulkImports::Pipeline::Runner) }
+
+    it 'has extractors' do
+      expect(described_class.get_extractor)
+        .to eq(
+          klass: BulkImports::Common::Extractors::GraphqlExtractor,
+          options: {
+            query: EE::BulkImports::Groups::Graphql::GetEpicEventsQuery
+          }
+        )
+    end
+
+    it 'has transformers' do
+      expect(described_class.transformers)
+        .to contain_exactly(
+          { klass: BulkImports::Common::Transformers::ProhibitedAttributesTransformer, options: nil },
+          { klass: BulkImports::Common::Transformers::UserReferenceTransformer, options: { reference: 'author' } }
+        )
+    end
+  end
+
+  def extractor_data(has_next_page:, cursor: nil)
+    data = [
+      {
+        'action' => 'CLOSED',
+        'created_at' => '2021-02-15T15:08:57Z',
+        'updated_at' => '2021-02-15T16:08:57Z',
+        'author' => {
+          'public_email' => user.email
+        }
+      }
+    ]
+
+    page_info = {
+      'end_cursor' => cursor,
+      'has_next_page' => has_next_page
+    }
+
+    BulkImports::Pipeline::ExtractedData.new(data: data, page_info: page_info)
+  end
+end

lib/bulk_imports/common/transformers/prohibited_attributes_transformer.rb

@@ -15,6 +15,8 @@ module BulkImports
         ).freeze
 
         def transform(context, data)
+          return unless data
+
           data.each_with_object({}) do |(key, value), result|
             prohibited = prohibited_key?(key)
 

lib/bulk_imports/common/transformers/award_emoji_transformer.rb → lib/bulk_imports/common/transformers/user_reference_transformer.rb

 # frozen_string_literal: true
 
+# UserReferenceTransformer replaces specified user
+# reference key with a user id being either:
+#   - A user id found by `public_email` in the group
+#   - Current user id
+# under a new key `"#{@reference}_id"`.
 module BulkImports
   module Common
     module Transformers
-      class AwardEmojiTransformer
+      class UserReferenceTransformer
+        DEFAULT_REFERENCE = 'user'
+
+        def initialize(options = {})
+          @reference = options[:reference] || DEFAULT_REFERENCE
+          @suffixed_reference = "#{@reference}_id"
+        end
+
         def transform(context, data)
-          user = find_user(context, data&.dig('user', 'public_email')) || context.current_user
+          return unless data
+
+          user = find_user(context, data&.dig(@reference, 'public_email')) || context.current_user
 
           data
-            .except('user')
-            .merge('user_id' => user.id)
+            .except(@reference)
+            .merge(@suffixed_reference => user.id)
         end
 
         private

lib/bulk_imports/pipeline.rb

@@ -3,6 +3,7 @@
 module BulkImports
   module Pipeline
     extend ActiveSupport::Concern
+    include Gitlab::Utils::StrongMemoize
     include Gitlab::ClassAttributes
     include Runner
 
@@ -60,12 +61,17 @@ module BulkImports
       #   end
       # end
       #
-      # In the example above `MyTransformerOne` is the first to run and
-      # the instance `#transform` method is the last.
+      # In the example above `#transform` is the first to run and
+      # `MyTransformerTwo` method is the last.
       def transformers
-        @transformers ||= self.class.transformers.map(&method(:instantiate))
-        @transformers << self if respond_to?(:transform) && @transformers.exclude?(self)
-        @transformers
+        strong_memoize(:transformers) do
+          defined_transformers = self.class.transformers.map(&method(:instantiate))
+
+          transformers = []
+          transformers << self if respond_to?(:transform)
+          transformers.concat(defined_transformers)
+          transformers
+        end
       end
 
       # Fetch pipeline loader.
@@ -126,7 +132,7 @@ module BulkImports
       end
 
       def transformers
-        class_attributes[:transformers]
+        class_attributes[:transformers] || []
       end
 
       def get_loader

spec/lib/bulk_imports/common/transformers/prohibited_attributes_transformer_spec.rb

@@ -68,5 +68,11 @@ RSpec.describe BulkImports::Common::Transformers::ProhibitedAttributesTransforme
 
       expect(transformed_hash).to eq(expected_hash)
     end
+
+    context 'when there is no data to transform' do
+      it 'returns' do
+        expect(subject.transform(nil, nil)).to be_nil
+      end
+    end
   end
 end

spec/lib/bulk_imports/common/transformers/award_emoji_transformer_spec.rb → spec/lib/bulk_imports/common/transformers/user_reference_transformer_spec.rb

@@ -2,7 +2,7 @@
 
 require 'spec_helper'
 
-RSpec.describe BulkImports::Common::Transformers::AwardEmojiTransformer do
+RSpec.describe BulkImports::Common::Transformers::UserReferenceTransformer do
   describe '#transform' do
     let_it_be(:user) { create(:user) }
     let_it_be(:group) { create(:group) }
@@ -12,7 +12,6 @@ RSpec.describe BulkImports::Common::Transformers::AwardEmojiTransformer do
 
     let(:hash) do
       {
-        'name' => 'thumbs up',
         'user' => {
           'public_email' => email
         }
@@ -44,5 +43,27 @@ RSpec.describe BulkImports::Common::Transformers::AwardEmojiTransformer do
 
       include_examples 'sets user_id and removes user key'
     end
+
+    context 'when there is no data to transform' do
+      it 'returns' do
+        expect(subject.transform(nil, nil)).to be_nil
+      end
+    end
+
+    context 'when custom reference is provided' do
+      it 'updates provided reference' do
+        hash = {
+          'author' => {
+            'public_email' => user.email
+          }
+        }
+
+        transformer = described_class.new(reference: 'author')
+        result = transformer.transform(context, hash)
+
+        expect(result['author']).to be_nil
+        expect(result['author_id']).to eq(user.id)
+      end
+    end
   end
 end

spec/lib/bulk_imports/pipeline_spec.rb

@@ -117,4 +117,27 @@ RSpec.describe BulkImports::Pipeline do
       end
     end
   end
+
+  describe '#transformers' do
+    before do
+      klass = Class.new do
+        include BulkImports::Pipeline
+
+        transformer BulkImports::Transformer
+
+        def transform; end
+      end
+
+      stub_const('BulkImports::TransformersPipeline', klass)
+    end
+
+    it 'has instance transform method first to run' do
+      transformer = double
+      allow(BulkImports::Transformer).to receive(:new).and_return(transformer)
+
+      pipeline = BulkImports::TransformersPipeline.new(nil)
+
+      expect(pipeline.send(:transformers)).to eq([pipeline, transformer])
+    end
+  end
 end