Merge branch 'acunskis-large-gl-import' into 'master'

E2E: Large repo gitlab migration test

See merge request gitlab-org/gitlab!84450

qa/qa/resource/bulk_import_group.rb

@@ -7,10 +7,14 @@ module QA
                  :destination_group,
                  :import_id
 
-      attribute :access_token do
+      attribute :import_access_token do
         api_client.personal_access_token
       end
 
+      attribute :gitlab_address do
+        QA::Runtime::Scenario.gitlab_address
+      end
+
       # In most cases we will want to set path the same as source group
       # but it can be set to a custom name as well when imported via API
       attribute :destination_group_path do
@@ -19,18 +23,16 @@ module QA
       # Can't define path as attribue since @path is set in base class initializer
       alias_method :path, :destination_group_path
 
-      delegate :gitlab_address, to: 'QA::Runtime::Scenario'
-
-      def fabricate_via_browser_ui!
+      def fabricate!
         Page::Main::Menu.perform(&:go_to_create_group)
 
         Page::Group::New.perform do |group|
           group.switch_to_import_tab
-          group.connect_gitlab_instance(gitlab_address, api_client.personal_access_token)
+          group.connect_gitlab_instance(gitlab_address, import_access_token)
         end
 
         Page::Group::BulkImport.perform do |import_page|
-          import_page.import_group(path, sandbox.path)
+          import_page.import_group(destination_group_path, sandbox.full_path)
         end
 
         reload!
@@ -49,7 +51,7 @@ module QA
         {
           configuration: {
             url: gitlab_address,
-            access_token: access_token
+            access_token: import_access_token
           },
           entities: [
             {

qa/qa/resource/project.rb

@@ -343,16 +343,17 @@ module QA
         parse_body(response)
       end
 
-      def pipelines
-        response = get(request_url(api_pipelines_path))
-        parse_body(response)
-      end
-
       def pipeline_schedules
         response = get(request_url(api_pipeline_schedules_path))
         parse_body(response)
       end
 
+      def pipelines(auto_paginate: false, attempts: 0)
+        return parse_body(api_get_from(api_pipelines_path)) unless auto_paginate
+
+        auto_paginated_response(request_url(api_pipelines_path, per_page: '100'), attempts: attempts)
+      end
+
       def issues(auto_paginate: false, attempts: 0)
         return parse_body(api_get_from(api_issues_path)) unless auto_paginate
 
@@ -387,9 +388,7 @@ module QA
           api_resource[:import_status] == "finished"
         end
 
-        unless mirror_succeeded
-          raise "Mirroring failed with error: #{api_resource[:import_error]}"
-        end
+        raise "Mirroring failed with error: #{api_resource[:import_error]}" unless mirror_succeeded
       end
 
       def remove_via_api!

qa/qa/specs/features/api/1_manage/import_large_github_repo_spec.rb

@@ -111,7 +111,7 @@ module QA
         user.remove_via_api! unless example.exception
         next unless defined?(@import_time)
 
-        # save data for comparison after run finished
+        # save data for comparison notification creation
         save_json(
           "data",
           {
@@ -121,26 +121,30 @@ module QA
             source: {
               name: "GitHub",
               project_name: github_repo,
-              branches: gh_branches.length,
-              commits: gh_commits.length,
-              labels: gh_labels.length,
-              milestones: gh_milestones.length,
-              mrs: gh_prs.length,
-              mr_comments: gh_prs.sum { |_k, v| v[:comments].length },
-              issues: gh_issues.length,
-              issue_comments: gh_issues.sum { |_k, v| v[:comments].length }
+              data: {
+                branches: gh_branches.length,
+                commits: gh_commits.length,
+                labels: gh_labels.length,
+                milestones: gh_milestones.length,
+                mrs: gh_prs.length,
+                mr_comments: gh_prs.sum { |_k, v| v[:comments].length },
+                issues: gh_issues.length,
+                issue_comments: gh_issues.sum { |_k, v| v[:comments].length }
+              }
             },
             target: {
               name: "GitLab",
               project_name: imported_project.path_with_namespace,
-              branches: gl_branches.length,
-              commits: gl_commits.length,
-              labels: gl_labels.length,
-              milestones: gl_milestones.length,
-              mrs: mrs.length,
-              mr_comments: mrs.sum { |_k, v| v[:comments].length },
-              issues: gl_issues.length,
-              issue_comments: gl_issues.sum { |_k, v| v[:comments].length }
+              data: {
+                branches: gl_branches.length,
+                commits: gl_commits.length,
+                labels: gl_labels.length,
+                milestones: gl_milestones.length,
+                mrs: mrs.length,
+                mr_comments: mrs.sum { |_k, v| v[:comments].length },
+                issues: gl_issues.length,
+                issue_comments: gl_issues.sum { |_k, v| v[:comments].length }
+              }
             },
             not_imported: {
               mrs: @mr_diff,
@@ -158,7 +162,7 @@ module QA
         start = Time.now
 
         # import the project and log gitlab path
-        Runtime::Logger.info("== Importing project '#{github_repo}' in to '#{imported_project.reload!.full_path}' ==")
+        logger.info("== Importing project '#{github_repo}' in to '#{imported_project.reload!.full_path}' ==")
         # fetch all objects right after import has started
         fetch_github_objects
 

qa/qa/specs/features/api/1_manage/migration/gitlab_project_migration_common.rb

@@ -4,10 +4,10 @@ module QA
   # Disable on live envs until bulk_import_projects toggle is on by default
   # Otherwise tests running in parallel can disable feature in the middle of other test
   RSpec.shared_context 'with gitlab project migration', requires_admin: 'creates a user via API',
-    feature_flag: {
-      name: 'bulk_import_projects',
-      scope: :global
-    } do
+                                                        feature_flag: {
+                                                          name: 'bulk_import_projects',
+                                                          scope: :global
+                                                        } do
     let(:source_project_with_readme) { false }
     let(:import_wait_duration) { { max_duration: 300, sleep_interval: 2 } }
     let(:admin_api_client) { Runtime::API::Client.as_admin }