Merge branch '267945-gitlab-migration-migrate-repositories-with-projects' into 'master'

Resolve "GitLab Migration - Migrate Repositories with Projects"

See merge request gitlab-org/gitlab!70326

app/models/project.rb

@@ -1178,7 +1178,7 @@ class Project < ApplicationRecord
   end
 
   def import?
-    external_import? || forked? || gitlab_project_import? || jira_import? || bare_repository_import?
+    external_import? || forked? || gitlab_project_import? || jira_import? || bare_repository_import? || gitlab_project_migration?
   end
 
   def external_import?
@@ -1201,6 +1201,10 @@ class Project < ApplicationRecord
     import_type == 'gitlab_project'
   end
 
+  def gitlab_project_migration?
+    import_type == 'gitlab_project_migration'
+  end
+
   def gitea_import?
     import_type == 'gitea'
   end

lib/bulk_imports/projects/graphql/get_repository_query.rb

+# frozen_string_literal: true
+
+module BulkImports
+  module Projects
+    module Graphql
+      module GetRepositoryQuery
+        extend self
+
+        def to_s
+          <<-'GRAPHQL'
+          query($full_path: ID!) {
+            project(fullPath: $full_path) {
+              httpUrlToRepo
+            }
+          }
+          GRAPHQL
+        end
+
+        def variables(context)
+          { full_path: context.entity.source_full_path }
+        end
+
+        def base_path
+          %w[data project]
+        end
+
+        def data_path
+          base_path
+        end
+
+        def page_info_path
+          base_path << 'page_info'
+        end
+      end
+    end
+  end
+end

lib/bulk_imports/projects/pipelines/repository_pipeline.rb

+# frozen_string_literal: true
+
+module BulkImports
+  module Projects
+    module Pipelines
+      class RepositoryPipeline
+        include Pipeline
+
+        abort_on_failure!
+
+        extractor Common::Extractors::GraphqlExtractor, query: Graphql::GetRepositoryQuery
+
+        def transform(_, data)
+          data.slice('httpUrlToRepo')
+        end
+
+        def load(context, data)
+          url = data['httpUrlToRepo']
+          url = url.sub("://", "://oauth2:#{context.configuration.access_token}@")
+
+          Gitlab::UrlBlocker.validate!(url, allow_local_network: allow_local_requests?, allow_localhost: allow_local_requests?)
+
+          context.portable.repository.import_repository(url)
+        end
+
+        private
+
+        def allow_local_requests?
+          Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services?
+        end
+      end
+    end
+  end
+end

lib/bulk_imports/projects/stage.rb

@@ -7,21 +7,25 @@ module BulkImports
 
       def config
         @config ||= {
-          group: {
+          project: {
             pipeline: BulkImports::Projects::Pipelines::ProjectPipeline,
             stage: 0
           },
+          repository: {
+            pipeline: BulkImports::Projects::Pipelines::RepositoryPipeline,
+            stage: 1
+          },
           labels: {
             pipeline: BulkImports::Common::Pipelines::LabelsPipeline,
-            stage: 1
+            stage: 2
           },
           issues: {
             pipeline: BulkImports::Projects::Pipelines::IssuesPipeline,
-            stage: 2
+            stage: 3
           },
           finisher: {
             pipeline: BulkImports::Common::Pipelines::EntityFinisher,
-            stage: 3
+            stage: 4
           }
         }
       end

spec/lib/bulk_imports/projects/graphql/get_repository_query_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe BulkImports::Projects::Graphql::GetRepositoryQuery do
+  describe 'query repository based on full_path' do
+    let(:entity)  { double(source_full_path: 'test', bulk_import: nil) }
+    let(:tracker) { double(entity: entity) }
+    let(:context) { BulkImports::Pipeline::Context.new(tracker) }
+
+    it 'returns project repository url' do
+      expect(described_class.to_s).to include('httpUrlToRepo')
+    end
+
+    it 'queries project based on source_full_path' do
+      expected = { full_path: entity.source_full_path }
+
+      expect(described_class.variables(context)).to eq(expected)
+    end
+  end
+end

spec/lib/bulk_imports/projects/pipelines/repository_pipeline_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe BulkImports::Projects::Pipelines::RepositoryPipeline do
+  describe '#run' do
+    let_it_be(:user) { create(:user) }
+    let_it_be(:parent) { create(:project) }
+    let_it_be(:bulk_import) { create(:bulk_import, user: user) }
+    let_it_be(:bulk_import_configuration) { create(:bulk_import_configuration, bulk_import: bulk_import) }
+
+    let_it_be(:entity) do
+      create(
+        :bulk_import_entity,
+        :project_entity,
+        bulk_import: bulk_import,
+        source_full_path: 'source/full/path',
+        destination_name: 'My Destination Repository',
+        destination_namespace: parent.full_path,
+        project: parent
+      )
+    end
+
+    let_it_be(:tracker) { create(:bulk_import_tracker, entity: entity) }
+    let_it_be(:context) { BulkImports::Pipeline::Context.new(tracker) }
+
+    context 'successfully imports repository' do
+      let(:project_data) do
+        {
+          'httpUrlToRepo' => 'http://test.git'
+        }
+      end
+
+      subject { described_class.new(context) }
+
+      it 'imports new repository into destination project' do
+        allow_next_instance_of(BulkImports::Common::Extractors::GraphqlExtractor) do |extractor|
+          allow(extractor).to receive(:extract).and_return(BulkImports::Pipeline::ExtractedData.new(data: project_data))
+        end
+
+        expect_next_instance_of(Gitlab::GitalyClient::RepositoryService) do |repository_service|
+          url = project_data['httpUrlToRepo'].sub("://", "://oauth2:#{bulk_import_configuration.access_token}@")
+          expect(repository_service).to receive(:import_repository).with(url).and_return 0
+        end
+
+        subject.run
+      end
+    end
+
+    context 'blocked local networks' do
+      let(:project_data) do
+        {
+          'httpUrlToRepo' => 'http://localhost/foo.git'
+        }
+      end
+
+      before do
+        allow(Gitlab.config.gitlab).to receive(:host).and_return('notlocalhost.gitlab.com')
+        allow(Gitlab::CurrentSettings).to receive(:allow_local_requests_from_web_hooks_and_services?).and_return(false)
+        allow_next_instance_of(BulkImports::Common::Extractors::GraphqlExtractor) do |extractor|
+          allow(extractor).to receive(:extract).and_return(BulkImports::Pipeline::ExtractedData.new(data: project_data))
+        end
+      end
+
+      subject { described_class.new(context) }
+
+      it 'imports new repository into destination project' do
+        subject.run
+        expect(context.entity.failed?).to be_truthy
+      end
+    end
+  end
+end

spec/lib/bulk_imports/projects/stage_spec.rb

@@ -6,9 +6,10 @@ RSpec.describe BulkImports::Projects::Stage do
   let(:pipelines) do
     [
       [0, BulkImports::Projects::Pipelines::ProjectPipeline],
-      [1, BulkImports::Common::Pipelines::LabelsPipeline],
-      [2, BulkImports::Projects::Pipelines::IssuesPipeline],
-      [3, BulkImports::Common::Pipelines::EntityFinisher]
+      [1, BulkImports::Projects::Pipelines::RepositoryPipeline],
+      [2, BulkImports::Common::Pipelines::LabelsPipeline],
+      [3, BulkImports::Projects::Pipelines::IssuesPipeline],
+      [4, BulkImports::Common::Pipelines::EntityFinisher]
     ]
   end