Add manifest import feature

It allows user to automatically import multiple repositories
with nested structure by uploading a manifest xml file.

AOSP project was used as an example during development of this feature.
Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>

app/controllers/import/manifest_controller.rb

+class Import::ManifestController < Import::BaseController
+  before_action :ensure_session, only: [:create, :status, :jobs]
+  before_action :group, only: [:status, :create]
+
+  def new
+  end
+
+  def status
+    @repos = session[:projects]
+
+    @already_added_projects = find_already_added_projects('manifest').where(namespace_id: group)
+    already_added_projects_names = @already_added_projects.pluck(:import_url)
+
+    @repos = @repos.to_a.reject { |repo| already_added_projects_names.include? repo[:url] }
+  end
+
+  def upload
+    group = Group.find(params[:group_id])
+
+    unless can?(current_user, :create_projects, group)
+      @errors = ["You don't have enough permissions to create projects in the selected group"]
+
+      render :new and return
+    end
+
+    manifest = Gitlab::ManifestImport::Manifest.new(params[:manifest].tempfile)
+
+    if manifest.valid?
+      session[:projects] = manifest.projects
+      session[:group_id] = group.id
+
+      flash[:notice] = "Import successfully started."
+
+      redirect_to status_import_manifest_path
+    else
+      @errors = manifest.errors
+
+      render :new
+    end
+  end
+
+  def jobs
+    render json: find_jobs('manifest')
+  end
+
+  def create
+    repository = session[:projects].find do |project|
+      project[:id] == params[:repo_id].to_i
+    end
+
+    project = Gitlab::ManifestImport::Importer.new(repository, group, current_user).execute
+
+    if project.persisted?
+      render json: ProjectSerializer.new.represent(project)
+    else
+      render json: { errors: project_save_error(project) }, status: :unprocessable_entity
+    end
+  end
+
+  private
+
+  def ensure_session
+    if session[:projects].blank? || session[:group_id].blank?
+      redirect_to(new_import_manifest_path)
+    end
+  end
+
+  def group
+    @group ||= Group.find(session[:group_id])
+  end
+end

app/services/groups/nested_create_service.rb

 module Groups
   class NestedCreateService < Groups::BaseService
-    attr_reader :group_path
+    attr_reader :group_path, :visibility_level
 
     def initialize(user, params)
       @current_user, @params = user, params.dup
-
       @group_path = @params.delete(:group_path)
+      @visibility_level = @params.delete(:visibility_level) ||
+        Gitlab::CurrentSettings.current_application_settings.default_group_visibility
     end
 
     def execute
@@ -36,11 +37,12 @@ module Groups
         new_params = params.reverse_merge(
           path: subgroup_name,
           name: subgroup_name,
-          parent: last_group
+          parent: last_group,
+          visibility_level: visibility_level
         )
-        new_params[:visibility_level] ||= Gitlab::CurrentSettings.current_application_settings.default_group_visibility
 
-        last_group = namespace_or_group(partial_path) || Groups::CreateService.new(current_user, new_params).execute
+        last_group = namespace_or_group(partial_path) ||
+          Groups::CreateService.new(current_user, new_params).execute
       end
 
       last_group

app/views/import/manifest/_form.html.haml

+= form_tag upload_import_manifest_path, multipart: true do
+  .form-group
+    = label_tag :manifest, class: 'label-light' do
+      Manifest
+    = file_field_tag :manifest, class: 'form-control-file', required: true
+    .form-text.text-muted
+      Import multiple repositories by uploading a manifest file.
+
+  .form-group
+    = label_tag :group_id, nil, class: 'label-light' do
+      Namespace
+    .input-group
+      .input-group-prepend.has-tooltip{ title: root_url }
+        .input-group-text
+          = root_url
+      = select_tag :group_id, namespaces_options(nil, display_path: true), { class: 'select2 js-select-namespace' }
+    .form-text.text-muted
+      Choose the top-level namespace for your repository imports.
+
+  .append-bottom-10
+    = submit_tag 'Import projects', class: 'btn btn-success'
+    = link_to 'Cancel', new_project_path, class: 'btn btn-cancel'

app/views/import/manifest/new.html.haml

+- page_title "Manifest Import"
+- header_title "Projects", root_path
+
+%h3.page-title
+  = icon('git')
+  Import multiple repositories
+
+- if @errors.present?
+  .alert.alert-danger
+    - @errors.each do |error|
+      = error
+
+= render 'form'

app/views/import/manifest/status.html.haml

+- page_title "Manifest import"
+- header_title "Projects", root_path
+- provider = 'manifest'
+
+%h3.page-title
+  = icon('git')
+  = _('Import multiple repositories')
+
+%p
+  = button_tag class: "btn btn-import btn-success js-import-all" do
+    = import_all_githubish_repositories_button_label
+    = icon("spinner spin", class: "loading-icon")
+
+.table-responsive
+  %table.table.import-jobs
+    %colgroup.import-jobs-from-col
+    %colgroup.import-jobs-to-col
+    %colgroup.import-jobs-status-col
+    %thead
+      %tr
+        %th= _('Repository URL')
+        %th= _('To GitLab')
+        %th= _('Status')
+    %tbody
+      - @already_added_projects.each do |project|
+        %tr{ id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}" }
+          %td
+            = link_to_project project
+          %td
+            = project.import_url
+          %td.job-status
+            - if project.import_status == 'finished'
+              %span
+                = icon('check')
+                = _('Done')
+            - elsif project.import_status == 'started'
+              = icon("spinner spin")
+              = _('Started')
+            - elsif project.import_status == 'failed'
+              = _('Failed')
+            - else
+              = project.human_import_status_name
+
+      - @repos.each do |repo|
+        %tr{ id: "repo_#{repo[:id]}" }
+          %td
+            = repo[:url]
+          %td.import-target
+            = import_project_target(@group.path, repo[:path])
+          %td.import-actions.job-status
+            = button_tag class: "btn btn-import js-add-to-import" do
+              Import
+              = icon("spinner spin", class: "loading-icon")
+
+.js-importer-status{ data: { jobs_import_path: "#{url_for([:jobs, :import, provider])}",
+                             import_path: "#{url_for([:import, provider])}",
+                             ci_cd_only: "#{has_ci_cd_only_params?}" } }

app/views/projects/_import_project_pane.html.haml

 - active_tab = local_assigns.fetch(:active_tab, 'blank')
-- f = local_assigns.fetch(:f)
 
 .project-import
   .form-group.import-btn-container.clearfix
-    = f.label :visibility_level, class: 'label-light' do #the label here seems wrong
+    %h5
       Import project from
     .import-buttons
       - if gitlab_project_import_enabled?
@@ -39,11 +38,20 @@
           = link_to new_import_gitea_path, class: 'btn import_gitea' do
             = custom_icon('go_logo')
             Gitea
-      %div
-        - if git_import_enabled?
+      - if git_import_enabled?
+        %div
           %button.btn.js-toggle-button.js-import-git-toggle-button{ type: "button", data: { toggle_open_class: 'active' } }
             = icon('git', text: 'Repo by URL')
+        %div
+          %button.btn{ type: "button" }
+            = icon('git', text: 'Manifest file')
+
   .js-toggle-content.toggle-import-form{ class: ('hide' if active_tab != 'import') }
-    %hr
+    = form_for @project, html: { class: 'new_project' } do |f|
+      %hr
       = render "shared/import_form", f: f
       = render 'new_project_fields', f: f, project_name_id: "import-url-name"
+
+  .toggle-manifest-form
+    %hr
+    = render 'import/manifest/form'

app/views/projects/new.html.haml

@@ -63,13 +63,12 @@
                   = render 'project_templates', f: f
 
         .tab-pane.import-project-pane.js-toggle-container{ id: 'import-project-pane', class: active_when(active_tab == 'import'), role: 'tabpanel' }
-          = form_for @project, html: { class: 'new_project' } do |f|
-            - if import_sources_enabled?
-              = render 'import_project_pane', f: f, active_tab: active_tab
-            - else
-              .nothing-here-block
-                %h4 No import options available
-                %p Contact an administrator to enable options for importing your project.
+          - if import_sources_enabled?
+            = render 'import_project_pane', active_tab: active_tab
+          - else
+            .nothing-here-block
+              %h4 No import options available
+              %p Contact an administrator to enable options for importing your project.
 
         -# EE-specific start
         = render 'new_ci_cd_only_project_pane', active_tab: active_tab

config/routes/import.rb

@@ -45,4 +45,10 @@ namespace :import do
   resource :gitlab_project, only: [:create, :new] do
     post :create
   end
+
+  resource :manifest, only: [:create, :new], controller: :manifest do
+    get :status
+    get :jobs
+    post :upload
+  end
 end

lib/gitlab/import_sources.rb

@@ -16,7 +16,8 @@ module Gitlab
       ImportSource.new('fogbugz',        'FogBugz',       Gitlab::FogbugzImport::Importer),
       ImportSource.new('git',            'Repo by URL',   nil),
       ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer),
-      ImportSource.new('gitea',          'Gitea',         Gitlab::LegacyGithubImport::Importer)
+      ImportSource.new('gitea',          'Gitea',         Gitlab::LegacyGithubImport::Importer),
+      ImportSource.new('manifest',       'Manifest',      Gitlab::ManifestImport::Importer)
     ].freeze
 
     class << self

lib/gitlab/manifest_import/importer.rb

+module Gitlab
+  module ManifestImport
+    class Importer
+      attr_reader :repository, :destination, :user
+
+      def initialize(repository, destination, user)
+        @repository = repository
+        @destination = destination
+        @user = user
+      end
+
+      def execute
+        import_project
+      end
+
+      private
+
+      def import_project
+        group_full_path, _, project_path = repository[:path].rpartition('/')
+        group_full_path = File.join(destination.path, group_full_path) if destination
+        group = Group.find_by_full_path(group_full_path) ||
+          create_group_with_parents(group_full_path)
+
+        params = {
+          import_url: repository[:url],
+          import_type: 'manifest',
+          namespace_id: group.id,
+          path: project_path,
+          name: project_path,
+          visibility_level: destination.visibility_level
+        }
+
+        Projects::CreateService.new(user, params).execute
+      end
+
+      def create_group_with_parents(full_path)
+        params = {
+          group_path: full_path,
+          visibility_level: destination.visibility_level
+        }
+
+        Groups::NestedCreateService.new(user, params).execute
+      end
+    end
+  end
+end

lib/gitlab/manifest_import/manifest.rb

+# Class to parse manifest file to import multiple projects at once
+#
+# <manifest>
+#   <remote review="https://android-review.googlesource.com/" />
+#   <project path="platform-common" name="platform" />
+#   <project path="platform/art" name="platform/art" />
+#   <project path="platform/device" name="platform/device" />
+# </manifest>
+#
+# 1. Project path must be uniq and can't be part of other project path.
+#    For example, you can't have projects with 'foo' and 'foo/bar' paths.
+# 2. Remote must be present with review attribute so GitLab knows
+#    where to fetch source code
+# 3. For each nested keyword in path a corresponding group will be created.
+#    For example if a path is 'foo/bar' then GitLab will create a group 'foo'
+#    and a project 'bar' in it.
+module Gitlab
+  module ManifestImport
+    class Manifest
+      attr_reader :parsed_xml, :errors
+
+      def initialize(file)
+        @parsed_xml = File.open(file) { |f| Nokogiri::XML(f) }
+        @errors = []
+      end
+
+      def projects
+        raw_projects.each_with_index.map do |project, i|
+          {
+            id: i,
+            name: project['name'],
+            path: project['path'],
+            url: repository_url(project['name'])
+          }
+        end
+      end
+
+      def valid?
+        unless validate_remote
+          @errors << 'Make sure a <remote> tag is present and is valid.'
+        end
+
+        unless validate_projects
+          @errors << 'Make sure every <project> tag has name and path attributes.'
+        end
+
+        @errors.empty?
+      end
+
+      private
+
+      def validate_remote
+        remote.present? && URI.parse(remote).host
+      rescue URI::Error
+        false
+      end
+
+      def validate_projects
+        raw_projects.all? do |project|
+          project['name'] && project['path']
+        end
+      end
+
+      def repository_url(name)
+        URI.join(remote, name).to_s
+      end
+
+      def remote
+        @remote ||= parsed_xml.css('manifest > remote').first['review']
+      end
+
+      def raw_projects
+        @raw_projects ||= parsed_xml.css('manifest > project')
+      end
+    end
+  end
+end

spec/lib/gitlab/manifest_import/importer_spec.rb

+require 'spec_helper'
+
+describe Gitlab::ManifestImport::Importer, :postgresql do
+  let(:group) { create(:group) }
+  let(:user) { create(:user) }
+  let(:repository) do
+    {
+      path: 'device/common',
+      url: 'https://android-review.googlesource.com/device/common'
+    }
+  end
+
+  before do
+    group.add_owner(user)
+  end
+
+  subject { described_class.new(repository, group, user) }
+
+  describe '#execute' do
+    it { expect(subject.execute).to be_a(Project) }
+    it { expect { subject.execute }.to change { Project.count }.by(1) }
+    it { expect { subject.execute }.to change { Group.count }.by(1) }
+
+    it 'creates project with valid full path and import url' do
+      subject.execute
+
+      project = Project.last
+
+      expect(project.full_path).to eq(File.join(group.path, 'device/common'))
+      expect(project.import_url).to eq('https://android-review.googlesource.com/device/common')
+    end
+  end
+end

spec/lib/gitlab/manifest_import/manifest_spec.rb

+require 'spec_helper'
+
+describe Gitlab::ManifestImport::Manifest, :postgresql do
+  let(:file) { Rails.root.join('spec/fixtures/aosp_manifest.xml') }
+  let(:manifest) { described_class.new(file) }
+
+  describe '#valid?' do
+    context 'valid file' do
+      it { expect(manifest.valid?).to be true }
+    end
+
+    context 'missing or invalid attributes' do
+      let(:file) { Tempfile.new('foo') }
+
+      before do
+        content = <<~EOS
+          <manifest>
+            <remote review="invalid-url" />
+            <project name="platform/build"/>
+          </manifest>
+        EOS
+
+        file.write(content)
+        file.rewind
+      end
+
+      it { expect(manifest.valid?).to be false }
+
+      describe 'errors' do
+        before do
+          manifest.valid?
+        end
+
+        it { expect(manifest.errors).to include('Make sure a <remote> tag is present and is valid.') }
+        it { expect(manifest.errors).to include('Make sure every <project> tag has name and path attributes.') }
+      end
+    end
+  end
+
+  describe '#projects' do
+    it { expect(manifest.projects.size).to eq(660) }
+    it { expect(manifest.projects[0][:name]).to eq('platform/build') }
+    it { expect(manifest.projects[0][:path]).to eq('build/make') }
+    it { expect(manifest.projects[0][:url]).to eq('https://android-review.googlesource.com/platform/build') }
+  end
+end