Merge branch 'ce-mirror-backport' into 'master'

Backport relevant changes from gitlab-org/gitlab-ee!51

To do:

- [x] Update gitlab-shell

See merge request !1822

GITLAB_SHELL_VERSION

-2.6.6
+2.6.7

app/assets/stylesheets/framework/common.scss

@@ -328,6 +328,10 @@ table {
   }
 }
 
+.well {
+  margin-bottom: 0;
+}
+
 .search_box {
   @extend .well;
   text-align: center;

app/assets/stylesheets/framework/tw_bootstrap.scss

@@ -172,7 +172,7 @@
   }
 
   .panel-body {
-    form {
+    form, pre {
       margin: 0;
     }
 

app/controllers/projects/imports_controller.rb

@@ -28,8 +28,8 @@ class Projects::ImportsController < Projects::ApplicationController
       if @project.import_finished?
         redirect_to(project_path(@project)) and return
       else
-        redirect_to new_namespace_project_import_path(@project.namespace,
-                                                      @project) && return
+        redirect_to(new_namespace_project_import_path(@project.namespace,
+                                                      @project)) and return
       end
     end
   end

app/helpers/projects_helper.rb

@@ -253,14 +253,6 @@ module ProjectsHelper
     filename_path(project, :version)
   end
 
-  def hidden_pass_url(original_url)
-    result = URI(original_url)
-    result.password = '*****' unless result.password.nil?
-    result
-  rescue
-    original_url
-  end
-
   def project_wiki_path_with_version(proj, page, version, is_newest)
     url_params = is_newest ? {} : { version_id: version }
     namespace_project_wiki_path(proj.namespace, proj, page, url_params)

app/models/project.rb

@@ -310,15 +310,17 @@ class Project < ActiveRecord::Base
 
   def add_import_job
     if forked?
-      unless RepositoryForkWorker.perform_async(id, forked_from_project.path_with_namespace, self.namespace.path)
-        import_fail
-      end
+      RepositoryForkWorker.perform_async(self.id, forked_from_project.path_with_namespace, self.namespace.path)
     else
-      RepositoryImportWorker.perform_async(id)
+      RepositoryImportWorker.perform_async(self.id)
     end
   end
 
   def clear_import_data
+    update(import_error: nil)
+
+    ProjectCacheWorker.perform_async(self.id)
+
     self.import_data.destroy if self.import_data
   end
 
@@ -346,6 +348,14 @@ class Project < ActiveRecord::Base
     import_status == 'finished'
   end
 
+  def safe_import_url
+    result = URI.parse(self.import_url)
+    result.password = '*****' unless result.password.nil?
+    result.to_s
+  rescue
+    original_url
+  end
+
   def check_limit
     unless creator.can_create_project? or namespace.kind == 'group'
       errors[:limit_reached] << ("Your project limit is #{creator.projects_limit} projects! Please contact your administrator to increase it")

app/models/repository.rb

@@ -6,7 +6,7 @@ class Repository
 
   include Gitlab::ShellAdapter
 
-  attr_accessor :raw_repository, :path_with_namespace, :project
+  attr_accessor :path_with_namespace, :project
 
   def self.clean_old_archives
     repository_downloads_path = Gitlab.config.gitlab.repository_downloads_path
@@ -19,15 +19,19 @@ class Repository
   def initialize(path_with_namespace, default_branch = nil, project = nil)
     @path_with_namespace = path_with_namespace
     @project = project
-
-    if path_with_namespace
-      @raw_repository = Gitlab::Git::Repository.new(path_to_repo)
-      @raw_repository.autocrlf = :input
   end
 
+  def raw_repository
+    return nil unless path_with_namespace
+
+    @raw_repository ||= begin
+      repo = Gitlab::Git::Repository.new(path_to_repo)
+      repo.autocrlf = :input
+      repo
     rescue Gitlab::Git::Repository::NoRepository
       nil
     end
+  end
 
   # Return absolute path to repository
   def path_to_repo
@@ -105,29 +109,25 @@ class Repository
   end
 
   def add_branch(branch_name, ref)
-    cache.expire(:branch_names)
-    @branches = nil
+    expire_branches_cache
 
     gitlab_shell.add_branch(path_with_namespace, branch_name, ref)
   end
 
   def add_tag(tag_name, ref, message = nil)
-    cache.expire(:tag_names)
-    @tags = nil
+    expire_tags_cache
 
     gitlab_shell.add_tag(path_with_namespace, tag_name, ref, message)
   end
 
   def rm_branch(branch_name)
-    cache.expire(:branch_names)
-    @branches = nil
+    expire_branches_cache
 
     gitlab_shell.rm_branch(path_with_namespace, branch_name)
   end
 
   def rm_tag(tag_name)
-    cache.expire(:tag_names)
-    @tags = nil
+    expire_tags_cache
 
     gitlab_shell.rm_tag(path_with_namespace, tag_name)
   end
@@ -169,6 +169,16 @@ class Repository
     end
   end
 
+  def expire_tags_cache
+    cache.expire(:tag_names)
+    @tags = nil
+  end
+
+  def expire_branches_cache
+    cache.expire(:branch_names)
+    @branches = nil
+  end
+
   def expire_cache
     cache_keys.each do |key|
       cache.expire(key)
@@ -484,7 +494,7 @@ class Repository
     root_ref_commit = commit(root_ref)
 
     if branch_commit
-      rugged.merge_base(root_ref_commit.id, branch_commit.id) == branch_commit.id
+      is_ancestor?(branch_commit.id, root_ref_commit.id)
     else
       nil
     end
@@ -494,6 +504,11 @@ class Repository
     rugged.merge_base(first_commit_id, second_commit_id)
   end
 
+  def is_ancestor?(ancestor_id, descendant_id)
+    merge_base(ancestor_id, descendant_id) == ancestor_id
+  end
+
+
   def search_files(query, ref)
     offset = 2
     args = %W(#{Gitlab.config.git.bin_path} grep -i -n --before-context #{offset} --after-context #{offset} -e #{query} #{ref || root_ref})

app/services/projects/create_service.rb

@@ -55,7 +55,9 @@ module Projects
         @project.save
 
         if @project.persisted? && !@project.import?
-          raise 'Failed to create repository' unless @project.create_repository
+          unless @project.create_repository
+            raise 'Failed to create repository'
+          end
         end
       end
 
@@ -94,9 +96,7 @@ module Projects
         @project.team << [current_user, :master, current_user]
       end
 
-      if @project.import?
-        @project.import_start
-      end
+      @project.import_start if @project.import?
     end
   end
 end

app/views/projects/_home_panel.html.haml

@@ -18,17 +18,12 @@
   .project-repo-buttons
     .split-one
       = render 'projects/buttons/star'
-
-      - unless empty_repo
       = render 'projects/buttons/fork'
 
     = render "shared/clone_panel"
-    .split-repo-buttons 
-      - unless empty_repo
-        - if can? current_user, :download_code, @project
-          = link_to archive_namespace_project_repository_path(@project.namespace, @project, ref: @ref, format: 'zip'), class: 'btn', rel: 'nofollow' do
-            = icon('download fw')
     
+    .split-repo-buttons
+      = render "projects/buttons/download"
       = render 'projects/buttons/dropdown'
-    = render 'projects/buttons/notifications'
 
+    = render 'projects/buttons/notifications'

app/views/projects/buttons/_download.html.haml

+- unless @project.empty_repo?
+  - if can? current_user, :download_code, @project
+    = link_to archive_namespace_project_repository_path(@project.namespace, @project, ref: @ref, format: 'zip'), class: 'btn has_tooltip', rel: 'nofollow', title: "Download ZIP" do
+      = icon('download')

app/views/projects/buttons/_dropdown.html.haml

@@ -32,5 +32,3 @@
           = link_to new_namespace_project_tag_path(@project.namespace, @project) do
             = icon('tags fw')
             New tag
-
-

app/views/projects/buttons/_fork.html.haml

-- if current_user && can?(current_user, :fork_project, @project)
+- unless @project.empty_repo?
+  - if current_user && can?(current_user, :fork_project, @project)
     - if current_user.already_forked?(@project) && current_user.manageable_namespaces.size < 2
-    = link_to namespace_project_path(current_user, current_user.fork_of(@project)), title: 'Go to your fork', class: 'btn' do
+      = link_to namespace_project_path(current_user, current_user.fork_of(@project)), title: 'Go to your fork', class: 'btn has_tooltip' do
         = icon('code-fork fw')
         Fork
         %span.count
           = @project.forks_count
     - else
-    = link_to new_namespace_project_fork_path(@project.namespace, @project), title: "Fork project", class: 'btn' do
+      = link_to new_namespace_project_fork_path(@project.namespace, @project), title: "Fork project", class: 'btn has_tooltip' do
         = icon('code-fork fw')
         %span.count
           = @project.forks_count

app/views/projects/buttons/_star.html.haml

 - if current_user
-  = link_to toggle_star_namespace_project_path(@project.namespace, @project), class: 'btn star-btn toggle-star', method: :post, remote: true do
+  = link_to toggle_star_namespace_project_path(@project.namespace, @project), class: 'btn star-btn toggle-star has_tooltip', method: :post, remote: true, title: "Star project" do
     = icon('star fw')
     %span.count
       = @project.star_count

app/views/projects/imports/new.html.haml

 - page_title "Import repository"
 %h3.page-title
-  - if @project.import_failed?
-    Import failed. Retry?
-  - else
   Import repository
 
 %hr
 
+- if @project.import_failed?
+  .panel.panel-danger
+    .panel-heading The repository could not be imported.
+    .panel-body
+      %pre
+        :preserve
+          #{@project.import_error.try(:strip)}
+
 = form_for @project, url: namespace_project_import_path(@project.namespace, @project), method: :post, html: { class: 'form-horizontal' } do |f|
-  .form-group.import-url-data
-    = f.label :import_url, class: 'control-label' do
-      %span Import existing git repo
-    .col-sm-10
-      = f.text_field :import_url, class: 'form-control', placeholder: 'https://github.com/randx/six.git'
-      .well.prepend-top-20
-        This URL must be publicly accessible or you can add a username and password like this: https://username:password@gitlab.com/company/project.git.
-        %br
-        The import will time out after 4 minutes. For big repositories, use a clone/push combination.
-        For SVN repositories, check #{link_to "this migrating from SVN doc.", "http://doc.gitlab.com/ce/workflow/importing/migrating_from_svn.html"}
+  = render "shared/import_form", f: f
+
   .form-actions
     = f.submit 'Start import', class: "btn btn-create", tabindex: 4

app/views/projects/imports/show.html.haml

@@ -8,7 +8,7 @@
       - else
         Import in progress.
     - unless @project.forked?
-      %p.monospace git clone --bare #{hidden_pass_url(@project.import_url)}
+      %p.monospace git clone --bare #{@project.safe_import_url}
     %p Please wait while we import the repository for you. Refresh at will.
     :javascript
       new ProjectImport();

app/views/projects/new.html.haml

@@ -23,7 +23,6 @@
             = f.select :namespace_id, namespaces_options(params[:namespace_id] || :current_user), {}, {class: 'select2', tabindex: 2}
 
       - if import_sources_enabled?
-
         .project-import.js-toggle-container
           .form-group
             %label.control-label Import project from
@@ -82,19 +81,7 @@
                   %span Any repo by URL
 
           .js-toggle-content.hide
-            .form-group.import-url-data
-              = f.label :import_url, class: 'control-label' do
-                %span Git repository URL
-              .col-sm-10
-                = f.text_field :import_url, class: 'form-control', placeholder: 'https://username:password@gitlab.company.com/group/project.git'
-                .well.prepend-top-20
-                  %ul
-                    %li
-                      The repository must be accessible over HTTP(S). If it is not publicly accessible, you can add authentication information to the URL: <code>https://username:password@gitlab.company.com/group/project.git</code>.
-                    %li
-                      The import will time out after 4 minutes. For big repositories, use a clone/push combination.
-                    %li
-                      To migrate an SVN repository, check out #{link_to "this document", "http://doc.gitlab.com/ce/workflow/importing/migrating_from_svn.html"}.
+            = render "shared/import_form", f: f
 
       .prepend-botton-10
 

app/views/shared/_clone_panel.html.haml

@@ -21,7 +21,6 @@
         = gitlab_config.protocol.upcase
   = text_field_tag :project_clone, default_url_to_repo(project), class: "js-select-on-focus form-control", readonly: true
   - if project.kind_of?(Project)
-    .input-group-addon
-      .visibility-level-label.has_tooltip{'data-title' => "#{visibility_level_label(project.visibility_level)} project" }
+    .input-group-addon.has_tooltip{title: "#{visibility_level_label(project.visibility_level)} project", data: { container: "body" } }
+      .visibility-level-label
         = visibility_level_icon(project.visibility_level)
-   

app/views/shared/_import_form.html.haml

+.form-group.import-url-data
+  = f.label :import_url, class: 'control-label' do
+    %span Git repository URL
+  .col-sm-10
+    = f.text_field :import_url, class: 'form-control', placeholder: 'https://username:password@gitlab.company.com/group/project.git'
+
+    .well.prepend-top-20
+      %ul
+        %li
+          The repository must be accessible over <code>http://</code>, <code>https://</code> or <code>git://</code>.
+        %li
+          If your HTTP repository is not publicly accessible, add authentication information to the URL: <code>https://username:password@gitlab.company.com/group/project.git</code>.
+        %li
+          The import will time out after 4 minutes. For big repositories, use a clone/push combination.
+        %li
+          To migrate an SVN repository, check out #{link_to "this document", "http://doc.gitlab.com/ce/workflow/importing/migrating_from_svn.html"}.

app/workers/repository_fork_worker.rb

@@ -13,22 +13,20 @@ class RepositoryForkWorker
     end
 
     result = gitlab_shell.fork_repository(source_path, target_path)
-
     unless result
       logger.error("Unable to fork project #{project_id} for repository #{source_path} -> #{target_path}")
+      project.update(import_error: "The project could not be forked.")
       project.import_fail
-      project.save
       return
     end
 
-    if project.valid_repo?
-      ProjectCacheWorker.perform_async(project.id)
-      project.import_finish
-    else
-      project.import_fail
+    unless project.valid_repo?
       logger.error("Project #{id} had an invalid repository after fork")
+      project.update(import_error: "The forked repository is invalid.")
+      project.import_fail
+      return
     end
 
-    project.save
+    project.import_finish
   end
 end

app/workers/repository_import_worker.rb

@@ -7,18 +7,25 @@ class RepositoryImportWorker
   def perform(project_id)
     project = Project.find(project_id)
 
-    unless project.import_url == Project::UNKNOWN_IMPORT_URL
-      import_result = gitlab_shell.send(:import_repository,
-                               project.path_with_namespace,
-                               project.import_url)
-      return project.import_fail unless import_result
-    else
+    if project.import_url == Project::UNKNOWN_IMPORT_URL
+      # In this case, we only want to import issues, not a repository.
       unless project.create_repository
-        return project.import_fail
+        project.update(import_error: "The repository could not be created.")
+        project.import_fail
+        return
+      end
+    else
+      begin
+        gitlab_shell.import_repository(project.path_with_namespace, project.import_url)
+      rescue Gitlab::Shell::Error => e
+        project.update(import_error: e.message)
+        project.import_fail
+        return
       end
     end
 
-    data_import_result = case project.import_type
+    data_import_result =
+      case project.import_type
       when 'github'
         Gitlab::GithubImport::Importer.new(project).execute
       when 'gitlab'
@@ -32,12 +39,20 @@ class RepositoryImportWorker
       else
         true
       end
-    return project.import_fail unless data_import_result
 
-    Gitlab::BitbucketImport::KeyDeleter.new(project).execute if project.import_type == 'bitbucket'
+    unless data_import_result
+      project.update(import_error: "The remote issue data could not be imported.")
+      project.import_fail
+      return
+    end
+
+    if project.import_type == 'bitbucket'
+      Gitlab::BitbucketImport::KeyDeleter.new(project).execute
+    end
 
     project.import_finish
-    project.save
-    ProjectCacheWorker.perform_async(project.id)
+
+    # Explicitly update mirror so that upstream remote is created and fetched
+    project.update_mirror
   end
 end

db/migrate/20151110125604_add_import_error_to_project.rb

+class AddImportErrorToProject < ActiveRecord::Migration
+  def change
+    add_column :projects, :import_error, :text
+  end
+end

db/schema.rb

@@ -641,6 +641,9 @@ ActiveRecord::Schema.define(version: 20151116144118) do
     t.string   "import_type"
     t.string   "import_source"
     t.integer  "commit_count",                   default: 0
+    t.boolean  "merge_requests_ff_only_enabled", default: false
+    t.text     "issues_template"
+    t.text     "import_error"
   end
 
   add_index "projects", ["created_at", "id"], name: "index_projects_on_created_at_and_id", using: :btree

features/steps/dashboard/new_project.rb

@@ -44,7 +44,6 @@ class Spinach::Features::NewProject < Spinach::FeatureSteps
     git_import_instructions = first('.js-toggle-content')
     expect(git_import_instructions).to be_visible
     expect(git_import_instructions).to have_content "Git repository URL"
-    expect(git_import_instructions).to have_content "The repository must be accessible over HTTP(S). If it is not publicly accessible, you can add authentication information to the URL:"
   end
 
   step 'I click on "Google Code"' do

lib/gitlab/backend/shell.rb

 module Gitlab
   class Shell
-    class AccessDenied < StandardError; end
+    class Error < StandardError; end
 
     class KeyAdder < Struct.new(:io)
       def add_key(id, key)
@@ -36,8 +36,9 @@ module Gitlab
     #   import_repository("gitlab/gitlab-ci", "https://github.com/randx/six.git")
     #
     def import_repository(name, url)
-      Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'import-project',
-                                   "#{name}.git", url, '240'])
+      output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '240'])
+      raise Error, output unless status.zero?
+      true
     end
 
     # Move repository

spec/services/projects/fork_service_spec.rb

@@ -25,13 +25,6 @@ describe Projects::ForkService do
       end
     end
 
-    context 'fork project failure' do
-      it "fails due to transaction failure" do
-        @to_project = fork_project(@from_project, @to_user, false)
-        expect(@to_project.import_failed?)
-      end
-    end
-
     context 'project already exists' do
       it "should fail due to validation, not transaction failure" do
         @existing_project = create(:project, creator_id: @to_user.id, name: @from_project.name, namespace: @to_namespace)
@@ -66,7 +59,7 @@ describe Projects::ForkService do
 
     context 'fork project for group' do
       it 'group owner successfully forks project into the group' do
-        to_project = fork_project(@project, @group_owner, true, @opts)
+        to_project = fork_project(@project, @group_owner, @opts)
         expect(to_project.owner).to       eq(@group)
         expect(to_project.namespace).to   eq(@group)
         expect(to_project.name).to        eq(@project.name)
@@ -78,7 +71,7 @@ describe Projects::ForkService do
 
     context 'fork project for group when user not owner' do
       it 'group developer should fail to fork project into the group' do
-        to_project = fork_project(@project, @developer, true, @opts)
+        to_project = fork_project(@project, @developer, @opts)
         expect(to_project.errors[:namespace]).to eq(['is not valid'])
       end
     end
@@ -87,7 +80,7 @@ describe Projects::ForkService do
       it 'should fail due to validation, not transaction failure' do
         existing_project = create(:project, name: @project.name,
                                             namespace: @group)
-        to_project = fork_project(@project, @group_owner, true, @opts)
+        to_project = fork_project(@project, @group_owner, @opts)
         expect(existing_project.persisted?).to be_truthy
         expect(to_project.errors[:name]).to eq(['has already been taken'])
         expect(to_project.errors[:path]).to eq(['has already been taken'])
@@ -95,8 +88,8 @@ describe Projects::ForkService do
     end
   end
 
-  def fork_project(from_project, user, fork_success = true, params = {})
-    allow(RepositoryForkWorker).to receive(:perform_async).and_return(fork_success)
+  def fork_project(from_project, user, params = {})
+    allow(RepositoryForkWorker).to receive(:perform_async).and_return(true)
     Projects::ForkService.new(from_project, user, params).execute
   end
 end

spec/workers/repository_fork_worker_spec.rb

@@ -12,7 +12,6 @@ describe RepositoryForkWorker do
                                                    project.path_with_namespace,
                                                    fork_project.namespace.path).
                                                    and_return(true)
-      expect(ProjectCacheWorker).to receive(:perform_async)
 
       subject.perform(project.id,
                       project.path_with_namespace,