Merge branch 'ce-to-ee-2018-01-15' into 'master'

CE upstream - Monday

Closes gitaly#907

See merge request gitlab-org/gitlab-ee!4098

GITALY_SERVER_VERSION

-0.67.0
+0.69.0

Gemfile

@@ -419,7 +419,7 @@ group :ed25519 do
 end
 
 # Gitaly GRPC client
-gem 'gitaly-proto', '~> 0.69.0', require: 'gitaly'
+gem 'gitaly-proto', '~> 0.73.0', require: 'gitaly'
 
 gem 'toml-rb', '~> 0.3.15', require: false
 

Gemfile.lock

@@ -308,7 +308,7 @@ GEM
       po_to_json (>= 1.0.0)
       rails (>= 3.2.0)
     gherkin-ruby (0.3.2)
-    gitaly-proto (0.69.0)
+    gitaly-proto (0.73.0)
       google-protobuf (~> 3.1)
       grpc (~> 1.0)
     github-linguist (4.7.6)
@@ -1089,7 +1089,7 @@ DEPENDENCIES
   gettext (~> 3.2.2)
   gettext_i18n_rails (~> 1.8.0)
   gettext_i18n_rails_js (~> 1.2.0)
-  gitaly-proto (~> 0.69.0)
+  gitaly-proto (~> 0.73.0)
   github-linguist (~> 4.7.0)
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-license (~> 1.0)

app/assets/javascripts/pages/projects/tree/show/index.js

@@ -20,4 +20,3 @@ export default () => {
     );
   }
 };
-

app/views/projects/jobs/_empty_state.html.haml

 - illustration = local_assigns.fetch(:illustration)
 - illustration_size = local_assigns.fetch(:illustration_size)
 - title = local_assigns.fetch(:title)
-- content = local_assigns.fetch(:content)
+- content = local_assigns.fetch(:content, nil)
 - action = local_assigns.fetch(:action, nil)
 
 .row.empty-state
@@ -11,7 +11,8 @@
   .col-xs-12
     .text-content
       %h4.text-center= title
-      %p= content
+      - if content
+        %p= content
       - if action
         .text-center
           = action

app/views/projects/jobs/show.html.haml

@@ -95,14 +95,13 @@
         illustration: 'illustrations/manual_action.svg',
         illustration_size: 'svg-394',
         title: _('This job requires a manual action'),
-        content: _('This job depends on a user to trigger its process. Often they are used to deploy code to production environments.'),
-        action: ( link_to _('Trigger this manual action'), play_project_job_path(@project, @build), class: 'btn btn-primary', title: _('Trigger this manual action') )
+        content: _('This job depends on a user to trigger its process. Often they are used to deploy code to production environments'),
+        action: ( link_to _('Trigger this manual action'), play_project_job_path(@project, @build), method: :post, class: 'btn btn-primary', title: _('Trigger this manual action') )
     - else
       = render 'empty_state',
         illustration: 'illustrations/job_not_triggered.svg',
         illustration_size: 'svg-306',
-        title: _('This job has not been triggered yet'),
-        content: _('This job depends on upstream jobs that need to succeed in order for this job to be triggered.')
+        title: _('This job has not been triggered yet')
 
   = render "sidebar"
 

app/workers/repository_fork_worker.rb

@@ -17,10 +17,7 @@ class RepositoryForkWorker
                                           project.repository_storage_path, project.disk_path)
     raise "Unable to fork project #{project_id} for repository #{source_disk_path} -> #{project.disk_path}" unless result
 
-    project.repository.after_import
-    raise "Project #{project_id} had an invalid repository after fork" unless project.valid_repo?
-
-    project.import_finish
+    project.after_import
   end
 
   private

changelogs/unreleased/19493-fork-does-not-protect-default-branch.yml

+---
+title: Makes forking protect default branch on completion
+merge_request:
+author:
+type: fixed

changelogs/unreleased/38540-ssh-env-file.yml

+---
+title: 'Closes #38540 - Remove .ssh/environment file that now breaks the gitlab:check
+  rake task'
+merge_request:
+author:
+type: fixed

changelogs/unreleased/42025-fix-issue-api.yml

+---
+title: "[API] Fix creating issue when assignee_id is empty"
+merge_request:
+author:
+type: fixed

lib/api/circuit_breakers.rb

@@ -17,11 +17,11 @@ module API
             end
 
             def storage_health
-              @failing_storage_health ||= Gitlab::Git::Storage::Health.for_all_storages
+              @storage_health ||= Gitlab::Git::Storage::Health.for_all_storages
             end
           end
 
-          desc 'Get all failing git storages' do
+          desc 'Get all git storages' do
             detail 'This feature was introduced in GitLab 9.5'
             success Entities::RepositoryStorageHealth
           end

lib/gitlab/git/gitlab_projects.rb

@@ -44,29 +44,13 @@ module Gitlab
       # Import project via git clone --bare
       # URL must be publicly cloneable
       def import_project(source, timeout)
-        # Skip import if repo already exists
-        return false if File.exist?(repository_absolute_path)
-
-        masked_source = mask_password_in_url(source)
-
-        logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>."
-        cmd = %W(git clone --bare -- #{source} #{repository_absolute_path})
-
-        success = run_with_timeout(cmd, timeout, nil)
-
-        unless success
-          logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.")
-          FileUtils.rm_rf(repository_absolute_path)
-          return false
+        Gitlab::GitalyClient.migrate(:import_repository) do |is_enabled|
+          if is_enabled
+            gitaly_import_repository(source)
+          else
+            git_import_repository(source, timeout)
+          end
         end
-
-        Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path)
-
-        # The project was imported successfully.
-        # Remove the origin URL since it may contain password.
-        remove_origin_in_repo
-
-        true
       end
 
       def fork_repository(new_shard_path, new_repository_relative_path)
@@ -231,6 +215,42 @@ module Gitlab
           raise(ShardNameNotFoundError, "no shard found for path '#{shard_path}'")
       end
 
+      def git_import_repository(source, timeout)
+        # Skip import if repo already exists
+        return false if File.exist?(repository_absolute_path)
+
+        masked_source = mask_password_in_url(source)
+
+        logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>."
+        cmd = %W(git clone --bare -- #{source} #{repository_absolute_path})
+
+        success = run_with_timeout(cmd, timeout, nil)
+
+        unless success
+          logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.")
+          FileUtils.rm_rf(repository_absolute_path)
+          return false
+        end
+
+        Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path)
+
+        # The project was imported successfully.
+        # Remove the origin URL since it may contain password.
+        remove_origin_in_repo
+
+        true
+      end
+
+      def gitaly_import_repository(source)
+        raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil)
+
+        Gitlab::GitalyClient::RepositoryService.new(raw_repository).import_repository(source)
+        true
+      rescue GRPC::BadStatus => e
+        @output << e.message
+        false
+      end
+
       def git_fork_repository(new_shard_path, new_repository_relative_path)
         from_path = repository_absolute_path
         to_path = File.join(new_shard_path, new_repository_relative_path)

lib/gitlab/gitaly_client/commit_service.rb

@@ -177,7 +177,7 @@ module Gitlab
 
         response = GitalyClient.call(@repository.storage, :commit_service, :list_commits_by_oid, request, timeout: GitalyClient.medium_timeout)
         consume_commits_response(response)
-      rescue GRPC::Unknown # If no repository is found, happens mainly during testing
+      rescue GRPC::NotFound # If no repository is found, happens mainly during testing
         []
       end
 

lib/gitlab/gitaly_client/repository_service.rb

@@ -100,6 +100,21 @@ module Gitlab
         )
       end
 
+      def import_repository(source)
+        request = Gitaly::CreateRepositoryFromURLRequest.new(
+          repository: @gitaly_repo,
+          url: source
+        )
+
+        GitalyClient.call(
+          @storage,
+          :repository_service,
+          :create_repository_from_url,
+          request,
+          timeout: GitalyClient.default_timeout
+        )
+      end
+
       def rebase_in_progress?(rebase_id)
         request = Gitaly::IsRebaseInProgressRequest.new(
           repository: @gitaly_repo,

lib/gitlab/kubernetes/helm/install_command.rb

@@ -36,7 +36,11 @@ module Gitlab
         def complete_command(namespace_name)
           return unless chart
 
-          "helm install #{chart} --name #{name} --namespace #{namespace_name} >/dev/null"
+          if chart_values_file
+            "helm install #{chart} --name #{name} --namespace #{namespace_name} -f /data/helm/#{name}/config/values.yaml >/dev/null"
+          else
+            "helm install #{chart} --name #{name} --namespace #{namespace_name} >/dev/null"
+          end
         end
 
         def install_dps_command

lib/gitlab/kubernetes/helm/pod.rb

@@ -10,9 +10,10 @@ module Gitlab
 
         def generate
           spec = { containers: [container_specification], restartPolicy: 'Never' }
+
           if command.chart_values_file
-            generate_config_map
-            spec['volumes'] = volumes_specification
+            create_config_map
+            spec[:volumes] = volumes_specification
           end
 
           ::Kubeclient::Resource.new(metadata: metadata, spec: spec)
@@ -35,19 +36,39 @@ module Gitlab
         end
 
         def labels
-          { 'gitlab.org/action': 'install', 'gitlab.org/application': command.name }
+          {
+            'gitlab.org/action': 'install',
+            'gitlab.org/application': command.name
+          }
         end
 
         def metadata
-          { name: command.pod_name, namespace: namespace_name, labels: labels }
+          {
+            name: command.pod_name,
+            namespace: namespace_name,
+            labels: labels
+          }
         end
 
         def volume_mounts_specification
-          [{ name: 'config-volume', mountPath: '/etc/config' }]
+          [
+            {
+              name: 'configuration-volume',
+              mountPath: "/data/helm/#{command.name}/config"
+            }
+          ]
         end
 
         def volumes_specification
-          [{ name: 'config-volume', configMap: { name: 'values-config' } }]
+          [
+            {
+              name: 'configuration-volume',
+              configMap: {
+                name: 'values-content-configuration',
+                items: [{ key: 'values', path: 'values.yaml' }]
+              }
+            }
+          ]
         end
 
         def generate_pod_env(command)
@@ -58,10 +79,10 @@ module Gitlab
           }.map { |key, value| { name: key, value: value } }
         end
 
-        def generate_config_map
+        def create_config_map
           resource = ::Kubeclient::Resource.new
-          resource.metadata = { name: 'values-config', namespace: namespace_name }
-          resource.data = YAML.load_file(command.chart_values_file)
+          resource.metadata = { name: 'values-content-configuration', namespace: namespace_name, labels: { name: 'values-content-configuration' } }
+          resource.data = { values: File.read(command.chart_values_file) }
           kubeclient.create_config_map(resource)
         end
       end

lib/tasks/gitlab/shell.rake

@@ -54,16 +54,6 @@ namespace :gitlab do
       # (Re)create hooks
       Rake::Task['gitlab:shell:create_hooks'].invoke
 
-      # Required for debian packaging with PKGR: Setup .ssh/environment with
-      # the current PATH, so that the correct ruby version gets loaded
-      # Requires to set "PermitUserEnvironment yes" in sshd config (should not
-      # be an issue since it is more than likely that there are no "normal"
-      # user accounts on a gitlab server). The alternative is for the admin to
-      # install a ruby (1.9.3+) in the global path.
-      File.open(File.join(user_home, ".ssh", "environment"), "w+") do |f|
-        f.puts "PATH=#{ENV['PATH']}"
-      end
-
       Gitlab::Shell.ensure_secret_token!
     end
 

spec/features/projects/jobs_spec.rb

@@ -391,9 +391,18 @@ feature 'Jobs' do
 
       it 'shows manual action empty state' do
         expect(page).to have_content('This job requires a manual action')
-        expect(page).to have_content('This job depends on a user to trigger its process. Often they are used to deploy code to production environments.')
+        expect(page).to have_content('This job depends on a user to trigger its process. Often they are used to deploy code to production environments')
         expect(page).to have_link('Trigger this manual action')
       end
+
+      it 'plays manual action', :js do
+        click_link 'Trigger this manual action'
+
+        wait_for_requests
+        expect(page).to have_content('This job has not been triggered')
+        expect(page).to have_content('This job is stuck, because the project doesn\'t have any runners online assigned to it.')
+        expect(page).to have_content('pending')
+      end
     end
 
     context 'Non triggered job' do
@@ -403,9 +412,8 @@ feature 'Jobs' do
         visit project_job_path(project, job)
       end
 
-      it 'shows manual action empty state' do
+      it 'shows empty state' do
         expect(page).to have_content('This job has not been triggered yet')
-        expect(page).to have_content('This job depends on upstream jobs that need to succeed in order for this job to be triggered.')
       end
     end
   end

spec/lib/gitlab/git/gitlab_projects_spec.rb

@@ -158,39 +158,55 @@ describe Gitlab::Git::GitlabProjects do
 
     subject { gl_projects.import_project(import_url, timeout) }
 
-    context 'success import' do
-      it 'imports a repo' do
-        expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_falsy
+    shared_examples 'importing repository' do
+      context 'success import' do
+        it 'imports a repo' do
+          expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_falsy
 
-        message = "Importing project from <#{import_url}> to <#{tmp_repo_path}>."
-        expect(logger).to receive(:info).with(message)
+          is_expected.to be_truthy
 
-        is_expected.to be_truthy
+          expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_truthy
+        end
+      end
 
-        expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_truthy
+      context 'already exists' do
+        it "doesn't import" do
+          FileUtils.mkdir_p(tmp_repo_path)
+
+          is_expected.to be_falsy
+        end
       end
     end
 
-    context 'already exists' do
-      it "doesn't import" do
-        FileUtils.mkdir_p(tmp_repo_path)
+    context 'when Gitaly import_repository feature is enabled' do
+      it_behaves_like 'importing repository'
+    end
+
+    context 'when Gitaly import_repository feature is disabled', :disable_gitaly do
+      describe 'logging' do
+        it 'imports a repo' do
+          message = "Importing project from <#{import_url}> to <#{tmp_repo_path}>."
+          expect(logger).to receive(:info).with(message)
 
-        is_expected.to be_falsy
+          subject
+        end
       end
-    end
 
-    context 'timeout' do
-      it 'does not import a repo' do
-        stub_spawn_timeout(cmd, timeout, nil)
+      context 'timeout' do
+        it 'does not import a repo' do
+          stub_spawn_timeout(cmd, timeout, nil)
 
-        message = "Importing project from <#{import_url}> to <#{tmp_repo_path}> failed."
-        expect(logger).to receive(:error).with(message)
+          message = "Importing project from <#{import_url}> to <#{tmp_repo_path}> failed."
+          expect(logger).to receive(:error).with(message)
 
-        is_expected.to be_falsy
+          is_expected.to be_falsy
 
-        expect(gl_projects.output).to eq("Timed out\n")
-        expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_falsy
+          expect(gl_projects.output).to eq("Timed out\n")
+          expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_falsy
+        end
       end
+
+      it_behaves_like 'importing repository'
     end
   end
 

spec/lib/gitlab/kubernetes/helm/install_command_spec.rb

@@ -100,6 +100,25 @@ describe Gitlab::Kubernetes::Helm::InstallCommand do
         is_expected.to eq(command)
       end
     end
+
+    context 'when chart values file is present' do
+      let(:install_command) { described_class.new(prometheus.name, chart: prometheus.chart, chart_values_file: prometheus.chart_values_file) }
+      let(:command) do
+        <<~MSG.chomp
+        set -eo pipefail
+        apk add -U ca-certificates openssl >/dev/null
+        wget -q -O - https://kubernetes-helm.storage.googleapis.com/helm-v2.7.0-linux-amd64.tar.gz | tar zxC /tmp >/dev/null
+        mv /tmp/linux-amd64/helm /usr/bin/
+
+        helm init --client-only >/dev/null
+        helm install #{prometheus.chart} --name #{prometheus.name} --namespace #{namespace.name} -f /data/helm/#{prometheus.name}/config/values.yaml >/dev/null
+        MSG
+      end
+
+      it 'should return appropriate command' do
+        is_expected.to eq(command)
+      end
+    end
   end
 
   describe "#pod_name" do

spec/lib/gitlab/kubernetes/helm/pod_spec.rb

@@ -52,18 +52,20 @@ describe Gitlab::Kubernetes::Helm::Pod do
 
       it 'should include volumes for the container' do
         container = subject.generate.spec.containers.first
-        expect(container.volumeMounts.first['name']).to eq('config-volume')
-        expect(container.volumeMounts.first['mountPath']).to eq('/etc/config')
+        expect(container.volumeMounts.first['name']).to eq('configuration-volume')
+        expect(container.volumeMounts.first['mountPath']).to eq("/data/helm/#{app.name}/config")
       end
 
       it 'should include a volume inside the specification' do
         spec = subject.generate.spec
-        expect(spec.volumes.first['name']).to eq('config-volume')
+        expect(spec.volumes.first['name']).to eq('configuration-volume')
       end
 
       it 'should mount configMap specification in the volume' do
         spec = subject.generate.spec
-        expect(spec.volumes.first.configMap['name']).to eq('values-config')
+        expect(spec.volumes.first.configMap['name']).to eq('values-content-configuration')
+        expect(spec.volumes.first.configMap['items'].first['key']).to eq('values')
+        expect(spec.volumes.first.configMap['items'].first['path']).to eq('values.yaml')
       end
     end
 

spec/spec_helper.rb

@@ -104,10 +104,12 @@ RSpec.configure do |config|
     TestEnv.init
   end
 
+  # EE-specific start
   config.before(:all) do
     License.destroy_all
     TestLicense.init
   end
+  # EE-specific stop
 
   config.before(:example) do
     # Skip pre-receive hook check so we can use the web editor and merge.

spec/workers/repository_fork_worker_spec.rb

@@ -47,6 +47,14 @@ describe RepositoryForkWorker do
       perform!
     end
 
+    it 'protects the default branch' do
+      expect_fork_repository.and_return(true)
+
+      perform!
+
+      expect(fork_project.protected_branches.first.name).to eq(fork_project.default_branch)
+    end
+
     it 'flushes various caches' do
       expect_fork_repository.and_return(true)
 

vendor/prometheus/values.yaml

-alertmanager: |
+alertmanager:
   enabled: false
 
-kubeStateMetrics: |
-  enabled: 'false'
+kubeStateMetrics:
+  enabled: false
 
-nodeExporter: |
-  enabled: 'false'
+nodeExporter:
+  enabled: false
 
-pushgateway: |
-  enabled: 'false'
+pushgateway:
+  enabled: false
 
-serverFiles: |
-  alerts: ''
-  rules: ''
+serverFiles:
+  alerts: ""
+  rules: ""
 
   prometheus.yml: |-
-    rule_files: |
+    rule_files:
       - /etc/config/rules
       - /etc/config/alerts
-    scrape_configs: |
+
+    scrape_configs:
       - job_name: prometheus
-        static_configs: |
+        static_configs:
           - targets:
             - localhost:9090
 
       - job_name: 'kubernetes-apiservers'
-        kubernetes_sd_configs: |
+
+        kubernetes_sd_configs:
           - role: endpoints
+
         scheme: https
 
         tls_config:
@@ -37,14 +40,17 @@ serverFiles: |
           - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
             action: keep
             regex: default;kubernetes;https
+
       - job_name: 'kubernetes-nodes'
         scheme: https
         tls_config:
           ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
           insecure_skip_verify: true
         bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+
         kubernetes_sd_configs:
           - role: node
+
         relabel_configs:
           - action: labelmap
             regex: __meta_kubernetes_node_label_(.+)
@@ -54,14 +60,15 @@ serverFiles: |
             regex: (.+)
             target_label: __metrics_path__
             replacement: /api/v1/nodes/${1}/proxy/metrics
- 
       - job_name: 'kubernetes-service-endpoints'
+
         kubernetes_sd_configs:
           - role: endpoints
-        relabel_configs: |
+
+        relabel_configs:
           - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
             action: keep
-            regex: 'true'
+            regex: true
           - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
             action: replace
             target_label: __scheme__
@@ -83,24 +90,30 @@ serverFiles: |
           - source_labels: [__meta_kubernetes_service_name]
             action: replace
             target_label: kubernetes_name
+
       - job_name: 'prometheus-pushgateway'
         honor_labels: true
-        kubernetes_sd_configs: |
+
+        kubernetes_sd_configs:
           - role: service
-        relabel_configs: |
+
+        relabel_configs:
           - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
             action: keep
             regex: pushgateway
       - job_name: 'kubernetes-services'
+
         metrics_path: /probe
-        params: |
+        params:
           module: [http_2xx]
-        kubernetes_sd_configs: |
+
+        kubernetes_sd_configs:
           - role: service
-        relabel_configs: |
+
+        relabel_configs:
           - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
             action: keep
-            regex: 'true'
+            regex: true
           - source_labels: [__address__]
             target_label: __param_target
           - target_label: __address__
@@ -113,17 +126,25 @@ serverFiles: |
             target_label: kubernetes_namespace
           - source_labels: [__meta_kubernetes_service_name]
             target_label: kubernetes_name
+
       - job_name: 'kubernetes-pods'
+
         kubernetes_sd_configs:
           - role: pod
+
         relabel_configs:
           - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
             action: keep
-            regex: 'true'
+            regex: true
           - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
             action: replace
             target_label: __metrics_path__
             regex: (.+)
+          - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+            action: replace
+            regex: (.+):(?:\d+);(\d+)
+            replacement: ${1}:${2}
+            target_label: __address__
           - action: labelmap
             regex: __meta_kubernetes_pod_label_(.+)
           - source_labels: [__meta_kubernetes_namespace]