Merge branch 'master' into 22643-manual-job-page

* master: (24 commits)
  Allow local tests to use a modified Gitaly
  Update check.md
  add deprecation and removal issue to docs
  Add status attribute to runner api entity
  Fix typos in a code comment
  Just try to detect and assign once
  Modify `LDAP::Person` to return username value based on attributes
  Create Kubernetes based on Application Templates
  Fix a bug where charlock_holmes was used needlessly to encode strings
  Ignore the Migration/Datetime cop in a migration that fix a column type to datetime_with_timezone
  Use `__` instead of `s__` when context is not required
  Enclose props in quotes
  Update some Gitaly annotations in Gitlab::Shell
  Prepare Gitlab::Git::Repository#rebase for Gitaly migration
  add missing changelog
  refactor spec, add docs
  deprecate check integrity task
  add lock specs
  add locks chek
  remove max-depth flag so it works with subgroups
  ...

app/assets/javascripts/groups/components/item_stats.vue

@@ -42,28 +42,28 @@ export default {
       v-if="isGroup"
       css-class="number-subgroups"
       icon-name="folder"
-      :title="s__('Subgroups')"
-      :value=item.subgroupCount
+      :title="__('Subgroups')"
+      :value="item.subgroupCount"
     />
     <item-stats-value
       v-if="isGroup"
       css-class="number-projects"
       icon-name="bookmark"
-      :title="s__('Projects')"
-      :value=item.projectCount
+      :title="__('Projects')"
+      :value="item.projectCount"
     />
     <item-stats-value
       v-if="isGroup"
       css-class="number-users"
       icon-name="users"
-      :title="s__('Members')"
-      :value=item.memberCount
+      :title="__('Members')"
+      :value="item.memberCount"
     />
     <item-stats-value
       v-if="isProject"
       css-class="project-stars"
       icon-name="star"
-      :value=item.starCount
+      :value="item.starCount"
     />
     <item-stats-value
       css-class="item-visibility"

app/models/concerns/deployment_platform.rb

+module DeploymentPlatform
+  def deployment_platform
+    @deployment_platform ||=
+      find_cluster_platform_kubernetes ||
+      find_kubernetes_service_integration ||
+      build_cluster_and_deployment_platform
+  end
+
+  private
+
+  def find_cluster_platform_kubernetes
+    clusters.find_by(enabled: true)&.platform_kubernetes
+  end
+
+  def find_kubernetes_service_integration
+    services.deployment.reorder(nil).find_by(active: true)
+  end
+
+  def build_cluster_and_deployment_platform
+    return unless kubernetes_service_template
+
+    cluster = ::Clusters::Cluster.create(cluster_attributes_from_service_template)
+    cluster.platform_kubernetes if cluster.persisted?
+  end
+
+  def kubernetes_service_template
+    @kubernetes_service_template ||= KubernetesService.active.find_by_template
+  end
+
+  def cluster_attributes_from_service_template
+    {
+      name: 'kubernetes-template',
+      projects: [self],
+      provider_type: :user,
+      platform_type: :kubernetes,
+      platform_kubernetes_attributes: platform_kubernetes_attributes_from_service_template
+    }
+  end
+
+  def platform_kubernetes_attributes_from_service_template
+    {
+      api_url: kubernetes_service_template.api_url,
+      ca_pem: kubernetes_service_template.ca_pem,
+      token: kubernetes_service_template.token,
+      namespace: kubernetes_service_template.namespace
+    }
+  end
+end

app/models/project.rb

@@ -19,6 +19,7 @@ class Project < ActiveRecord::Base
   include Routable
   include GroupDescendant
   include Gitlab::SQL::Pattern
+  include DeploymentPlatform
 
   extend Gitlab::ConfigHelper
   extend Gitlab::CurrentSettings
@@ -904,12 +905,6 @@ class Project < ActiveRecord::Base
     @ci_service ||= ci_services.reorder(nil).find_by(active: true)
   end
 
-  # TODO: This will be extended for multiple enviroment clusters
-  def deployment_platform
-    @deployment_platform ||= clusters.find_by(enabled: true)&.platform_kubernetes
-    @deployment_platform ||= services.where(category: :deployment).reorder(nil).find_by(active: true)
-  end
-
   def monitoring_services
     services.where(category: :monitoring)
   end

app/models/service.rb

@@ -44,6 +44,7 @@ class Service < ActiveRecord::Base
   scope :pipeline_hooks, -> { where(pipeline_events: true, active: true) }
   scope :wiki_page_hooks, -> { where(wiki_page_events: true, active: true) }
   scope :external_issue_trackers, -> { issue_trackers.active.without_defaults }
+  scope :deployment, -> { where(category: 'deployment') }
 
   default_value_for :category, 'common'
 
@@ -271,6 +272,10 @@ class Service < ActiveRecord::Base
     nil
   end
 
+  def self.find_by_template
+    find_by(template: true)
+  end
+
   private
 
   def cache_project_has_external_issue_tracker

changelogs/unreleased/40228-verify-integrity-of-repositories.yml

+---
+title: Fix gitlab-rake gitlab:import:repos import schedule
+merge_request: 15931
+author:
+type: fixed

changelogs/unreleased/41056-create-cluster-from-kubernetes-integration-application-template.yml

+---
+title: Allow automatic creation of Kubernetes Integration from template
+merge_request: 16104
+author:
+type: added

changelogs/unreleased/feature-api_runners_online.yml

+---
+title: Add online and status attribute to runner api entity
+merge_request: 11750
+author:
+type: added

changelogs/unreleased/ldap_username_attributes.yml

+---
+title: Modify `LDAP::Person` to return username value based on attributes
+merge_request:
+author:
+type: fixed

db/post_migrate/20171221140220_schedule_issues_closed_at_type_change.rb

 # See http://doc.gitlab.com/ce/development/migration_style_guide.html
 # for more information on how to write migrations for GitLab.
-
+# rubocop:disable Migration/Datetime
 class ScheduleIssuesClosedAtTypeChange < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 

doc/administration/raketasks/check.md

@@ -28,19 +28,25 @@ exactly which repositories are causing the trouble.
 
 ### Check all GitLab repositories
 
+>**Note:**
+>
+>  - `gitlab:repo:check` has been deprecated in favor of `gitlab:git:fsck`
+>  - [Deprecated][ce-15931] in GitLab 10.4.
+>  - `gitlab:repo:check` will be removed in the future. [Removal issue][ce-41699]
+
 This task loops through all repositories on the GitLab server and runs the
 3 integrity checks described previously.
 
 **Omnibus Installation**
 
 ```
-sudo gitlab-rake gitlab:repo:check
+sudo gitlab-rake gitlab:git:fsck
 ```
 
 **Source Installation**
 
 ```bash
-sudo -u git -H bundle exec rake gitlab:repo:check RAILS_ENV=production
+sudo -u git -H bundle exec rake gitlab:git:fsck RAILS_ENV=production
 ```
 
 ### Check repositories for a specific user
@@ -76,3 +82,6 @@ The LDAP check Rake task will test the bind_dn and password credentials
 (if configured) and will list a sample of LDAP users. This task is also
 executed as part of the `gitlab:check` task, but can run independently.
 See [LDAP Rake Tasks - LDAP Check](ldap.md#check) for details.
+
+[ce-15931]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/15931
+[ce-41699]: https://gitlab.com/gitlab-org/gitlab-ce/issues/41699

doc/api/runners.md

@@ -30,14 +30,18 @@ Example response:
         "description": "test-1-20150125",
         "id": 6,
         "is_shared": false,
-        "name": null
+        "name": null,
+        "online": true,
+        "status": "online"
     },
     {
         "active": true,
         "description": "test-2-20150125",
         "id": 8,
         "is_shared": false,
-        "name": null
+        "name": null,
+        "online": false,
+        "status": "offline"
     }
 ]
 ```
@@ -69,28 +73,36 @@ Example response:
         "description": "shared-runner-1",
         "id": 1,
         "is_shared": true,
-        "name": null
+        "name": null,
+        "online": true,
+        "status": "online"
     },
     {
         "active": true,
         "description": "shared-runner-2",
         "id": 3,
         "is_shared": true,
-        "name": null
+        "name": null,
+        "online": false
+        "status": "offline"
     },
     {
         "active": true,
         "description": "test-1-20150125",
         "id": 6,
         "is_shared": false,
-        "name": null
+        "name": null,
+        "online": true
+        "status": "paused"
     },
     {
         "active": true,
         "description": "test-2-20150125",
         "id": 8,
         "is_shared": false,
-        "name": null
+        "name": null,
+        "online": false,
+        "status": "offline"
     }
 ]
 ```
@@ -122,6 +134,8 @@ Example response:
     "is_shared": false,
     "contacted_at": "2016-01-25T16:39:48.066Z",
     "name": null,
+    "online": true,
+    "status": "online",
     "platform": null,
     "projects": [
         {
@@ -176,6 +190,8 @@ Example response:
     "is_shared": false,
     "contacted_at": "2016-01-25T16:39:48.066Z",
     "name": null,
+    "online": true,
+    "status": "online",
     "platform": null,
     "projects": [
         {
@@ -327,14 +343,18 @@ Example response:
         "description": "test-2-20150125",
         "id": 8,
         "is_shared": false,
-        "name": null
+        "name": null,
+        "online": false,
+        "status": "offline"
     },
     {
         "active": true,
         "description": "development_runner",
         "id": 5,
         "is_shared": true,
-        "name": null
+        "name": null,
+        "online": true
+        "status": "paused"
     }
 ]
 ```
@@ -364,7 +384,9 @@ Example response:
     "description": "test-2016-02-01",
     "id": 9,
     "is_shared": false,
-    "name": null
+    "name": null,
+    "online": true,
+    "status": "online"
 }
 ```
 

doc/development/gitaly.md

@@ -97,6 +97,29 @@ describe 'Gitaly Request count tests' do
 end
 ```
 
+## Running tests with a locally modified version of Gitaly
+
+Normally, gitlab-ce/ee tests use a local clone of Gitaly in `tmp/tests/gitaly`
+pinned at the version specified in GITALY_SERVER_VERSION. If you want
+to run tests locally against a modified version of Gitaly you can
+replace `tmp/tests/gitaly` with a symlink.
+
+```shell
+rm -rf tmp/tests/gitaly
+ln -s /path/to/gitaly tmp/tests/gitaly
+```
+
+Make sure you run `make` in your local Gitaly directory before running
+tests. Otherwise, Gitaly will fail to boot.
+
+If you make changes to your local Gitaly in between test runs you need
+to manually run `make` again.
+
+Note that CI tests will not use your locally modified version of
+Gitaly. To use a custom Gitaly version in CI you need to update
+GITALY_SERVER_VERSION. You can use the format `=revision` to use a
+non-tagged commit from https://gitlab.com/gitlab-org/gitaly in CI.
+
 ---
 
 [Return to Development documentation](README.md)

lib/api/entities.rb

@@ -862,6 +862,8 @@ module API
       expose :active
       expose :is_shared
       expose :name
+      expose :online?, as: :online
+      expose :status
     end
 
     class RunnerDetails < Runner

lib/gitlab/encoding_helper.rb

@@ -14,14 +14,7 @@ module Gitlab
     ENCODING_CONFIDENCE_THRESHOLD = 50
 
     def encode!(message)
-      return nil unless message.respond_to?(:force_encoding)
-      return message if message.encoding == Encoding::UTF_8 && message.valid_encoding?
-
-      if message.respond_to?(:frozen?) && message.frozen?
-        message = message.dup
-      end
-
-      message.force_encoding("UTF-8")
+      message = force_encode_utf8(message)
       return message if message.valid_encoding?
 
       # return message if message type is binary
@@ -35,6 +28,8 @@ module Gitlab
 
       # encode and clean the bad chars
       message.replace clean(message)
+    rescue ArgumentError
+      return nil
     rescue
       encoding = detect ? detect[:encoding] : "unknown"
       "--broken encoding: #{encoding}"
@@ -54,8 +49,8 @@ module Gitlab
     end
 
     def encode_utf8(message)
-      return nil unless message.is_a?(String)
-      return message if message.encoding == Encoding::UTF_8 && message.valid_encoding?
+      message = force_encode_utf8(message)
+      return message if message.valid_encoding?
 
       detect = CharlockHolmes::EncodingDetector.detect(message)
       if detect && detect[:encoding]
@@ -69,6 +64,8 @@ module Gitlab
       else
         clean(message)
       end
+    rescue ArgumentError
+      return nil
     end
 
     def encode_binary(s)
@@ -83,6 +80,15 @@ module Gitlab
 
     private
 
+    def force_encode_utf8(message)
+      raise ArgumentError unless message.respond_to?(:force_encoding)
+      return message if message.encoding == Encoding::UTF_8 && message.valid_encoding?
+
+      message = message.dup if message.respond_to?(:frozen?) && message.frozen?
+
+      message.force_encoding("UTF-8")
+    end
+
     def clean(message)
       message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "")
         .encode("UTF-8")

lib/gitlab/git.rb

@@ -11,7 +11,7 @@ module Gitlab
       include Gitlab::EncodingHelper
 
       def ref_name(ref)
-        encode_utf8(ref).sub(/\Arefs\/(tags|heads|remotes)\//, '')
+        encode!(ref).sub(/\Arefs\/(tags|heads|remotes)\//, '')
       end
 
       def branch_name(ref)

lib/gitlab/git/blob.rb

@@ -50,12 +50,21 @@ module Gitlab
         # to the caller to limit the number of blobs and blob_size_limit.
         #
         # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/798
-        def batch(repository, blob_references, blob_size_limit: nil)
-          blob_size_limit ||= MAX_DATA_DISPLAY_SIZE
+        def batch(repository, blob_references, blob_size_limit: MAX_DATA_DISPLAY_SIZE)
+          Gitlab::GitalyClient.migrate(:list_blobs_by_sha_path) do |is_enabled|
+            if is_enabled
+              Gitlab::GitalyClient.allow_n_plus_1_calls do
+                blob_references.map do |sha, path|
+                  find_by_gitaly(repository, sha, path, limit: blob_size_limit)
+                end
+              end
+            else
               blob_references.map do |sha, path|
                 find_by_rugged(repository, sha, path, limit: blob_size_limit)
               end
             end
+          end
+        end
 
         # Find LFS blobs given an array of sha ids
         # Returns array of Gitlab::Git::Blob
@@ -122,13 +131,23 @@ module Gitlab
           )
         end
 
-        def find_by_gitaly(repository, sha, path)
+        def find_by_gitaly(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE)
           path = path.sub(/\A\/*/, '')
           path = '/' if path.empty?
           name = File.basename(path)
-          entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE)
+
+          # Gitaly will think that setting the limit to 0 means unlimited, while
+          # the client might only need the metadata and thus set the limit to 0.
+          # In this method we'll then set the limit to 1, but clear the byte of data
+          # that we got back so for the outside world it looks like the limit was
+          # actually 0.
+          req_limit = limit == 0 ? 1 : limit
+
+          entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, req_limit)
           return unless entry
 
+          entry.data = "" if limit == 0
+
           case entry.type
           when :COMMIT
             new(

lib/gitlab/git/operation_service.rb

@@ -97,6 +97,11 @@ module Gitlab
         end
       end
 
+      def update_branch(branch_name, newrev, oldrev)
+        ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name
+        update_ref_in_hooks(ref, newrev, oldrev)
+      end
+
       private
 
       # Returns [newrev, should_run_after_create, should_run_after_create_branch]

lib/gitlab/git/repository.rb

@@ -1219,9 +1219,16 @@ module Gitlab
         rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id)
         env = git_env_for_user(user)
 
+        if remote_repository.is_a?(RemoteRepository)
+          env.merge!(remote_repository.fetch_env)
+          remote_repo_path = GITALY_INTERNAL_URL
+        else
+          remote_repo_path = remote_repository.path
+        end
+
         with_worktree(rebase_path, branch, env: env) do
           run_git!(
-            %W(pull --rebase #{remote_repository.path} #{remote_branch}),
+            %W(pull --rebase #{remote_repo_path} #{remote_branch}),
             chdir: rebase_path, env: env
           )
 

lib/gitlab/ldap/adapter.rb

@@ -74,7 +74,7 @@ module Gitlab
 
       def user_options(fields, value, limit)
         options = {
-          attributes: Gitlab::LDAP::Person.ldap_attributes(config).compact.uniq,
+          attributes: Gitlab::LDAP::Person.ldap_attributes(config),
           base: config.base
         }
 

lib/gitlab/ldap/config.rb

@@ -148,7 +148,7 @@ module Gitlab
 
       def default_attributes
         {
-          'username'    => %w(uid userid sAMAccountName),
+          'username'    => %w(uid sAMAccountName userid),
           'email'       => %w(mail email userPrincipalName),
           'name'        => 'cn',
           'first_name'  => 'givenName',

lib/gitlab/ldap/person.rb

@@ -6,6 +6,8 @@ module Gitlab
       # Source: http://ctogonewild.com/2009/09/03/bitmask-searches-in-ldap/
       AD_USER_DISABLED = Net::LDAP::Filter.ex("userAccountControl:1.2.840.113556.1.4.803", "2")
 
+      InvalidEntryError = Class.new(StandardError)
+
       attr_accessor :entry, :provider
 
       def self.find_by_uid(uid, adapter)
@@ -29,11 +31,12 @@ module Gitlab
 
       def self.ldap_attributes(config)
         [
-          'dn', # Used in `dn`
-          config.uid, # Used in `uid`
-          *config.attributes['name'], # Used in `name`
-          *config.attributes['email'] # Used in `email`
-        ]
+          'dn',
+          config.uid,
+          *config.attributes['name'],
+          *config.attributes['email'],
+          *config.attributes['username']
+        ].compact.uniq
       end
 
       def self.normalize_dn(dn)
@@ -60,6 +63,8 @@ module Gitlab
         Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" }
         @entry = entry
         @provider = provider
+
+        validate_entry
       end
 
       def name
@@ -71,7 +76,13 @@ module Gitlab
       end
 
       def username
-        uid
+        username = attribute_value(:username)
+
+        # Depending on the attribute, multiple values may
+        # be returned. We need only one for username.
+        # Ex. `uid` returns only one value but `mail` may
+        # return an array of multiple email addresses.
+        [username].flatten.first
       end
 
       def email
@@ -104,6 +115,19 @@ module Gitlab
 
         entry.public_send(selected_attr) # rubocop:disable GitlabSecurity/PublicSend
       end
+
+      def validate_entry
+        allowed_attrs = self.class.ldap_attributes(config).map(&:downcase)
+
+        # Net::LDAP::Entry transforms keys to symbols. Change to strings to compare.
+        entry_attrs = entry.attribute_names.map { |n| n.to_s.downcase }
+        invalid_attrs = entry_attrs - allowed_attrs
+
+        if invalid_attrs.any?
+          raise InvalidEntryError,
+                "#{self.class.name} initialized with Net::LDAP::Entry containing invalid attributes(s): #{invalid_attrs}"
+        end
+      end
     end
   end
 end

lib/gitlab/setup_helper.rb

+module Gitlab
+  module SetupHelper
+    class << self
+      # We cannot create config.toml files for all possible Gitaly configuations.
+      # For instance, if Gitaly is running on another machine then it makes no
+      # sense to write a config.toml file on the current machine. This method will
+      # only generate a configuration for the most common and simplest case: when
+      # we have exactly one Gitaly process and we are sure it is running locally
+      # because it uses a Unix socket.
+      # For development and testing purposes, an extra storage is added to gitaly,
+      # which is not known to Rails, but must be explicitly stubbed.
+      def gitaly_configuration_toml(gitaly_dir, gitaly_ruby: true)
+        storages = []
+        address = nil
+
+        Gitlab.config.repositories.storages.each do |key, val|
+          if address
+            if address != val['gitaly_address']
+              raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address."
+            end
+          elsif URI(val['gitaly_address']).scheme != 'unix'
+            raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses."
+          else
+            address = val['gitaly_address']
+          end
+
+          storages << { name: key, path: val['path'] }
+        end
+
+        if Rails.env.test?
+          storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s }
+        end
+
+        config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages }
+        config[:auth] = { token: 'secret' } if Rails.env.test?
+        config[:'gitaly-ruby'] = { dir: File.join(gitaly_dir, 'ruby') } if gitaly_ruby
+        config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path }
+        config[:bin_dir] = Gitlab.config.gitaly.client_path
+
+        TOML.dump(config)
+      end
+
+      # rubocop:disable Rails/Output
+      def create_gitaly_configuration(dir, force: false)
+        config_path = File.join(dir, 'config.toml')
+        FileUtils.rm_f(config_path) if force
+
+        File.open(config_path, File::WRONLY | File::CREAT | File::EXCL) do |f|
+          f.puts gitaly_configuration_toml(dir)
+        end
+      rescue Errno::EEXIST
+        puts "Skipping config.toml generation:"
+        puts "A configuration file already exists."
+      rescue ArgumentError => e
+        puts "Skipping config.toml generation:"
+        puts e.message
+      end
+      # rubocop:enable Rails/Output
+    end
+  end
+end

lib/gitlab/shell.rb

@@ -71,7 +71,6 @@ module Gitlab
     # Ex.
     #   add_repository("/path/to/storage", "gitlab/gitlab-ci")
     #
-    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
     def add_repository(storage, name)
       relative_path = name.dup
       relative_path << '.git' unless relative_path.end_with?('.git')
@@ -100,7 +99,7 @@ module Gitlab
     # Ex.
     #   import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git")
     #
-    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
+    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/874
     def import_repository(storage, name, url)
       # The timeout ensures the subprocess won't hang forever
       cmd = gitlab_projects(storage, "#{name}.git")
@@ -122,7 +121,6 @@ module Gitlab
     # Ex.
     #   fetch_remote(my_repo, "upstream")
     #
-    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
     def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false)
       gitaly_migrate(:fetch_remote) do |is_enabled|
         if is_enabled
@@ -142,7 +140,7 @@ module Gitlab
     # Ex.
     #   mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new")
     #
-    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
+    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873
     def mv_repository(storage, path, new_path)
       gitlab_projects(storage, "#{path}.git").mv_project("#{new_path}.git")
     end
@@ -156,7 +154,7 @@ module Gitlab
     # Ex.
     #  fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "new-namespace/gitlab-ci")
     #
-    # Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one.
+    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/817
     def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path)
       gitlab_projects(forked_from_storage, "#{forked_from_disk_path}.git")
         .fork_repository(forked_to_storage, "#{forked_to_disk_path}.git")
@@ -170,7 +168,7 @@ module Gitlab
     # Ex.
     #   remove_repository("/path/to/storage", "gitlab/gitlab-ci")
     #
-    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
+    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873
     def remove_repository(storage, name)
       gitlab_projects(storage, "#{name}.git").rm_project
     end
@@ -221,7 +219,6 @@ module Gitlab
     # Ex.
     #   add_namespace("/path/to/storage", "gitlab")
     #
-    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
     def add_namespace(storage, name)
       Gitlab::GitalyClient.migrate(:add_namespace) do |enabled|
         if enabled
@@ -243,7 +240,6 @@ module Gitlab
     # Ex.
     #   rm_namespace("/path/to/storage", "gitlab")
     #
-    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
     def rm_namespace(storage, name)
       Gitlab::GitalyClient.migrate(:remove_namespace) do |enabled|
         if enabled
@@ -261,7 +257,6 @@ module Gitlab
     # Ex.
     #   mv_namespace("/path/to/storage", "gitlab", "gitlabhq")
     #
-    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
     def mv_namespace(storage, old_name, new_name)
       Gitlab::GitalyClient.migrate(:rename_namespace) do |enabled|
         if enabled

lib/tasks/gitlab/check.rake

@@ -387,14 +387,8 @@ namespace :gitlab do
   namespace :repo do
     desc "GitLab | Check the integrity of the repositories managed by GitLab"
     task check: :environment do
-      Gitlab.config.repositories.storages.each do |name, repository_storage|
-        namespace_dirs = Dir.glob(File.join(repository_storage['path'], '*'))
-
-        namespace_dirs.each do |namespace_dir|
-          repo_dirs = Dir.glob(File.join(namespace_dir, '*'))
-          repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
-        end
-      end
+      puts "This task is deprecated. Please use gitlab:git:fsck instead".color(:red)
+      Rake::Task["gitlab:git:fsck"].execute
     end
   end
 
@@ -461,35 +455,4 @@ namespace :gitlab do
       puts "FAIL. Please update gitlab-shell to #{required_version} from #{current_version}".color(:red)
     end
   end
-
-  def check_repo_integrity(repo_dir)
-    puts "\nChecking repo at #{repo_dir.color(:yellow)}"
-
-    git_fsck(repo_dir)
-    check_config_lock(repo_dir)
-    check_ref_locks(repo_dir)
-  end
-
-  def git_fsck(repo_dir)
-    puts "Running `git fsck`".color(:yellow)
-    system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: repo_dir)
-  end
-
-  def check_config_lock(repo_dir)
-    config_exists = File.exist?(File.join(repo_dir, 'config.lock'))
-    config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green)
-    puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}"
-  end
-
-  def check_ref_locks(repo_dir)
-    lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock'))
-    if lock_files.present?
-      puts "Ref lock files exist:".color(:red)
-      lock_files.each do |lock_file|
-        puts "  #{lock_file}"
-      end
-    else
-      puts "No ref lock files exist".color(:green)
-    end
-  end
 end

lib/tasks/gitlab/git.rake

@@ -30,6 +30,20 @@ namespace :gitlab do
       end
     end
 
+    desc 'GitLab | Git | Check all repos integrity'
+    task fsck: :environment do
+      failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} fsck --name-objects --no-progress), "Checking integrity") do |repo|
+        check_config_lock(repo)
+        check_ref_locks(repo)
+      end
+
+      if failures.empty?
+        puts "Done".color(:green)
+      else
+        output_failures(failures)
+      end
+    end
+
     def perform_git_cmd(cmd, message)
       puts "Starting #{message} on all repositories"
 
@@ -40,6 +54,8 @@ namespace :gitlab do
         else
           failures << repo
         end
+
+        yield(repo) if block_given?
       end
 
       failures
@@ -49,5 +65,24 @@ namespace :gitlab do
       puts "The following repositories reported errors:".color(:red)
       failures.each { |f| puts "- #{f}" }
     end
+
+    def check_config_lock(repo_dir)
+      config_exists = File.exist?(File.join(repo_dir, 'config.lock'))
+      config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green)
+
+      puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}"
+    end
+
+    def check_ref_locks(repo_dir)
+      lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock'))
+
+      if lock_files.present?
+        puts "Ref lock files exist:".color(:red)
+
+        lock_files.each { |lock_file| puts "  #{lock_file}" }
+      else
+        puts "No ref lock files exist".color(:green)
+      end
+    end
   end
 end

lib/tasks/gitlab/gitaly.rake

@@ -21,8 +21,8 @@ namespace :gitlab do
 
       command << 'BUNDLE_FLAGS=--no-deployment' if Rails.env.test?
 
+      Gitlab::SetupHelper.create_gitaly_configuration(args.dir)
       Dir.chdir(args.dir) do
-        create_gitaly_configuration
         # In CI we run scripts/gitaly-test-build instead of this command
         unless ENV['CI'].present?
           Bundler.with_original_env { run_command!(command) }
@@ -39,60 +39,7 @@ namespace :gitlab do
 
       # Exclude gitaly-ruby configuration because that depends on the gitaly
       # installation directory.
-      puts gitaly_configuration_toml(gitaly_ruby: false)
-    end
-
-    private
-
-    # We cannot create config.toml files for all possible Gitaly configuations.
-    # For instance, if Gitaly is running on another machine then it makes no
-    # sense to write a config.toml file on the current machine. This method will
-    # only generate a configuration for the most common and simplest case: when
-    # we have exactly one Gitaly process and we are sure it is running locally
-    # because it uses a Unix socket.
-    # For development and testing purposes, an extra storage is added to gitaly,
-    # which is not known to Rails, but must be explicitly stubbed.
-    def gitaly_configuration_toml(gitaly_ruby: true)
-      storages = []
-      address = nil
-
-      Gitlab.config.repositories.storages.each do |key, val|
-        if address
-          if address != val['gitaly_address']
-            raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address."
-          end
-        elsif URI(val['gitaly_address']).scheme != 'unix'
-          raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses."
-        else
-          address = val['gitaly_address']
-        end
-
-        storages << { name: key, path: val['path'] }
-      end
-
-      if Rails.env.test?
-        storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s }
-      end
-
-      config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages }
-      config[:auth] = { token: 'secret' } if Rails.env.test?
-      config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby
-      config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path }
-      config[:bin_dir] = Gitlab.config.gitaly.client_path
-
-      TOML.dump(config)
-    end
-
-    def create_gitaly_configuration
-      File.open("config.toml", File::WRONLY | File::CREAT | File::EXCL) do |f|
-        f.puts gitaly_configuration_toml
-      end
-    rescue Errno::EEXIST
-      puts "Skipping config.toml generation:"
-      puts "A configuration file already exists."
-    rescue ArgumentError => e
-      puts "Skipping config.toml generation:"
-      puts e.message
+      puts Gitlab::SetupHelper.gitaly_configuration_toml('', gitaly_ruby: false)
     end
   end
 end

lib/tasks/gitlab/task_helpers.rb

@@ -130,7 +130,7 @@ module Gitlab
 
     def all_repos
       Gitlab.config.repositories.storages.each_value do |repository_storage|
-        IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find|
+        IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -type d -name *.git)) do |find|
           find.each_line do |path|
             yield path.chomp
           end

spec/features/projects/import_export/export_file_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 
 # Integration test that exports a file using the Import/Export feature
 # It looks up for any sensitive word inside the JSON, so if a sensitive word is found
-# we''l have to either include it adding the model that includes it to the +safe_list+
+# we'll have to either include it adding the model that includes it to the +safe_list+
 # or make sure the attribute is blacklisted in the +import_export.yml+ configuration
 feature 'Import/Export - project export integration test', :js do
   include Select2Helper

spec/lib/gitlab/encoding_helper_spec.rb

@@ -120,6 +120,24 @@ describe Gitlab::EncodingHelper do
     it 'returns empty string on conversion errors' do
       expect { ext_class.encode_utf8('') }.not_to raise_error(ArgumentError)
     end
+
+    context 'with strings that can be forcefully encoded into utf8' do
+      let(:test_string) do
+        "refs/heads/FixSymbolsTitleDropdown".encode("ASCII-8BIT")
+      end
+      let(:expected_string) do
+        "refs/heads/FixSymbolsTitleDropdown".encode("UTF-8")
+      end
+
+      subject { ext_class.encode_utf8(test_string) }
+
+      it "doesn't use CharlockHolmes if the encoding can be forced into utf_8" do
+        expect(CharlockHolmes::EncodingDetector).not_to receive(:detect)
+
+        expect(subject).to eq(expected_string)
+        expect(subject.encoding.name).to eq('UTF-8')
+      end
+    end
   end
 
   describe '#clean' do

spec/lib/gitlab/git/blob_spec.rb

@@ -202,16 +202,6 @@ describe Gitlab::Git::Blob, seed_helper: true do
     context 'limiting' do
       subject { described_class.batch(repository, blob_references, blob_size_limit: blob_size_limit) }
 
-      context 'default' do
-        let(:blob_size_limit) { nil }
-
-        it 'limits to MAX_DATA_DISPLAY_SIZE' do
-          stub_const('Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE', 100)
-
-          expect(subject.first.data.size).to eq(100)
-        end
-      end
-
       context 'positive' do
         let(:blob_size_limit) { 10 }
 
@@ -221,7 +211,10 @@ describe Gitlab::Git::Blob, seed_helper: true do
       context 'zero' do
         let(:blob_size_limit) { 0 }
 
-        it { expect(subject.first.data).to eq('') }
+        it 'only loads the metadata' do
+          expect(subject.first.size).not_to be(0)
+          expect(subject.first.data).to eq('')
+        end
       end
 
       context 'negative' do

spec/lib/gitlab/ldap/adapter_spec.rb

@@ -16,7 +16,7 @@ describe Gitlab::LDAP::Adapter do
       expect(adapter).to receive(:ldap_search) do |arg|
         expect(arg[:filter].to_s).to eq('(uid=johndoe)')
         expect(arg[:base]).to eq('dc=example,dc=com')
-        expect(arg[:attributes]).to match(%w{dn uid cn mail email userPrincipalName})
+        expect(arg[:attributes]).to match(ldap_attributes)
       end.and_return({})
 
       adapter.users('uid', 'johndoe')
@@ -26,7 +26,7 @@ describe Gitlab::LDAP::Adapter do
       expect(adapter).to receive(:ldap_search).with(
         base: 'uid=johndoe,ou=users,dc=example,dc=com',
         scope: Net::LDAP::SearchScope_BaseObject,
-        attributes: %w{dn uid cn mail email userPrincipalName},
+        attributes: ldap_attributes,
         filter: nil
       ).and_return({})
 
@@ -63,7 +63,7 @@ describe Gitlab::LDAP::Adapter do
     it 'uses the right uid attribute when non-default' do
       stub_ldap_config(uid: 'sAMAccountName')
       expect(adapter).to receive(:ldap_search).with(
-        hash_including(attributes: %w{dn sAMAccountName cn mail email userPrincipalName})
+        hash_including(attributes: ldap_attributes)
       ).and_return({})
 
       adapter.users('sAMAccountName', 'johndoe')
@@ -137,4 +137,8 @@ describe Gitlab::LDAP::Adapter do
       end
     end
   end
+
+  def ldap_attributes
+    Gitlab::LDAP::Person.ldap_attributes(Gitlab::LDAP::Config.new('ldapmain'))
+  end
 end

spec/lib/gitlab/ldap/person_spec.rb

@@ -8,13 +8,16 @@ describe Gitlab::LDAP::Person do
   before do
     stub_ldap_config(
       options: {
+        'uid'        => 'uid',
         'attributes' => {
           'name'     => 'cn',
-          'email' => %w(mail email userPrincipalName)
+          'email'    => %w(mail email userPrincipalName),
+          'username' => username_attribute
         }
       }
     )
   end
+  let(:username_attribute) { %w(uid sAMAccountName userid) }
 
   describe '.normalize_dn' do
     subject { described_class.normalize_dn(given) }
@@ -44,6 +47,34 @@ describe Gitlab::LDAP::Person do
     end
   end
 
+  describe '.ldap_attributes' do
+    it 'returns a compact and unique array' do
+      stub_ldap_config(
+        options: {
+          'uid'        => nil,
+          'attributes' => {
+            'name'     => 'cn',
+            'email'    => 'mail',
+            'username' => %w(uid mail memberof)
+          }
+        }
+      )
+      config = Gitlab::LDAP::Config.new('ldapmain')
+      ldap_attributes = described_class.ldap_attributes(config)
+
+      expect(ldap_attributes).to match_array(%w(dn uid cn mail memberof))
+    end
+  end
+
+  describe '.validate_entry' do
+    it 'raises InvalidEntryError' do
+      entry['foo'] = 'bar'
+
+      expect { described_class.new(entry, 'ldapmain') }
+        .to raise_error(Gitlab::LDAP::Person::InvalidEntryError)
+    end
+  end
+
   describe '#name' do
     it 'uses the configured name attribute and handles values as an array' do
       name = 'John Doe'
@@ -72,6 +103,44 @@ describe Gitlab::LDAP::Person do
     end
   end
 
+  describe '#username' do
+    context 'with default uid username attribute' do
+      let(:username_attribute) { 'uid' }
+
+      it 'returns the proper username value' do
+        attr_value = 'johndoe'
+        entry[username_attribute] = attr_value
+        person = described_class.new(entry, 'ldapmain')
+
+        expect(person.username).to eq(attr_value)
+      end
+    end
+
+    context 'with a different username attribute' do
+      let(:username_attribute) { 'sAMAccountName' }
+
+      it 'returns the proper username value' do
+        attr_value = 'johndoe'
+        entry[username_attribute] = attr_value
+        person = described_class.new(entry, 'ldapmain')
+
+        expect(person.username).to eq(attr_value)
+      end
+    end
+
+    context 'with a non-standard username attribute' do
+      let(:username_attribute) { 'mail' }
+
+      it 'returns the proper username value' do
+        attr_value = 'john.doe@example.com'
+        entry[username_attribute] = attr_value
+        person = described_class.new(entry, 'ldapmain')
+
+        expect(person.username).to eq(attr_value)
+      end
+    end
+  end
+
   def assert_generic_test(test_description, got, expected)
     test_failure_message = "Failed test description: '#{test_description}'\n\n    expected: #{expected}\n         got: #{got}"
     expect(got).to eq(expected), test_failure_message

spec/lib/gitlab/o_auth/user_spec.rb

@@ -275,6 +275,26 @@ describe Gitlab::OAuth::User do
             end
           end
 
+          context 'and a corresponding LDAP person with a non-default username' do
+            before do
+              allow(ldap_user).to receive(:uid) { uid }
+              allow(ldap_user).to receive(:username) { 'johndoe@example.com' }
+              allow(ldap_user).to receive(:email) { %w(johndoe@example.com john2@example.com) }
+              allow(ldap_user).to receive(:dn) { dn }
+            end
+
+            context 'and no account for the LDAP user' do
+              it 'creates a user favoring the LDAP username and strips email domain' do
+                allow(Gitlab::LDAP::Person).to receive(:find_by_uid).and_return(ldap_user)
+
+                oauth_user.save
+
+                expect(gl_user).to be_valid
+                expect(gl_user.username).to eql 'johndoe'
+              end
+            end
+          end
+
           context "and no corresponding LDAP person" do
             before do
               allow(Gitlab::LDAP::Person).to receive(:find_by_uid).and_return(nil)

spec/models/concerns/deployment_platform_spec.rb

+require 'rails_helper'
+
+describe DeploymentPlatform do
+  let(:project) { create(:project) }
+
+  describe '#deployment_platform' do
+    subject { project.deployment_platform }
+
+    context 'with no Kubernetes configuration on CI/CD, no Kubernetes Service and a Kubernetes template configured' do
+      let!(:kubernetes_service) { create(:kubernetes_service, template: true) }
+
+      it 'returns a platform kubernetes' do
+        expect(subject).to be_a_kind_of(Clusters::Platforms::Kubernetes)
+      end
+
+      it 'creates a cluster and a platform kubernetes' do
+        expect { subject }
+          .to change { Clusters::Cluster.count }.by(1)
+          .and change { Clusters::Platforms::Kubernetes.count }.by(1)
+      end
+
+      it 'includes appropriate attributes for Cluster' do
+        cluster = subject.cluster
+        expect(cluster.name).to eq('kubernetes-template')
+        expect(cluster.project).to eq(project)
+        expect(cluster.provider_type).to eq('user')
+        expect(cluster.platform_type).to eq('kubernetes')
+      end
+
+      it 'creates a platform kubernetes' do
+        expect { subject }.to change { Clusters::Platforms::Kubernetes.count }.by(1)
+      end
+
+      it 'copies attributes from Clusters::Platform::Kubernetes template into the new Cluster::Platforms::Kubernetes' do
+        expect(subject.api_url).to eq(kubernetes_service.api_url)
+        expect(subject.ca_pem).to eq(kubernetes_service.ca_pem)
+        expect(subject.token).to eq(kubernetes_service.token)
+        expect(subject.namespace).to eq(kubernetes_service.namespace)
+      end
+    end
+
+    context 'with no Kubernetes configuration on CI/CD, no Kubernetes Service and no Kubernetes template configured' do
+      it { is_expected.to be_nil }
+    end
+
+    context 'when user configured kubernetes from CI/CD > Clusters' do
+      let!(:cluster) { create(:cluster, :provided_by_gcp, projects: [project]) }
+      let(:platform_kubernetes) { cluster.platform_kubernetes }
+
+      it 'returns the Kubernetes platform' do
+        expect(subject).to eq(platform_kubernetes)
+      end
+    end
+
+    context 'when user configured kubernetes integration from project services' do
+      let!(:kubernetes_service) { create(:kubernetes_service, project: project) }
+
+      it 'returns the Kubernetes service' do
+        expect(subject).to eq(kubernetes_service)
+      end
+    end
+
+    context 'when the cluster creation fails' do
+      let!(:kubernetes_service) { create(:kubernetes_service, template: true) }
+
+      before do
+        allow_any_instance_of(Clusters::Cluster).to receive(:persisted?).and_return(false)
+      end
+
+      it { is_expected.to be_nil }
+    end
+  end
+end

spec/models/project_spec.rb

@@ -3137,25 +3137,6 @@ describe Project do
     end
   end
 
-  describe '#deployment_platform' do
-    subject { project.deployment_platform }
-
-    let(:project) { create(:project) }
-
-    context 'when user configured kubernetes from Integration > Kubernetes' do
-      let!(:kubernetes_service) { create(:kubernetes_service, project: project) }
-
-      it { is_expected.to eq(kubernetes_service) }
-    end
-
-    context 'when user configured kubernetes from CI/CD > Clusters' do
-      let!(:cluster) { create(:cluster, :provided_by_gcp, projects: [project]) }
-      let(:platform_kubernetes) { cluster.platform_kubernetes }
-
-      it { is_expected.to eq(platform_kubernetes) }
-    end
-  end
-
   describe '#write_repository_config' do
     set(:project) { create(:project, :repository) }
 

spec/models/service_spec.rb

@@ -272,4 +272,12 @@ describe Service do
       expect(service.deprecation_message).to be_nil
     end
   end
+
+  describe '.find_by_template' do
+    let!(:kubernetes_service) { create(:kubernetes_service, template: true) }
+
+    it 'returns service template' do
+      expect(KubernetesService.find_by_template).to eq(kubernetes_service)
+    end
+  end
 end

spec/support/test_env.rb

 require 'rspec/mocks'
+require 'toml'
 
 module TestEnv
   extend self
@@ -147,6 +148,9 @@ module TestEnv
       version: Gitlab::GitalyClient.expected_server_version,
       task: "gitlab:gitaly:install[#{gitaly_dir}]") do
 
+      # Always re-create config, in case it's outdated. This is fast anyway.
+      Gitlab::SetupHelper.create_gitaly_configuration(gitaly_dir, force: true)
+
       start_gitaly(gitaly_dir)
     end
   end
@@ -347,6 +351,9 @@ module TestEnv
   end
 
   def component_needs_update?(component_folder, expected_version)
+    # Allow local overrides of the component for tests during development
+    return false if Rails.env.test? && File.symlink?(component_folder)
+
     version = File.read(File.join(component_folder, 'VERSION')).strip
 
     # Notice that this will always yield true when using branch versions

spec/tasks/gitlab/git_rake_spec.rb

+require 'rake_helper'
+
+describe 'gitlab:git rake tasks' do
+  before do
+    Rake.application.rake_require 'tasks/gitlab/git'
+
+    storages = { 'default' => { 'path' => Settings.absolute('tmp/tests/default_storage') } }
+
+    FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git'))
+    allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
+    allow_any_instance_of(String).to receive(:color) { |string, _color| string }
+
+    stub_warn_user_is_not_gitlab
+  end
+
+  after do
+    FileUtils.rm_rf(Settings.absolute('tmp/tests/default_storage'))
+  end
+
+  describe 'fsck' do
+    it 'outputs the integrity check for a repo' do
+      expect { run_rake_task('gitlab:git:fsck') }.to output(/Performed Checking integrity at .*@hashed\/1\/2\/test.git/).to_stdout
+    end
+
+    it 'errors out about config.lock issues' do
+      FileUtils.touch(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git/config.lock'))
+
+      expect { run_rake_task('gitlab:git:fsck') }.to output(/file exists\? ... yes/).to_stdout
+    end
+
+    it 'errors out about ref lock issues' do
+      FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git/refs/heads'))
+      FileUtils.touch(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git/refs/heads/blah.lock'))
+
+      expect { run_rake_task('gitlab:git:fsck') }.to output(/Ref lock files exist:/).to_stdout
+    end
+  end
+end