repository.rb 55.9 KB
Newer Older
Robert Speicher's avatar
Robert Speicher committed
1 2 3 4 5 6 7 8
# Gitlab::Git::Repository is a wrapper around native Rugged::Repository object
require 'tempfile'
require 'forwardable'
require "rubygems/package"

module Gitlab
  module Git
    class Repository
9
      include Gitlab::Git::RepositoryMirroring
Robert Speicher's avatar
Robert Speicher committed
10
      include Gitlab::Git::Popen
11
      include Gitlab::EncodingHelper
12
      include Gitlab::Utils::StrongMemoize
Robert Speicher's avatar
Robert Speicher committed
13

14 15 16 17
      ALLOWED_OBJECT_DIRECTORIES_VARIABLES = %w[
        GIT_OBJECT_DIRECTORY
        GIT_ALTERNATE_OBJECT_DIRECTORIES
      ].freeze
18 19 20 21
      ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES = %w[
        GIT_OBJECT_DIRECTORY_RELATIVE
        GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE
      ].freeze
Robert Speicher's avatar
Robert Speicher committed
22
      SEARCH_CONTEXT_LINES = 3
23 24 25
      # In https://gitlab.com/gitlab-org/gitaly/merge_requests/698
      # We copied these two prefixes into gitaly-go, so don't change these
      # or things will break! (REBASE_WORKTREE_PREFIX and SQUASH_WORKTREE_PREFIX)
26 27
      REBASE_WORKTREE_PREFIX = 'rebase'.freeze
      SQUASH_WORKTREE_PREFIX = 'squash'.freeze
28
      GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze
29
      GITLAB_PROJECTS_TIMEOUT = Gitlab.config.gitlab_shell.git_timeout
30
      EMPTY_REPOSITORY_CHECKSUM = '0000000000000000000000000000000000000000'.freeze
Robert Speicher's avatar
Robert Speicher committed
31

32
      NoRepository = Class.new(StandardError)
33
      InvalidRepository = Class.new(StandardError)
34 35
      InvalidBlobName = Class.new(StandardError)
      InvalidRef = Class.new(StandardError)
36
      GitError = Class.new(StandardError)
37
      DeleteBranchError = Class.new(StandardError)
38
      CreateTreeError = Class.new(StandardError)
39
      TagExistsError = Class.new(StandardError)
40
      ChecksumError = Class.new(StandardError)
Robert Speicher's avatar
Robert Speicher committed
41

42
      class << self
43 44
        # Unlike `new`, `create` takes the repository path
        def create(repo_path, bare: true, symlink_hooks_to: nil)
45 46 47 48 49 50
          FileUtils.mkdir_p(repo_path, mode: 0770)

          # Equivalent to `git --git-path=#{repo_path} init [--bare]`
          repo = Rugged::Repository.init_at(repo_path, bare)
          repo.close

51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67
          create_hooks(repo_path, symlink_hooks_to) if symlink_hooks_to.present?

          true
        end

        def create_hooks(repo_path, global_hooks_path)
          local_hooks_path = File.join(repo_path, 'hooks')
          real_local_hooks_path = :not_found

          begin
            real_local_hooks_path = File.realpath(local_hooks_path)
          rescue Errno::ENOENT
            # real_local_hooks_path == :not_found
          end

          # Do nothing if hooks already exist
          unless real_local_hooks_path == File.realpath(global_hooks_path)
Rémy Coutable's avatar
Rémy Coutable committed
68 69 70 71 72 73
            if File.exist?(local_hooks_path)
              # Move the existing hooks somewhere safe
              FileUtils.mv(
                local_hooks_path,
                "#{local_hooks_path}.old.#{Time.now.to_i}")
            end
74 75 76

            # Create the hooks symlink
            FileUtils.ln_sf(global_hooks_path, local_hooks_path)
77 78 79 80 81 82
          end

          true
        end
      end

Robert Speicher's avatar
Robert Speicher committed
83 84 85
      # Directory name of repo
      attr_reader :name

86 87 88
      # Relative path of repo
      attr_reader :relative_path

Robert Speicher's avatar
Robert Speicher committed
89 90 91
      # Rugged repo object
      attr_reader :rugged

92
      attr_reader :gitlab_projects, :storage, :gl_repository, :relative_path
Jacob Vosmaer's avatar
Jacob Vosmaer committed
93

94 95
      # This initializer method is only used on the client side (gitlab-ce).
      # Gitaly-ruby uses a different initializer.
96
      def initialize(storage, relative_path, gl_repository)
Jacob Vosmaer's avatar
Jacob Vosmaer committed
97
        @storage = storage
98
        @relative_path = relative_path
99
        @gl_repository = gl_repository
100

101
        @gitlab_projects = Gitlab::Git::GitlabProjects.new(
102
          storage,
103 104 105 106
          relative_path,
          global_hooks_path: Gitlab.config.gitlab_shell.hooks_path,
          logger: Rails.logger
        )
107

108
        @name = @relative_path.split("/").last
Robert Speicher's avatar
Robert Speicher committed
109 110
      end

111
      def ==(other)
112
        [storage, relative_path] == [other.storage, other.relative_path]
113 114
      end

115 116 117 118 119 120
      def path
        @path ||= File.join(
          Gitlab.config.repositories.storages[@storage].legacy_disk_path, @relative_path
        )
      end

Robert Speicher's avatar
Robert Speicher committed
121 122
      # Default branch in the repository
      def root_ref
123 124 125 126 127
        gitaly_ref_client.default_branch_name
      rescue GRPC::NotFound => e
        raise NoRepository.new(e.message)
      rescue GRPC::Unknown => e
        raise Gitlab::Git::CommandError.new(e.message)
Robert Speicher's avatar
Robert Speicher committed
128 129 130
      end

      def rugged
131 132 133
        @rugged ||= circuit_breaker.perform do
          Rugged::Repository.new(path, alternates: alternate_object_directories)
        end
Robert Speicher's avatar
Robert Speicher committed
134 135 136 137
      rescue Rugged::RepositoryError, Rugged::OSError
        raise NoRepository.new('no repository for such path')
      end

138 139 140 141
      def cleanup
        @rugged&.close
      end

142 143 144 145
      def circuit_breaker
        @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage)
      end

146
      def exists?
147
        gitaly_repository_client.exists?
148 149
      end

Robert Speicher's avatar
Robert Speicher committed
150 151 152
      # Returns an Array of branch names
      # sorted by name ASC
      def branch_names
153 154
        wrapped_gitaly_errors do
          gitaly_ref_client.branch_names
155
        end
Robert Speicher's avatar
Robert Speicher committed
156 157 158
      end

      # Returns an Array of Branches
159
      def branches
160 161
        wrapped_gitaly_errors do
          gitaly_ref_client.branches
162
        end
Robert Speicher's avatar
Robert Speicher committed
163 164 165 166 167 168 169 170 171 172 173 174 175 176
      end

      def reload_rugged
        @rugged = nil
      end

      # Directly find a branch with a simple name (e.g. master)
      #
      # force_reload causes a new Rugged repository to be instantiated
      #
      # This is to work around a bug in libgit2 that causes in-memory refs to
      # be stale/invalid when packed-refs is changed.
      # See https://gitlab.com/gitlab-org/gitlab-ce/issues/15392#note_14538333
      def find_branch(name, force_reload = false)
177 178 179 180 181
        gitaly_migrate(:find_branch) do |is_enabled|
          if is_enabled
            gitaly_ref_client.find_branch(name)
          else
            reload_rugged if force_reload
Robert Speicher's avatar
Robert Speicher committed
182

183 184 185 186 187 188
            rugged_ref = rugged.branches[name]
            if rugged_ref
              target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
              Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
            end
          end
189
        end
Robert Speicher's avatar
Robert Speicher committed
190 191
      end

192
      def local_branches(sort_by: nil)
193 194
        wrapped_gitaly_errors do
          gitaly_ref_client.local_branches(sort_by: sort_by)
Robert Speicher's avatar
Robert Speicher committed
195 196 197 198 199
        end
      end

      # Returns the number of valid branches
      def branch_count
200
        gitaly_migrate(:branch_names) do |is_enabled|
201 202 203
          if is_enabled
            gitaly_ref_client.count_branch_names
          else
204
            rugged.branches.each(:local).count do |ref|
205 206 207 208 209 210 211 212 213 214 215
              begin
                ref.name && ref.target # ensures the branch is valid

                true
              rescue Rugged::ReferenceError
                false
              end
            end
          end
        end
      end
Robert Speicher's avatar
Robert Speicher committed
216

217 218 219 220
      def expire_has_local_branches_cache
        clear_memoization(:has_local_branches)
      end

221
      def has_local_branches?
222 223 224 225 226
        strong_memoize(:has_local_branches) do
          uncached_has_local_branches?
        end
      end

227 228 229 230 231 232 233
      # Git repository can contains some hidden refs like:
      #   /refs/notes/*
      #   /refs/git-as-svn/*
      #   /refs/pulls/*
      # This refs by default not visible in project page and not cloned to client side.
      alias_method :has_visible_content?, :has_local_branches?

234 235
      # Returns the number of valid tags
      def tag_count
236
        gitaly_migrate(:tag_names) do |is_enabled|
237 238 239 240
          if is_enabled
            gitaly_ref_client.count_tag_names
          else
            rugged.tags.count
Robert Speicher's avatar
Robert Speicher committed
241 242 243 244 245 246
          end
        end
      end

      # Returns an Array of tag names
      def tag_names
247 248 249
        wrapped_gitaly_errors do
          gitaly_ref_client.tag_names
        end
Robert Speicher's avatar
Robert Speicher committed
250 251 252
      end

      # Returns an Array of Tags
253
      #
Robert Speicher's avatar
Robert Speicher committed
254
      def tags
255 256
        wrapped_gitaly_errors do
          gitaly_ref_client.tags
257
        end
Robert Speicher's avatar
Robert Speicher committed
258 259
      end

260 261 262 263
      # Returns true if the given ref name exists
      #
      # Ref names must start with `refs/`.
      def ref_exists?(ref_name)
264 265
        gitaly_migrate(:ref_exists,
                      status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
266 267 268 269 270 271 272 273
          if is_enabled
            gitaly_ref_exists?(ref_name)
          else
            rugged_ref_exists?(ref_name)
          end
        end
      end

Robert Speicher's avatar
Robert Speicher committed
274 275 276 277
      # Returns true if the given tag exists
      #
      # name - The name of the tag as a String.
      def tag_exists?(name)
278
        gitaly_migrate(:ref_exists_tags, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
279 280 281 282 283 284
          if is_enabled
            gitaly_ref_exists?("refs/tags/#{name}")
          else
            rugged_tag_exists?(name)
          end
        end
Robert Speicher's avatar
Robert Speicher committed
285 286 287 288 289 290
      end

      # Returns true if the given branch exists
      #
      # name - The name of the branch as a String.
      def branch_exists?(name)
291
        gitaly_migrate(:ref_exists_branches, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
292 293 294 295 296 297
          if is_enabled
            gitaly_ref_exists?("refs/heads/#{name}")
          else
            rugged_branch_exists?(name)
          end
        end
Robert Speicher's avatar
Robert Speicher committed
298 299
      end

300 301 302 303 304 305
      def batch_existence(object_ids, existing: true)
        filter_method = existing ? :select : :reject

        object_ids.public_send(filter_method) do |oid| # rubocop:disable GitlabSecurity/PublicSend
          rugged.exists?(oid)
        end
Robert Speicher's avatar
Robert Speicher committed
306 307 308 309 310 311 312
      end

      # Returns an Array of branch and tag names
      def ref_names
        branch_names + tag_names
      end

313
      def delete_all_refs_except(prefixes)
314 315 316 317 318 319 320
        gitaly_migrate(:ref_delete_refs) do |is_enabled|
          if is_enabled
            gitaly_ref_client.delete_refs(except_with_prefixes: prefixes)
          else
            delete_refs(*all_ref_names_except(prefixes))
          end
        end
321 322
      end

Lin Jen-Shin's avatar
Lin Jen-Shin committed
323 324 325
      # Returns an Array of all ref names, except when it's matching pattern
      #
      # regexp - The pattern for ref names we don't want
326 327 328 329
      def all_ref_names_except(prefixes)
        rugged.references.reject do |ref|
          prefixes.any? { |p| ref.name.start_with?(p) }
        end.map(&:name)
Lin Jen-Shin's avatar
Lin Jen-Shin committed
330 331
      end

Robert Speicher's avatar
Robert Speicher committed
332 333 334 335 336 337
      def rugged_head
        rugged.head
      rescue Rugged::ReferenceError
        nil
      end

338
      def archive_metadata(ref, storage_path, project_path, format = "tar.gz", append_sha:)
Robert Speicher's avatar
Robert Speicher committed
339 340 341 342
        ref ||= root_ref
        commit = Gitlab::Git::Commit.find(self, ref)
        return {} if commit.nil?

343
        prefix = archive_prefix(ref, commit.id, project_path, append_sha: append_sha)
Robert Speicher's avatar
Robert Speicher committed
344 345 346

        {
          'ArchivePrefix' => prefix,
347
          'ArchivePath' => archive_file_path(storage_path, commit.id, prefix, format),
348 349
          'CommitId' => commit.id,
          'GitalyRepository' => gitaly_repository.to_h
Robert Speicher's avatar
Robert Speicher committed
350 351 352
        }
      end

353 354
      # This is both the filename of the archive (missing the extension) and the
      # name of the top-level member of the archive under which all files go
355
      def archive_prefix(ref, sha, project_path, append_sha:)
356 357 358 359
        append_sha = (ref != sha) if append_sha.nil?

        formatted_ref = ref.tr('/', '-')

360
        prefix_segments = [project_path, formatted_ref]
361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381
        prefix_segments << sha if append_sha

        prefix_segments.join('-')
      end
      private :archive_prefix

      # The full path on disk where the archive should be stored. This is used
      # to cache the archive between requests.
      #
      # The path is a global namespace, so needs to be globally unique. This is
      # achieved by including `gl_repository` in the path.
      #
      # Archives relating to a particular ref when the SHA is not present in the
      # filename must be invalidated when the ref is updated to point to a new
      # SHA. This is achieved by including the SHA in the path.
      #
      # As this is a full path on disk, it is not "cloud native". This should
      # be resolved by either removing the cache, or moving the implementation
      # into Gitaly and removing the ArchivePath parameter from the git-archive
      # senddata response.
      def archive_file_path(storage_path, sha, name, format = "tar.gz")
Robert Speicher's avatar
Robert Speicher committed
382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398
        # Build file path
        return nil unless name

        extension =
          case format
          when "tar.bz2", "tbz", "tbz2", "tb2", "bz2"
            "tar.bz2"
          when "tar"
            "tar"
          when "zip"
            "zip"
          else
            # everything else should fall back to tar.gz
            "tar.gz"
          end

        file_name = "#{name}.#{extension}"
399
        File.join(storage_path, self.gl_repository, sha, file_name)
Robert Speicher's avatar
Robert Speicher committed
400
      end
401
      private :archive_file_path
Robert Speicher's avatar
Robert Speicher committed
402 403 404

      # Return repo size in megabytes
      def size
405
        size = gitaly_repository_client.repository_size
406

Robert Speicher's avatar
Robert Speicher committed
407 408 409 410 411 412 413 414 415 416 417 418 419 420
        (size.to_f / 1024).round(2)
      end

      # Use the Rugged Walker API to build an array of commits.
      #
      # Usage.
      #   repo.log(
      #     ref: 'master',
      #     path: 'app/models',
      #     limit: 10,
      #     offset: 5,
      #     after: Time.new(2016, 4, 21, 14, 32, 10)
      #   )
      #
421
      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/446
Robert Speicher's avatar
Robert Speicher committed
422
      def log(options)
423 424 425 426 427 428 429
        default_options = {
          limit: 10,
          offset: 0,
          path: nil,
          follow: false,
          skip_merges: false,
          after: nil,
Tiago Botelho's avatar
Tiago Botelho committed
430 431
          before: nil,
          all: false
432 433 434 435 436
        }

        options = default_options.merge(options)
        options[:offset] ||= 0

437 438 439 440 441
        limit = options[:limit]
        if limit == 0 || !limit.is_a?(Integer)
          raise ArgumentError.new("invalid Repository#log limit: #{limit.inspect}")
        end

442 443
        wrapped_gitaly_errors do
          gitaly_commit_client.find_commits(options)
444
        end
Robert Speicher's avatar
Robert Speicher committed
445 446
      end

447
      def count_commits(options)
448
        options = process_count_commits_options(options.dup)
449

450 451 452 453 454 455 456 457 458 459 460
        wrapped_gitaly_errors do
          if options[:left_right]
            from = options[:from]
            to = options[:to]

            right_count = gitaly_commit_client
              .commit_count("#{from}..#{to}", options)
            left_count = gitaly_commit_client
              .commit_count("#{to}..#{from}", options)

            [left_count, right_count]
461
          else
462
            gitaly_commit_client.commit_count(options[:ref], options)
463 464
          end
        end
465 466
      end

Robert Speicher's avatar
Robert Speicher committed
467 468 469 470 471 472 473 474
      # Return the object that +revspec+ points to.  If +revspec+ is an
      # annotated tag, then return the tag's target instead.
      def rev_parse_target(revspec)
        obj = rugged.rev_parse(revspec)
        Ref.dereference_object(obj)
      end

      # Counts the amount of commits between `from` and `to`.
475 476
      def count_commits_between(from, to, options = {})
        count_commits(from: from, to: to, **options)
Robert Speicher's avatar
Robert Speicher committed
477 478
      end

Rubén Dávila's avatar
Rubén Dávila committed
479 480 481
      # old_rev and new_rev are commit ID's
      # the result of this method is an array of Gitlab::Git::RawDiffChange
      def raw_changes_between(old_rev, new_rev)
482 483
        @raw_changes_between ||= {}

484 485 486
        @raw_changes_between[[old_rev, new_rev]] ||=
          begin
            return [] if new_rev.blank? || new_rev == Gitlab::Git::BLANK_SHA
487

488
            wrapped_gitaly_errors do
489 490 491 492 493
              gitaly_repository_client.raw_changes_between(old_rev, new_rev)
                .each_with_object([]) do |msg, arr|
                msg.raw_changes.each { |change| arr << ::Gitlab::Git::RawDiffChange.new(change) }
              end
            end
Rubén Dávila's avatar
Rubén Dávila committed
494
          end
495 496
      rescue ArgumentError => e
        raise Gitlab::Git::Repository::GitError.new(e)
Rubén Dávila's avatar
Rubén Dávila committed
497 498
      end

Robert Speicher's avatar
Robert Speicher committed
499
      # Returns the SHA of the most recent common ancestor of +from+ and +to+
500
      def merge_base(from, to)
501 502 503 504
        gitaly_migrate(:merge_base) do |is_enabled|
          if is_enabled
            gitaly_repository_client.find_merge_base(from, to)
          else
505
            rugged_merge_base(from, to)
506 507
          end
        end
Robert Speicher's avatar
Robert Speicher committed
508 509
      end

510
      # Returns true is +from+ is direct ancestor to +to+, otherwise false
511
      def ancestor?(from, to)
512
        gitaly_commit_client.ancestor?(from, to)
513 514
      end

515
      def merged_branch_names(branch_names = [])
516 517 518 519 520 521 522 523 524 525 526 527 528 529 530
        return [] unless root_ref

        root_sha = find_branch(root_ref)&.target

        return [] unless root_sha

        branches = gitaly_migrate(:merged_branch_names) do |is_enabled|
          if is_enabled
            gitaly_merged_branch_names(branch_names, root_sha)
          else
            git_merged_branch_names(branch_names, root_sha)
          end
        end

        Set.new(branches)
531 532
      end

Robert Speicher's avatar
Robert Speicher committed
533 534 535 536 537
      # Return an array of Diff objects that represent the diff
      # between +from+ and +to+.  See Diff::filter_diff_options for the allowed
      # diff options.  The +options+ hash can also include :break_rewrites to
      # split larger rewrites into delete/add pairs.
      def diff(from, to, options = {}, *paths)
538
        iterator = gitaly_commit_client.diff(from, to, options.merge(paths: paths))
539 540

        Gitlab::Git::DiffCollection.new(iterator, options)
Robert Speicher's avatar
Robert Speicher committed
541 542
      end

543 544
      # Returns a RefName for a given SHA
      def ref_name_for_sha(ref_path, sha)
545 546
        raise ArgumentError, "sha can't be empty" unless sha.present?

547
        gitaly_ref_client.find_ref_name(sha, ref_path)
548 549
      end

550 551 552
      # Get refs hash which key is is the commit id
      # and value is a Gitlab::Git::Tag or Gitlab::Git::Branch
      # Note that both inherit from Gitlab::Git::Ref
Robert Speicher's avatar
Robert Speicher committed
553
      def refs_hash
554 555 556 557 558 559 560 561
        return @refs_hash if @refs_hash

        @refs_hash = Hash.new { |h, k| h[k] = [] }

        (tags + branches).each do |ref|
          next unless ref.target && ref.name

          @refs_hash[ref.dereferenced_target.id] << ref.name
Robert Speicher's avatar
Robert Speicher committed
562
        end
563

Robert Speicher's avatar
Robert Speicher committed
564 565 566 567 568 569 570 571
        @refs_hash
      end

      # Lookup for rugged object by oid or ref name
      def lookup(oid_or_ref_name)
        rugged.rev_parse(oid_or_ref_name)
      end

572
      # Returns url for submodule
Robert Speicher's avatar
Robert Speicher committed
573 574
      #
      # Ex.
575 576
      #   @repository.submodule_url_for('master', 'rack')
      #   # => git@localhost:rack.git
Robert Speicher's avatar
Robert Speicher committed
577
      #
578
      def submodule_url_for(ref, path)
Jacob Vosmaer's avatar
Jacob Vosmaer committed
579 580
        wrapped_gitaly_errors do
          gitaly_submodule_url_for(ref, path)
Robert Speicher's avatar
Robert Speicher committed
581 582 583 584 585
        end
      end

      # Return total commits count accessible from passed ref
      def commit_count(ref)
586 587
        wrapped_gitaly_errors do
          gitaly_commit_client.commit_count(ref)
588
        end
Robert Speicher's avatar
Robert Speicher committed
589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614
      end

      # Mimic the `git clean` command and recursively delete untracked files.
      # Valid keys that can be passed in the +options+ hash are:
      #
      # :d - Remove untracked directories
      # :f - Remove untracked directories that are managed by a different
      #      repository
      # :x - Remove ignored files
      #
      # The value in +options+ must evaluate to true for an option to take
      # effect.
      #
      # Examples:
      #
      #   repo.clean(d: true, f: true) # Enable the -d and -f options
      #
      #   repo.clean(d: false, x: true) # -x is enabled, -d is not
      def clean(options = {})
        strategies = [:remove_untracked]
        strategies.push(:force) if options[:f]
        strategies.push(:remove_ignored) if options[:x]

        # TODO: implement this method
      end

615
      def add_branch(branch_name, user:, target:)
616 617 618
        wrapped_gitaly_errors do
          gitaly_operation_client.user_create_branch(branch_name, user, target)
        end
619 620
      end

621
      def add_tag(tag_name, user:, target:, message: nil)
622 623
        wrapped_gitaly_errors do
          gitaly_operation_client.add_tag(tag_name, user, target, message)
624 625 626
        end
      end

627
      def update_branch(branch_name, user:, newrev:, oldrev:)
628 629 630 631 632 633 634
        gitaly_migrate(:operation_user_update_branch) do |is_enabled|
          if is_enabled
            gitaly_operations_client.user_update_branch(branch_name, user, newrev, oldrev)
          else
            OperationService.new(user, self).update_branch(branch_name, newrev, oldrev)
          end
        end
635 636
      end

637
      def rm_branch(branch_name, user:)
638 639
        wrapped_gitaly_errors do
          gitaly_operation_client.user_delete_branch(branch_name, user)
640
        end
641 642
      end

643
      def rm_tag(tag_name, user:)
644 645
        wrapped_gitaly_errors do
          gitaly_operation_client.rm_tag(tag_name, user)
646
        end
647 648 649 650 651 652
      end

      def find_tag(name)
        tags.find { |tag| tag.name == name }
      end

653
      def merge(user, source_sha, target_branch, message, &block)
654 655
        wrapped_gitaly_errors do
          gitaly_operation_client.user_merge_branch(user, source_sha, target_branch, message, &block)
656 657 658
        end
      end

659
      def ff_merge(user, source_sha, target_branch)
660 661
        wrapped_gitaly_errors do
          gitaly_operation_client.user_ff_branch(user, source_sha, target_branch)
662 663 664
        end
      end

665
      def revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
666 667 668 669 670 671 672 673
        args = {
          user: user,
          commit: commit,
          branch_name: branch_name,
          message: message,
          start_branch_name: start_branch_name,
          start_repository: start_repository
        }
674

675 676
        wrapped_gitaly_errors do
          gitaly_operation_client.user_revert(args)
677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693
        end
      end

      def check_revert_content(target_commit, source_sha)
        args = [target_commit.sha, source_sha]
        args << { mainline: 1 } if target_commit.merge_commit?

        revert_index = rugged.revert_commit(*args)
        return false if revert_index.conflicts?

        tree_id = revert_index.write_tree(rugged)
        return false unless diff_exists?(source_sha, tree_id)

        tree_id
      end

      def cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
694 695 696 697 698 699 700 701
        args = {
          user: user,
          commit: commit,
          branch_name: branch_name,
          message: message,
          start_branch_name: start_branch_name,
          start_repository: start_repository
        }
702

703 704
        wrapped_gitaly_errors do
          gitaly_operation_client.user_cherry_pick(args)
705 706 707 708 709 710 711 712 713 714 715
        end
      end

      def diff_exists?(sha1, sha2)
        rugged.diff(sha1, sha2).size > 0
      end

      def user_to_committer(user)
        Gitlab::Git.committer_hash(email: user.email, name: user.name)
      end

716 717 718 719 720 721
      def create_commit(params = {})
        params[:message].delete!("\r")

        Rugged::Commit.create(rugged, params)
      end

Robert Speicher's avatar
Robert Speicher committed
722 723
      # Delete the specified branch from the repository
      def delete_branch(branch_name)
724
        gitaly_migrate(:delete_branch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
725 726 727 728 729 730 731 732
          if is_enabled
            gitaly_ref_client.delete_branch(branch_name)
          else
            rugged.branches.delete(branch_name)
          end
        end
      rescue Rugged::ReferenceError, CommandError => e
        raise DeleteBranchError, e
Robert Speicher's avatar
Robert Speicher committed
733
      end
Lin Jen-Shin's avatar
Lin Jen-Shin committed
734

735
      def delete_refs(*ref_names)
736 737
        gitaly_migrate(:delete_refs,
                      status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
738 739 740 741 742
          if is_enabled
            gitaly_delete_refs(*ref_names)
          else
            git_delete_refs(*ref_names)
          end
743
        end
Lin Jen-Shin's avatar
Lin Jen-Shin committed
744
      end
Robert Speicher's avatar
Robert Speicher committed
745 746 747 748 749 750 751

      # Create a new branch named **ref+ based on **stat_point+, HEAD by default
      #
      # Examples:
      #   create_branch("feature")
      #   create_branch("other-feature", "master")
      def create_branch(ref, start_point = "HEAD")
752
        gitaly_migrate(:create_branch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
753 754 755 756 757 758
          if is_enabled
            gitaly_ref_client.create_branch(ref, start_point)
          else
            rugged_create_branch(ref, start_point)
          end
        end
Robert Speicher's avatar
Robert Speicher committed
759 760
      end

761 762
      # If `mirror_refmap` is present the remote is set as mirror with that mapping
      def add_remote(remote_name, url, mirror_refmap: nil)
763
        gitaly_migrate(:remote_add_remote) do |is_enabled|
764 765 766 767 768 769
          if is_enabled
            gitaly_remote_client.add_remote(remote_name, url, mirror_refmap)
          else
            rugged_add_remote(remote_name, url, mirror_refmap)
          end
        end
Robert Speicher's avatar
Robert Speicher committed
770 771
      end

772
      def remove_remote(remote_name)
773
        gitaly_migrate(:remote_remove_remote) do |is_enabled|
774 775 776 777 778
          if is_enabled
            gitaly_remote_client.remove_remote(remote_name)
          else
            rugged_remove_remote(remote_name)
          end
779
        end
780 781
      end

Robert Speicher's avatar
Robert Speicher committed
782 783 784 785
      # Update the specified remote using the values in the +options+ hash
      #
      # Example
      # repo.update_remote("origin", url: "path/to/repo")
Jacob Vosmaer's avatar
Jacob Vosmaer committed
786
      def remote_update(remote_name, url:)
Robert Speicher's avatar
Robert Speicher committed
787
        # TODO: Implement other remote options
Jacob Vosmaer's avatar
Jacob Vosmaer committed
788
        rugged.remotes.set_url(remote_name, url)
Jacob Vosmaer's avatar
Jacob Vosmaer committed
789
        nil
Robert Speicher's avatar
Robert Speicher committed
790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811
      end

      AUTOCRLF_VALUES = {
        "true" => true,
        "false" => false,
        "input" => :input
      }.freeze

      def autocrlf
        AUTOCRLF_VALUES[rugged.config['core.autocrlf']]
      end

      def autocrlf=(value)
        rugged.config['core.autocrlf'] = AUTOCRLF_VALUES.invert[value]
      end

      # Returns result like "git ls-files" , recursive and full file path
      #
      # Ex.
      #   repo.ls_files('master')
      #
      def ls_files(ref)
812
        gitaly_commit_client.ls_files(ref)
Robert Speicher's avatar
Robert Speicher committed
813 814 815
      end

      def copy_gitattributes(ref)
Jacob Vosmaer's avatar
Jacob Vosmaer committed
816 817
        wrapped_gitaly_errors do
          gitaly_repository_client.apply_gitattributes(ref)
Robert Speicher's avatar
Robert Speicher committed
818 819 820
        end
      end

821 822 823
      def info_attributes
        return @info_attributes if @info_attributes

824
        content = gitaly_repository_client.info_attributes
825 826 827
        @info_attributes = AttributesParser.new(content)
      end

Robert Speicher's avatar
Robert Speicher committed
828 829 830 831
      # Returns the Git attributes for the given file path.
      #
      # See `Gitlab::Git::Attributes` for more information.
      def attributes(path)
832
        info_attributes.attributes(path)
Robert Speicher's avatar
Robert Speicher committed
833 834
      end

Sean McGivern's avatar
Sean McGivern committed
835 836 837 838
      def gitattribute(path, name)
        attributes(path)[name]
      end

839 840 841 842 843
      # Check .gitattributes for a given ref
      #
      # This only checks the root .gitattributes file,
      # it does not traverse subfolders to find additional .gitattributes files
      #
844 845 846
      # This method is around 30 times slower than `attributes`, which uses
      # `$GIT_DIR/info/attributes`. Consider caching AttributesAtRefParser
      # and reusing that for multiple calls instead of this method.
847 848 849 850 851
      def attributes_at(ref, file_path)
        parser = AttributesAtRefParser.new(self, ref)
        parser.attributes(file_path)
      end

852
      def languages(ref = nil)
Zeger-Jan van de Weg's avatar
Zeger-Jan van de Weg committed
853 854
        wrapped_gitaly_errors do
          gitaly_commit_client.languages(ref)
855 856 857
        end
      end

858
      def license_short_name
859 860
        wrapped_gitaly_errors do
          gitaly_repository_client.license_short_name
861 862 863
        end
      end

864
      def with_repo_branch_commit(start_repository, start_branch_name)
865
        Gitlab::Git.check_namespace!(start_repository)
866
        start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository)
867

868
        return yield nil if start_repository.empty?
869

870
        if start_repository.same_repository?(self)
871 872
          yield commit(start_branch_name)
        else
873
          start_commit_id = start_repository.commit_id(start_branch_name)
874

875
          return yield nil unless start_commit_id
876

877
          if branch_commit = commit(start_commit_id)
878 879 880
            yield branch_commit
          else
            with_repo_tmp_commit(
881
              start_repository, start_branch_name, start_commit_id) do |tmp_commit|
882 883 884 885 886 887 888
              yield tmp_commit
            end
          end
        end
      end

      def with_repo_tmp_commit(start_repository, start_branch_name, sha)
889 890 891 892 893 894
        source_ref = start_branch_name

        unless Gitlab::Git.branch_ref?(source_ref)
          source_ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{source_ref}"
        end

895
        tmp_ref = fetch_ref(
896
          start_repository,
897
          source_ref: source_ref,
898
          target_ref: "refs/tmp/#{SecureRandom.hex}"
899 900 901 902 903 904 905
        )

        yield commit(sha)
      ensure
        delete_refs(tmp_ref) if tmp_ref
      end

906
      def fetch_source_branch!(source_repository, source_branch, local_ref)
907 908 909
        Gitlab::GitalyClient.migrate(:fetch_source_branch) do |is_enabled|
          if is_enabled
            gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref)
910
          else
911
            rugged_fetch_source_branch(source_repository, source_branch, local_ref)
912 913 914 915 916
          end
        end
      end

      def compare_source_branch(target_branch_name, source_repository, source_branch_name, straight:)
917 918 919 920 921 922 923 924 925 926 927 928
        tmp_ref = "refs/tmp/#{SecureRandom.hex}"

        return unless fetch_source_branch!(source_repository, source_branch_name, tmp_ref)

        Gitlab::Git::Compare.new(
          self,
          target_branch_name,
          tmp_ref,
          straight: straight
        )
      ensure
        delete_refs(tmp_ref)
929 930
      end

931
      def write_ref(ref_path, ref, old_ref: nil, shell: true)
932 933 934 935 936 937 938 939
        ref_path = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{ref_path}" unless ref_path.start_with?("refs/") || ref_path == "HEAD"

        gitaly_migrate(:write_ref) do |is_enabled|
          if is_enabled
            gitaly_repository_client.write_ref(ref_path, ref, old_ref, shell)
          else
            local_write_ref(ref_path, ref, old_ref: old_ref, shell: shell)
          end
940 941 942
        end
      end

943
      def fetch_ref(source_repository, source_ref:, target_ref:)
944 945 946
        Gitlab::Git.check_namespace!(source_repository)
        source_repository = RemoteRepository.new(source_repository) unless source_repository.is_a?(RemoteRepository)

947 948 949 950
        message, status = GitalyClient.migrate(:fetch_ref) do |is_enabled|
          if is_enabled
            gitaly_fetch_ref(source_repository, source_ref: source_ref, target_ref: target_ref)
          else
951 952
            # When removing this code, also remove source_repository#path
            # to remove deprecated method calls
953 954 955
            local_fetch_ref(source_repository.path, source_ref: source_ref, target_ref: target_ref)
          end
        end
956 957 958 959 960 961 962 963 964 965 966 967

        # Make sure ref was created, and raise Rugged::ReferenceError when not
        raise Rugged::ReferenceError, message if status != 0

        target_ref
      end

      # Refactoring aid; allows us to copy code from app/models/repository.rb
      def commit(ref = 'HEAD')
        Gitlab::Git::Commit.find(self, ref)
      end

968 969
      def empty?
        !has_visible_content?
970 971
      end

972
      def fetch_repository_as_mirror(repository)
973
        gitaly_migrate(:remote_fetch_internal_remote) do |is_enabled|
974
          if is_enabled
975
            gitaly_remote_client.fetch_internal_remote(repository)
976
          else
977
            rugged_fetch_repository_as_mirror(repository)
978 979
          end
        end
980 981
      end

982 983 984 985
      def blob_at(sha, path)
        Gitlab::Git::Blob.find(self, sha, path) unless Gitlab::Git.blank_ref?(sha)
      end

986
      # Items should be of format [[commit_id, path], [commit_id1, path1]]
987
      def batch_blobs(items, blob_size_limit: Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE)
988 989 990
        Gitlab::Git::Blob.batch(self, items, blob_size_limit: blob_size_limit)
      end

991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004
      def commit_index(user, branch_name, index, options)
        committer = user_to_committer(user)

        OperationService.new(user, self).with_branch(branch_name) do
          commit_params = options.merge(
            tree: index.write_tree(rugged),
            author: committer,
            committer: committer
          )

          create_commit(commit_params)
        end
      end

1005
      def fsck
1006
        msg, status = gitaly_repository_client.fsck
1007

1008
        raise GitError.new("Could not fsck repository: #{msg}") unless status.zero?
1009 1010
      end

1011
      def create_from_bundle(bundle_path)
1012
        gitaly_repository_client.create_from_bundle(bundle_path)
1013 1014
      end

1015 1016 1017 1018
      def create_from_snapshot(url, auth)
        gitaly_repository_client.create_from_snapshot(url, auth)
      end

1019
      def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:)
1020 1021 1022 1023 1024 1025
        wrapped_gitaly_errors do
          gitaly_operation_client.user_rebase(user, rebase_id,
                                            branch: branch,
                                            branch_sha: branch_sha,
                                            remote_repository: remote_repository,
                                            remote_branch: remote_branch)
1026 1027 1028 1029
        end
      end

      def rebase_in_progress?(rebase_id)
1030 1031
        wrapped_gitaly_errors do
          gitaly_repository_client.rebase_in_progress?(rebase_id)
1032
        end
1033 1034 1035
      end

      def squash(user, squash_id, branch:, start_sha:, end_sha:, author:, message:)
1036 1037
        wrapped_gitaly_errors do
          gitaly_operation_client.user_squash(user, squash_id, branch,
1038
              start_sha, end_sha, author, message)
1039 1040 1041 1042
        end
      end

      def squash_in_progress?(squash_id)
1043 1044
        wrapped_gitaly_errors do
          gitaly_repository_client.squash_in_progress?(squash_id)
1045
        end
1046 1047
      end

1048 1049 1050 1051 1052 1053
      def push_remote_branches(remote_name, branch_names, forced: true)
        success = @gitlab_projects.push_branches(remote_name, GITLAB_PROJECTS_TIMEOUT, forced, branch_names)

        success || gitlab_projects_error
      end

1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065
      def delete_remote_branches(remote_name, branch_names)
        success = @gitlab_projects.delete_remote_branches(remote_name, branch_names)

        success || gitlab_projects_error
      end

      def delete_remote_branches(remote_name, branch_names)
        success = @gitlab_projects.delete_remote_branches(remote_name, branch_names)

        success || gitlab_projects_error
      end

1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077
      def bundle_to_disk(save_path)
        gitaly_migrate(:bundle_to_disk) do |is_enabled|
          if is_enabled
            gitaly_repository_client.create_bundle(save_path)
          else
            run_git!(%W(bundle create #{save_path} --all))
          end
        end

        true
      end

1078 1079 1080 1081 1082
      def multi_action(
        user, branch_name:, message:, actions:,
        author_email: nil, author_name: nil,
        start_branch_name: nil, start_repository: self)

1083 1084
        wrapped_gitaly_errors do
          gitaly_operation_client.user_commit_files(user, branch_name,
1085 1086
              message, actions, author_email, author_name,
              start_branch_name, start_repository)
1087 1088 1089
        end
      end

1090
      def write_config(full_path:)
1091 1092
        return unless full_path.present?

1093
        # This guard avoids Gitaly log/error spam
1094
        raise NoRepository, 'repository does not exist' unless exists?
1095

1096 1097 1098 1099 1100 1101 1102 1103 1104 1105
        set_config('gitlab.fullpath' => full_path)
      end

      def set_config(entries)
        wrapped_gitaly_errors do
          gitaly_repository_client.set_config(entries)
        end
      end

      def delete_config(*keys)
1106
        wrapped_gitaly_errors do
1107
          gitaly_repository_client.delete_config(keys)
1108
        end
1109 1110
      end

1111
      def gitaly_repository
1112
        Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository)
1113 1114
      end

1115 1116 1117 1118 1119 1120
      def gitaly_ref_client
        @gitaly_ref_client ||= Gitlab::GitalyClient::RefService.new(self)
      end

      def gitaly_commit_client
        @gitaly_commit_client ||= Gitlab::GitalyClient::CommitService.new(self)
1121 1122 1123 1124
      end

      def gitaly_repository_client
        @gitaly_repository_client ||= Gitlab::GitalyClient::RepositoryService.new(self)
1125 1126
      end

1127 1128 1129 1130
      def gitaly_operation_client
        @gitaly_operation_client ||= Gitlab::GitalyClient::OperationService.new(self)
      end

1131 1132 1133 1134
      def gitaly_remote_client
        @gitaly_remote_client ||= Gitlab::GitalyClient::RemoteService.new(self)
      end

1135 1136 1137 1138
      def gitaly_blob_client
        @gitaly_blob_client ||= Gitlab::GitalyClient::BlobService.new(self)
      end

1139 1140
      def gitaly_conflicts_client(our_commit_oid, their_commit_oid)
        Gitlab::GitalyClient::ConflictsService.new(self, our_commit_oid, their_commit_oid)
1141 1142
      end

1143 1144
      def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block)
        Gitlab::GitalyClient.migrate(method, status: status, &block)
1145 1146
      rescue GRPC::NotFound => e
        raise NoRepository.new(e)
1147 1148
      rescue GRPC::InvalidArgument => e
        raise ArgumentError.new(e)
1149 1150
      rescue GRPC::BadStatus => e
        raise CommandError.new(e)
1151 1152 1153 1154 1155 1156 1157 1158 1159 1160
      end

      def wrapped_gitaly_errors(&block)
        yield block
      rescue GRPC::NotFound => e
        raise NoRepository.new(e)
      rescue GRPC::InvalidArgument => e
        raise ArgumentError.new(e)
      rescue GRPC::BadStatus => e
        raise CommandError.new(e)
1161 1162
      end

1163 1164 1165 1166 1167
      def clean_stale_repository_files
        gitaly_migrate(:repository_cleanup, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
          gitaly_repository_client.cleanup if is_enabled && exists?
        end
      rescue Gitlab::Git::CommandError => e # Don't fail if we can't cleanup
1168
        Rails.logger.error("Unable to clean repository on storage #{storage} with relative path #{relative_path}: #{e.message}")
1169 1170 1171 1172 1173 1174
        Gitlab::Metrics.counter(
          :failed_repository_cleanup_total,
          'Number of failed repository cleanup events'
        ).increment
      end

1175
      def branch_names_contains_sha(sha)
1176
        gitaly_ref_client.branch_names_contains_sha(sha)
1177
      end
1178

1179
      def tag_names_contains_sha(sha)
1180
        gitaly_ref_client.tag_names_contains_sha(sha)
1181 1182 1183 1184 1185
      end

      def search_files_by_content(query, ref)
        return [] if empty? || query.blank?

1186 1187 1188
        safe_query = Regexp.escape(query)
        ref ||= root_ref

1189
        gitaly_repository_client.search_files_by_content(ref, safe_query)
1190 1191
      end

1192
      def can_be_merged?(source_sha, target_branch)
1193 1194 1195 1196 1197
        if target_sha = find_branch(target_branch, true)&.target
          !gitaly_conflicts_client(source_sha, target_sha).conflicts?
        else
          false
        end
1198 1199
      end

1200
      def search_files_by_name(query, ref)
1201
        safe_query = Regexp.escape(query.sub(%r{^/*}, ""))
1202
        ref ||= root_ref
1203 1204 1205

        return [] if empty? || safe_query.blank?

1206
        gitaly_repository_client.search_files_by_name(ref, safe_query)
1207 1208 1209
      end

      def find_commits_by_message(query, ref, path, limit, offset)
1210 1211 1212 1213
        wrapped_gitaly_errors do
          gitaly_commit_client
            .commits_by_message(query, revision: ref, path: path, limit: limit, offset: offset)
            .map { |c| commit(c) }
1214 1215 1216
        end
      end

1217 1218 1219 1220 1221
      def shell_blame(sha, path)
        output, _status = run_git(%W(blame -p #{sha} -- #{path}))
        output
      end

1222
      def last_commit_for_path(sha, path)
1223 1224
        wrapped_gitaly_errors do
          gitaly_commit_client.last_commit_for_path(sha, path)
1225 1226 1227
        end
      end

1228
      def rev_list(including: [], excluding: [], options: [], objects: false, &block)
1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240
        args = ['rev-list']

        args.push(*rev_list_param(including))

        exclude_param = *rev_list_param(excluding)
        if exclude_param.any?
          args.push('--not')
          args.push(*exclude_param)
        end

        args.push('--objects') if objects

1241 1242 1243 1244
        if options.any?
          args.push(*options)
        end

1245 1246 1247
        run_git!(args, lazy_block: block)
      end

1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274
      def with_worktree(worktree_path, branch, sparse_checkout_files: nil, env:)
        base_args = %w(worktree add --detach)

        # Note that we _don't_ want to test for `.present?` here: If the caller
        # passes an non nil empty value it means it still wants sparse checkout
        # but just isn't interested in any file, perhaps because it wants to
        # checkout files in by a changeset but that changeset only adds files.
        if sparse_checkout_files
          # Create worktree without checking out
          run_git!(base_args + ['--no-checkout', worktree_path], env: env)
          worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path).chomp

          configure_sparse_checkout(worktree_git_path, sparse_checkout_files)

          # After sparse checkout configuration, checkout `branch` in worktree
          run_git!(%W(checkout --detach #{branch}), chdir: worktree_path, env: env)
        else
          # Create worktree and checkout `branch` in it
          run_git!(base_args + [worktree_path, branch], env: env)
        end

        yield
      ensure
        FileUtils.rm_rf(worktree_path) if File.exist?(worktree_path)
        FileUtils.rm_rf(worktree_git_path) if worktree_git_path && File.exist?(worktree_git_path)
      end

1275
      def checksum
1276 1277 1278 1279 1280 1281
        # The exists? RPC is much cheaper, so we perform this request first
        raise NoRepository, "Repository does not exists" unless exists?

        gitaly_repository_client.calculate_checksum
      rescue GRPC::NotFound
        raise NoRepository # Guard against data races.
1282 1283
      end

Robert Speicher's avatar
Robert Speicher committed
1284 1285
      private

1286
      def uncached_has_local_branches?
1287 1288
        wrapped_gitaly_errors do
          gitaly_repository_client.has_local_branches?
1289 1290 1291
        end
      end

1292 1293 1294 1295 1296 1297 1298 1299
      def local_write_ref(ref_path, ref, old_ref: nil, shell: true)
        if shell
          shell_write_ref(ref_path, ref, old_ref)
        else
          rugged_write_ref(ref_path, ref)
        end
      end

1300 1301 1302 1303
      def rugged_write_config(full_path:)
        rugged.config['gitlab.fullpath'] = full_path
      end

1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315
      def shell_write_ref(ref_path, ref, old_ref)
        raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ')
        raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00")
        raise ArgumentError, "invalid old_ref #{old_ref.inspect}" if !old_ref.nil? && old_ref.include?("\x00")

        input = "update #{ref_path}\x00#{ref}\x00#{old_ref}\x00"
        run_git!(%w[update-ref --stdin -z]) { |stdin| stdin.write(input) }
      end

      def rugged_write_ref(ref_path, ref)
        rugged.references.create(ref_path, ref, force: true)
      rescue Rugged::ReferenceError => ex
1316
        Rails.logger.error "Unable to create #{ref_path} reference for repository #{path}: #{ex}"
1317
      rescue Rugged::OSError => ex
1318 1319 1320
        raise unless ex.message =~ /Failed to create locked file/ && ex.message =~ /File exists/

        Rails.logger.error "Unable to create #{ref_path} reference for repository #{path}: #{ex}"
1321 1322
      end

1323
      def run_git(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block)
1324 1325
        cmd = [Gitlab.config.git.bin_path, *args]
        cmd.unshift("nice") if nice
1326 1327 1328 1329 1330 1331

        object_directories = alternate_object_directories
        if object_directories.any?
          env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = object_directories.join(File::PATH_SEPARATOR)
        end

1332
        circuit_breaker.perform do
1333
          popen(cmd, chdir, env, lazy_block: lazy_block, &block)
1334 1335 1336
        end
      end

1337 1338
      def run_git!(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block)
        output, status = run_git(args, chdir: chdir, env: env, nice: nice, lazy_block: lazy_block, &block)
1339 1340 1341 1342 1343 1344

        raise GitError, output unless status.zero?

        output
      end

1345 1346 1347 1348 1349 1350
      def run_git_with_timeout(args, timeout, env: {})
        circuit_breaker.perform do
          popen_with_timeout([Gitlab.config.git.bin_path, *args], timeout, path, env)
        end
      end

1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363
      # Adding a worktree means checking out the repository. For large repos,
      # this can be very expensive, so set up sparse checkout for the worktree
      # to only check out the files we're interested in.
      def configure_sparse_checkout(worktree_git_path, files)
        run_git!(%w(config core.sparseCheckout true))

        return if files.empty?

        worktree_info_path = File.join(worktree_git_path, 'info')
        FileUtils.mkdir_p(worktree_info_path)
        File.write(File.join(worktree_info_path, 'sparse-checkout'), files)
      end

1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374
      def rugged_fetch_source_branch(source_repository, source_branch, local_ref)
        with_repo_branch_commit(source_repository, source_branch) do |commit|
          if commit
            write_ref(local_ref, commit.sha)
            true
          else
            false
          end
        end
      end

1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392
      def worktree_path(prefix, id)
        id = id.to_s
        raise ArgumentError, "worktree id can't be empty" unless id.present?
        raise ArgumentError, "worktree id can't contain slashes " if id.include?("/")

        File.join(path, 'gitlab-worktree', "#{prefix}-#{id}")
      end

      def git_env_for_user(user)
        {
          'GIT_COMMITTER_NAME' => user.name,
          'GIT_COMMITTER_EMAIL' => user.email,
          'GL_ID' => Gitlab::GlId.gl_id(user),
          'GL_PROTOCOL' => Gitlab::Git::Hook::GL_PROTOCOL,
          'GL_REPOSITORY' => gl_repository
        }
      end

1393
      def git_merged_branch_names(branch_names, root_sha)
1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404
        git_arguments =
          %W[branch --merged #{root_sha}
             --format=%(refname:short)\ %(objectname)] + branch_names

        lines = run_git(git_arguments).first.lines

        lines.each_with_object([]) do |line, branches|
          name, sha = line.strip.split(' ', 2)

          branches << name if sha != root_sha
        end
1405 1406
      end

1407 1408 1409 1410 1411 1412 1413 1414
      def gitaly_merged_branch_names(branch_names, root_sha)
        qualified_branch_names = branch_names.map { |b| "refs/heads/#{b}" }

        gitaly_ref_client.merged_branches(qualified_branch_names)
          .reject { |b| b.target == root_sha }
          .map(&:name)
      end

1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434
      def process_count_commits_options(options)
        if options[:from] || options[:to]
          ref =
            if options[:left_right] # Compare with merge-base for left-right
              "#{options[:from]}...#{options[:to]}"
            else
              "#{options[:from]}..#{options[:to]}"
            end

          options.merge(ref: ref)

        elsif options[:ref] && options[:left_right]
          from, to = options[:ref].match(/\A([^\.]*)\.{2,3}([^\.]*)\z/)[1..2]

          options.merge(from: from, to: to)
        else
          options
        end
      end

1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451
      # We are trying to deprecate this method because it does a lot of work
      # but it seems to be used only to look up submodule URL's.
      # https://gitlab.com/gitlab-org/gitaly/issues/329
      def submodules(ref)
        commit = rev_parse_target(ref)
        return {} unless commit

        begin
          content = blob_content(commit, ".gitmodules")
        rescue InvalidBlobName
          return {}
        end

        parser = GitmodulesParser.new(content)
        fill_submodule_ids(commit, parser.parse)
      end

1452 1453 1454 1455 1456 1457
      def gitaly_submodule_url_for(ref, path)
        # We don't care about the contents so 1 byte is enough. Can't request 0 bytes, 0 means unlimited.
        commit_object = gitaly_commit_client.tree_entry(ref, path, 1)

        return unless commit_object && commit_object.type == :COMMIT

Clement Ho's avatar
Clement Ho committed
1458
        gitmodules = gitaly_commit_client.tree_entry(ref, '.gitmodules', Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE)
1459 1460
        return unless gitmodules

1461 1462 1463 1464 1465
        found_module = GitmodulesParser.new(gitmodules.data).parse[path]

        found_module && found_module['url']
      end

1466
      def alternate_object_directories
1467
        relative_object_directories.map { |d| File.join(path, d) }
1468 1469
      end

1470
      def relative_object_directories
1471
        Gitlab::Git::HookEnv.all(gl_repository).values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact
1472 1473
      end

Robert Speicher's avatar
Robert Speicher committed
1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492
      # Get the content of a blob for a given commit.  If the blob is a commit
      # (for submodules) then return the blob's OID.
      def blob_content(commit, blob_name)
        blob_entry = tree_entry(commit, blob_name)

        unless blob_entry
          raise InvalidBlobName.new("Invalid blob name: #{blob_name}")
        end

        case blob_entry[:type]
        when :commit
          blob_entry[:oid]
        when :tree
          raise InvalidBlobName.new("#{blob_name} is a tree, not a blob")
        when :blob
          rugged.lookup(blob_entry[:oid]).content
        end
      end

1493 1494 1495 1496 1497 1498 1499 1500 1501
      # Fill in the 'id' field of a submodule hash from its values
      # as-of +commit+. Return a Hash consisting only of entries
      # from the submodule hash for which the 'id' field is filled.
      def fill_submodule_ids(commit, submodule_data)
        submodule_data.each do |path, data|
          id = begin
            blob_content(commit, path)
          rescue InvalidBlobName
            nil
Robert Speicher's avatar
Robert Speicher committed
1502
          end
1503
          data['id'] = id
Robert Speicher's avatar
Robert Speicher committed
1504
        end
1505
        submodule_data.select { |path, data| data['id'] }
Robert Speicher's avatar
Robert Speicher committed
1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520
      end

      # Find the entry for +path+ in the tree for +commit+
      def tree_entry(commit, path)
        pathname = Pathname.new(path)
        first = true
        tmp_entry = nil

        pathname.each_filename do |dir|
          if first
            tmp_entry = commit.tree[dir]
            first = false
          elsif tmp_entry.nil?
            return nil
          else
1521 1522 1523 1524 1525 1526
            begin
              tmp_entry = rugged.lookup(tmp_entry[:oid])
            rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError
              return nil
            end

Robert Speicher's avatar
Robert Speicher committed
1527
            return nil unless tmp_entry.type == :tree
1528

Robert Speicher's avatar
Robert Speicher committed
1529 1530 1531 1532 1533 1534 1535
            tmp_entry = tmp_entry[dir]
          end
        end

        tmp_entry
      end

1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546
      # Return the Rugged patches for the diff between +from+ and +to+.
      def diff_patches(from, to, options = {}, *paths)
        options ||= {}
        break_rewrites = options[:break_rewrites]
        actual_options = Gitlab::Git::Diff.filter_diff_options(options.merge(paths: paths))

        diff = rugged.diff(from, to, actual_options)
        diff.find_similar!(break_rewrites: break_rewrites)
        diff.each_patch
      end

1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563
      def sort_branches(branches, sort_by)
        case sort_by
        when 'name'
          branches.sort_by(&:name)
        when 'updated_desc'
          branches.sort do |a, b|
            b.dereferenced_target.committed_date <=> a.dereferenced_target.committed_date
          end
        when 'updated_asc'
          branches.sort do |a, b|
            a.dereferenced_target.committed_date <=> b.dereferenced_target.committed_date
          end
        else
          branches
        end
      end

1564 1565 1566
      # Returns true if the given ref name exists
      #
      # Ref names must start with `refs/`.
1567 1568
      def rugged_ref_exists?(ref_name)
        raise ArgumentError, 'invalid refname' unless ref_name.start_with?('refs/')
1569

1570 1571 1572 1573 1574 1575 1576 1577
        rugged.references.exist?(ref_name)
      rescue Rugged::ReferenceError
        false
      end

      # Returns true if the given ref name exists
      #
      # Ref names must start with `refs/`.
1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602
      def gitaly_ref_exists?(ref_name)
        gitaly_ref_client.ref_exists?(ref_name)
      end

      # Returns true if the given tag exists
      #
      # name - The name of the tag as a String.
      def rugged_tag_exists?(name)
        !!rugged.tags[name]
      end

      # Returns true if the given branch exists
      #
      # name - The name of the branch as a String.
      def rugged_branch_exists?(name)
        rugged.branches.exists?(name)

      # If the branch name is invalid (e.g. ".foo") Rugged will raise an error.
      # Whatever code calls this method shouldn't have to deal with that so
      # instead we just return `false` (which is true since a branch doesn't
      # exist when it has an invalid name).
      rescue Rugged::ReferenceError
        false
      end

1603 1604 1605 1606 1607
      def rugged_create_branch(ref, start_point)
        rugged_ref = rugged.branches.create(ref, start_point)
        target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
        Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
      rescue Rugged::ReferenceError => e
1608
        raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ %r{'refs/heads/#{ref}'}
1609

1610 1611 1612
        raise InvalidRef.new("Invalid reference #{start_point}")
      end

1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645
      def gitaly_copy_gitattributes(revision)
        gitaly_repository_client.apply_gitattributes(revision)
      end

      def rugged_copy_gitattributes(ref)
        begin
          commit = lookup(ref)
        rescue Rugged::ReferenceError
          raise InvalidRef.new("Ref #{ref} is invalid")
        end

        # Create the paths
        info_dir_path = File.join(path, 'info')
        info_attributes_path = File.join(info_dir_path, 'attributes')

        begin
          # Retrieve the contents of the blob
          gitattributes_content = blob_content(commit, '.gitattributes')
        rescue InvalidBlobName
          # No .gitattributes found. Should now remove any info/attributes and return
          File.delete(info_attributes_path) if File.exist?(info_attributes_path)
          return
        end

        # Create the info directory if needed
        Dir.mkdir(info_dir_path) unless File.directory?(info_dir_path)

        # Write the contents of the .gitattributes file to info/attributes
        # Use binary mode to prevent Rails from converting ASCII-8BIT to UTF-8
        File.open(info_attributes_path, "wb") do |file|
          file.write(gitattributes_content)
        end
      end
1646

1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685
      def rugged_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
        OperationService.new(user, self).with_branch(
          branch_name,
          start_branch_name: start_branch_name,
          start_repository: start_repository
        ) do |start_commit|

          Gitlab::Git.check_namespace!(commit, start_repository)

          cherry_pick_tree_id = check_cherry_pick_content(commit, start_commit.sha)
          raise CreateTreeError unless cherry_pick_tree_id

          committer = user_to_committer(user)

          create_commit(message: message,
                        author: {
                            email: commit.author_email,
                            name: commit.author_name,
                            time: commit.authored_date
                        },
                        committer: committer,
                        tree: cherry_pick_tree_id,
                        parents: [start_commit.sha])
        end
      end

      def check_cherry_pick_content(target_commit, source_sha)
        args = [target_commit.sha, source_sha]
        args << 1 if target_commit.merge_commit?

        cherry_pick_index = rugged.cherrypick_commit(*args)
        return false if cherry_pick_index.conflicts?

        tree_id = cherry_pick_index.write_tree(rugged)
        return false unless diff_exists?(source_sha, tree_id)

        tree_id
      end

1686 1687 1688 1689 1690 1691
      def local_fetch_ref(source_path, source_ref:, target_ref:)
        args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref})
        run_git(args)
      end

      def gitaly_fetch_ref(source_repository, source_ref:, target_ref:)
1692
        args = %W(fetch --no-tags -f #{GITALY_INTERNAL_URL} #{source_ref}:#{target_ref})
1693

1694
        run_git(args, env: source_repository.fetch_env)
1695
      end
1696

1697 1698 1699 1700 1701 1702 1703 1704
      def rugged_add_remote(remote_name, url, mirror_refmap)
        rugged.remotes.create(remote_name, url)

        set_remote_as_mirror(remote_name, refmap: mirror_refmap) if mirror_refmap
      rescue Rugged::ConfigError
        remote_update(remote_name, url: url)
      end

1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719
      def git_delete_refs(*ref_names)
        instructions = ref_names.map do |ref|
          "delete #{ref}\x00\x00"
        end

        message, status = run_git(%w[update-ref --stdin -z]) do |stdin|
          stdin.write(instructions.join)
        end

        unless status.zero?
          raise GitError.new("Could not delete refs #{ref_names}: #{message}")
        end
      end

      def gitaly_delete_refs(*ref_names)
1720
        gitaly_ref_client.delete_refs(refs: ref_names) if ref_names.any?
1721 1722
      end

1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737
      def rugged_remove_remote(remote_name)
        # When a remote is deleted all its remote refs are deleted too, but in
        # the case of mirrors we map its refs (that would usualy go under
        # [remote_name]/) to the top level namespace. We clean the mapping so
        # those don't get deleted.
        if rugged.config["remote.#{remote_name}.mirror"]
          rugged.config.delete("remote.#{remote_name}.fetch")
        end

        rugged.remotes.delete(remote_name)
        true
      rescue Rugged::ConfigError
        false
      end

1738 1739 1740 1741 1742 1743 1744 1745 1746 1747
      def rugged_fetch_repository_as_mirror(repository)
        remote_name = "tmp-#{SecureRandom.hex}"
        repository = RemoteRepository.new(repository) unless repository.is_a?(RemoteRepository)

        add_remote(remote_name, GITALY_INTERNAL_URL, mirror_refmap: :all_refs)
        fetch_remote(remote_name, env: repository.fetch_env)
      ensure
        remove_remote(remote_name)
      end

1748 1749 1750
      def fetch_remote(remote_name = 'origin', env: nil)
        run_git(['fetch', remote_name], env: env).last.zero?
      end
1751 1752 1753 1754

      def gitlab_projects_error
        raise CommandError, @gitlab_projects.output
      end
1755

1756 1757 1758 1759 1760 1761
      def rugged_merge_base(from, to)
        rugged.merge_base(from, to)
      rescue Rugged::ReferenceError
        nil
      end

1762 1763 1764
      def rev_list_param(spec)
        spec == :all ? ['--all'] : spec
      end
1765 1766 1767 1768

      def sha_from_ref(ref)
        rev_parse_target(ref).oid
      end
Robert Speicher's avatar
Robert Speicher committed
1769 1770 1771
    end
  end
end