• Timothy Andrew's avatar
    Improve performance of the cycle analytics page. · ba25e2f1
    Timothy Andrew authored
    1. These changes bring down page load time for 100 issues from more than
       a minute to about 1.5 seconds.
    
    2. This entire commit is composed of these types of performance
       enhancements:
    
         - Cache relevant data in `IssueMetrics` wherever possible.
         - Cache relevant data in `MergeRequestMetrics` wherever possible.
         - Preload metrics
    
    3. Given these improvements, we now only need to make 4 SQL calls:
    
        - Load all issues
        - Load all merge requests
        - Load all metrics for the issues
        - Load all metrics for the merge requests
    
    4. A list of all the data points that are now being pre-calculated:
    
        a. The first time an issue is mentioned in a commit
    
          - In `GitPushService`, find all issues mentioned by the given commit
            using `ReferenceExtractor`. Set the `first_mentioned_in_commit_at`
            flag for each of them.
    
          - There seems to be a (pre-existing) bug here - files (and
            therefore commits) created using the Web CI don't have
            cross-references created, and issues are not closed even when
            the commit title is "Fixes #xx".
    
        b. The first time a merge request is deployed to production
    
          When a `Deployment` is created, find all merge requests that
          were merged in before the deployment, and set the
          `first_deployed_to_production_at` flag for each of them.
    
        c. The start / end time for a merge request pipeline
    
          Hook into the `Pipeline` state machine. When the `status` moves to
          `running`, find the merge requests whose tip commit matches the
          pipeline, and record the `latest_build_started_at` time for each
          of them. When the `status` moves to `success`, record the
          `latest_build_finished_at` time.
    
        d. The merge requests that close an issue
    
          - This was a big cause of the performance problems we were having
            with Cycle Analytics. We need to use `ReferenceExtractor` to make
            this calculation, which is slow when we have to run it on a large
            number of merge requests.
    
          - When a merge request is created, updated, or refreshed, find the
            issues it closes, and create an instance of
            `MergeRequestsClosingIssues`, which acts as a join model between
            merge requests and issues.
    
          - If a `MergeRequestsClosingIssues` instance links a merge request
            and an issue, that issue closes that merge request.
    
    5. The `Queries` module was changed into a class, so we can cache the
       results of `issues` and `merge_requests_closing_issues` across
       various cycle analytics stages.
    
    6. The code added in this commit is untested. Tests will be added in the
       next commit.
    ba25e2f1
refresh_service.rb 5.97 KB
module MergeRequests
  class RefreshService < MergeRequests::BaseService
    def execute(oldrev, newrev, ref)
      return true unless Gitlab::Git.branch_ref?(ref)

      @oldrev, @newrev = oldrev, newrev
      @branch_name = Gitlab::Git.ref_name(ref)

      find_new_commits
      # Be sure to close outstanding MRs before reloading them to avoid generating an
      # empty diff during a manual merge
      close_merge_requests
      reload_merge_requests
      reset_merge_when_build_succeeds
      mark_pending_todos_done
      cache_merge_requests_closing_issues

      # Leave a system note if a branch was deleted/added
      if branch_added? || branch_removed?
        comment_mr_branch_presence_changed
      end

      comment_mr_with_commits
      execute_mr_web_hooks

      true
    end

    private

    # Collect open merge requests that target same branch we push into
    # and close if push to master include last commit from merge request
    # We need this to close(as merged) merge requests that were merged into
    # target branch manually
    def close_merge_requests
      commit_ids = @commits.map(&:id)
      merge_requests = @project.merge_requests.opened.where(target_branch: @branch_name).to_a
      merge_requests = merge_requests.select(&:diff_head_commit)

      merge_requests = merge_requests.select do |merge_request|
        commit_ids.include?(merge_request.diff_head_sha)
      end

      merge_requests.uniq.select(&:source_project).each do |merge_request|
        MergeRequests::PostMergeService.
          new(merge_request.target_project, @current_user).
          execute(merge_request)
      end
    end

    def force_push?
      Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev)
    end

    # Refresh merge request diff if we push to source or target branch of merge request
    # Note: we should update merge requests from forks too
    def reload_merge_requests
      merge_requests = @project.merge_requests.opened.by_branch(@branch_name).to_a
      merge_requests += fork_merge_requests.by_branch(@branch_name).to_a
      merge_requests = filter_merge_requests(merge_requests)

      merge_requests.each do |merge_request|
        if merge_request.source_branch == @branch_name || force_push?
          merge_request.reload_diff
        else
          mr_commit_ids = merge_request.commits.map(&:id)
          push_commit_ids = @commits.map(&:id)
          matches = mr_commit_ids & push_commit_ids
          merge_request.reload_diff if matches.any?
        end

        merge_request.mark_as_unchecked
      end
    end

    def reset_merge_when_build_succeeds
      merge_requests_for_source_branch.each(&:reset_merge_when_build_succeeds)
    end

    def mark_pending_todos_done
      merge_requests_for_source_branch.each do |merge_request|
        todo_service.merge_request_push(merge_request, @current_user)
      end
    end

    def find_new_commits
      if branch_added?
        @commits = []

        merge_request = merge_requests_for_source_branch.first
        return unless merge_request

        begin
          # Since any number of commits could have been made to the restored branch,
          # find the common root to see what has been added.
          common_ref = @project.repository.merge_base(merge_request.diff_head_sha, @newrev)
          # If the a commit no longer exists in this repo, gitlab_git throws
          # a Rugged::OdbError. This is fixed in https://gitlab.com/gitlab-org/gitlab_git/merge_requests/52
          @commits = @project.repository.commits_between(common_ref, @newrev) if common_ref
        rescue
        end
      elsif branch_removed?
        # No commits for a deleted branch.
        @commits = []
      else
        @commits = @project.repository.commits_between(@oldrev, @newrev)
      end
    end

    # Add comment about branches being deleted or added to merge requests
    def comment_mr_branch_presence_changed
      presence = branch_added? ? :add : :delete

      merge_requests_for_source_branch.each do |merge_request|
        SystemNoteService.change_branch_presence(
          merge_request, merge_request.project, @current_user,
            :source, @branch_name, presence)
      end
    end

    # Add comment about pushing new commits to merge requests
    def comment_mr_with_commits
      return unless @commits.present?

      merge_requests_for_source_branch.each do |merge_request|
        mr_commit_ids = Set.new(merge_request.commits.map(&:id))

        new_commits, existing_commits = @commits.partition do |commit|
          mr_commit_ids.include?(commit.id)
        end

        SystemNoteService.add_commits(merge_request, merge_request.project,
                                      @current_user, new_commits,
                                      existing_commits, @oldrev)
      end
    end

    # Call merge request webhook with update branches
    def execute_mr_web_hooks
      merge_requests_for_source_branch.each do |merge_request|
        execute_hooks(merge_request, 'update', @oldrev)
      end
    end

    # If the merge requests closes any issues, save this information in the
    # `MergeRequestsClosingIssues` model (as a performance optimization).
    def cache_merge_requests_closing_issues
      merge_requests_for_source_branch.each do |merge_request|
        merge_request.cache_merge_request_closes_issues!(@current_user)
      end
    end

    def filter_merge_requests(merge_requests)
      merge_requests.uniq.select(&:source_project)
    end

    def merge_requests_for_source_branch
      @source_merge_requests ||= begin
        merge_requests = @project.origin_merge_requests.opened.where(source_branch: @branch_name).to_a
        merge_requests += fork_merge_requests.where(source_branch: @branch_name).to_a
        filter_merge_requests(merge_requests)
      end
    end

    def fork_merge_requests
      @fork_merge_requests ||= @project.fork_merge_requests.opened
    end

    def branch_added?
      Gitlab::Git.blank_ref?(@oldrev)
    end

    def branch_removed?
      Gitlab::Git.blank_ref?(@newrev)
    end
  end
end