ee_compat_check.rb 11.6 KB
Newer Older
1
# rubocop: disable Rails/Output
2 3 4
module Gitlab
  # Checks if a set of migrations requires downtime or not.
  class EeCompatCheck
5
    CE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ce.git'.freeze
6
    EE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze
7 8
    CHECK_DIR = Rails.root.join('ee_compat_check')
    IGNORED_FILES_REGEX = /(VERSION|CHANGELOG\.md:\d+)/.freeze
Rémy Coutable's avatar
Rémy Coutable committed
9 10 11 12 13 14 15 16 17 18
    PLEASE_READ_THIS_BANNER = %Q{
      ============================================================
      ===================== PLEASE READ THIS =====================
      ============================================================
    }.freeze
    THANKS_FOR_READING_BANNER = %Q{
      ============================================================
      ==================== THANKS FOR READING ====================
      ============================================================\n
    }.freeze
19

20 21
    attr_reader :ee_repo_dir, :patches_dir, :ce_repo, :ce_branch, :ee_branch_found
    attr_reader :failed_files
22

23
    def initialize(branch:, ce_repo: CE_REPO)
24
      @ee_repo_dir = CHECK_DIR.join('ee-repo')
25
      @patches_dir = CHECK_DIR.join('patches')
26
      @ce_branch = branch
27
      @ce_repo = ce_repo
28 29 30
    end

    def check
31
      ensure_patches_dir
32 33
      generate_patch(ce_branch, ce_patch_full_path)

34 35 36
      ensure_ee_repo
      Dir.chdir(ee_repo_dir) do
        step("In the #{ee_repo_dir} directory")
37 38 39

        status = catch(:halt_check) do
          ce_branch_compat_check!
40
          delete_ee_branches_locally!
41 42 43 44
          ee_branch_presence_check!
          ee_branch_compat_check!
        end

45
        delete_ee_branches_locally!
46 47 48 49 50 51 52 53 54 55 56 57

        if status.nil?
          true
        else
          false
        end
      end
    end

    private

    def ensure_ee_repo
58 59
      if Dir.exist?(ee_repo_dir)
        step("#{ee_repo_dir} already exists")
60
      else
61 62 63 64
        step(
          "Cloning #{EE_REPO} into #{ee_repo_dir}",
          %W[git clone --branch master --single-branch --depth=200 #{EE_REPO} #{ee_repo_dir}]
        )
65 66 67
      end
    end

68 69 70 71 72 73 74
    def ensure_patches_dir
      FileUtils.mkdir_p(patches_dir)
    end

    def generate_patch(branch, patch_path)
      FileUtils.rm(patch_path, force: true)

75
      find_merge_base_with_master(branch: branch)
76

77 78
      step(
        "Generating the patch against origin/master in #{patch_path}",
79
        %w[git diff --binary origin/master...HEAD]
80
      ) do |output, status|
81 82 83 84 85
        throw(:halt_check, :ko) unless status.zero?

        File.write(patch_path, output)

        throw(:halt_check, :ko) unless File.exist?(patch_path)
86
      end
87 88 89 90 91
    end

    def ce_branch_compat_check!
      if check_patch(ce_patch_full_path).zero?
        puts applies_cleanly_msg(ce_branch)
92 93 94 95 96
        throw(:halt_check)
      end
    end

    def ee_branch_presence_check!
97
      _, status = step("Fetching origin/#{ee_branch_prefix}", %W[git fetch origin #{ee_branch_prefix}])
98

99 100 101 102 103 104 105 106 107
      if status.zero?
        @ee_branch_found = ee_branch_prefix
      else
        _, status = step("Fetching origin/#{ee_branch_suffix}", %W[git fetch origin #{ee_branch_suffix}])
      end

      if status.zero?
        @ee_branch_found = ee_branch_suffix
      else
108 109 110 111 112 113 114 115
        puts
        puts ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg

        throw(:halt_check, :ko)
      end
    end

    def ee_branch_compat_check!
116
      step("Checking out origin/#{ee_branch_found}", %W[git checkout -b #{ee_branch_found} FETCH_HEAD])
117

118
      generate_patch(ee_branch_found, ee_patch_full_path)
119

120
      unless check_patch(ee_patch_full_path).zero?
121 122 123 124 125 126 127
        puts
        puts ee_branch_doesnt_apply_cleanly_msg

        throw(:halt_check, :ko)
      end

      puts
128
      puts applies_cleanly_msg(ee_branch_found)
129 130
    end

131 132
    def check_patch(patch_path)
      step("Checking out master", %w[git checkout master])
133
      step("Resetting to latest master", %w[git reset --hard origin/master])
134
      step("Fetching CE/#{ce_branch}", %W[git fetch #{CE_REPO} #{ce_branch}])
135 136
      step(
        "Checking if #{patch_path} applies cleanly to EE/master",
137 138 139 140 141 142 143 144 145
        # Don't use --check here because it can result in a 0-exit status even
        # though the patch doesn't apply cleanly, e.g.:
        #   > git apply --check --3way foo.patch
        #   error: patch failed: lib/gitlab/ee_compat_check.rb:74
        #   Falling back to three-way merge...
        #   Applied patch to 'lib/gitlab/ee_compat_check.rb' with conflicts.
        #   > echo $?
        #   0
        %W[git apply --3way #{patch_path}]
146
      ) do |output, status|
147
        puts output
148 149 150 151 152 153 154
        unless status.zero?
          @failed_files = output.lines.reduce([]) do |memo, line|
            if line.start_with?('error: patch failed:')
              file = line.sub(/\Aerror: patch failed: /, '')
              memo << file unless file =~ IGNORED_FILES_REGEX
            end
            memo
155
          end
156 157

          status = 0 if failed_files.empty?
158
        end
159

160
        command(%w[git reset --hard])
161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178
        status
      end
    end

    def delete_ee_branches_locally!
      command(%w[git checkout master])
      command(%W[git branch --delete --force #{ee_branch_prefix}])
      command(%W[git branch --delete --force #{ee_branch_suffix}])
    end

    def merge_base_found?
      step(
        "Finding merge base with master",
        %w[git merge-base origin/master HEAD]
      ) do |output, status|
        if status.zero?
          puts "Merge base was found: #{output}"
          true
179 180
        end
      end
181
    end
182

183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200
    def find_merge_base_with_master(branch:)
      return if merge_base_found?

      # Start with (Math.exp(3).to_i = 20) until (Math.exp(6).to_i = 403)
      # In total we go (20 + 54 + 148 + 403 = 625) commits deeper
      depth = 20
      success =
        (3..6).any? do |factor|
          depth += Math.exp(factor).to_i
          # Repository is initially cloned with a depth of 20 so we need to fetch
          # deeper in the case the branch has more than 20 commits on top of master
          fetch(branch: branch, depth: depth)
          fetch(branch: 'master', depth: depth)

          merge_base_found?
        end

      raise "\n#{branch} is too far behind master, please rebase it!\n" unless success
201 202
    end

203 204 205 206 207 208 209
    def fetch(branch:, depth:)
      step(
        "Fetching deeper...",
        %W[git fetch --depth=#{depth} --prune origin +refs/heads/#{branch}:refs/remotes/origin/#{branch}]
      ) do |output, status|
        raise "Fetch failed: #{output}" unless status.zero?
      end
210 211 212
    end

    def ce_patch_name
213
      @ce_patch_name ||= patch_name_from_branch(ce_branch)
214 215 216
    end

    def ce_patch_full_path
217
      @ce_patch_full_path ||= patches_dir.join(ce_patch_name)
218 219
    end

220 221 222 223 224 225
    def ee_branch_suffix
      @ee_branch_suffix ||= "#{ce_branch}-ee"
    end

    def ee_branch_prefix
      @ee_branch_prefix ||= "ee-#{ce_branch}"
226 227 228
    end

    def ee_patch_name
229
      @ee_patch_name ||= patch_name_from_branch(ee_branch_found)
230 231 232
    end

    def ee_patch_full_path
233
      @ee_patch_full_path ||= patches_dir.join(ee_patch_name)
234 235
    end

236 237 238 239
    def patch_name_from_branch(branch_name)
      branch_name.parameterize << '.patch'
    end

240 241 242 243
    def step(desc, cmd = nil)
      puts "\n=> #{desc}\n"

      if cmd
244
        start = Time.now
245
        puts "\n$ #{cmd.join(' ')}"
246 247 248 249 250 251 252 253 254

        output, status = command(cmd)
        puts "\n==> Finished in #{Time.now - start} seconds"

        if block_given?
          yield(output, status)
        else
          [output, status]
        end
255 256 257 258
      end
    end

    def command(cmd)
259
      Gitlab::Popen.popen(cmd)
260 261
    end

262
    def applies_cleanly_msg(branch)
Rémy Coutable's avatar
Rémy Coutable committed
263 264
      %Q{
        #{PLEASE_READ_THIS_BANNER}
265 266
        🎉 Congratulations!! 🎉

Rémy Coutable's avatar
Rémy Coutable committed
267
        The `#{branch}` branch applies cleanly to EE/master!
268

269 270
        Much ❤️! For more information, see
        https://docs.gitlab.com/ce/development/limit_ee_conflicts.html#check-the-rake-ee_compat_check-in-your-merge-requests
Rémy Coutable's avatar
Rémy Coutable committed
271 272
        #{THANKS_FOR_READING_BANNER}
      }
273 274 275
    end

    def ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg
Rémy Coutable's avatar
Rémy Coutable committed
276 277
      %Q{
        #{PLEASE_READ_THIS_BANNER}
278 279
        💥 Oh no! 💥

Rémy Coutable's avatar
Rémy Coutable committed
280
        The `#{ce_branch}` branch does not apply cleanly to the current
281 282 283 284
        EE/master, and no `#{ee_branch_prefix}` or `#{ee_branch_suffix}` branch
        was found in the EE repository.

        #{conflicting_files_msg}
285

286
        We advise you to create a `#{ee_branch_prefix}` or `#{ee_branch_suffix}`
Rémy Coutable's avatar
Rémy Coutable committed
287
        branch that includes changes from `#{ce_branch}` but also specific changes
288 289 290
        than can be applied cleanly to EE/master. In some cases, the conflicts
        are trivial and you can ignore the warning from this job. As always,
        use your best judgment!
291 292 293

        There are different ways to create such branch:

294
        1. Create a new branch from master and cherry-pick your CE commits
295 296 297

          # In the EE repo
          $ git fetch origin
298 299 300
          $ git checkout -b #{ee_branch_prefix} origin/master
          $ git fetch #{ce_repo} #{ce_branch}
          $ git cherry-pick SHA # Repeat for all the commits you want to pick
301

Rémy Coutable's avatar
Rémy Coutable committed
302
          You can squash the `#{ce_branch}` commits into a single "Port of #{ce_branch} to EE" commit.
303

304
        2. Apply your branch's patch to EE
305

306 307
          # In the CE repo
          $ git fetch origin master
308
          $ git diff --binary origin/master...HEAD -- > #{ce_branch}.patch
309 310

          # In the EE repo
311 312 313
          $ git fetch origin master
          $ git checkout -b #{ee_branch_prefix} origin/master
          $ git apply --3way path/to/#{ce_branch}.patch
314

315 316 317 318 319 320
          At this point you might have conflicts such as:

            error: patch failed: lib/gitlab/ee_compat_check.rb:5
            Falling back to three-way merge...
            Applied patch to 'lib/gitlab/ee_compat_check.rb' with conflicts.
            U lib/gitlab/ee_compat_check.rb
321

322
          Resolve them, stage the changes and commit them.
323

324
          If the patch couldn't be applied cleanly, use the following command:
325

326 327
          # In the EE repo
          $ git apply --reject path/to/#{ce_branch}.patch
328

329 330 331 332 333
          This option makes git apply the parts of the patch that are applicable,
          and leave the rejected hunks in corresponding `.rej` files.
          You can then resolve the conflicts highlighted in `.rej` by
          manually applying the correct diff from the `.rej` file to the file with conflicts.
          When finished, you can delete the `.rej` files and commit your changes.
334 335

        ⚠️ Don't forget to push your branch to gitlab-ee:
336 337

          # In the EE repo
338 339 340 341
          $ git push origin #{ee_branch_prefix}

        ⚠️ Also, don't forget to create a new merge request on gitlab-ce and
        cross-link it with the CE merge request.
342

343
        Once this is done, you can retry this failed build, and it should pass.
344

345 346
        Stay 💪 ! For more information, see
        https://docs.gitlab.com/ce/development/limit_ee_conflicts.html#check-the-rake-ee_compat_check-in-your-merge-requests
Rémy Coutable's avatar
Rémy Coutable committed
347 348
        #{THANKS_FOR_READING_BANNER}
      }
349 350 351
    end

    def ee_branch_doesnt_apply_cleanly_msg
Rémy Coutable's avatar
Rémy Coutable committed
352 353
      %Q{
        #{PLEASE_READ_THIS_BANNER}
354 355
        💥 Oh no! 💥

Rémy Coutable's avatar
Rémy Coutable committed
356
        The `#{ce_branch}` does not apply cleanly to the current EE/master, and
357 358 359 360 361
        even though a `#{ee_branch_found}` branch
        exists in the EE repository, it does not apply cleanly either to
        EE/master!

        #{conflicting_files_msg}
362

363
        Please update the `#{ee_branch_found}`, push it again to gitlab-ee, and
364 365
        retry this build.

366 367
        Stay 💪 ! For more information, see
        https://docs.gitlab.com/ce/development/limit_ee_conflicts.html#check-the-rake-ee_compat_check-in-your-merge-requests
Rémy Coutable's avatar
Rémy Coutable committed
368 369
        #{THANKS_FOR_READING_BANNER}
      }
370
    end
371 372 373 374 375 376

    def conflicting_files_msg
      failed_files.reduce("The conflicts detected were as follows:\n") do |memo, file|
        memo << "\n        - #{file}"
      end
    end
377 378
  end
end