Merge branch 'gitaly-redis-refactor' into 'master'

Gitaly redis refactor

Closes gitaly#522

See merge request !164

CHANGELOG

 v5.9.0
+  - Support new /internal/pre-receive API endpoint for post-receive operations
   - Support new /internal/post-receive API endpoint for post-receive operations
+  - Support `redis` field on /internal/check API endpoint
 
 v5.8.1
   - Support old versions of ruby without monotonic clock

bin/check

@@ -3,6 +3,17 @@
 require_relative '../lib/gitlab_init'
 require_relative '../lib/gitlab_net'
 
+def ping_redis
+  print "Send ping to redis server: "
+  if GitlabNet.new.redis_client.ping
+    print 'OK'
+  else
+    abort 'FAILED'
+  end
+
+  puts "\n"
+end
+
 #
 # GitLab shell check task
 #
@@ -10,11 +21,25 @@ require_relative '../lib/gitlab_net'
 print "Check GitLab API access: "
 begin
   resp = GitlabNet.new.check
-  if resp.code == "200"
-    print 'OK'
-  else
+
+  if resp.code != "200"
     abort "FAILED. code: #{resp.code}"
   end
+
+  puts 'OK'
+
+  check_values = JSON.parse(resp.body)
+
+  if check_values.key?('redis')
+    print 'Redis available via internal API: '
+    if check_values['redis']
+      puts 'OK'
+    else
+      abort 'FAILED'
+    end
+  else
+    ping_redis
+  end
 rescue GitlabNet::ApiUnreachableError
   abort "FAILED: Failed to connect to internal API"
 end
@@ -30,12 +55,3 @@ else
   abort "FAILED"
 end
 puts "\n"
-
-print "Send ping to redis server: "
-if GitlabNet.new.redis_client.ping
-  print 'OK'
-else
-  abort 'FAILED'
-end
-
-puts "\n"

hooks/pre-receive

@@ -9,9 +9,18 @@ protocol = ENV.delete('GL_PROTOCOL')
 repo_path = Dir.pwd
 gl_repository = ENV['GL_REPOSITORY']
 
+def increase_reference_counter(gl_repository)
+  result = GitlabNet.new.pre_receive(gl_repository)
+
+  result['reference_counter_increased']
+rescue GitlabNet::NotFound
+  GitlabReferenceCounter.new(repo_path).increase
+end
+
 require_relative '../lib/gitlab_custom_hook'
 require_relative '../lib/gitlab_reference_counter'
 require_relative '../lib/gitlab_access'
+require_relative '../lib/gitlab_net'
 
 # It's important that on pre-receive `increase_reference_counter` gets executed
 # last so that it only runs if everything else succeeded. On post-receive on the
@@ -19,7 +28,7 @@ require_relative '../lib/gitlab_access'
 # and we don't want to skip it if the custom hook fails.
 if GitlabAccess.new(gl_repository, repo_path, key_id, refs, protocol).exec &&
     GitlabCustomHook.new(repo_path, key_id).pre_receive(refs) &&
-    GitlabReferenceCounter.new(repo_path).increase
+    increase_reference_counter(gl_repository)
   exit 0
 else
   exit 1

lib/gitlab_net.rb

@@ -126,6 +126,14 @@ class GitlabNet
     JSON.parse(resp.body) if resp.code == '200'
   end
 
+  def pre_receive(gl_repository)
+    resp = post("#{host}/pre_receive", gl_repository: gl_repository)
+
+    raise NotFound if resp.code == '404'
+
+    JSON.parse(resp.body) if resp.code == '200'
+  end
+
   def redis_client
     redis_config = config.redis
     database = redis_config['database'] || 0

spec/gitlab_net_spec.rb

@@ -126,6 +126,32 @@ describe GitlabNet, vcr: true do
     end
   end
 
+  describe :pre_receive do
+    let(:gl_repository) { "project-1" }
+    let(:params) { { gl_repository: gl_repository } }
+
+    subject { gitlab_net.pre_receive(gl_repository) }
+
+    it 'sends the correct parameters and returns the request body parsed' do
+      Net::HTTP::Post.any_instance.should_receive(:set_form_data)
+        .with(hash_including(params))
+
+      VCR.use_cassette("pre-receive") { subject }
+    end
+
+    it 'calls /internal/pre-receive' do
+      VCR.use_cassette("pre-receive") do
+        expect(subject['reference_counter_increased']).to be(true)
+      end
+    end
+
+    it 'throws a NotFound error when pre-receive is not available' do
+      VCR.use_cassette("pre-receive-not-found") do
+        expect { subject }.to raise_error(GitlabNet::NotFound)
+      end
+    end
+  end
+
   describe :post_receive do
     let(:gl_repository) { "project-1" }
     let(:changes) { "123456 789012 refs/heads/test\n654321 210987 refs/tags/tag" }

spec/vcr_cassettes/pre-receive-not-found.yml

+---
+http_interactions:
+- request:
+    method: post
+    uri: http://localhost:3000/api/v4/internal/pre_receive
+    body:
+      encoding: US-ASCII
+      string: gl_repository=project-1&secret_token=0a3938d9d95d807e94d937af3a4fbbea%0A
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+      Content-Type:
+      - application/x-www-form-urlencoded
+  response:
+    status:
+      code: 404
+      message: Not Found
+    headers:
+      Cache-Control:
+      - no-cache
+      Content-Length:
+      - '25'
+      Content-Type:
+      - application/json
+      Date:
+      - Thu, 31 Aug 2017 16:41:13 GMT
+      Vary:
+      - Origin
+      X-Request-Id:
+      - 0b845e9a-5417-488d-bc5a-07d8c585b2da
+      X-Runtime:
+      - '0.295361'
+    body:
+      encoding: UTF-8
+      string: '{"error":"404 Not Found"}'
+    http_version: 
+  recorded_at: Thu, 31 Aug 2017 16:41:13 GMT
+recorded_with: VCR 2.4.0

spec/vcr_cassettes/pre-receive.yml

+---
+http_interactions:
+- request:
+    method: post
+    uri: http://localhost:3000/api/v4/internal/pre_receive
+    body:
+      encoding: US-ASCII
+      string: gl_repository=project-1&secret_token=0a3938d9d95d807e94d937af3a4fbbea%0A
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+      Content-Type:
+      - application/x-www-form-urlencoded
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Cache-Control:
+      - max-age=0, private, must-revalidate
+      Content-Length:
+      - '36'
+      Content-Type:
+      - application/json
+      Date:
+      - Thu, 31 Aug 2017 20:17:41 GMT
+      Etag:
+      - W/"7d4df85c493bd3d421351aa791a8fbf6"
+      Vary:
+      - Origin
+      X-Frame-Options:
+      - SAMEORIGIN
+      X-Request-Id:
+      - f0c84103-8dc0-48ea-a142-62554f6bca3d
+      X-Runtime:
+      - '0.612997'
+    body:
+      encoding: UTF-8
+      string: '{"reference_counter_increased":true}'
+    http_version:
+  recorded_at: Thu, 31 Aug 2017 20:17:41 GMT
+recorded_with: VCR 2.4.0