Geo SSH clone/pull for not yet replicated repos

Support cloning/pulling a repo from a Geo
secondary where the repo only exists on the
primary by proxying the request.

ee/changelogs/unreleased/202037-geo-ssh-clone-pull-redirect-to-primary-when-selective-sync-enab.yml

+---
+title: 'Geo: Proxy SSH git operations for repositories that are not yet replicated'
+merge_request: 27994
+author:
+type: added

ee/lib/api/geo.rb

@@ -87,6 +87,51 @@ module API
       resource 'proxy_git_ssh' do
         format :json
 
+        # For git clone/pull
+
+        # Responsible for making HTTP GET /repo.git/info/refs?service=git-upload-pack
+        # request *from* secondary gitlab-shell to primary
+        #
+        params do
+          requires :secret_token, type: String
+          requires :data, type: Hash do
+            requires :gl_id, type: String
+            requires :primary_repo, type: String
+          end
+        end
+        post 'info_refs_upload_pack' do
+          authenticate_by_gitlab_shell_token!
+          params.delete(:secret_token)
+
+          response = Gitlab::Geo::GitSSHProxy.new(params['data']).info_refs_upload_pack
+
+          status(response.code)
+          response.body
+        end
+
+        # Responsible for making HTTP POST /repo.git/git-upload-pack
+        # request *from* secondary gitlab-shell to primary
+        #
+        params do
+          requires :secret_token, type: String
+          requires :data, type: Hash do
+            requires :gl_id, type: String
+            requires :primary_repo, type: String
+          end
+          requires :output, type: String, desc: 'Output from git-upload-pack'
+        end
+        post 'upload_pack' do
+          authenticate_by_gitlab_shell_token!
+          params.delete(:secret_token)
+
+          response = Gitlab::Geo::GitSSHProxy.new(params['data']).upload_pack(params['output'])
+
+          status(response.code)
+          response.body
+        end
+
+        # For git push
+
         # Responsible for making HTTP GET /repo.git/info/refs?service=git-receive-pack
         # request *from* secondary gitlab-shell to primary
         #

ee/lib/ee/gitlab/geo_git_access.rb

@@ -12,20 +12,13 @@ module EE
 
       private
 
-      def custom_action_for?(cmd)
-        return unless receive_pack?(cmd) # git push
-        return unless ::Gitlab::Database.read_only?
-
-        ::Gitlab::Geo.secondary_with_primary?
-      end
-
       def custom_action_for(cmd)
         return unless custom_action_for?(cmd)
 
         payload = {
           'action' => 'geo_proxy_to_primary',
           'data' => {
-            'api_endpoints' => custom_action_api_endpoints,
+            'api_endpoints' => custom_action_api_endpoints_for(cmd),
             'primary_repo' => primary_http_repo_url
           }
         }
@@ -33,6 +26,17 @@ module EE
         ::Gitlab::GitAccessResult::CustomAction.new(payload, messages)
       end
 
+      def custom_action_for?(cmd)
+        return unless ::Gitlab::Database.read_only?
+        return unless ::Gitlab::Geo.secondary_with_primary?
+
+        receive_pack?(cmd) || upload_pack_and_not_replicated?(cmd)
+      end
+
+      def upload_pack_and_not_replicated?(cmd)
+        upload_pack?(cmd) && !::Geo::ProjectRegistry.repository_replicated_for?(project.id)
+      end
+
       def messages
         messages = ::Gitlab::Geo.interacting_with_primary_message(primary_ssh_url_to_repo).split("\n")
         lag_message = current_replication_lag_message
@@ -79,7 +83,18 @@ module EE
         @current_replication_lag ||= ::Gitlab::Geo::HealthCheck.new.db_replication_lag_seconds
       end
 
-      def custom_action_api_endpoints
+      def custom_action_api_endpoints_for(cmd)
+        receive_pack?(cmd) ? custom_action_push_api_endpoints : custom_action_pull_api_endpoints
+      end
+
+      def custom_action_pull_api_endpoints
+        [
+         api_v4_geo_proxy_git_ssh_info_refs_upload_pack_path,
+         api_v4_geo_proxy_git_ssh_upload_pack_path
+        ]
+      end
+
+      def custom_action_push_api_endpoints
         [
           api_v4_geo_proxy_git_ssh_info_refs_receive_pack_path,
           api_v4_geo_proxy_git_ssh_receive_pack_path

ee/lib/ee/gitlab/middleware/read_only/controller.rb

@@ -38,14 +38,9 @@ module EE
           end
 
           def geo_proxy_git_ssh_route?
-            routes = ::Gitlab::Middleware::ReadOnly::API_VERSIONS.map do |version|
-              %W(
-                /api/v#{version}/geo/proxy_git_ssh/info_refs_receive_pack
-                /api/v#{version}/geo/proxy_git_ssh/receive_pack
-              )
+            ::Gitlab::Middleware::ReadOnly::API_VERSIONS.any? do |version|
+              request.path.start_with?("/api/v#{version}/geo/proxy_git_ssh")
             end
-
-            routes.flatten.include?(request.path)
           end
 
           def geo_api_route?

ee/lib/gitlab/geo/git_ssh_proxy.rb

@@ -5,6 +5,9 @@ module Gitlab
     class GitSSHProxy
       HTTP_READ_TIMEOUT = 60
 
+      UPLOAD_PACK_REQUEST_CONTENT_TYPE = 'application/x-git-upload-pack-request'.freeze
+      UPLOAD_PACK_RESULT_CONTENT_TYPE = 'application/x-git-upload-pack-result'.freeze
+
       RECEIVE_PACK_REQUEST_CONTENT_TYPE = 'application/x-git-receive-pack-request'.freeze
       RECEIVE_PACK_RESULT_CONTENT_TYPE = 'application/x-git-receive-pack-result'.freeze
 
@@ -49,7 +52,38 @@ module Gitlab
         @data = data
       end
 
+      # For git clone/pull
+
+      def info_refs_upload_pack
+        ensure_secondary!
+
+        url = "#{primary_repo}/info/refs?service=git-upload-pack"
+
+        resp = get(url)
+        resp.body = remove_upload_pack_http_service_fragment_from(resp.body) if resp.is_a?(Net::HTTPSuccess)
+
+        APIResponse.from_http_response(resp, primary_repo)
+      rescue => e
+        handle_exception(e)
+      end
+
+      def upload_pack(encoded_response)
+        ensure_secondary!
+
+        url = "#{primary_repo}/git-upload-pack"
+        headers = { 'Content-Type' => UPLOAD_PACK_REQUEST_CONTENT_TYPE, 'Accept' => UPLOAD_PACK_RESULT_CONTENT_TYPE }
+        decoded_response = Base64.decode64(encoded_response)
+        decoded_response = convert_upload_pack_from_http_to_ssh(decoded_response)
+
+        resp = post(url, decoded_response, headers)
+
+        APIResponse.from_http_response(resp, primary_repo)
+      rescue => e
+        handle_exception(e)
+      end
+
       # For git push
+
       def info_refs_receive_pack
         ensure_secondary!
 
@@ -63,7 +97,6 @@ module Gitlab
         handle_exception(e)
       end
 
-      # For git push
       def receive_pack(encoded_info_refs_response)
         ensure_secondary!
 
@@ -130,16 +163,38 @@ module Gitlab
         http.start { http.request(req) }
       end
 
+      # HTTP(S) and SSH responses are very similar, except for the fragment below.
+      # As we're performing a git HTTP(S) request here, we'll get a HTTP(s)
+      # suitable git response.  However, we're executing in the context of an
+      # SSH session so we need to make the response suitable for what git over
+      # SSH expects.
+
+      # See Uploading Data > HTTP(S) section at:
+      # https://git-scm.com/book/en/v2/Git-Internals-Transfer-Protocols
+      #
+      def remove_upload_pack_http_service_fragment_from(body)
+        body.gsub(/\A001e# service=git-upload-pack\n0000/, '')
+      end
+
+      # See Uploading Data > HTTP(S) section at:
+      # https://git-scm.com/book/en/v2/Git-Internals-Transfer-Protocols
+      #
+      def convert_upload_pack_from_http_to_ssh(body)
+        clone_operation = /\n0032want \w{40}/
+
+        if body.match?(clone_operation)
+          # git clone
+          body.gsub(clone_operation, '') + "\n0000"
+        else
+          # git pull
+          body.gsub(/\n0000$/, "\n0009done\n0000")
+        end
+      end
+
+      # See Downloading Data > HTTP(S) section at:
+      # https://git-scm.com/book/en/v2/Git-Internals-Transfer-Protocols
+      #
       def remove_receive_pack_http_service_fragment_from(body)
-        # HTTP(S) and SSH responses are very similar, except for the fragment below.
-        # As we're performing a git HTTP(S) request here, we'll get a HTTP(s)
-        # suitable git response.  However, we're executing in the context of an
-        # SSH session so we need to make the response suitable for what git over
-        # SSH expects.
-        #
-        # See Downloading Data > HTTP(S) section at:
-        # https://git-scm.com/book/en/v2/Git-Internals-Transfer-Protocols
-        #
         body.gsub(/\A001f# service=git-receive-pack\n0000/, '')
       end
 

ee/spec/lib/gitlab/git_access_spec.rb

@@ -331,19 +331,41 @@ describe Gitlab::GitAccess do
           end
         end
 
-        context 'that has no DB replication lag' do
-          let(:current_replication_lag) { 0 }
+        context 'for a repository that has been replicated' do
+          context 'that has no DB replication lag' do
+            let(:current_replication_lag) { 0 }
+
+            it 'does not return a replication lag message in the console messages' do
+              expect(pull_changes.console_messages).to be_empty
+            end
+          end
+
+          context 'that has DB replication lag > 0' do
+            let(:current_replication_lag) { 7 }
 
-          it 'does not return a replication lag message in the console messages' do
-            expect(pull_changes.console_messages).to be_empty
+            it 'returns a replication lag message in the console messages' do
+              expect(pull_changes.console_messages).to eq(['Current replication lag: 7 seconds'])
+            end
           end
         end
 
-        context 'that has DB replication lag > 0' do
-          let(:current_replication_lag) { 7 }
+        context 'for a repository that has yet to be replicated' do
+          let(:project) { create(:project) }
+          let(:current_replication_lag) { 0 }
+
+          before do
+            create(:geo_node, :primary)
+          end
+
+          it 'returns a custom action' do
+            expected_payload = { "action" => "geo_proxy_to_primary", "data" => { "api_endpoints" => ["/api/v4/geo/proxy_git_ssh/info_refs_upload_pack", "/api/v4/geo/proxy_git_ssh/upload_pack"], "primary_repo" => geo_primary_http_url_to_repo(project) } }
+            expected_console_messages = ["This request to a Geo secondary node will be forwarded to the", "Geo primary node:", "", "  #{geo_primary_ssh_url_to_repo(project)}"]
+
+            response = pull_changes
 
-          it 'returns a replication lag message in the console messages' do
-            expect(pull_changes.console_messages).to eq(['Current replication lag: 7 seconds'])
+            expect(response).to be_instance_of(Gitlab::GitAccessResult::CustomAction)
+            expect(response.payload).to eq(expected_payload)
+            expect(response.console_messages).to eq(expected_console_messages)
           end
         end
       end
@@ -351,6 +373,25 @@ describe Gitlab::GitAccess do
 
     context 'git push' do
       it { expect { push_changes }.to raise_forbidden(Gitlab::GitAccess::ERROR_MESSAGES[:upload]) }
+
+      context 'for a secondary' do
+        before do
+          stub_licensed_features(geo: true)
+          create(:geo_node, :primary)
+          stub_current_geo_node(create(:geo_node))
+        end
+
+        it 'returns a custom action' do
+          expected_payload = { "action" => "geo_proxy_to_primary", "data" => { "api_endpoints" => ["/api/v4/geo/proxy_git_ssh/info_refs_receive_pack", "/api/v4/geo/proxy_git_ssh/receive_pack"], "primary_repo" => geo_primary_http_url_to_repo(project) } }
+          expected_console_messages = ["This request to a Geo secondary node will be forwarded to the", "Geo primary node:", "", "  #{geo_primary_ssh_url_to_repo(project)}"]
+
+          response = push_changes
+
+          expect(response).to be_instance_of(Gitlab::GitAccessResult::CustomAction)
+          expect(response.payload).to eq(expected_payload)
+          expect(response.console_messages).to eq(expected_console_messages)
+        end
+      end
     end
   end
 

ee/spec/lib/gitlab/git_access_wiki_spec.rb

@@ -39,6 +39,10 @@ describe Gitlab::GitAccessWiki do
 
   private
 
+  def pull_changes(changes = Gitlab::GitAccess::ANY)
+    access.check('git-upload-pack', changes)
+  end
+
   def push_changes(changes = Gitlab::GitAccess::ANY)
     access.check('git-receive-pack', changes)
   end

ee/spec/requests/api/geo_spec.rb

@@ -364,6 +364,126 @@ describe API::Geo do
       stub_current_geo_node(secondary_node)
     end
 
+    describe 'POST /geo/proxy_git_ssh/info_refs_upload_pack' do
+      context 'with all required params missing' do
+        it 'responds with 400' do
+          post api('/geo/proxy_git_ssh/info_refs_upload_pack'), params: nil
+
+          expect(response).to have_gitlab_http_status(:bad_request)
+          expect(json_response['error']).to eql('secret_token is missing, data is missing, data[gl_id] is missing, data[primary_repo] is missing')
+        end
+      end
+
+      context 'with all required params' do
+        let(:git_push_ssh_proxy) { double(Gitlab::Geo::GitSSHProxy) }
+
+        before do
+          allow(Gitlab::Geo::GitSSHProxy).to receive(:new).with(data).and_return(git_push_ssh_proxy)
+        end
+
+        context 'with an invalid secret_token' do
+          it 'responds with 401' do
+            post(api('/geo/proxy_git_ssh/info_refs_upload_pack'), params: { secret_token: 'invalid', data: data })
+
+            expect(response).to have_gitlab_http_status(:unauthorized)
+            expect(json_response['error']).to be_nil
+          end
+        end
+
+        context 'where an exception occurs' do
+          it 'responds with 500' do
+            expect(git_push_ssh_proxy).to receive(:info_refs_upload_pack).and_raise('deliberate exception raised')
+
+            post api('/geo/proxy_git_ssh/info_refs_upload_pack'), params: { secret_token: secret_token, data: data }
+
+            expect(response).to have_gitlab_http_status(:internal_server_error)
+            expect(json_response['message']).to include('RuntimeError (deliberate exception raised)')
+            expect(json_response['result']).to be_nil
+          end
+        end
+
+        context 'with a valid secret token' do
+          let(:http_response) { double(Net::HTTPOK, code: 200, body: 'something here') }
+          let(:api_response) { Gitlab::Geo::GitSSHProxy::APIResponse.from_http_response(http_response, primary_repo) }
+
+          before do
+            # Mocking a real Net::HTTPSuccess is very difficult as it's not
+            # easy to instantiate the class due to the way it sets the body
+            expect(http_response).to receive(:is_a?).with(Net::HTTPSuccess).and_return(true)
+          end
+
+          it 'responds with 200' do
+            expect(git_push_ssh_proxy).to receive(:info_refs_upload_pack).and_return(api_response)
+
+            post api('/geo/proxy_git_ssh/info_refs_upload_pack'), params: { secret_token: secret_token, data: data }
+
+            expect(response).to have_gitlab_http_status(:ok)
+            expect(Base64.decode64(json_response['result'])).to eql('something here')
+          end
+        end
+      end
+    end
+
+    describe 'POST /geo/proxy_git_ssh/upload_pack' do
+      context 'with all required params missing' do
+        it 'responds with 400' do
+          post api('/geo/proxy_git_ssh/upload_pack'), params: nil
+
+          expect(response).to have_gitlab_http_status(:bad_request)
+          expect(json_response['error']).to eql('secret_token is missing, data is missing, data[gl_id] is missing, data[primary_repo] is missing, output is missing')
+        end
+      end
+
+      context 'with all required params' do
+        let(:output) { Base64.encode64('info_refs content') }
+        let(:git_push_ssh_proxy) { double(Gitlab::Geo::GitSSHProxy) }
+
+        before do
+          allow(Gitlab::Geo::GitSSHProxy).to receive(:new).with(data).and_return(git_push_ssh_proxy)
+        end
+
+        context 'with an invalid secret_token' do
+          it 'responds with 401' do
+            post(api('/geo/proxy_git_ssh/upload_pack'), params: { secret_token: 'invalid', data: data, output: output })
+
+            expect(response).to have_gitlab_http_status(:unauthorized)
+            expect(json_response['error']).to be_nil
+          end
+        end
+
+        context 'where an exception occurs' do
+          it 'responds with 500' do
+            expect(git_push_ssh_proxy).to receive(:upload_pack).and_raise('deliberate exception raised')
+            post api('/geo/proxy_git_ssh/upload_pack'), params: { secret_token: secret_token, data: data, output: output }
+
+            expect(response).to have_gitlab_http_status(:internal_server_error)
+            expect(json_response['message']).to include('RuntimeError (deliberate exception raised)')
+            expect(json_response['result']).to be_nil
+          end
+        end
+
+        context 'with a valid secret token' do
+          let(:http_response) { double(Net::HTTPCreated, code: 201, body: 'something here', class: Net::HTTPCreated) }
+          let(:api_response) { Gitlab::Geo::GitSSHProxy::APIResponse.from_http_response(http_response, primary_repo) }
+
+          before do
+            # Mocking a real Net::HTTPSuccess is very difficult as it's not
+            # easy to instantiate the class due to the way it sets the body
+            expect(http_response).to receive(:is_a?).with(Net::HTTPSuccess).and_return(true)
+          end
+
+          it 'responds with 201' do
+            expect(git_push_ssh_proxy).to receive(:upload_pack).with(output).and_return(api_response)
+
+            post api('/geo/proxy_git_ssh/upload_pack'), params: { secret_token: secret_token, data: data, output: output }
+
+            expect(response).to have_gitlab_http_status(:created)
+            expect(Base64.decode64(json_response['result'])).to eql('something here')
+          end
+        end
+      end
+    end
+
     describe 'POST /geo/proxy_git_ssh/info_refs_receive_pack' do
       context 'with all required params missing' do
         it 'responds with 400' do

ee/spec/support/shared_examples/lib/gitlab/git_access_shared_examples.rb

@@ -28,15 +28,6 @@ RSpec.shared_examples 'a read-only GitLab instance' do
 
     context 'that is correctly set up' do
       let(:secondary_with_primary) { true }
-      let(:payload) do
-        {
-          'action' => 'geo_proxy_to_primary',
-          'data' => {
-            'api_endpoints' => %w{/api/v4/geo/proxy_git_ssh/info_refs_receive_pack /api/v4/geo/proxy_git_ssh/receive_pack},
-            'primary_repo' => primary_repo_url
-          }
-        }
-      end
       let(:console_messages) do
         [
           "This request to a Geo secondary node will be forwarded to the",
@@ -46,12 +37,44 @@ RSpec.shared_examples 'a read-only GitLab instance' do
         ]
       end
 
-      it 'attempts to proxy to the primary' do
-        project.add_maintainer(user)
+      context 'for a git clone/pull' do
+        let(:payload) do
+          {
+            'action' => 'geo_proxy_to_primary',
+            'data' => {
+              'api_endpoints' => %w{/api/v4/geo/proxy_git_ssh/info_refs_upload_pack /api/v4/geo/proxy_git_ssh/upload_pack},
+              'primary_repo' => primary_repo_url
+            }
+          }
+        end
+
+        it 'attempts to proxy to the primary' do
+          project.add_maintainer(user)
+
+          expect(pull_changes).to be_a(Gitlab::GitAccessResult::CustomAction)
+          expect(pull_changes.payload).to eql(payload)
+          expect(pull_changes.console_messages).to include(*console_messages)
+        end
+      end
+
+      context 'for a git push' do
+        let(:payload) do
+          {
+            'action' => 'geo_proxy_to_primary',
+            'data' => {
+              'api_endpoints' => %w{/api/v4/geo/proxy_git_ssh/info_refs_receive_pack /api/v4/geo/proxy_git_ssh/receive_pack},
+              'primary_repo' => primary_repo_url
+            }
+          }
+        end
+
+        it 'attempts to proxy to the primary' do
+          project.add_maintainer(user)
 
-        expect(push_changes).to be_a(Gitlab::GitAccessResult::CustomAction)
-        expect(push_changes.payload).to eql(payload)
-        expect(push_changes.console_messages).to include(*console_messages)
+          expect(push_changes).to be_a(Gitlab::GitAccessResult::CustomAction)
+          expect(push_changes.payload).to eql(payload)
+          expect(push_changes.console_messages).to include(*console_messages)
+        end
       end
     end
   end