Backend for looking up SSH host keys

Gemfile

@@ -405,6 +405,16 @@ gem 'sys-filesystem', '~> 1.1.6'
 # NTP client
 gem 'net-ntp'
 
+# SSH host key support
+gem 'net-ssh', '~> 4.1.0'
+
+# Required for ED25519 SSH host key support
+group :ed25519 do
+  gem 'rbnacl-libsodium'
+  gem 'rbnacl', '~> 3.2'
+  gem 'bcrypt_pbkdf', '~> 1.0'
+end
+
 # Gitaly GRPC client
 gem 'gitaly', '~> 0.24.0'
 

Gemfile.lock

@@ -83,6 +83,7 @@ GEM
     babosa (1.0.2)
     base32 (0.3.2)
     bcrypt (3.1.11)
+    bcrypt_pbkdf (1.0.0)
     benchmark-ips (2.3.0)
     better_errors (2.1.1)
       coderay (>= 1.0.0)
@@ -504,6 +505,7 @@ GEM
     mysql2 (0.4.5)
     net-ldap (0.16.0)
     net-ntp (2.1.3)
+    net-ssh (4.1.0)
     netrc (0.11.0)
     nokogiri (1.6.8.1)
       mini_portile2 (~> 2.1.0)
@@ -691,6 +693,10 @@ GEM
     rake (12.0.0)
     rblineprof (0.3.6)
       debugger-ruby_core_source (~> 1.3)
+    rbnacl (3.4.0)
+      ffi
+    rbnacl-libsodium (1.0.11)
+      rbnacl (>= 3.0.1)
     rdoc (4.2.2)
       json (~> 1.4)
     re2 (1.1.1)
@@ -954,6 +960,7 @@ DEPENDENCIES
   aws-sdk
   babosa (~> 1.0.2)
   base32 (~> 0.3.0)
+  bcrypt_pbkdf (~> 1.0)
   benchmark-ips (~> 2.3.0)
   better_errors (~> 2.1.0)
   binding_of_caller (~> 0.7.2)
@@ -1053,6 +1060,7 @@ DEPENDENCIES
   mysql2 (~> 0.4.5)
   net-ldap
   net-ntp
+  net-ssh (~> 4.1.0)
   nokogiri (~> 1.6.7, >= 1.6.7.2)
   oauth2 (~> 1.4)
   octokit (~> 4.6.2)
@@ -1099,6 +1107,8 @@ DEPENDENCIES
   rainbow (~> 2.2)
   raindrops (~> 0.18)
   rblineprof (~> 0.3.6)
+  rbnacl (~> 3.2)
+  rbnacl-libsodium
   rdoc (~> 4.2)
   re2 (~> 1.1.1)
   recaptcha (~> 3.0)

app/controllers/projects/mirrors_controller.rb

@@ -13,6 +13,22 @@ class Projects::MirrorsController < Projects::ApplicationController
     redirect_to_repository_settings(@project)
   end
 
+  def ssh_host_keys
+    lookup = SshHostKey.new(project: project, url: params[:ssh_url])
+
+    if lookup.error.present?
+      # Failed to read keys
+      render json: { message: lookup.error }, status: 400
+    elsif lookup.known_hosts.nil?
+      # Still working, come back later
+      render body: nil, status: 204
+    else
+      render json: lookup
+    end
+  rescue ArgumentError => err
+    render json: { message: err.message }, status: 400
+  end
+
   def update
     if @project.update_attributes(safe_mirror_params)
       if @project.mirror?

config/routes/project.rb

@@ -189,6 +189,7 @@ constraints(ProjectUrlConstrainer.new) do
       ## EE-specific
       resource :mirror, only: [:show, :update] do
         member do
+          get :ssh_host_keys, constraints: { format: :json }
           post :update_now
         end
       end

ee/app/models/ssh_host_key.rb

+# Detected SSH host keys are transiently stored in Redis
+class SshHostKey
+  class Fingerprint < Gitlab::KeyFingerprint
+    attr_reader :index
+
+    def initialize(key, index: nil)
+      super(key)
+
+      @index = index
+    end
+
+    def as_json(*)
+      { bits: bits, fingerprint: fingerprint, type: type, index: index }
+    end
+  end
+
+  include ReactiveCaching
+
+  self.reactive_cache_key = ->(key) { [key.class.to_s, key.id] }
+
+  # Do not refresh the data in the background - it is not expected to change
+  self.reactive_cache_refresh_interval = 15.minutes
+  self.reactive_cache_lifetime = 10.minutes
+
+  def self.find_by(opts = {})
+    id = opts.fetch(:id, "")
+    project_id, url = id.split(':', 2)
+
+    project = Project.where(id: project_id).includes(:import_data).first
+
+    project.presence && new(project: project, url: url)
+  end
+
+  def self.fingerprint_host_keys(data)
+    return [] unless data.is_a?(String)
+
+    data
+      .each_line
+      .each_with_index
+      .map { |line, index| Fingerprint.new(line, index: index) }
+      .select(&:valid?)
+  end
+
+  attr_reader :project, :url
+
+  def initialize(project:, url:)
+    @project = project
+    @url = normalize_url(url)
+  end
+
+  def id
+    [project.id, url].join(':')
+  end
+
+  def as_json(*)
+    {
+      known_hosts: known_hosts,
+      fingerprints: fingerprints
+    }
+  end
+
+  def known_hosts
+    with_reactive_cache { |data| data[:known_hosts] }
+  end
+
+  def fingerprints
+    @fingerprints ||= self.class.fingerprint_host_keys(known_hosts)
+  end
+
+  def error
+    with_reactive_cache { |data| data[:error] }
+  end
+
+  def calculate_reactive_cache
+    known_hosts, errors, status =
+      Open3.popen3({}, *%W[ssh-keyscan -T 5 -p #{url.port} -f-]) do |stdin, stdout, stderr, wait_thr|
+        stdin.puts(url.host)
+        stdin.close
+
+        [
+          cleanup(stdout.read),
+          cleanup(stderr.read),
+          wait_thr.value
+        ]
+      end
+
+    # ssh-keyscan returns an exit code 0 in several error conditions, such as an
+    # unknown hostname, so check both STDERR and the exit code
+    if !status.success? || errors.present?
+      Rails.logger.debug("Failed to detect SSH host keys for #{id}: #{errors}")
+
+      return { error: 'Failed to detect SSH host keys' }
+    end
+
+    { known_hosts: known_hosts }
+  end
+
+  private
+
+  # Remove comments and duplicate entries
+  def cleanup(data)
+    data
+      .each_line
+      .map { |line| line unless line.start_with?('#') || line.chomp.empty? }
+      .compact
+      .uniq
+      .sort
+      .join
+  end
+
+  def normalize_url(url)
+    full_url = ::Addressable::URI.parse(url)
+    raise ArgumentError.new("Invalid URL") unless full_url&.scheme == 'ssh'
+
+    Addressable::URI.parse("ssh://#{full_url.host}:#{full_url.inferred_port}")
+  end
+end

lib/gitlab/key_fingerprint.rb

 module Gitlab
   class KeyFingerprint
-    include Gitlab::Popen
+    attr_reader :key, :ssh_key
 
-    attr_accessor :key
+    # Unqualified MD5 fingerprint for compatibility
+    delegate :fingerprint, to: :ssh_key, allow_nil: true
 
     def initialize(key)
       @key = key
-    end
-
-    def fingerprint
-      cmd_status = 0
-      cmd_output = ''
-
-      Tempfile.open('gitlab_key_file') do |file|
-        file.puts key
-        file.rewind
-
-        cmd = []
-        cmd.push('ssh-keygen')
-        cmd.push('-E', 'md5') if explicit_fingerprint_algorithm?
-        cmd.push('-lf', file.path)
-
-        cmd_output, cmd_status = popen(cmd, '/tmp')
-      end
-
-      return nil unless cmd_status.zero?
 
-      # 16 hex bytes separated by ':', optionally starting with "MD5:"
-      fingerprint_matches = cmd_output.match(/(MD5:)?(?<fingerprint>(\h{2}:){15}\h{2})/)
-      return nil unless fingerprint_matches
-
-      fingerprint_matches[:fingerprint]
+      @ssh_key =
+        begin
+          Net::SSH::KeyFactory.load_data_public_key(key)
+        rescue Net::SSH::Exception, NotImplementedError
+        end
     end
 
-    private
-
-    def explicit_fingerprint_algorithm?
-      # OpenSSH 6.8 introduces a new default output format for fingerprints.
-      # Check the version and decide which command to use.
-
-      version_output, version_status = popen(%w(ssh -V))
-      return false unless version_status.zero?
+    def valid?
+      ssh_key.present?
+    end
 
-      version_matches = version_output.match(/OpenSSH_(?<major>\d+)\.(?<minor>\d+)/)
-      return false unless version_matches
+    def type
+      return unless valid?
 
-      version_info = Gitlab::VersionInfo.new(version_matches[:major].to_i, version_matches[:minor].to_i)
+      parts = ssh_key.ssh_type.split('-')
+      parts.shift if parts[0] == 'ssh'
 
-      required_version_info = Gitlab::VersionInfo.new(6, 8)
+      parts[0].upcase
+    end
 
-      version_info >= required_version_info
+    def bits
+      return unless valid?
+
+      case type
+      when 'RSA'
+        ssh_key.n.num_bits
+      when 'DSS', 'DSA'
+        ssh_key.p.num_bits
+      when 'ECDSA'
+        ssh_key.group.order.num_bits
+      when 'ED25519'
+        256
+      else
+        raise "Unsupported key type: #{type}"
+      end
     end
   end
 end

spec/controllers/projects/mirrors_controller_spec.rb

 require 'spec_helper'
 
 describe Projects::MirrorsController do
+  include ReactiveCachingHelpers
+
   describe 'setting up a remote mirror' do
     context 'when the current project is a mirror' do
       let(:project) { create(:project, :repository, :mirror) }
@@ -126,6 +128,63 @@ describe Projects::MirrorsController do
     end
   end
 
+  describe '#ssh_host_keys', use_clean_rails_memory_store_caching: true do
+    let(:project) { create(:project) }
+    let(:cache) { SshHostKey.new(project_id: project.id, url: "ssh://example.com:22") }
+
+    before do
+      sign_in(project.owner)
+    end
+
+    context 'invalid URL' do
+      it 'returns an error with a 400 response' do
+        do_get(project, 'INVALID URL')
+
+        expect(response).to have_http_status(400)
+        expect(json_response).to eq('message' => 'Invalid URL')
+      end
+    end
+
+    context 'no data in cache' do
+      it 'requests the cache to be filled and returns a 204 response' do
+        expect(ReactiveCachingWorker).to receive(:perform_async).with(cache.class, cache.id).at_least(:once)
+
+        do_get(project)
+
+        expect(response).to have_http_status(204)
+      end
+    end
+
+    context 'error in the cache' do
+      it 'returns the error with a 400 response' do
+        stub_reactive_cache(cache, error: 'An error')
+
+        do_get(project)
+
+        expect(response).to have_http_status(400)
+        expect(json_response).to eq('message' => 'An error')
+      end
+    end
+
+    context 'data in the cache' do
+      let(:ssh_key) { 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfuCHKVTjquxvt6CM6tdG4SLp1Btn/nOeHHE5UOzRdf' }
+      let(:ssh_fp) { { type: 'ED25519', bits: 256, fingerprint: '2e:65:6a:c8:cf:bf:b2:8b:9a:bd:6d:9f:11:5c:12:16', index: 0 } }
+
+      it 'returns the data with a 200 response' do
+        stub_reactive_cache(cache, known_hosts: ssh_key)
+
+        do_get(project)
+
+        expect(response).to have_http_status(200)
+        expect(json_response).to eq('known_hosts' => ssh_key, 'fingerprints' => [ssh_fp.stringify_keys])
+      end
+    end
+
+    def do_get(project, url = 'ssh://example.com')
+      get :ssh_host_keys, namespace_id: project.namespace, project_id: project, ssh_url: url
+    end
+  end
+
   def do_put(project, options)
     attrs = { namespace_id: project.namespace.to_param, project_id: project.to_param }
     attrs[:project] = options

spec/ee/spec/models/ssh_host_key_spec.rb

+require 'spec_helper'
+
+describe SshHostKey do
+  include ReactiveCachingHelpers
+
+  keys = [
+    SSHKeygen.generate,
+    SSHKeygen.generate
+  ]
+
+  # Purposefully ordered so that `sort` will make changes
+  known_hosts = <<-EOF.strip_heredoc
+    example.com #{keys[0]} git@localhost
+    @revoked other.example.com #{keys[1]} git@localhost
+  EOF
+
+  def stub_ssh_keyscan(args, status: true, stdout: "", stderr: "")
+    stdin = StringIO.new
+    stdout = double(:stdout, read: stdout)
+    stderr = double(:stderr, read: stderr)
+    wait_thr = double(:wait_thr, value: double(success?: status))
+
+    expect(Open3).to receive(:popen3).with({}, 'ssh-keyscan', *args).and_yield(stdin, stdout, stderr, wait_thr)
+
+    stdin
+  end
+
+  let(:project) { build(:project, :mirror) }
+
+  subject(:ssh_host_key) { described_class.new(project: project, url: 'ssh://example.com:2222') }
+
+  describe '#fingerprints', use_clean_rails_memory_store_caching: true do
+    it 'returns an array of indexed fingerprints when the cache is filled' do
+      stub_reactive_cache(ssh_host_key, known_hosts: known_hosts)
+
+      expected = keys
+        .map { |data| Gitlab::KeyFingerprint.new(data) }
+        .each_with_index
+        .map { |key, i| { bits: key.bits, fingerprint: key.fingerprint, type: key.type, index: i } } 
+
+      expect(ssh_host_key.fingerprints.as_json).to eq(expected)
+    end
+
+    it 'returns an empty array when the cache is empty' do
+      expect(ssh_host_key.fingerprints).to eq([])
+    end
+  end
+
+  describe '#calculate_reactive_cache' do
+    subject(:cache) { ssh_host_key.calculate_reactive_cache }
+
+    it 'writes the hostname to STDIN' do
+      stdin = stub_ssh_keyscan(%w[-T 5 -p 2222 -f-])
+
+      cache
+
+      expect(stdin.string).to eq("example.com\n")
+    end
+
+    context 'successful key scan' do
+      it 'stores the cleaned known_hosts data' do
+        stub_ssh_keyscan(%w[-T 5 -p 2222 -f-], stdout: "KEY 1\nKEY 1\n\n# comment\nKEY 2\n")
+
+        is_expected.to eq(known_hosts: "KEY 1\nKEY 2\n")
+      end
+    end
+
+    context 'failed key scan (exit code 1)' do
+      it 'returns a generic error' do
+        stub_ssh_keyscan(%w[-T 5 -p 2222 -f-], stdout: 'blarg', status: false)
+
+        is_expected.to eq(error: 'Failed to detect SSH host keys')
+      end
+    end
+
+    context 'failed key scan (exit code 0)' do
+      it 'returns a generic error' do
+        stub_ssh_keyscan(%w[-T 5 -p 2222 -f-], stderr: 'Unknown host')
+
+        is_expected.to eq(error: 'Failed to detect SSH host keys')
+      end
+    end
+  end
+end

spec/lib/gitlab/insecure_key_fingerprint_spec.rb

+require 'spec_helper'
+
+describe Gitlab::InsecureKeyFingerprint do
+  let(:key) do
+    'ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAIEAiPWx6WM4lhHNedGfBpPJNPpZ7yKu+dnn' \
+    '1SJejgt4596k6YjzGGphH2TUxwKzxcKDKKezwkpfnxPkSMkuEspGRt/aZZ9wa++Oi7Qk' \
+    'r8prgHc4soW6NUlfDzpvZK2H5E7eQaSeP3SAwGmQKUFHCddNaP0L+hM7zhFNzjFvpaMg' \
+    'Jw0='
+  end
+
+  let(:fingerprint) { "3f:a2:ee:de:b5:de:53:c3:aa:2f:9c:45:24:4c:47:7b" }
+
+  describe "#fingerprint" do
+    it "generates the key's fingerprint" do
+      expect(described_class.new(key.split[1]).fingerprint).to eq(fingerprint)
+    end
+  end
+end

spec/lib/gitlab/key_fingerprint_spec.rb

-require "spec_helper"
+require 'spec_helper'
 
-describe "SSH Keys" do
-  let(:key)         { "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAIEAiPWx6WM4lhHNedGfBpPJNPpZ7yKu+dnn1SJejgt4596k6YjzGGphH2TUxwKzxcKDKKezwkpfnxPkSMkuEspGRt/aZZ9wa++Oi7Qkr8prgHc4soW6NUlfDzpvZK2H5E7eQaSeP3SAwGmQKUFHCddNaP0L+hM7zhFNzjFvpaMgJw0=" }
-  let(:fingerprint) { "3f:a2:ee:de:b5:de:53:c3:aa:2f:9c:45:24:4c:47:7b" }
+describe Gitlab::KeyFingerprint, lib: true do
+  KEYS = {
+    rsa:
+      'example.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC5z65PwQ1GE6foJgwk' \
+      '9rmQi/glaXbUeVa5uvQpnZ3Z5+forcI7aTngh3aZ/H2UDP2L70TGy7kKNyp0J3a8/OdG' \
+      'Z08y5yi3JlbjFARO1NyoFEjw2H1SJxeJ43L6zmvTlu+hlK1jSAlidl7enS0ufTlzEEj4' \
+      'iJcuTPKdVzKRgZuTRVm9woWNVKqIrdRC0rJiTinERnfSAp/vNYERMuaoN4oJt8p/NEek' \
+      'rmFoDsQOsyDW5RAnCnjWUU+jFBKDpfkJQ1U2n6BjJewC9dl6ODK639l3yN4WOLZEk4tN' \
+      'UysfbGeF3rmMeflaD6O1Jplpv3YhwVGFNKa7fMq6k3Z0tszTJPYh',
+    ecdsa:
+      'example.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAI' \
+      'bmlzdHAyNTYAAABBBKTJy43NZzJSfNxpv/e2E6Zy3qoHoTQbmOsU5FEfpWfWa1MdTeXQ' \
+      'YvKOi+qz/1AaNx6BK421jGu74JCDJtiZWT8=',
+    ed25519:
+      '@revoked example.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfuCHKVTjq' \
+      'uxvt6CM6tdG4SLp1Btn/nOeHHE5UOzRdf',
+    dss:
+      'example.com ssh-dss AAAAB3NzaC1kc3MAAACBAP1/U4EddRIpUt9KnC7s5Of2EbdS' \
+      'PO9EAMMeP4C2USZpRV1AIlH7WT2NWPq/xfW6MPbLm1Vs14E7gB00b/JmYLdrmVClpJ+f' \
+      '6AR7ECLCT7up1/63xhv4O1fnxqimFQ8E+4P208UewwI1VBNaFpEy9nXzrith1yrv8iID' \
+      'GZ3RSAHHAAAAFQCXYFCPFSMLzLKSuYKi64QL8Fgc9QAAAIEA9+GghdabPd7LvKtcNrhX' \
+      'uXmUr7v6OuqC+VdMCz0HgmdRWVeOutRZT+ZxBxCBgLRJFnEj6EwoFhO3zwkyjMim4TwW' \
+      'eotUfI0o4KOuHiuzpnWRbqN/C/ohNWLx+2J6ASQ7zKTxvqhRkImog9/hWuWfBpKLZl6A' \
+      'e1UlZAFMO/7PSSoAAACBAJcQ4JODqhuGbXIEpqxetm7PWbdbCcr3y/GzIZ066pRovpL6' \
+      'qm3qCVIym4cyChxWwb8qlyCIi+YRUUWm1z/wiBYT2Vf3S4FXBnyymCkKEaV/EY7+jd4X' \
+      '1bXI58OD2u+bLCB/sInM4fGB8CZUIWT9nJH0Ve9jJUge2ms348/QOJ1+'
+  }.freeze
 
-  describe Gitlab::KeyFingerprint do
-    describe "#fingerprint" do
-      it "generates the key's fingerprint" do
-        expect(described_class.new(key).fingerprint).to eq(fingerprint)
+  MD5_FINGERPRINTS = {
+    rsa: '06:b2:8a:92:df:0e:11:2c:ca:7b:8f:a4:ba:6e:4b:fd',
+    ecdsa: '45:ff:5b:98:9a:b6:8a:41:13:c1:30:8b:09:5e:7b:4e', 
+    ed25519: '2e:65:6a:c8:cf:bf:b2:8b:9a:bd:6d:9f:11:5c:12:16', 
+    dss: '57:98:86:02:5f:9c:f4:9b:ad:5a:1e:51:92:0e:fd:2b'
+  }.freeze
+
+  BIT_COUNTS = {
+    rsa: 2048,
+    ecdsa: 256,
+    ed25519: 256,
+    dss: 1024
+  }.freeze
+
+  describe '#type' do
+    KEYS.each do |type, key|
+      it "calculates the type of #{type} keys" do
+        calculated_type = described_class.new(key).type
+
+        expect(calculated_type).to eq(type.to_s.upcase)
       end
     end
   end
 
-  describe Gitlab::InsecureKeyFingerprint do
-    describe "#fingerprint" do
-      it "generates the key's fingerprint" do
-        expect(Gitlab::InsecureKeyFingerprint.new(key.split[1]).fingerprint).to eq(fingerprint)
+  describe '#fingerprint' do
+    KEYS.each do |type, key|
+      it "calculates the MD5 fingerprint for #{type} keys" do
+        fp = described_class.new(key).fingerprint
+
+        expect(fp).to eq(MD5_FINGERPRINTS[type])
       end
     end
   end
+
+  describe '#bits' do
+    KEYS.each do |type, key|
+      it "calculates the number of bits in #{type} keys" do
+        bits = described_class.new(key).bits
+
+        expect(bits).to eq(BIT_COUNTS[type])
+      end
+    end
+  end
+
+  describe '#key' do
+    it 'carries the unmodified key data' do
+      key = described_class.new(KEYS[:rsa]).key
+
+      expect(key).to eq(KEYS[:rsa])
+    end
+  end
 end

spec/models/key_spec.rb

@@ -83,15 +83,6 @@ describe Key, :mailer do
       expect(build(:key)).to be_valid
     end
 
-    it 'rejects an unfingerprintable key that contains a space' do
-      key = build(:key)
-
-      # Not always the middle, but close enough
-      key.key = key.key[0..100] + ' ' + key.key[101..-1]
-
-      expect(key).not_to be_valid
-    end
-
     it 'accepts a key with newline charecters after stripping them' do
       key = build(:key)
       key.key = key.key.insert(100, "\n")
@@ -102,7 +93,6 @@ describe Key, :mailer do
     it 'rejects the unfingerprintable key (not a key)' do
       expect(build(:key, key: 'ssh-rsa an-invalid-key==')).not_to be_valid
     end
-    
   end
 
   context 'callbacks' do