Don't generate MD5 fingerprint in FIPS mode

Contributes to https://gitlab.com/gitlab-org/gitlab/-/issues/336421

**Problem**

OpenSSL library raises an exception for users with FIPS mode on.

**Solution**

Don't generate MD5 fingerprints for SSH keys when FIPS mode detected.

Changelog: changed

app/models/key.rb

@@ -26,7 +26,13 @@ class Key < ApplicationRecord
 
   validates :fingerprint,
     uniqueness: true,
-    presence: { message: 'cannot be generated' }
+    presence: { message: 'cannot be generated' },
+    unless: -> { Gitlab::FIPS.enabled? }
+
+  validates :fingerprint_sha256,
+    uniqueness: true,
+    presence: { message: 'cannot be generated' },
+    if: -> { Gitlab::FIPS.enabled? }
 
   validate :key_meets_restrictions
 
@@ -129,7 +135,7 @@ class Key < ApplicationRecord
 
     return unless public_key.valid?
 
-    self.fingerprint_md5 = public_key.fingerprint
+    self.fingerprint_md5 = public_key.fingerprint unless Gitlab::FIPS.enabled?
     self.fingerprint_sha256 = public_key.fingerprint_sha256.gsub("SHA256:", "")
   end
 

spec/controllers/profiles/keys_controller_spec.rb

@@ -17,7 +17,25 @@ RSpec.describe Profiles::KeysController do
         post :create, params: { key: build(:key, expires_at: expires_at).attributes }
       end.to change { Key.count }.by(1)
 
-      expect(Key.last.expires_at).to be_like_time(expires_at)
+      key = Key.last
+      expect(key.expires_at).to be_like_time(expires_at)
+      expect(key.fingerprint_md5).to be_present
+      expect(key.fingerprint_sha256).to be_present
+    end
+
+    context 'with FIPS mode', :fips_mode do
+      it 'creates a new key without MD5 fingerprint' do
+        expires_at = 3.days.from_now
+
+        expect do
+          post :create, params: { key: build(:rsa_key_4096, expires_at: expires_at).attributes }
+        end.to change { Key.count }.by(1)
+
+        key = Key.last
+        expect(key.expires_at).to be_like_time(expires_at)
+        expect(key.fingerprint_md5).to be_nil
+        expect(key.fingerprint_sha256).to be_present
+      end
     end
   end
 end

spec/models/key_spec.rb

@@ -123,25 +123,55 @@ RSpec.describe Key, :mailer do
     end
   end
 
-  context "validation of uniqueness (based on fingerprint uniqueness)" do
+  context 'validation of uniqueness (based on fingerprint uniqueness)' do
     let(:user) { create(:user) }
 
-    it "accepts the key once" do
-      expect(build(:key, user: user)).to be_valid
+    shared_examples 'fingerprint uniqueness' do
+      it 'accepts the key once' do
+        expect(build(:rsa_key_4096, user: user)).to be_valid
+      end
+
+      it 'does not accept the exact same key twice' do
+        first_key = create(:rsa_key_4096, user: user)
+
+        expect(build(:key, user: user, key: first_key.key)).not_to be_valid
+      end
+
+      it 'does not accept a duplicate key with a different comment' do
+        first_key = create(:rsa_key_4096, user: user)
+        duplicate = build(:key, user: user, key: first_key.key)
+        duplicate.key << ' extra comment'
+
+        expect(duplicate).not_to be_valid
+      end
     end
 
-    it "does not accept the exact same key twice" do
-      first_key = create(:key, user: user)
+    context 'with FIPS mode off' do
+      it_behaves_like 'fingerprint uniqueness'
+    end
 
-      expect(build(:key, user: user, key: first_key.key)).not_to be_valid
+    context 'with FIPS mode', :fips_mode do
+      it_behaves_like 'fingerprint uniqueness'
     end
+  end
 
-    it "does not accept a duplicate key with a different comment" do
-      first_key = create(:key, user: user)
-      duplicate = build(:key, user: user, key: first_key.key)
-      duplicate.key << ' extra comment'
+  context 'fingerprint generation' do
+    it 'generates both md5 and sha256 fingerprints' do
+      key = build(:rsa_key_4096)
+
+      expect(key).to be_valid
+      expect(key.fingerprint).to be_kind_of(String)
+      expect(key.fingerprint_sha256).to be_kind_of(String)
+    end
 
-      expect(duplicate).not_to be_valid
+    context 'with FIPS mode', :fips_mode do
+      it 'generates only sha256 fingerprint' do
+        key = build(:rsa_key_4096)
+
+        expect(key).to be_valid
+        expect(key.fingerprint).to be_nil
+        expect(key.fingerprint_sha256).to be_kind_of(String)
+      end
     end
   end