Use structured logging for DB load balancer

As part of an effort to increase the observability of our database load
balancing, we are moving towards structured logging.

doc/administration/logs.md

@@ -338,3 +338,12 @@ installations from source.
 [repocheck]: repository_checks.md
 [Rack Attack]: ../security/rack_attack.md
 [Rate Limit]: ../user/admin_area/settings/rate_limits_on_raw_endpoints.md
+
+## `database_load_balancing.log`
+
+Introduced in GitLab 12.3 for observability of [Database Load
+Balancing](https://docs.gitlab.com/ee/administration/database_load_balancing.html)
+when enabled. This file lives in
+`/var/log/gitlab/gitlab-rails/database_load_balancing.log` for Omnibus GitLab
+packages or in `/home/git/gitlab/log/database_load_balancing.log` for
+installations from source.

ee/lib/gitlab/database/load_balancing.rb

 # frozen_string_literal: true
 
-# rubocop:disable GitlabSecurity/PublicSend
-
 module Gitlab
   module Database
     module LoadBalancing
       # The connection proxy to use for load balancing (if enabled).
       cattr_accessor :proxy
 
-      LOG_TAG = 'DB-LB'.freeze
-
       # The exceptions raised for connection errors.
       CONNECTION_ERRORS = if defined?(PG)
                             [
@@ -70,14 +66,6 @@ module Gitlab
         }
       end
 
-      # rubocop:disable Gitlab/RailsLogger
-      def self.log(level, message)
-        Rails.logger.tagged(LOG_TAG) do
-          Rails.logger.send(level, message)
-        end
-      end
-      # rubocop:enable Gitlab/RailsLogger
-
       def self.pool_size
         ActiveRecord::Base.configurations[Rails.env]['pool']
       end

ee/lib/gitlab/database/load_balancing/host.rb

@@ -54,7 +54,12 @@ module Gitlab
         end
 
         def offline!
-          LoadBalancing.log(:warn, "Marking host #{@host} as offline")
+          LoadBalancing::Logger.warn(
+            event: :host_offline,
+            message: 'Marking host as offline',
+            db_host: @host,
+            db_port: @port
+          )
 
           @online = false
           @pool.disconnect!
@@ -66,7 +71,14 @@ module Gitlab
 
           refresh_status
 
-          LoadBalancing.log(:info, "Host #{@host} came back online") if @online
+          if @online
+            LoadBalancing::Logger.info(
+              event: :host_online,
+              message: 'Host came back online',
+              db_host: @host,
+              db_port: @port
+            )
+          end
 
           @online
         rescue *CONNECTION_ERRORS

ee/lib/gitlab/database/load_balancing/load_balancer.rb

@@ -38,7 +38,19 @@ module Gitlab
                 #
                 # In this event we'll cycle through the secondaries at most 3
                 # times before using the primary instead.
-                if conflict_retried < @host_list.length * 3
+                will_retry = conflict_retried < @host_list.length * 3
+
+                LoadBalancing::Logger.warn(
+                  event: :host_query_conflict,
+                  message: 'Query conflict on host',
+                  conflict_retried: conflict_retried,
+                  will_retry: will_retry,
+                  db_host: host.host,
+                  db_port: host.port,
+                  host_list_length: @host_list.length
+                )
+
+                if will_retry
                   conflict_retried += 1
                   release_host
                 else
@@ -53,8 +65,12 @@ module Gitlab
             end
           end
 
-          LoadBalancing
-            .log(:warn, 'No secondaries were available, using primary instead')
+          LoadBalancing::Logger.warn(
+            event: :no_secondaries_available,
+            message: 'No secondaries were available, using primary instead',
+            conflict_retried: conflict_retried,
+            host_list_length: @host_list.length
+          )
 
           read_write(&block)
         end

ee/lib/gitlab/database/load_balancing/logger.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module LoadBalancing
+      class Logger < ::Gitlab::JsonLogger
+        def self.file_name_noext
+          'database_load_balancing'
+        end
+      end
+    end
+  end
+end

ee/spec/lib/gitlab/database/load_balancing_spec.rb

 require 'spec_helper'
 
 describe Gitlab::Database::LoadBalancing do
-  describe '.log' do
-    it 'logs a message' do
-      expect(Rails.logger).to receive(:info).with('boop')
-
-      described_class.log(:info, 'boop')
-    end
-  end
-
   describe '.configuration' do
     it 'returns a Hash' do
       config = { 'hosts' => %w(foo) }