Merge branch 'bvl-sidekiq-logger-structured' into 'master'

Add a separate logger for Sidekiq clients

See merge request gitlab-org/gitlab!26586

app/workers/namespaces/root_statistics_worker.rb

@@ -16,13 +16,7 @@ module Namespaces
 
       namespace.aggregation_schedule.destroy
     rescue ::Namespaces::StatisticsRefresherService::RefresherError, ActiveRecord::RecordNotFound => ex
-      log_error(namespace.full_path, ex.message) if namespace
-    end
-
-    private
-
-    def log_error(namespace_path, error_message)
-      Gitlab::SidekiqLogger.error("Namespace statistics can't be updated for #{namespace_path}: #{error_message}")
+      Gitlab::ErrorTracking.track_exception(ex, namespace_id: namespace_id, namespace: namespace&.full_path)
     end
   end
 end

app/workers/namespaces/schedule_aggregation_worker.rb

@@ -16,8 +16,8 @@ module Namespaces
       return if root_ancestor.aggregation_scheduled?
 
       Namespace::AggregationSchedule.safe_find_or_create_by!(namespace_id: root_ancestor.id)
-    rescue ActiveRecord::RecordNotFound
-      log_error(namespace_id)
+    rescue ActiveRecord::RecordNotFound => ex
+      Gitlab::ErrorTracking.track_exception(ex, namespace_id: namespace_id)
     end
 
     private
@@ -34,9 +34,5 @@ module Namespaces
 
       Namespace::AggregationSchedule.table_exists?
     end
-
-    def log_error(root_ancestor_id)
-      Gitlab::SidekiqLogger.error("Namespace can't be scheduled for aggregation: #{root_ancestor_id} does not exist")
-    end
   end
 end

config/initializers/sidekiq.rb

@@ -108,6 +108,11 @@ end
 
 Sidekiq.configure_client do |config|
   config.redis = queues_config_hash
+  # We only need to do this for other clients. If Sidekiq-server is the
+  # client scheduling jobs, we have access to the regular sidekiq logger that
+  # writes to STDOUT
+  Sidekiq.logger = Gitlab::SidekiqLogging::ClientLogger.build
+  Sidekiq.logger.formatter = Gitlab::SidekiqLogging::JSONFormatter.new if enable_json_logs
 
   config.client_middleware(&Gitlab::SidekiqMiddleware.client_configurator)
 end

doc/administration/logs.md

@@ -400,6 +400,21 @@ For source installations, edit the `gitlab.yml` and set the Sidekiq
     log_format: json
 ```
 
+## `sidekiq_client.log`
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26586) in GitLab 12.9.
+
+This file lives in `/var/log/gitlab/gitlab-rails/sidekiq_client.log` for
+Omnibus GitLab packages or in `/home/git/gitlab/log/sidekiq_client.log` for
+installations from source.
+
+This file contains logging information about jobs before they are start
+being processed by Sidekiq, for example before being enqueued.
+
+This logfile follows the same structure as
+[`sidekiq.log`](#sidekiqlog), so it will be structured as JSON if
+you've configured this for Sidekiq as mentioned above.
+
 ## `gitlab-shell.log`
 
 This file lives in `/var/log/gitlab/gitaly/gitlab-shell.log` for

lib/gitlab/sidekiq_logging/client_logger.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqLogging
+    class ClientLogger < Gitlab::Logger
+      def self.file_name_noext
+        'sidekiq_client'
+      end
+    end
+  end
+end

spec/workers/namespaces/root_statistics_worker_spec.rb

@@ -42,7 +42,7 @@ describe Namespaces::RootStatisticsWorker, '#perform' do
         allow_any_instance_of(Namespace::AggregationSchedule)
           .to receive(:schedule_root_storage_statistics).and_return(nil)
 
-        expect(Gitlab::SidekiqLogger).to receive(:error).once
+        expect(Gitlab::ErrorTracking).to receive(:track_exception).once
 
         worker.perform(group.id)
       end

spec/workers/namespaces/schedule_aggregation_worker_spec.rb

@@ -48,7 +48,7 @@ describe Namespaces::ScheduleAggregationWorker, '#perform', :clean_gitlab_redis_
 
   context 'when namespace does not exist' do
     it 'logs the error' do
-      expect(Gitlab::SidekiqLogger).to receive(:error).once
+      expect(Gitlab::ErrorTracking).to receive(:track_exception).once
 
       worker.perform(12345)
     end