Merge branch 'estimate-json-size-for-log-limited-array' into 'master'

Reduce object allocation when using LogLimitedArray

See merge request gitlab-org/gitlab!26257

lib/gitlab/utils/json_size_estimator.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Utils
+    # This class estimates the JSON blob byte size of a ruby object using as
+    # little allocations as possible.
+    # The estimation should be quite accurate when using simple objects.
+    #
+    # Example:
+    #
+    # Gitlab::Utils::JsonSizeEstimator.estimate(["a", { b: 12, c: nil }])
+    class JsonSizeEstimator
+      ARRAY_BRACKETS_SIZE = 2 # []
+      OBJECT_BRACKETS_SIZE = 2 # {}
+      DOUBLEQUOTE_SIZE = 2 # ""
+      COLON_SIZE = 1 # : character size from {"a": 1}
+      MINUS_SIGN_SIZE = 1 # - character size from -1
+      NULL_SIZE = 4 # null
+
+      class << self
+        # Returns: integer (number of bytes)
+        def estimate(object)
+          case object
+          when Hash
+            estimate_hash(object)
+          when Array
+            estimate_array(object)
+          when String
+            estimate_string(object)
+          when Integer
+            estimate_integer(object)
+          when Float
+            estimate_float(object)
+          when DateTime, Time
+            estimate_time(object)
+          when NilClass
+            NULL_SIZE
+          else
+            # might be incorrect, but #to_s is safe, #to_json might be disabled for some objects: User
+            estimate_string(object.to_s)
+          end
+        end
+
+        private
+
+        def estimate_hash(hash)
+          size = 0
+          item_count = 0
+
+          hash.each do |key, value|
+            item_count += 1
+
+            size += estimate(key.to_s) + COLON_SIZE + estimate(value)
+          end
+
+          size + OBJECT_BRACKETS_SIZE + comma_count(item_count)
+        end
+
+        def estimate_array(array)
+          size = 0
+          item_count = 0
+
+          array.each do |item|
+            item_count += 1
+
+            size += estimate(item)
+          end
+
+          size + ARRAY_BRACKETS_SIZE + comma_count(item_count)
+        end
+
+        def estimate_string(string)
+          string.bytesize + DOUBLEQUOTE_SIZE
+        end
+
+        def estimate_float(float)
+          float.to_s.bytesize
+        end
+
+        def estimate_integer(integer)
+          if integer > 0
+            integer_string_size(integer)
+          elsif integer < 0
+            integer_string_size(integer.abs) + MINUS_SIGN_SIZE
+          else # 0
+            1
+          end
+        end
+
+        def estimate_time(time)
+          time.to_json.size
+        end
+
+        def integer_string_size(integer)
+          Math.log10(integer).floor + 1
+        end
+
+        def comma_count(item_count)
+          item_count == 0 ? 0 : item_count - 1
+        end
+      end
+    end
+  end
+end

lib/gitlab/utils/log_limited_array.rb

@@ -13,7 +13,7 @@ module Gitlab
 
         total_length = 0
         limited_array = array.take_while do |arg|
-          total_length += arg.to_json.length
+          total_length += JsonSizeEstimator.estimate(arg)
 
           total_length <= MAXIMUM_ARRAY_LENGTH
         end

spec/lib/gitlab/lograge/custom_options_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Lograge::CustomOptions do
+  describe '.call' do
+    let(:params) do
+      {
+        'controller' => 'ApplicationController',
+        'action' => 'show',
+        'format' => 'html',
+        'a' => 'b'
+      }
+    end
+
+    let(:event) do
+      ActiveSupport::Notifications::Event.new(
+        'test',
+        1,
+        2,
+        'transaction_id',
+        { params: params, user_id: 'test' }
+      )
+    end
+
+    subject { described_class.call(event) }
+
+    it 'ignores some parameters' do
+      param_keys = subject[:params].map { |param| param[:key] }
+
+      expect(param_keys).not_to include(*described_class::IGNORE_PARAMS)
+    end
+
+    it 'formats the parameters' do
+      expect(subject[:params]).to eq([{ key: 'a', value: 'b' }])
+    end
+
+    it 'adds the current time' do
+      travel_to(5.days.ago) do
+        expected_time = Time.now.utc.iso8601(3)
+
+        expect(subject[:time]).to eq(expected_time)
+      end
+    end
+
+    it 'adds the user id' do
+      expect(subject[:user_id]).to eq('test')
+    end
+  end
+end

spec/lib/gitlab/utils/json_size_estimator_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Utils::JsonSizeEstimator do
+  RSpec::Matchers.define :match_json_bytesize_of do |expected|
+    match do |actual|
+      actual == expected.to_json.bytesize
+    end
+  end
+
+  def estimate(object)
+    described_class.estimate(object)
+  end
+
+  [
+    [],
+    [[[[]]]],
+    [1, "str", 3.14, ["str", { a: -1 }]],
+    {},
+    { a: {} },
+    { a: { b: { c: [1, 2, 3], e: Time.now, f: nil } } },
+    { 100 => 500 },
+    { '狸' => '狸' },
+    nil
+  ].each do |example|
+    it { expect(estimate(example)).to match_json_bytesize_of(example) }
+  end
+
+  it 'calls #to_s on unknown object' do
+    klass = Class.new do
+      def to_s
+        'hello'
+      end
+    end
+
+    expect(estimate(klass.new)).to match_json_bytesize_of(klass.new.to_s) # "hello"
+  end
+end