Merge branch 'gitlab-json-multi' into 'master'

Enable oj in Gitlab::Json

See merge request gitlab-org/gitlab!35527

Gemfile

@@ -500,3 +500,5 @@ gem 'valid_email', '~> 0.1'
 # JSON
 gem 'json', '~> 2.3.0'
 gem 'json-schema', '~> 2.8.0'
+gem 'oj', '~> 3.10.6'
+gem 'multi_json', '~> 1.14.1'

Gemfile.lock

@@ -687,6 +687,7 @@ GEM
     octokit (4.15.0)
       faraday (>= 0.9)
       sawyer (~> 0.8.0, >= 0.5.3)
+    oj (3.10.6)
     omniauth (1.9.0)
       hashie (>= 3.4.6, < 3.7.0)
       rack (>= 1.6.2, < 3)
@@ -1312,6 +1313,7 @@ DEPENDENCIES
   mimemagic (~> 0.3.2)
   mini_magick
   minitest (~> 5.11.0)
+  multi_json (~> 1.14.1)
   nakayoshi_fork (~> 0.0.4)
   net-ldap
   net-ntp
@@ -1319,6 +1321,7 @@ DEPENDENCIES
   nokogiri (~> 1.10.9)
   oauth2 (~> 1.4)
   octokit (~> 4.15)
+  oj (~> 3.10.6)
   omniauth (~> 1.8)
   omniauth-auth0 (~> 2.0.0)
   omniauth-authentiq (~> 0.3.3)

changelogs/unreleased/swap-to-oj.yml

+---
+title: Add oj gem for faster JSON
+merge_request: 35527
+author:
+type: performance

config/initializers/multi_json.rb

+# frozen_string_literal: true
+
+# Explicitly set the JSON adapter used by MultiJson
+# Currently we want this to default to the existing json gem
+MultiJson.use(:json_gem)

config/initializers/oj.rb

+# frozen_string_literal: true
+
+# Ensure Oj runs in json-gem compatibility mode by default
+Oj.default_options = { mode: :rails }

lib/gitlab/json.rb

 # frozen_string_literal: true
 
+# This is a GitLab-specific JSON interface. You should use this instead
+# of using `JSON` directly. This allows us to swap the adapter and handle
+# legacy issues.
+
 module Gitlab
   module Json
     INVALID_LEGACY_TYPES = [String, TrueClass, FalseClass].freeze
 
     class << self
-      def parse(string, *args, **named_args)
-        legacy_mode = legacy_mode_enabled?(named_args.delete(:legacy_mode))
-        data = adapter.parse(string, *args, **named_args)
+      # Parse a string and convert it to a Ruby object
+      #
+      # @param string [String] the JSON string to convert to Ruby objects
+      # @param opts [Hash] an options hash in the standard JSON gem format
+      # @return [Boolean, String, Array, Hash]
+      # @raise [JSON::ParserError] raised if parsing fails
+      def parse(string, opts = {})
+        # First we should ensure this really is a string, not some other
+        # type which purports to be a string. This handles some legacy
+        # usage of the JSON class.
+        string = string.to_s unless string.is_a?(String)
+
+        legacy_mode = legacy_mode_enabled?(opts.delete(:legacy_mode))
+        data = adapter_load(string, opts)
 
         handle_legacy_mode!(data) if legacy_mode
 
         data
       end
 
-      def parse!(string, *args, **named_args)
-        legacy_mode = legacy_mode_enabled?(named_args.delete(:legacy_mode))
-        data = adapter.parse!(string, *args, **named_args)
-
-        handle_legacy_mode!(data) if legacy_mode
-
-        data
-      end
+      alias_method :parse!, :parse
 
-      def dump(*args)
-        adapter.dump(*args)
+      # Take a Ruby object and convert it to a string
+      #
+      # @param object [Boolean, String, Array, Hash, Object] depending on the adapter this can be a variety of types
+      # @param opts [Hash] an options hash in the standard JSON gem format
+      # @return [String]
+      def dump(object, opts = {})
+        adapter_dump(object, opts)
       end
 
+      # Legacy method used in our codebase that might just be an alias for `parse`.
+      # Will be updated to use our `parse` method.
       def generate(*args)
-        adapter.generate(*args)
+        ::JSON.generate(*args)
       end
 
+      # Generates a JSON string and formats it nicely.
+      # Varies depending on adapter and will be updated to use our methods.
       def pretty_generate(*args)
-        adapter.pretty_generate(*args)
+        ::JSON.pretty_generate(*args)
       end
 
       private
 
-      def adapter
-        ::JSON
+      # Convert JSON string into Ruby through toggleable adapters.
+      #
+      # Must rescue adapter-specific errors and return `parser_error`, and
+      # must also standardize the options hash to support each adapter as
+      # they all take different options.
+      #
+      # @param string [String] the JSON string to convert to Ruby objects
+      # @param opts [Hash] an options hash in the standard JSON gem format
+      # @return [Boolean, String, Array, Hash]
+      # @raise [JSON::ParserError]
+      def adapter_load(string, opts = {})
+        opts = standardize_opts(opts)
+
+        if enable_oj?
+          Oj.load(string, opts)
+        else
+          ::JSON.parse(string, opts)
+        end
+      rescue Oj::ParseError, Encoding::UndefinedConversionError => ex
+        raise parser_error.new(ex)
       end
 
+      # Convert Ruby object to JSON string through toggleable adapters.
+      #
+      # @param object [Boolean, String, Array, Hash, Object] depending on the adapter this can be a variety of types
+      # @param opts [Hash] an options hash in the standard JSON gem format
+      # @return [String]
+      def adapter_dump(thing, opts = {})
+        opts = standardize_opts(opts)
+
+        if enable_oj?
+          Oj.dump(thing, opts)
+        else
+          ::JSON.dump(thing, opts)
+        end
+      end
+
+      # Take a JSON standard options hash and standardize it to work across adapters
+      # An example of this is Oj taking :symbol_keys instead of :symbolize_names
+      #
+      # @param opts [Hash]
+      # @return [Hash]
+      def standardize_opts(opts = {})
+        if enable_oj?
+          opts[:mode] = :rails
+          opts[:symbol_keys] = opts[:symbolize_keys] || opts[:symbolize_names]
+        end
+
+        opts
+      end
+
+      # The standard parser error we should be returning. Defined in a method
+      # so we can potentially override it later.
+      #
+      # @return [JSON::ParserError]
       def parser_error
         ::JSON::ParserError
       end
 
+      # @param [Nil, Boolean] an extracted :legacy_mode key from the opts hash
+      # @return [Boolean]
       def legacy_mode_enabled?(arg_value)
         arg_value.nil? ? false : arg_value
       end
 
+      # If legacy mode is enabled, we need to raise an error depending on the values
+      # provided in the string. This will be deprecated.
+      #
+      # @param data [Boolean, String, Array, Hash, Object]
+      # @return [Boolean, String, Array, Hash, Object]
+      # @raise [JSON::ParserError]
       def handle_legacy_mode!(data)
         return data unless Feature.enabled?(:json_wrapper_legacy_mode, default_enabled: true)
 
         raise parser_error if INVALID_LEGACY_TYPES.any? { |type| data.is_a?(type) }
       end
+
+      # @return [Boolean]
+      def enable_oj?
+        Feature.enabled?(:oj_json, default_enabled: true)
+      end
     end
   end
 end

lib/gitlab/json_logger.rb

@@ -19,7 +19,7 @@ module Gitlab
         data.merge!(message)
       end
 
-      data.to_json + "\n"
+      Gitlab::Json.dump(data) + "\n"
     end
   end
 end

spec/lib/gitlab/ci/parsers/accessibility/pa11y_spec.rb

 # frozen_string_literal: true
 
-require 'fast_spec_helper'
+require 'spec_helper'
 
 RSpec.describe Gitlab::Ci::Parsers::Accessibility::Pa11y do
   describe '#parse!' do
@@ -108,7 +108,7 @@ RSpec.describe Gitlab::Ci::Parsers::Accessibility::Pa11y do
       it "sets error_message" do
         expect { subject }.not_to raise_error
 
-        expect(accessibility_report.error_message).to include('Pa11y parsing failed')
+        expect(accessibility_report.error_message).to include('JSON parsing failed')
         expect(accessibility_report.errors_count).to eq(0)
         expect(accessibility_report.passes_count).to eq(0)
         expect(accessibility_report.scans_count).to eq(0)

spec/lib/gitlab/phabricator_import/conduit/response_spec.rb

@@ -30,7 +30,7 @@ RSpec.describe Gitlab::PhabricatorImport::Conduit::Response do
                        body: 'This is no JSON')
 
       expect { described_class.parse!(fake_response) }
-        .to raise_error(Gitlab::PhabricatorImport::Conduit::ResponseError, /unexpected token at/)
+        .to raise_error(Gitlab::PhabricatorImport::Conduit::ResponseError, /unexpected character/)
     end
 
     it 'returns a parsed response for valid input' do