Basic setup for an RSpec based benchmark suite

This benchmark suite uses benchmark-ips
(https://github.com/evanphx/benchmark-ips) behind the scenes. Specs can
be turned into benchmark specs by setting "benchmark" to "true" in the
top-level describe block like so:

    describe SomeClass, benchmark: true do

    end

Writing benchmarks can be done using custom RSpec matchers, for example:

    describe MaruTheCat, benchmark: true do
      describe '#jump_in_box' do
        it 'should run 1000 iterations per second' do
          maru = described_class.new

          expect { maru.jump_in_box }.to iterate_per_second(1000)
        end
      end
    end

By default the "iterate_per_second" expectation requires a standard
deviation under 30% (this is just an arbitrary default for now). You can
change this by chaining "with_maximum_stddev" on the expectation:

    expect { maru.jump_in_box }.to iterate_per_second(1000)
      .with_maximum_stddev(10)

This will change the expectation to require a maximum deviation of 10%.

Alternatively you can use the it block style to write specs:

    describe MaruTheCat, benchmark: true do
      describe '#jump_in_box' do
        subject { -> { described_class.new } }

        it { is_expected.to iterate_per_second(1000) }
      end
    end

Because "iterate_per_second" operates on a block, opposed to a static
value, the "subject" method must return a Proc. This looks a bit goofy
but I have been unable to find a nice way around this.

.gitlab-ci.yml

@@ -24,6 +24,13 @@ spec:api:
     - ruby
     - mysql
 
+spec:benchmark:
+  script:
+    - RAILS_ENV=test bundle exec rake spec:benchmark
+  tags:
+    - ruby
+    - mysql
+
 spec:other:
   script:
     - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:other

lib/tasks/spec.rake

@@ -19,11 +19,20 @@ namespace :spec do
     run_commands(cmds)
   end
 
+  desc 'GitLab | Rspec | Run benchmark specs'
+  task :benchmark do
+    cmds = [
+      %W(rake gitlab:setup),
+      %W(rspec spec --tag @benchmark)
+    ]
+    run_commands(cmds)
+  end
+
   desc 'GitLab | Rspec | Run other specs'
   task :other do
     cmds = [
       %W(rake gitlab:setup),
-      %W(rspec spec --tag ~@api --tag ~@feature)
+      %W(rspec spec --tag ~@api --tag ~@feature --tag ~@benchmark)
     ]
     run_commands(cmds)
   end

spec/benchmarks/models/user_spec.rb

+require 'spec_helper'
+
+describe User, benchmark: true do
+  describe '.by_login' do
+    before do
+      %w{Alice Bob Eve}.each do |name|
+        create(:user,
+               email: "#{name}@gitlab.com",
+               username: name,
+               name: name)
+      end
+    end
+
+    let(:iterations) { 1000 }
+
+    describe 'using a capitalized username' do
+      subject { -> { User.by_login('Alice') } }
+
+      it { is_expected.to iterate_per_second(iterations) }
+    end
+
+    describe 'using a lowercase username' do
+      subject { -> { User.by_login('alice') } }
+
+      it { is_expected.to iterate_per_second(iterations) }
+    end
+
+    describe 'using a capitalized Email address' do
+      subject { -> { User.by_login('Alice@gitlab.com') } }
+
+      it { is_expected.to iterate_per_second(iterations) }
+    end
+
+    describe 'using a lowercase Email address' do
+      subject { -> { User.by_login('alice@gitlab.com') } }
+
+      it { is_expected.to iterate_per_second(iterations) }
+    end
+  end
+end

spec/spec_helper.rb

@@ -14,6 +14,7 @@ require File.expand_path("../../config/environment", __FILE__)
 require 'rspec/rails'
 require 'shoulda/matchers'
 require 'sidekiq/testing/inline'
+require 'benchmark/ips'
 
 # Requires supporting ruby files with custom matchers and macros, etc,
 # in spec/support/ and its subdirectories.
@@ -32,7 +33,7 @@ RSpec.configure do |config|
   config.include TestEnv
   config.include StubGitlabCalls
   config.include StubGitlabData
-
+  config.include BenchmarkMatchers, benchmark: true
 
   config.infer_spec_type_from_file_location!
   config.raise_errors_for_deprecations!

spec/support/matchers/benchmark_matchers.rb

+module BenchmarkMatchers
+  extend RSpec::Matchers::DSL
+
+  matcher :iterate_per_second do |min_iterations|
+    supports_block_expectations
+
+    match do |block|
+      @max_stddev ||= 30
+
+      @entry = benchmark(&block)
+
+      expect(@entry.ips).to be >= min_iterations
+      expect(@entry.stddev_percentage).to be <= @max_stddev
+    end
+
+    chain :with_maximum_stddev do |value|
+      @max_stddev = value
+    end
+
+    description do
+      "run at least #{min_iterations} iterations per second"
+    end
+
+    failure_message do
+      ips    = @entry.ips.round(2)
+      stddev = @entry.stddev_percentage.round(2)
+
+      "expected at least #{min_iterations} iterations per second " \
+        "with a maximum stddev of #{@max_stddev}%, instead of " \
+        "#{ips} iterations per second with a stddev of #{stddev}%"
+    end
+  end
+
+  # Benchmarks the given block and returns a Benchmark::IPS::Report::Entry.
+  def benchmark(&block)
+    report = Benchmark.ips(quiet: true) do |bench|
+      bench.report(&block)
+    end
+
+    report.entries[0]
+  end
+end