Start versioning cached markdown fields

app/models/concerns/cache_markdown_field.rb

@@ -8,6 +8,14 @@
 #
 # Corresponding foo_html, bar_html and baz_html fields should exist.
 module CacheMarkdownField
+  extend ActiveSupport::Concern
+
+  # Increment this number every time the renderer changes its output
+  CACHE_VERSION = 1
+
+  # changes to these attributes cause the cache to be invalidates
+  INVALIDATED_BY = %w[author project].freeze
+
   # Knows about the relationship between markdown and html field names, and
   # stores the rendering contexts for the latter
   class FieldData
@@ -34,34 +42,67 @@ module CacheMarkdownField
     false
   end
 
-  extend ActiveSupport::Concern
+  # Returns the default Banzai render context for the cached markdown field.
+  def banzai_render_context(field)
+    raise ArgumentError.new("Unknown field: #{field.inspect}") unless
+      cached_markdown_fields.markdown_fields.include?(field)
 
-  included do
-    cattr_reader :cached_markdown_fields do
-      FieldData.new
-    end
+    # Always include a project key, or Banzai complains
+    project = self.project if self.respond_to?(:project)
+    context = cached_markdown_fields[field].merge(project: project)
+
+    # Banzai is less strict about authors, so don't always have an author key
+    context[:author] = self.author if self.respond_to?(:author)
 
-    # Returns the default Banzai render context for the cached markdown field.
-    def banzai_render_context(field)
-      raise ArgumentError.new("Unknown field: #{field.inspect}") unless
-        cached_markdown_fields.markdown_fields.include?(field)
+    context
+  end
 
-      # Always include a project key, or Banzai complains
-      project = self.project if self.respond_to?(:project)
-      context = cached_markdown_fields[field].merge(project: project)
+  # Update every column in a row if any one is invalidated, as we only store
+  # one version per row
+  def refresh_markdown_cache!(do_update: false)
+    options = { skip_project_check: skip_project_check? }
 
-      # Banzai is less strict about authors, so don't always have an author key
-      context[:author] = self.author if self.respond_to?(:author)
+    updates = cached_markdown_fields.markdown_fields.map do |markdown_field|
+      [
+        cached_markdown_fields.html_field(markdown_field),
+        Banzai::Renderer.cacheless_render_field(self, markdown_field, options)
+      ]
+    end.to_h
+    updates['cached_markdown_version'] = CacheMarkdownField::CACHE_VERSION
 
-      context
-    end
+    updates.each {|html_field, data| write_attribute(html_field, data) }
+
+    update_columns(updates) if persisted? && do_update
+  end
+
+  def cached_html_up_to_date?(markdown_field)
+    html_field = cached_markdown_fields.html_field(markdown_field)
+
+    markdown_changed = attribute_changed?(markdown_field) || false
+    html_changed = attribute_changed?(html_field) || false
 
-    # Allow callers to look up the cache field name, rather than hardcoding it
-    def markdown_cache_field_for(field)
-      raise ArgumentError.new("Unknown field: #{field}") unless
-        cached_markdown_fields.markdown_fields.include?(field)
+    CacheMarkdownField::CACHE_VERSION == cached_markdown_version &&
+      (html_changed || markdown_changed == html_changed)
+  end
+
+  def invalidated_markdown_cache?
+    cached_markdown_fields.html_fields.any? {|html_field| attribute_invalidated?(html_field) }
+  end
+
+  def attribute_invalidated?(attr)
+    __send__("#{attr}_invalidated?")
+  end
+
+  def cached_html_for(markdown_field)
+    raise ArgumentError.new("Unknown field: #{field}") unless
+      cached_markdown_fields.markdown_fields.include?(markdown_field)
+
+    __send__(cached_markdown_fields.html_field(markdown_field))
+  end
 
-      cached_markdown_fields.html_field(field)
+  included do
+    cattr_reader :cached_markdown_fields do
+      FieldData.new
     end
 
     # Always exclude _html fields from attributes (including serialization).
@@ -70,12 +111,16 @@ module CacheMarkdownField
     def attributes
       attrs = attributes_before_markdown_cache
 
+      attrs.delete('cached_markdown_version')
+
       cached_markdown_fields.html_fields.each do |field|
         attrs.delete(field)
       end
 
       attrs
     end
+
+    before_save :refresh_markdown_cache!, if: :invalidated_markdown_cache?
   end
 
   class_methods do
@@ -88,25 +133,15 @@ module CacheMarkdownField
       cached_markdown_fields[markdown_field] = context
 
       html_field = cached_markdown_fields.html_field(markdown_field)
-      cache_method = "#{markdown_field}_cache_refresh".to_sym
       invalidation_method = "#{html_field}_invalidated?".to_sym
 
-      define_method(cache_method) do
-        options = { skip_project_check: skip_project_check? }
-        html = Banzai::Renderer.cacheless_render_field(self, markdown_field, options)
-        __send__("#{html_field}=", html)
-        true
-      end
-
       # The HTML becomes invalid if any dependent fields change. For now, assume
       # author and project invalidate the cache in all circumstances.
       define_method(invalidation_method) do
         changed_fields = changed_attributes.keys
-        invalidations = changed_fields & [markdown_field.to_s, "author", "project"]
-        !invalidations.empty?
+        invalidations = changed_fields & [markdown_field.to_s, *INVALIDATED_BY]
+        !invalidations.empty? || !cached_html_up_to_date?(markdown_field)
       end
-
-      before_save cache_method, if: invalidation_method
     end
   end
 end

changelogs/unreleased/30672-versioned-markdown-cache.yml

+---
+title: Replace rake cache:clear:db with an automatic mechanism
+merge_request: 10597
+author:

db/migrate/20170410133135_add_version_field_to_markdown_cache.rb

+class AddVersionFieldToMarkdownCache < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    %i[
+      abuse_reports
+      appearances
+      application_settings
+      broadcast_messages
+      issues
+      labels
+      merge_requests
+      milestones
+      namespaces
+      notes
+      projects
+      releases
+      snippets
+    ].each do |table|
+      add_column table, :cached_markdown_version, :integer, limit: 4
+    end
+  end
+end

db/schema.rb

@@ -24,6 +24,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do
     t.datetime "created_at"
     t.datetime "updated_at"
     t.text "message_html"
+    t.integer "cached_markdown_version"
   end
 
   create_table "appearances", force: :cascade do |t|
@@ -34,6 +35,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.text "description_html"
+    t.integer "cached_markdown_version"
   end
 
   create_table "application_settings", force: :cascade do |t|
@@ -116,6 +118,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do
     t.integer "unique_ips_limit_time_window"
     t.boolean "unique_ips_limit_enabled", default: false, null: false
     t.decimal "polling_interval_multiplier", default: 1.0, null: false
+    t.integer "cached_markdown_version"
     t.boolean "usage_ping_enabled", default: true, null: false
     t.string "uuid"
   end
@@ -161,6 +164,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do
     t.string "color"
     t.string "font"
     t.text "message_html"
+    t.integer "cached_markdown_version"
   end
 
   create_table "chat_names", force: :cascade do |t|
@@ -479,6 +483,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do
     t.integer "time_estimate"
     t.integer "relative_position"
     t.datetime "closed_at"
+    t.integer "cached_markdown_version"
   end
 
   add_index "issues", ["assignee_id"], name: "index_issues_on_assignee_id", using: :btree
@@ -543,6 +548,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do
     t.text "description_html"
     t.string "type"
     t.integer "group_id"
+    t.integer "cached_markdown_version"
   end
 
   add_index "labels", ["group_id", "project_id", "title"], name: "index_labels_on_group_id_and_project_id_and_title", unique: true, using: :btree
@@ -663,6 +669,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do
     t.text "title_html"
     t.text "description_html"
     t.integer "time_estimate"
+    t.integer "cached_markdown_version"
   end
 
   add_index "merge_requests", ["assignee_id"], name: "index_merge_requests_on_assignee_id", using: :btree
@@ -700,6 +707,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do
     t.text "title_html"
     t.text "description_html"
     t.date "start_date"
+    t.integer "cached_markdown_version"
   end
 
   add_index "milestones", ["description"], name: "index_milestones_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"}
@@ -726,6 +734,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do
     t.integer "parent_id"
     t.boolean "require_two_factor_authentication", default: false, null: false
     t.integer "two_factor_grace_period", default: 48, null: false
+    t.integer "cached_markdown_version"
   end
 
   add_index "namespaces", ["created_at"], name: "index_namespaces_on_created_at", using: :btree
@@ -760,6 +769,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do
     t.integer "resolved_by_id"
     t.string "discussion_id"
     t.text "note_html"
+    t.integer "cached_markdown_version"
   end
 
   add_index "notes", ["author_id"], name: "index_notes_on_author_id", using: :btree
@@ -956,6 +966,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do
     t.integer "auto_cancel_pending_pipelines", default: 0, null: false
     t.boolean "printing_merge_request_link_enabled", default: true, null: false
     t.string "import_jid"
+    t.integer "cached_markdown_version"
   end
 
   add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree
@@ -1028,6 +1039,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do
     t.datetime "created_at"
     t.datetime "updated_at"
     t.text "description_html"
+    t.integer "cached_markdown_version"
   end
 
   add_index "releases", ["project_id", "tag"], name: "index_releases_on_project_id_and_tag", using: :btree
@@ -1099,6 +1111,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do
     t.integer "visibility_level", default: 0, null: false
     t.text "title_html"
     t.text "content_html"
+    t.integer "cached_markdown_version"
   end
 
   add_index "snippets", ["author_id"], name: "index_snippets_on_author_id", using: :btree

lib/banzai/renderer.rb

@@ -33,20 +33,12 @@ module Banzai
     # of HTML. This method is analogous to calling render(object.field), but it
     # can cache the rendered HTML in the object, rather than Redis.
     #
-    # The context to use is learned from the passed-in object by calling
-    # #banzai_render_context(field), and cannot be changed. Use #render, passing
-    # it the field text, if a custom rendering is needed. The generated context
-    # is returned along with the HTML.
+    # The context to use is managed by the object and cannot be changed.
+    # Use #render, passing it the field text, if a custom rendering is needed.
     def self.render_field(object, field)
-      html_field = object.markdown_cache_field_for(field)
+      object.refresh_markdown_cache!(do_update: update_object?(object)) unless object.cached_html_up_to_date?(field)
 
-      html = object.__send__(html_field)
-      return html if html.present?
-
-      html = cacheless_render_field(object, field)
-      update_object(object, html_field, html) unless object.new_record? || object.destroyed?
-
-      html
+      object.cached_html_for(field)
     end
 
     # Same as +render_field+, but without consulting or updating the cache field
@@ -165,8 +157,9 @@ module Banzai
       Rails.cache.send(:expanded_key, full_cache_key(cache_key, pipeline_name))
     end
 
-    def self.update_object(object, html_field, html)
-      object.update_column(html_field, html)
+    # GitLab EE needs to disable updates on GET requests in Geo
+    def self.update_object?(object)
+      true
     end
   end
 end

spec/lib/banzai/object_renderer_spec.rb

@@ -4,13 +4,13 @@ describe Banzai::ObjectRenderer do
   let(:project) { create(:empty_project) }
   let(:user) { project.owner }
   let(:renderer) { described_class.new(project, user, custom_value: 'value') }
-  let(:object) { Note.new(note: 'hello', note_html: '<p>hello</p>') }
+  let(:object) { Note.new(note: 'hello', note_html: '<p dir="auto">hello</p>', cached_markdown_version: CacheMarkdownField::CACHE_VERSION) }
 
   describe '#render' do
     it 'renders and redacts an Array of objects' do
       renderer.render([object], :note)
 
-      expect(object.redacted_note_html).to eq '<p>hello</p>'
+      expect(object.redacted_note_html).to eq '<p dir="auto">hello</p>'
       expect(object.user_visible_reference_count).to eq 0
     end
 

spec/lib/banzai/renderer_spec.rb

 require 'spec_helper'
 
 describe Banzai::Renderer do
-  def expect_render(project = :project)
-    expected_context = { project: project }
-    expect(renderer).to receive(:cacheless_render) { :html }.with(:markdown, expected_context)
-  end
-
-  def expect_cache_update
-    expect(object).to receive(:update_column).with("field_html", :html)
-  end
-
-  def fake_object(*features)
-    markdown = :markdown if features.include?(:markdown)
-    html = :html if features.include?(:html)
-
-    object = double(
-      "object",
-      banzai_render_context: { project: :project },
-      field: markdown,
-      field_html: html
-    )
+  def fake_object(fresh:)
+    object = double('object')
 
-    allow(object).to receive(:markdown_cache_field_for).with(:field).and_return("field_html")
-    allow(object).to receive(:new_record?).and_return(features.include?(:new))
-    allow(object).to receive(:destroyed?).and_return(features.include?(:destroyed))
+    allow(object).to receive(:cached_html_up_to_date?).with(:field).and_return(fresh)
+    allow(object).to receive(:cached_html_for).with(:field).and_return('field_html')
 
     object
   end
 
-  describe "#render_field" do
+  describe '#render_field' do
     let(:renderer) { Banzai::Renderer }
-    let(:subject) { renderer.render_field(object, :field) }
+    subject { renderer.render_field(object, :field) }
 
-    context "with an empty cache" do
-      let(:object) { fake_object(:markdown) }
-      it "caches and returns the result" do
-        expect_render
-        expect_cache_update
-        expect(subject).to eq(:html)
-      end
-    end
+    context 'with a stale cache' do
+      let(:object) { fake_object(fresh: false) }
 
-    context "with a filled cache" do
-      let(:object) { fake_object(:markdown, :html) }
+      it 'caches and returns the result' do
+        expect(object).to receive(:refresh_markdown_cache!).with(do_update: true)
 
-      it "uses the cache" do
-        expect_render.never
-        expect_cache_update.never
-        should eq(:html)
+        is_expected.to eq('field_html')
       end
     end
 
-    context "new object" do
-      let(:object) { fake_object(:new, :markdown) }
-
-      it "doesn't cache the result" do
-        expect_render
-        expect_cache_update.never
-        expect(subject).to eq(:html)
-      end
-    end
+    context 'with an up-to-date cache' do
+      let(:object) { fake_object(fresh: true) }
 
-    context "destroyed object" do
-      let(:object) { fake_object(:destroyed, :markdown) }
+      it 'uses the cache' do
+        expect(object).to receive(:refresh_markdown_cache!).never
 
-      it "doesn't cache the result" do
-        expect_render
-        expect_cache_update.never
-        expect(subject).to eq(:html)
+        is_expected.to eq('field_html')
       end
     end
   end