Merge branch '1052-advanced-search-syntax' into 'master'

Support advanced search queries using elasticsearch

Closes #2037

See merge request !1770

Gemfile

@@ -113,7 +113,7 @@ gem 'seed-fu', '~> 2.3.5'
 gem 'elasticsearch-model', '~> 0.1.9'
 gem 'elasticsearch-rails', '~> 0.1.9'
 gem 'elasticsearch-api',   '5.0.3'
-gem 'gitlab-elasticsearch-git', '1.1.1', require: "elasticsearch/git"
+gem 'gitlab-elasticsearch-git', '1.2.0', require: "elasticsearch/git"
 gem 'aws-sdk'
 gem 'faraday_middleware-aws-signers-v4'
 

Gemfile.lock

@@ -287,7 +287,7 @@ GEM
       mime-types (>= 1.19)
       rugged (>= 0.23.0b)
     github-markup (1.4.0)
-    gitlab-elasticsearch-git (1.1.1)
+    gitlab-elasticsearch-git (1.2.0)
       activemodel (~> 4.2)
       activesupport (~> 4.2)
       charlock_holmes (~> 0.7)
@@ -949,7 +949,7 @@ DEPENDENCIES
   gemojione (~> 3.0)
   gitaly (~> 0.5.0)
   github-linguist (~> 4.7.0)
-  gitlab-elasticsearch-git (= 1.1.1)
+  gitlab-elasticsearch-git (= 1.2.0)
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-license (~> 1.0)
   gitlab-markup (~> 1.5.1)

app/models/concerns/elastic/application_search.rb

@@ -117,10 +117,10 @@ module Elastic
                          query: {
                            bool: {
                              must: [{
-                               multi_match: {
+                               simple_query_string: {
                                  fields: fields,
                                  query: query,
-                                 operator: :and
+                                 default_operator: :and
                                }
                              }]
                            }

app/models/concerns/elastic/notes_search.rb

@@ -50,14 +50,7 @@ module Elastic
       def self.elastic_search(query, options: {})
         options[:in] = ['note']
 
-        query_hash = {
-          query: {
-            bool: {
-              must: [{ match: { note: query } }],
-            },
-          }
-        }
-
+        query_hash = basic_query_hash(%w[note], query)
         query_hash = project_ids_filter(query_hash, options)
         query_hash = confidentiality_filter(query_hash, options[:current_user])
 

app/models/concerns/elastic/snippets_search.rb

@@ -52,23 +52,9 @@ module Elastic
       end
 
       def self.elastic_search_code(query, options: {})
-        query_hash = {
-          query: {
-            bool: {
-              must: [{ match: { content: query } }]
-            }
-          }
-        }
-
+        query_hash = basic_query_hash(%w(content), query)
         query_hash = filter(query_hash, options[:user])
 
-        query_hash[:sort] = [
-          { updated_at: { order: :desc } },
-          :_score
-        ]
-
-        query_hash[:highlight] = { fields: { content: {} } }
-
         self.__elasticsearch__.search(query_hash)
       end
 

app/views/search/_form.html.haml

@@ -13,3 +13,7 @@
     - unless params[:snippets].eql? 'true'
       = render 'filter' if current_user
     = button_tag "Search", class: "btn btn-success btn-search"
+  - if current_application_settings.elasticsearch_search?
+    .help-block
+      = link_to 'Advanced search functionality', help_page_path('user/search/advanced-search-syntax.md'), target: '_blank'
+      is enabled.

changelogs/unreleased-ee/1052-advanced-search-syntax.yml

+---
+title: Support advanced search queries using elasticsearch
+merge_request: 1770
+author:

doc/user/search/advanced-search-syntax.md

+## Advanced search syntax
+
+If your site administrator has enabled [Elasticsearch integration](../../integration/elasticsearch.md)
+then some advanced search functionality is available.
+
+Full details can be found in the
+[Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html#_simple_query_string_syntax)
+but here's a quick guide:
+
+* Searches look for all the words in a query, in any order - e.g.: searching
+  issues for `display bug` will return all issues matching both those words, in any order.
+* To find the exact term, use double quotes: `"display bug"`
+* To find bugs not mentioning display, use `-`: `bug -display`
+* To find a bug in display or sound, use `|`: `bug display | sound`
+* To group terms together, use parentheses: `bug | (display +sound)`
+* To match a partial word, use `*`: `bug find_by_*`
+* To find a term containing one of these symbols, use `\`: `argument \-last`
+

lib/gitlab/elastic/search_results.rb

@@ -10,7 +10,7 @@ module Gitlab
       def initialize(current_user, query, limit_project_ids, public_and_internal_projects = true)
         @current_user = current_user
         @limit_project_ids = limit_project_ids
-        @query = Shellwords.shellescape(query) if query.present?
+        @query = query
         @public_and_internal_projects = public_and_internal_projects
       end
 

spec/models/concerns/elastic/issue_spec.rb

@@ -15,19 +15,19 @@ describe Issue, elastic: true do
 
   it "searches issues" do
     Sidekiq::Testing.inline! do
-      create :issue, title: 'bla-bla term', project: project
-      create :issue, description: 'bla-bla term', project: project
+      create :issue, title: 'bla-bla term1', project: project
+      create :issue, description: 'bla-bla term2', project: project
       create :issue, project: project
 
       # The issue I have no access to
-      create :issue, title: 'bla-bla term'
+      create :issue, title: 'bla-bla term3'
 
       Gitlab::Elastic::Helper.refresh_index
     end
 
     options = { project_ids: [project.id] }
 
-    expect(described_class.elastic_search('term', options: options).total_count).to eq(2)
+    expect(described_class.elastic_search('(term1 | term2 | term3) +bla-bla', options: options).total_count).to eq(2)
   end
 
   it "returns json with all needed elements" do

spec/models/concerns/elastic/merge_request_spec.rb

@@ -15,19 +15,19 @@ describe MergeRequest, elastic: true do
     project = create :project
 
     Sidekiq::Testing.inline! do
-      create :merge_request, title: 'bla-bla term', source_project: project
-      create :merge_request, description: 'term in description', source_project: project, target_branch: "feature2"
+      create :merge_request, title: 'bla-bla term1', source_project: project
+      create :merge_request, description: 'term2 in description', source_project: project, target_branch: "feature2"
       create :merge_request, source_project: project, target_branch: "feature3"
 
       # The merge request you have no access to
-      create :merge_request, title: 'also with term'
+      create :merge_request, title: 'also with term3'
 
       Gitlab::Elastic::Helper.refresh_index
     end
 
     options = { project_ids: [project.id] }
 
-    expect(described_class.elastic_search('term', options: options).total_count).to eq(2)
+    expect(described_class.elastic_search('term1 | term2 | term3', options: options).total_count).to eq(2)
   end
 
   it "returns json with all needed elements" do

spec/models/concerns/elastic/milestone_spec.rb

@@ -15,19 +15,19 @@ describe Milestone, elastic: true do
     project = create :empty_project
 
     Sidekiq::Testing.inline! do
-      create :milestone, title: 'bla-bla term', project: project
-      create :milestone, description: 'bla-bla term', project: project
+      create :milestone, title: 'bla-bla term1', project: project
+      create :milestone, description: 'bla-bla term2', project: project
       create :milestone, project: project
 
       # The milestone you have no access to
-      create :milestone, title: 'bla-bla term'
+      create :milestone, title: 'bla-bla term3'
 
       Gitlab::Elastic::Helper.refresh_index
     end
 
     options = { project_ids: [project.id] }
 
-    expect(described_class.elastic_search('term', options: options).total_count).to eq(2)
+    expect(described_class.elastic_search('(term1 | term2 | term3) +bla-bla', options: options).total_count).to eq(2)
   end
 
   it "returns json with all needed elements" do

spec/models/concerns/elastic/note_spec.rb

@@ -15,18 +15,18 @@ describe Note, elastic: true do
     issue = create :issue
 
     Sidekiq::Testing.inline! do
-      create :note, note: 'bla-bla term', project: issue.project
+      create :note, note: 'bla-bla term1', project: issue.project
       create :note, project: issue.project
 
       # The note in the project you have no access to
-      create :note, note: 'bla-bla term'
+      create :note, note: 'bla-bla term2'
 
       Gitlab::Elastic::Helper.refresh_index
     end
 
     options = { project_ids: [issue.project.id] }
 
-    expect(described_class.elastic_search('term', options: options).total_count).to eq(1)
+    expect(described_class.elastic_search('term1 | term2', options: options).total_count).to eq(1)
   end
 
   it "indexes && searches diff notes" do

spec/models/concerns/elastic/project_spec.rb

@@ -15,8 +15,8 @@ describe Project, elastic: true do
     project_ids = []
 
     Sidekiq::Testing.inline! do
-      project = create :empty_project, name: 'test'
-      project1 = create :empty_project, path: 'test1'
+      project = create :empty_project, name: 'test1'
+      project1 = create :empty_project, path: 'test2'
       project2 = create :empty_project
       create :empty_project, path: 'someone_elses_project'
       project_ids += [project.id, project1.id, project2.id]
@@ -24,8 +24,9 @@ describe Project, elastic: true do
       Gitlab::Elastic::Helper.refresh_index
     end
 
-    expect(described_class.elastic_search('test', options: { project_ids: project_ids }).total_count).to eq(1)
     expect(described_class.elastic_search('test1', options: { project_ids: project_ids }).total_count).to eq(1)
+    expect(described_class.elastic_search('test2', options: { project_ids: project_ids }).total_count).to eq(1)
+    expect(described_class.elastic_search('test*', options: { project_ids: project_ids }).total_count).to eq(2)
     expect(described_class.elastic_search('someone_elses_project', options: { project_ids: project_ids }).total_count).to eq(0)
   end
 

spec/models/concerns/elastic/project_wiki_spec.rb

@@ -15,12 +15,14 @@ describe ProjectWiki, elastic: true do
     project = create :empty_project
 
     Sidekiq::Testing.inline! do
-      project.wiki.create_page("index_page", "Bla bla")
+      project.wiki.create_page("index_page", "Bla bla term1")
+      project.wiki.create_page("omega_page", "Bla bla term2")
       project.wiki.index_blobs
 
       Gitlab::Elastic::Helper.refresh_index
     end
 
-    expect(project.wiki.search('bla', type: :blob)[:blobs][:total_count]).to eq(1)
+    expect(project.wiki.search('term1', type: :blob)[:blobs][:total_count]).to eq(1)
+    expect(project.wiki.search('term1 | term2', type: :blob)[:blobs][:total_count]).to eq(2)
   end
 end

spec/models/concerns/elastic/repository_spec.rb

@@ -22,6 +22,7 @@ describe Repository, elastic: true do
     end
 
     expect(project.repository.search('def popen')[:blobs][:total_count]).to eq(1)
+    expect(project.repository.search('def | popen')[:blobs][:total_count] > 1).to be_truthy
     expect(project.repository.search('initial')[:commits][:total_count]).to eq(1)
   end
 

spec/models/concerns/elastic/snippet_spec.rb

@@ -19,9 +19,9 @@ describe Snippet, elastic: true do
     let!(:internal_snippet) { create(:snippet, :internal, content: 'password: XXX') }
     let!(:private_snippet)  { create(:snippet, :private, content: 'password: XXX', author: author) }
 
-    let!(:project_public_snippet)   { create(:snippet, :public, project: project, content: 'password: XXX') }
-    let!(:project_internal_snippet) { create(:snippet, :internal, project: project, content: 'password: XXX') }
-    let!(:project_private_snippet)  { create(:snippet, :private, project: project, content: 'password: XXX') }
+    let!(:project_public_snippet)   { create(:snippet, :public, project: project, content: 'password: 123') }
+    let!(:project_internal_snippet) { create(:snippet, :internal, project: project, content: 'password: 456') }
+    let!(:project_private_snippet)  { create(:snippet, :private, project: project, content: 'password: 789') }
 
     before do
       Gitlab::Elastic::Helper.refresh_index
@@ -60,6 +60,16 @@ describe Snippet, elastic: true do
       expect(result.records).to match_array [public_snippet, internal_snippet, private_snippet]
     end
 
+    it 'supports advanced search syntax' do
+      member = create(:user)
+      project.add_reporter(member)
+
+      result = described_class.elastic_search_code('password +(123 | 789)', options: { user: member })
+
+      expect(result.total_count).to eq(2)
+      expect(result.records).to match_array [project_public_snippet, project_private_snippet]
+    end
+
     [:admin, :auditor].each do |user_type|
       it "returns all snippets for #{user_type}" do
         superuser = create(user_type)