Skip to content

G
gitlab-ce
Commit fe605235 authored by Valery Sizov's avatar Valery Sizov
Browse files
Options

ES: all searches are done but it reqiores lots of refactoring

parent c0f6df43
Hide whitespace changes
Inline Side-by-side
Showing with 862 additions and 593 deletions
Gemfile
View file @ fe605235
...@@ -91,6 +91,11 @@ gem "six", '~> 0.2.0' ...@@ -91,6 +91,11 @@ gem "six", '~> 0.2.0'
# Seed data # Seed data
gem "seed-fu", '~> 2.3.5' gem "seed-fu", '~> 2.3.5'
# Search
gem 'elasticsearch-model'
gem 'elasticsearch-rails'
gem 'gitlab-elasticsearch-git', require: "elasticsearch/git"
# Markdown and HTML processing # Markdown and HTML processing
gem 'html-pipeline', '~> 1.11.0' gem 'html-pipeline', '~> 1.11.0'
gem 'task_list', '~> 1.0.2', require: 'task_list/railtie' gem 'task_list', '~> 1.0.2', require: 'task_list/railtie'
......
app/controllers/search_controller.rb
View file @ fe605235
...@@ -4,56 +4,45 @@ class SearchController < ApplicationController ...@@ -4,56 +4,45 @@ class SearchController < ApplicationController
layout 'search' layout 'search'
def show def show
# return if params[:search].nil? || params[:search].blank? return if params[:search].nil? || params[:search].blank?
# @search_term = params[:search] @search_term = params[:search]
# if params[:project_id].present? if params[:project_id].present?
# @project = Project.find_by(id: params[:project_id]) @project = Project.find_by(id: params[:project_id])
# @project = nil unless can?(current_user, :download_code, @project) @project = nil unless can?(current_user, :download_code, @project)
# end end
# if params[:group_id].present?
# @group = Group.find_by(id: params[:group_id])
# @group = nil unless can?(current_user, :read_group, @group)
# end
# @scope = params[:scope]
# @show_snippets = params[:snippets].eql? 'true'
# @search_results =
# if @project
# unless %w(blobs notes issues merge_requests milestones wiki_blobs
# commits).include?(@scope)
# @scope = 'blobs'
# end
# Search::ProjectService.new(@project, current_user, params).execute
# elsif @show_snippets
# unless %w(snippet_blobs snippet_titles).include?(@scope)
# @scope = 'snippet_blobs'
# end
# Search::SnippetService.new(current_user, params).execute
# else
# unless %w(projects issues merge_requests milestones).include?(@scope)
# @scope = 'projects'
# end
# Search::GlobalService.new(current_user, params).execute
# end
# @objects = @search_results.objects(@scope, params[:page])
@group = Group.find_by(id: params[:group_id]) if params[:group_id].present?
if project if params[:group_id].present?
return access_denied! unless can?(current_user, :download_code, project) @group = Group.find_by(id: params[:group_id])
@search_results = SearchService.new(current_user, params).project_search(project) @group = nil unless can?(current_user, :read_group, @group)
else
@search_results = SearchService.new(current_user, params).global_search
end end
@search_results = SearchDecorator.new(@search_results, params[:type]) @scope = params[:scope]
@show_snippets = params[:snippets].eql? 'true'
@search_results =
if @project
unless %w(blobs notes issues merge_requests milestones wiki_blobs
commits).include?(@scope)
@scope = 'blobs'
end
Search::ProjectService.new(@project, current_user, params).execute
elsif @show_snippets
unless %w(snippet_blobs snippet_titles).include?(@scope)
@scope = 'snippet_blobs'
end
Search::SnippetService.new(current_user, params).execute
else
unless %w(projects issues merge_requests milestones).include?(@scope)
@scope = 'projects'
end
Search::GlobalService.new(current_user, params).execute
end
@objects = @search_results.objects(@scope, params[:page])
end end
def autocomplete def autocomplete
......
app/elastic/application_search.rb
View file @ fe605235
...@@ -4,52 +4,49 @@ module ApplicationSearch ...@@ -4,52 +4,49 @@ module ApplicationSearch
included do included do
include Elasticsearch::Model include Elasticsearch::Model
# $ host git-elasticsearch-1.production.infra.home
# git-elasticsearch-1.production.infra.home has address 10.40.56.23
self.__elasticsearch__.client = Elasticsearch::Client.new host: Gitlab.config.elasticsearch.host, port: Gitlab.config.elasticsearch.port self.__elasticsearch__.client = Elasticsearch::Client.new host: Gitlab.config.elasticsearch.host, port: Gitlab.config.elasticsearch.port
index_name [Rails.application.class.parent_name.downcase, self.name.downcase, Rails.env].join('-') index_name [Rails.application.class.parent_name.downcase, self.name.downcase, Rails.env].join('-')
settings \ settings \
index: { index: {
query: { query: {
default_field: :name default_field: :name
},
analysis: {
:analyzer => {
:index_analyzer => {
type: "custom",
tokenizer: "ngram_tokenizer",
filter: %w(lowercase asciifolding name_ngrams)
},
:search_analyzer => {
type: "custom",
tokenizer: "standard",
filter: %w(lowercase asciifolding )
}
},
tokenizer: {
ngram_tokenizer: {
type: "NGram",
min_gram: 1,
max_gram: 20,
token_chars: %w(letter digit connector_punctuation punctuation)
}
}, },
filter: { analysis: {
name_ngrams: { :analyzer => {
type: "NGram", :my_analyzer => {
max_gram: 20, type: "custom",
min_gram: 1 tokenizer: "ngram_tokenizer",
filter: %w(lowercase asciifolding name_ngrams)
},
:search_analyzer => {
type: "custom",
tokenizer: "standard",
filter: %w(lowercase asciifolding)
}
},
tokenizer: {
ngram_tokenizer: {
type: "nGram",
min_gram: 1,
max_gram: 20,
token_chars: %w(letter digit connector_punctuation punctuation)
}
},
filter: {
name_ngrams: {
type: "nGram",
max_gram: 20,
min_gram: 1
}
} }
} }
} }
}
after_commit lambda { Resque.enqueue(Elastic::BaseIndexer, :index, self.class.to_s, self.id) }, on: :create after_commit ->{ ElasticIndexerWorker.perform_async(:index, self.class.to_s, self.id) }, on: :create
after_commit lambda { Resque.enqueue(Elastic::BaseIndexer, :update, self.class.to_s, self.id) }, on: :update after_commit ->{ ElasticIndexerWorker.perform_async(:update, self.class.to_s, self.id) }, on: :update
after_commit lambda { Resque.enqueue(Elastic::BaseIndexer, :delete, self.class.to_s, self.id) }, on: :destroy after_commit ->{ ElasticIndexerWorker.perform_async(:delete, self.class.to_s, self.id) }, on: :destroy
after_touch lambda { Resque.enqueue(Elastic::BaseIndexer, :update, self.class.to_s, self.id) }
end end
module ClassMethods module ClassMethods
...@@ -59,11 +56,7 @@ module ApplicationSearch ...@@ -59,11 +56,7 @@ module ApplicationSearch
memo memo
end end
{ { fields: es_fields }
pre_tags: ["gitlabelasticsearch→"],
post_tags: ["←gitlabelasticsearch"],
fields: es_fields
}
end end
end end
end end
app/elastic/issues_search.rb
View file @ fe605235
...@@ -8,49 +8,33 @@ module IssuesSearch ...@@ -8,49 +8,33 @@ module IssuesSearch
indexes :id, type: :integer, index: :not_analyzed indexes :id, type: :integer, index: :not_analyzed
indexes :iid, type: :integer, index: :not_analyzed indexes :iid, type: :integer, index: :not_analyzed
indexes :title, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer indexes :title, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :description, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer indexes :description, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :created_at, type: :date indexes :created_at, type: :date
indexes :updated_at, type: :date indexes :updated_at, type: :date
indexes :state, type: :string indexes :state, type: :string
indexes :project_id, type: :integer, index: :not_analyzed indexes :project_id, type: :integer, index: :not_analyzed
indexes :author_id, type: :integer, index: :not_analyzed indexes :author_id, type: :integer, index: :not_analyzed
#indexes :assignee_id, type: :integer, index: :not_analyzed
indexes :project, type: :nested indexes :project, type: :nested
indexes :author, type: :nested indexes :author, type: :nested
#indexes :assignee, type: :nested
indexes :title_sort, type: :string, index: 'not_analyzed'
indexes :updated_at_sort, type: :date, index: :not_analyzed indexes :updated_at_sort, type: :date, index: :not_analyzed
indexes :created_at_sort, type: :string, index: :not_analyzed
end end
def as_indexed_json(options = {}) def as_indexed_json(options = {})
as_json( as_json(
include: { include: {
project: { only: :id }, project: { only: :id },
author: { only: :id }, author: { only: :id }
#assignee: { only: :id }
} }
).merge({ ).merge({ updated_at_sort: updated_at })
title_sort: title.downcase,
updated_at_sort: updated_at,
created_at_sort: created_at
})
end end
def self.search(query, page: 1, per: 20, options: {}) def self.elastic_search(query, options: {})
options[:in] = %w(title^2 description)
page ||= 1
if options[:in].blank?
options[:in] = %w(title^2 description)
else
options[:in].push(%w(title^2 description) - options[:in])
end
query_hash = { query_hash = {
query: { query: {
filtered: { filtered: {
...@@ -62,9 +46,7 @@ module IssuesSearch ...@@ -62,9 +46,7 @@ module IssuesSearch
} }
}, },
}, },
}, }
size: per,
from: per * (page.to_i - 1)
} }
if query.blank? if query.blank?
...@@ -81,30 +63,13 @@ module IssuesSearch ...@@ -81,30 +63,13 @@ module IssuesSearch
} }
end end
options[:order] = :default if options[:order].blank?
order = case options[:order].to_sym
when :newest
{ created_at_sort: { order: :asc, mode: :min } }
when :oldest
{ created_at_sort: { order: :desc, mode: :min } }
when :recently_updated
{ updated_at_sort: { order: :asc, mode: :min } }
when :last_updated
{ updated_at_sort: { order: :desc, mode: :min } }
else
{ title_sort: { order: :asc, mode: :min } }
end
query_hash[:sort] = [ query_hash[:sort] = [
order, { updated_at_sort: { order: :desc, mode: :min } },
:_score :_score
] ]
if options[:highlight] query_hash[:highlight] = { fields: options[:in].inject({}) { |a, o| a[o.to_sym] = {} } }
query_hash[:highlight] = { fields: options[:in].inject({}) { |a, o| a[o.to_sym] = {} } }
end
self.__elasticsearch__.search(query_hash) self.__elasticsearch__.search(query_hash)
end end
end end
......
app/elastic/merge_requests_search.rb
View file @ fe605235
...@@ -8,10 +8,10 @@ module MergeRequestsSearch ...@@ -8,10 +8,10 @@ module MergeRequestsSearch
indexes :id, type: :integer, index: :not_analyzed indexes :id, type: :integer, index: :not_analyzed
indexes :iid, type: :integer, index: :not_analyzed indexes :iid, type: :integer, index: :not_analyzed
indexes :target_branch, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer indexes :target_branch, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :source_branch, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer indexes :source_branch, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :title, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer indexes :title, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :description, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer indexes :description, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :created_at, type: :date indexes :created_at, type: :date
indexes :updated_at, type: :date indexes :updated_at, type: :date
indexes :state, type: :string indexes :state, type: :string
...@@ -20,15 +20,11 @@ module MergeRequestsSearch ...@@ -20,15 +20,11 @@ module MergeRequestsSearch
indexes :source_project_id, type: :integer, index: :not_analyzed indexes :source_project_id, type: :integer, index: :not_analyzed
indexes :target_project_id, type: :integer, index: :not_analyzed indexes :target_project_id, type: :integer, index: :not_analyzed
indexes :author_id, type: :integer, index: :not_analyzed indexes :author_id, type: :integer, index: :not_analyzed
#indexes :assignee_id, type: :integer, index: :not_analyzed
indexes :source_project, type: :nested indexes :source_project, type: :nested
indexes :target_project, type: :nested indexes :target_project, type: :nested
indexes :author, type: :nested indexes :author, type: :nested
#indexes :assignee, type: :nested
indexes :title_sort, type: :string, index: 'not_analyzed'
indexes :created_at_sort, type: :string, index: 'not_analyzed'
indexes :updated_at_sort, type: :string, index: 'not_analyzed' indexes :updated_at_sort, type: :string, index: 'not_analyzed'
end end
...@@ -37,25 +33,13 @@ module MergeRequestsSearch ...@@ -37,25 +33,13 @@ module MergeRequestsSearch
include: { include: {
source_project: { only: :id }, source_project: { only: :id },
target_project: { only: :id }, target_project: { only: :id },
author: { only: :id }, author: { only: :id }
#assignee: { only: :id }
} }
).merge({ ).merge({ updated_at_sort: updated_at })
title_sort: title.downcase,
updated_at_sort: updated_at,
created_at_sort: created_at
})
end end
def self.search(query, page: 1, per: 20, options: {}) def self.elastic_search(query, options: {})
options[:in] = %w(title^2 description)
page ||= 1
if options[:in].blank?
options[:in] = %w(title^2 description)
else
options[:in].push(%w(title^2 description) - options[:in])
end
query_hash = { query_hash = {
query: { query: {
...@@ -68,18 +52,7 @@ module MergeRequestsSearch ...@@ -68,18 +52,7 @@ module MergeRequestsSearch
} }
}, },
}, },
}, }
facets: {
targetProjectFacet: {
terms: {
field: :target_project_id,
all_terms: true,
size: Project.count
}
}
},
size: per,
from: per * (page.to_i - 1)
} }
if query.blank? if query.blank?
...@@ -101,30 +74,13 @@ module MergeRequestsSearch ...@@ -101,30 +74,13 @@ module MergeRequestsSearch
} }
end end
options[:order] = :default if options[:order].blank?
order = case options[:order].to_sym
when :newest
{ created_at_sort: { order: :asc, mode: :min } }
when :oldest
{ created_at_sort: { order: :desc, mode: :min } }
when :recently_updated
{ updated_at_sort: { order: :asc, mode: :min } }
when :last_updated
{ updated_at_sort: { order: :desc, mode: :min } }
else
{ title_sort: { order: :asc, mode: :min } }
end
query_hash[:sort] = [ query_hash[:sort] = [
order, { updated_at_sort: { order: :desc, mode: :min } },
:_score :_score
] ]
if options[:highlight] query_hash[:highlight] = highlight_options(options[:in])
query_hash[:highlight] = highlight_options(options[:in])
end
self.__elasticsearch__.search(query_hash) self.__elasticsearch__.search(query_hash)
end end
end end
......
app/elastic/groups_search.rb → app/elastic/milestones_search.rb
View file @ fe605235
module GroupsSearch module MilestonesSearch
extend ActiveSupport::Concern extend ActiveSupport::Concern
included do included do
...@@ -6,33 +6,20 @@ module GroupsSearch ...@@ -6,33 +6,20 @@ module GroupsSearch
mappings do mappings do
indexes :id, type: :integer indexes :id, type: :integer
indexes :name, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer indexes :title, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :path, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer indexes :description, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :description, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer indexes :project_id, type: :integer, index: :not_analyzed
indexes :created_at, type: :date indexes :created_at, type: :date
indexes :name_sort, type: :string, index: :not_analyzed
indexes :created_at_sort, type: :string, index: 'not_analyzed'
indexes :updated_at_sort, type: :string, index: 'not_analyzed' indexes :updated_at_sort, type: :string, index: 'not_analyzed'
end end
def as_indexed_json(options = {}) def as_indexed_json(options = {})
as_json.merge({ as_json.merge({ updated_at_sort: updated_at })
name_sort: name.downcase,
created_at_sort: created_at,
updated_at_sort: updated_at
})
end end
def self.search(query, page: 1, per: 20, options: {}) def self.elastic_search(query, options: {})
options[:in] = %w(title^2 description)
page ||= 1
if options[:in].blank?
options[:in] = %w(name^10 path^5)
else
options[:in].push(%w(name^10 path^5) - options[:in])
end
query_hash = { query_hash = {
query: { query: {
...@@ -45,9 +32,7 @@ module GroupsSearch ...@@ -45,9 +32,7 @@ module GroupsSearch
} }
}, },
}, },
}, }
size: per,
from: per * (page.to_i - 1)
} }
if query.blank? if query.blank?
...@@ -55,42 +40,22 @@ module GroupsSearch ...@@ -55,42 +40,22 @@ module GroupsSearch
query_hash[:track_scores] = true query_hash[:track_scores] = true
end end
if options[:gids] if options[:project_ids]
query_hash[:query][:filtered][:filter] ||= { and: [] } query_hash[:query][:filtered][:filter] ||= { and: [] }
query_hash[:query][:filtered][:filter][:and] << { query_hash[:query][:filtered][:filter][:and] << {
ids: { terms: {
values: options[:gids] project_id: [options[:project_ids]].flatten
} }
} }
end end
options[:order] = :default if options[:order].blank?
order = case options[:order].to_sym
when :newest
{ created_at_sort: { order: :asc, mode: :min } }
when :oldest
{ created_at_sort: { order: :desc, mode: :min } }
when :recently_updated
{ updated_at_sort: { order: :asc, mode: :min } }
when :last_updated
{ updated_at_sort: { order: :desc, mode: :min } }
else
{ name_sort: { order: :asc, mode: :min } }
end
query_hash[:sort] = [ query_hash[:sort] = [
order, { updated_at_sort: { order: :desc, mode: :min } },
:_score :_score
] ]
#query_hash[:sort] = [ query_hash[:highlight] = highlight_options(options[:in])
#{ name_sort: { order: :asc, mode: :min }},
#:_score
#]
if options[:highlight]
query_hash[:highlight] = highlight_options(options[:in])
end
self.__elasticsearch__.search(query_hash) self.__elasticsearch__.search(query_hash)
end end
......
app/elastic/notes_search.rb 0 → 100644
View file @ fe605235
module NotesSearch
extend ActiveSupport::Concern
included do
include ApplicationSearch
mappings do
indexes :id, type: :integer
indexes :note, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :project_id, type: :integer, index: :not_analyzed
indexes :created_at, type: :date
indexes :updated_at_sort, type: :string, index: 'not_analyzed'
end
def as_indexed_json(options = {})
as_json.merge({ updated_at_sort: updated_at })
end
def self.elastic_search(query, options: {})
options[:in] = ["note"]
query_hash = {
query: {
filtered: {
query: {match: {note: query}},
},
}
}
if query.blank?
query_hash[:query][:filtered][:query] = { match_all: {}}
query_hash[:track_scores] = true
end
if options[:project_ids]
query_hash[:query][:filtered][:filter] ||= { and: [] }
query_hash[:query][:filtered][:filter][:and] << {
terms: {
project_id: [options[:project_ids]].flatten
}
}
end
query_hash[:sort] = [
{ updated_at_sort: { order: :desc, mode: :min } },
:_score
]
query_hash[:highlight] = highlight_options(options[:in])
self.__elasticsearch__.search(query_hash)
end
end
end
app/elastic/projects_search.rb
View file @ fe605235
...@@ -7,11 +7,11 @@ module ProjectsSearch ...@@ -7,11 +7,11 @@ module ProjectsSearch
mappings do mappings do
indexes :id, type: :integer, index: 'not_analyzed' indexes :id, type: :integer, index: 'not_analyzed'
indexes :name, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer indexes :name, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :path, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer indexes :path, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :name_with_namespace, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer indexes :name_with_namespace, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :path_with_namespace, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer indexes :path_with_namespace, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :description, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer indexes :description, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :namespace_id, type: :integer, index: 'not_analyzed' indexes :namespace_id, type: :integer, index: 'not_analyzed'
...@@ -20,49 +20,17 @@ module ProjectsSearch ...@@ -20,49 +20,17 @@ module ProjectsSearch
indexes :visibility_level, type: :integer, index: 'not_analyzed' indexes :visibility_level, type: :integer, index: 'not_analyzed'
indexes :last_activity_at, type: :date indexes :last_activity_at, type: :date
indexes :last_pushed_at, type: :date indexes :last_pushed_at, type: :date
indexes :owners, type: :nested
indexes :masters, type: :nested
indexes :developers, type: :nested
indexes :reporters, type: :nested
indexes :guests, type: :nested
indexes :categories, type: :nested
indexes :name_with_namespace_sort, type: :string, index: 'not_analyzed'
indexes :created_at_sort, type: :string, index: 'not_analyzed'
indexes :updated_at_sort, type: :string, index: 'not_analyzed'
end end
def as_indexed_json(options={}) def as_indexed_json(options = {})
as_json( as_json.merge({
include: {
owners: { only: :id },
masters: { only: :id },
developers: { only: :id },
reporters: { only: :id },
guests: { only: :id },
categories: { only: :name}
}
).merge({
name_with_namespace: name_with_namespace, name_with_namespace: name_with_namespace,
name_with_namespace_sort: name_with_namespace.downcase, path_with_namespace: path_with_namespace
path_with_namespace: path_with_namespace,
updated_at_sort: updated_at,
created_at_sort: created_at
}) })
end end
def self.search(query, page: 1, per: 20, options: {}) def self.elastic_search(query, options: {})
options[:in] = %w(name^10 name_with_namespace^2 path_with_namespace path^9)
page ||= 1
if options[:in].blank?
options[:in] = %w(name^10 name_with_namespace^2 path_with_namespace path^9)
else
options[:in].push(%w(name^10 name_with_namespace^2 path_with_namespace path^9) - options[:in])
end
query_hash = { query_hash = {
query: { query: {
...@@ -76,25 +44,6 @@ module ProjectsSearch ...@@ -76,25 +44,6 @@ module ProjectsSearch
}, },
}, },
}, },
facets: {
namespaceFacet: {
terms: {
field: :namespace_id,
all_terms: true,
size: Namespace.count
}
},
categoryFacet: {
terms: {
field: "categories.name",
all_terms: true,
# FIXME. Remove to_a
size: Project.category_counts.to_a.count
}
}
},
size: per,
from: per * (page.to_i - 1)
} }
if query.blank? if query.blank?
...@@ -135,18 +84,6 @@ module ProjectsSearch ...@@ -135,18 +84,6 @@ module ProjectsSearch
} }
end end
if options[:category]
query_hash[:query][:filtered][:filter] ||= { and: [] }
query_hash[:query][:filtered][:filter][:and] << {
nested: {
path: :categories,
filter: {
term: { "categories.name" => options[:category] }
}
}
}
end
if options[:non_archived] if options[:non_archived]
query_hash[:query][:filtered][:filter] ||= { and: [] } query_hash[:query][:filtered][:filter] ||= { and: [] }
query_hash[:query][:filtered][:filter][:and] << { query_hash[:query][:filtered][:filter][:and] << {
...@@ -169,9 +106,9 @@ module ProjectsSearch ...@@ -169,9 +106,9 @@ module ProjectsSearch
query_hash[:query][:filtered][:filter] ||= { and: [] } query_hash[:query][:filtered][:filter] ||= { and: [] }
query_hash[:query][:filtered][:filter][:and] << { query_hash[:query][:filtered][:filter][:and] << {
nested: { nested: {
path: :owners, path: :owner,
filter: { filter: {
term: { "owners.id" => options[:owner_id] } term: { "owner.id" => options[:owner_id] }
} }
} }
} }
...@@ -186,28 +123,10 @@ module ProjectsSearch ...@@ -186,28 +123,10 @@ module ProjectsSearch
} }
end end
options[:order] = :default if options[:order].blank? query_hash[:sort] = [:_score]
order = case options[:order].to_sym
when :newest query_hash[:highlight] = highlight_options(options[:in])
{ created_at_sort: { order: :asc, mode: :min } }
when :oldest
{ created_at_sort: { order: :desc, mode: :min } }
when :recently_updated
{ updated_at_sort: { order: :asc, mode: :min } }
when :last_updated
{ updated_at_sort: { order: :desc, mode: :min } }
else
{ name_with_namespace_sort: { order: :asc, mode: :min } }
end
query_hash[:sort] = [
order,
:_score
]
if options[:highlight]
query_hash[:highlight] = highlight_options(options[:in])
end
self.__elasticsearch__.search(query_hash) self.__elasticsearch__.search(query_hash)
end end
......
app/elastic/repositories_search.rb
View file @ fe605235
...@@ -23,14 +23,8 @@ module RepositoriesSearch ...@@ -23,14 +23,8 @@ module RepositoriesSearch
Project.find_each do |project| Project.find_each do |project|
if project.repository.exists? && !project.repository.empty? if project.repository.exists? && !project.repository.empty?
begin project.repository.index_commits
project.repository.index_commits project.repository.index_blobs
rescue
end
begin
project.repository.index_blobs
rescue
end
end end
end end
end end
......
app/elastic/snippets_search.rb 0 → 100644
View file @ fe605235
module SnippetsSearch
extend ActiveSupport::Concern
included do
include ApplicationSearch
mappings do
indexes :id, type: :integer, index: :not_analyzed
indexes :title, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :file_name, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :content, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :created_at, type: :date
indexes :updated_at, type: :date
indexes :state, type: :string
indexes :project_id, type: :integer, index: :not_analyzed
indexes :author_id, type: :integer, index: :not_analyzed
indexes :project, type: :nested
indexes :author, type: :nested
indexes :updated_at_sort, type: :date, index: :not_analyzed
end
def as_indexed_json(options = {})
as_json(
include: {
project: { only: :id },
author: { only: :id }
}
)
end
def self.elastic_search(query, options: {})
options[:in] = %w(title file_name)
query_hash = {
query: {
filtered: {
query: {
multi_match: {
fields: options[:in],
query: "#{query}",
operator: :and
}
},
},
}
}
if query.blank?
query_hash[:query][:filtered][:query] = { match_all: {}}
query_hash[:track_scores] = true
end
if options[:ids]
query_hash[:query][:filtered][:filter] ||= { and: [] }
query_hash[:query][:filtered][:filter][:and] << {
terms: {
id: [options[:ids]].flatten
}
}
end
query_hash[:sort] = [
{ updated_at_sort: { order: :desc, mode: :min } },
:_score
]
query_hash[:highlight] = { fields: options[:in].inject({}) { |a, o| a[o.to_sym] = {} } }
self.__elasticsearch__.search(query_hash)
end
def self.elastic_search_code(query, options: {})
options[:in] = %w(title file_name)
query_hash = {
query: {
filtered: {
query: {match: {content: query}},
},
}
}
if options[:ids]
query_hash[:query][:filtered][:filter] ||= { and: [] }
query_hash[:query][:filtered][:filter][:and] << {
terms: {
id: [options[:ids]].flatten
}
}
end
query_hash[:sort] = [
{ updated_at_sort: { order: :desc, mode: :min } },
:_score
]
query_hash[:highlight] = { fields: options[:in].inject({}) { |a, o| a[o.to_sym] = {} } }
self.__elasticsearch__.search(query_hash)
end
end
end
app/elastic/users_search.rb
View file @ fe605235
...@@ -6,21 +6,17 @@ module UsersSearch ...@@ -6,21 +6,17 @@ module UsersSearch
mappings do mappings do
indexes :id, type: :integer indexes :id, type: :integer
indexes :email, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer indexes :email, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :name, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer indexes :name, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :username, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer indexes :username, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :bio, type: :string indexes :bio, type: :string
indexes :skype, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer indexes :skype, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :linkedin, type: :string indexes :linkedin, type: :string
indexes :twitter, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer indexes :twitter, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :state, type: :string indexes :state, type: :string
indexes :website_url, type: :string indexes :website_url, type: :string
indexes :created_at, type: :date indexes :created_at, type: :date
indexes :admin, type: :boolean indexes :admin, type: :boolean
indexes :name_sort, type: :string, index: 'not_analyzed'
indexes :created_at_sort, type: :string, index: 'not_analyzed'
indexes :updated_at_sort, type: :string, index: 'not_analyzed'
end end
def as_indexed_json(options = {}) def as_indexed_json(options = {})
...@@ -31,16 +27,8 @@ module UsersSearch ...@@ -31,16 +27,8 @@ module UsersSearch
}) })
end end
def self.search(query, page: 1, per: 20, options: {}) def self.elastic_search(query, options: {})
options[:in] = %w(name^3 username^2 email)
page ||= 1
per ||= 20
if options[:in].blank?
options[:in] = %w(name^3 username^2 email)
else
options[:in].push(%w(name^3 username^2 email) - options[:in])
end
query_hash = { query_hash = {
query: { query: {
...@@ -53,9 +41,7 @@ module UsersSearch ...@@ -53,9 +41,7 @@ module UsersSearch
} }
}, },
}, },
}, }
size: per,
from: per * (page.to_i - 1)
} }
if query.blank? if query.blank?
...@@ -81,30 +67,10 @@ module UsersSearch ...@@ -81,30 +67,10 @@ module UsersSearch
} }
end end
options[:order] = :default if options[:order].blank? query_hash[:sort] = [:_score]
order = case options[:order].to_sym
when :newest
{ created_at_sort: { order: :asc, mode: :min } }
when :oldest
{ created_at_sort: { order: :desc, mode: :min } }
when :recently_updated
{ updated_at_sort: { order: :asc, mode: :min } }
when :last_updated
{ updated_at_sort: { order: :desc, mode: :min } }
else
{ name_sort: { order: :asc, mode: :min } }
end
query_hash[:sort] = [
order,
:_score
]
if options[:highlight]
query_hash[:highlight] = highlight_options(options[:in])
end
query_hash[:highlight] = highlight_options(options[:in])
self.__elasticsearch__.search(query_hash) self.__elasticsearch__.search(query_hash)
end end
end end
......
app/elastic/wiki_repositories_search.rb 0 → 100644
View file @ fe605235
module WikiRepositoriesSearch
extend ActiveSupport::Concern
included do
include Elasticsearch::Git::Repository
self.__elasticsearch__.client = Elasticsearch::Client.new host: Gitlab.config.elasticsearch.host, port: Gitlab.config.elasticsearch.port
def repository_id
"wiki_#{project.id}"
end
def self.repositories_count
Project.where(wiki_enabled: true).count
end
def client_for_indexing
self.__elasticsearch__.client
end
def self.import
ProjectWiki.__elasticsearch__.create_index! force: true
Project.where(wiki_enabled: true).find_each do |project|
unless project.wiki.empty?
project.wiki.index_blobs
end
end
end
end
end
app/models/group.rb
View file @ fe605235
...@@ -19,7 +19,6 @@ require 'file_size_validator' ...@@ -19,7 +19,6 @@ require 'file_size_validator'
class Group < Namespace class Group < Namespace
include Gitlab::ConfigHelper include Gitlab::ConfigHelper
include Referable include Referable
include GroupsSearch
has_many :group_members, dependent: :destroy, as: :source, class_name: 'GroupMember' has_many :group_members, dependent: :destroy, as: :source, class_name: 'GroupMember'
alias_method :members, :group_members alias_method :members, :group_members
......
app/models/milestone.rb
View file @ fe605235
...@@ -24,6 +24,7 @@ class Milestone < ActiveRecord::Base ...@@ -24,6 +24,7 @@ class Milestone < ActiveRecord::Base
include Sortable include Sortable
include Referable include Referable
include StripAttribute include StripAttribute
include MilestonesSearch
belongs_to :project belongs_to :project
has_many :issues has_many :issues
......
app/models/note.rb
View file @ fe605235
...@@ -26,6 +26,7 @@ class Note < ActiveRecord::Base ...@@ -26,6 +26,7 @@ class Note < ActiveRecord::Base
include Gitlab::CurrentSettings include Gitlab::CurrentSettings
include Participable include Participable
include Mentionable include Mentionable
include NotesSearch
default_value_for :system, false default_value_for :system, false
......
app/models/project.rb
View file @ fe605235
...@@ -943,6 +943,10 @@ class Project < ActiveRecord::Base ...@@ -943,6 +943,10 @@ class Project < ActiveRecord::Base
false false
end end
def wiki
ProjectWiki.new(self, self.owner)
end
def reference_issue_tracker? def reference_issue_tracker?
default_issues_tracker? || jira_tracker_active? default_issues_tracker? || jira_tracker_active?
end end
......
app/models/project_wiki.rb
View file @ fe605235
class ProjectWiki class ProjectWiki
include Gitlab::ShellAdapter include Gitlab::ShellAdapter
include WikiRepositoriesSearch
MARKUPS = { MARKUPS = {
'Markdown' => :md, 'Markdown' => :md,
...@@ -12,6 +13,8 @@ class ProjectWiki ...@@ -12,6 +13,8 @@ class ProjectWiki
# Returns a string describing what went wrong after # Returns a string describing what went wrong after
# an operation fails. # an operation fails.
attr_reader :error_message attr_reader :error_message
attr_reader :project
def initialize(project, user = nil) def initialize(project, user = nil)
@project = project @project = project
......
app/models/repository.rb
View file @ fe605235
...@@ -108,6 +108,12 @@ class Repository ...@@ -108,6 +108,12 @@ class Repository
commits commits
end end
def find_commits_by_message_with_elastic(query)
project.repository.search(query, type: :commit)[:commits][:results].map do |result|
commit result["_source"]["commit"]["sha"]
end
end
def find_branch(name) def find_branch(name)
raw_repository.branches.find { |branch| branch.name == name } raw_repository.branches.find { |branch| branch.name == name }
end end
...@@ -660,7 +666,53 @@ class Repository ...@@ -660,7 +666,53 @@ class Repository
Gitlab::Popen.popen(args, path_to_repo).first.scrub.split(/^--$/) Gitlab::Popen.popen(args, path_to_repo).first.scrub.split(/^--$/)
end end
def parse_search_result(result) def parse_search_result(result, elastic = false)
if elastic
parse_search_result_from_elastic(result)
else
parse_search_result_from_grep(result)
end
end
def parse_search_result_from_elastic(result)
ref = result["_source"]["blob"]["oid"]
filename = result["_source"]["blob"]["path"]
content = result["_source"]["blob"]["content"]
total_lines = content.lines.size
term = result["highlight"]["blob.content"][0].match(/gitlabelasticsearch→(.*)←gitlabelasticsearch/)[1]
found_line_number = 0
content.each_line.each_with_index do |line, index|
if line.include?(term)
found_line_number = index
break
end
end
from = if found_line_number >= 2
found_line_number - 2
else
found_line_number
end
to = if (total_lines - found_line_number) > 3
found_line_number + 2
else
found_line_number
end
data = content.lines[from..to]
OpenStruct.new(
filename: filename,
ref: ref,
startline: from + 1,
data: data.join
)
end
def parse_search_result_from_grep(result)
ref = nil ref = nil
filename = nil filename = nil
startline = 0 startline = 0
......
app/models/snippet.rb
View file @ fe605235
...@@ -21,6 +21,7 @@ class Snippet < ActiveRecord::Base ...@@ -21,6 +21,7 @@ class Snippet < ActiveRecord::Base
include Participable include Participable
include Referable include Referable
include Sortable include Sortable
include SnippetsSearch
default_value_for :visibility_level, Snippet::PRIVATE default_value_for :visibility_level, Snippet::PRIVATE
......
app/services/search/global_service.rb
View file @ fe605235
...@@ -12,7 +12,11 @@ module Search ...@@ -12,7 +12,11 @@ module Search
projects = projects.in_namespace(group.id) if group projects = projects.in_namespace(group.id) if group
project_ids = projects.pluck(:id) project_ids = projects.pluck(:id)
Gitlab::SearchResults.new(project_ids, params[:search]) if Gitlab.config.elasticsearch.enabled
Gitlab::Elastic::SearchResults.new(project_ids, params[:search])
else
Gitlab::SearchResults.new(project_ids, params[:search])
end
end end
end end
end end
app/services/search/project_service.rb
View file @ fe605235
...@@ -7,9 +7,15 @@ module Search ...@@ -7,9 +7,15 @@ module Search
end end
def execute def execute
Gitlab::ProjectSearchResults.new(project.id, if Gitlab.config.elasticsearch.enabled
params[:search], Gitlab::Elastic::ProjectSearchResults.new(project.id,
params[:repository_ref]) params[:search],
params[:repository_ref])
else
Gitlab::ProjectSearchResults.new(project.id,
params[:search],
params[:repository_ref])
end
end end
end end
end end
app/services/search/snippet_service.rb
View file @ fe605235
...@@ -8,7 +8,12 @@ module Search ...@@ -8,7 +8,12 @@ module Search
def execute def execute
snippet_ids = Snippet.accessible_to(current_user).pluck(:id) snippet_ids = Snippet.accessible_to(current_user).pluck(:id)
Gitlab::SnippetSearchResults.new(snippet_ids, params[:search])
if Gitlab.config.elasticsearch.enabled
Gitlab::Elastic::SnippetSearchResults.new(snippet_ids, params[:search])
else
Gitlab::SnippetSearchResults.new(snippet_ids, params[:search])
end
end end
end end
end end
app/services/search_service.rb deleted 100644 → 0
View file @ c0f6df43
class SearchService < BaseService
def global_search
query = params[:search]
{
groups: search_in_groups(query),
users: search_in_users(query),
projects: search_in_projects(query),
merge_requests: search_in_merge_requests(query),
issues: search_in_issues(query),
repositories: search_in_repository(query),
}
end
def project_search(project)
query = params[:search]
{
groups: {},
users: {},
projects: {},
merge_requests: search_in_merge_requests(query, project),
issues: search_in_issues(query, project),
repositories: search_in_repository(query, project),
}
end
private
def search_in_projects(query)
opt = {
pids: projects_ids,
order: params[:order],
fields: %w(name^10 path^9 description^5
name_with_namespace^2 path_with_namespace),
highlight: true
}
group = Group.find_by(id: params[:group_id]) if params[:group_id].present?
opt[:namespace_id] = group.id if group
opt[:category] = params[:category] if params[:category].present?
begin
response = Project.search(query, options: opt, page: page)
categories_list = if query.blank?
Project.category_counts.map do |category|
{ category: category.name, count: category.count }
end
else
response.response["facets"]["categoryFacet"]["terms"].map do |term|
{ category: term["term"], count: term["count"] }
end
end
{
records: response.records,
results: response.results,
response: response.response,
total_count: response.total_count,
namespaces: namespaces(response.response["facets"]["namespaceFacet"]["terms"]),
categories: categories_list
}
rescue Exception => e
{}
end
end
def search_in_groups(query)
opt = {
gids: current_user ? current_user.authorized_groups.ids : [],
order: params[:order],
fields: %w(name^10 path^5 description),
highlight: true
}
begin
response = Group.search(query, options: opt, page: page)
{
records: response.records,
results: response.results,
response: response.response,
total_count: response.total_count
}
rescue Exception => e
{}
end
end
def search_in_users(query)
opt = {
active: true,
order: params[:order],
highlight: true
}
begin
response = User.search(query, options: opt, page: page)
{
records: response.records,
results: response.results,
response: response.response,
total_count: response.total_count
}
rescue Exception => e
{}
end
end
def search_in_merge_requests(query, project = nil)
opt = {
projects_ids: project ? [project.id] : projects_ids,
order: params[:order],
highlight: true
}
begin
response = MergeRequest.search(query, options: opt, page: page)
{
records: response.records,
results: response.results,
response: response.response,
total_count: response.total_count
}
rescue Exception => e
{}
end
end
def search_in_issues(query, project = nil)
opt = {
projects_ids: project ? [project.id] : projects_ids,
order: params[:order]
}
begin
response = Issue.search(query, options: opt, page: page)
{
records: response.records,
results: response.results,
response: response.response,
total_count: response.total_count
}
rescue Exception => e
{}
end
end
def search_in_repository(query, project = nil)
opt = {
repository_id: project ? [project.id] : projects_ids,
highlight: true,
order: params[:order]
}
if params[:language].present? && params[:language] != 'All'
opt.merge!({ language: params[:language] })
end
begin
res = Repository.search(query, options: opt, page: page)
res[:blobs][:projects] = project_filter(res[:blobs][:repositories]) || []
res[:commits][:projects] = project_filter(res[:commits][:repositories]) || []
res
rescue Exception => e
{}
end
end
def projects_ids
@allowed_projects_ids ||= begin
if params[:namespace].present?
namespace = Namespace.find_by(path: params[:namespace])
if namespace
return namespace.projects.where(id: known_projects_ids).pluck(:id)
end
end
known_projects_ids
end
end
def page
return @current_page if defined?(@current_page)
@current_page = params[:page].to_i
@current_page = 1 if @current_page == 0
@current_page
end
def known_projects_ids
known_projects_ids = []
known_projects_ids += current_user.known_projects.pluck(:id) if current_user
known_projects_ids + Project.public_or_internal_only(current_user).pluck(:id)
end
def project_filter(es_results)
terms = es_results.
select { |term| term['count'] > 0 }.
inject({}) do |memo, term|
memo[term["term"]] = term["count"]
memo
end
projects_meta_data = Project.joins(:namespace).where(id: terms.keys).
pluck(['projects.name','projects.path',
'namespaces.name as namespace_name',
'namespaces.path as namespace_path',
'projects.id'].join(","))
if projects_meta_data.any?
projects_meta_data.map do |meta|
{
name: meta[2] + ' / ' + meta[0],
path: meta[3] + ' / ' + meta[1],
count: terms[meta[4]]
}
end.sort { |x, y| y[:count] <=> x[:count] }
else
[]
end
end
def namespaces(terms)
founded_terms = terms.select { |term| term['count'] > 0 }
grouped_terms = founded_terms.inject({}) do |memo, term|
memo[term["term"]] = term["count"]
memo
end
namespaces_meta_data = Namespace.find(grouped_terms.keys)
if namespaces_meta_data.any?
namespaces_meta_data.map do |namespace|
{ namespace: namespace, count: grouped_terms[namespace.id] }
end.sort { |x, y| y[:count] <=> x[:count] }
else
[]
end
end
end
app/views/search/results/_blob.html.haml
View file @ fe605235
- blob = @project.repository.parse_search_result(blob) - blob = @project.repository.parse_search_result(blob, true)
.blob-result .blob-result
.file-holder .file-holder
.file-title .file-title
......
app/views/search/results/_wiki_blob.html.haml
View file @ fe605235
- wiki_blob = @project.repository.parse_search_result(wiki_blob) - wiki_blob = @project.repository.parse_search_result(wiki_blob, true)
.blob-result .blob-result
.file-holder .file-holder
.file-title .file-title
......
app/workers/elastic_indexer_worker.rb 0 → 100644
View file @ fe605235
class ElasticIndexerWorker
include Sidekiq::Worker
sidekiq_options queue: :elasticsearch
Client = Elasticsearch::Client.new(host: Gitlab.config.elasticsearch.host,
port: Gitlab.config.elasticsearch.port)
def perform(operation, klass, record_id, options = {})
klass = "Snippet" if klass =~ /Snippet$/
cklass = klass.constantize
case operation.to_s
when /index|update/
record = cklass.find(record_id)
record.__elasticsearch__.client = Client
record.__elasticsearch__.__send__ "#{operation}_document"
when /delete/
Client.delete index: cklass.index_name, type: cklass.document_type, id: record_id
end
end
end
config/gitlab.yml.example
View file @ fe605235
...@@ -136,6 +136,14 @@ production: &base ...@@ -136,6 +136,14 @@ production: &base
# The location where LFS objects are stored (default: shared/lfs-objects). # The location where LFS objects are stored (default: shared/lfs-objects).
# storage_path: shared/lfs-objects # storage_path: shared/lfs-objects
## Elasticsearch (EE only)
# Enable it if you are going to use elasticsearch instead of
# regular database search
elasticsearch:
enabled: false
host: localhost
port: 9200
## GitLab Pages ## GitLab Pages
pages: pages:
enabled: false enabled: false
......
config/initializers/1_settings.rb
View file @ fe605235
...@@ -246,6 +246,12 @@ Settings.gitlab['restricted_signup_domains'] ||= [] ...@@ -246,6 +246,12 @@ Settings.gitlab['restricted_signup_domains'] ||= []
Settings.gitlab['import_sources'] ||= ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'] Settings.gitlab['import_sources'] ||= ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git']
#
# Elasticseacrh
#
Settings['elasticsearch'] ||= Settingslogic.new({})
Settings.elasticsearch['enabled'] = false if Settings.elasticsearch['enabled'].nil?
# #
# CI # CI
# #
......
lib/gitlab/elastic/project_search_results.rb 0 → 100644
View file @ fe605235
module Gitlab
module Elastic
class ProjectSearchResults < SearchResults
attr_reader :project, :repository_ref
def initialize(project_id, query, repository_ref = nil)
@project = Project.find(project_id)
@repository_ref = if repository_ref.present?
repository_ref
else
nil
end
@query = query
end
def objects(scope, page = nil)
case scope
when 'notes'
notes.records.page(page).per(per_page)
when 'blobs'
blobs.response.page(page).per(per_page)
when 'wiki_blobs'
wiki_blobs.response.page(page).per(per_page)
when 'commits'
Kaminari.paginate_array(commits).page(page).per(per_page)
else
super
end
end
def total_count
@total_count ||= issues_count + merge_requests_count + blobs_count +
notes_count + wiki_blobs_count + commits_count
end
def blobs_count
@blobs_count ||= blobs.total_count
end
def notes_count
@notes_count ||= notes.total_count
end
def wiki_blobs_count
@wiki_blobs_count ||= wiki_blobs.total_count
end
def commits_count
@commits_count ||= commits.count
end
private
def blobs
if project.empty_repo? || query.blank?
Kaminari.paginate_array([])
else
project.repository.search(query, type: :blob, options: {highlight: true})[:blobs][:results]
end
end
def wiki_blobs
if project.wiki_enabled? && !project.wiki.empty? && query.present?
project.wiki.search(query, type: :blob, options: {highlight: true})[:blobs][:results]
else
Kaminari.paginate_array([])
end
end
def notes
opt = {
project_ids: limit_project_ids
}
Note.elastic_search(query, options: opt)
end
def commits
if project.empty_repo? || query.blank?
Kaminari.paginate_array([])
else
project.repository.find_commits_by_message_with_elastic(query)
end
end
def limit_project_ids
[project.id]
end
end
end
end
\ No newline at end of file
lib/gitlab/elastic/search_results.rb 0 → 100644
View file @ fe605235
module Gitlab
module Elastic
class SearchResults
attr_reader :query
# Limit search results by passed project ids
# It allows us to search only for projects user has access to
attr_reader :limit_project_ids
def initialize(limit_project_ids, query)
@limit_project_ids = limit_project_ids || Project.all
@query = Shellwords.shellescape(query) if query.present?
end
def objects(scope, page = nil)
case scope
when 'projects'
projects.records.page(page).per(per_page)
when 'issues'
issues.records.page(page).per(per_page)
when 'merge_requests'
merge_requests.records.page(page).per(per_page)
when 'milestones'
milestones.records.page(page).per(per_page)
else
Kaminari.paginate_array([])
end
end
def total_count
@total_count ||= projects_count + issues_count + merge_requests_count + milestones_count
end
def projects_count
@projects_count ||= projects.total_count
end
def issues_count
@issues_count ||= issues.total_count
end
def merge_requests_count
@merge_requests_count ||= merge_requests.total_count
end
def milestones_count
@milestones_count ||= milestones.total_count
end
def empty?
total_count.zero?
end
private
def projects
opt = {
pids: limit_project_ids
}
@projects = Project.elastic_search(query, options: opt)
end
def issues
opt = {
projects_ids: limit_project_ids
}
if query =~ /#(\d+)\z/
issues = Issue.where(project_id: limit_project_ids).where(iid: $1)
else
issues = Issue.elastic_search(query, options: opt)
end
end
def milestones
opt = {
projects_ids: limit_project_ids
}
milestones = Milestone.elastic_search(query, options: opt)
end
def merge_requests
opt = {
projects_ids: limit_project_ids
}
if query =~ /[#!](\d+)\z/
merge_requests = MergeRequest.in_projects(limit_project_ids).where(iid: $1)
else
merge_requests = MergeRequest.elastic_search(query, options: opt)
end
end
def default_scope
'projects'
end
def per_page
20
end
end
end
end
lib/gitlab/elastic/snippet_search_results.rb 0 → 100644
View file @ fe605235
module Gitlab
module Elastic
class SnippetSearchResults < SearchResults
attr_reader :limit_snippet_ids
def initialize(limit_snippet_ids, query)
@limit_snippet_ids = limit_snippet_ids
@query = query
end
def objects(scope, page = nil)
case scope
when 'snippet_titles'
snippet_titles.records.page(page).per(per_page)
when 'snippet_blobs'
# We process whole list of items then paginate it. Not too smart
# Should be refactored in the CE side first to prevent conflicts hell
Kaminari.paginate_array(
snippet_blobs.records.map do
|snippet| chunk_snippet(snippet)
end
).page(page).per(per_page)
else
super
end
end
def total_count
@total_count ||= snippet_titles_count + snippet_blobs_count
end
def snippet_titles_count
@snippet_titles_count ||= snippet_titles.total_count
end
def snippet_blobs_count
@snippet_blobs_count ||= snippet_blobs.total_count
end
private
def snippet_titles
opt = {
ids: limit_snippet_ids
}
Snippet.elastic_search(query, options: opt)
end
def snippet_blobs
opt = {
ids: limit_snippet_ids
}
Snippet.elastic_search_code(query, options: opt)
end
def default_scope
'snippet_blobs'
end
# Get an array of line numbers surrounding a matching
# line, bounded by min/max.
#
# @returns Array of line numbers
def bounded_line_numbers(line, min, max)
lower = line - surrounding_lines > min ? line - surrounding_lines : min
upper = line + surrounding_lines < max ? line + surrounding_lines : max
(lower..upper).to_a
end
# Returns a sorted set of lines to be included in a snippet preview.
# This ensures matching adjacent lines do not display duplicated
# surrounding code.
#
# @returns Array, unique and sorted.
def matching_lines(lined_content)
used_lines = []
lined_content.each_with_index do |line, line_number|
used_lines.concat bounded_line_numbers(
line_number,
0,
lined_content.size
) if line.include?(query)
end
used_lines.uniq.sort
end
# 'Chunkify' entire snippet. Splits the snippet data into matching lines +
# surrounding_lines() worth of unmatching lines.
#
# @returns a hash with {snippet_object, snippet_chunks:{data,start_line}}
def chunk_snippet(snippet)
lined_content = snippet.content.split("\n")
used_lines = matching_lines(lined_content)
snippet_chunk = []
snippet_chunks = []
snippet_start_line = 0
last_line = -1
# Go through each used line, and add consecutive lines as a single chunk
# to the snippet chunk array.
used_lines.each do |line_number|
if last_line < 0
# Start a new chunk.
snippet_start_line = line_number
snippet_chunk << lined_content[line_number]
elsif last_line == line_number - 1
# Consecutive line, continue chunk.
snippet_chunk << lined_content[line_number]
else
# Non-consecutive line, add chunk to chunk array.
snippet_chunks << {
data: snippet_chunk.join("\n"),
start_line: snippet_start_line + 1
}
# Start a new chunk.
snippet_chunk = [lined_content[line_number]]
snippet_start_line = line_number
end
last_line = line_number
end
# Add final chunk to chunk array
snippet_chunks << {
data: snippet_chunk.join("\n"),
start_line: snippet_start_line + 1
}
# Return snippet with chunk array
{ snippet_object: snippet, snippet_chunks: snippet_chunks }
end
# Defines how many unmatching lines should be
# included around the matching lines in a snippet
def surrounding_lines
3
end
end
end
end
lib/tasks/gitlab/elastic.rake 0 → 100644
View file @ fe605235
namespace :gitlab do
namespace :elastic do
desc "Indexing repositories"
task index_repository: :environment do
Repository.import
end
desc "Indexing all wikis"
task index_wiki: :environment do
ProjectWiki.import
end
desc "Create indexes in the Elasticsearch from database records"
task create_index: :environment do
[Project, User, Issue, MergeRequest, Snippet, Note, Milestone].each do |klass|
klass.__elasticsearch__.create_index!
klass.import
end
end
end
end
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7