Commit 15b602ab authored by Valery Sizov's avatar Valery Sizov

Merge branch 'es_application_settings' into 'master'

ES settings in the ApplicationSetting



See merge request !359
parents 648424a4 756f9624
......@@ -7,6 +7,7 @@ v 8.9.0 (unreleased)
- Instrument instance methods of Gitlab::InsecureKeyFingerprint class
- Add API endpoint for Merge Request Approvals !449
- Distribute RepositoryUpdateMirror jobs in time and add exclusive lease on them by project_id
- [Elastic] Move ES settings to application settings
v 8.8.5
- Make sure OAuth routes that we generate for Geo matches with the ones in Rails routes !444
......
......@@ -42,6 +42,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
def application_setting_params
restricted_levels = params[:application_setting][:restricted_visibility_levels]
if restricted_levels.nil?
params[:application_setting][:restricted_visibility_levels] = []
else
......@@ -51,6 +52,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
end
import_sources = params[:application_setting][:import_sources]
if import_sources.nil?
params[:application_setting][:import_sources] = []
else
......@@ -111,6 +113,10 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
:metrics_packet_size,
:send_user_confirmation_email,
:container_registry_token_expire_delay,
:elasticsearch_indexing,
:elasticsearch_search,
:elasticsearch_host,
:elasticsearch_port,
restricted_visibility_levels: [],
import_sources: [],
disabled_oauth_sign_in_sources: []
......
......@@ -55,6 +55,14 @@ class ApplicationSetting < ActiveRecord::Base
presence: true,
numericality: { only_integer: true, greater_than: 0 }
validates :elasticsearch_host,
presence: { message: "can't be blank when indexing is enabled" },
if: :elasticsearch_indexing?
validates :elasticsearch_port,
presence: { message: "can't be blank when indexing is enabled" },
if: :elasticsearch_indexing?
validates_each :restricted_visibility_levels do |record, attr, value|
unless value.nil?
value.each do |level|
......@@ -134,9 +142,15 @@ class ApplicationSetting < ActiveRecord::Base
disabled_oauth_sign_in_sources: [],
send_user_confirmation_email: false,
container_registry_token_expire_delay: 5,
elasticsearch_host: ENV['ELASTIC_HOST'] || 'localhost',
elasticsearch_port: ENV['ELASTIC_PORT'] || '9200'
)
end
def elasticsearch_host
read_attribute(:elasticsearch_host).split(',').map(&:strip)
end
def home_page_url_column_exist
ActiveRecord::Base.connection.column_exists?(:application_settings, :home_page_url)
end
......
module Elastic
module ApplicationSearch
extend ActiveSupport::Concern
extend Gitlab::CurrentSettings
included do
include Elasticsearch::Model
self.__elasticsearch__.client = Elasticsearch::Client.new(
host: Gitlab.config.elasticsearch.host,
port: Gitlab.config.elasticsearch.port
)
index_name [Rails.application.class.parent_name.downcase, self.name.downcase, Rails.env].join('-')
settings \
......@@ -31,13 +27,13 @@ module Elastic
}
after_commit on: :create do
if Gitlab.config.elasticsearch.enabled && self.searchable?
if current_application_settings.elasticsearch_indexing? && self.searchable?
ElasticIndexerWorker.perform_async(:index, self.class.to_s, self.id)
end
end
after_commit on: :update do
if Gitlab.config.elasticsearch.enabled && self.searchable?
if current_application_settings.elasticsearch_indexing? && self.searchable?
ElasticIndexerWorker.perform_async(
:update,
self.class.to_s,
......@@ -48,12 +44,12 @@ module Elastic
end
after_commit on: :destroy do
if Gitlab.config.elasticsearch.enabled && self.searchable?
if current_application_settings.elasticsearch_indexing? && self.searchable?
ElasticIndexerWorker.perform_async(:delete, self.class.to_s, self.id)
end
end
# Should be overridden in the models where not eveything should be indexed
# Should be overridden in the models where some records should be skipped
def searchable?
true
end
......
......@@ -5,11 +5,6 @@ module Elastic
included do
include Elasticsearch::Git::Repository
self.__elasticsearch__.client = Elasticsearch::Client.new(
host: Gitlab.config.elasticsearch.host,
port: Gitlab.config.elasticsearch.port
)
def repository_id
project.id
end
......
......@@ -5,11 +5,6 @@ module Elastic
included do
include Elasticsearch::Git::Repository
self.__elasticsearch__.client = Elasticsearch::Client.new(
host: Gitlab.config.elasticsearch.host,
port: Gitlab.config.elasticsearch.port
)
def repository_id
"wiki_#{project.id}"
end
......
......@@ -229,7 +229,7 @@ class Project < ActiveRecord::Base
project.save
end
if Gitlab.config.elasticsearch.enabled
if current_application_settings.elasticsearch_indexing?
project.repository.index_blobs
project.repository.index_commits
end
......
class ProjectWiki
include Gitlab::ShellAdapter
include Elastic::WikiRepositoriesSearch
include Gitlab::CurrentSettings
MARKUPS = {
'Markdown' => :markdown,
......@@ -193,6 +194,6 @@ class ProjectWiki
end
def update_elastic_index
index_blobs if Gitlab.config.elasticsearch.enabled
index_blobs if current_application_settings.elasticsearch_indexing?
end
end
......@@ -74,7 +74,7 @@ class GitPushService < BaseService
CreateCommitBuildsService.new.execute(@project, current_user, build_push_data, mirror_update: mirror_update)
ProjectCacheWorker.perform_async(@project.id)
index_commits_blobs if Gitlab.config.elasticsearch.enabled
index_commits_blobs if current_application_settings.elasticsearch_indexing?
end
def index_commits_blobs
......
module Search
class GlobalService
include Gitlab::CurrentSettings
attr_accessor :current_user, :params
def initialize(user, params)
......@@ -11,7 +13,7 @@ module Search
projects = ProjectsFinder.new.execute(current_user)
projects = projects.in_namespace(group.id) if group
if Gitlab.config.elasticsearch.enabled
if current_application_settings.elasticsearch_search?
Gitlab::Elastic::SearchResults.new(current_user, projects.pluck(:id), params[:search])
else
Gitlab::SearchResults.new(current_user, projects, params[:search])
......
module Search
class ProjectService
include Gitlab::CurrentSettings
attr_accessor :project, :current_user, :params
def initialize(project, user, params)
......@@ -7,7 +9,7 @@ module Search
end
def execute
if Gitlab.config.elasticsearch.enabled
if current_application_settings.elasticsearch_search?
Gitlab::Elastic::ProjectSearchResults.new(current_user,
project.id,
params[:search],
......
module Search
class SnippetService
include Gitlab::CurrentSettings
attr_accessor :current_user, :params
def initialize(user, params)
......@@ -7,7 +8,7 @@ module Search
end
def execute
if Gitlab.config.elasticsearch.enabled
if current_application_settings.elasticsearch_search?
Gitlab::Elastic::SnippetSearchResults.new(current_user,
params[:search])
else
......
......@@ -340,6 +340,34 @@
.help-block
If you got a lot of false alarms from repository checks you can choose to clear all repository check information from the database.
%fieldset
%legend Elasticsearch
.form-group
.col-sm-offset-2.col-sm-10
.checkbox
= f.label :elasticsearch_indexing do
= f.check_box :elasticsearch_indexing
Elasticsearch indexing
.form-group
.col-sm-offset-2.col-sm-10
.checkbox
= f.label :elasticsearch_search do
= f.check_box :elasticsearch_search
Search with Elasticsearch enabled
.form-group
= f.label :elasticsearch_host, 'Host', class: 'control-label col-sm-2'
.col-sm-10
= f.text_field :elasticsearch_host, value: @application_setting.elasticsearch_host.join(', '), class: 'form-control', placeholder: 'localhost'
.help-block
The TCP/IP host to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "host1, host2").
.form-group
= f.label :elasticsearch_port, 'Port', class: 'control-label col-sm-2'
.col-sm-10
= f.text_field :elasticsearch_port, class: 'form-control', placeholder: ApplicationSetting.current.elasticsearch_port
.form-actions
= f.submit 'Save', class: 'btn btn-save'
......@@ -66,7 +66,7 @@
%p
Elasticsearch
%span.light.pull-right
= boolean_to_icon Gitlab.config.elasticsearch.enabled
= boolean_to_icon current_application_settings.elasticsearch_search?
%p
Geo
%span.light.pull-right
......
class ElasticIndexerWorker
include Sidekiq::Worker
include Elasticsearch::Model::Client::ClassMethods
sidekiq_options queue: :elasticsearch
ISSUE_TRACKED_FIELDS = %w(assignee_id author_id confidential)
Client = Elasticsearch::Client.new(host: Gitlab.config.elasticsearch.host,
port: Gitlab.config.elasticsearch.port)
def perform(operation, class_name, record_id, options = {})
klass = class_name.constantize
case operation.to_s
when /index|update/
record = klass.find(record_id)
record.__elasticsearch__.client = Client
record.__elasticsearch__.client = client
record.__elasticsearch__.__send__ "#{operation}_document"
update_issue_notes(record, options["changed_fields"]) if klass == Issue
when /delete/
Client.delete index: klass.index_name, type: klass.document_type, id: record_id
client.delete index: klass.index_name, type: klass.document_type, id: record_id
clear_project_indexes(record_id) if klass == Project
end
......@@ -35,7 +33,7 @@ class ElasticIndexerWorker
def clear_project_indexes(record_id)
# Remove repository index
Client.delete_by_query({
client.delete_by_query({
index: Repository.__elasticsearch__.index_name,
body: {
query: {
......
class PostReceive
include Sidekiq::Worker
extend Gitlab::CurrentSettings
sidekiq_options queue: :post_receive
......@@ -50,7 +51,7 @@ class PostReceive
end
def update_wiki_es_indexes(post_received)
return unless Gitlab.config.elasticsearch.enabled
return unless current_application_settings.elasticsearch_indexing?
post_received.project.wiki.index_blobs
end
......
......@@ -150,14 +150,6 @@ production: &base
# The location where LFS objects are stored (default: shared/lfs-objects).
# storage_path: shared/lfs-objects
## Elasticsearch (EE only)
# Enable it if you are going to use elasticsearch instead of
# regular database search
elasticsearch:
enabled: false
# host: localhost
# port: 9200
## GitLab Pages
pages:
enabled: false
......
# Be sure to restart your server when you modify this file.
require 'gitlab/current_settings'
module Elasticsearch
module Model
module Client
module ClassMethods
include Gitlab::CurrentSettings
def client(client = nil)
if @client.nil? || es_configuration_changed?
@es_host = current_application_settings.elasticsearch_host
@es_port = current_application_settings.elasticsearch_port
@client = Elasticsearch::Client.new(
host: current_application_settings.elasticsearch_host,
port: current_application_settings.elasticsearch_port
)
end
@client
end
def es_configuration_changed?
@es_host != current_application_settings.elasticsearch_host ||
@es_port != current_application_settings.elasticsearch_port
end
end
end
end
end
class AddEsToApplicationSettings < ActiveRecord::Migration
def up
add_column :application_settings, :elasticsearch_indexing, :boolean, default: false, null: false
add_column :application_settings, :elasticsearch_search, :boolean, default: false, null: false
add_column :application_settings, :elasticsearch_host, :string, default: 'localhost'
add_column :application_settings, :elasticsearch_port, :string, default: '9200'
es_enabled = Settings.elasticsearch['enabled']
es_host = Settings.elasticsearch['host']
es_host = es_host.join(',') if es_host.is_a?(Array)
es_port = Settings.elasticsearch['port']
execute <<-SQL.strip_heredoc
UPDATE application_settings
SET
elasticsearch_indexing = #{es_enabled},
elasticsearch_search = #{es_enabled},
elasticsearch_host = '#{es_host}',
elasticsearch_port = '#{es_port}'
SQL
end
def down
remove_column :application_settings, :elasticsearch_indexing
remove_column :application_settings, :elasticsearch_search
remove_column :application_settings, :elasticsearch_host
remove_column :application_settings, :elasticsearch_port
end
end
......@@ -88,6 +88,10 @@ ActiveRecord::Schema.define(version: 20160615142710) do
t.boolean "send_user_confirmation_email", default: false
t.integer "container_registry_token_expire_delay", default: 5
t.text "after_sign_up_text"
t.boolean "elasticsearch_indexing", default: false, null: false
t.boolean "elasticsearch_search", default: false, null: false
t.string "elasticsearch_host", default: "localhost"
t.string "elasticsearch_port", default: "9200"
end
create_table "approvals", force: :cascade do |t|
......
......@@ -22,7 +22,7 @@ searching in:
- snippets
- wiki repositories
Once the data is added to the database, search indexes will be updated
Once the data is added to the database or repository, search indexes will be updated
automatically. Elasticsearch can be installed on the same machine that GitLab
is installed or on a separate server.
......@@ -45,63 +45,34 @@ use the packages that are available for your OS.
## Enable Elasticsearch
In order to enable Elasticsearch you need to have access to the server that
GitLab is hosted on.
In order to enable Elasticsearch you need to have access to the server that GitLab is hosted on, and an administrator account on your GitLab instance. Go to **Admin > Settings** and find the "Elasticsearch" section.
The following three parameters are needed to enable Elasticsearch:
The following Elasticsearch settings are available:
| Parameter | Description |
| --------- | ----------- |
| `enabled` | Enables/disables the Elasticsearch integration. Can be either `true` or `false` |
| `host` | The host where Elasticsearch is installed on. Can be either an IP or a domain name which correctly resolves to an IP. It can be changed in the [Elasticsearch configuration settings][elastic-settings]. The default value is `localhost` |
| `port` | The TCP port that Elasticsearch listens to. It can be changed in the [Elasticsearch configuration settings][elastic-settings]. The default value is `9200` |
| Parameter | Description |
| --------- | ----------- |
| `Elasticsearch indexing` | Enables/disables Elasticsearch indexing. You may want to enable indexing but disable search in order to give the index time to be fully completed, for example. |
| `Search with Elasticsearch enabled` | Enables/disables using Elasticsearch in search. |
| `Host` | The TCP/IP host to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "host1, host2"). |
| `Port` | The TCP port that Elasticsearch listens to. The default value is 9200 |
### Enable Elasticsearch in Omnibus installations
If you have used one of the [Omnibus packages][pkg] to install GitLab, all
you have to do is edit `/etc/gitlab/gitlab.rb` and add the following lines:
```ruby
gitlab_rails['elasticsearch_enabled'] = true
gitlab_rails['elasticsearch_host'] = "localhost"
gitlab_rails['elasticsearch_port'] = 9200
```
Replace the values as you see fit according to the
[settings table above](#enable-elasticsearch).
Save the file and reconfigure GitLab for the changes to take effect:
`sudo gitlab-ctl reconfigure`.
As a last step, move on to
[add GitLab's data to the Elasticsearch index](#add-gitlabs-data-to-the-elasticsearch-index).
### Enable Elasticsearch in source installations
If you have installed GitLab from source, edit `/home/git/gitlab/config/gitlab.yml`:
## Add GitLab's data to the Elasticsearch index
```yaml
elasticsearch:
enabled: true
host: localhost
port: 9200
```
Configure Elasticsearch's host and port in **Admin > Settings**. Then create empty indexes using one of the following commands:
Replace the values as you see fit according to the
[settings table above](#enable-elasticsearch).
Save the file and restart GitLab for the changes to take effect:
`sudo service gitlab restart`.
```
# Omnibus installations
sudo gitlab-rake gitlab:elastic:create_empty_indexes
As a last step, move on to
[add GitLab's data to the Elasticsearch index](#add-gitlabs-data-to-the-elasticsearch-index).
# Installations from source
bundle exec rake gitlab:elastic:create_empty_indexes
```
## Add GitLab's data to the Elasticsearch index
After [enabling Elasticsearch](#enable-elasticsearch), you must run the
following rake tasks to add GitLab's data to the Elasticsearch index.
It might take a while depending on how big your Git repositories are (see
Then enable Elasticsearch indexing and run indexing tasks. It might take a while depending on how big your Git repositories are (see
[Indexing large repositories](#indexing-large-repositories)).
---
......@@ -167,7 +138,8 @@ sudo gitlab-rake gitlab:elastic:index_database
# Installations from source
bundle exec rake gitlab:elastic:index_database RAILS_ENV=production
```
To index all available entities:
Or everything at once (database records, repositories, wikis):
```
# Omnibus installations
......@@ -179,10 +151,7 @@ bundle exec rake gitlab:elastic:index RAILS_ENV=production
## Disable Elasticsearch
Disabling the Elasticsearch integration is as easy as setting `enabled` to
`false` in your GitLab settings. See [Enable Elasticsearch](#enable-elasticsearch)
to find where those settings are and don't forget to reconfigure/restart GitLab
for the changes to take effect.
Disabling the Elasticsearch integration is as easy as unchecking `Search with Elasticsearch enabled` and `Elasticsearch indexing` in **Admin > Settings**.
## Special recommendations
......@@ -241,8 +210,7 @@ time drop.
To minimize downtime of the search feature we recommend the following:
1. Configure Elasticsearch in `gitlab.yml`, or `gitlab.rb` for Omnibus
installations, but do not enable it, just set a host and port.
1. Configure Elasticsearch in **Admin > Settings**, but do not enable it, just set a host and port.
1. Create empty indexes:
......@@ -257,7 +225,7 @@ To minimize downtime of the search feature we recommend the following:
1. Index all repositories using the `gitlab:elastic:index_repositories` Rake
task (see above). You'll probably want to do this in parallel.
1. Enable Elasticsearch and restart GitLab. Note that once enabled the index will be updated when new data is pushed to the GitLab server.
1. Enable Elasticsearch indexing.
1. Run indexers for database (with the `UPDATE_INDEX=1` parameter), wikis, and
repositories. By running the repository indexer twice you will be sure that
......
require Rails.root.join('spec', 'support', 'stub_configuration')
class Spinach::Features::GlobalSearch < Spinach::FeatureSteps
include SharedAuthentication
include SharedPaths
include SharedProject
include SharedElastic
include StubConfiguration
before do
[::Project, Issue, MergeRequest, Milestone].each do |model|
......@@ -15,7 +18,7 @@ class Spinach::Features::GlobalSearch < Spinach::FeatureSteps
model.__elasticsearch__.delete_index!
end
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
end
step 'project has all data available for the search' do
......
......@@ -3,6 +3,7 @@ class Spinach::Features::ProjectSearch < Spinach::FeatureSteps
include SharedPaths
include SharedProject
include SharedElastic
include StubConfiguration
before do
[::Project, Repository, Note, MergeRequest, Milestone, ::ProjectWiki, Issue].each do |model|
......@@ -15,7 +16,7 @@ class Spinach::Features::ProjectSearch < Spinach::FeatureSteps
model.__elasticsearch__.delete_index!
end
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
end
step 'project has all data available for the search' do
......
......@@ -3,6 +3,7 @@ class Spinach::Features::SnippetsSearch < Spinach::FeatureSteps
include SharedPaths
include SharedProject
include SharedElastic
include StubConfiguration
before do
Snippet.__elasticsearch__.create_index!
......@@ -11,7 +12,7 @@ class Spinach::Features::SnippetsSearch < Spinach::FeatureSteps
after do
Snippet.__elasticsearch__.delete_index!
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
end
step 'there is a snippet "index" with "php rocks" string' do
......
......@@ -25,7 +25,7 @@ module SharedElastic
end
step 'Elasticsearch is enabled' do
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(true)
stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
end
def select_filter(name)
......
......@@ -44,6 +44,10 @@ module Gitlab
akismet_enabled: false,
repository_checks_enabled: true,
container_registry_token_expire_delay: 5,
elasticsearch_search: false,
elasticsearch_indexing: false,
elasticsearch_host: ENV['ELASTIC_HOST'] || 'localhost',
elasticsearch_port: ENV['ELASTIC_PORT'] || '9200'
)
end
......
module Gitlab
module Elastic
class Indexer
include Gitlab::CurrentSettings
Error = Class.new(StandardError)
def initialize
connection_info = {
host: Gitlab.config.elasticsearch.host,
port: Gitlab.config.elasticsearch.port
host: current_application_settings.elasticsearch_host,
port: current_application_settings.elasticsearch_port
}.to_json
# We accept any form of settings, including string and array
......
......@@ -30,7 +30,7 @@ namespace :gitlab do
check_ruby_version
check_git_version
check_active_users
check_elasticsearch if Gitlab.config.elasticsearch.enabled
check_elasticsearch if ApplicationSetting.current.elasticsearch_indexing?
finished_checking "GitLab"
end
......@@ -972,8 +972,8 @@ namespace :gitlab do
end
def check_elasticsearch
client = Elasticsearch::Client.new(host: Gitlab.config.elasticsearch.host,
port: Gitlab.config.elasticsearch.port)
client = Elasticsearch::Client.new(host: ApplicationSetting.current.elasticsearch_host,
port: ApplicationSetting.current.elasticsearch_port)
print "Elasticsearch version >= 2.0? ... "
......
......@@ -2,13 +2,14 @@ require 'spec_helper'
describe "Indexer" do
it "runs commands" do
stub_application_setting(es_host: ['elastic-host1', 'elastic-host2'])
expect(Gitlab::Popen).to receive(:popen).with(
[File.join(Rails.root, 'bin/elastic_repo_indexer'), '1', 'full_repo_path'],
nil,
hash_including(
'ELASTIC_CONNECTION_INFO' => {
host: Gitlab.config.elasticsearch.host,
port: Gitlab.config.elasticsearch.port
host: ApplicationSetting.current.elasticsearch_host,
port: ApplicationSetting.current.elasticsearch_port
}.to_json,
'RAILS_ENV' => Rails.env,
'FROM_SHA' => '000000',
......
......@@ -6,13 +6,13 @@ describe Gitlab::Elastic::ProjectSearchResults, lib: true do
let(:query) { 'hello world' }
before do
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(true)
stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
Project.__elasticsearch__.create_index!
Issue.__elasticsearch__.create_index!
end
after do
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
Project.__elasticsearch__.delete_index!
Issue.__elasticsearch__.delete_index!
end
......
......@@ -2,11 +2,11 @@ require 'spec_helper'
describe Gitlab::Elastic::SearchResults, lib: true do
before do
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(true)
stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
end
after do
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
end
let(:user) { create(:user) }
......@@ -89,14 +89,17 @@ describe Gitlab::Elastic::SearchResults, lib: true do
let(:non_member) { create(:user) }
let(:member) { create(:user) }
let(:admin) { create(:admin) }
let!(:issue) { create(:issue, project: project_1, title: 'Issue 1', iid: 1) }
let!(:security_issue_1) { create(:issue, :confidential, project: project_1, title: 'Security issue 1', author: author, iid: 2) }
let!(:security_issue_2) { create(:issue, :confidential, title: 'Security issue 2', project: project_1, assignee: assignee, iid: 3) }
let!(:security_issue_3) { create(:issue, :confidential, project: project_2, title: 'Security issue 3', author: author, iid: 1) }
let!(:security_issue_4) { create(:issue, :confidential, project: project_3, title: 'Security issue 4', assignee: assignee, iid: 1) }
let!(:security_issue_5) { create(:issue, :confidential, project: project_4, title: 'Security issue 5', iid: 1) }
before do
Issue.__elasticsearch__.create_index!
@issue = create(:issue, project: project_1, title: 'Issue 1', iid: 1)
@security_issue_1 = create(:issue, :confidential, project: project_1, title: 'Security issue 1', author: author, iid: 2)
@security_issue_2 = create(:issue, :confidential, title: 'Security issue 2', project: project_1, assignee: assignee, iid: 3)
@security_issue_3 = create(:issue, :confidential, project: project_2, title: 'Security issue 3', author: author, iid: 1)
@security_issue_4 = create(:issue, :confidential, project: project_3, title: 'Security issue 4', assignee: assignee, iid: 1)
@security_issue_5 = create(:issue, :confidential, project: project_4, title: 'Security issue 5', iid: 1)
Issue.__elasticsearch__.refresh_index!
end
......@@ -107,12 +110,12 @@ describe Gitlab::Elastic::SearchResults, lib: true do
results = described_class.new(nil, limit_project_ids, query)
issues = results.objects('issues')
expect(issues).to include issue
expect(issues).not_to include security_issue_1
expect(issues).not_to include security_issue_2
expect(issues).not_to include security_issue_3
expect(issues).not_to include security_issue_4
expect(issues).not_to include security_issue_5
expect(issues).to include @issue
expect(issues).not_to include @security_issue_1
expect(issues).not_to include @security_issue_2
expect(issues).not_to include @security_issue_3
expect(issues).not_to include @security_issue_4
expect(issues).not_to include @security_issue_5
expect(results.issues_count).to eq 1
end
......@@ -120,12 +123,12 @@ describe Gitlab::Elastic::SearchResults, lib: true do
results = described_class.new(non_member, limit_project_ids, query)
issues = results.objects('issues')
expect(issues).to include issue
expect(issues).not_to include security_issue_1
expect(issues).not_to include security_issue_2
expect(issues).not_to include security_issue_3
expect(issues).not_to include security_issue_4
expect(issues).not_to include security_issue_5
expect(issues).to include @issue
expect(issues).not_to include @security_issue_1
expect(issues).not_to include @security_issue_2
expect(issues).not_to include @security_issue_3
expect(issues).not_to include @security_issue_4
expect(issues).not_to include @security_issue_5
expect(results.issues_count).to eq 1
end
......@@ -133,12 +136,12 @@ describe Gitlab::Elastic::SearchResults, lib: true do
results = described_class.new(author, limit_project_ids, query)
issues = results.objects('issues')
expect(issues).to include issue
expect(issues).to include security_issue_1
expect(issues).not_to include security_issue_2
expect(issues).to include security_issue_3
expect(issues).not_to include security_issue_4
expect(issues).not_to include security_issue_5
expect(issues).to include @issue
expect(issues).to include @security_issue_1
expect(issues).not_to include @security_issue_2
expect(issues).to include @security_issue_3
expect(issues).not_to include @security_issue_4
expect(issues).not_to include @security_issue_5
expect(results.issues_count).to eq 3
end
......@@ -146,12 +149,12 @@ describe Gitlab::Elastic::SearchResults, lib: true do
results = described_class.new(assignee, limit_project_ids, query)
issues = results.objects('issues')
expect(issues).to include issue
expect(issues).not_to include security_issue_1
expect(issues).to include security_issue_2
expect(issues).not_to include security_issue_3
expect(issues).to include security_issue_4
expect(issues).not_to include security_issue_5
expect(issues).to include @issue
expect(issues).not_to include @security_issue_1
expect(issues).to include @security_issue_2
expect(issues).not_to include @security_issue_3
expect(issues).to include @security_issue_4
expect(issues).not_to include @security_issue_5
expect(results.issues_count).to eq 3
end
......@@ -162,12 +165,12 @@ describe Gitlab::Elastic::SearchResults, lib: true do
results = described_class.new(member, limit_project_ids, query)
issues = results.objects('issues')
expect(issues).to include issue
expect(issues).to include security_issue_1
expect(issues).to include security_issue_2
expect(issues).to include security_issue_3
expect(issues).not_to include security_issue_4
expect(issues).not_to include security_issue_5
expect(issues).to include @issue
expect(issues).to include @security_issue_1
expect(issues).to include @security_issue_2
expect(issues).to include @security_issue_3
expect(issues).not_to include @security_issue_4
expect(issues).not_to include @security_issue_5
expect(results.issues_count).to eq 4
end
......@@ -175,12 +178,12 @@ describe Gitlab::Elastic::SearchResults, lib: true do
results = described_class.new(admin, limit_project_ids, query)
issues = results.objects('issues')
expect(issues).to include issue
expect(issues).to include security_issue_1
expect(issues).to include security_issue_2
expect(issues).to include security_issue_3
expect(issues).to include security_issue_4
expect(issues).not_to include security_issue_5
expect(issues).to include @issue
expect(issues).to include @security_issue_1
expect(issues).to include @security_issue_2
expect(issues).to include @security_issue_3
expect(issues).to include @security_issue_4
expect(issues).not_to include @security_issue_5
expect(results.issues_count).to eq 5
end
end
......@@ -192,12 +195,12 @@ describe Gitlab::Elastic::SearchResults, lib: true do
results = described_class.new(nil, limit_project_ids, query)
issues = results.objects('issues')
expect(issues).to include issue
expect(issues).not_to include security_issue_1
expect(issues).not_to include security_issue_2
expect(issues).not_to include security_issue_3
expect(issues).not_to include security_issue_4
expect(issues).not_to include security_issue_5
expect(issues).to include @issue
expect(issues).not_to include @security_issue_1
expect(issues).not_to include @security_issue_2
expect(issues).not_to include @security_issue_3
expect(issues).not_to include @security_issue_4
expect(issues).not_to include @security_issue_5
expect(results.issues_count).to eq 1
end
......@@ -205,12 +208,12 @@ describe Gitlab::Elastic::SearchResults, lib: true do
results = described_class.new(non_member, limit_project_ids, query)
issues = results.objects('issues')
expect(issues).to include issue
expect(issues).not_to include security_issue_1
expect(issues).not_to include security_issue_2
expect(issues).not_to include security_issue_3
expect(issues).not_to include security_issue_4
expect(issues).not_to include security_issue_5
expect(issues).to include @issue
expect(issues).not_to include @security_issue_1
expect(issues).not_to include @security_issue_2
expect(issues).not_to include @security_issue_3
expect(issues).not_to include @security_issue_4
expect(issues).not_to include @security_issue_5
expect(results.issues_count).to eq 1
end
......@@ -218,12 +221,12 @@ describe Gitlab::Elastic::SearchResults, lib: true do
results = described_class.new(author, limit_project_ids, query)
issues = results.objects('issues')
expect(issues).to include issue
expect(issues).not_to include security_issue_1
expect(issues).not_to include security_issue_2
expect(issues).to include security_issue_3
expect(issues).not_to include security_issue_4
expect(issues).not_to include security_issue_5
expect(issues).to include @issue
expect(issues).not_to include @security_issue_1
expect(issues).not_to include @security_issue_2
expect(issues).to include @security_issue_3
expect(issues).not_to include @security_issue_4
expect(issues).not_to include @security_issue_5
expect(results.issues_count).to eq 2
end
......@@ -231,12 +234,12 @@ describe Gitlab::Elastic::SearchResults, lib: true do
results = described_class.new(assignee, limit_project_ids, query)
issues = results.objects('issues')
expect(issues).to include issue
expect(issues).not_to include security_issue_1
expect(issues).not_to include security_issue_2
expect(issues).not_to include security_issue_3
expect(issues).to include security_issue_4
expect(issues).not_to include security_issue_5
expect(issues).to include @issue
expect(issues).not_to include @security_issue_1
expect(issues).not_to include @security_issue_2
expect(issues).not_to include @security_issue_3
expect(issues).to include @security_issue_4
expect(issues).not_to include @security_issue_5
expect(results.issues_count).to eq 2
end
......@@ -247,12 +250,12 @@ describe Gitlab::Elastic::SearchResults, lib: true do
results = described_class.new(member, limit_project_ids, query)
issues = results.objects('issues')
expect(issues).to include issue
expect(issues).not_to include security_issue_1
expect(issues).not_to include security_issue_2
expect(issues).to include security_issue_3
expect(issues).to include security_issue_4
expect(issues).not_to include security_issue_5
expect(issues).to include @issue
expect(issues).not_to include @security_issue_1
expect(issues).not_to include @security_issue_2
expect(issues).to include @security_issue_3
expect(issues).to include @security_issue_4
expect(issues).not_to include @security_issue_5
expect(results.issues_count).to eq 3
end
......@@ -260,12 +263,12 @@ describe Gitlab::Elastic::SearchResults, lib: true do
results = described_class.new(admin, limit_project_ids, query)
issues = results.objects('issues')
expect(issues).to include issue
expect(issues).not_to include security_issue_1
expect(issues).not_to include security_issue_2
expect(issues).to include security_issue_3
expect(issues).to include security_issue_4
expect(issues).not_to include security_issue_5
expect(issues).to include @issue
expect(issues).not_to include @security_issue_1
expect(issues).not_to include @security_issue_2
expect(issues).to include @security_issue_3
expect(issues).to include @security_issue_4
expect(issues).not_to include @security_issue_5
expect(results.issues_count).to eq 3
end
end
......
require 'spec_helper'
describe "Issue", elastic: true do
describe Issue, elastic: true do
before do
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(true)
Issue.__elasticsearch__.create_index!
stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
described_class.__elasticsearch__.create_index!
end
after do
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
Issue.__elasticsearch__.delete_index!
described_class.__elasticsearch__.delete_index!
stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
end
it "searches issues" do
......@@ -21,11 +21,11 @@ describe "Issue", elastic: true do
# The issue I have no access to
create :issue, title: 'bla-bla term'
Issue.__elasticsearch__.refresh_index!
described_class.__elasticsearch__.refresh_index!
options = { project_ids: [project.id] }
expect(Issue.elastic_search('term', options: options).total_count).to eq(2)
expect(described_class.elastic_search('term', options: options).total_count).to eq(2)
end
it "returns json with all needed elements" do
......
require 'spec_helper'
describe "MergeRequest", elastic: true do
describe MergeRequest, elastic: true do
before do
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(true)
MergeRequest.__elasticsearch__.create_index!
stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
described_class.__elasticsearch__.create_index!
end
after do
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
MergeRequest.__elasticsearch__.delete_index!
described_class.__elasticsearch__.delete_index!
stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
end
it "searches merge requests" do
......@@ -21,11 +21,11 @@ describe "MergeRequest", elastic: true do
# The merge request you have no access to
create :merge_request, title: 'also with term'
MergeRequest.__elasticsearch__.refresh_index!
described_class.__elasticsearch__.refresh_index!
options = { project_ids: [project.id] }
expect(MergeRequest.elastic_search('term', options: options).total_count).to eq(2)
expect(described_class.elastic_search('term', options: options).total_count).to eq(2)
end
it "returns json with all needed elements" do
......
require 'spec_helper'
describe "Milestone", elastic: true do
describe Milestone, elastic: true do
before do
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(true)
Milestone.__elasticsearch__.create_index!
stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
described_class.__elasticsearch__.create_index!
end
after do
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
Milestone.__elasticsearch__.delete_index!
described_class.__elasticsearch__.delete_index!
stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
end
it "searches milestones" do
......@@ -21,11 +21,11 @@ describe "Milestone", elastic: true do
# The milestone you have no access to
create :milestone, title: 'bla-bla term'
Milestone.__elasticsearch__.refresh_index!
described_class.__elasticsearch__.refresh_index!
options = { project_ids: [project.id] }
expect(Milestone.elastic_search('term', options: options).total_count).to eq(2)
expect(described_class.elastic_search('term', options: options).total_count).to eq(2)
end
it "returns json with all needed elements" do
......
require 'spec_helper'
describe "Note", elastic: true do
describe Note, elastic: true do
before do
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(true)
Note.__elasticsearch__.create_index!
stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
described_class.__elasticsearch__.create_index!
end
after do
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
Note.__elasticsearch__.delete_index!
described_class.__elasticsearch__.delete_index!
stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
end
it "searches notes" do
......@@ -20,11 +20,11 @@ describe "Note", elastic: true do
# The note in the project you have no access to
create :note, note: 'bla-bla term'
Note.__elasticsearch__.refresh_index!
described_class.__elasticsearch__.refresh_index!
options = { project_ids: [issue.project.id] }
expect(Note.elastic_search('term', options: options).total_count).to eq(1)
expect(described_class.elastic_search('term', options: options).total_count).to eq(1)
end
it "returns json with all needed elements" do
......
require 'spec_helper'
describe "Projects", elastic: true do
describe Project, elastic: true do
before do
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(true)
Project.__elasticsearch__.create_index!
stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
described_class.__elasticsearch__.create_index!
end
after do
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
Project.__elasticsearch__.delete_index!
described_class.__elasticsearch__.delete_index!
stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
end
it "searches projects" do
......@@ -18,11 +18,11 @@ describe "Projects", elastic: true do
create :empty_project, path: 'someone_elses_project'
project_ids = [project.id, project1.id, project2.id]
Project.__elasticsearch__.refresh_index!
described_class.__elasticsearch__.refresh_index!
expect(Project.elastic_search('test', options: { pids: project_ids }).total_count).to eq(1)
expect(Project.elastic_search('test1', options: { pids: project_ids }).total_count).to eq(1)
expect(Project.elastic_search('someone_elses_project', options: { pids: project_ids }).total_count).to eq(0)
expect(described_class.elastic_search('test', options: { pids: project_ids }).total_count).to eq(1)
expect(described_class.elastic_search('test1', options: { pids: project_ids }).total_count).to eq(1)
expect(described_class.elastic_search('someone_elses_project', options: { pids: project_ids }).total_count).to eq(0)
end
it "returns json with all needed elements" do
......
require 'spec_helper'
describe "Repository", elastic: true do
describe Repository, elastic: true do
before do
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(true)
Repository.__elasticsearch__.create_index!
stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
described_class.__elasticsearch__.create_index!
end
after do
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
Repository.__elasticsearch__.delete_index!
described_class.__elasticsearch__.delete_index!
stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
end
it "searches blobs and commits" do
......@@ -16,8 +16,8 @@ describe "Repository", elastic: true do
project.repository.index_blobs
project.repository.index_commits
Repository.__elasticsearch__.refresh_index!
described_class.__elasticsearch__.refresh_index!
expect(project.repository.search('def popen')[:blobs][:total_count]).to eq(1)
expect(project.repository.search('initial')[:commits][:total_count]).to eq(1)
......
require 'spec_helper'
describe "Snippet", elastic: true do
describe Snippet, elastic: true do
before do
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(true)
Snippet.__elasticsearch__.create_index!
stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
described_class.__elasticsearch__.create_index!
end
after do
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
Snippet.__elasticsearch__.delete_index!
described_class.__elasticsearch__.delete_index!
stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
end
it "searches snippets by code" do
......@@ -20,11 +20,11 @@ describe "Snippet", elastic: true do
snippet3 = create :personal_snippet, :public, content: 'genius code'
Snippet.__elasticsearch__.refresh_index!
described_class.__elasticsearch__.refresh_index!
options = { author_id: user.id }
result = Snippet.elastic_search_code('genius code', options: options)
result = described_class.elastic_search_code('genius code', options: options)
expect(result.total_count).to eq(2)
expect(result.records.map(&:id)).to include(snippet.id, snippet3.id)
......@@ -38,12 +38,12 @@ describe "Snippet", elastic: true do
create :snippet, :public, file_name: 'index.php'
create :snippet
Snippet.__elasticsearch__.refresh_index!
described_class.__elasticsearch__.refresh_index!
options = { author_id: user.id }
expect(Snippet.elastic_search('home', options: options).total_count).to eq(1)
expect(Snippet.elastic_search('index.php', options: options).total_count).to eq(1)
expect(described_class.elastic_search('home', options: options).total_count).to eq(1)
expect(described_class.elastic_search('index.php', options: options).total_count).to eq(1)
end
it "returns json with all needed elements" do
......
require 'spec_helper'
describe "ProjectWiki", elastic: true do
describe ProjectWiki, elastic: true do
before do
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(true)
ProjectWiki.__elasticsearch__.create_index!
stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
described_class.__elasticsearch__.create_index!
end
after do
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
ProjectWiki.__elasticsearch__.delete_index!
described_class.__elasticsearch__.delete_index!
stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
end
it "searches wiki page" do
......@@ -17,8 +17,8 @@ describe "ProjectWiki", elastic: true do
project.wiki.create_page("index_page", "Bla bla")
project.wiki.index_blobs
ProjectWiki.__elasticsearch__.refresh_index!
described_class.__elasticsearch__.refresh_index!
expect(project.wiki.search('bla', type: :blob)[:blobs][:total_count]).to eq(1)
end
......
......@@ -800,11 +800,13 @@ describe Repository, models: true do
describe "Elastic search", elastic: true do
before do
stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
Repository.__elasticsearch__.create_index!
end
after do
Repository.__elasticsearch__.delete_index!
stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
end
describe :find_commits_by_message_with_elastic do
......
......@@ -139,11 +139,11 @@ describe GitPushService, services: true do
describe "ES indexing" do
before do
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(true)
stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
end
after do
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
end
it "triggers indexer" do
......
......@@ -75,7 +75,7 @@ describe PostReceive do
it "triggers wiki index update" do
expect(Project).to receive(:find_with_namespace).with("#{project.path_with_namespace}.wiki").and_return(nil)
expect(Project).to receive(:find_with_namespace).with(project.path_with_namespace).and_return(project)
allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(true)
stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
expect_any_instance_of(ProjectWiki).to receive(:index_blobs)
repo_path = "#{pwd(project)}.wiki"
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment