Skip to content

G
gitlab-ce
Commit a6013817 authored by Dmytro Zaporozhets (DZ)'s avatar Dmytro Zaporozhets (DZ)
Browse files
Options

Merge branch '326979-remove-waf-backend-code' into 'master' 

Remove GitLab WAF related models, services and workers [RUN ALL RSPEC] [RUN AS-IF-FOSS]

See merge request gitlab-org/gitlab!61556
parents 9ff47402 d1146081
Show whitespace changes
Inline Side-by-side
Showing with 15 additions and 1240 deletions
app/controllers/clusters/applications_controller.rb
View file @ a6013817
......@@ -47,7 +47,7 @@ class Clusters::ApplicationsController < Clusters::BaseController
end
def cluster_application_params
params.permit(:application, :hostname, :pages_domain_id, :email, :stack, :modsecurity_enabled, :modsecurity_mode, :host, :port, :protocol, :waf_log_enabled, :cilium_log_enabled)
params.permit(:application, :hostname, :pages_domain_id, :email, :stack, :host, :port, :protocol, :cilium_log_enabled)
end
def cluster_application_destroy_params
......
app/models/clusters/applications/fluentd.rb
View file @ a6013817
......@@ -12,11 +12,13 @@ module Clusters
include ::Clusters::Concerns::ApplicationStatus
include ::Clusters::Concerns::ApplicationVersion
include ::Clusters::Concerns::ApplicationData
include IgnorableColumns
default_value_for :version, VERSION
default_value_for :port, 514
default_value_for :protocol, :tcp
default_value_for :waf_log_enabled, false
ignore_column :waf_log_enabled, remove_with: '14.2', remove_after: '2021-07-22'
enum protocol: { tcp: 0, udp: 1 }
......@@ -48,9 +50,7 @@ module Clusters
private
def has_at_least_one_log_enabled?
if !waf_log_enabled && !cilium_log_enabled
errors.add(:base, _("At least one logging option is required to be enabled"))
end
errors.add(:base, _("At least one logging option is required to be enabled")) unless cilium_log_enabled
end
def content_values
......@@ -113,7 +113,6 @@ module Clusters
def path_to_logs
path = []
path << "/var/log/containers/*#{Ingress::MODSECURITY_LOG_CONTAINER_NAME}*.log" if waf_log_enabled
path << "/var/log/containers/*#{CILIUM_CONTAINER_NAME}*.log" if cilium_log_enabled
path.join(',')
end
......
app/models/clusters/applications/ingress.rb
View file @ a6013817
......@@ -7,10 +7,6 @@ module Clusters
class Ingress < ApplicationRecord
VERSION = '1.40.2'
INGRESS_CONTAINER_NAME = 'nginx-ingress-controller'
MODSECURITY_LOG_CONTAINER_NAME = 'modsecurity-log'
MODSECURITY_MODE_LOGGING = "DetectionOnly"
MODSECURITY_MODE_BLOCKING = "On"
MODSECURITY_OWASP_RULES_FILE = "/etc/nginx/owasp-modsecurity-crs/nginx-modsecurity.conf"
self.table_name = 'clusters_applications_ingress'
......@@ -20,22 +16,18 @@ module Clusters
include ::Clusters::Concerns::ApplicationData
include AfterCommitQueue
include UsageStatistics
include IgnorableColumns
default_value_for :ingress_type, :nginx
default_value_for :modsecurity_enabled, true
default_value_for :version, VERSION
default_value_for :modsecurity_mode, :logging
ignore_column :modsecurity_enabled, remove_with: '14.2', remove_after: '2021-07-22'
ignore_column :modsecurity_mode, remove_with: '14.2', remove_after: '2021-07-22'
enum ingress_type: {
nginx: 1
}
enum modsecurity_mode: { logging: 0, blocking: 1 }
scope :modsecurity_not_installed, -> { where(modsecurity_enabled: nil) }
scope :modsecurity_enabled, -> { where(modsecurity_enabled: true) }
scope :modsecurity_disabled, -> { where(modsecurity_enabled: false) }
FETCH_IP_ADDRESS_DELAY = 30.seconds
state_machine :status do
......@@ -92,96 +84,13 @@ module Clusters
private
def specification
return {} unless modsecurity_enabled
{
"controller" => {
"config" => {
"enable-modsecurity" => "true",
"enable-owasp-modsecurity-crs" => "false",
"modsecurity-snippet" => modsecurity_snippet_content,
"modsecurity.conf" => modsecurity_config_content
},
"extraContainers" => [
{
"name" => MODSECURITY_LOG_CONTAINER_NAME,
"image" => "busybox",
"args" => [
"/bin/sh",
"-c",
"tail -F /var/log/modsec/audit.log"
],
"volumeMounts" => [
{
"name" => "modsecurity-log-volume",
"mountPath" => "/var/log/modsec",
"readOnly" => true
}
],
"livenessProbe" => {
"exec" => {
"command" => [
"ls",
"/var/log/modsec/audit.log"
]
}
}
}
],
"extraVolumeMounts" => [
{
"name" => "modsecurity-template-volume",
"mountPath" => "/etc/nginx/modsecurity/modsecurity.conf",
"subPath" => "modsecurity.conf"
},
{
"name" => "modsecurity-log-volume",
"mountPath" => "/var/log/modsec"
}
],
"extraVolumes" => [
{
"name" => "modsecurity-template-volume",
"configMap" => {
"name" => "ingress-#{INGRESS_CONTAINER_NAME}",
"items" => [
{
"key" => "modsecurity.conf",
"path" => "modsecurity.conf"
}
]
}
},
{
"name" => "modsecurity-log-volume",
"emptyDir" => {}
}
]
}
}
end
def modsecurity_config_content
File.read(modsecurity_config_file_path)
end
def modsecurity_config_file_path
Rails.root.join('vendor', 'ingress', 'modsecurity.conf')
end
def content_values
YAML.load_file(chart_values_file).deep_merge!(specification)
YAML.load_file(chart_values_file)
end
def application_jupyter_installed?
cluster.application_jupyter&.installed?
end
def modsecurity_snippet_content
sec_rule_engine = logging? ? MODSECURITY_MODE_LOGGING : MODSECURITY_MODE_BLOCKING
"SecRuleEngine #{sec_rule_engine}\nInclude #{MODSECURITY_OWASP_RULES_FILE}"
end
end
end
end
app/models/clusters/cluster.rb
View file @ a6013817
......@@ -138,7 +138,6 @@ module Clusters
scope :gcp_installed, -> { gcp_provided.joins(:provider_gcp).merge(Clusters::Providers::Gcp.with_status(:created)) }
scope :aws_installed, -> { aws_provided.joins(:provider_aws).merge(Clusters::Providers::Aws.with_status(:created)) }
scope :with_enabled_modsecurity, -> { joins(:application_ingress).merge(::Clusters::Applications::Ingress.modsecurity_enabled) }
scope :with_available_elasticstack, -> { joins(:application_elastic_stack).merge(::Clusters::Applications::ElasticStack.available) }
scope :with_available_cilium, -> { joins(:application_cilium).merge(::Clusters::Applications::Cilium.available) }
scope :distinct_with_deployed_environments, -> { joins(:environments).merge(::Deployment.success).distinct }
......
app/serializers/cluster_application_entity.rb
View file @ a6013817
......@@ -10,15 +10,12 @@ class ClusterApplicationEntity < Grape::Entity
expose :hostname, if: -> (e, _) { e.respond_to?(:hostname) }
expose :email, if: -> (e, _) { e.respond_to?(:email) }
expose :stack, if: -> (e, _) { e.respond_to?(:stack) }
expose :modsecurity_enabled, if: -> (e, _) { e.respond_to?(:modsecurity_enabled) }
expose :update_available?, as: :update_available, if: -> (e, _) { e.respond_to?(:update_available?) }
expose :can_uninstall?, as: :can_uninstall
expose :available_domains, using: Serverless::DomainEntity, if: -> (e, _) { e.respond_to?(:available_domains) }
expose :pages_domain, using: Serverless::DomainEntity, if: -> (e, _) { e.respond_to?(:pages_domain) }
expose :modsecurity_mode, if: -> (e, _) { e.respond_to?(:modsecurity_mode) }
expose :host, if: -> (e, _) { e.respond_to?(:host) }
expose :port, if: -> (e, _) { e.respond_to?(:port) }
expose :protocol, if: -> (e, _) { e.respond_to?(:protocol) }
expose :waf_log_enabled, if: -> (e, _) { e.respond_to?(:waf_log_enabled) }
expose :cilium_log_enabled, if: -> (e, _) { e.respond_to?(:cilium_log_enabled) }
end
app/services/clusters/applications/base_service.rb
View file @ a6013817
......@@ -29,14 +29,6 @@ module Clusters
application.stack = params[:stack]
end
if application.has_attribute?(:modsecurity_enabled)
application.modsecurity_enabled = params[:modsecurity_enabled] || false
end
if application.has_attribute?(:modsecurity_mode)
application.modsecurity_mode = params[:modsecurity_mode] || 0
end
apply_fluentd_related_attributes(application)
if application.respond_to?(:oauth_application)
......
ee/app/controllers/projects/security/waf_anomalies_controller.rb deleted 100644 → 0
View file @ 9ff47402
# frozen_string_literal: true
module Projects
module Security
class WafAnomaliesController < Projects::ApplicationController
include SecurityAndCompliancePermissions
POLLING_INTERVAL = 5_000
before_action :authorize_read_waf_anomalies!
before_action :set_polling_interval
feature_category :web_firewall
def summary
return not_found unless anomaly_summary_service.elasticsearch_client
result = anomaly_summary_service.execute
respond_to do |format|
format.json do
status = result[:status] == :success ? :ok : :bad_request
render status: status, json: result
end
end
end
private
def anomaly_summary_service
@anomaly_summary_service ||= ::Security::WafAnomalySummaryService.new(
environment: environment,
**query_params.to_h.symbolize_keys
)
end
def query_params
params.permit(:interval, :from, :to)
end
def set_polling_interval
Gitlab::PollingInterval.set_header(response, interval: POLLING_INTERVAL)
end
def environment
@environment ||= project.environments.find(params.delete("environment_id"))
end
def authorize_read_waf_anomalies!
render_403 unless can?(current_user, :read_threat_monitoring, project)
end
end
end
end
ee/app/services/security/waf_anomaly_summary_service.rb deleted 100644 → 0
View file @ 9ff47402
# frozen_string_literal: true
module Security
# Service for fetching summary statistics from ElasticSearch.
# Queries ES and retrieves both total nginx requests & modsec violations
#
class WafAnomalySummaryService < ::BaseService
def initialize(environment:, cluster: environment.deployment_platform&.cluster, interval: 'day', from: 30.days.ago.iso8601, to: Time.zone.now.iso8601, options: {})
@environment = environment
@cluster = cluster
@interval = interval
@from = from
@to = to
@options = options
end
def execute(totals_only: false)
return if elasticsearch_client.nil?
return unless @environment.external_url
# Use multi-search with single query as we'll be adding nginx later
# with https://gitlab.com/gitlab-org/gitlab/issues/14707
aggregate_results = elasticsearch_client.msearch(body: body)
nginx_results, modsec_results = aggregate_results['responses']
if chart_above_v3?
nginx_total_requests = nginx_results.dig('hits', 'total', 'value').to_f
modsec_total_requests = modsec_results.dig('hits', 'total', 'value').to_f
else
nginx_total_requests = nginx_results.dig('hits', 'total').to_f
modsec_total_requests = modsec_results.dig('hits', 'total').to_f
end
return { total_traffic: nginx_total_requests, total_anomalous_traffic: modsec_total_requests } if totals_only
anomalous_traffic_count = nginx_total_requests == 0 ? 0 : (modsec_total_requests / nginx_total_requests).round(2)
{
total_traffic: nginx_total_requests,
anomalous_traffic: anomalous_traffic_count,
history: {
nominal: histogram_from(nginx_results),
anomalous: histogram_from(modsec_results)
},
interval: @interval,
from: @from,
to: @to,
status: :success
}
end
def elasticsearch_client
@elasticsearch_client ||= elastic_stack_adapter&.elasticsearch_client(timeout: @options[:timeout])
end
private
def elastic_stack_adapter
@elastic_stack_adapter ||= @cluster&.elastic_stack_adapter
end
def chart_above_v3?
elastic_stack_adapter.chart_above_v3?
end
def body
[
{ index: indices },
{
query: nginx_requests_query,
aggs: aggregations(@interval),
size: 0 # no docs needed, only counts
},
{ index: indices },
{
query: modsec_requests_query,
aggs: aggregations(@interval),
size: 0 # no docs needed, only counts
}
]
end
# Construct a list of daily indices to be searched. We do this programmatically
# based on the requested timeframe to reduce the load of querying all previous
# indices
def indices
(@from.to_date..@to.to_date).map do |day|
"filebeat-*-#{day.strftime('%Y.%m.%d')}"
end
end
def nginx_requests_query
{
bool: {
must: [
{
range: {
'@timestamp' => {
gte: @from,
lte: @to
}
}
},
{
terms_set: {
message: {
terms: environment_proxy_upstream_name_tokens,
minimum_should_match_script: {
source: 'params.num_terms'
}
}
}
},
{
match_phrase: {
'kubernetes.container.name' => {
query: ::Clusters::Applications::Ingress::INGRESS_CONTAINER_NAME
}
}
},
{
match_phrase: {
'kubernetes.namespace' => {
query: Gitlab::Kubernetes::Helm::NAMESPACE
}
}
},
{
match_phrase: {
stream: {
query: 'stdout'
}
}
}
]
}
}
end
def modsec_requests_query
{
bool: {
must: [
{
range: {
'@timestamp' => {
gte: @from,
lte: @to
}
}
},
{
prefix: {
'transaction.unique_id': application_server_name
}
},
{
match_phrase: {
'kubernetes.container.name' => {
query: ::Clusters::Applications::Ingress::MODSECURITY_LOG_CONTAINER_NAME
}
}
},
{
match_phrase: {
'kubernetes.namespace' => {
query: Gitlab::Kubernetes::Helm::NAMESPACE
}
}
}
]
}
}
end
def aggregations(interval)
{
counts: {
date_histogram: {
field: '@timestamp',
interval: interval,
order: {
'_key': 'asc'
}
}
}
}
end
def histogram_from(results)
buckets = results.dig('aggregations', 'counts', 'buckets') || []
buckets.map { |bucket| [bucket['key_as_string'], bucket['doc_count']] }
end
# Derive server_name to filter modsec audit log by environment
def application_server_name
@environment.formatted_external_url
end
# Derive proxy upstream name to filter nginx log by environment
# See https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/log-format/
def environment_proxy_upstream_name_tokens
[
*@environment.deployment_namespace.split('-'),
@environment.slug # $RELEASE_NAME
]
end
end
end
ee/app/views/projects/threat_monitoring/show.html.haml
View file @ a6013817
......@@ -8,9 +8,7 @@
#js-threat-monitoring-app{ data: { documentation_path: 'https://docs.gitlab.com/ee/user/application_security/threat_monitoring/',
empty_state_svg_path: image_path('illustrations/monitoring/unable_to_connect.svg'),
waf_no_data_svg_path: image_path('illustrations/firewall-not-detected-sm.svg'),
network_policy_no_data_svg_path: image_path('illustrations/network-policies-not-detected-sm.svg'),
waf_statistics_endpoint: summary_project_security_waf_anomalies_path(@project, format: :json),
network_policy_statistics_endpoint: summary_project_security_network_policies_path(@project, format: :json),
environments_endpoint: project_environments_path(@project),
network_policies_endpoint: project_security_network_policies_path(@project),
......
ee/config/routes/project.rb
View file @ a6013817
......@@ -53,10 +53,6 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
resources :audit_events, only: [:index]
namespace :security do
resources :waf_anomalies, only: [] do
get :summary, on: :collection
end
resources :network_policies, only: [:index, :create, :update, :destroy] do
get :summary, on: :collection
end
......
ee/spec/controllers/projects/security/waf_anomalies_controller_spec.rb deleted 100644 → 0
View file @ 9ff47402
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Projects::Security::WafAnomaliesController do
let_it_be(:group) { create(:group) }
let_it_be(:user) { create(:user) }
let_it_be(:project) { create(:project, :public, :repository, group: group) }
let_it_be(:environment) { create(:environment, :with_review_app, project: project) }
let_it_be(:cluster) { create(:cluster, :provided_by_gcp, environment_scope: '*', projects: [environment.project]) }
let_it_be(:action_params) { { project_id: project, namespace_id: project.namespace, environment_id: environment } }
let(:es_client) { nil }
describe 'GET #summary' do
subject(:request) { get :summary, params: action_params, format: :json }
before do
stub_licensed_features(threat_monitoring: true)
sign_in(user)
allow_next_instance_of(::Security::WafAnomalySummaryService) do |instance|
allow(instance).to receive(:elasticsearch_client).at_most(3).times { es_client }
allow(instance).to receive(:chart_above_v3?) { true }
end
end
include_context '"Security & Compliance" permissions' do
let(:valid_request) { request }
before_request do
group.add_developer(user)
end
end
context 'with authorized user' do
before do
group.add_developer(user)
end
context 'with elastic_stack' do
let(:es_client) { double(Elasticsearch::Client) }
before do
allow(es_client).to receive(:msearch) { { "responses" => [{}, {}] } }
end
it 'returns anomaly summary' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(json_response['total_traffic']).to eq(0)
expect(json_response['anomalous_traffic']).to eq(0)
expect(response).to match_response_schema('vulnerabilities/summary', dir: 'ee')
end
end
context 'without elastic_stack' do
it 'returns not found' do
subject
expect(response).to have_gitlab_http_status(:not_found)
end
end
it 'sets a polling interval header' do
subject
expect(response.headers['Poll-Interval']).to eq('5000')
end
end
context 'with unauthorized user' do
it 'returns unauthorized' do
subject
expect(response).to have_gitlab_http_status(:forbidden)
end
end
end
end
ee/spec/frontend/threat_monitoring/components/threat_monitoring_section_spec.js
View file @ a6013817
......@@ -37,7 +37,7 @@ describe('ThreatMonitoringSection component', () => {
wrapper = shallowMount(ThreatMonitoringSection, {
propsData: {
storeNamespace: 'threatMonitoringNetworkPolicy',
title: 'Web Application Firewall',
title: 'Container Network Policy',
subtitle: 'Requests',
nominalTitle: 'Total Requests',
anomalousTitle: 'Anomalous Requests',
......
ee/spec/services/security/waf_anomaly_summary_service_spec.rb deleted 100644 → 0
View file @ 9ff47402
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Security::WafAnomalySummaryService do
let(:environment) { create(:environment, :with_review_app, environment_type: 'review') }
let!(:cluster) do
create(:cluster, :provided_by_gcp, environment_scope: '*', projects: [environment.project])
end
let(:es_client) { double(Elasticsearch::Client) }
let(:chart_above_v3) { true }
let(:empty_response) do
{
'took' => 40,
'timed_out' => false,
'_shards' => { 'total' => 11, 'successful' => 11, 'skipped' => 0, 'failed' => 0 },
'hits' => { 'total' => { 'value' => 0, 'relation' => 'gte' }, 'max_score' => 0.0, 'hits' => [] },
'aggregations' => {
'counts' => {
'buckets' => []
}
},
'status' => 200
}
end
let(:nginx_response) do
empty_response.deep_merge(
'hits' => { 'total' => { 'value' => 3 } },
'aggregations' => {
'counts' => {
'buckets' => [
{ 'key_as_string' => '2020-02-14T23:00:00.000Z', 'key' => 1575500400000, 'doc_count' => 1 },
{ 'key_as_string' => '2020-02-15T00:00:00.000Z', 'key' => 1575504000000, 'doc_count' => 0 },
{ 'key_as_string' => '2020-02-15T01:00:00.000Z', 'key' => 1575507600000, 'doc_count' => 0 },
{ 'key_as_string' => '2020-02-15T08:00:00.000Z', 'key' => 1575532800000, 'doc_count' => 2 }
]
}
}
)
end
let(:modsec_response) do
empty_response.deep_merge(
'hits' => { 'total' => { 'value' => 1 } },
'aggregations' => {
'counts' => {
'buckets' => [
{ 'key_as_string' => '2019-12-04T23:00:00.000Z', 'key' => 1575500400000, 'doc_count' => 0 },
{ 'key_as_string' => '2019-12-05T00:00:00.000Z', 'key' => 1575504000000, 'doc_count' => 0 },
{ 'key_as_string' => '2019-12-05T01:00:00.000Z', 'key' => 1575507600000, 'doc_count' => 0 },
{ 'key_as_string' => '2019-12-05T08:00:00.000Z', 'key' => 1575532800000, 'doc_count' => 1 }
]
}
}
)
end
let(:nginx_response_es6) do
empty_response.deep_merge(
'hits' => { 'total' => 3 },
'aggregations' => {
'counts' => {
'buckets' => [
{ 'key_as_string' => '2020-02-14T23:00:00.000Z', 'key' => 1575500400000, 'doc_count' => 1 },
{ 'key_as_string' => '2020-02-15T00:00:00.000Z', 'key' => 1575504000000, 'doc_count' => 0 },
{ 'key_as_string' => '2020-02-15T01:00:00.000Z', 'key' => 1575507600000, 'doc_count' => 0 },
{ 'key_as_string' => '2020-02-15T08:00:00.000Z', 'key' => 1575532800000, 'doc_count' => 2 }
]
}
}
)
end
let(:modsec_response_es6) do
empty_response.deep_merge(
'hits' => { 'total' => 1 },
'aggregations' => {
'counts' => {
'buckets' => [
{ 'key_as_string' => '2019-12-04T23:00:00.000Z', 'key' => 1575500400000, 'doc_count' => 0 },
{ 'key_as_string' => '2019-12-05T00:00:00.000Z', 'key' => 1575504000000, 'doc_count' => 0 },
{ 'key_as_string' => '2019-12-05T01:00:00.000Z', 'key' => 1575507600000, 'doc_count' => 0 },
{ 'key_as_string' => '2019-12-05T08:00:00.000Z', 'key' => 1575532800000, 'doc_count' => 1 }
]
}
}
)
end
subject { described_class.new(environment: environment) }
describe '#execute' do
context 'without cluster' do
before do
allow(environment).to receive(:deployment_platform) { nil }
end
it 'returns no results' do
expect(subject.execute).to be_nil
end
end
context 'without elastic_stack' do
it 'returns no results' do
expect(subject.execute).to be_nil
end
end
context 'with environment missing external_url' do
before do
allow(environment.deployment_platform.cluster).to receive_message_chain(
:integration_elastic_stack, :elasticsearch_client
) { es_client }
allow(environment).to receive(:external_url) { nil }
end
it 'returns nil' do
expect(subject.execute).to be_nil
end
end
context 'with default histogram' do
before do
allow(es_client).to receive(:msearch) do
{ 'responses' => [nginx_results, modsec_results] }
end
allow(environment.deployment_platform.cluster).to receive_message_chain(
:integration_elastic_stack, :elasticsearch_client
) { es_client }
allow(environment.deployment_platform.cluster).to receive_message_chain(
:integration_elastic_stack, :chart_above_v3?
) { chart_above_v3 }
end
context 'no requests' do
let(:nginx_results) { empty_response }
let(:modsec_results) { empty_response }
it 'returns results', :aggregate_failures do
results = subject.execute
expect(results.fetch(:status)).to eq :success
expect(results.fetch(:interval)).to eq 'day'
expect(results.fetch(:total_traffic)).to eq 0
expect(results.fetch(:anomalous_traffic)).to eq 0.0
end
context 'when totals_only is set to true' do
it 'returns totals only', :aggregate_failures do
results = subject.execute(totals_only: true)
expect(results).to eq(total_traffic: 0.0, total_anomalous_traffic: 0.0)
end
end
end
context 'no violations' do
let(:nginx_results) { nginx_response }
let(:modsec_results) { empty_response }
it 'returns results', :aggregate_failures do
results = subject.execute
expect(results.fetch(:status)).to eq :success
expect(results.fetch(:interval)).to eq 'day'
expect(results.fetch(:total_traffic)).to eq 3
expect(results.fetch(:anomalous_traffic)).to eq 0.0
end
context 'when totals_only is set to true' do
it 'returns totals only', :aggregate_failures do
results = subject.execute(totals_only: true)
expect(results).to eq(total_traffic: 3.0, total_anomalous_traffic: 0.0)
end
end
end
context 'with violations' do
let(:nginx_results) { nginx_response }
let(:modsec_results) { modsec_response }
it 'returns results', :aggregate_failures do
results = subject.execute
expect(results.fetch(:status)).to eq :success
expect(results.fetch(:interval)).to eq 'day'
expect(results.fetch(:total_traffic)).to eq 3
expect(results.fetch(:anomalous_traffic)).to eq 0.33
end
context 'when totals_only is set to true' do
it 'returns totals only', :aggregate_failures do
results = subject.execute(totals_only: true)
expect(results).to eq(total_traffic: 3.0, total_anomalous_traffic: 1.0)
end
end
end
context 'with legacy es6 cluster' do
let(:chart_above_v3) { false }
let(:nginx_results) { nginx_response_es6 }
let(:modsec_results) { modsec_response_es6 }
it 'returns results', :aggregate_failures do
results = subject.execute
expect(results.fetch(:status)).to eq :success
expect(results.fetch(:interval)).to eq 'day'
expect(results.fetch(:total_traffic)).to eq 3
expect(results.fetch(:anomalous_traffic)).to eq 0.33
end
end
end
context 'with review app' do
it 'resolves transaction_id from external_url' do
allow(subject).to receive(:elasticsearch_client) { es_client }
allow(subject).to receive(:chart_above_v3?) { chart_above_v3 }
expect(es_client).to receive(:msearch).with(
body: array_including(
hash_including(
query: hash_including(
bool: hash_including(
must: array_including(
hash_including(
prefix: hash_including(
'transaction.unique_id': environment.formatted_external_url
)
)
)
)
)
)
)
).and_return({ 'responses' => [{}, {}] })
subject.execute
end
end
context 'with time window' do
it 'passes time frame to ElasticSearch' do
from = 1.day.ago
to = Time.current
subject = described_class.new(
environment: environment,
from: from,
to: to
)
allow(subject).to receive(:elasticsearch_client) { es_client }
allow(subject).to receive(:chart_above_v3?) { chart_above_v3 }
expect(es_client).to receive(:msearch).with(
body: array_including(
hash_including(
query: hash_including(
bool: hash_including(
must: array_including(
hash_including(
range: hash_including(
'@timestamp' => {
gte: from,
lte: to
}
)
)
)
)
)
)
)
).and_return({ 'responses' => [{}, {}] })
subject.execute
end
end
context 'with interval' do
it 'passes interval to ElasticSearch' do
interval = 'hour'
subject = described_class.new(
environment: environment,
interval: interval
)
allow(subject).to receive(:elasticsearch_client) { es_client }
allow(subject).to receive(:chart_above_v3?) { chart_above_v3 }
expect(es_client).to receive(:msearch).with(
body: array_including(
hash_including(
aggs: hash_including(
counts: hash_including(
date_histogram: hash_including(
interval: interval
)
)
)
)
)
).and_return({ 'responses' => [{}, {}] })
subject.execute
end
end
end
end
spec/factories/clusters/applications/helm.rb
View file @ a6013817
......@@ -96,26 +96,7 @@ FactoryBot.define do
end
factory :clusters_applications_ingress, class: 'Clusters::Applications::Ingress' do
modsecurity_enabled { false }
cluster factory: %i(cluster with_installed_helm provided_by_gcp)
trait :modsecurity_blocking do
modsecurity_enabled { true }
modsecurity_mode { :blocking }
end
trait :modsecurity_logging do
modsecurity_enabled { true }
modsecurity_mode { :logging }
end
trait :modsecurity_disabled do
modsecurity_enabled { false }
end
trait :modsecurity_not_installed do
modsecurity_enabled { nil }
end
end
factory :clusters_applications_cert_manager, class: 'Clusters::Applications::CertManager' do
......@@ -153,7 +134,6 @@ FactoryBot.define do
factory :clusters_applications_fluentd, class: 'Clusters::Applications::Fluentd' do
host { 'example.com' }
waf_log_enabled { true }
cilium_log_enabled { true }
cluster factory: %i(cluster with_installed_helm provided_by_gcp)
end
......
spec/fixtures/api/schemas/cluster_status.json
View file @ a6013817
......@@ -42,7 +42,6 @@
"host": {"type": ["string", "null"]},
"port": {"type": ["integer", "514"]},
"protocol": {"type": ["integer", "0"]},
"waf_log_enabled": {"type": ["boolean", "true"]},
"cilium_log_enabled": {"type": ["boolean", "true"]},
"update_available": { "type": ["boolean", "null"] },
"can_uninstall": { "type": "boolean" },
......
spec/lib/gitlab/usage/metrics/name_suggestion_spec.rb
View file @ a6013817
......@@ -38,22 +38,6 @@ RSpec.describe Gitlab::Usage::Metrics::NameSuggestion do
end
context 'joined relations' do
context 'counted attribute comes from joined relation' do
it_behaves_like 'name suggestion' do
let(:operation) { :distinct_count }
let(:column) { ::Deployment.arel_table[:environment_id] }
let(:relation) do
::Clusters::Applications::Ingress.modsecurity_enabled.logging
.joins(cluster: :deployments)
.merge(::Clusters::Cluster.enabled)
.merge(Deployment.success)
end
let(:constraints) { /'\(clusters_applications_ingress\.modsecurity_enabled = TRUE AND clusters_applications_ingress\.modsecurity_mode = \d+ AND clusters.enabled = TRUE AND deployments.status = \d+\)'/ }
let(:name_suggestion) { /count_distinct_environment_id_from_<adjective describing\: #{constraints}>_deployments_<with>_<adjective describing\: #{constraints}>_clusters_<having>_<adjective describing\: #{constraints}>_clusters_applications_ingress/ }
end
end
context 'counted attribute comes from source relation' do
it_behaves_like 'name suggestion' do
# corresponding metric is collected with count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id)
......
spec/models/clusters/applications/fluentd_spec.rb
View file @ a6013817
......@@ -3,9 +3,8 @@
require 'spec_helper'
RSpec.describe Clusters::Applications::Fluentd do
let(:waf_log_enabled) { true }
let(:cilium_log_enabled) { true }
let(:fluentd) { create(:clusters_applications_fluentd, waf_log_enabled: waf_log_enabled, cilium_log_enabled: cilium_log_enabled) }
let(:fluentd) { create(:clusters_applications_fluentd, cilium_log_enabled: cilium_log_enabled) }
include_examples 'cluster application core specs', :clusters_applications_fluentd
include_examples 'cluster application status specs', :clusters_applications_fluentd
......@@ -51,13 +50,11 @@ RSpec.describe Clusters::Applications::Fluentd do
end
describe '#values' do
let(:modsecurity_log_path) { "/var/log/containers/*#{Clusters::Applications::Ingress::MODSECURITY_LOG_CONTAINER_NAME}*.log" }
let(:cilium_log_path) { "/var/log/containers/*#{described_class::CILIUM_CONTAINER_NAME}*.log" }
subject { fluentd.values }
context 'with both logs variables set to false' do
let(:waf_log_enabled) { false }
context 'with cilium_log_enabled set to false' do
let(:cilium_log_enabled) { false }
it "raises ActiveRecord::RecordInvalid" do
......@@ -65,18 +62,8 @@ RSpec.describe Clusters::Applications::Fluentd do
end
end
context 'with both logs variables set to true' do
it { is_expected.to include("#{modsecurity_log_path},#{cilium_log_path}") }
end
context 'with waf_log_enabled set to true' do
let(:cilium_log_enabled) { false }
it { is_expected.to include(modsecurity_log_path) }
end
context 'with cilium_log_enabled set to true' do
let(:waf_log_enabled) { false }
let(:cilium_log_enabled) { true }
it { is_expected.to include(cilium_log_path) }
end
......
spec/models/clusters/applications/ingress_spec.rb
View file @ a6013817
......@@ -172,94 +172,4 @@ RSpec.describe Clusters::Applications::Ingress do
expect(values).to include('clusterIP')
end
end
describe '#values' do
subject { ingress }
context 'when modsecurity_enabled is enabled' do
before do
allow(subject).to receive(:modsecurity_enabled).and_return(true)
end
it 'includes modsecurity module enablement' do
expect(subject.values).to include("enable-modsecurity: 'true'")
end
it 'includes modsecurity core ruleset enablement set to false' do
expect(subject.values).to include("enable-owasp-modsecurity-crs: 'false'")
end
it 'includes modsecurity snippet with information related to security rules' do
expect(subject.values).to include("SecRuleEngine DetectionOnly")
expect(subject.values).to include("Include #{described_class::MODSECURITY_OWASP_RULES_FILE}")
end
context 'when modsecurity_mode is set to :blocking' do
before do
subject.blocking!
end
it 'includes modsecurity snippet with information related to security rules' do
expect(subject.values).to include("SecRuleEngine On")
expect(subject.values).to include("Include #{described_class::MODSECURITY_OWASP_RULES_FILE}")
end
end
it 'includes modsecurity.conf content' do
expect(subject.values).to include('modsecurity.conf')
# Includes file content from Ingress#modsecurity_config_content
expect(subject.values).to include('SecAuditLog')
expect(subject.values).to include('extraVolumes')
expect(subject.values).to include('extraVolumeMounts')
end
it 'includes modsecurity sidecar container' do
expect(subject.values).to include('modsecurity-log-volume')
expect(subject.values).to include('extraContainers')
end
it 'executes command to tail modsecurity logs with -F option' do
args = YAML.safe_load(subject.values).dig('controller', 'extraContainers', 0, 'args')
expect(args).to eq(['/bin/sh', '-c', 'tail -F /var/log/modsec/audit.log'])
end
it 'includes livenessProbe for modsecurity sidecar container' do
probe_config = YAML.safe_load(subject.values).dig('controller', 'extraContainers', 0, 'livenessProbe')
expect(probe_config).to eq('exec' => { 'command' => ['ls', '/var/log/modsec/audit.log'] })
end
end
context 'when modsecurity_enabled is disabled' do
before do
allow(subject).to receive(:modsecurity_enabled).and_return(false)
end
it 'excludes modsecurity module enablement' do
expect(subject.values).not_to include('enable-modsecurity')
end
it 'excludes modsecurity core ruleset enablement' do
expect(subject.values).not_to include('enable-owasp-modsecurity-crs')
end
it 'excludes modsecurity.conf content' do
expect(subject.values).not_to include('modsecurity.conf')
# Excludes file content from Ingress#modsecurity_config_content
expect(subject.values).not_to include('SecAuditLog')
expect(subject.values).not_to include('extraVolumes')
expect(subject.values).not_to include('extraVolumeMounts')
end
it 'excludes modsecurity sidecar container' do
expect(subject.values).not_to include('modsecurity-log-volume')
expect(subject.values).not_to include('extraContainers')
end
end
end
end
spec/models/clusters/cluster_spec.rb
View file @ a6013817
......@@ -196,28 +196,6 @@ RSpec.describe Clusters::Cluster, :use_clean_rails_memory_store_caching do
end
end
describe '.with_enabled_modsecurity' do
subject { described_class.with_enabled_modsecurity }
let_it_be(:cluster) { create(:cluster) }
context 'cluster has ingress application with enabled modsecurity' do
let!(:application) { create(:clusters_applications_ingress, :installed, :modsecurity_logging, cluster: cluster) }
it { is_expected.to include(cluster) }
end
context 'cluster has ingress application with disabled modsecurity' do
let!(:application) { create(:clusters_applications_ingress, :installed, :modsecurity_disabled, cluster: cluster) }
it { is_expected.not_to include(cluster) }
end
context 'cluster does not have ingress application' do
it { is_expected.not_to include(cluster) }
end
end
describe '.with_available_elasticstack' do
subject { described_class.with_available_elasticstack }
......
spec/serializers/cluster_application_entity_spec.rb
View file @ a6013817
......@@ -85,7 +85,6 @@ RSpec.describe ClusterApplicationEntity do
expect(subject[:port]).to eq(514)
expect(subject[:host]).to eq("example.com")
expect(subject[:protocol]).to eq("tcp")
expect(subject[:waf_log_enabled]).to be true
expect(subject[:cilium_log_enabled]).to be true
end
end
......
spec/services/clusters/applications/create_service_spec.rb
View file @ a6013817
......@@ -46,8 +46,7 @@ RSpec.describe Clusters::Applications::CreateService do
context 'ingress application' do
let(:params) do
{
application: 'ingress',
modsecurity_enabled: true
application: 'ingress'
}
end
......@@ -64,10 +63,6 @@ RSpec.describe Clusters::Applications::CreateService do
cluster.reload
end.to change(cluster, :application_ingress)
end
it 'sets modsecurity_enabled' do
expect(subject.modsecurity_enabled).to eq(true)
end
end
context 'cert manager application' do
......
vendor/elastic_stack/values.yaml
View file @ a6013817
......@@ -61,12 +61,6 @@ filebeat:
target_field: tie_breaker_id
- add_cloud_metadata: ~
- add_kubernetes_metadata: ~
- decode_json_fields:
fields: ["message"]
when:
equals:
kubernetes.container.namespace: "gitlab-managed-apps"
kubernetes.container.name: "modsecurity-log"
kibana:
enabled: false
elasticsearchHosts: "http://elastic-stack-elasticsearch-master:9200"
......
vendor/ingress/modsecurity.conf deleted 100644 → 0
View file @ 9ff47402
# -- GitLab Customization ----------------------------------------------
# Based on https://github.com/SpiderLabs/ModSecurity/blob/v3.0.3/modsecurity.conf-recommended
# Our base modsecurity.conf includes some minor customization:
# - `SecRuleEngine` is disabled, defaulting to `DetectionOnly`. Overridable at project-level
# - `SecAuditLogType` is disabled, defaulting to `Serial`. Overridable at project-level
# - `SecStatusEngine` is disabled, to disallow usage reporting
#
# ----------------------------------------------------------------------------
# -- Rule engine initialization ----------------------------------------------
# Enable ModSecurity, attaching it to every transaction. Use detection
# only to start with, because that minimises the chances of post-installation
# disruption.
#
# SecRuleEngine DetectionOnly
# -- Request body handling ---------------------------------------------------
# Allow ModSecurity to access request bodies. If you don't, ModSecurity
# won't be able to see any POST parameters, which opens a large security
# hole for attackers to exploit.
#
SecRequestBodyAccess On
# Enable XML request body parser.
# Initiate XML Processor in case of xml content-type
#
SecRule REQUEST_HEADERS:Content-Type "(?:application(?:/soap\+|/)|text/)xml" \
"id:'200000',phase:1,t:none,t:lowercase,pass,nolog,ctl:requestBodyProcessor=XML"
# Enable JSON request body parser.
# Initiate JSON Processor in case of JSON content-type; change accordingly
# if your application does not use 'application/json'
#
SecRule REQUEST_HEADERS:Content-Type "application/json" \
"id:'200001',phase:1,t:none,t:lowercase,pass,nolog,ctl:requestBodyProcessor=JSON"
# Maximum request body size we will accept for buffering. If you support
# file uploads then the value given on the first line has to be as large
# as the largest file you are willing to accept. The second value refers
# to the size of data, with files excluded. You want to keep that value as
# low as practical.
#
SecRequestBodyLimit 13107200
SecRequestBodyNoFilesLimit 131072
# What do do if the request body size is above our configured limit.
# Keep in mind that this setting will automatically be set to ProcessPartial
# when SecRuleEngine is set to DetectionOnly mode in order to minimize
# disruptions when initially deploying ModSecurity.
#
SecRequestBodyLimitAction Reject
# Verify that we've correctly processed the request body.
# As a rule of thumb, when failing to process a request body
# you should reject the request (when deployed in blocking mode)
# or log a high-severity alert (when deployed in detection-only mode).
#
SecRule REQBODY_ERROR "!@eq 0" \
"id:'200002', phase:2,t:none,log,deny,status:400,msg:'Failed to parse request body.',logdata:'%{reqbody_error_msg}',severity:2"
# By default be strict with what we accept in the multipart/form-data
# request body. If the rule below proves to be too strict for your
# environment consider changing it to detection-only. You are encouraged
# _not_ to remove it altogether.
#
SecRule MULTIPART_STRICT_ERROR "!@eq 0" \
"id:'200003',phase:2,t:none,log,deny,status:400, \
msg:'Multipart request body failed strict validation: \
PE %{REQBODY_PROCESSOR_ERROR}, \
BQ %{MULTIPART_BOUNDARY_QUOTED}, \
BW %{MULTIPART_BOUNDARY_WHITESPACE}, \
DB %{MULTIPART_DATA_BEFORE}, \
DA %{MULTIPART_DATA_AFTER}, \
HF %{MULTIPART_HEADER_FOLDING}, \
LF %{MULTIPART_LF_LINE}, \
SM %{MULTIPART_MISSING_SEMICOLON}, \
IQ %{MULTIPART_INVALID_QUOTING}, \
IP %{MULTIPART_INVALID_PART}, \
IH %{MULTIPART_INVALID_HEADER_FOLDING}, \
FL %{MULTIPART_FILE_LIMIT_EXCEEDED}'"
# Did we see anything that might be a boundary?
#
# Here is a short description about the ModSecurity Multipart parser: the
# parser returns with value 0, if all "boundary-like" line matches with
# the boundary string which given in MIME header. In any other cases it returns
# with different value, eg. 1 or 2.
#
# The RFC 1341 descript the multipart content-type and its syntax must contains
# only three mandatory lines (above the content):
# * Content-Type: multipart/mixed; boundary=BOUNDARY_STRING
# * --BOUNDARY_STRING
# * --BOUNDARY_STRING--
#
# First line indicates, that this is a multipart content, second shows that
# here starts a part of the multipart content, third shows the end of content.
#
# If there are any other lines, which starts with "--", then it should be
# another boundary id - or not.
#
# After 3.0.3, there are two kinds of types of boundary errors: strict and permissive.
#
# If multipart content contains the three necessary lines with correct order, but
# there are one or more lines with "--", then parser returns with value 2 (non-zero).
#
# If some of the necessary lines (usually the start or end) misses, or the order
# is wrong, then parser returns with value 1 (also a non-zero).
#
# You can choose, which one is what you need. The example below contains the
# 'strict' mode, which means if there are any lines with start of "--", then
# ModSecurity blocked the content. But the next, commented example contains
# the 'permissive' mode, then you check only if the necessary lines exists in
# correct order. Whit this, you can enable to upload PEM files (eg "----BEGIN.."),
# or other text files, which contains eg. HTTP headers.
#
# The difference is only the operator - in strict mode (first) the content blocked
# in case of any non-zero value. In permissive mode (second, commented) the
# content blocked only if the value is explicit 1. If it 0 or 2, the content will
# allowed.
#
#
# See #1747 and #1924 for further information on the possible values for
# MULTIPART_UNMATCHED_BOUNDARY.
#
SecRule MULTIPART_UNMATCHED_BOUNDARY "@eq 1" \
"id:'200004',phase:2,t:none,log,deny,msg:'Multipart parser detected a possible unmatched boundary.'"
# PCRE Tuning
# We want to avoid a potential RegEx DoS condition
#
SecPcreMatchLimit 1000
SecPcreMatchLimitRecursion 1000
# Some internal errors will set flags in TX and we will need to look for these.
# All of these are prefixed with "MSC_". The following flags currently exist:
#
# MSC_PCRE_LIMITS_EXCEEDED: PCRE match limits were exceeded.
#
SecRule TX:/^MSC_/ "!@streq 0" \
"id:'200005',phase:2,t:none,deny,msg:'ModSecurity internal error flagged: %{MATCHED_VAR_NAME}'"
# -- Response body handling --------------------------------------------------
# Allow ModSecurity to access response bodies.
# You should have this directive enabled in order to identify errors
# and data leakage issues.
#
# Do keep in mind that enabling this directive does increases both
# memory consumption and response latency.
#
SecResponseBodyAccess On
# Which response MIME types do you want to inspect? You should adjust the
# configuration below to catch documents but avoid static files
# (e.g., images and archives).
#
SecResponseBodyMimeType text/plain text/html text/xml
# Buffer response bodies of up to 512 KB in length.
SecResponseBodyLimit 524288
# What happens when we encounter a response body larger than the configured
# limit? By default, we process what we have and let the rest through.
# That's somewhat less secure, but does not break any legitimate pages.
#
SecResponseBodyLimitAction ProcessPartial
# -- Filesystem configuration ------------------------------------------------
# The location where ModSecurity stores temporary files (for example, when
# it needs to handle a file upload that is larger than the configured limit).
#
# This default setting is chosen due to all systems have /tmp available however,
# this is less than ideal. It is recommended that you specify a location that's private.
#
SecTmpDir /tmp/
# The location where ModSecurity will keep its persistent data. This default setting
# is chosen due to all systems have /tmp available however, it
# too should be updated to a place that other users can't access.
#
SecDataDir /tmp/
# -- File uploads handling configuration -------------------------------------
# The location where ModSecurity stores intercepted uploaded files. This
# location must be private to ModSecurity. You don't want other users on
# the server to access the files, do you?
#
#SecUploadDir /opt/modsecurity/var/upload/
# By default, only keep the files that were determined to be unusual
# in some way (by an external inspection script). For this to work you
# will also need at least one file inspection rule.
#
#SecUploadKeepFiles RelevantOnly
# Uploaded files are by default created with permissions that do not allow
# any other user to access them. You may need to relax that if you want to
# interface ModSecurity to an external program (e.g., an anti-virus).
#
#SecUploadFileMode 0600
# -- Debug log configuration -------------------------------------------------
# The default debug log configuration is to duplicate the error, warning
# and notice messages from the error log.
#
#SecDebugLog /opt/modsecurity/var/log/debug.log
#SecDebugLogLevel 3
# -- Audit log configuration -------------------------------------------------
# Log the transactions that are marked by a rule, as well as those that
# trigger a server error (determined by a 5xx or 4xx, excluding 404,
# level response status codes).
#
SecAuditEngine RelevantOnly
SecAuditLogRelevantStatus "^(?:5|4(?!04))"
# Log everything we know about a transaction.
SecAuditLogParts ABIJDEFHZ
# Use a single file for logging. This is much easier to look at, but
# assumes that you will use the audit log only ocassionally.
#
# SecAuditLogType Serial
SecAuditLogFormat JSON
SecAuditLog /var/log/modsec/audit.log
# Specify the path for concurrent audit logging.
#SecAuditLogStorageDir /opt/modsecurity/var/audit/
# -- Miscellaneous -----------------------------------------------------------
# Use the most commonly used application/x-www-form-urlencoded parameter
# separator. There's probably only one application somewhere that uses
# something else so don't expect to change this value.
#
SecArgumentSeparator &
# Settle on version 0 (zero) cookies, as that is what most applications
# use. Using an incorrect cookie version may open your installation to
# evasion attacks (against the rules that examine named cookies).
#
SecCookieFormat 0
# Specify your Unicode Code Point.
# This mapping is used by the t:urlDecodeUni transformation function
# to properly map encoded data to your language. Properly setting
# these directives helps to reduce false positives and negatives.
#
SecUnicodeMapFile unicode.mapping 20127
# Improve the quality of ModSecurity by sharing information about your
# current ModSecurity version and dependencies versions.
# The following information will be shared: ModSecurity version,
# Web Server version, APR version, PCRE version, Lua version, Libxml2
# version, Anonymous unique id for host.
# SecStatusEngine On
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7