Skip to content

G
gitlab-ce
Commit d1146081 authored by Alan Paruszewski's avatar Alan Paruszewski Committed by Alan (Maciej) Paruszewski
Browse files
Options

Remove GitLab WAF related models, services and workers 

This change removes all code related to WAF (ModSecurity) feature.

Changelog: removed
EE: true
parent 6dfed16d
Hide whitespace changes
Inline Side-by-side
Showing with 15 additions and 1240 deletions
app/controllers/clusters/applications_controller.rb
View file @ d1146081
......@@ -47,7 +47,7 @@ class Clusters::ApplicationsController < Clusters::BaseController
end
def cluster_application_params
params.permit(:application, :hostname, :pages_domain_id, :email, :stack, :modsecurity_enabled, :modsecurity_mode, :host, :port, :protocol, :waf_log_enabled, :cilium_log_enabled)
params.permit(:application, :hostname, :pages_domain_id, :email, :stack, :host, :port, :protocol, :cilium_log_enabled)
end
def cluster_application_destroy_params
......
app/models/clusters/applications/fluentd.rb
View file @ d1146081
......@@ -12,11 +12,13 @@ module Clusters
include ::Clusters::Concerns::ApplicationStatus
include ::Clusters::Concerns::ApplicationVersion
include ::Clusters::Concerns::ApplicationData
include IgnorableColumns
default_value_for :version, VERSION
default_value_for :port, 514
default_value_for :protocol, :tcp
default_value_for :waf_log_enabled, false
ignore_column :waf_log_enabled, remove_with: '14.2', remove_after: '2021-07-22'
enum protocol: { tcp: 0, udp: 1 }
......@@ -48,9 +50,7 @@ module Clusters
private
def has_at_least_one_log_enabled?
if !waf_log_enabled && !cilium_log_enabled
errors.add(:base, _("At least one logging option is required to be enabled"))
end
errors.add(:base, _("At least one logging option is required to be enabled")) unless cilium_log_enabled
end
def content_values
......@@ -113,7 +113,6 @@ module Clusters
def path_to_logs
path = []
path << "/var/log/containers/*#{Ingress::MODSECURITY_LOG_CONTAINER_NAME}*.log" if waf_log_enabled
path << "/var/log/containers/*#{CILIUM_CONTAINER_NAME}*.log" if cilium_log_enabled
path.join(',')
end
......
app/models/clusters/applications/ingress.rb
View file @ d1146081
......@@ -7,10 +7,6 @@ module Clusters
class Ingress < ApplicationRecord
VERSION = '1.40.2'
INGRESS_CONTAINER_NAME = 'nginx-ingress-controller'
MODSECURITY_LOG_CONTAINER_NAME = 'modsecurity-log'
MODSECURITY_MODE_LOGGING = "DetectionOnly"
MODSECURITY_MODE_BLOCKING = "On"
MODSECURITY_OWASP_RULES_FILE = "/etc/nginx/owasp-modsecurity-crs/nginx-modsecurity.conf"
self.table_name = 'clusters_applications_ingress'
......@@ -20,22 +16,18 @@ module Clusters
include ::Clusters::Concerns::ApplicationData
include AfterCommitQueue
include UsageStatistics
include IgnorableColumns
default_value_for :ingress_type, :nginx
default_value_for :modsecurity_enabled, true
default_value_for :version, VERSION
default_value_for :modsecurity_mode, :logging
ignore_column :modsecurity_enabled, remove_with: '14.2', remove_after: '2021-07-22'
ignore_column :modsecurity_mode, remove_with: '14.2', remove_after: '2021-07-22'
enum ingress_type: {
nginx: 1
}
enum modsecurity_mode: { logging: 0, blocking: 1 }
scope :modsecurity_not_installed, -> { where(modsecurity_enabled: nil) }
scope :modsecurity_enabled, -> { where(modsecurity_enabled: true) }
scope :modsecurity_disabled, -> { where(modsecurity_enabled: false) }
FETCH_IP_ADDRESS_DELAY = 30.seconds
state_machine :status do
......@@ -92,96 +84,13 @@ module Clusters
private
def specification
return {} unless modsecurity_enabled
{
"controller" => {
"config" => {
"enable-modsecurity" => "true",
"enable-owasp-modsecurity-crs" => "false",
"modsecurity-snippet" => modsecurity_snippet_content,
"modsecurity.conf" => modsecurity_config_content
},
"extraContainers" => [
{
"name" => MODSECURITY_LOG_CONTAINER_NAME,
"image" => "busybox",
"args" => [
"/bin/sh",
"-c",
"tail -F /var/log/modsec/audit.log"
],
"volumeMounts" => [
{
"name" => "modsecurity-log-volume",
"mountPath" => "/var/log/modsec",
"readOnly" => true
}
],
"livenessProbe" => {
"exec" => {
"command" => [
"ls",
"/var/log/modsec/audit.log"
]
}
}
}
],
"extraVolumeMounts" => [
{
"name" => "modsecurity-template-volume",
"mountPath" => "/etc/nginx/modsecurity/modsecurity.conf",
"subPath" => "modsecurity.conf"
},
{
"name" => "modsecurity-log-volume",
"mountPath" => "/var/log/modsec"
}
],
"extraVolumes" => [
{
"name" => "modsecurity-template-volume",
"configMap" => {
"name" => "ingress-#{INGRESS_CONTAINER_NAME}",
"items" => [
{
"key" => "modsecurity.conf",
"path" => "modsecurity.conf"
}
]
}
},
{
"name" => "modsecurity-log-volume",
"emptyDir" => {}
}
]
}
}
end
def modsecurity_config_content
File.read(modsecurity_config_file_path)
end
def modsecurity_config_file_path
Rails.root.join('vendor', 'ingress', 'modsecurity.conf')
end
def content_values
YAML.load_file(chart_values_file).deep_merge!(specification)
YAML.load_file(chart_values_file)
end
def application_jupyter_installed?
cluster.application_jupyter&.installed?
end
def modsecurity_snippet_content
sec_rule_engine = logging? ? MODSECURITY_MODE_LOGGING : MODSECURITY_MODE_BLOCKING
"SecRuleEngine #{sec_rule_engine}\nInclude #{MODSECURITY_OWASP_RULES_FILE}"
end
end
end
end
app/models/clusters/cluster.rb
View file @ d1146081
......@@ -138,7 +138,6 @@ module Clusters
scope :gcp_installed, -> { gcp_provided.joins(:provider_gcp).merge(Clusters::Providers::Gcp.with_status(:created)) }
scope :aws_installed, -> { aws_provided.joins(:provider_aws).merge(Clusters::Providers::Aws.with_status(:created)) }
scope :with_enabled_modsecurity, -> { joins(:application_ingress).merge(::Clusters::Applications::Ingress.modsecurity_enabled) }
scope :with_available_elasticstack, -> { joins(:application_elastic_stack).merge(::Clusters::Applications::ElasticStack.available) }
scope :with_available_cilium, -> { joins(:application_cilium).merge(::Clusters::Applications::Cilium.available) }
scope :distinct_with_deployed_environments, -> { joins(:environments).merge(::Deployment.success).distinct }
......
app/serializers/cluster_application_entity.rb
View file @ d1146081
......@@ -10,15 +10,12 @@ class ClusterApplicationEntity < Grape::Entity
expose :hostname, if: -> (e, _) { e.respond_to?(:hostname) }
expose :email, if: -> (e, _) { e.respond_to?(:email) }
expose :stack, if: -> (e, _) { e.respond_to?(:stack) }
expose :modsecurity_enabled, if: -> (e, _) { e.respond_to?(:modsecurity_enabled) }
expose :update_available?, as: :update_available, if: -> (e, _) { e.respond_to?(:update_available?) }
expose :can_uninstall?, as: :can_uninstall
expose :available_domains, using: Serverless::DomainEntity, if: -> (e, _) { e.respond_to?(:available_domains) }
expose :pages_domain, using: Serverless::DomainEntity, if: -> (e, _) { e.respond_to?(:pages_domain) }
expose :modsecurity_mode, if: -> (e, _) { e.respond_to?(:modsecurity_mode) }
expose :host, if: -> (e, _) { e.respond_to?(:host) }
expose :port, if: -> (e, _) { e.respond_to?(:port) }
expose :protocol, if: -> (e, _) { e.respond_to?(:protocol) }
expose :waf_log_enabled, if: -> (e, _) { e.respond_to?(:waf_log_enabled) }
expose :cilium_log_enabled, if: -> (e, _) { e.respond_to?(:cilium_log_enabled) }
end
app/services/clusters/applications/base_service.rb
View file @ d1146081
......@@ -29,14 +29,6 @@ module Clusters
application.stack = params[:stack]
end
if application.has_attribute?(:modsecurity_enabled)
application.modsecurity_enabled = params[:modsecurity_enabled] || false
end
if application.has_attribute?(:modsecurity_mode)
application.modsecurity_mode = params[:modsecurity_mode] || 0
end
apply_fluentd_related_attributes(application)
if application.respond_to?(:oauth_application)
......
ee/app/controllers/projects/security/waf_anomalies_controller.rb deleted 100644 → 0
View file @ 6dfed16d
# frozen_string_literal: true
module Projects
module Security
class WafAnomaliesController < Projects::ApplicationController
include SecurityAndCompliancePermissions
POLLING_INTERVAL = 5_000
before_action :authorize_read_waf_anomalies!
before_action :set_polling_interval
feature_category :web_firewall
def summary
return not_found unless anomaly_summary_service.elasticsearch_client
result = anomaly_summary_service.execute
respond_to do |format|
format.json do
status = result[:status] == :success ? :ok : :bad_request
render status: status, json: result
end
end
end
private
def anomaly_summary_service
@anomaly_summary_service ||= ::Security::WafAnomalySummaryService.new(
environment: environment,
**query_params.to_h.symbolize_keys
)
end
def query_params
params.permit(:interval, :from, :to)
end
def set_polling_interval
Gitlab::PollingInterval.set_header(response, interval: POLLING_INTERVAL)
end
def environment
@environment ||= project.environments.find(params.delete("environment_id"))
end
def authorize_read_waf_anomalies!
render_403 unless can?(current_user, :read_threat_monitoring, project)
end
end
end
end
ee/app/services/security/waf_anomaly_summary_service.rb deleted 100644 → 0
View file @ 6dfed16d
# frozen_string_literal: true
module Security
# Service for fetching summary statistics from ElasticSearch.
# Queries ES and retrieves both total nginx requests & modsec violations
#
class WafAnomalySummaryService < ::BaseService
def initialize(environment:, cluster: environment.deployment_platform&.cluster, interval: 'day', from: 30.days.ago.iso8601, to: Time.zone.now.iso8601, options: {})
@environment = environment
@cluster = cluster
@interval = interval
@from = from
@to = to
@options = options
end
def execute(totals_only: false)
return if elasticsearch_client.nil?
return unless @environment.external_url
# Use multi-search with single query as we'll be adding nginx later
# with https://gitlab.com/gitlab-org/gitlab/issues/14707
aggregate_results = elasticsearch_client.msearch(body: body)
nginx_results, modsec_results = aggregate_results['responses']
if chart_above_v3?
nginx_total_requests = nginx_results.dig('hits', 'total', 'value').to_f
modsec_total_requests = modsec_results.dig('hits', 'total', 'value').to_f
else
nginx_total_requests = nginx_results.dig('hits', 'total').to_f
modsec_total_requests = modsec_results.dig('hits', 'total').to_f
end
return { total_traffic: nginx_total_requests, total_anomalous_traffic: modsec_total_requests } if totals_only
anomalous_traffic_count = nginx_total_requests == 0 ? 0 : (modsec_total_requests / nginx_total_requests).round(2)
{
total_traffic: nginx_total_requests,
anomalous_traffic: anomalous_traffic_count,
history: {
nominal: histogram_from(nginx_results),
anomalous: histogram_from(modsec_results)
},
interval: @interval,
from: @from,
to: @to,
status: :success
}
end
def elasticsearch_client
@elasticsearch_client ||= elastic_stack_adapter&.elasticsearch_client(timeout: @options[:timeout])
end
private
def elastic_stack_adapter
@elastic_stack_adapter ||= @cluster&.elastic_stack_adapter
end
def chart_above_v3?
elastic_stack_adapter.chart_above_v3?
end
def body
[
{ index: indices },
{
query: nginx_requests_query,
aggs: aggregations(@interval),
size: 0 # no docs needed, only counts
},
{ index: indices },
{
query: modsec_requests_query,
aggs: aggregations(@interval),
size: 0 # no docs needed, only counts
}
]
end
# Construct a list of daily indices to be searched. We do this programmatically
# based on the requested timeframe to reduce the load of querying all previous
# indices
def indices
(@from.to_date..@to.to_date).map do |day|
"filebeat-*-#{day.strftime('%Y.%m.%d')}"
end
end
def nginx_requests_query
{
bool: {
must: [
{
range: {
'@timestamp' => {
gte: @from,
lte: @to
}
}
},
{
terms_set: {
message: {
terms: environment_proxy_upstream_name_tokens,
minimum_should_match_script: {
source: 'params.num_terms'
}
}
}
},
{
match_phrase: {
'kubernetes.container.name' => {
query: ::Clusters::Applications::Ingress::INGRESS_CONTAINER_NAME
}
}
},
{
match_phrase: {
'kubernetes.namespace' => {
query: Gitlab::Kubernetes::Helm::NAMESPACE
}
}
},
{
match_phrase: {
stream: {
query: 'stdout'
}
}
}
]
}
}
end
def modsec_requests_query
{
bool: {
must: [
{
range: {
'@timestamp' => {
gte: @from,
lte: @to
}
}
},
{
prefix: {
'transaction.unique_id': application_server_name
}
},
{
match_phrase: {
'kubernetes.container.name' => {
query: ::Clusters::Applications::Ingress::MODSECURITY_LOG_CONTAINER_NAME
}
}
},
{
match_phrase: {
'kubernetes.namespace' => {
query: Gitlab::Kubernetes::Helm::NAMESPACE
}
}
}
]
}
}
end
def aggregations(interval)
{
counts: {
date_histogram: {
field: '@timestamp',
interval: interval,
order: {
'_key': 'asc'
}
}
}
}
end
def histogram_from(results)
buckets = results.dig('aggregations', 'counts', 'buckets') || []
buckets.map { |bucket| [bucket['key_as_string'], bucket['doc_count']] }
end
# Derive server_name to filter modsec audit log by environment
def application_server_name
@environment.formatted_external_url
end
# Derive proxy upstream name to filter nginx log by environment
# See https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/log-format/
def environment_proxy_upstream_name_tokens
[
*@environment.deployment_namespace.split('-'),
@environment.slug # $RELEASE_NAME
]
end
end
end
ee/app/views/projects/threat_monitoring/show.html.haml
View file @ d1146081
......@@ -8,9 +8,7 @@
#js-threat-monitoring-app{ data: { documentation_path: 'https://docs.gitlab.com/ee/user/application_security/threat_monitoring/',
empty_state_svg_path: image_path('illustrations/monitoring/unable_to_connect.svg'),
waf_no_data_svg_path: image_path('illustrations/firewall-not-detected-sm.svg'),
network_policy_no_data_svg_path: image_path('illustrations/network-policies-not-detected-sm.svg'),
waf_statistics_endpoint: summary_project_security_waf_anomalies_path(@project, format: :json),
network_policy_statistics_endpoint: summary_project_security_network_policies_path(@project, format: :json),
environments_endpoint: project_environments_path(@project),
network_policies_endpoint: project_security_network_policies_path(@project),
......
ee/config/routes/project.rb
View file @ d1146081
......@@ -53,10 +53,6 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
resources :audit_events, only: [:index]
namespace :security do
resources :waf_anomalies, only: [] do
get :summary, on: :collection
end
resources :network_policies, only: [:index, :create, :update, :destroy] do
get :summary, on: :collection
end
......
ee/spec/controllers/projects/security/waf_anomalies_controller_spec.rb deleted 100644 → 0
View file @ 6dfed16d
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Projects::Security::WafAnomaliesController do
let_it_be(:group) { create(:group) }
let_it_be(:user) { create(:user) }
let_it_be(:project) { create(:project, :public, :repository, group: group) }
let_it_be(:environment) { create(:environment, :with_review_app, project: project) }
let_it_be(:cluster) { create(:cluster, :provided_by_gcp, environment_scope: '*', projects: [environment.project]) }
let_it_be(:action_params) { { project_id: project, namespace_id: project.namespace, environment_id: environment } }
let(:es_client) { nil }
describe 'GET #summary' do
subject(:request) { get :summary, params: action_params, format: :json }
before do
stub_licensed_features(threat_monitoring: true)
sign_in(user)
allow_next_instance_of(::Security::WafAnomalySummaryService) do |instance|
allow(instance).to receive(:elasticsearch_client).at_most(3).times { es_client }
allow(instance).to receive(:chart_above_v3?) { true }
end
end
include_context '"Security & Compliance" permissions' do
let(:valid_request) { request }
before_request do
group.add_developer(user)
end
end
context 'with authorized user' do
before do
group.add_developer(user)
end
context 'with elastic_stack' do
let(:es_client) { double(Elasticsearch::Client) }
before do
allow(es_client).to receive(:msearch) { { "responses" => [{}, {}] } }
end
it 'returns anomaly summary' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(json_response['total_traffic']).to eq(0)
expect(json_response['anomalous_traffic']).to eq(0)
expect(response).to match_response_schema('vulnerabilities/summary', dir: 'ee')
end
end
context 'without elastic_stack' do
it 'returns not found' do
subject
expect(response).to have_gitlab_http_status(:not_found)
end
end
it 'sets a polling interval header' do
subject
expect(response.headers['Poll-Interval']).to eq('5000')
end
end
context 'with unauthorized user' do
it 'returns unauthorized' do
subject
expect(response).to have_gitlab_http_status(:forbidden)
end
end
end
end
ee/spec/frontend/threat_monitoring/components/threat_monitoring_section_spec.js
View file @ d1146081
......@@ -37,7 +37,7 @@ describe('ThreatMonitoringSection component', () => {
wrapper = shallowMount(ThreatMonitoringSection, {
propsData: {
storeNamespace: 'threatMonitoringNetworkPolicy',
title: 'Web Application Firewall',
title: 'Container Network Policy',
subtitle: 'Requests',
nominalTitle: 'Total Requests',
anomalousTitle: 'Anomalous Requests',
......
ee/spec/services/security/waf_anomaly_summary_service_spec.rb deleted 100644 → 0
View file @ 6dfed16d
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Security::WafAnomalySummaryService do
let(:environment) { create(:environment, :with_review_app, environment_type: 'review') }
let!(:cluster) do
create(:cluster, :provided_by_gcp, environment_scope: '*', projects: [environment.project])
end
let(:es_client) { double(Elasticsearch::Client) }
let(:chart_above_v3) { true }
let(:empty_response) do
{
'took' => 40,
'timed_out' => false,
'_shards' => { 'total' => 11, 'successful' => 11, 'skipped' => 0, 'failed' => 0 },
'hits' => { 'total' => { 'value' => 0, 'relation' => 'gte' }, 'max_score' => 0.0, 'hits' => [] },
'aggregations' => {
'counts' => {
'buckets' => []
}
},
'status' => 200
}
end
let(:nginx_response) do
empty_response.deep_merge(
'hits' => { 'total' => { 'value' => 3 } },
'aggregations' => {
'counts' => {
'buckets' => [
{ 'key_as_string' => '2020-02-14T23:00:00.000Z', 'key' => 1575500400000, 'doc_count' => 1 },
{ 'key_as_string' => '2020-02-15T00:00:00.000Z', 'key' => 1575504000000, 'doc_count' => 0 },
{ 'key_as_string' => '2020-02-15T01:00:00.000Z', 'key' => 1575507600000, 'doc_count' => 0 },
{ 'key_as_string' => '2020-02-15T08:00:00.000Z', 'key' => 1575532800000, 'doc_count' => 2 }
]
}
}
)
end
let(:modsec_response) do
empty_response.deep_merge(
'hits' => { 'total' => { 'value' => 1 } },
'aggregations' => {
'counts' => {
'buckets' => [
{ 'key_as_string' => '2019-12-04T23:00:00.000Z', 'key' => 1575500400000, 'doc_count' => 0 },
{ 'key_as_string' => '2019-12-05T00:00:00.000Z', 'key' => 1575504000000, 'doc_count' => 0 },
{ 'key_as_string' => '2019-12-05T01:00:00.000Z', 'key' => 1575507600000, 'doc_count' => 0 },
{ 'key_as_string' => '2019-12-05T08:00:00.000Z', 'key' => 1575532800000, 'doc_count' => 1 }
]
}
}
)
end
let(:nginx_response_es6) do
empty_response.deep_merge(
'hits' => { 'total' => 3 },
'aggregations' => {
'counts' => {
'buckets' => [
{ 'key_as_string' => '2020-02-14T23:00:00.000Z', 'key' => 1575500400000, 'doc_count' => 1 },
{ 'key_as_string' => '2020-02-15T00:00:00.000Z', 'key' => 1575504000000, 'doc_count' => 0 },
{ 'key_as_string' => '2020-02-15T01:00:00.000Z', 'key' => 1575507600000, 'doc_count' => 0 },
{ 'key_as_string' => '2020-02-15T08:00:00.000Z', 'key' => 1575532800000, 'doc_count' => 2 }
]
}
}
)
end
let(:modsec_response_es6) do
empty_response.deep_merge(
'hits' => { 'total' => 1 },
'aggregations' => {
'counts' => {
'buckets' => [
{ 'key_as_string' => '2019-12-04T23:00:00.000Z', 'key' => 1575500400000, 'doc_count' => 0 },
{ 'key_as_string' => '2019-12-05T00:00:00.000Z', 'key' => 1575504000000, 'doc_count' => 0 },
{ 'key_as_string' => '2019-12-05T01:00:00.000Z', 'key' => 1575507600000, 'doc_count' => 0 },
{ 'key_as_string' => '2019-12-05T08:00:00.000Z', 'key' => 1575532800000, 'doc_count' => 1 }
]
}
}
)
end
subject { described_class.new(environment: environment) }
describe '#execute' do
context 'without cluster' do
before do
allow(environment).to receive(:deployment_platform) { nil }
end
it 'returns no results' do
expect(subject.execute).to be_nil
end
end
context 'without elastic_stack' do
it 'returns no results' do
expect(subject.execute).to be_nil
end
end
context 'with environment missing external_url' do
before do
allow(environment.deployment_platform.cluster).to receive_message_chain(
:integration_elastic_stack, :elasticsearch_client
) { es_client }
allow(environment).to receive(:external_url) { nil }
end
it 'returns nil' do
expect(subject.execute).to be_nil
end
end
context 'with default histogram' do
before do
allow(es_client).to receive(:msearch) do
{ 'responses' => [nginx_results, modsec_results] }
end
allow(environment.deployment_platform.cluster).to receive_message_chain(
:integration_elastic_stack, :elasticsearch_client
) { es_client }
allow(environment.deployment_platform.cluster).to receive_message_chain(
:integration_elastic_stack, :chart_above_v3?
) { chart_above_v3 }
end
context 'no requests' do
let(:nginx_results) { empty_response }
let(:modsec_results) { empty_response }
it 'returns results', :aggregate_failures do
results = subject.execute
expect(results.fetch(:status)).to eq :success
expect(results.fetch(:interval)).to eq 'day'
expect(results.fetch(:total_traffic)).to eq 0
expect(results.fetch(:anomalous_traffic)).to eq 0.0
end
context 'when totals_only is set to true' do
it 'returns totals only', :aggregate_failures do
results = subject.execute(totals_only: true)
expect(results).to eq(total_traffic: 0.0, total_anomalous_traffic: 0.0)
end
end
end
context 'no violations' do
let(:nginx_results) { nginx_response }
let(:modsec_results) { empty_response }
it 'returns results', :aggregate_failures do
results = subject.execute
expect(results.fetch(:status)).to eq :success
expect(results.fetch(:interval)).to eq 'day'
expect(results.fetch(:total_traffic)).to eq 3
expect(results.fetch(:anomalous_traffic)).to eq 0.0
end
context 'when totals_only is set to true' do
it 'returns totals only', :aggregate_failures do
results = subject.execute(totals_only: true)
expect(results).to eq(total_traffic: 3.0, total_anomalous_traffic: 0.0)
end
end
end
context 'with violations' do
let(:nginx_results) { nginx_response }
let(:modsec_results) { modsec_response }
it 'returns results', :aggregate_failures do
results = subject.execute
expect(results.fetch(:status)).to eq :success
expect(results.fetch(:interval)).to eq 'day'
expect(results.fetch(:total_traffic)).to eq 3
expect(results.fetch(:anomalous_traffic)).to eq 0.33
end
context 'when totals_only is set to true' do
it 'returns totals only', :aggregate_failures do
results = subject.execute(totals_only: true)
expect(results).to eq(total_traffic: 3.0, total_anomalous_traffic: 1.0)
end
end
end
context 'with legacy es6 cluster' do
let(:chart_above_v3) { false }
let(:nginx_results) { nginx_response_es6 }
let(:modsec_results) { modsec_response_es6 }
it 'returns results', :aggregate_failures do
results = subject.execute
expect(results.fetch(:status)).to eq :success
expect(results.fetch(:interval)).to eq 'day'
expect(results.fetch(:total_traffic)).to eq 3
expect(results.fetch(:anomalous_traffic)).to eq 0.33
end
end
end
context 'with review app' do
it 'resolves transaction_id from external_url' do
allow(subject).to receive(:elasticsearch_client) { es_client }
allow(subject).to receive(:chart_above_v3?) { chart_above_v3 }
expect(es_client).to receive(:msearch).with(
body: array_including(
hash_including(
query: hash_including(
bool: hash_including(
must: array_including(
hash_including(
prefix: hash_including(
'transaction.unique_id': environment.formatted_external_url
)
)
)
)
)
)
)
).and_return({ 'responses' => [{}, {}] })
subject.execute
end
end
context 'with time window' do
it 'passes time frame to ElasticSearch' do
from = 1.day.ago
to = Time.current
subject = described_class.new(
environment: environment,
from: from,
to: to
)
allow(subject).to receive(:elasticsearch_client) { es_client }
allow(subject).to receive(:chart_above_v3?) { chart_above_v3 }
expect(es_client).to receive(:msearch).with(
body: array_including(
hash_including(
query: hash_including(
bool: hash_including(
must: array_including(
hash_including(
range: hash_including(
'@timestamp' => {
gte: from,
lte: to
}
)
)
)
)
)
)
)
).and_return({ 'responses' => [{}, {}] })
subject.execute
end
end
context 'with interval' do
it 'passes interval to ElasticSearch' do
interval = 'hour'
subject = described_class.new(
environment: environment,
interval: interval
)
allow(subject).to receive(:elasticsearch_client) { es_client }
allow(subject).to receive(:chart_above_v3?) { chart_above_v3 }
expect(es_client).to receive(:msearch).with(
body: array_including(
hash_including(
aggs: hash_including(
counts: hash_including(
date_histogram: hash_including(
interval: interval
)
)
)
)
)
).and_return({ 'responses' => [{}, {}] })
subject.execute
end
end
end
end
spec/factories/clusters/applications/helm.rb
View file @ d1146081
......@@ -96,26 +96,7 @@ FactoryBot.define do
end
factory :clusters_applications_ingress, class: 'Clusters::Applications::Ingress' do
modsecurity_enabled { false }
cluster factory: %i(cluster with_installed_helm provided_by_gcp)
trait :modsecurity_blocking do
modsecurity_enabled { true }
modsecurity_mode { :blocking }
end
trait :modsecurity_logging do
modsecurity_enabled { true }
modsecurity_mode { :logging }
end
trait :modsecurity_disabled do
modsecurity_enabled { false }
end
trait :modsecurity_not_installed do
modsecurity_enabled { nil }
end
end
factory :clusters_applications_cert_manager, class: 'Clusters::Applications::CertManager' do
......@@ -153,7 +134,6 @@ FactoryBot.define do
factory :clusters_applications_fluentd, class: 'Clusters::Applications::Fluentd' do
host { 'example.com' }
waf_log_enabled { true }
cilium_log_enabled { true }
cluster factory: %i(cluster with_installed_helm provided_by_gcp)
end
......
spec/fixtures/api/schemas/cluster_status.json
View file @ d1146081
......@@ -42,7 +42,6 @@
"host": {"type": ["string", "null"]},
"port": {"type": ["integer", "514"]},
"protocol": {"type": ["integer", "0"]},
"waf_log_enabled": {"type": ["boolean", "true"]},
"cilium_log_enabled": {"type": ["boolean", "true"]},
"update_available": { "type": ["boolean", "null"] },
"can_uninstall": { "type": "boolean" },
......
spec/lib/gitlab/usage/metrics/name_suggestion_spec.rb
View file @ d1146081
......@@ -38,22 +38,6 @@ RSpec.describe Gitlab::Usage::Metrics::NameSuggestion do
end
context 'joined relations' do
context 'counted attribute comes from joined relation' do
it_behaves_like 'name suggestion' do
let(:operation) { :distinct_count }
let(:column) { ::Deployment.arel_table[:environment_id] }
let(:relation) do
::Clusters::Applications::Ingress.modsecurity_enabled.logging
.joins(cluster: :deployments)
.merge(::Clusters::Cluster.enabled)
.merge(Deployment.success)
end
let(:constraints) { /'\(clusters_applications_ingress\.modsecurity_enabled = TRUE AND clusters_applications_ingress\.modsecurity_mode = \d+ AND clusters.enabled = TRUE AND deployments.status = \d+\)'/ }
let(:name_suggestion) { /count_distinct_environment_id_from_<adjective describing\: #{constraints}>_deployments_<with>_<adjective describing\: #{constraints}>_clusters_<having>_<adjective describing\: #{constraints}>_clusters_applications_ingress/ }
end
end
context 'counted attribute comes from source relation' do
it_behaves_like 'name suggestion' do
# corresponding metric is collected with count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id)
......
spec/models/clusters/applications/fluentd_spec.rb
View file @ d1146081
......@@ -3,9 +3,8 @@
require 'spec_helper'
RSpec.describe Clusters::Applications::Fluentd do
let(:waf_log_enabled) { true }
let(:cilium_log_enabled) { true }
let(:fluentd) { create(:clusters_applications_fluentd, waf_log_enabled: waf_log_enabled, cilium_log_enabled: cilium_log_enabled) }
let(:fluentd) { create(:clusters_applications_fluentd, cilium_log_enabled: cilium_log_enabled) }
include_examples 'cluster application core specs', :clusters_applications_fluentd
include_examples 'cluster application status specs', :clusters_applications_fluentd
......@@ -51,13 +50,11 @@ RSpec.describe Clusters::Applications::Fluentd do
end
describe '#values' do
let(:modsecurity_log_path) { "/var/log/containers/*#{Clusters::Applications::Ingress::MODSECURITY_LOG_CONTAINER_NAME}*.log" }
let(:cilium_log_path) { "/var/log/containers/*#{described_class::CILIUM_CONTAINER_NAME}*.log" }
subject { fluentd.values }
context 'with both logs variables set to false' do
let(:waf_log_enabled) { false }
context 'with cilium_log_enabled set to false' do
let(:cilium_log_enabled) { false }
it "raises ActiveRecord::RecordInvalid" do
......@@ -65,18 +62,8 @@ RSpec.describe Clusters::Applications::Fluentd do
end
end
context 'with both logs variables set to true' do
it { is_expected.to include("#{modsecurity_log_path},#{cilium_log_path}") }
end
context 'with waf_log_enabled set to true' do
let(:cilium_log_enabled) { false }
it { is_expected.to include(modsecurity_log_path) }
end
context 'with cilium_log_enabled set to true' do
let(:waf_log_enabled) { false }
let(:cilium_log_enabled) { true }
it { is_expected.to include(cilium_log_path) }
end
......
spec/models/clusters/applications/ingress_spec.rb
View file @ d1146081
......@@ -172,94 +172,4 @@ RSpec.describe Clusters::Applications::Ingress do
expect(values).to include('clusterIP')
end
end
describe '#values' do
subject { ingress }
context 'when modsecurity_enabled is enabled' do
before do
allow(subject).to receive(:modsecurity_enabled).and_return(true)
end
it 'includes modsecurity module enablement' do
expect(subject.values).to include("enable-modsecurity: 'true'")
end
it 'includes modsecurity core ruleset enablement set to false' do
expect(subject.values).to include("enable-owasp-modsecurity-crs: 'false'")
end
it 'includes modsecurity snippet with information related to security rules' do
expect(subject.values).to include("SecRuleEngine DetectionOnly")
expect(subject.values).to include("Include #{described_class::MODSECURITY_OWASP_RULES_FILE}")
end
context 'when modsecurity_mode is set to :blocking' do
before do
subject.blocking!
end
it 'includes modsecurity snippet with information related to security rules' do
expect(subject.values).to include("SecRuleEngine On")
expect(subject.values).to include("Include #{described_class::MODSECURITY_OWASP_RULES_FILE}")
end
end
it 'includes modsecurity.conf content' do
expect(subject.values).to include('modsecurity.conf')
# Includes file content from Ingress#modsecurity_config_content
expect(subject.values).to include('SecAuditLog')
expect(subject.values).to include('extraVolumes')
expect(subject.values).to include('extraVolumeMounts')
end
it 'includes modsecurity sidecar container' do
expect(subject.values).to include('modsecurity-log-volume')
expect(subject.values).to include('extraContainers')
end
it 'executes command to tail modsecurity logs with -F option' do
args = YAML.safe_load(subject.values).dig('controller', 'extraContainers', 0, 'args')
expect(args).to eq(['/bin/sh', '-c', 'tail -F /var/log/modsec/audit.log'])
end
it 'includes livenessProbe for modsecurity sidecar container' do
probe_config = YAML.safe_load(subject.values).dig('controller', 'extraContainers', 0, 'livenessProbe')
expect(probe_config).to eq('exec' => { 'command' => ['ls', '/var/log/modsec/audit.log'] })
end
end
context 'when modsecurity_enabled is disabled' do
before do
allow(subject).to receive(:modsecurity_enabled).and_return(false)
end
it 'excludes modsecurity module enablement' do
expect(subject.values).not_to include('enable-modsecurity')
end
it 'excludes modsecurity core ruleset enablement' do
expect(subject.values).not_to include('enable-owasp-modsecurity-crs')
end
it 'excludes modsecurity.conf content' do
expect(subject.values).not_to include('modsecurity.conf')
# Excludes file content from Ingress#modsecurity_config_content
expect(subject.values).not_to include('SecAuditLog')
expect(subject.values).not_to include('extraVolumes')
expect(subject.values).not_to include('extraVolumeMounts')
end
it 'excludes modsecurity sidecar container' do
expect(subject.values).not_to include('modsecurity-log-volume')
expect(subject.values).not_to include('extraContainers')
end
end
end
end
spec/models/clusters/cluster_spec.rb
View file @ d1146081
......@@ -196,28 +196,6 @@ RSpec.describe Clusters::Cluster, :use_clean_rails_memory_store_caching do
end
end
describe '.with_enabled_modsecurity' do
subject { described_class.with_enabled_modsecurity }
let_it_be(:cluster) { create(:cluster) }
context 'cluster has ingress application with enabled modsecurity' do
let!(:application) { create(:clusters_applications_ingress, :installed, :modsecurity_logging, cluster: cluster) }
it { is_expected.to include(cluster) }
end
context 'cluster has ingress application with disabled modsecurity' do
let!(:application) { create(:clusters_applications_ingress, :installed, :modsecurity_disabled, cluster: cluster) }
it { is_expected.not_to include(cluster) }
end
context 'cluster does not have ingress application' do
it { is_expected.not_to include(cluster) }
end
end
describe '.with_available_elasticstack' do
subject { described_class.with_available_elasticstack }
......
spec/serializers/cluster_application_entity_spec.rb
View file @ d1146081
......@@ -85,7 +85,6 @@ RSpec.describe ClusterApplicationEntity do
expect(subject[:port]).to eq(514)
expect(subject[:host]).to eq("example.com")
expect(subject[:protocol]).to eq("tcp")
expect(subject[:waf_log_enabled]).to be true
expect(subject[:cilium_log_enabled]).to be true
end
end
......
spec/services/clusters/applications/create_service_spec.rb
View file @ d1146081
......@@ -46,8 +46,7 @@ RSpec.describe Clusters::Applications::CreateService do
context 'ingress application' do
let(:params) do
{
application: 'ingress',
modsecurity_enabled: true
application: 'ingress'
}
end
......@@ -64,10 +63,6 @@ RSpec.describe Clusters::Applications::CreateService do
cluster.reload
end.to change(cluster, :application_ingress)
end
it 'sets modsecurity_enabled' do
expect(subject.modsecurity_enabled).to eq(true)
end
end
context 'cert manager application' do
......
vendor/elastic_stack/values.yaml
View file @ d1146081
......@@ -61,12 +61,6 @@ filebeat:
target_field: tie_breaker_id
- add_cloud_metadata: ~
- add_kubernetes_metadata: ~
- decode_json_fields:
fields: ["message"]
when:
equals:
kubernetes.container.namespace: "gitlab-managed-apps"
kubernetes.container.name: "modsecurity-log"
kibana:
enabled: false
elasticsearchHosts: "http://elastic-stack-elasticsearch-master:9200"
......
vendor/ingress/modsecurity.conf deleted 100644 → 0
View file @ 6dfed16d
# -- GitLab Customization ----------------------------------------------
# Based on https://github.com/SpiderLabs/ModSecurity/blob/v3.0.3/modsecurity.conf-recommended
# Our base modsecurity.conf includes some minor customization:
# - `SecRuleEngine` is disabled, defaulting to `DetectionOnly`. Overridable at project-level
# - `SecAuditLogType` is disabled, defaulting to `Serial`. Overridable at project-level
# - `SecStatusEngine` is disabled, to disallow usage reporting
#
# ----------------------------------------------------------------------------
# -- Rule engine initialization ----------------------------------------------
# Enable ModSecurity, attaching it to every transaction. Use detection
# only to start with, because that minimises the chances of post-installation
# disruption.
#
# SecRuleEngine DetectionOnly
# -- Request body handling ---------------------------------------------------
# Allow ModSecurity to access request bodies. If you don't, ModSecurity
# won't be able to see any POST parameters, which opens a large security
# hole for attackers to exploit.
#
SecRequestBodyAccess On
# Enable XML request body parser.
# Initiate XML Processor in case of xml content-type
#
SecRule REQUEST_HEADERS:Content-Type "(?:application(?:/soap\+|/)|text/)xml" \
"id:'200000',phase:1,t:none,t:lowercase,pass,nolog,ctl:requestBodyProcessor=XML"
# Enable JSON request body parser.
# Initiate JSON Processor in case of JSON content-type; change accordingly
# if your application does not use 'application/json'
#
SecRule REQUEST_HEADERS:Content-Type "application/json" \
"id:'200001',phase:1,t:none,t:lowercase,pass,nolog,ctl:requestBodyProcessor=JSON"
# Maximum request body size we will accept for buffering. If you support
# file uploads then the value given on the first line has to be as large
# as the largest file you are willing to accept. The second value refers
# to the size of data, with files excluded. You want to keep that value as
# low as practical.
#
SecRequestBodyLimit 13107200
SecRequestBodyNoFilesLimit 131072
# What do do if the request body size is above our configured limit.
# Keep in mind that this setting will automatically be set to ProcessPartial
# when SecRuleEngine is set to DetectionOnly mode in order to minimize
# disruptions when initially deploying ModSecurity.
#
SecRequestBodyLimitAction Reject
# Verify that we've correctly processed the request body.
# As a rule of thumb, when failing to process a request body
# you should reject the request (when deployed in blocking mode)
# or log a high-severity alert (when deployed in detection-only mode).
#
SecRule REQBODY_ERROR "!@eq 0" \
"id:'200002', phase:2,t:none,log,deny,status:400,msg:'Failed to parse request body.',logdata:'%{reqbody_error_msg}',severity:2"
# By default be strict with what we accept in the multipart/form-data
# request body. If the rule below proves to be too strict for your
# environment consider changing it to detection-only. You are encouraged
# _not_ to remove it altogether.
#
SecRule MULTIPART_STRICT_ERROR "!@eq 0" \
"id:'200003',phase:2,t:none,log,deny,status:400, \
msg:'Multipart request body failed strict validation: \
PE %{REQBODY_PROCESSOR_ERROR}, \
BQ %{MULTIPART_BOUNDARY_QUOTED}, \
BW %{MULTIPART_BOUNDARY_WHITESPACE}, \
DB %{MULTIPART_DATA_BEFORE}, \
DA %{MULTIPART_DATA_AFTER}, \
HF %{MULTIPART_HEADER_FOLDING}, \
LF %{MULTIPART_LF_LINE}, \
SM %{MULTIPART_MISSING_SEMICOLON}, \
IQ %{MULTIPART_INVALID_QUOTING}, \
IP %{MULTIPART_INVALID_PART}, \
IH %{MULTIPART_INVALID_HEADER_FOLDING}, \
FL %{MULTIPART_FILE_LIMIT_EXCEEDED}'"
# Did we see anything that might be a boundary?
#
# Here is a short description about the ModSecurity Multipart parser: the
# parser returns with value 0, if all "boundary-like" line matches with
# the boundary string which given in MIME header. In any other cases it returns
# with different value, eg. 1 or 2.
#
# The RFC 1341 descript the multipart content-type and its syntax must contains
# only three mandatory lines (above the content):
# * Content-Type: multipart/mixed; boundary=BOUNDARY_STRING
# * --BOUNDARY_STRING
# * --BOUNDARY_STRING--
#
# First line indicates, that this is a multipart content, second shows that
# here starts a part of the multipart content, third shows the end of content.
#
# If there are any other lines, which starts with "--", then it should be
# another boundary id - or not.
#
# After 3.0.3, there are two kinds of types of boundary errors: strict and permissive.
#
# If multipart content contains the three necessary lines with correct order, but
# there are one or more lines with "--", then parser returns with value 2 (non-zero).
#
# If some of the necessary lines (usually the start or end) misses, or the order
# is wrong, then parser returns with value 1 (also a non-zero).
#
# You can choose, which one is what you need. The example below contains the
# 'strict' mode, which means if there are any lines with start of "--", then
# ModSecurity blocked the content. But the next, commented example contains
# the 'permissive' mode, then you check only if the necessary lines exists in
# correct order. Whit this, you can enable to upload PEM files (eg "----BEGIN.."),
# or other text files, which contains eg. HTTP headers.
#
# The difference is only the operator - in strict mode (first) the content blocked
# in case of any non-zero value. In permissive mode (second, commented) the
# content blocked only if the value is explicit 1. If it 0 or 2, the content will
# allowed.
#
#
# See #1747 and #1924 for further information on the possible values for
# MULTIPART_UNMATCHED_BOUNDARY.
#
SecRule MULTIPART_UNMATCHED_BOUNDARY "@eq 1" \
"id:'200004',phase:2,t:none,log,deny,msg:'Multipart parser detected a possible unmatched boundary.'"
# PCRE Tuning
# We want to avoid a potential RegEx DoS condition
#
SecPcreMatchLimit 1000
SecPcreMatchLimitRecursion 1000
# Some internal errors will set flags in TX and we will need to look for these.
# All of these are prefixed with "MSC_". The following flags currently exist:
#
# MSC_PCRE_LIMITS_EXCEEDED: PCRE match limits were exceeded.
#
SecRule TX:/^MSC_/ "!@streq 0" \
"id:'200005',phase:2,t:none,deny,msg:'ModSecurity internal error flagged: %{MATCHED_VAR_NAME}'"
# -- Response body handling --------------------------------------------------
# Allow ModSecurity to access response bodies.
# You should have this directive enabled in order to identify errors
# and data leakage issues.
#
# Do keep in mind that enabling this directive does increases both
# memory consumption and response latency.
#
SecResponseBodyAccess On
# Which response MIME types do you want to inspect? You should adjust the
# configuration below to catch documents but avoid static files
# (e.g., images and archives).
#
SecResponseBodyMimeType text/plain text/html text/xml
# Buffer response bodies of up to 512 KB in length.
SecResponseBodyLimit 524288
# What happens when we encounter a response body larger than the configured
# limit? By default, we process what we have and let the rest through.
# That's somewhat less secure, but does not break any legitimate pages.
#
SecResponseBodyLimitAction ProcessPartial
# -- Filesystem configuration ------------------------------------------------
# The location where ModSecurity stores temporary files (for example, when
# it needs to handle a file upload that is larger than the configured limit).
#
# This default setting is chosen due to all systems have /tmp available however,
# this is less than ideal. It is recommended that you specify a location that's private.
#
SecTmpDir /tmp/
# The location where ModSecurity will keep its persistent data. This default setting
# is chosen due to all systems have /tmp available however, it
# too should be updated to a place that other users can't access.
#
SecDataDir /tmp/
# -- File uploads handling configuration -------------------------------------
# The location where ModSecurity stores intercepted uploaded files. This
# location must be private to ModSecurity. You don't want other users on
# the server to access the files, do you?
#
#SecUploadDir /opt/modsecurity/var/upload/
# By default, only keep the files that were determined to be unusual
# in some way (by an external inspection script). For this to work you
# will also need at least one file inspection rule.
#
#SecUploadKeepFiles RelevantOnly
# Uploaded files are by default created with permissions that do not allow
# any other user to access them. You may need to relax that if you want to
# interface ModSecurity to an external program (e.g., an anti-virus).
#
#SecUploadFileMode 0600
# -- Debug log configuration -------------------------------------------------
# The default debug log configuration is to duplicate the error, warning
# and notice messages from the error log.
#
#SecDebugLog /opt/modsecurity/var/log/debug.log
#SecDebugLogLevel 3
# -- Audit log configuration -------------------------------------------------
# Log the transactions that are marked by a rule, as well as those that
# trigger a server error (determined by a 5xx or 4xx, excluding 404,
# level response status codes).
#
SecAuditEngine RelevantOnly
SecAuditLogRelevantStatus "^(?:5|4(?!04))"
# Log everything we know about a transaction.
SecAuditLogParts ABIJDEFHZ
# Use a single file for logging. This is much easier to look at, but
# assumes that you will use the audit log only ocassionally.
#
# SecAuditLogType Serial
SecAuditLogFormat JSON
SecAuditLog /var/log/modsec/audit.log
# Specify the path for concurrent audit logging.
#SecAuditLogStorageDir /opt/modsecurity/var/audit/
# -- Miscellaneous -----------------------------------------------------------
# Use the most commonly used application/x-www-form-urlencoded parameter
# separator. There's probably only one application somewhere that uses
# something else so don't expect to change this value.
#
SecArgumentSeparator &
# Settle on version 0 (zero) cookies, as that is what most applications
# use. Using an incorrect cookie version may open your installation to
# evasion attacks (against the rules that examine named cookies).
#
SecCookieFormat 0
# Specify your Unicode Code Point.
# This mapping is used by the t:urlDecodeUni transformation function
# to properly map encoded data to your language. Properly setting
# these directives helps to reduce false positives and negatives.
#
SecUnicodeMapFile unicode.mapping 20127
# Improve the quality of ModSecurity by sharing information about your
# current ModSecurity version and dependencies versions.
# The following information will be shared: ModSecurity version,
# Web Server version, APR version, PCRE version, Lua version, Libxml2
# version, Anonymous unique id for host.
# SecStatusEngine On
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7