Merge branch '6954_add_group_level_vulnerability_history_API_endpoint' into 'master'

Add API endpoint for vulnerability history at group level

See merge request gitlab-org/gitlab-ee!8603

ee/app/controllers/groups/security/vulnerabilities_controller.rb

 # frozen_string_literal: true
 class Groups::Security::VulnerabilitiesController < Groups::Security::ApplicationController
+  HISTORY_RANGE = 3.months
+
+  before_action :check_group_security_dashboard_history_feature_flag!, only: [:history]
+
   def index
     @vulnerabilities = group.latest_vulnerabilities
       .sast # FIXME: workaround until https://gitlab.com/gitlab-org/gitlab-ee/issues/6240
@@ -23,4 +27,16 @@ class Groups::Security::VulnerabilitiesController < Groups::Security::Applicatio
       end
     end
   end
+
+  def history
+    respond_to do |format|
+      format.json do
+        render json: Vulnerabilities::HistorySerializer.new.represent(group.all_vulnerabilities.count_by_day_and_severity(HISTORY_RANGE))
+      end
+    end
+  end
+
+  def check_group_security_dashboard_history_feature_flag!
+    render_404 unless ::Feature.enabled?(:group_security_dashboard_history, group, default_enabled: true)
+  end
 end

ee/app/models/ee/group.rb

@@ -93,6 +93,11 @@ module EE
         .for_pipelines(all_pipelines.with_vulnerabilities.latest_successful_ids_per_project)
     end
 
+    def all_vulnerabilities
+      Vulnerabilities::Occurrence
+        .for_pipelines(all_pipelines.with_vulnerabilities.success)
+    end
+
     def human_ldap_access
       ::Gitlab::Access.options_with_owner.key(ldap_access)
     end

ee/app/models/vulnerabilities/occurrence.rb

@@ -73,6 +73,14 @@ module Vulnerabilities
         .where(vulnerability_occurrence_pipelines: { pipeline_id: pipelines })
     end
 
+    def self.count_by_day_and_severity(period)
+      joins(:occurrence_pipelines)
+        .select('CAST(vulnerability_occurrence_pipelines.created_at AS DATE) AS day', :severity, 'COUNT(distinct vulnerability_occurrences.id) as count')
+        .where(['vulnerability_occurrence_pipelines.created_at >= ?', Date.today - period])
+        .group(:day, :severity)
+        .order('day')
+    end
+
     def feedback(feedback_type:)
       params = {
         project_id: project_id,

ee/app/serializers/vulnerabilities/history_entity.rb

+# frozen_string_literal: true
+
+class Vulnerabilities::HistoryEntity < Grape::Entity
+  present_collection true
+
+  Vulnerabilities::Occurrence::LEVELS.keys.each do |level|
+    expose level do |object|
+      counts(by_severity[level]&.group_by(&:day) || {})
+    end
+  end
+
+  expose :total do |object|
+    counts(by_days)
+  end
+
+  private
+
+  def by_days
+    items.group_by(&:day)
+  end
+
+  def by_severity
+    items.group_by(&:severity)
+  end
+
+  def items
+    object[:items]
+  end
+
+  def counts(hash)
+    hash.transform_values { |items| items.sum(&:count) } # rubocop: disable CodeReuse/ActiveRecord
+  end
+end

ee/app/serializers/vulnerabilities/history_serializer.rb

+# frozen_string_literal: true
+
+class Vulnerabilities::HistorySerializer < BaseSerializer
+  entity Vulnerabilities::HistoryEntity
+end

ee/changelogs/unreleased/6954_add_group_level_vulnerability_history_API_endpoint.yml

+---
+title: Add vulnerability history at group level
+merge_request: 8603
+author:
+type: added

ee/config/routes/group.rb

@@ -70,6 +70,7 @@ constraints(::Constraints::GroupUrlConstrainer.new) do
       resources :vulnerabilities, only: [:index], controller: :vulnerabilities do
         collection do
           get :summary
+          get :history
         end
       end
     end

ee/spec/controllers/groups/security/vulnerabilities_controller_spec.rb

@@ -214,4 +214,123 @@ describe Groups::Security::VulnerabilitiesController do
       end
     end
   end
+
+  describe 'GET history.json' do
+    subject { get :history,  group_id: group, format: :json }
+
+    context 'when security dashboard feature is disabled' do
+      before do
+        stub_licensed_features(security_dashboard: false)
+      end
+
+      it 'returns 404' do
+        subject
+
+        expect(response).to have_gitlab_http_status(404)
+      end
+    end
+
+    context 'when group security dashboard history feature flag is disabled' do
+      before do
+        stub_licensed_features(security_dashboard: true)
+        stub_feature_flags(group_security_dashboard_history: false)
+        group.add_developer(user)
+      end
+
+      it 'returns 404' do
+        subject
+        expect(response).to have_gitlab_http_status(404)
+      end
+    end
+
+    context 'when security dashboard feature is enabled' do
+      before do
+        stub_licensed_features(security_dashboard: true)
+
+        travel_to(Time.zone.parse('2018-11-10')) do
+          pipeline_1 = create(:ci_pipeline, :success, project: project_dev)
+          pipeline_2 = create(:ci_pipeline, :success, project: project_dev)
+
+          create_list(:vulnerabilities_occurrence, 2,
+            pipelines: [pipeline_1], project: project_dev, report_type: :sast, severity: :high)
+
+          create_list(:vulnerabilities_occurrence, 1,
+            pipelines: [pipeline_1], project: project_dev, report_type: :dependency_scanning, severity: :low)
+
+          create_list(:vulnerabilities_occurrence, 1,
+            pipelines: [pipeline_1, pipeline_2], project: project_dev, report_type: :sast, severity: :critical)
+
+          create_list(:vulnerabilities_occurrence, 1,
+            pipelines: [pipeline_1, pipeline_2], project: project_dev, report_type: :dependency_scanning, severity: :low)
+        end
+
+        travel_to(Time.zone.parse('2018-11-12')) do
+          pipeline = create(:ci_pipeline, :success, project: project_dev)
+
+          create_list(:vulnerabilities_occurrence, 2,
+            pipelines: [pipeline], project: project_dev, report_type: :dependency_scanning, severity: :low)
+
+          create_list(:vulnerabilities_occurrence, 1,
+            pipelines: [pipeline], project: project_dev, report_type: :dast, severity: :medium)
+
+          create_list(:vulnerabilities_occurrence, 1,
+            pipelines: [pipeline], project: project_dev, report_type: :dast, severity: :low)
+        end
+      end
+
+      context 'when user has guest access' do
+        before do
+          group.add_guest(user)
+        end
+
+        it 'returns 403' do
+          subject
+
+          expect(response).to have_gitlab_http_status(403)
+        end
+      end
+
+      context 'when user has developer access' do
+        before do
+          group.add_developer(user)
+        end
+
+        it 'returns vulnerability history within last 90 days' do
+          travel_to(Time.zone.parse('2019-02-10')) do
+            subject
+          end
+
+          expect(response).to have_gitlab_http_status(200)
+          expect(json_response).to be_an(Hash)
+          expect(json_response['total']).to eq({ '2018-11-10' => 5, '2018-11-12' => 4 })
+          expect(json_response['critical']).to eq({ '2018-11-10' => 1 })
+          expect(json_response['high']).to eq({ '2018-11-10' => 2 })
+          expect(json_response['medium']).to eq({ '2018-11-12' => 1 })
+          expect(json_response['low']).to eq({ '2018-11-10' => 2, '2018-11-12' => 3 })
+          expect(response).to match_response_schema('vulnerabilities/history', dir: 'ee')
+        end
+
+        it 'returns empty history if there are no vulnerabilities within last 90 days' do
+          travel_to(Time.zone.parse('2019-02-13')) do
+            subject
+          end
+
+          expect(response).to have_gitlab_http_status(200)
+          expect(json_response).to be_an(Hash)
+          expect(json_response).to eq({
+            "undefined" => {},
+            "ignore" => {},
+            "unknown" => {},
+            "experimental" => {},
+            "low" => {},
+            "medium" => {},
+            "high" => {},
+            "critical" => {},
+            "total" => {}
+          })
+          expect(response).to match_response_schema('vulnerabilities/history', dir: 'ee')
+        end
+      end
+    end
+  end
 end

ee/spec/fixtures/api/schemas/vulnerabilities/history.json

+{
+  "type" : "object",
+  "required" : [
+    "total"
+  ],
+  "properties" : {
+    "total": { "type" : "object" },
+    "undefined": { "type" : "object" },
+    "ignore": { "type" : "object" },
+    "unknown": { "type" : "object" },
+    "experimental": { "type" : "object" },
+    "low": { "type" : "object" },
+    "medium": { "type" : "object" },
+    "high": { "type" : "object" },
+    "critical": { "type" : "object" }
+  },
+  "additional_properties" : false
+}

ee/spec/models/group_spec.rb

@@ -293,6 +293,39 @@ describe Group do
     end
   end
 
+  describe '#all_vulnerabilities' do
+    let(:project) { create(:project, namespace: group) }
+    let(:external_project) { create(:project) }
+    let(:failed_pipeline) { create(:ci_pipeline, :failed, project: project) }
+
+    let!(:old_vuln) { create_vulnerability(project) }
+    let!(:new_vuln) { create_vulnerability(project) }
+    let!(:external_vuln) { create_vulnerability(external_project) }
+    let!(:failed_vuln) { create_vulnerability(project, failed_pipeline) }
+
+    subject { group.all_vulnerabilities }
+
+    def create_vulnerability(project, pipeline = nil)
+      pipeline ||= create(:ci_pipeline, :success, project: project)
+      create(:vulnerabilities_occurrence, pipelines: [pipeline], project: project)
+    end
+
+    it 'returns vulns for all successful pipelines of projects belonging to the group' do
+      is_expected.to contain_exactly(old_vuln, new_vuln)
+    end
+
+    context 'with vulnerabilities from other branches' do
+      let!(:branch_pipeline) { create(:ci_pipeline, :success, project: project, ref: 'feature-x') }
+      let!(:branch_vuln) { create(:vulnerabilities_occurrence, pipelines: [branch_pipeline], project: project) }
+
+      # TODO: This should actually fail and we must scope vulns
+      # per branch as soon as we store them for other branches
+      it 'includes vulnerabilities from all branches' do
+        is_expected.to contain_exactly(old_vuln, new_vuln, branch_vuln)
+      end
+    end
+  end
+
   describe '#saml_discovery_token' do
     it 'returns existing tokens' do
       group = create(:group, saml_discovery_token: 'existing')

ee/spec/models/vulnerabilities/occurrence_spec.rb

@@ -81,4 +81,48 @@ describe Vulnerabilities::Occurrence do
       end
     end
   end
+
+  describe '.count_by_day_and_severity' do
+    let(:project) { create(:project) }
+    let(:date_1) { Time.zone.parse('2018-11-10') }
+    let(:date_2) { Time.zone.parse('2018-11-12') }
+
+    before do
+      travel_to(date_1) do
+        pipeline = create(:ci_pipeline, :success, project: project)
+
+        create_list(:vulnerabilities_occurrence, 2,
+          pipelines: [pipeline], project: project, report_type: :sast, severity: :high)
+      end
+
+      travel_to(date_2) do
+        pipeline = create(:ci_pipeline, :success, project: project)
+
+        create_list(:vulnerabilities_occurrence, 2,
+          pipelines: [pipeline], project: project, report_type: :dependency_scanning, severity: :low)
+
+        create_list(:vulnerabilities_occurrence, 1,
+          pipelines: [pipeline], project: project, report_type: :dast, severity: :medium)
+
+        create_list(:vulnerabilities_occurrence, 1,
+          pipelines: [pipeline], project: project, report_type: :dast, severity: :low)
+      end
+    end
+
+    subject do
+      travel_to(Time.zone.parse('2018-11-15')) do
+        described_class.count_by_day_and_severity(3.days)
+      end
+    end
+
+    it 'returns expected counts for occurrences within given period' do
+      first, second = subject
+      expect(first.day).to eq(date_2)
+      expect(first.severity).to eq('low')
+      expect(first.count).to eq(3)
+      expect(second.day).to eq(date_2)
+      expect(second.severity).to eq('medium')
+      expect(second.count).to eq(1)
+    end
+  end
 end

ee/spec/routing/group_routing_spec.rb

@@ -61,4 +61,30 @@ describe 'Group routing', "routing" do
       end
     end
   end
+
+  describe 'security' do
+    it 'shows group dashboard' do
+      allow(Group).to receive(:find_by_full_path).with('gitlabhq', any_args).and_return(true)
+
+      expect(get('/groups/gitlabhq/-/security/dashboard')).to route_to('groups/security/dashboard#show', group_id: 'gitlabhq')
+    end
+
+    it 'lists vulnerabilities' do
+      allow(Group).to receive(:find_by_full_path).with('gitlabhq', any_args).and_return(true)
+
+      expect(get('/groups/gitlabhq/-/security/vulnerabilities')).to route_to('groups/security/vulnerabilities#index', group_id: 'gitlabhq')
+    end
+
+    it 'shows vulnerability summary' do
+      allow(Group).to receive(:find_by_full_path).with('gitlabhq', any_args).and_return(true)
+
+      expect(get('/groups/gitlabhq/-/security/vulnerabilities/summary')).to route_to('groups/security/vulnerabilities#summary', group_id: 'gitlabhq')
+    end
+
+    it 'shows vulnerability history' do
+      allow(Group).to receive(:find_by_full_path).with('gitlabhq', any_args).and_return(true)
+
+      expect(get('/groups/gitlabhq/-/security/vulnerabilities/history')).to route_to('groups/security/vulnerabilities#history', group_id: 'gitlabhq')
+    end
+  end
 end

ee/spec/serializers/vulnerabilities/history_entity_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Vulnerabilities::HistoryEntity do
+  let(:group) { create(:group) }
+  let(:project) { create(:project, group: group) }
+  let(:time) { Time.zone.parse('2018-11-10') }
+
+  let(:entity) do
+    travel_to(Time.zone.parse('2018-11-15')) do
+      described_class.represent(group.all_vulnerabilities.count_by_day_and_severity(3.months))
+    end
+  end
+
+  before do
+    travel_to(time) do
+      pipeline_1 = create(:ci_pipeline, :success, project: project)
+
+      create_list(:vulnerabilities_occurrence, 2,
+        pipelines: [pipeline_1], project: project, report_type: :sast, severity: :high)
+
+      create_list(:vulnerabilities_occurrence, 1,
+        pipelines: [pipeline_1], project: project, report_type: :dependency_scanning, severity: :low)
+    end
+  end
+
+  describe '#as_json' do
+    subject { entity.as_json }
+
+    it 'contains required fields' do
+      expect(subject[:total]).to eq({ time.to_date => 3 })
+      expect(subject[:high]).to eq({ time.to_date => 2 })
+      expect(subject[:low]).to eq({ time.to_date => 1 })
+    end
+  end
+end