Merge branch '34824-exclude-policies' into 'master'

Add a detected licenses search filter

See merge request gitlab-org/gitlab!21603

ee/app/controllers/projects/licenses_controller.rb

@@ -15,7 +15,7 @@ module Projects
 
           license_compliance = project.license_compliance
           render json: serializer.represent(
-            pageable(license_compliance.policies),
+            pageable(matching_policies_from(license_compliance)),
             build: license_compliance.latest_build_for_default_branch
           )
         end
@@ -64,5 +64,13 @@ module Projects
     def render_error_for(result)
       render json: { errors: result[:message].as_json }, status: result.fetch(:http_status, :unprocessable_entity)
     end
+
+    def matching_policies_from(license_compliance)
+      if params[:detected]
+        license_compliance.detected_policies
+      else
+        license_compliance.policies
+      end
+    end
   end
 end

ee/app/models/sca/license_compliance.rb

@@ -10,10 +10,14 @@ module SCA
 
     def policies
       strong_memoize(:policies) do
-        new_policies.merge(known_policies).sort.map(&:last)
+        unclassified_policies.merge(known_policies).sort.map(&:last)
       end
     end
 
+    def detected_policies
+      policies.reject { |policy| policy.dependencies.count.zero? }
+    end
+
     def latest_build_for_default_branch
       return if pipeline.blank?
 
@@ -38,7 +42,7 @@ module SCA
       end
     end
 
-    def new_policies
+    def unclassified_policies
       license_scan_report.licenses.map do |reported_license|
         next if known_policies[reported_license.canonical_id]
 

ee/app/serializers/license_entity.rb

@@ -8,7 +8,9 @@ class LicenseEntity < Grape::Entity
 
   expose :id
   expose :name
-  expose :url
+  expose :url do |license|
+    license.url.presence
+  end
   expose :spdx_identifier
   expose :classification
   expose :dependencies, using: ComponentEntity, as: :components

ee/app/views/projects/licenses/index.html.haml

 - breadcrumb_title _('License Compliance')
 - page_title _('License Compliance')
 
-#js-licenses-app{ data: { endpoint: project_licenses_path(@project, format: :json),
+#js-licenses-app{ data: { endpoint: project_licenses_path(@project, detected: true, format: :json),
   documentation_path: help_page_path('user/application_security/license_compliance/index'),
   empty_state_svg_path: image_path('illustrations/Dependency-list-empty-state.svg') } }

ee/spec/controllers/projects/licenses_controller_spec.rb

@@ -76,46 +76,101 @@ describe Projects::LicensesController do
         end
 
         context "when software policies are applied to some of the most recently detected licenses" do
-          let_it_be(:raw_report) { fixture_file_upload(Rails.root.join('ee/spec/fixtures/security_reports/gl-license-management-report-v2.json'), 'application/json') }
           let_it_be(:mit) { create(:software_license, :mit) }
           let_it_be(:mit_policy) { create(:software_license_policy, :denied, software_license: mit, project: project) }
-          let_it_be(:pipeline) do
-            create(:ee_ci_pipeline, :with_license_management_report, project: project).tap do |pipeline|
-              pipeline.job_artifacts.license_management.last.update!(file: raw_report)
+          let_it_be(:other_license) { create(:software_license, spdx_identifier: "Other-Id") }
+          let_it_be(:other_license_policy) { create(:software_license_policy, :allowed, software_license: other_license, project: project) }
+          let_it_be(:pipeline) { create(:ee_ci_pipeline, project: project, builds: [create(:ee_ci_build, :license_scan_v2, :success)]) }
+
+          context "when loading all policies" do
+            before do
+              get :index, params: { namespace_id: project.namespace, project_id: project }, format: :json
             end
-          end
 
-          before do
-            get :index, params: { namespace_id: project.namespace, project_id: project }, format: :json
+            it { expect(response).to have_http_status(:ok) }
+            it { expect(json_response["licenses"].count).to be(4) }
+
+            it 'includes a policy for an unclassified and known license that was detected in the scan report' do
+              expect(json_response.dig("licenses", 0)).to include({
+                "id" => nil,
+                "spdx_identifier" => "BSD-3-Clause",
+                "name" => "BSD 3-Clause \"New\" or \"Revised\" License",
+                "url" => "http://spdx.org/licenses/BSD-3-Clause.json",
+                "classification" => "unclassified"
+              })
+            end
+
+            it 'includes a policy for a denied license found in the scan report' do
+              expect(json_response.dig("licenses", 1)).to include({
+                "id" => mit_policy.id,
+                "spdx_identifier" => "MIT",
+                "name" => mit.name,
+                "url" => "http://spdx.org/licenses/MIT.json",
+                "classification" => "denied"
+              })
+            end
+
+            it 'includes a policy for an allowed license NOT found in the latest scan report' do
+              expect(json_response.dig("licenses", 2)).to include({
+                "id" => other_license_policy.id,
+                "spdx_identifier" => other_license.spdx_identifier,
+                "name" => other_license.name,
+                "url" => nil,
+                "classification" => "allowed"
+              })
+            end
+
+            it 'includes an entry for an unclassified and unknown license found in the scan report' do
+              expect(json_response.dig("licenses", 3)).to include({
+                "id" => nil,
+                "spdx_identifier" => nil,
+                "name" => "unknown",
+                "url" => nil,
+                "classification" => "unclassified"
+              })
+            end
           end
 
-          it { expect(response).to have_http_status(:ok) }
+          context "when loading software policies that match licenses detected in the most recent license scan report" do
+            before do
+              get :index, params: {
+                namespace_id: project.namespace,
+                project_id: project,
+                detected: true
+              }, format: :json
+            end
 
-          it 'generates the proper JSON response' do
-            expect(json_response["licenses"].count).to be(3)
-            expect(json_response.dig("licenses", 0)).to include({
-              "id" => nil,
-              "spdx_identifier" => "BSD-3-Clause",
-              "name" => "BSD 3-Clause \"New\" or \"Revised\" License",
-              "url" => "http://spdx.org/licenses/BSD-3-Clause.json",
-              "classification" => "unclassified"
-            })
+            it { expect(response).to have_http_status(:ok) }
 
-            expect(json_response.dig("licenses", 1)).to include({
-              "id" => mit_policy.id,
-              "spdx_identifier" => "MIT",
-              "name" => mit.name,
-              "url" => "http://spdx.org/licenses/MIT.json",
-              "classification" => "denied"
-            })
+            it 'only includes policies for licenses detected in the most recent scan report' do
+              expect(json_response["licenses"].count).to be(3)
+            end
 
-            expect(json_response.dig("licenses", 2)).to include({
-              "id" => nil,
-              "spdx_identifier" => nil,
-              "name" => "unknown",
-              "url" => "",
-              "classification" => "unclassified"
-            })
+            it 'includes an unclassified policy for a known license detected in the scan report' do
+              expect(json_response.dig("licenses", 0)).to include({
+                "id" => nil,
+                "spdx_identifier" => "BSD-3-Clause",
+                "classification" => "unclassified"
+              })
+            end
+
+            it 'includes a classified license for a known license detected in the scan report' do
+              expect(json_response.dig("licenses", 1)).to include({
+                "id" => mit_policy.id,
+                "spdx_identifier" => "MIT",
+                "classification" => "denied"
+              })
+            end
+
+            it 'includes an unclassified and unknown license discovered in the scan report' do
+              expect(json_response.dig("licenses", 2)).to include({
+                "id" => nil,
+                "spdx_identifier" => nil,
+                "name" => "unknown",
+                "url" => nil,
+                "classification" => "unclassified"
+              })
+            end
           end
         end
 

ee/spec/factories/ci/builds.rb

@@ -104,7 +104,7 @@ FactoryBot.define do
 
     trait :corrupted_license_management_report do
       after(:build) do |build|
-        build.job_artifacts << create(:ee_ci_job_artifact, :corrupted_license_management_report, job: build)
+        build.job_artifacts << create(:ee_ci_job_artifact, :license_scan, :with_corrupted_data, job: build)
       end
     end
 
@@ -113,5 +113,13 @@ FactoryBot.define do
         build.job_artifacts << create(:ee_ci_job_artifact, :low_severity_dast_report, job: build)
       end
     end
+
+    %w[1 1_1 2].each do |version|
+      trait :"license_scan_v#{version}" do
+        after :build do |build|
+          build.job_artifacts << build(:ee_ci_job_artifact, :license_scan, :"v#{version}", job: build)
+        end
+      end
+    end
   end
 end

ee/spec/factories/ci/job_artifacts.rb

@@ -122,16 +122,6 @@ FactoryBot.define do
       end
     end
 
-    trait :corrupted_license_management_report do
-      file_type { :license_management }
-      file_format { :raw }
-
-      after(:build) do |artifact, _|
-        artifact.file = fixture_file_upload(
-          Rails.root.join('spec/fixtures/trace/sample_trace'), 'application/json')
-      end
-    end
-
     trait :performance do
       file_format { :raw }
       file_type { :performance }
@@ -142,16 +132,6 @@ FactoryBot.define do
       end
     end
 
-    trait :license_management do
-      file_format { :raw }
-      file_type { :license_management }
-
-      after(:build) do |artifact, _|
-        artifact.file = fixture_file_upload(
-          Rails.root.join('spec/fixtures/trace/sample_trace'), 'text/plain')
-      end
-    end
-
     trait :dependency_scanning do
       file_format { :raw }
       file_type { :dependency_scanning }
@@ -271,5 +251,27 @@ FactoryBot.define do
           Rails.root.join('ee/spec/fixtures/security_reports/dependency_list/gl-dependency-scanning-report.json'), 'application/json')
       end
     end
+
+    trait :license_scan do
+      file_type { :license_management }
+      file_format { :raw }
+    end
+
+    %w[1 1_1 2].each do |version|
+      trait :"v#{version}" do
+        after(:build) do |artifact, _|
+          filename = "gl-#{artifact.file_type.dasherize}-report-v#{version.sub(/_/, '.')}.json"
+          path = Rails.root.join("ee/spec/fixtures/security_reports/#{filename}")
+          artifact.file = fixture_file_upload(path, "application/json")
+        end
+      end
+    end
+
+    trait :with_corrupted_data do
+      after :build do |artifact, _|
+        path = Rails.root.join('spec/fixtures/trace/sample_trace')
+        artifact.file = fixture_file_upload(path, 'application/json')
+      end
+    end
   end
 end

ee/spec/factories/ci/reports/license_scanning/license.rb

@@ -9,5 +9,11 @@ FactoryBot.define do
       name { 'MIT License' }
       url { 'https://opensource.org/licenses/MIT' }
     end
+
+    trait :unknown do
+      id { 'unknown' }
+      name { 'Unknown' }
+      url { '' }
+    end
   end
 end

ee/spec/models/ci/build_spec.rb

@@ -198,7 +198,7 @@ describe Ci::Build do
 
       context 'when there is a corrupted license management report' do
         before do
-          create(:ee_ci_job_artifact, :corrupted_license_management_report, job: job, project: job.project)
+          create(:ee_ci_job_artifact, :license_scan, :with_corrupted_data, job: job, project: job.project)
         end
 
         it 'raises an error' do

ee/spec/models/sca/license_compliance_spec.rb

@@ -6,6 +6,8 @@ RSpec.describe SCA::LicenseCompliance do
   subject { described_class.new(project) }
 
   let(:project) { create(:project, :repository, :private) }
+  let(:mit) { create(:software_license, :mit) }
+  let(:other_license) { create(:software_license, spdx_identifier: "Other-Id") }
 
   before do
     stub_licensed_features(license_management: true)
@@ -16,16 +18,15 @@ RSpec.describe SCA::LicenseCompliance do
       it { expect(subject.policies.count).to be_zero }
 
       context "when the project has policies configured" do
-        it "includes an entry for each policy that was not detected in the latest report" do
-          mit = create(:software_license, :mit)
-          mit_policy = create(:software_license_policy, :denied, software_license: mit, project: project)
+        let!(:mit_policy) { create(:software_license_policy, :denied, software_license: mit, project: project) }
 
+        it "includes an a policy for a classified license that was not detected in the scan report" do
           expect(subject.policies.count).to be(1)
           expect(subject.policies[0].id).to eq(mit_policy.id)
           expect(subject.policies[0].name).to eq(mit.name)
           expect(subject.policies[0].url).to be_nil
           expect(subject.policies[0].classification).to eq("denied")
-          expect(subject.policies[0].spdx_identifier).to eq("MIT")
+          expect(subject.policies[0].spdx_identifier).to eq(mit.spdx_identifier)
         end
       end
     end
@@ -48,69 +49,56 @@ RSpec.describe SCA::LicenseCompliance do
       end
 
       context "when the license scan produces a poorly formatted report" do
-        let(:builds) { [create(:ci_build, :running, job_artifacts: [artifact])] }
-        let(:artifact) { create(:ci_job_artifact, file_type: :license_management, file_format: :raw, file: invalid_file) }
-        let(:invalid_file) { fixture_file_upload(Rails.root.join("ee/spec/fixtures/metrics.txt"), "text/plain") }
-
-        before do
-          artifact.update!(file: invalid_file)
-        end
+        let(:builds) { [create(:ee_ci_build, :running, :corrupted_license_management_report)] }
 
         it { expect(subject.policies).to be_empty }
       end
 
       context "when the dependency scan produces a poorly formatted report" do
-        let(:builds) { [license_scan_build, dependency_scan_build] }
-        let(:license_scan_build) { create(:ci_build, :success, job_artifacts: [license_scan_artifact]) }
-        let(:license_scan_artifact) { create(:ci_job_artifact, file_type: :license_management, file_format: :raw) }
-        let(:license_scan_file) { fixture_file_upload(Rails.root.join("ee/spec/fixtures/security_reports/gl-license-management-report-v2.json"), "application/json") }
-
-        let(:dependency_scan_build) { create(:ci_build, :success, job_artifacts: [dependency_scan_artifact]) }
-        let(:dependency_scan_artifact) { create(:ci_job_artifact, file_type: :dependency_scanning, file_format: :raw) }
-        let(:invalid_file) { fixture_file_upload(Rails.root.join("ee/spec/fixtures/metrics.txt"), "text/plain") }
-
-        before do
-          license_scan_artifact.update!(file: license_scan_file)
-          dependency_scan_artifact.update!(file: invalid_file)
+        let(:builds) do
+          [
+            create(:ee_ci_build, :success, :license_scan_v2),
+            create(:ee_ci_build, :success, :corrupted_dependency_scanning_report)
+          ]
         end
 
         it { expect(subject.policies.map(&:spdx_identifier)).to contain_exactly("BSD-3-Clause", "MIT", nil) }
       end
 
       context "when a pipeline has successfully produced a v2.0 license scan report" do
-        let(:builds) { [license_scan_build] }
-
-        let(:license_scan_build) { create(:ci_build, :success, job_artifacts: [license_scan_artifact]) }
-        let(:license_scan_artifact) { create(:ci_job_artifact, file_type: :license_management, file_format: :raw) }
-        let(:license_scan_file) { fixture_file_upload(Rails.root.join("ee/spec/fixtures/security_reports/gl-license-management-report-v2.json"), "application/json") }
-
-        it "adds an entry for each detected license and each policy" do
-          mit = create(:software_license, :mit)
-          mit_policy = create(:software_license_policy, :denied, software_license: mit, project: project)
-          other_license = create(:software_license, spdx_identifier: "Other-Id")
-          other_license_policy = create(:software_license_policy, :allowed, software_license: other_license, project: project)
-
-          license_scan_artifact.update!(file: license_scan_file)
+        let(:builds) { [create(:ee_ci_build, :success, :license_scan_v2)] }
+        let!(:mit_policy) { create(:software_license_policy, :denied, software_license: mit, project: project) }
+        let!(:other_license_policy) { create(:software_license_policy, :allowed, software_license: other_license, project: project) }
 
+        it "includes a policy for each detected license and classified license" do
           expect(subject.policies.count).to eq(4)
+        end
+
+        it 'includes a policy for a detected license that is unclassified' do
           expect(subject.policies[0].id).to be_nil
           expect(subject.policies[0].name).to eq("BSD 3-Clause \"New\" or \"Revised\" License")
           expect(subject.policies[0].url).to eq("http://spdx.org/licenses/BSD-3-Clause.json")
           expect(subject.policies[0].classification).to eq("unclassified")
           expect(subject.policies[0].spdx_identifier).to eq("BSD-3-Clause")
+        end
 
+        it 'includes a policy for a classified license that was also detected in the scan report' do
           expect(subject.policies[1].id).to eq(mit_policy.id)
           expect(subject.policies[1].name).to eq(mit.name)
           expect(subject.policies[1].url).to eq("http://spdx.org/licenses/MIT.json")
           expect(subject.policies[1].classification).to eq("denied")
           expect(subject.policies[1].spdx_identifier).to eq("MIT")
+        end
 
+        it 'includes a policy for a classified license that was not detected in the scan report' do
           expect(subject.policies[2].id).to eq(other_license_policy.id)
           expect(subject.policies[2].name).to eq(other_license.name)
           expect(subject.policies[2].url).to be_blank
           expect(subject.policies[2].classification).to eq("allowed")
           expect(subject.policies[2].spdx_identifier).to eq(other_license.spdx_identifier)
+        end
 
+        it 'includes a policy for an unclassified and unknown license that was detected in the scan report' do
           expect(subject.policies[3].id).to be_nil
           expect(subject.policies[3].name).to eq("unknown")
           expect(subject.policies[3].url).to be_blank
@@ -120,40 +108,35 @@ RSpec.describe SCA::LicenseCompliance do
       end
 
       context "when a pipeline has successfully produced a v1.1 license scan report" do
-        let(:builds) { [license_scan_build] }
-
-        let(:license_scan_build) { create(:ci_build, :success, job_artifacts: [license_scan_artifact]) }
-        let(:license_scan_artifact) { create(:ci_job_artifact, file_type: :license_management, file_format: :raw) }
-        let(:license_scan_file) { fixture_file_upload(Rails.root.join("ee/spec/fixtures/security_reports/gl-license-management-report-v1.1.json"), "application/json") }
-
-        it "adds an entry for each detected license and each policy" do
-          mit = create(:software_license, :mit)
-          mit_policy = create(:software_license_policy, :denied, software_license: mit, project: project)
-          other_license = create(:software_license, spdx_identifier: "Other-Id")
-          other_license_policy = create(:software_license_policy, :allowed, software_license: other_license, project: project)
-
-          license_scan_artifact.update!(file: license_scan_file)
-
-          expect(subject.policies.count).to eq(4)
+        let(:builds) { [create(:ee_ci_build, :license_scan_v1_1, :success)] }
+        let!(:mit_policy) { create(:software_license_policy, :denied, software_license: mit, project: project) }
+        let!(:other_license_policy) { create(:software_license_policy, :allowed, software_license: other_license, project: project) }
 
+        it 'includes a policy for an unclassified license detected in the scan report' do
           expect(subject.policies[0].id).to be_nil
           expect(subject.policies[0].name).to eq("BSD")
           expect(subject.policies[0].url).to eq("http://spdx.org/licenses/BSD-4-Clause.json")
           expect(subject.policies[0].classification).to eq("unclassified")
           expect(subject.policies[0].spdx_identifier).to eq("BSD-4-Clause")
+        end
 
+        it 'includes a policy for a denied license found in the scan report' do
           expect(subject.policies[1].id).to eq(mit_policy.id)
           expect(subject.policies[1].name).to eq(mit.name)
           expect(subject.policies[1].url).to eq("http://opensource.org/licenses/mit-license")
           expect(subject.policies[1].classification).to eq("denied")
           expect(subject.policies[1].spdx_identifier).to eq("MIT")
+        end
 
+        it 'includes a policy for an allowed license NOT found in the scan report' do
           expect(subject.policies[2].id).to eq(other_license_policy.id)
           expect(subject.policies[2].name).to eq(other_license.name)
           expect(subject.policies[2].url).to be_blank
           expect(subject.policies[2].classification).to eq("allowed")
           expect(subject.policies[2].spdx_identifier).to eq(other_license.spdx_identifier)
+        end
 
+        it 'includes a policy for an unclassified and unknown license found in the scan report' do
           expect(subject.policies[3].id).to be_nil
           expect(subject.policies[3].name).to eq("unknown")
           expect(subject.policies[3].url).to be_blank
@@ -164,10 +147,44 @@ RSpec.describe SCA::LicenseCompliance do
     end
   end
 
+  describe "#detected_policies" do
+    let!(:pipeline) { create(:ci_pipeline, :success, project: project, builds: [create(:ee_ci_build, :success, :license_scan_v2)]) }
+    let!(:mit_policy) { create(:software_license_policy, :denied, software_license: mit, project: project) }
+    let!(:other_license_policy) { create(:software_license_policy, :allowed, software_license: other_license, project: project) }
+    let(:results) { subject.detected_policies }
+
+    it 'excludes policies for licenses that do not appear in the latest license scan report' do
+      expect(results.count).to eq(3)
+    end
+
+    it 'includes a policy for an unclassified and known license that was detected in the scan report' do
+      expect(results[0].id).to be_nil
+      expect(results[0].name).to eq("BSD 3-Clause \"New\" or \"Revised\" License")
+      expect(results[0].url).to eq("http://spdx.org/licenses/BSD-3-Clause.json")
+      expect(results[0].classification).to eq("unclassified")
+      expect(results[0].spdx_identifier).to eq("BSD-3-Clause")
+    end
+
+    it 'includes an entry for a denied license found in the scan report' do
+      expect(results[1].id).to eq(mit_policy.id)
+      expect(results[1].name).to eq(mit.name)
+      expect(results[1].url).to eq("http://spdx.org/licenses/MIT.json")
+      expect(results[1].classification).to eq("denied")
+      expect(results[1].spdx_identifier).to eq("MIT")
+    end
+
+    it 'includes an entry for an allowed license found in the scan report' do
+      expect(results[2].id).to be_nil
+      expect(results[2].name).to eq("unknown")
+      expect(results[2].url).to be_blank
+      expect(results[2].classification).to eq("unclassified")
+      expect(results[2].spdx_identifier).to be_nil
+    end
+  end
+
   describe "#latest_build_for_default_branch" do
     let(:regular_build) { create(:ci_build, :success) }
-    let(:license_scan_build) { create(:ci_build, :success, job_artifacts: [license_scan_artifact]) }
-    let(:license_scan_artifact) { create(:ci_job_artifact, file_type: :license_management, file_format: :raw) }
+    let(:license_scan_build) { create(:ee_ci_build, :license_scan_v2, :success) }
 
     context "when a pipeline has never been completed for the project" do
       it { expect(subject.latest_build_for_default_branch).to be_nil }

ee/spec/serializers/license_entity_spec.rb

@@ -26,5 +26,15 @@ describe LicenseEntity do
         components: [{ name: 'rails', blob_path: path }]
       })
     end
+
+    context "when the url is blank" do
+      where(url: ['', nil])
+
+      with_them do
+        let(:license) { build(:license_scanning_license, :unknown) }
+
+        it { expect(subject[:url]).to be_nil }
+      end
+    end
   end
 end