Commit 10b53fb7 authored by Robert Speicher's avatar Robert Speicher

Merge branch '34824-license-classifications' into 'master'

Add id, classification to the software license policy json entity

See merge request gitlab-org/gitlab!19261
parents 0a805493 d5835db7
......@@ -10,30 +10,24 @@ module Projects
format.json do
::Gitlab::UsageDataCounters::LicensesList.count(:views)
render json: serializer.represent(licenses, build: report_service.build)
license_compliance = ::SCA::LicenseCompliance.new(project)
render json: serializer.represent(
pageable(license_compliance.policies),
build: license_compliance.latest_build_for_default_branch
)
end
end
end
private
def licenses
found_licenses = report_service.able_to_fetch? ? service.execute : []
::Gitlab::ItemsCollection.new(found_licenses)
end
def report_service
@report_service ||= ::Security::ReportFetchService.new(project, ::Ci::JobArtifact.license_management_reports)
end
def serializer
::LicensesListSerializer.new(project: project, user: current_user)
.with_pagination(request, response)
end
def service
::Security::LicensesListService.new(pipeline: report_service.pipeline)
def pageable(items)
::Gitlab::ItemsCollection.new(items)
end
end
end
......
......@@ -21,6 +21,8 @@ module EE
after_save :stick_build_if_status_changed
delegate :service_specification, to: :runner_session, allow_nil: true
scope :license_scan, -> { joins(:job_artifacts).merge(::Ci::JobArtifact.license_management) }
end
def shared_runners_minutes_limit_enabled?
......
# frozen_string_literal: true
# This is the Secure Composition Analysis namespace
# https://about.gitlab.com/handbook/product/categories/#composition-analysis-group
module SCA
end
# frozen_string_literal: true
module SCA
class LicenseCompliance
include ::Gitlab::Utils::StrongMemoize
def initialize(project)
@project = project
end
def policies
strong_memoize(:policies) do
configured_policies = project.software_license_policies.index_by { |policy| policy.software_license.canonical_id }
detected_licenses = license_scan_report.licenses.map do |reported_license|
policy = configured_policies[reported_license.canonical_id]
configured_policies.delete(reported_license.canonical_id) if policy
build_policy(reported_license, policy)
end
undetected_licenses = configured_policies.map do |id, policy|
build_policy(license_scan_report.fetch(id, nil), policy)
end
(detected_licenses + undetected_licenses).sort_by(&:name)
end
end
def latest_build_for_default_branch
return if pipeline.blank?
strong_memoize(:latest_build_for_default_branch) do
pipeline.builds.latest.license_scan.last
end
end
private
attr_reader :project
def pipeline
strong_memoize(:pipeline) do
project.all_pipelines.latest_successful_for_ref(project.default_branch)
end
end
def license_scan_report
return empty_report if pipeline.blank?
strong_memoize(:license_scan_report) do
pipeline.license_scanning_report.tap do |report|
report.apply_details_from!(dependency_list_report)
end
rescue ::Gitlab::Ci::Parsers::LicenseCompliance::LicenseScanning::LicenseScanningParserError
empty_report
end
end
def dependency_list_report
pipeline.dependency_list_report
rescue ::Gitlab::Ci::Parsers::LicenseCompliance::LicenseScanning::LicenseScanningParserError
end
def empty_report
::Gitlab::Ci::Reports::LicenseScanning::Report.new
end
def build_policy(reported_license, software_license_policy)
::SCA::LicensePolicy.new(reported_license, software_license_policy)
end
end
end
# frozen_string_literal: true
module SCA
class LicensePolicy
attr_reader :id, :name, :url, :dependencies, :spdx_identifier, :classification
def initialize(reported_license, software_policy)
@id = software_policy&.id
@name = software_policy&.name || reported_license.name
@url = reported_license&.url
@dependencies = reported_license&.dependencies || []
@spdx_identifier = software_policy&.spdx_identifier || reported_license.id
@classification = classify(software_policy)
end
private
def classify(policy)
if policy&.approved?
'allowed'
elsif policy&.blacklisted?
'denied'
else
'unclassified'
end
end
end
end
......@@ -19,4 +19,8 @@ class SoftwareLicense < ApplicationRecord
software_license: safe_find_or_create_by!(name: name)
)
end
def canonical_id
spdx_identifier || name.downcase
end
end
......@@ -42,7 +42,7 @@ class SoftwareLicensePolicy < ApplicationRecord
with_license.where(software_licenses: { spdx_identifier: spdx_identifier })
end
delegate :name, to: :software_license
delegate :name, :spdx_identifier, to: :software_license
def self.workaround_cache_key
pluck(:id, :approval_status).flatten
......
......@@ -6,7 +6,10 @@ class LicenseEntity < Grape::Entity
expose :path, as: :blob_path
end
expose :id
expose :name
expose :url
expose :spdx_identifier
expose :classification
expose :dependencies, using: ComponentEntity, as: :components
end
# frozen_string_literal: true
module Security
class LicensesListService
# @param pipeline [Ci::Pipeline]
def initialize(pipeline:)
@pipeline = pipeline
end
def execute
report.merge_dependencies_info!(dependencies) if dependencies.any?
report.licenses
end
private
attr_reader :pipeline
def dependencies
@dependencies ||= pipeline.dependency_list_report.dependencies_with_licenses
end
def report
@report ||= pipeline.license_scanning_report
end
end
end
......@@ -8,8 +8,9 @@ module Gitlab
attr_accessor :path
attr_reader :name
def initialize(name)
def initialize(name, path: nil)
@name = name
@path = path
end
def hash
......
......@@ -37,7 +37,14 @@ module Gitlab
licenses.find { |license| license.name == name }
end
def apply_details_from!(dependency_list_report)
return if dependency_list_report.blank?
merge_dependencies_info!(dependency_list_report.dependencies_with_licenses)
end
def merge_dependencies_info!(dependencies_with_licenses)
return if dependencies_with_licenses.blank?
return if found_licenses.empty?
found_licenses.values.each do |license|
......
......@@ -43,6 +43,17 @@ describe Projects::Security::LicensesController do
it 'returns a hash with licenses' do
expect(json_response).to be_a(Hash)
expect(json_response['licenses'].length).to eq(4)
expect(json_response['licenses'][0]).to include({
'id' => nil,
'spdx_identifier' => nil,
'classification' => 'unclassified',
'name' => 'Apache 2.0',
'url' => 'http://www.apache.org/licenses/LICENSE-2.0.txt',
'components' => [{
"blob_path" => nil,
"name" => "thread_safe"
}]
})
end
it 'returns status ok' do
......@@ -58,6 +69,47 @@ describe Projects::Security::LicensesController do
end
end
context "when software policies are applied to some of the most recently detected licenses" do
let!(:raw_report) { fixture_file_upload(Rails.root.join('ee/spec/fixtures/security_reports/gl-license-management-report-v2.json'), 'application/json') }
let!(:pipeline) { create(:ee_ci_pipeline, :with_license_management_report, project: project) }
let!(:mit) { create(:software_license, :mit) }
let!(:mit_policy) { create(:software_license_policy, :denied, software_license: mit, project: project) }
before do
pipeline.job_artifacts.license_management.last.update!(file: raw_report)
get :index, params: { namespace_id: project.namespace, project_id: project }, format: :json
end
it { expect(response).to have_http_status(:ok) }
it 'generates the proper JSON response' do
expect(json_response["licenses"].count).to be(3)
expect(json_response.dig("licenses", 0)).to include({
"id" => nil,
"spdx_identifier" => "BSD-3-Clause",
"name" => "BSD 3-Clause \"New\" or \"Revised\" License",
"url" => "http://spdx.org/licenses/BSD-3-Clause.json",
"classification" => "unclassified"
})
expect(json_response.dig("licenses", 1)).to include({
"id" => mit_policy.id,
"spdx_identifier" => "MIT",
"name" => mit.name,
"url" => "http://spdx.org/licenses/MIT.json",
"classification" => "denied"
})
expect(json_response.dig("licenses", 2)).to include({
"id" => nil,
"spdx_identifier" => nil,
"name" => "unknown",
"url" => "",
"classification" => "unclassified"
})
end
end
context 'without existing report' do
let!(:pipeline) { create(:ee_ci_pipeline, :with_dependency_list_report, project: project) }
......
# frozen_string_literal: true
FactoryBot.define do
factory :license_scanning_dependency, class: ::Gitlab::Ci::Reports::LicenseScanning::Dependency do
initialize_with { new(name, path: path) }
trait :rails do
name { 'rails' }
path { '.' }
end
end
end
# frozen_string_literal: true
FactoryBot.define do
factory :license_scanning_license, class: ::Gitlab::Ci::Reports::LicenseScanning::License do
initialize_with { new(id: id, name: name, url: url) }
trait :mit do
id { 'MIT' }
name { 'MIT License' }
url { 'https://opensource.org/licenses/MIT' }
end
end
end
......@@ -9,5 +9,13 @@ FactoryBot.define do
trait :blacklist do
approval_status { :blacklisted }
end
trait :allowed do
approval_status { :approved }
end
trait :denied do
approval_status { :blacklisted }
end
end
end
......@@ -10,15 +10,27 @@
"items": {
"type": "object",
"required": [
"id",
"name",
"url",
"spdx_identifier",
"classification",
"components"
],
"properties": {
"id": {
"type": "integer, null"
},
"name": {
"type": "string"
},
"url": {
"type": "string, null"
},
"spdx_identifier": {
"type": "string, null"
},
"classification": {
"type": "string"
},
"components": {
......
......@@ -350,4 +350,13 @@ describe Ci::Build do
it { is_expected.to be false }
end
end
describe ".license_scan" do
it 'returns only license artifacts' do
create(:ci_build, job_artifacts: [create(:ci_job_artifact, :zip)])
build_with_license_scan = create(:ci_build, job_artifacts: [create(:ci_job_artifact, file_type: :license_management, file_format: :raw)])
expect(described_class.license_scan).to contain_exactly(build_with_license_scan)
end
end
end
# frozen_string_literal: true
require "spec_helper"
RSpec.describe SCA::LicenseCompliance do
subject { described_class.new(project) }
let(:project) { create(:project, :repository, :private) }
before do
stub_licensed_features(licenses_list: true, license_management: true)
end
describe "#policies" do
context "when a pipeline has not been run for this project" do
it { expect(subject.policies.count).to be_zero }
context "when the project has policies configured" do
it 'includes an entry for each policy that was not detected in the latest report' do
mit = create(:software_license, :mit)
mit_policy = create(:software_license_policy, :denied, software_license: mit, project: project)
expect(subject.policies.count).to be(1)
expect(subject.policies[0]&.id).to eq(mit_policy.id)
expect(subject.policies[0]&.name).to eq(mit.name)
expect(subject.policies[0]&.url).to be_nil
expect(subject.policies[0]&.classification).to eq("denied")
expect(subject.policies[0]&.spdx_identifier).to eq("MIT")
end
end
end
context "when a pipeline has run" do
let!(:pipeline) { create(:ci_pipeline, :success, project: project, builds: builds) }
let(:builds) { [] }
context "when a license scan job is not configured" do
let(:builds) { [create(:ci_build, :success)] }
it { expect(subject.policies).to be_empty }
end
context "when the license scan job has not finished" do
let(:builds) { [create(:ci_build, :running, job_artifacts: [artifact])] }
let(:artifact) { create(:ci_job_artifact, file_type: :license_management, file_format: :raw) }
it { expect(subject.policies).to be_empty }
end
context "when the license scan produces a poorly formatted report" do
let(:builds) { [create(:ci_build, :running, job_artifacts: [artifact])] }
let(:artifact) { create(:ci_job_artifact, file_type: :license_management, file_format: :raw, file: invalid_file) }
let(:invalid_file) { fixture_file_upload(Rails.root.join("ee/spec/fixtures/metrics.txt"), "text/plain") }
before do
artifact.update!(file: invalid_file)
end
it { expect(subject.policies).to be_empty }
end
context "when the dependency scan produces a poorly formatted report" do
let(:builds) { [license_scan_build, dependency_scan_build] }
let(:license_scan_build) { create(:ci_build, :success, job_artifacts: [license_scan_artifact]) }
let(:license_scan_artifact) { create(:ci_job_artifact, file_type: :license_management, file_format: :raw) }
let(:license_scan_file) { fixture_file_upload(Rails.root.join("ee/spec/fixtures/security_reports/gl-license-management-report-v2.json"), "application/json") }
let(:dependency_scan_build) { create(:ci_build, :success, job_artifacts: [dependency_scan_artifact]) }
let(:dependency_scan_artifact) { create(:ci_job_artifact, file_type: :dependency_scanning, file_format: :raw) }
let(:invalid_file) { fixture_file_upload(Rails.root.join("ee/spec/fixtures/metrics.txt"), "text/plain") }
before do
license_scan_artifact.update!(file: license_scan_file)
dependency_scan_artifact.update!(file: invalid_file)
end
it { expect(subject.policies.map(&:spdx_identifier)).to contain_exactly('BSD-3-Clause', 'MIT', nil) }
end
context "when a pipeline has successfully produced a license scan report" do
let(:builds) { [license_scan_build] }
let(:license_scan_build) { create(:ci_build, :success, job_artifacts: [license_scan_artifact]) }
let(:license_scan_artifact) { create(:ci_job_artifact, file_type: :license_management, file_format: :raw) }
let(:license_scan_file) { fixture_file_upload(Rails.root.join("ee/spec/fixtures/security_reports/gl-license-management-report-v2.json"), "application/json") }
it 'adds an entry for each detected license and each policy' do
mit = create(:software_license, :mit)
mit_policy = create(:software_license_policy, :denied, software_license: mit, project: project)
other_license = create(:software_license, spdx_identifier: "Other-Id")
other_license_policy = create(:software_license_policy, :allowed, software_license: other_license, project: project)
license_scan_artifact.update!(file: license_scan_file)
expect(subject.policies.count).to eq(4)
expect(subject.policies[0]&.id).to be_nil
expect(subject.policies[0]&.name).to eq("BSD 3-Clause \"New\" or \"Revised\" License")
expect(subject.policies[0]&.url).to eq("http://spdx.org/licenses/BSD-3-Clause.json")
expect(subject.policies[0]&.classification).to eq("unclassified")
expect(subject.policies[0]&.spdx_identifier).to eq("BSD-3-Clause")
expect(subject.policies[1]&.id).to eq(mit_policy.id)
expect(subject.policies[1]&.name).to eq(mit.name)
expect(subject.policies[1]&.url).to eq("http://spdx.org/licenses/MIT.json")
expect(subject.policies[1]&.classification).to eq("denied")
expect(subject.policies[1]&.spdx_identifier).to eq("MIT")
expect(subject.policies[2]&.id).to eq(other_license_policy.id)
expect(subject.policies[2]&.name).to eq(other_license.name)
expect(subject.policies[2]&.url).to be_blank
expect(subject.policies[2]&.classification).to eq("allowed")
expect(subject.policies[2]&.spdx_identifier).to eq(other_license.spdx_identifier)
expect(subject.policies[3]&.id).to be_nil
expect(subject.policies[3]&.name).to eq("unknown")
expect(subject.policies[3]&.url).to be_blank
expect(subject.policies[3]&.classification).to eq("unclassified")
expect(subject.policies[3]&.spdx_identifier).to be_nil
end
end
end
end
describe "#latest_build_for_default_branch" do
let(:regular_build) { create(:ci_build, :success) }
let(:license_scan_build) { create(:ci_build, :success, job_artifacts: [license_scan_artifact]) }
let(:license_scan_artifact) { create(:ci_job_artifact, file_type: :license_management, file_format: :raw) }
context "when a pipeline has never been completed for the project" do
it { expect(subject.latest_build_for_default_branch).to be_nil }
end
context "when a pipeline has completed successfully and produced a license scan report" do
let!(:pipeline) { create(:ci_pipeline, :success, project: project, builds: [regular_build, license_scan_build]) }
it { expect(subject.latest_build_for_default_branch).to eq(license_scan_build) }
end
context "when a pipeline has completed but does not contain a license scan report" do
let!(:pipeline) { create(:ci_pipeline, :success, project: project, builds: [regular_build]) }
it { expect(subject.latest_build_for_default_branch).to be_nil }
end
end
end
# frozen_string_literal: true
require "spec_helper"
RSpec.describe SCA::LicensePolicy do
let(:license) { build(:license_scanning_license, :mit) }
let(:policy) { build(:software_license_policy, software_license: software_license) }
let(:software_license) { build(:software_license, :mit) }
describe "#id" do
context "when a software_policy is provided" do
it { expect(described_class.new(license, policy).id).to eq(policy.id) }
end
context "when a software_policy is NOT provided" do
it { expect(described_class.new(license, nil).id).to be_nil }
end
end
describe "#name" do
context "when a software_policy is provided" do
it { expect(described_class.new(license, policy).name).to eq(policy.software_license.name) }
end
context "when a software_policy is NOT provided" do
it { expect(described_class.new(license, nil).name).to eq(license.name) }
end
end
describe "#url" do
context "when a license is provided" do
it { expect(described_class.new(license, policy).url).to eq(license.url) }
end
context "when a license is NOT provided" do
it { expect(described_class.new(nil, policy).id).to be_nil }
end
end
describe "#dependencies" do
context "when a license is provided" do
it { expect(described_class.new(license, policy).dependencies).to eq(license.dependencies) }
end
context "when a license is NOT provided" do
it { expect(described_class.new(nil, policy).dependencies).to be_empty }
end
end
describe "#classification" do
let(:allowed_policy) { build(:software_license_policy, :allowed, software_license: software_license) }
let(:denied_policy) { build(:software_license_policy, :denied, software_license: software_license) }
context "when a allowed software_policy is provided" do
it { expect(described_class.new(license, allowed_policy).classification).to eq("allowed") }
end
context "when a denied software_policy is provided" do
it { expect(described_class.new(license, denied_policy).classification).to eq("denied") }
end
context "when a software_policy is NOT provided" do
it { expect(described_class.new(license, nil).classification).to eq("unclassified") }
end
end
describe "#spdx_identifier" do
context "when a software_policy is provided" do
it { expect(described_class.new(license, policy).spdx_identifier).to eq(policy.software_license.spdx_identifier) }
end
context "when a software_policy is provided but does not have a SPDX Id" do
let(:software_license) { build(:software_license, spdx_identifier: nil) }
it { expect(described_class.new(license, policy).spdx_identifier).to eq(license.id) }
end
context "when a software_policy is NOT provided" do
it { expect(described_class.new(license, nil).spdx_identifier).to eq(license.id) }
end
end
end
......@@ -58,4 +58,14 @@ describe SoftwareLicense do
it { expect(subject.ordered.pluck(:name)).to eql([apache_2.name, mit.name]) }
end
end
describe "#canonical_id" do
context "when an SPDX identifier is available" do
it { expect(build(:software_license, spdx_identifier: 'MIT').canonical_id).to eq('MIT') }
end
context "when an SPDX identifier is not available" do
it { expect(build(:software_license, name: 'MIT License', spdx_identifier: nil).canonical_id).to eq('mit license') }
end
end
end
# frozen_string_literal: true
require 'spec_helper'
require "spec_helper"
describe LicenseEntity do
describe '#as_json' do
subject { described_class.represent(license).as_json }
describe "#as_json" do
subject { described_class.represent(license_policy).as_json }
let(:license) { build(:ci_reports_license_scanning_report, :mit).licenses.first }
let(:assert_license) do
{
name: 'MIT',
url: 'https://opensource.org/licenses/mit',
components: [{
name: 'rails',
blob_path: 'some_path'
}]
}
end
let(:license) { build(:license_scanning_license, :mit) }
let(:license_policy) { ::SCA::LicensePolicy.new(license, software_policy) }
let(:software_policy) { build(:software_license_policy) }
let(:path) { 'some_path' }
before do
license.dependencies.first.path = 'some_path'
license.add_dependency('rails')
allow(license.dependencies.first).to receive(:path).and_return(path)
end
it { is_expected.to eq(assert_license) }
it "produces the correct representation" do
is_expected.to eq({
id: license_policy.id,
name: license_policy.name,
url: license_policy.url,
spdx_identifier: license_policy.spdx_identifier,
classification: license_policy.classification,
components: [{ name: 'rails', blob_path: path }]
})
end
end
end
......@@ -3,11 +3,16 @@
require 'spec_helper'
describe LicensesListEntity do
let(:report) { build(:ci_reports_license_scanning_report, :mit) }
let!(:pipeline) { create(:ee_ci_pipeline, :with_license_management_report, project: project) }
let(:license_compliance) { ::SCA::LicenseCompliance.new(project) }
before do
stub_licensed_features(license_management: true)
end
it_behaves_like 'report list' do
let(:name) { :licenses }
let(:collection) { report.licenses }
let(:collection) { license_compliance.policies }
let(:no_items_status) { :no_licenses }
end
end
......@@ -6,17 +6,19 @@ describe LicensesListSerializer do
describe '#to_json' do
subject do
described_class.new(project: project, user: user)
.represent(report.licenses, build: ci_build)
.represent(license_compliance.policies, build: ci_build)
.to_json
end
let(:project) { create(:project, :repository) }
let!(:pipeline) { create(:ee_ci_pipeline, :with_license_management_report, project: project) }
let(:license_compliance) { ::SCA::LicenseCompliance.new(project) }
let(:user) { create(:user) }
let(:ci_build) { create(:ee_ci_build, :success) }
let(:report) { build(:ci_reports_license_scanning_report, :mit) }
before do
project.add_guest(user)
stub_licensed_features(license_management: true)
end
it { is_expected.to match_schema('licenses_list', dir: 'ee') }
......
# frozen_string_literal: true
require 'spec_helper'
describe Security::LicensesListService do
include LicenseScanningReportHelpers
describe '#execute' do
subject { described_class.new(pipeline: pipeline).execute }
let!(:pipeline) { create(:ee_ci_pipeline, :with_license_management_report) }
let(:project) { pipeline.project }
let(:mit_license) { find_license_by_name(subject, 'MIT') }
before do
stub_licensed_features(license_management: true, dependency_list: true)
end
context 'with matching dependency list' do
let!(:build) { create(:ci_build, :success, name: 'dependency_list', pipeline: pipeline, project: project) }
let!(:artifact) { create(:ee_ci_job_artifact, :dependency_list, job: build, project: project) }
it 'merges dependency location for found dependencies' do
nokogiri = dependency_by_name(mit_license, 'nokogiri')
actioncable = dependency_by_name(mit_license, 'actioncable')
nokogiri_path = "/#{project.full_path}/blob/#{pipeline.sha}/rails/Gemfile.lock"
expect(nokogiri.path).to eq(nokogiri_path)
expect(actioncable.path).to be_nil
end
end
context 'without matching dependency list' do
it 'returns array of Licenses' do
is_expected.to be_an(Array)
end
it 'returns empty path in dependencies' do
nokogiri = dependency_by_name(mit_license, 'nokogiri')
expect(nokogiri.path).to be_nil
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment