Merge branch '212388-offline-spdx-copy' into 'master'

Offline copy of SPDX catalogue

See merge request gitlab-org/gitlab!38691

ee/changelogs/unreleased/212388-offline-spdx-copy.yml

+---
+title: Offline copy of SPDX catalogue
+merge_request: 38691
+author:
+type: added

ee/config/feature_flags/development/offline_spdx_catalogue.yml

+---
+name: offline_spdx_catalogue
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38691
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/212388
+group: group::composition analysis
+type: development
+default_enabled: false

ee/lib/gitlab/spdx/catalogue_gateway.rb

@@ -3,9 +3,12 @@
 module Gitlab
   module SPDX
     class CatalogueGateway
-      URL = 'https://spdx.org/licenses/licenses.json'
+      URL = 'https://spdx.org/licenses/licenses.json'.freeze
+      OFFLINE_CATALOGUE = Rails.root.join('vendor/spdx.json').freeze
 
       def fetch
+        return offline_catalogue if Feature.enabled?(:offline_spdx_catalogue)
+
         response = ::Gitlab::HTTP.get(URL)
 
         if response.success?
@@ -33,6 +36,10 @@ module Gitlab
         build_catalogue(licenses: [])
       end
 
+      def offline_catalogue
+        parse(File.read(OFFLINE_CATALOGUE))
+      end
+
       def build_catalogue(hash)
         ::Gitlab::SPDX::Catalogue.new(hash)
       end

ee/lib/tasks/gitlab/spdx.rake

+# frozen_string_literal: true
+
+require 'net/http'
+require 'gitlab/json'
+
+namespace :gitlab do
+  namespace :spdx do
+    desc 'GitLab | SPDX | Import copy of the catalogue to store it offline'
+    task :import do
+      spdx_url = Gitlab::SPDX::CatalogueGateway::URL
+      resp = Net::HTTP.get_response(URI.parse(spdx_url))
+      data = Gitlab::Json.parse(resp.body)
+
+      path = Gitlab::SPDX::CatalogueGateway::OFFLINE_CATALOGUE
+      IO.write(path, data.to_json, mode: 'w')
+
+      puts "Local copy of SPDX catalogue is saved to #{path}"
+    rescue => e
+      puts "Import of SPDX catalogue failed: #{e}"
+    end
+  end
+end

ee/spec/lib/gitlab/spdx/catalogue_gateway_spec.rb

@@ -8,39 +8,51 @@ RSpec.describe Gitlab::SPDX::CatalogueGateway do
   describe "#fetch" do
     let(:result) { subject.fetch }
     let(:url) { described_class::URL }
+    let(:catalogue_hash) { Gitlab::Json.parse(spdx_json, symbolize_names: true) }
 
-    context "when the licenses.json endpoint is healthy" do
-      let(:spdx_json) { IO.read(Rails.root.join("spec", "fixtures", "spdx.json")) }
-      let(:catalogue_hash) { Gitlab::Json.parse(spdx_json, symbolize_names: true) }
-
-      before do
-        stub_full_request(url, method: :get).to_return(status: 200, body: spdx_json)
-      end
+    context 'when feature flag is enabled' do
+      let(:spdx_json) { described_class::OFFLINE_CATALOGUE.read }
 
       it { expect(result.count).to be(catalogue_hash[:licenses].count) }
     end
 
-    context "when the licenses.json endpoint is not reachable" do
+    context 'when feature flag is disabled' do
       before do
-        allow(Gitlab::Metrics).to receive(:add_event)
-        stub_full_request(url, method: :get).to_return(status: 404)
-        result
+        stub_feature_flags(offline_spdx_catalogue: false)
       end
 
-      it { expect(result.count).to be_zero }
-      it { expect(Gitlab::Metrics).to have_received(:add_event).with(:spdx_fetch_failed, http_status_code: 404) }
-    end
+      context 'when endpoint is healthy' do
+        let(:spdx_json) { Rails.root.join("spec", "fixtures", "spdx.json").read }
 
-    Gitlab::HTTP::HTTP_ERRORS.each do |error|
-      context "when an `#{error}` is raised while trying to connect to the endpoint" do
+        before do
+          stub_full_request(url, method: :get).to_return(status: 200, body: spdx_json)
+        end
+
+        it { expect(result.count).to be(catalogue_hash[:licenses].count) }
+      end
+
+      context 'when the licenses.json endpoint is not reachable' do
         before do
           allow(Gitlab::Metrics).to receive(:add_event)
-          stub_full_request(url, method: :get).and_raise(error)
+          stub_full_request(url, method: :get).to_return(status: 404)
           result
         end
 
         it { expect(result.count).to be_zero }
-        it { expect(Gitlab::Metrics).to have_received(:add_event).with(:spdx_fetch_failed, anything) }
+        it { expect(Gitlab::Metrics).to have_received(:add_event).with(:spdx_fetch_failed, http_status_code: 404) }
+      end
+
+      Gitlab::HTTP::HTTP_ERRORS.each do |error|
+        context "when an `#{error}` is raised while trying to connect to the endpoint" do
+          before do
+            allow(Gitlab::Metrics).to receive(:add_event)
+            stub_full_request(url, method: :get).and_raise(error)
+            result
+          end
+
+          it { expect(result.count).to be_zero }
+          it { expect(Gitlab::Metrics).to have_received(:add_event).with(:spdx_fetch_failed, anything) }
+        end
       end
     end
   end