Offline copy of SPDX catalogue

ee/changelogs/unreleased/212388-offline-spdx-copy.yml

+---
+title: Offline copy of SPDX catalogue
+merge_request: 38691
+author:
+type: added

ee/config/feature_flags/development/offline_spdx_catalogue.yml

+---
+name: offline_spdx_catalogue
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38691
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/212388
+group: group::composition analysis
+type: development
+default_enabled: false

ee/lib/gitlab/spdx/catalogue_gateway.rb

@@ -3,9 +3,12 @@
 module Gitlab
   module SPDX
     class CatalogueGateway
-      URL = 'https://spdx.org/licenses/licenses.json'
+      URL = 'https://spdx.org/licenses/licenses.json'.freeze
+      OFFLINE_CATALOGUE = Rails.root.join('vendor/spdx.json').freeze
 
       def fetch
+        return offline_catalogue if Feature.enabled?(:offline_spdx_catalogue)
+
         response = ::Gitlab::HTTP.get(URL)
 
         if response.success?
@@ -33,6 +36,10 @@ module Gitlab
         build_catalogue(licenses: [])
       end
 
+      def offline_catalogue
+        parse(File.read(OFFLINE_CATALOGUE))
+      end
+
       def build_catalogue(hash)
         ::Gitlab::SPDX::Catalogue.new(hash)
       end

ee/lib/tasks/gitlab/spdx.rake

+# frozen_string_literal: true
+
+require 'net/http'
+require 'gitlab/json'
+
+namespace :gitlab do
+  namespace :spdx do
+    desc 'GitLab | SPDX | Import copy of the catalogue to store it offline'
+    task :import do
+      spdx_url = Gitlab::SPDX::CatalogueGateway::URL
+      resp = Net::HTTP.get_response(URI.parse(spdx_url))
+      data = Gitlab::Json.parse(resp.body)
+
+      path = Gitlab::SPDX::CatalogueGateway::OFFLINE_CATALOGUE
+      IO.write(path, data.to_json, mode: 'w')
+
+      puts "Local copy of SPDX catalogue is saved to #{path}"
+    rescue => e
+      puts "Import of SPDX catalogue failed: #{e}"
+    end
+  end
+end

ee/spec/lib/gitlab/spdx/catalogue_gateway_spec.rb

@@ -8,11 +8,22 @@ RSpec.describe Gitlab::SPDX::CatalogueGateway do
   describe "#fetch" do
     let(:result) { subject.fetch }
     let(:url) { described_class::URL }
-
-    context "when the licenses.json endpoint is healthy" do
-      let(:spdx_json) { IO.read(Rails.root.join("spec", "fixtures", "spdx.json")) }
     let(:catalogue_hash) { Gitlab::Json.parse(spdx_json, symbolize_names: true) }
 
+    context 'when feature flag is enabled' do
+      let(:spdx_json) { described_class::OFFLINE_CATALOGUE.read }
+
+      it { expect(result.count).to be(catalogue_hash[:licenses].count) }
+    end
+
+    context 'when feature flag is disabled' do
+      before do
+        stub_feature_flags(offline_spdx_catalogue: false)
+      end
+
+      context 'when endpoint is healthy' do
+        let(:spdx_json) { Rails.root.join("spec", "fixtures", "spdx.json").read }
+
         before do
           stub_full_request(url, method: :get).to_return(status: 200, body: spdx_json)
         end
@@ -20,7 +31,7 @@ RSpec.describe Gitlab::SPDX::CatalogueGateway do
         it { expect(result.count).to be(catalogue_hash[:licenses].count) }
       end
 
-    context "when the licenses.json endpoint is not reachable" do
+      context 'when the licenses.json endpoint is not reachable' do
         before do
           allow(Gitlab::Metrics).to receive(:add_event)
           stub_full_request(url, method: :get).to_return(status: 404)
@@ -44,4 +55,5 @@ RSpec.describe Gitlab::SPDX::CatalogueGateway do
         end
       end
     end
+  end
 end