Merge branch 'pedropombeiro/356689/1-create-releases-service' into 'master'

Implement service to retrieve latest Runner releases

See merge request gitlab-org/gitlab!83549

app/models/application_setting.rb

@@ -576,6 +576,8 @@ class ApplicationSetting < ApplicationRecord
           length: { maximum: 100, message: N_('is too long (maximum is 100 entries)') },
           allow_nil: false
 
+  validates :public_runner_releases_url, addressable_url: true, presence: true
+
   attr_encrypted :asset_proxy_secret_key,
                  mode: :per_attribute_iv,
                  key: Settings.attr_encrypted_db_key_base_truncated,

db/migrate/20220324171254_add_public_git_lab_runner_releases_url_to_application_settings.rb

+# frozen_string_literal: true
+
+class AddPublicGitLabRunnerReleasesUrlToApplicationSettings < Gitlab::Database::Migration[1.0]
+  # rubocop:disable Migration/AddLimitToTextColumns
+  # limit is added in 20220324173554_add_text_limit_to_public_git_lab_runner_releases_url_application_settings
+  def change
+    add_column :application_settings, :public_runner_releases_url, :text, null: false, default: 'https://gitlab.com/api/v4/projects/gitlab-org%2Fgitlab-runner/releases'
+  end
+  # rubocop:enable Migration/AddLimitToTextColumns
+end

db/migrate/20220324173554_add_text_limit_to_public_git_lab_runner_releases_url_application_settings.rb

+# frozen_string_literal: true
+
+class AddTextLimitToPublicGitLabRunnerReleasesUrlApplicationSettings < Gitlab::Database::Migration[1.0]
+  disable_ddl_transaction!
+
+  def up
+    add_text_limit :application_settings, :public_runner_releases_url, 255
+  end
+
+  def down
+    remove_text_limit :application_settings, :public_runner_releases_url
+  end
+end

db/schema_migrations/20220324171254

+610c5ded785f174d195a660062bb74e718bfd5a38b13773215e20e8f95c59da4
\ No newline at end of file

db/schema_migrations/20220324173554

+9f597a462768531b0c6ad23e6e1a52edb765724518e1cebc0684160b030d6225
\ No newline at end of file

db/structure.sql

@@ -11257,6 +11257,7 @@ CREATE TABLE application_settings (
     encrypted_database_grafana_api_key_iv bytea,
     database_grafana_api_url text,
     database_grafana_tag text,
+    public_runner_releases_url text DEFAULT 'https://gitlab.com/api/v4/projects/gitlab-org%2Fgitlab-runner/releases'::text NOT NULL,
     CONSTRAINT app_settings_container_reg_cleanup_tags_max_list_size_positive CHECK ((container_registry_cleanup_tags_service_max_list_size >= 0)),
     CONSTRAINT app_settings_dep_proxy_ttl_policies_worker_capacity_positive CHECK ((dependency_proxy_ttl_group_policy_worker_capacity >= 0)),
     CONSTRAINT app_settings_ext_pipeline_validation_service_url_text_limit CHECK ((char_length(external_pipeline_validation_service_url) <= 255)),
@@ -11279,6 +11280,7 @@ CREATE TABLE application_settings (
     CONSTRAINT check_718b4458ae CHECK ((char_length(personal_access_token_prefix) <= 20)),
     CONSTRAINT check_7227fad848 CHECK ((char_length(rate_limiting_response_text) <= 255)),
     CONSTRAINT check_85a39b68ff CHECK ((char_length(encrypted_ci_jwt_signing_key_iv) <= 255)),
+    CONSTRAINT check_8dca35398a CHECK ((char_length(public_runner_releases_url) <= 255)),
     CONSTRAINT check_9a719834eb CHECK ((char_length(secret_detection_token_revocation_url) <= 255)),
     CONSTRAINT check_9c6c447a13 CHECK ((char_length(maintenance_mode_message) <= 255)),
     CONSTRAINT check_a5704163cc CHECK ((char_length(secret_detection_revocation_token_types_url) <= 255)),

lib/gitlab/ci/runner_releases.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class RunnerReleases
+      include Singleton
+
+      RELEASES_VALIDITY_PERIOD = 1.day
+      RELEASES_VALIDITY_AFTER_ERROR_PERIOD = 5.seconds
+
+      INITIAL_BACKOFF = 5.seconds
+      MAX_BACKOFF = 1.hour
+      BACKOFF_GROWTH_FACTOR = 2.0
+
+      def initialize
+        reset!
+      end
+
+      # Returns a sorted list of the publicly available GitLab Runner releases
+      #
+      def releases
+        return @releases unless Time.now.utc >= @expire_time
+
+        @releases = fetch_new_releases
+      end
+
+      def reset!
+        @expire_time = Time.now.utc
+        @releases = nil
+        @backoff_count = 0
+      end
+
+      public_class_method :instance
+
+      private
+
+      def fetch_new_releases
+        response = Gitlab::HTTP.try_get(::Gitlab::CurrentSettings.current_application_settings.public_runner_releases_url)
+
+        releases = response.success? ? extract_releases(response) : nil
+      ensure
+        @expire_time = (releases ? RELEASES_VALIDITY_PERIOD : next_backoff).from_now
+      end
+
+      def extract_releases(response)
+        response.parsed_response.map { |release| parse_runner_release(release) }.sort!
+      end
+
+      def parse_runner_release(release)
+        ::Gitlab::VersionInfo.parse(release['name'].delete_prefix('v'))
+      end
+
+      def next_backoff
+        return MAX_BACKOFF if @backoff_count >= 11 # optimization to prevent expensive exponentiation and possible overflows
+
+        backoff = (INITIAL_BACKOFF * (BACKOFF_GROWTH_FACTOR**@backoff_count))
+          .clamp(INITIAL_BACKOFF, MAX_BACKOFF)
+          .seconds
+        @backoff_count += 1
+
+        backoff
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/runner_releases_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Ci::RunnerReleases do
+  subject { described_class.instance }
+
+  describe '#releases' do
+    before do
+      subject.reset!
+
+      stub_application_setting(public_runner_releases_url: 'the release API URL')
+      allow(Gitlab::HTTP).to receive(:try_get).with('the release API URL').once { mock_http_response(response) }
+    end
+
+    def releases
+      subject.releases
+    end
+
+    shared_examples 'requests that follow cache status' do |validity_period|
+      context "almost #{validity_period.inspect} later" do
+        let(:followup_request_interval) { validity_period - 0.001.seconds }
+
+        it 'returns cached releases' do
+          releases
+
+          travel followup_request_interval do
+            expect(Gitlab::HTTP).not_to receive(:try_get)
+
+            expect(releases).to eq(expected_result)
+          end
+        end
+      end
+
+      context "after #{validity_period.inspect}" do
+        let(:followup_request_interval) { validity_period + 1.second }
+        let(:followup_response) { (response || []) + [{ 'name' => 'v14.9.2' }] }
+
+        it 'checks new releases' do
+          releases
+
+          travel followup_request_interval do
+            expect(Gitlab::HTTP).to receive(:try_get).with('the release API URL').once { mock_http_response(followup_response) }
+
+            expect(releases).to eq((expected_result || []) + [Gitlab::VersionInfo.new(14, 9, 2)])
+          end
+        end
+      end
+    end
+
+    context 'when response is nil' do
+      let(:response) { nil }
+      let(:expected_result) { nil }
+
+      it 'returns nil' do
+        expect(releases).to be_nil
+      end
+
+      it_behaves_like 'requests that follow cache status', 5.seconds
+
+      it 'performs exponential backoff on requests', :aggregate_failures do
+        start_time = Time.now.utc.change(usec: 0)
+
+        http_call_timestamp_offsets = []
+        allow(Gitlab::HTTP).to receive(:try_get).with('the release API URL') do
+          http_call_timestamp_offsets << Time.now.utc - start_time
+          mock_http_response(response)
+        end
+
+        # An initial HTTP request fails
+        travel_to(start_time)
+        subject.reset!
+        expect(releases).to be_nil
+
+        # Successive failed requests result in HTTP requests only after specific backoff periods
+        backoff_periods = [5, 10, 20, 40, 80, 160, 320, 640, 1280, 2560, 3600].map(&:seconds)
+        backoff_periods.each do |period|
+          travel(period - 1.second)
+          expect(releases).to be_nil
+
+          travel 1.second
+          expect(releases).to be_nil
+        end
+
+        expect(http_call_timestamp_offsets).to eq([0, 5, 15, 35, 75, 155, 315, 635, 1275, 2555, 5115, 8715])
+
+        # Finally a successful HTTP request results in releases being returned
+        allow(Gitlab::HTTP).to receive(:try_get).with('the release API URL').once { mock_http_response([{ 'name' => 'v14.9.1' }]) }
+        travel 1.hour
+        expect(releases).not_to be_nil
+      end
+    end
+
+    context 'when response is not nil' do
+      let(:response) { [{ 'name' => 'v14.9.1' }, { 'name' => 'v14.9.0' }] }
+      let(:expected_result) { [Gitlab::VersionInfo.new(14, 9, 0), Gitlab::VersionInfo.new(14, 9, 1)] }
+
+      it 'returns parsed and sorted Gitlab::VersionInfo objects' do
+        expect(releases).to eq(expected_result)
+      end
+
+      it_behaves_like 'requests that follow cache status', 1.day
+    end
+
+    def mock_http_response(response)
+      http_response = instance_double(HTTParty::Response)
+
+      allow(http_response).to receive(:success?).and_return(response.present?)
+      allow(http_response).to receive(:parsed_response).and_return(response)
+
+      http_response
+    end
+  end
+end