Add Harbor integration

This adds the Harbor package registry as an optional project
integration.

When enabled, this integration adds the following environment variable
that CI jobs can use to download images from Harbor:

* `HARBOR_URL`
* `HARBOR_PROJECT_NAME`
* `HARBOR_USERNAME`
* `HARBOR_PASSWORD`

Part of https://gitlab.com/groups/gitlab-org/-/epics/7650

Changelog: added

app/models/ci/build.rb

@@ -72,6 +72,7 @@ module Ci
     delegate :terminal_specification, to: :runner_session, allow_nil: true
     delegate :service_specification, to: :runner_session, allow_nil: true
     delegate :gitlab_deploy_token, to: :project
+    delegate :harbor_integration, to: :project
     delegate :trigger_short_token, to: :trigger_request, allow_nil: true
 
     ##
@@ -583,6 +584,7 @@ module Ci
           .append(key: 'CI_REGISTRY_PASSWORD', value: token.to_s, public: false, masked: true)
           .append(key: 'CI_REPOSITORY_URL', value: repo_url.to_s, public: false)
           .concat(deploy_token_variables)
+          .concat(harbor_variables)
       end
     end
 
@@ -619,6 +621,12 @@ module Ci
       end
     end
 
+    def harbor_variables
+      return [] unless harbor_integration.try(:activated?)
+
+      Gitlab::Ci::Variables::Collection.new(harbor_integration.ci_variables)
+    end
+
     def features
       {
         trace_sections: true,

app/models/integration.rb

@@ -20,7 +20,7 @@ class Integration < ApplicationRecord
 
   INTEGRATION_NAMES = %w[
     asana assembla bamboo bugzilla buildkite campfire confluence custom_issue_tracker datadog discord
-    drone_ci emails_on_push ewm external_wiki flowdock hangouts_chat irker jira
+    drone_ci emails_on_push ewm external_wiki flowdock hangouts_chat harbor irker jira
     mattermost mattermost_slash_commands microsoft_teams packagist pipelines_email
     pivotaltracker prometheus pushover redmine slack slack_slash_commands teamcity unify_circuit webex_teams youtrack zentao
   ].freeze

app/models/integrations/harbor.rb

+# frozen_string_literal: true
+
+module Integrations
+  class Harbor < Integration
+    prop_accessor :url, :project_name, :username, :password
+
+    validates :url, public_url: true, presence: true, if: :activated?
+    validates :project_name, presence: true, if: :activated?
+    validates :username, presence: true, if: :activated?
+    validates :password, format: { with: ::Ci::Maskable::REGEX }, if: :activated?
+
+    before_validation :reset_username_and_password
+
+    def title
+      'Harbor'
+    end
+
+    def description
+      s_("HarborIntegration|Use Harbor as this project's container registry.")
+    end
+
+    def help
+      s_("HarborIntegration|After the Harbor integration is activated, global variables ‘$HARBOR_USERNAME’, ‘$HARBOR_PASSWORD’, ‘$HARBOR_URL’ and ‘$HARBOR_PROJECT’ will be created for CI/CD use.")
+    end
+
+    class << self
+      def to_param
+        name.demodulize.downcase
+      end
+
+      def supported_events
+        []
+      end
+
+      def supported_event_actions
+        []
+      end
+    end
+
+    def test(*_args)
+      client.ping
+    end
+
+    def fields
+      [
+        {
+          type: 'text',
+          name: 'url',
+          title: s_('HarborIntegration|Harbor URL'),
+          placeholder: 'https://demo.goharbor.io',
+          help: s_('HarborIntegration|Base URL of the Harbor instance.'),
+          required: true
+        },
+        {
+          type: 'text',
+          name: 'project_name',
+          title: s_('HarborIntegration|Harbor project name'),
+          help: s_('HarborIntegration|The name of the project in Harbor.')
+        },
+        {
+          type: 'text',
+          name: 'username',
+          title: s_('HarborIntegration|Harbor username'),
+          required: true
+        },
+        {
+          type: 'text',
+          name: 'password',
+          title: s_('HarborIntegration|Harbor password'),
+          non_empty_password_title: s_('HarborIntegration|Enter Harbor password'),
+          non_empty_password_help: s_('HarborIntegration|Password for your Harbor username.'),
+          required: true
+        }
+      ]
+    end
+
+    def ci_variables
+      return [] unless activated?
+
+      [
+        { key: 'HARBOR_URL', value: url },
+        { key: 'HARBOR_PROJECT', value: project_name },
+        { key: 'HARBOR_USERNAME', value: username },
+        { key: 'HARBOR_PASSWORD', value: password, public: false, masked: true }
+      ]
+    end
+
+    private
+
+    def client
+      @client ||= ::Gitlab::Harbor::Client.new(self)
+    end
+
+    def reset_username_and_password
+      if url_changed? && !password_touched?
+        self.password = nil
+      end
+
+      if url_changed? && !username_touched?
+        self.username = nil
+      end
+    end
+  end
+end

app/models/project.rb

@@ -196,6 +196,7 @@ class Project < ApplicationRecord
   has_one :external_wiki_integration, class_name: 'Integrations::ExternalWiki'
   has_one :flowdock_integration, class_name: 'Integrations::Flowdock'
   has_one :hangouts_chat_integration, class_name: 'Integrations::HangoutsChat'
+  has_one :harbor_integration, class_name: 'Integrations::Harbor'
   has_one :irker_integration, class_name: 'Integrations::Irker'
   has_one :jenkins_integration, class_name: 'Integrations::Jenkins'
   has_one :jira_integration, class_name: 'Integrations::Jira'

config/metrics/counts_all/20220315180122_projects_harbor_active.yml

+---
+data_category: optional
+key_path: counts.projects_harbor_active
+description: Count of projects with active integrations for Harbor
+product_section: dev
+product_stage: ecosystem
+product_group: group::integrations
+product_category: integrations
+value_type: number
+status: active
+time_frame: all
+data_source: database
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+performance_indicator_type: []
+milestone: "14.9"

config/metrics/counts_all/20220315180124_groups_harbor_active.yml

+---
+data_category: optional
+key_path: counts.groups_harbor_active
+description: Count of groups with active integrations for Harbor
+product_section: dev
+product_stage: ecosystem
+product_group: group::integrations
+product_category: integrations
+value_type: number
+status: active
+time_frame: all
+data_source: database
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+performance_indicator_type: []
+milestone: "14.9"

config/metrics/counts_all/20220315180127_instances_harbor_active.yml

+---
+data_category: optional
+key_path: counts.instances_harbor_active
+description: Count of active instance-level integrations for Harbor
+product_section: dev
+product_stage: ecosystem
+product_group: group::integrations
+product_category: integrations
+value_type: number
+status: active
+time_frame: all
+data_source: database
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+performance_indicator_type: []
+milestone: "14.9"

config/metrics/counts_all/20220315180129_projects_inheriting_harbor_active.yml

+---
+data_category: optional
+key_path: counts.projects_inheriting_harbor_active
+description: Count of active projects inheriting integrations for Harbor
+product_section: dev
+product_stage: ecosystem
+product_group: group::integrations
+product_category: integrations
+value_type: number
+status: active
+time_frame: all
+data_source: database
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+performance_indicator_type: []
+milestone: "14.9"

config/metrics/counts_all/20220315180131_groups_inheriting_harbor_active.yml

+---
+data_category: optional
+key_path: counts.groups_inheriting_harbor_active
+description: Count of active groups inheriting integrations for Harbor
+product_section: dev
+product_stage: ecosystem
+product_group: group::integrations
+product_category: integrations
+value_type: number
+status: active
+time_frame: all
+data_source: database
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+performance_indicator_type: []
+milestone: "14.9"

doc/api/graphql/reference/index.md

@@ -18891,6 +18891,7 @@ State of a Sentry error.
 | <a id="servicetypegithub_service"></a>`GITHUB_SERVICE` | GithubService type. |
 | <a id="servicetypegitlab_slack_application_service"></a>`GITLAB_SLACK_APPLICATION_SERVICE` | GitlabSlackApplicationService type (Gitlab.com only). |
 | <a id="servicetypehangouts_chat_service"></a>`HANGOUTS_CHAT_SERVICE` | HangoutsChatService type. |
+| <a id="servicetypeharbor_service"></a>`HARBOR_SERVICE` | HarborService type. |
 | <a id="servicetypeirker_service"></a>`IRKER_SERVICE` | IrkerService type. |
 | <a id="servicetypejenkins_service"></a>`JENKINS_SERVICE` | JenkinsService type. |
 | <a id="servicetypejira_service"></a>`JIRA_SERVICE` | JiraService type. |

lib/api/helpers/integrations_helpers.rb

@@ -440,6 +440,32 @@ module API
             },
             chat_notification_events
           ].flatten,
+          'harbor' => [
+            {
+              required: true,
+              name: :url,
+              type: String,
+              desc: 'The base URL to the Harbor instance which is being linked to this GitLab project. For example, https://demo.goharbor.io.'
+            },
+            {
+              required: true,
+              name: :project_name,
+              type: String,
+              desc: 'The Project name to the Harbor instance. For example, testproject.'
+            },
+            {
+              required: true,
+              name: :username,
+              type: String,
+              desc: 'The username created from Harbor interface.'
+            },
+            {
+              required: true,
+              name: :password,
+              type: String,
+              desc: 'The password of the user.'
+            }
+          ],
           'irker' => [
             {
               required: true,
@@ -856,6 +882,7 @@ module API
           ::Integrations::ExternalWiki,
           ::Integrations::Flowdock,
           ::Integrations::HangoutsChat,
+          ::Integrations::Harbor,
           ::Integrations::Irker,
           ::Integrations::Jenkins,
           ::Integrations::Jira,

lib/gitlab/harbor/client.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Harbor
+    class Client
+      Error = Class.new(StandardError)
+      ConfigError = Class.new(Error)
+
+      attr_reader :integration
+
+      def initialize(integration)
+        raise ConfigError, 'Please check your integration configuration.' unless integration
+
+        @integration = integration
+      end
+
+      def ping
+        options = { headers: headers.merge!('Accept': 'text/plain') }
+        response = Gitlab::HTTP.get(url('ping'), options)
+
+        { success: response.success? }
+      end
+
+      private
+
+      def url(path)
+        Gitlab::Utils.append_path(base_url, path)
+      end
+
+      def base_url
+        Gitlab::Utils.append_path(integration.url, '/api/v2.0/')
+      end
+
+      def headers
+        auth = Base64.strict_encode64("#{integration.username}:#{integration.password}")
+        {
+          'Content-Type': 'application/json',
+          'Authorization': "Basic #{auth}"
+        }
+      end
+    end
+  end
+end

lib/gitlab/integrations/sti_type.rb

@@ -5,7 +5,7 @@ module Gitlab
     class StiType < ActiveRecord::Type::String
       NAMESPACED_INTEGRATIONS = Set.new(%w(
         Asana Assembla Bamboo Bugzilla Buildkite Campfire Confluence CustomIssueTracker Datadog
-        Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Irker Jenkins Jira Mattermost
+        Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Harbor Irker Jenkins Jira Mattermost
         MattermostSlashCommands MicrosoftTeams MockCi MockMonitoring Packagist PipelinesEmail Pivotaltracker
         Prometheus Pushover Redmine Shimo Slack SlackSlashCommands Teamcity UnifyCircuit WebexTeams Youtrack Zentao
       )).freeze

locale/gitlab.pot

@@ -18071,6 +18071,36 @@ msgstr ""
 msgid "Harbor Registry"
 msgstr ""
 
+msgid "HarborIntegration|After the Harbor integration is activated, global variables ‘$HARBOR_USERNAME’, ‘$HARBOR_PASSWORD’, ‘$HARBOR_URL’ and ‘$HARBOR_PROJECT’ will be created for CI/CD use."
+msgstr ""
+
+msgid "HarborIntegration|Base URL of the Harbor instance."
+msgstr ""
+
+msgid "HarborIntegration|Enter Harbor password"
+msgstr ""
+
+msgid "HarborIntegration|Harbor URL"
+msgstr ""
+
+msgid "HarborIntegration|Harbor password"
+msgstr ""
+
+msgid "HarborIntegration|Harbor project name"
+msgstr ""
+
+msgid "HarborIntegration|Harbor username"
+msgstr ""
+
+msgid "HarborIntegration|Password for your Harbor username."
+msgstr ""
+
+msgid "HarborIntegration|The name of the project in Harbor."
+msgstr ""
+
+msgid "HarborIntegration|Use Harbor as this project's container registry."
+msgstr ""
+
 msgid "Hashed Storage must be enabled to use Geo"
 msgstr ""
 

spec/factories/integrations.rb

@@ -230,6 +230,17 @@ FactoryBot.define do
     token { 'test' }
   end
 
+  factory :harbor_integration, class: 'Integrations::Harbor' do
+    project
+    active { true }
+    type { 'HarborService' }
+
+    url { 'https://demo.goharbor.io' }
+    project_name { 'testproject' }
+    username { 'harborusername' }
+    password { 'harborpassword' }
+  end
+
   # this is for testing storing values inside properties, which is deprecated and will be removed in
   # https://gitlab.com/gitlab-org/gitlab/issues/29404
   trait :without_properties_callback do

spec/lib/gitlab/harbor/client_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Harbor::Client do
+  let(:harbor_integration) { build(:harbor_integration) }
+
+  subject(:client) { described_class.new(harbor_integration) }
+
+  describe '#ping' do
+    let!(:harbor_ping_request) { stub_harbor_request("https://demo.goharbor.io/api/v2.0/ping") }
+
+    it "calls api/v2.0/ping successfully" do
+      expect(client.ping).to eq(success: true)
+    end
+  end
+
+  private
+
+  def stub_harbor_request(url, body: {}, status: 200, headers: {})
+    stub_request(:get, url)
+      .to_return(
+        status: status,
+        headers: { 'Content-Type' => 'application/json' }.merge(headers),
+        body: body.to_json
+      )
+  end
+end

spec/lib/gitlab/import_export/all_models.yml

@@ -395,6 +395,7 @@ project:
 - mattermost_slash_commands_integration
 - shimo_integration
 - slack_slash_commands_integration
+- harbor_integration
 - irker_integration
 - packagist_integration
 - pivotaltracker_integration

spec/models/ci/build_spec.rb

@@ -3510,6 +3510,38 @@ RSpec.describe Ci::Build do
       end
     end
 
+    context 'for harbor integration' do
+      let(:harbor_integration) { create(:harbor_integration) }
+
+      let(:harbor_variables) do
+        [
+          { key: 'HARBOR_URL', value: harbor_integration.url, public: true, masked: false },
+          { key: 'HARBOR_PROJECT', value: harbor_integration.project_name, public: true, masked: false },
+          { key: 'HARBOR_USERNAME', value: harbor_integration.username, public: true, masked: false },
+          { key: 'HARBOR_PASSWORD', value: harbor_integration.password, public: false, masked: true }
+        ]
+      end
+
+      context 'when harbor_integration exists' do
+        before do
+          build.project.update!(harbor_integration: harbor_integration)
+        end
+
+        it 'includes harbor variables' do
+          is_expected.to include(*harbor_variables)
+        end
+      end
+
+      context 'when harbor_integration does not exist' do
+        it 'does not include harbor variables' do
+          expect(subject.find { |v| v[:key] == 'HARBOR_URL'}).to be_nil
+          expect(subject.find { |v| v[:key] == 'HARBOR_PROJECT_NAME'}).to be_nil
+          expect(subject.find { |v| v[:key] == 'HARBOR_USERNAME'}).to be_nil
+          expect(subject.find { |v| v[:key] == 'HARBOR_PASSWORD'}).to be_nil
+        end
+      end
+    end
+
     context 'when build has dependency which has dotenv variable' do
       let!(:prepare) { create(:ci_build, pipeline: pipeline, stage_idx: 0) }
       let!(:build) { create(:ci_build, pipeline: pipeline, stage_idx: 1, options: { dependencies: [prepare.name] }) }

spec/models/integrations/harbor_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Integrations::Harbor do
+  let(:url) { 'https://demo.goharbor.io' }
+  let(:project_name) { 'testproject' }
+  let(:username) { 'harborusername' }
+  let(:password) { 'harborpassword' }
+  let(:harbor_integration) { create(:harbor_integration) }
+
+  describe "masked password" do
+    subject { build(:harbor_integration) }
+
+    it { is_expected.not_to allow_value('hello').for(:password) }
+    it { is_expected.not_to allow_value('hello world').for(:password) }
+    it { is_expected.not_to allow_value('hello$VARIABLEworld').for(:password) }
+    it { is_expected.not_to allow_value('hello\rworld').for(:password) }
+    it { is_expected.to allow_value('helloworld').for(:password) }
+  end
+
+  describe '#fields' do
+    it 'returns custom fields' do
+      expect(harbor_integration.fields.pluck(:name)).to eq(%w[url project_name username password])
+    end
+  end
+
+  describe '#test' do
+    let(:test_response) { "pong" }
+
+    before do
+      allow_next_instance_of(Gitlab::Harbor::Client) do |client|
+        allow(client).to receive(:ping).and_return(test_response)
+      end
+    end
+
+    it 'gets response from Gitlab::Harbor::Client#ping' do
+      expect(harbor_integration.test).to eq(test_response)
+    end
+  end
+
+  describe '#help' do
+    it 'renders prompt information' do
+      expect(harbor_integration.help).not_to be_empty
+    end
+  end
+
+  describe '.to_param' do
+    it 'returns the name of the integration' do
+      expect(described_class.to_param).to eq('harbor')
+    end
+  end
+
+  context 'ci variables' do
+    it 'returns vars when harbor_integration is activated' do
+      ci_vars = [
+        { key: 'HARBOR_URL', value: url },
+        { key: 'HARBOR_PROJECT', value: project_name },
+        { key: 'HARBOR_USERNAME', value: username },
+        { key: 'HARBOR_PASSWORD', value: password, public: false, masked: true }
+      ]
+
+      expect(harbor_integration.ci_variables).to match_array(ci_vars)
+    end
+
+    it 'returns [] when harbor_integration is inactive' do
+      harbor_integration.update!(active: false)
+      expect(harbor_integration.ci_variables).to match_array([])
+    end
+  end
+
+  describe 'before_validation :reset_username_and_password' do
+    context 'when username/password was previously set' do
+      it 'resets username and password if url changed' do
+        harbor_integration.url = 'https://anotherharbor.com'
+        harbor_integration.valid?
+
+        expect(harbor_integration.password).to be_nil
+        expect(harbor_integration.username).to be_nil
+      end
+
+      it 'does not reset password if username changed' do
+        harbor_integration.username = 'newusername'
+        harbor_integration.valid?
+
+        expect(harbor_integration.password).to eq('harborpassword')
+      end
+
+      it 'does not reset username if password changed' do
+        harbor_integration.password = 'newpassword'
+        harbor_integration.valid?
+
+        expect(harbor_integration.username).to eq('harborusername')
+      end
+
+      it "does not reset password if new url is set together with password, even if it's the same password" do
+        harbor_integration.url = 'https://anotherharbor.com'
+        harbor_integration.password = 'harborpassword'
+        harbor_integration.valid?
+
+        expect(harbor_integration.password).to eq('harborpassword')
+        expect(harbor_integration.username).to be_nil
+        expect(harbor_integration.url).to eq('https://anotherharbor.com')
+      end
+
+      it "does not reset username if new url is set together with username, even if it's the same username" do
+        harbor_integration.url = 'https://anotherharbor.com'
+        harbor_integration.username = 'harborusername'
+        harbor_integration.valid?
+
+        expect(harbor_integration.password).to be_nil
+        expect(harbor_integration.username).to eq('harborusername')
+        expect(harbor_integration.url).to eq('https://anotherharbor.com')
+      end
+    end
+
+    it 'saves password if new url is set together with password when no password was previously set' do
+      harbor_integration.password = nil
+      harbor_integration.username = nil
+
+      harbor_integration.url = 'https://anotherharbor.com'
+      harbor_integration.password = 'newpassword'
+      harbor_integration.username = 'newusername'
+      harbor_integration.save!
+
+      expect(harbor_integration).to have_attributes(
+        url: 'https://anotherharbor.com',
+        password: 'newpassword',
+        username: 'newusername'
+      )
+    end
+  end
+end

spec/models/project_spec.rb

@@ -64,6 +64,7 @@ RSpec.describe Project, factory_default: :keep do
     it { is_expected.to have_one(:bamboo_integration) }
     it { is_expected.to have_one(:teamcity_integration) }
     it { is_expected.to have_one(:jira_integration) }
+    it { is_expected.to have_one(:harbor_integration) }
     it { is_expected.to have_one(:redmine_integration) }
     it { is_expected.to have_one(:youtrack_integration) }
     it { is_expected.to have_one(:custom_issue_tracker_integration) }