Process files related to cluster_applications artifact

- Add parser for Helm list (v2) JSON
- Only process known applications

app/models/clusters/cluster.rb

@@ -223,11 +223,19 @@ module Clusters
     end
 
     def applications
-      APPLICATIONS_ASSOCIATIONS.map do |association_name|
-        public_send(association_name) || public_send("build_#{association_name}") # rubocop:disable GitlabSecurity/PublicSend
+      APPLICATIONS.each_value.map do |application_class|
+        find_or_build_application(application_class)
       end
     end
 
+    def find_or_build_application(application_class)
+      raise ArgumentError, "#{application_class} is not in APPLICATIONS" unless APPLICATIONS.value?(application_class)
+
+      association_name = application_class.association_name
+
+      public_send(association_name) || public_send("build_#{association_name}") # rubocop:disable GitlabSecurity/PublicSend
+    end
+
     def provider
       if gcp?
         provider_gcp

app/models/clusters/concerns/application_status.rb

@@ -27,6 +27,7 @@ module Clusters
           state :update_errored, value: 6
           state :uninstalling, value: 7
           state :uninstall_errored, value: 8
+          state :uninstalled, value: 10
 
           # Used for applications that are pre-installed by the cluster,
           # e.g. Knative in GCP Cloud Run enabled clusters
@@ -35,6 +36,14 @@ module Clusters
           # and no exit transitions.
           state :pre_installed, value: 9
 
+          event :make_externally_installed do
+            transition any => :installed
+          end
+
+          event :make_externally_uninstalled do
+            transition any => :uninstalled
+          end
+
           event :make_scheduled do
             transition [:installable, :errored, :installed, :updated, :update_errored, :uninstall_errored] => :scheduled
           end

app/services/ci/create_job_artifacts_service.rb

@@ -61,6 +61,7 @@ module Ci
 
       case artifact.file_type
       when 'dotenv' then parse_dotenv_artifact(job, artifact)
+      when 'cluster_applications' then parse_cluster_applications_artifact(job, artifact)
       else success
       end
     end
@@ -111,5 +112,9 @@ module Ci
     def parse_dotenv_artifact(job, artifact)
       Ci::ParseDotenvArtifactService.new(job.project, current_user).execute(artifact)
     end
+
+    def parse_cluster_applications_artifact(job, artifact)
+      Clusters::ParseClusterApplicationsArtifactService.new(job, job.user).execute(artifact)
+    end
   end
 end

app/services/clusters/parse_cluster_applications_artifact_service.rb

+# frozen_string_literal: true
+
+module Clusters
+  class ParseClusterApplicationsArtifactService < ::BaseService
+    include Gitlab::Utils::StrongMemoize
+
+    MAX_ACCEPTABLE_ARTIFACT_SIZE = 5.kilobytes
+    RELEASE_NAMES = %w[prometheus].freeze
+
+    def initialize(job, current_user)
+      @job = job
+
+      super(job.project, current_user)
+    end
+
+    def execute(artifact)
+      return success unless Feature.enabled?(:cluster_applications_artifact, project)
+
+      raise ArgumentError, 'Artifact is not cluster_applications file type' unless artifact&.cluster_applications?
+
+      unless artifact.file.size < MAX_ACCEPTABLE_ARTIFACT_SIZE
+        return error(too_big_error_message, :bad_request)
+      end
+
+      unless cluster
+        return error(s_('ClusterIntegration|No deployment cluster found for this job'))
+      end
+
+      parse!(artifact)
+
+      success
+    rescue Gitlab::Kubernetes::Helm::Parsers::ListV2::ParserError, ActiveRecord::RecordInvalid => error
+      Gitlab::ErrorTracking.track_exception(error, job_id: artifact.job_id)
+      error(error.message, :bad_request)
+    end
+
+    private
+
+    attr_reader :job
+
+    def cluster
+      strong_memoize(:cluster) do
+        deployment_cluster = job.deployment&.cluster
+
+        deployment_cluster if Ability.allowed?(current_user, :admin_cluster, deployment_cluster)
+      end
+    end
+
+    def parse!(artifact)
+      releases = []
+
+      artifact.each_blob do |blob|
+        releases.concat(Gitlab::Kubernetes::Helm::Parsers::ListV2.new(blob).releases)
+      end
+
+      update_cluster_application_statuses!(releases)
+    end
+
+    def update_cluster_application_statuses!(releases)
+      release_by_name = releases.index_by { |release| release['Name'] }
+
+      Clusters::Cluster.transaction do
+        RELEASE_NAMES.each do |release_name|
+          application = find_or_build_application(release_name)
+
+          release = release_by_name[release_name]
+
+          if release
+            case release['Status']
+            when 'DEPLOYED'
+              application.make_externally_installed!
+            when 'FAILED'
+              application.make_errored!(s_('ClusterIntegration|Helm release failed to install'))
+            end
+          else
+            # missing, so by definition, we consider this uninstalled
+            application.make_externally_uninstalled! if application.persisted?
+          end
+        end
+      end
+    end
+
+    def find_or_build_application(application_name)
+      application_class = Clusters::Cluster::APPLICATIONS[application_name]
+
+      cluster.find_or_build_application(application_class)
+    end
+
+    def too_big_error_message
+      human_size = ActiveSupport::NumberHelper.number_to_human_size(MAX_ACCEPTABLE_ARTIFACT_SIZE)
+
+      s_('ClusterIntegration|Cluster_applications artifact too big. Maximum allowable size: %{human_size}') % { human_size: human_size }
+    end
+  end
+end

lib/gitlab/kubernetes/helm/parsers/list_v2.rb

@@ -18,7 +18,17 @@ module Gitlab
           end
 
           def releases
-            @releases ||= json["Releases"] || []
+            @releases = helm_releases
+          end
+
+          private
+
+          def helm_releases
+            helm_releases = json['Releases'] || []
+
+            raise ParserError, 'Invalid format for Releases' unless helm_releases.all? { |item| item.is_a?(Hash) }
+
+            helm_releases
           end
         end
       end

locale/gitlab.pot

@@ -4571,6 +4571,9 @@ msgstr ""
 msgid "ClusterIntegration|Cluster name is required."
 msgstr ""
 
+msgid "ClusterIntegration|Cluster_applications artifact too big. Maximum allowable size: %{human_size}"
+msgstr ""
+
 msgid "ClusterIntegration|Clusters are utilized by selecting the nearest ancestor with a matching environment scope. For example, project clusters will override group clusters."
 msgstr ""
 
@@ -4751,6 +4754,9 @@ msgstr ""
 msgid "ClusterIntegration|Helm Tiller"
 msgstr ""
 
+msgid "ClusterIntegration|Helm release failed to install"
+msgstr ""
+
 msgid "ClusterIntegration|Helm streamlines installing and managing Kubernetes applications. Tiller runs inside of your Kubernetes Cluster, and manages releases of your charts."
 msgstr ""
 
@@ -4913,6 +4919,9 @@ msgstr ""
 msgid "ClusterIntegration|No VPCs found"
 msgstr ""
 
+msgid "ClusterIntegration|No deployment cluster found for this job"
+msgstr ""
+
 msgid "ClusterIntegration|No instance type found"
 msgstr ""
 

spec/factories/ci/job_artifacts.rb

@@ -252,6 +252,21 @@ FactoryBot.define do
       end
     end
 
+    trait :cluster_applications do
+      file_type { :cluster_applications }
+      file_format { :gzip }
+
+      transient do
+        file do
+          fixture_file_upload(Rails.root.join('spec/fixtures/helm/helm_list_v2_prometheus_missing.json.gz'), 'application/x-gzip')
+        end
+      end
+
+      after(:build) do |artifact, evaluator|
+        artifact.file = evaluator.file
+      end
+    end
+
     trait :correct_checksum do
       after(:build) do |artifact, evaluator|
         artifact.file_sha256 = Digest::SHA256.file(artifact.file.path).hexdigest

spec/factories/clusters/applications/helm.rb

@@ -65,6 +65,10 @@ FactoryBot.define do
       status_reason { 'something went wrong' }
     end
 
+    trait :uninstalled do
+      status { 10 }
+    end
+
     trait :timed_out do
       installing
       updated_at { ClusterWaitForAppInstallationWorker::TIMEOUT.ago }

spec/lib/gitlab/kubernetes/helm/parsers/list_v2_spec.rb

@@ -82,5 +82,19 @@ describe Gitlab::Kubernetes::Helm::Parsers::ListV2 do
         expect(list_v2.releases).to eq([])
       end
     end
+
+    context 'invalid Releases' do
+      let(:invalid_file_contents) do
+        '{ "Releases" : ["a", "b"] }'
+      end
+
+      subject(:list_v2) { described_class.new(invalid_file_contents) }
+
+      it 'raises an error' do
+        expect do
+          list_v2.releases
+        end.to raise_error(described_class::ParserError, 'Invalid format for Releases')
+      end
+    end
   end
 end

spec/models/clusters/cluster_spec.rb

@@ -590,6 +590,60 @@ describe Clusters::Cluster, :use_clean_rails_memory_store_caching do
     end
   end
 
+  describe '#find_or_build_application' do
+    let_it_be(:cluster, reload: true) { create(:cluster) }
+
+    it 'rejects classes that are not applications' do
+      expect do
+        cluster.find_or_build_application(Project)
+      end.to raise_error(ArgumentError)
+    end
+
+    context 'when none of applications are created' do
+      it 'returns the new application', :aggregate_failures do
+        described_class::APPLICATIONS.values.each do |application_class|
+          application = cluster.find_or_build_application(application_class)
+
+          expect(application).to be_a(application_class)
+          expect(application).not_to be_persisted
+        end
+      end
+    end
+
+    context 'when application is persisted' do
+      let!(:helm) { create(:clusters_applications_helm, cluster: cluster) }
+      let!(:ingress) { create(:clusters_applications_ingress, cluster: cluster) }
+      let!(:cert_manager) { create(:clusters_applications_cert_manager, cluster: cluster) }
+      let!(:crossplane) { create(:clusters_applications_crossplane, cluster: cluster) }
+      let!(:prometheus) { create(:clusters_applications_prometheus, cluster: cluster) }
+      let!(:runner) { create(:clusters_applications_runner, cluster: cluster) }
+      let!(:jupyter) { create(:clusters_applications_jupyter, cluster: cluster) }
+      let!(:knative) { create(:clusters_applications_knative, cluster: cluster) }
+      let!(:elastic_stack) { create(:clusters_applications_elastic_stack, cluster: cluster) }
+      let!(:fluentd) { create(:clusters_applications_fluentd, cluster: cluster) }
+
+      it 'returns the persisted application', :aggregate_failures do
+        {
+          Clusters::Applications::Helm => helm,
+          Clusters::Applications::Ingress => ingress,
+          Clusters::Applications::CertManager => cert_manager,
+          Clusters::Applications::Crossplane => crossplane,
+          Clusters::Applications::Prometheus => prometheus,
+          Clusters::Applications::Runner => runner,
+          Clusters::Applications::Jupyter => jupyter,
+          Clusters::Applications::Knative => knative,
+          Clusters::Applications::ElasticStack => elastic_stack,
+          Clusters::Applications::Fluentd => fluentd
+        }.each do |application_class, expected_object|
+          application = cluster.find_or_build_application(application_class)
+
+          expect(application).to eq(expected_object)
+          expect(application).to be_persisted
+        end
+      end
+    end
+  end
+
   describe '#allow_user_defined_namespace?' do
     subject { cluster.allow_user_defined_namespace? }
 

spec/services/ci/create_job_artifacts_service_spec.rb

@@ -177,6 +177,53 @@ describe Ci::CreateJobArtifactsService do
       end
     end
 
+    context 'when artifact type is cluster_applications' do
+      let(:artifacts_file) do
+        file_to_upload('spec/fixtures/helm/helm_list_v2_prometheus_missing.json.gz', sha256: artifacts_sha256)
+      end
+
+      let(:params) do
+        {
+          'artifact_type' => 'cluster_applications',
+          'artifact_format' => 'gzip'
+        }
+      end
+
+      it 'calls cluster applications parse service' do
+        expect_next_instance_of(Clusters::ParseClusterApplicationsArtifactService) do |service|
+          expect(service).to receive(:execute).once.and_call_original
+        end
+
+        subject
+      end
+
+      context 'when there is a deployment cluster' do
+        let(:user) { project.owner }
+
+        before do
+          job.update!(user: user)
+        end
+
+        it 'calls cluster applications parse service with job and job user', :aggregate_failures do
+          expect(Clusters::ParseClusterApplicationsArtifactService).to receive(:new).with(job, user).and_call_original
+
+          subject
+        end
+      end
+
+      context 'when ci_synchronous_artifact_parsing feature flag is disabled' do
+        before do
+          stub_feature_flags(ci_synchronous_artifact_parsing: false)
+        end
+
+        it 'does not call parse service' do
+          expect(Clusters::ParseClusterApplicationsArtifactService).not_to receive(:new)
+
+          expect(subject[:status]).to eq(:success)
+        end
+      end
+    end
+
     shared_examples 'rescues object storage error' do |klass, message, expected_message|
       it "handles #{klass}" do
         allow_next_instance_of(JobArtifactUploader) do |uploader|

spec/services/clusters/parse_cluster_applications_artifact_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Clusters::ParseClusterApplicationsArtifactService do
+  let_it_be(:project) { create(:project) }
+  let_it_be(:user) { create(:user) }
+
+  before do
+    project.add_maintainer(user)
+  end
+
+  describe 'RELEASE_NAMES' do
+    it 'is included in Cluster application names', :aggregate_failures do
+      described_class::RELEASE_NAMES.each do |release_name|
+        expect(Clusters::Cluster::APPLICATIONS).to include(release_name)
+      end
+    end
+  end
+
+  describe '.new' do
+    let(:job) { build(:ci_build) }
+
+    it 'sets the project and current user', :aggregate_failures do
+      service = described_class.new(job, user)
+
+      expect(service.project).to eq(job.project)
+      expect(service.current_user).to eq(user)
+    end
+  end
+
+  describe '#execute' do
+    let_it_be(:cluster, reload: true) { create(:cluster, projects: [project]) }
+    let_it_be(:deployment, reload: true) { create(:deployment, cluster: cluster) }
+
+    let(:job) { deployment.deployable }
+    let(:artifact) { create(:ci_job_artifact, :cluster_applications, job: job) }
+
+    context 'when cluster_applications_artifact feature flag is disabled' do
+      before do
+        stub_feature_flags(cluster_applications_artifact: false)
+      end
+
+      it 'does not call Gitlab::Kubernetes::Helm::Parsers::ListV2 and returns success immediately' do
+        expect(Gitlab::Kubernetes::Helm::Parsers::ListV2).not_to receive(:new)
+
+        result = described_class.new(job, user).execute(artifact)
+
+        expect(result[:status]).to eq(:success)
+      end
+    end
+
+    context 'when cluster_applications_artifact feature flag is enabled for project' do
+      before do
+        stub_feature_flags(cluster_applications_artifact: job.project)
+      end
+
+      it 'calls Gitlab::Kubernetes::Helm::Parsers::ListV2' do
+        expect(Gitlab::Kubernetes::Helm::Parsers::ListV2).to receive(:new).and_call_original
+
+        result = described_class.new(job, user).execute(artifact)
+
+        expect(result[:status]).to eq(:success)
+      end
+
+      context 'artifact is not of cluster_applications type' do
+        let(:artifact) { create(:ci_job_artifact, :archive) }
+        let(:job) { artifact.job }
+
+        it 'raise ArgumentError' do
+          expect do
+            described_class.new(job, user).execute(artifact)
+          end.to raise_error(ArgumentError, 'Artifact is not cluster_applications file type')
+        end
+      end
+
+      context 'artifact exceeds acceptable size' do
+        it 'returns an error' do
+          stub_const("#{described_class}::MAX_ACCEPTABLE_ARTIFACT_SIZE", 1.byte)
+
+          result = described_class.new(job, user).execute(artifact)
+
+          expect(result[:status]).to eq(:error)
+          expect(result[:message]).to eq('Cluster_applications artifact too big. Maximum allowable size: 1 Byte')
+        end
+      end
+
+      context 'job has no deployment cluster' do
+        let(:job) { build(:ci_build) }
+
+        it 'returns an error' do
+          result = described_class.new(job, user).execute(artifact)
+
+          expect(result[:status]).to eq(:error)
+          expect(result[:message]).to eq('No deployment cluster found for this job')
+        end
+      end
+
+      context 'job has deployment cluster' do
+        context 'current user does not have access to deployment cluster' do
+          let(:other_user) { create(:user) }
+
+          it 'returns an error' do
+            result = described_class.new(job, other_user).execute(artifact)
+
+            expect(result[:status]).to eq(:error)
+            expect(result[:message]).to eq('No deployment cluster found for this job')
+          end
+        end
+
+        context 'release is missing' do
+          let(:fixture) { 'spec/fixtures/helm/helm_list_v2_prometheus_missing.json.gz' }
+          let(:file) { fixture_file_upload(Rails.root.join(fixture)) }
+          let(:artifact) { create(:ci_job_artifact, :cluster_applications, job: job, file: file) }
+
+          context 'application does not exist' do
+            it 'does not create or destroy an application' do
+              expect do
+                described_class.new(job, user).execute(artifact)
+              end.not_to change(Clusters::Applications::Prometheus, :count)
+            end
+          end
+
+          context 'application exists' do
+            before do
+              create(:clusters_applications_prometheus, :installed, cluster: cluster)
+            end
+
+            it 'marks the application as uninstalled' do
+              described_class.new(job, user).execute(artifact)
+
+              cluster.application_prometheus.reload
+              expect(cluster.application_prometheus).to be_uninstalled
+            end
+          end
+        end
+
+        context 'release is deployed' do
+          let(:fixture) { 'spec/fixtures/helm/helm_list_v2_prometheus_deployed.json.gz' }
+          let(:file) { fixture_file_upload(Rails.root.join(fixture)) }
+          let(:artifact) { create(:ci_job_artifact, :cluster_applications, job: job, file: file) }
+
+          context 'application does not exist' do
+            it 'creates an application and marks it as installed' do
+              expect do
+                described_class.new(job, user).execute(artifact)
+              end.to change(Clusters::Applications::Prometheus, :count)
+
+              expect(cluster.application_prometheus).to be_persisted
+              expect(cluster.application_prometheus).to be_installed
+            end
+          end
+
+          context 'application exists' do
+            before do
+              create(:clusters_applications_prometheus, :errored, cluster: cluster)
+            end
+
+            it 'marks the application as installed' do
+              described_class.new(job, user).execute(artifact)
+
+              expect(cluster.application_prometheus).to be_installed
+            end
+          end
+        end
+
+        context 'release is failed' do
+          let(:fixture) { 'spec/fixtures/helm/helm_list_v2_prometheus_failed.json.gz' }
+          let(:file) { fixture_file_upload(Rails.root.join(fixture)) }
+          let(:artifact) { create(:ci_job_artifact, :cluster_applications, job: job, file: file) }
+
+          context 'application does not exist' do
+            it 'creates an application and marks it as errored' do
+              expect do
+                described_class.new(job, user).execute(artifact)
+              end.to change(Clusters::Applications::Prometheus, :count)
+
+              expect(cluster.application_prometheus).to be_persisted
+              expect(cluster.application_prometheus).to be_errored
+              expect(cluster.application_prometheus.status_reason).to eq('Helm release failed to install')
+            end
+          end
+
+          context 'application exists' do
+            before do
+              create(:clusters_applications_prometheus, :installed, cluster: cluster)
+            end
+
+            it 'marks the application as errored' do
+              described_class.new(job, user).execute(artifact)
+
+              expect(cluster.application_prometheus).to be_errored
+              expect(cluster.application_prometheus.status_reason).to eq('Helm release failed to install')
+            end
+          end
+        end
+      end
+    end
+  end
+end

spec/support/shared_examples/models/cluster_application_status_shared_examples.rb

@@ -194,6 +194,66 @@ RSpec.shared_examples 'cluster application status specs' do |application_name|
       end
     end
 
+    describe '#make_externally_installed' do
+      subject { create(application_name, :installing) }
+
+      it 'is installed' do
+        subject.make_externally_installed
+
+        expect(subject).to be_installed
+      end
+
+      context 'application is updated' do
+        subject { create(application_name, :updated) }
+
+        it 'is installed' do
+          subject.make_externally_installed
+
+          expect(subject).to be_installed
+        end
+      end
+
+      context 'application is errored' do
+        subject { create(application_name, :errored) }
+
+        it 'is installed' do
+          subject.make_externally_installed
+
+          expect(subject).to be_installed
+        end
+      end
+    end
+
+    describe '#make_externally_uninstalled' do
+      subject { create(application_name, :installed) }
+
+      it 'is uninstalled' do
+        subject.make_externally_uninstalled
+
+        expect(subject).to be_uninstalled
+      end
+
+      context 'application is updated' do
+        subject { create(application_name, :updated) }
+
+        it 'is uninstalled' do
+          subject.make_externally_uninstalled
+
+          expect(subject).to be_uninstalled
+        end
+      end
+
+      context 'application is errored' do
+        subject { create(application_name, :errored) }
+
+        it 'is uninstalled' do
+          subject.make_externally_uninstalled
+
+          expect(subject).to be_uninstalled
+        end
+      end
+    end
+
     describe '#make_scheduled' do
       subject { create(application_name, :installable) }
 
@@ -278,6 +338,7 @@ RSpec.shared_examples 'cluster application status specs' do |application_name|
       :update_errored    | false
       :uninstalling      | false
       :uninstall_errored | false
+      :uninstalled       | false
       :timed_out         | false
     end