Skip to content

G
gitlab-ce
Commit b6181d8e authored by Hordur Freyr Yngvason's avatar Hordur Freyr Yngvason Committed by Dmytro Zaporozhets (DZ)
Browse files
Options

Remove cluster applications report and parser

parent 045c11d1
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 529 deletions
app/models/ci/job_artifact.rb
View file @ b6181d8e
...@@ -43,7 +43,7 @@ module Ci ...@@ -43,7 +43,7 @@ module Ci
dotenv: '.env', dotenv: '.env',
cobertura: 'cobertura-coverage.xml', cobertura: 'cobertura-coverage.xml',
terraform: 'tfplan.json', terraform: 'tfplan.json',
cluster_applications: 'gl-cluster-applications.json', cluster_applications: 'gl-cluster-applications.json', # DEPRECATED: https://gitlab.com/gitlab-org/gitlab/-/issues/333441
requirements: 'requirements.json', requirements: 'requirements.json',
coverage_fuzzing: 'gl-coverage-fuzzing.json', coverage_fuzzing: 'gl-coverage-fuzzing.json',
api_fuzzing: 'gl-api-fuzzing-report.json' api_fuzzing: 'gl-api-fuzzing-report.json'
......
app/services/ci/job_artifacts/create_service.rb
View file @ b6181d8e
...@@ -115,7 +115,6 @@ module Ci ...@@ -115,7 +115,6 @@ module Ci
case artifact.file_type case artifact.file_type
when 'dotenv' then parse_dotenv_artifact(artifact) when 'dotenv' then parse_dotenv_artifact(artifact)
when 'cluster_applications' then parse_cluster_applications_artifact(artifact)
else success else success
end end
end end
...@@ -165,10 +164,6 @@ module Ci ...@@ -165,10 +164,6 @@ module Ci
def parse_dotenv_artifact(artifact) def parse_dotenv_artifact(artifact)
Ci::ParseDotenvArtifactService.new(project, current_user).execute(artifact) Ci::ParseDotenvArtifactService.new(project, current_user).execute(artifact)
end end
def parse_cluster_applications_artifact(artifact)
Clusters::ParseClusterApplicationsArtifactService.new(job, job.user).execute(artifact)
end
end end
end end
end end
app/services/clusters/parse_cluster_applications_artifact_service.rb deleted 100644 → 0
View file @ 045c11d1
# frozen_string_literal: true
module Clusters
class ParseClusterApplicationsArtifactService < ::BaseService
include Gitlab::Utils::StrongMemoize
MAX_ACCEPTABLE_ARTIFACT_SIZE = 5.kilobytes
RELEASE_NAMES = %w[cilium].freeze
def initialize(job, current_user)
@job = job
super(job.project, current_user)
end
def execute(artifact)
raise ArgumentError, 'Artifact is not cluster_applications file type' unless artifact&.cluster_applications?
return error(too_big_error_message, :bad_request) unless artifact.file.size < MAX_ACCEPTABLE_ARTIFACT_SIZE
return error(no_deployment_message, :bad_request) unless job.deployment
return error(no_deployment_cluster_message, :bad_request) unless cluster
parse!(artifact)
success
rescue Gitlab::Kubernetes::Helm::Parsers::ListV2::ParserError, ActiveRecord::RecordInvalid => error
Gitlab::ErrorTracking.track_exception(error, job_id: artifact.job_id)
error(error.message, :bad_request)
end
private
attr_reader :job
def cluster
strong_memoize(:cluster) do
deployment_cluster = job.deployment&.cluster
deployment_cluster if Ability.allowed?(current_user, :admin_cluster, deployment_cluster)
end
end
def parse!(artifact)
releases = []
artifact.each_blob do |blob|
next if blob.empty?
releases.concat(Gitlab::Kubernetes::Helm::Parsers::ListV2.new(blob).releases)
end
update_cluster_application_statuses!(releases)
end
def update_cluster_application_statuses!(releases)
release_by_name = releases.index_by { |release| release['Name'] }
Clusters::Cluster.transaction do
RELEASE_NAMES.each do |release_name|
application_class = Clusters::Cluster::APPLICATIONS[release_name]
application = cluster.find_or_build_application(application_class)
release = release_by_name[release_name]
if release
case release['Status']
when 'DEPLOYED'
application.make_externally_installed!
when 'FAILED'
application.make_errored!(s_('ClusterIntegration|Helm release failed to install'))
end
else
# missing, so by definition, we consider this uninstalled
application.make_externally_uninstalled! if application.persisted?
end
end
end
end
def too_big_error_message
human_size = ActiveSupport::NumberHelper.number_to_human_size(MAX_ACCEPTABLE_ARTIFACT_SIZE)
s_('ClusterIntegration|Cluster_applications artifact too big. Maximum allowable size: %{human_size}') % { human_size: human_size }
end
def no_deployment_message
s_('ClusterIntegration|No deployment found for this job')
end
def no_deployment_cluster_message
s_('ClusterIntegration|No deployment cluster found for this job')
end
end
end
lib/gitlab/ci/config/entry/reports.rb
View file @ b6181d8e
...@@ -43,7 +43,7 @@ module Gitlab ...@@ -43,7 +43,7 @@ module Gitlab
validates :cobertura, array_of_strings_or_string: true validates :cobertura, array_of_strings_or_string: true
validates :terraform, array_of_strings_or_string: true validates :terraform, array_of_strings_or_string: true
validates :accessibility, array_of_strings_or_string: true validates :accessibility, array_of_strings_or_string: true
validates :cluster_applications, array_of_strings_or_string: true validates :cluster_applications, array_of_strings_or_string: true # DEPRECATED: https://gitlab.com/gitlab-org/gitlab/-/issues/333441
validates :requirements, array_of_strings_or_string: true validates :requirements, array_of_strings_or_string: true
end end
end end
......
lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml
View file @ b6181d8e
...@@ -12,8 +12,6 @@ apply: ...@@ -12,8 +12,6 @@ apply:
variables: variables:
- $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
artifacts: artifacts:
reports:
cluster_applications: gl-cluster-applications.json
when: on_failure when: on_failure
paths: paths:
- tiller.log - tiller.log
lib/gitlab/kubernetes/helm/parsers/list_v2.rb deleted 100644 → 0
View file @ 045c11d1
# frozen_string_literal: true
module Gitlab
module Kubernetes
module Helm
module Parsers
# Parses Helm v2 list (JSON) output
class ListV2
ParserError = Class.new(StandardError)
attr_reader :contents, :json
def initialize(contents)
@contents = contents
@json = Gitlab::Json.parse(contents)
rescue JSON::ParserError => e
raise ParserError, e.message
end
def releases
@releases = helm_releases
end
private
def helm_releases
helm_releases = json['Releases'] || []
raise ParserError, 'Invalid format for Releases' unless helm_releases.all? { |item| item.is_a?(Hash) }
helm_releases
end
end
end
end
end
end
locale/gitlab.pot
View file @ b6181d8e
...@@ -7075,9 +7075,6 @@ msgstr "" ...@@ -7075,9 +7075,6 @@ msgstr ""
msgid "ClusterIntegration|Cluster name is required." msgid "ClusterIntegration|Cluster name is required."
msgstr "" msgstr ""
msgid "ClusterIntegration|Cluster_applications artifact too big. Maximum allowable size: %{human_size}"
msgstr ""
msgid "ClusterIntegration|Clusters are utilized by selecting the nearest ancestor with a matching environment scope. For example, project clusters will override group clusters. %{linkStart}More information%{linkEnd}" msgid "ClusterIntegration|Clusters are utilized by selecting the nearest ancestor with a matching environment scope. For example, project clusters will override group clusters. %{linkStart}More information%{linkEnd}"
msgstr "" msgstr ""
...@@ -7282,9 +7279,6 @@ msgstr "" ...@@ -7282,9 +7279,6 @@ msgstr ""
msgid "ClusterIntegration|HTTP Error" msgid "ClusterIntegration|HTTP Error"
msgstr "" msgstr ""
msgid "ClusterIntegration|Helm release failed to install"
msgstr ""
msgid "ClusterIntegration|If you are setting up multiple clusters and are using Auto DevOps, %{help_link_start}read this first%{help_link_end}." msgid "ClusterIntegration|If you are setting up multiple clusters and are using Auto DevOps, %{help_link_start}read this first%{help_link_end}."
msgstr "" msgstr ""
...@@ -7444,12 +7438,6 @@ msgstr "" ...@@ -7444,12 +7438,6 @@ msgstr ""
msgid "ClusterIntegration|No VPCs found" msgid "ClusterIntegration|No VPCs found"
msgstr "" msgstr ""
msgid "ClusterIntegration|No deployment cluster found for this job"
msgstr ""
msgid "ClusterIntegration|No deployment found for this job"
msgstr ""
msgid "ClusterIntegration|No instance type found" msgid "ClusterIntegration|No instance type found"
msgstr "" msgstr ""
......
spec/factories/ci/job_artifacts.rb
View file @ b6181d8e
...@@ -317,21 +317,6 @@ FactoryBot.define do ...@@ -317,21 +317,6 @@ FactoryBot.define do
end end
end end
trait :cluster_applications do
file_type { :cluster_applications }
file_format { :gzip }
transient do
file do
fixture_file_upload(Rails.root.join('spec/fixtures/helm/helm_list_v2_prometheus_missing.json.gz'), 'application/x-gzip')
end
end
after(:build) do |artifact, evaluator|
artifact.file = evaluator.file
end
end
trait :correct_checksum do trait :correct_checksum do
after(:build) do |artifact, evaluator| after(:build) do |artifact, evaluator|
artifact.file_sha256 = Digest::SHA256.file(artifact.file.path).hexdigest artifact.file_sha256 = Digest::SHA256.file(artifact.file.path).hexdigest
......
spec/lib/gitlab/kubernetes/helm/parsers/list_v2_spec.rb deleted 100644 → 0
View file @ 045c11d1
# frozen_string_literal: true
require 'fast_spec_helper'
RSpec.describe Gitlab::Kubernetes::Helm::Parsers::ListV2 do
let(:valid_file_contents) do
<<~EOF
{
"Next": "",
"Releases": [
{
"Name": "certmanager",
"Revision": 2,
"Updated": "Sun Mar 29 06:55:42 2020",
"Status": "DEPLOYED",
"Chart": "cert-manager-v0.10.1",
"AppVersion": "v0.10.1",
"Namespace": "gitlab-managed-apps"
},
{
"Name": "certmanager-crds",
"Revision": 2,
"Updated": "Sun Mar 29 06:55:32 2020",
"Status": "DEPLOYED",
"Chart": "cert-manager-crds-v0.2.0",
"AppVersion": "release-0.10",
"Namespace": "gitlab-managed-apps"
},
{
"Name": "certmanager-issuer",
"Revision": 1,
"Updated": "Tue Feb 18 10:04:04 2020",
"Status": "FAILED",
"Chart": "cert-manager-issuer-v0.1.0",
"AppVersion": "",
"Namespace": "gitlab-managed-apps"
},
{
"Name": "runner",
"Revision": 2,
"Updated": "Sun Mar 29 07:01:01 2020",
"Status": "DEPLOYED",
"Chart": "gitlab-runner-0.14.0",
"AppVersion": "12.8.0",
"Namespace": "gitlab-managed-apps"
}
]
}
EOF
end
describe '#initialize' do
it 'initializes without error' do
expect do
described_class.new(valid_file_contents)
end.not_to raise_error
end
it 'raises an error on invalid JSON' do
expect do
described_class.new('')
end.to raise_error(described_class::ParserError)
end
end
describe '#releases' do
subject(:list_v2) { described_class.new(valid_file_contents) }
it 'returns list of releases' do
expect(list_v2.releases).to match([
a_hash_including('Name' => 'certmanager', 'Status' => 'DEPLOYED'),
a_hash_including('Name' => 'certmanager-crds', 'Status' => 'DEPLOYED'),
a_hash_including('Name' => 'certmanager-issuer', 'Status' => 'FAILED'),
a_hash_including('Name' => 'runner', 'Status' => 'DEPLOYED')
])
end
context 'empty Releases' do
let(:valid_file_contents) { '{}' }
it 'returns an empty array' do
expect(list_v2.releases).to eq([])
end
end
context 'invalid Releases' do
let(:invalid_file_contents) do
'{ "Releases" : ["a", "b"] }'
end
subject(:list_v2) { described_class.new(invalid_file_contents) }
it 'raises an error' do
expect do
list_v2.releases
end.to raise_error(described_class::ParserError, 'Invalid format for Releases')
end
end
end
end
spec/services/ci/job_artifacts/create_service_spec.rb
View file @ b6181d8e
...@@ -203,53 +203,6 @@ RSpec.describe Ci::JobArtifacts::CreateService do ...@@ -203,53 +203,6 @@ RSpec.describe Ci::JobArtifacts::CreateService do
end end
end end
context 'when artifact type is cluster_applications' do
let(:artifacts_file) do
file_to_upload('spec/fixtures/helm/helm_list_v2_prometheus_missing.json.gz', sha256: artifacts_sha256)
end
let(:params) do
{
'artifact_type' => 'cluster_applications',
'artifact_format' => 'gzip'
}.with_indifferent_access
end
it 'calls cluster applications parse service' do
expect_next_instance_of(Clusters::ParseClusterApplicationsArtifactService) do |service|
expect(service).to receive(:execute).once.and_call_original
end
subject
end
context 'when there is a deployment cluster' do
let(:user) { project.owner }
before do
job.update!(user: user)
end
it 'calls cluster applications parse service with job and job user', :aggregate_failures do
expect(Clusters::ParseClusterApplicationsArtifactService).to receive(:new).with(job, user).and_call_original
subject
end
end
context 'when ci_synchronous_artifact_parsing feature flag is disabled' do
before do
stub_feature_flags(ci_synchronous_artifact_parsing: false)
end
it 'does not call parse service' do
expect(Clusters::ParseClusterApplicationsArtifactService).not_to receive(:new)
expect(subject[:status]).to eq(:success)
end
end
end
shared_examples 'rescues object storage error' do |klass, message, expected_message| shared_examples 'rescues object storage error' do |klass, message, expected_message|
it "handles #{klass}" do it "handles #{klass}" do
allow_next_instance_of(JobArtifactUploader) do |uploader| allow_next_instance_of(JobArtifactUploader) do |uploader|
......
spec/services/clusters/parse_cluster_applications_artifact_service_spec.rb deleted 100644 → 0
View file @ 045c11d1
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Clusters::ParseClusterApplicationsArtifactService do
let_it_be(:project) { create(:project) }
let_it_be(:user) { create(:user) }
before do
project.add_maintainer(user)
end
describe 'RELEASE_NAMES' do
it 'is included in Cluster application names', :aggregate_failures do
described_class::RELEASE_NAMES.each do |release_name|
expect(Clusters::Cluster::APPLICATIONS).to include(release_name)
end
end
end
describe '.new' do
let(:job) { build(:ci_build) }
it 'sets the project and current user', :aggregate_failures do
service = described_class.new(job, user)
expect(service.project).to eq(job.project)
expect(service.current_user).to eq(user)
end
end
describe '#execute' do
let_it_be(:cluster, reload: true) { create(:cluster, projects: [project]) }
let_it_be(:deployment, reload: true) { create(:deployment, cluster: cluster) }
let(:job) { deployment.deployable }
let(:artifact) { create(:ci_job_artifact, :cluster_applications, job: job) }
it 'calls Gitlab::Kubernetes::Helm::Parsers::ListV2' do
expect(Gitlab::Kubernetes::Helm::Parsers::ListV2).to receive(:new).and_call_original
result = described_class.new(job, user).execute(artifact)
expect(result[:status]).to eq(:success)
end
context 'artifact is not of cluster_applications type' do
let(:artifact) { create(:ci_job_artifact, :archive) }
let(:job) { artifact.job }
it 'raise ArgumentError' do
expect do
described_class.new(job, user).execute(artifact)
end.to raise_error(ArgumentError, 'Artifact is not cluster_applications file type')
end
end
context 'artifact exceeds acceptable size' do
it 'returns an error' do
stub_const("#{described_class}::MAX_ACCEPTABLE_ARTIFACT_SIZE", 1.byte)
result = described_class.new(job, user).execute(artifact)
expect(result[:status]).to eq(:error)
expect(result[:message]).to eq('Cluster_applications artifact too big. Maximum allowable size: 1 Byte')
end
end
context 'job has no deployment' do
let(:job) { build(:ci_build) }
it 'returns an error' do
result = described_class.new(job, user).execute(artifact)
expect(result[:status]).to eq(:error)
expect(result[:message]).to eq('No deployment found for this job')
end
end
context 'job has no deployment cluster' do
let(:deployment) { create(:deployment) }
let(:job) { deployment.deployable }
it 'returns an error' do
result = described_class.new(job, user).execute(artifact)
expect(result[:status]).to eq(:error)
expect(result[:message]).to eq('No deployment cluster found for this job')
end
end
context 'blob is empty' do
let(:file) { fixture_file_upload(Rails.root.join("spec/fixtures/helm/helm_list_v2_empty_blob.json.gz")) }
let(:artifact) { create(:ci_job_artifact, :cluster_applications, job: job, file: file) }
it 'returns success' do
result = described_class.new(job, user).execute(artifact)
expect(result[:status]).to eq(:success)
end
end
context 'job has deployment cluster' do
context 'current user does not have access to deployment cluster' do
let(:other_user) { create(:user) }
it 'returns an error' do
result = described_class.new(job, other_user).execute(artifact)
expect(result[:status]).to eq(:error)
expect(result[:message]).to eq('No deployment cluster found for this job')
end
end
it 'does not affect unpermitted cluster applications' do
expect(Clusters::ParseClusterApplicationsArtifactService::RELEASE_NAMES).to contain_exactly('cilium')
end
Clusters::ParseClusterApplicationsArtifactService::RELEASE_NAMES.each do |release_name|
context release_name do
include_examples 'parse cluster applications artifact', release_name
end
end
end
end
end
spec/support/shared_examples/services/clusters/parse_cluster_applications_artifact_shared_examples.rb deleted 100644 → 0
View file @ 045c11d1
# frozen_string_literal: true
RSpec.shared_examples 'parse cluster applications artifact' do |release_name|
let(:application_class) { Clusters::Cluster::APPLICATIONS[release_name] }
let(:cluster_application) { cluster.public_send("application_#{release_name}") }
let(:file) { fixture_file_upload(Rails.root.join(fixture)) }
let(:artifact) { create(:ci_job_artifact, :cluster_applications, job: job, file: file) }
context 'release is missing' do
let(:fixture) { "spec/fixtures/helm/helm_list_v2_#{release_name}_missing.json.gz" }
context 'application does not exist' do
it 'does not create or destroy an application' do
expect do
described_class.new(job, user).execute(artifact)
end.not_to change(application_class, :count)
end
end
context 'application exists' do
before do
create("clusters_applications_#{release_name}".to_sym, :installed, cluster: cluster)
end
it 'marks the application as uninstalled' do
described_class.new(job, user).execute(artifact)
cluster_application.reload
expect(cluster_application).to be_uninstalled
end
end
end
context 'release is deployed' do
let(:fixture) { "spec/fixtures/helm/helm_list_v2_#{release_name}_deployed.json.gz" }
context 'application does not exist' do
it 'creates an application and marks it as installed' do
expect do
described_class.new(job, user).execute(artifact)
end.to change(application_class, :count)
expect(cluster_application).to be_persisted
expect(cluster_application).to be_externally_installed
end
end
context 'application exists' do
before do
create("clusters_applications_#{release_name}".to_sym, :errored, cluster: cluster)
end
it 'marks the application as installed' do
described_class.new(job, user).execute(artifact)
expect(cluster_application).to be_externally_installed
end
end
end
context 'release is failed' do
let(:fixture) { "spec/fixtures/helm/helm_list_v2_#{release_name}_failed.json.gz" }
context 'application does not exist' do
it 'creates an application and marks it as errored' do
expect do
described_class.new(job, user).execute(artifact)
end.to change(application_class, :count)
expect(cluster_application).to be_persisted
expect(cluster_application).to be_errored
expect(cluster_application.status_reason).to eq('Helm release failed to install')
end
end
context 'application exists' do
before do
create("clusters_applications_#{release_name}".to_sym, :installed, cluster: cluster)
end
it 'marks the application as errored' do
described_class.new(job, user).execute(artifact)
expect(cluster_application).to be_errored
expect(cluster_application.status_reason).to eq('Helm release failed to install')
end
end
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7