Commit 2f6d081d authored by Stan Hu's avatar Stan Hu

Merge branch '11424-sync-maven-metadata-for-maven-plugins' into 'master'

Support maven plugins packaging in the maven metadata sync worker

See merge request gitlab-org/gitlab!56229
parents cb20b848 6c9f6489
......@@ -18,6 +18,13 @@ class Packages::Maven::Metadatum < ApplicationRecord
validate :maven_package_type
scope :for_package_ids, -> (package_ids) { where(package_id: package_ids) }
scope :order_created, -> { reorder('created_at ASC') }
def self.pluck_app_name
pluck(:app_name)
end
private
def maven_package_type
......
# frozen_string_literal: true
module Packages
module Maven
module Metadata
class BaseCreateXmlService
include Gitlab::Utils::StrongMemoize
INDENT_SPACE = 2
def initialize(metadata_content:, package:)
@metadata_content = metadata_content
@package = package
end
private
def xml_doc
strong_memoize(:xml_doc) do
Nokogiri::XML(@metadata_content) do |config|
config.default_xml.noblanks
end
end
end
def xml_node(name, content)
xml_doc.create_element(name).tap { |e| e.content = content }
end
end
end
end
end
# frozen_string_literal: true
module Packages
module Maven
module Metadata
class CreatePluginsXmlService < BaseCreateXmlService
XPATH_PLUGIN_ARTIFACT_ID = '//plugin/artifactId'
XPATH_PLUGINS = '//metadata/plugins'
EMPTY_PLUGINS_PAYLOAD = {
changes_exist: true,
empty_plugins: true
}.freeze
def execute
return ServiceResponse.error(message: 'package not set') unless @package
return ServiceResponse.error(message: 'metadata_content not set') unless @metadata_content
return ServiceResponse.error(message: 'metadata_content is invalid') unless plugins_xml_node.present?
return ServiceResponse.success(payload: EMPTY_PLUGINS_PAYLOAD) if plugin_artifact_ids_from_database.empty?
changes_exist = update_plugins_list
payload = { changes_exist: changes_exist, empty_versions: false }
payload[:metadata_content] = xml_doc.to_xml(indent: INDENT_SPACE) if changes_exist
ServiceResponse.success(payload: payload)
end
private
def update_plugins_list
return false if plugin_artifact_ids_from_xml == plugin_artifact_ids_from_database
plugins_xml_node.children.remove
plugin_artifact_ids_from_database.each do |artifact_id|
plugins_xml_node.add_child(plugin_node_for(artifact_id))
end
true
end
def plugins_xml_node
strong_memoize(:plugins_xml_node) do
xml_doc.xpath(XPATH_PLUGINS)
.first
end
end
def plugin_artifact_ids_from_xml
strong_memoize(:plugin_artifact_ids_from_xml) do
plugins_xml_node.xpath(XPATH_PLUGIN_ARTIFACT_ID)
.map(&:content)
end
end
def plugin_artifact_ids_from_database
strong_memoize(:plugin_artifact_ids_from_database) do
package_names = plugin_artifact_ids_from_xml.map do |artifact_id|
"#{@package.name}/#{artifact_id}"
end
packages = @package.project.packages
.maven
.displayable
.with_name(package_names)
.has_version
::Packages::Maven::Metadatum.for_package_ids(packages.select(:id))
.order_created
.pluck_app_name
.uniq
end
end
def plugin_node_for(artifact_id)
xml_doc.create_element('plugin').tap do |plugin_node|
plugin_node.add_child(xml_node('name', artifact_id))
plugin_node.add_child(xml_node('prefix', prefix_from(artifact_id)))
plugin_node.add_child(xml_node('artifactId', artifact_id))
end
end
# Maven plugin prefix generation from
# https://github.com/apache/maven/blob/c3dba0e5ba71ee7cbd62620f669a8c206e71b5e2/maven-plugin-api/src/main/java/org/apache/maven/plugin/descriptor/PluginDescriptor.java#L189
def prefix_from(artifact_id)
artifact_id.gsub(/-?maven-?/, '')
.gsub(/-?plugin-?/, '')
end
end
end
end
end
......@@ -3,9 +3,7 @@
module Packages
module Maven
module Metadata
class CreateVersionsXmlService
include Gitlab::Utils::StrongMemoize
class CreateVersionsXmlService < BaseCreateXmlService
XPATH_VERSIONING = '//metadata/versioning'
XPATH_VERSIONS = '//versions'
XPATH_VERSION = '//version'
......@@ -13,18 +11,11 @@ module Packages
XPATH_RELEASE = '//release'
XPATH_LAST_UPDATED = '//lastUpdated'
INDENT_SPACE = 2
EMPTY_VERSIONS_PAYLOAD = {
changes_exist: true,
empty_versions: true
}.freeze
def initialize(metadata_content:, package:)
@metadata_content = metadata_content
@package = package
end
def execute
return ServiceResponse.error(message: 'package not set') unless @package
return ServiceResponse.error(message: 'metadata_content not set') unless @metadata_content
......@@ -57,7 +48,7 @@ module Packages
version_xml_nodes.remove
versions_from_database.each do |version|
versions_xml_node.add_child(version_node_for(version))
versions_xml_node.add_child(xml_node('version', version))
end
true
end
......@@ -131,10 +122,6 @@ module Packages
end
end
def version_node_for(version)
Nokogiri::XML::Node.new('version', xml_doc).tap { |node| node.content = version }
end
def versions_from_xml
strong_memoize(:versions_from_xml) do
versions_xml_node.xpath(XPATH_VERSION)
......@@ -172,14 +159,6 @@ module Packages
non_snapshot_versions_from_database.last
end
end
def xml_doc
strong_memoize(:xml_doc) do
Nokogiri::XML(@metadata_content) do |config|
config.default_xml.noblanks
end
end
end
end
end
end
......
......@@ -16,26 +16,50 @@ module Packages
return error('Non existing versionless package') unless versionless_package_for_versions
return error('Non existing metadata file for versions') unless metadata_package_file_for_versions
if metadata_package_file_for_plugins
result = update_plugins_xml
return result if result.error?
end
update_versions_xml
end
private
def update_versions_xml
return error('Metadata file for versions is too big') if metadata_package_file_for_versions.size > MAX_FILE_SIZE
update_xml(
kind: :versions,
package_file: metadata_package_file_for_versions,
service_class: CreateVersionsXmlService,
payload_empty_field: :empty_versions
)
end
metadata_package_file_for_versions.file.use_open_file do |file|
result = CreateVersionsXmlService.new(metadata_content: file, package: versionless_package_for_versions)
.execute
def update_plugins_xml
update_xml(
kind: :plugins,
package_file: metadata_package_file_for_plugins,
service_class: CreatePluginsXmlService,
payload_empty_field: :empty_plugins
)
end
def update_xml(kind:, package_file:, service_class:, payload_empty_field:)
return error("Metadata file for #{kind} is too big") if package_file.size > MAX_FILE_SIZE
package_file.file.use_open_file do |file|
result = service_class.new(metadata_content: file, package: package_file.package)
.execute
next result unless result.success?
next success('No changes for versions xml') unless result.payload[:changes_exist]
next success("No changes for #{kind} xml") unless result.payload[:changes_exist]
if result.payload[:empty_versions]
versionless_package_for_versions.destroy!
success('Versionless package for versions destroyed')
if result.payload[payload_empty_field]
package_file.package.destroy!
success("Versionless package for #{kind} destroyed")
else
AppendPackageFileService.new(metadata_content: result.payload[:metadata_content], package: versionless_package_for_versions)
AppendPackageFileService.new(metadata_content: result.payload[:metadata_content], package: package_file.package)
.execute
end
end
......@@ -43,28 +67,49 @@ module Packages
def metadata_package_file_for_versions
strong_memoize(:metadata_file_for_versions) do
versionless_package_for_versions.package_files
.with_file_name(Metadata.filename)
.recent
.first
metadata_package_file_for(versionless_package_for_versions)
end
end
def versionless_package_for_versions
strong_memoize(:versionless_package_for_versions) do
project.packages
.maven
.displayable
.with_name(package_name)
.with_version(nil)
.first
versionless_package_named(package_name)
end
end
def metadata_package_file_for_plugins
strong_memoize(:metadata_package_file_for_plugins) do
metadata_package_file_for(versionless_package_named(package_name_for_plugins))
end
end
def metadata_package_file_for(package)
return unless package
package.package_files
.with_file_name(Metadata.filename)
.recent
.first
end
def versionless_package_named(name)
project.packages
.maven
.displayable
.with_name(name)
.with_version(nil)
.first
end
def package_name
params[:package_name]
end
def package_name_for_plugins
group = versionless_package_for_versions.maven_metadatum.app_group
group.tr('.', '/')
end
def error(message)
ServiceResponse.error(message: message)
end
......
---
title: Support maven plugins packaging in the maven metadata sync worker
merge_request: 56229
author:
type: fixed
......@@ -36,5 +36,38 @@ RSpec.describe Packages::Maven::Metadatum, type: :model do
expect(maven_metadatum.errors.to_a).to include('Package type must be Maven')
end
end
context 'with a package' do
let_it_be(:package) { create(:package) }
describe '.for_package_ids' do
let_it_be(:metadata) { create_list(:maven_metadatum, 3, package: package) }
subject { Packages::Maven::Metadatum.for_package_ids(package.id) }
it { is_expected.to match_array(metadata) }
end
describe '.order_created' do
let_it_be(:metadatum1) { create(:maven_metadatum, package: package) }
let_it_be(:metadatum2) { create(:maven_metadatum, package: package) }
let_it_be(:metadatum3) { create(:maven_metadatum, package: package) }
let_it_be(:metadatum4) { create(:maven_metadatum, package: package) }
subject { Packages::Maven::Metadatum.for_package_ids(package.id).order_created }
it { is_expected.to eq([metadatum1, metadatum2, metadatum3, metadatum4]) }
end
describe '.pluck_app_name' do
let_it_be(:metadatum1) { create(:maven_metadatum, package: package, app_name: 'one') }
let_it_be(:metadatum2) { create(:maven_metadatum, package: package, app_name: 'two') }
let_it_be(:metadatum3) { create(:maven_metadatum, package: package, app_name: 'three') }
subject { Packages::Maven::Metadatum.for_package_ids(package.id).pluck_app_name }
it { is_expected.to match_array([metadatum1, metadatum2, metadatum3].map(&:app_name)) }
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe ::Packages::Maven::Metadata::CreatePluginsXmlService do
let_it_be(:group_id) { 'my/test' }
let_it_be(:package) { create(:maven_package, name: group_id, version: nil) }
let(:plugins_in_database) { %w[one-maven-plugin two three-maven-plugin] }
let(:plugins_in_xml) { %w[one-maven-plugin two three-maven-plugin] }
let(:service) { described_class.new(metadata_content: metadata_xml, package: package) }
describe '#execute' do
subject { service.execute }
before do
next unless package
plugins_in_database.each do |plugin|
create(
:maven_package,
name: "#{group_id}/#{plugin}",
version: '1.0.0',
project: package.project,
maven_metadatum_attributes: {
app_group: group_id.tr('/', '.'),
app_name: plugin,
app_version: '1.0.0'
}
)
end
end
shared_examples 'returning an xml with plugins from the database' do
it 'returns an metadata versions xml with versions in the database', :aggregate_failures do
expect(subject).to be_success
expect(subject.payload[:changes_exist]).to eq(true)
expect(subject.payload[:empty_versions]).to eq(false)
expect(plugins_from(subject.payload[:metadata_content])).to match_array(plugins_in_database)
end
end
shared_examples 'returning no changes' do
it 'returns no changes', :aggregate_failures do
expect(subject).to be_success
expect(subject.payload).to eq(changes_exist: false, empty_versions: false)
end
end
context 'with same plugins on both sides' do
it_behaves_like 'returning no changes'
end
context 'with more plugins' do
let(:additional_plugins) { %w[four-maven-plugin five] }
context 'in database' do
let(:plugins_in_database) { plugins_in_xml + additional_plugins }
# we can't distinguish that the additional plugin are actually maven plugins
it_behaves_like 'returning no changes'
end
context 'in xml' do
let(:plugins_in_xml) { plugins_in_database + additional_plugins }
it_behaves_like 'returning an xml with plugins from the database'
end
end
context 'with no versions in the database' do
let(:plugins_in_database) { [] }
it 'returns a success', :aggregate_failures do
result = subject
expect(result).to be_success
expect(result.payload).to eq(changes_exist: true, empty_plugins: true)
end
end
context 'with an incomplete metadata content' do
let(:metadata_xml) { '<metadata></metadata>' }
it_behaves_like 'returning an error service response', message: 'metadata_content is invalid'
end
context 'with an invalid metadata content' do
let(:metadata_xml) { '<meta></metadata>' }
it_behaves_like 'returning an error service response', message: 'metadata_content is invalid'
end
it_behaves_like 'handling metadata content pointing to a file for the create xml service'
it_behaves_like 'handling invalid parameters for create xml service'
end
def metadata_xml
Nokogiri::XML::Builder.new do |xml|
xml.metadata do
xml.plugins do
plugins_in_xml.each do |plugin|
xml.plugin do
xml.name(plugin)
xml.prefix(prefix_from(plugin))
xml.artifactId(plugin)
end
end
end
end
end.to_xml
end
def prefix_from(artifact_id)
artifact_id.gsub(/-?maven-?/, '')
.gsub(/-?plugin-?/, '')
end
def plugins_from(xml_content)
doc = Nokogiri::XML(xml_content)
doc.xpath('//metadata/plugins/plugin/artifactId').map(&:content)
end
end
......@@ -181,59 +181,9 @@ RSpec.describe ::Packages::Maven::Metadata::CreateVersionsXmlService do
it_behaves_like 'returning an error service response', message: 'metadata_content is invalid'
end
context 'with metadata content pointing to a file' do
let(:service) { described_class.new(metadata_content: file, package: package) }
let(:file) do
Tempfile.new('metadata').tap do |file|
if file_contents
file.write(file_contents)
file.flush
file.rewind
end
end
end
after do
file.close
file.unlink
end
context 'with valid content' do
let(:file_contents) { metadata_xml }
it 'returns no changes' do
result = subject
expect(result).to be_success
expect(result.payload).to eq(changes_exist: false, empty_versions: false)
end
end
it_behaves_like 'handling metadata content pointing to a file for the create xml service'
context 'with invalid content' do
let(:file_contents) { '<meta></metadata>' }
it_behaves_like 'returning an error service response', message: 'metadata_content is invalid'
end
context 'with no content' do
let(:file_contents) { nil }
it_behaves_like 'returning an error service response', message: 'metadata_content is invalid'
end
end
context 'with no package' do
let(:metadata_xml) { '' }
let(:package) { nil }
it_behaves_like 'returning an error service response', message: 'package not set'
end
context 'with no metadata content' do
let(:metadata_xml) { nil }
it_behaves_like 'returning an error service response', message: 'metadata_content not set'
end
it_behaves_like 'handling invalid parameters for create xml service'
end
def metadata_xml
......
# frozen_string_literal: true
RSpec.shared_examples 'handling metadata content pointing to a file for the create xml service' do
context 'with metadata content pointing to a file' do
let(:service) { described_class.new(metadata_content: file, package: package) }
let(:file) do
Tempfile.new('metadata').tap do |file|
if file_contents
file.write(file_contents)
file.flush
file.rewind
end
end
end
after do
file.close
file.unlink
end
context 'with valid content' do
let(:file_contents) { metadata_xml }
it 'returns no changes' do
expect(subject).to be_success
expect(subject.payload).to eq(changes_exist: false, empty_versions: false)
end
end
context 'with invalid content' do
let(:file_contents) { '<meta></metadata>' }
it_behaves_like 'returning an error service response', message: 'metadata_content is invalid'
end
context 'with no content' do
let(:file_contents) { nil }
it_behaves_like 'returning an error service response', message: 'metadata_content is invalid'
end
end
end
RSpec.shared_examples 'handling invalid parameters for create xml service' do
context 'with no package' do
let(:metadata_xml) { '' }
let(:package) { nil }
it_behaves_like 'returning an error service response', message: 'package not set'
end
context 'with no metadata content' do
let(:metadata_xml) { nil }
it_behaves_like 'returning an error service response', message: 'metadata_content not set'
end
end
......@@ -22,18 +22,23 @@ RSpec.describe Packages::Maven::Metadata::SyncWorker, type: :worker do
subject { worker.perform(user.id, project.id, package_name) }
context 'with a valid package name' do
before do
file = CarrierWaveStringFile.new_file(file_content: versions_xml_content, filename: 'maven-metadata.xml', content_type: 'application/xml')
metadata_package_file.update!(file: file)
versions.each do |version|
create(:maven_package, name: versionless_package_for_versions.name, version: version, project: versionless_package_for_versions.project)
context 'with a jar' do
context 'with a valid package name' do
before do
metadata_package_file.update!(
file: CarrierWaveStringFile.new_file(
file_content: versions_xml_content,
filename: 'maven-metadata.xml',
content_type: 'application/xml'
)
)
versions.each do |version|
create(:maven_package, name: versionless_package_for_versions.name, version: version, project: versionless_package_for_versions.project)
end
end
end
context 'idempotent worker' do
include_examples 'an idempotent worker' do
it_behaves_like 'an idempotent worker' do
let(:job_args) { [user.id, project.id, package_name] }
it 'creates the updated metadata files', :aggregate_failures do
......@@ -45,31 +50,116 @@ RSpec.describe Packages::Maven::Metadata::SyncWorker, type: :worker do
expect(most_recent_versions.versions).to match_array(versions)
end
end
end
it 'logs the message from the service' do
expect(worker).to receive(:log_extra_metadata_on_done).with(:message, 'New metadata package file created')
it 'logs the message from the service' do
expect(worker).to receive(:log_extra_metadata_on_done).with(:message, 'New metadata package file created')
subject
end
subject
end
context 'not in the passed project' do
let(:project) { create(:project) }
context 'not in the passed project' do
let(:project) { create(:project) }
it 'does not create the updated metadata files' do
expect { subject }
.to change { ::Packages::PackageFile.count }.by(0)
.and raise_error(described_class::SyncError, 'Non existing versionless package')
it 'does not create the updated metadata files' do
expect { subject }
.to change { ::Packages::PackageFile.count }.by(0)
.and raise_error(described_class::SyncError, 'Non existing versionless package')
end
end
context 'with a user with not enough permissions' do
let(:role) { :guest }
it 'does not create the updated metadata files' do
expect { subject }
.to change { ::Packages::PackageFile.count }.by(0)
.and raise_error(described_class::SyncError, 'Not allowed')
end
end
end
end
context 'with a user with not enough permissions' do
let(:role) { :guest }
context 'with a maven plugin' do
let_it_be(:versionless_package_name_for_plugins) { versionless_package_for_versions.maven_metadatum.app_group.tr('.', '/') }
let_it_be(:versionless_package_for_versions) { create(:maven_package, name: "#{versionless_package_name_for_plugins}/one-maven-plugin", version: nil) }
let_it_be(:metadata_package_file) { create(:package_file, :xml, package: versionless_package_for_versions) }
let_it_be(:versionless_package_for_plugins) { create(:maven_package, name: versionless_package_name_for_plugins, version: nil, project: versionless_package_for_versions.project) }
let_it_be(:metadata_package_file_for_plugins) { create(:package_file, :xml, package: versionless_package_for_plugins) }
let_it_be(:addtional_maven_package_for_same_group_id) { create(:maven_package, name: "#{versionless_package_name_for_plugins}/maven-package", project: versionless_package_for_versions.project) }
let(:plugins) { %w[one-maven-plugin three-maven-plugin] }
let(:most_recent_metadata_file_for_plugins) { versionless_package_for_plugins.package_files.recent.with_file_name(Packages::Maven::Metadata.filename).first }
context 'with a valid package name' do
before do
versionless_package_for_versions.update!(name: package_name)
metadata_package_file.update!(
file: CarrierWaveStringFile.new_file(
file_content: versions_xml_content,
filename: 'maven-metadata.xml',
content_type: 'application/xml'
)
)
metadata_package_file_for_plugins.update!(
file: CarrierWaveStringFile.new_file(
file_content: plugins_xml_content,
filename: 'maven-metadata.xml',
content_type: 'application/xml'
)
)
plugins.each do |plugin|
versions.each do |version|
pkg = create(:maven_package, name: "#{versionless_package_name_for_plugins}/#{plugin}", version: version, project: versionless_package_for_versions.project)
pkg.maven_metadatum.update!(app_name: plugin)
end
end
end
it_behaves_like 'an idempotent worker' do
let(:job_args) { [user.id, project.id, package_name] }
it 'creates the updated metadata files', :aggregate_failures do
expect { subject }.to change { ::Packages::PackageFile.count }.by(5 * 2) # the two xml files are updated
most_recent_versions = versions_from(most_recent_metadata_file_for_versions.file.read)
expect(most_recent_versions.latest).to eq('3.0-SNAPSHOT')
expect(most_recent_versions.release).to eq('2.1')
expect(most_recent_versions.versions).to match_array(versions)
it 'does not create the updated metadata files' do
expect { subject }
.to change { ::Packages::PackageFile.count }.by(0)
.and raise_error(described_class::SyncError, 'Not allowed')
plugins_from_xml = plugins_from(most_recent_metadata_file_for_plugins.file.read)
expect(plugins_from_xml).to match_array(plugins)
end
end
it 'logs the message from the service' do
expect(worker).to receive(:log_extra_metadata_on_done).with(:message, 'New metadata package file created')
subject
end
context 'not in the passed project' do
let(:project) { create(:project) }
it 'does not create the updated metadata files' do
expect { subject }
.to change { ::Packages::PackageFile.count }.by(0)
.and raise_error(described_class::SyncError, 'Non existing versionless package')
end
end
context 'with a user with not enough permissions' do
let(:role) { :guest }
it 'does not create the updated metadata files' do
expect { subject }
.to change { ::Packages::PackageFile.count }.by(0)
.and raise_error(described_class::SyncError, 'Not allowed')
end
end
end
end
......@@ -112,6 +202,12 @@ RSpec.describe Packages::Maven::Metadata::SyncWorker, type: :worker do
)
end
def plugins_from(xml_content)
xml_doc = Nokogiri::XML(xml_content)
xml_doc.xpath('//metadata/plugins/plugin/name').map(&:content)
end
def versions_xml_content
Nokogiri::XML::Builder.new do |xml|
xml.metadata do
......@@ -130,4 +226,28 @@ RSpec.describe Packages::Maven::Metadata::SyncWorker, type: :worker do
end
end.to_xml
end
def plugins_xml_content
Nokogiri::XML::Builder.new do |xml|
xml.metadata do
xml.plugins do
xml.plugin do
xml.name('one-maven-plugin')
xml.prefix('one')
xml.artifactId('one-maven-plugin')
end
xml.plugin do
xml.name('two-maven-plugin')
xml.prefix('two')
xml.artifactId('two-maven-plugin')
end
xml.plugin do
xml.name('three-maven-plugin')
xml.prefix('three')
xml.artifactId('three-maven-plugin')
end
end
end
end.to_xml
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment