Commit 6c9f6489 authored by David Fernandez's avatar David Fernandez Committed by Stan Hu

Add maven plugin support to the sync worker

Add the create plugins xml service
Update the maven metadata sync worker to also update
the `maven-metadata.xml` file for maven plugins
parent d071b6ae
......@@ -18,6 +18,13 @@ class Packages::Maven::Metadatum < ApplicationRecord
validate :maven_package_type
scope :for_package_ids, -> (package_ids) { where(package_id: package_ids) }
scope :order_created, -> { reorder('created_at ASC') }
def self.pluck_app_name
pluck(:app_name)
end
private
def maven_package_type
......
# frozen_string_literal: true
module Packages
module Maven
module Metadata
class BaseCreateXmlService
include Gitlab::Utils::StrongMemoize
INDENT_SPACE = 2
def initialize(metadata_content:, package:)
@metadata_content = metadata_content
@package = package
end
private
def xml_doc
strong_memoize(:xml_doc) do
Nokogiri::XML(@metadata_content) do |config|
config.default_xml.noblanks
end
end
end
def xml_node(name, content)
xml_doc.create_element(name).tap { |e| e.content = content }
end
end
end
end
end
# frozen_string_literal: true
module Packages
module Maven
module Metadata
class CreatePluginsXmlService < BaseCreateXmlService
XPATH_PLUGIN_ARTIFACT_ID = '//plugin/artifactId'
XPATH_PLUGINS = '//metadata/plugins'
EMPTY_PLUGINS_PAYLOAD = {
changes_exist: true,
empty_plugins: true
}.freeze
def execute
return ServiceResponse.error(message: 'package not set') unless @package
return ServiceResponse.error(message: 'metadata_content not set') unless @metadata_content
return ServiceResponse.error(message: 'metadata_content is invalid') unless plugins_xml_node.present?
return ServiceResponse.success(payload: EMPTY_PLUGINS_PAYLOAD) if plugin_artifact_ids_from_database.empty?
changes_exist = update_plugins_list
payload = { changes_exist: changes_exist, empty_versions: false }
payload[:metadata_content] = xml_doc.to_xml(indent: INDENT_SPACE) if changes_exist
ServiceResponse.success(payload: payload)
end
private
def update_plugins_list
return false if plugin_artifact_ids_from_xml == plugin_artifact_ids_from_database
plugins_xml_node.children.remove
plugin_artifact_ids_from_database.each do |artifact_id|
plugins_xml_node.add_child(plugin_node_for(artifact_id))
end
true
end
def plugins_xml_node
strong_memoize(:plugins_xml_node) do
xml_doc.xpath(XPATH_PLUGINS)
.first
end
end
def plugin_artifact_ids_from_xml
strong_memoize(:plugin_artifact_ids_from_xml) do
plugins_xml_node.xpath(XPATH_PLUGIN_ARTIFACT_ID)
.map(&:content)
end
end
def plugin_artifact_ids_from_database
strong_memoize(:plugin_artifact_ids_from_database) do
package_names = plugin_artifact_ids_from_xml.map do |artifact_id|
"#{@package.name}/#{artifact_id}"
end
packages = @package.project.packages
.maven
.displayable
.with_name(package_names)
.has_version
::Packages::Maven::Metadatum.for_package_ids(packages.select(:id))
.order_created
.pluck_app_name
.uniq
end
end
def plugin_node_for(artifact_id)
xml_doc.create_element('plugin').tap do |plugin_node|
plugin_node.add_child(xml_node('name', artifact_id))
plugin_node.add_child(xml_node('prefix', prefix_from(artifact_id)))
plugin_node.add_child(xml_node('artifactId', artifact_id))
end
end
# Maven plugin prefix generation from
# https://github.com/apache/maven/blob/c3dba0e5ba71ee7cbd62620f669a8c206e71b5e2/maven-plugin-api/src/main/java/org/apache/maven/plugin/descriptor/PluginDescriptor.java#L189
def prefix_from(artifact_id)
artifact_id.gsub(/-?maven-?/, '')
.gsub(/-?plugin-?/, '')
end
end
end
end
end
......@@ -3,9 +3,7 @@
module Packages
module Maven
module Metadata
class CreateVersionsXmlService
include Gitlab::Utils::StrongMemoize
class CreateVersionsXmlService < BaseCreateXmlService
XPATH_VERSIONING = '//metadata/versioning'
XPATH_VERSIONS = '//versions'
XPATH_VERSION = '//version'
......@@ -13,18 +11,11 @@ module Packages
XPATH_RELEASE = '//release'
XPATH_LAST_UPDATED = '//lastUpdated'
INDENT_SPACE = 2
EMPTY_VERSIONS_PAYLOAD = {
changes_exist: true,
empty_versions: true
}.freeze
def initialize(metadata_content:, package:)
@metadata_content = metadata_content
@package = package
end
def execute
return ServiceResponse.error(message: 'package not set') unless @package
return ServiceResponse.error(message: 'metadata_content not set') unless @metadata_content
......@@ -57,7 +48,7 @@ module Packages
version_xml_nodes.remove
versions_from_database.each do |version|
versions_xml_node.add_child(version_node_for(version))
versions_xml_node.add_child(xml_node('version', version))
end
true
end
......@@ -131,10 +122,6 @@ module Packages
end
end
def version_node_for(version)
Nokogiri::XML::Node.new('version', xml_doc).tap { |node| node.content = version }
end
def versions_from_xml
strong_memoize(:versions_from_xml) do
versions_xml_node.xpath(XPATH_VERSION)
......@@ -172,14 +159,6 @@ module Packages
non_snapshot_versions_from_database.last
end
end
def xml_doc
strong_memoize(:xml_doc) do
Nokogiri::XML(@metadata_content) do |config|
config.default_xml.noblanks
end
end
end
end
end
end
......
......@@ -16,26 +16,50 @@ module Packages
return error('Non existing versionless package') unless versionless_package_for_versions
return error('Non existing metadata file for versions') unless metadata_package_file_for_versions
if metadata_package_file_for_plugins
result = update_plugins_xml
return result if result.error?
end
update_versions_xml
end
private
def update_versions_xml
return error('Metadata file for versions is too big') if metadata_package_file_for_versions.size > MAX_FILE_SIZE
update_xml(
kind: :versions,
package_file: metadata_package_file_for_versions,
service_class: CreateVersionsXmlService,
payload_empty_field: :empty_versions
)
end
metadata_package_file_for_versions.file.use_open_file do |file|
result = CreateVersionsXmlService.new(metadata_content: file, package: versionless_package_for_versions)
.execute
def update_plugins_xml
update_xml(
kind: :plugins,
package_file: metadata_package_file_for_plugins,
service_class: CreatePluginsXmlService,
payload_empty_field: :empty_plugins
)
end
def update_xml(kind:, package_file:, service_class:, payload_empty_field:)
return error("Metadata file for #{kind} is too big") if package_file.size > MAX_FILE_SIZE
package_file.file.use_open_file do |file|
result = service_class.new(metadata_content: file, package: package_file.package)
.execute
next result unless result.success?
next success('No changes for versions xml') unless result.payload[:changes_exist]
next success("No changes for #{kind} xml") unless result.payload[:changes_exist]
if result.payload[:empty_versions]
versionless_package_for_versions.destroy!
success('Versionless package for versions destroyed')
if result.payload[payload_empty_field]
package_file.package.destroy!
success("Versionless package for #{kind} destroyed")
else
AppendPackageFileService.new(metadata_content: result.payload[:metadata_content], package: versionless_package_for_versions)
AppendPackageFileService.new(metadata_content: result.payload[:metadata_content], package: package_file.package)
.execute
end
end
......@@ -43,28 +67,49 @@ module Packages
def metadata_package_file_for_versions
strong_memoize(:metadata_file_for_versions) do
versionless_package_for_versions.package_files
.with_file_name(Metadata.filename)
.recent
.first
metadata_package_file_for(versionless_package_for_versions)
end
end
def versionless_package_for_versions
strong_memoize(:versionless_package_for_versions) do
project.packages
.maven
.displayable
.with_name(package_name)
.with_version(nil)
.first
versionless_package_named(package_name)
end
end
def metadata_package_file_for_plugins
strong_memoize(:metadata_package_file_for_plugins) do
metadata_package_file_for(versionless_package_named(package_name_for_plugins))
end
end
def metadata_package_file_for(package)
return unless package
package.package_files
.with_file_name(Metadata.filename)
.recent
.first
end
def versionless_package_named(name)
project.packages
.maven
.displayable
.with_name(name)
.with_version(nil)
.first
end
def package_name
params[:package_name]
end
def package_name_for_plugins
group = versionless_package_for_versions.maven_metadatum.app_group
group.tr('.', '/')
end
def error(message)
ServiceResponse.error(message: message)
end
......
---
title: Support maven plugins packaging in the maven metadata sync worker
merge_request: 56229
author:
type: fixed
......@@ -36,5 +36,38 @@ RSpec.describe Packages::Maven::Metadatum, type: :model do
expect(maven_metadatum.errors.to_a).to include('Package type must be Maven')
end
end
context 'with a package' do
let_it_be(:package) { create(:package) }
describe '.for_package_ids' do
let_it_be(:metadata) { create_list(:maven_metadatum, 3, package: package) }
subject { Packages::Maven::Metadatum.for_package_ids(package.id) }
it { is_expected.to match_array(metadata) }
end
describe '.order_created' do
let_it_be(:metadatum1) { create(:maven_metadatum, package: package) }
let_it_be(:metadatum2) { create(:maven_metadatum, package: package) }
let_it_be(:metadatum3) { create(:maven_metadatum, package: package) }
let_it_be(:metadatum4) { create(:maven_metadatum, package: package) }
subject { Packages::Maven::Metadatum.for_package_ids(package.id).order_created }
it { is_expected.to eq([metadatum1, metadatum2, metadatum3, metadatum4]) }
end
describe '.pluck_app_name' do
let_it_be(:metadatum1) { create(:maven_metadatum, package: package, app_name: 'one') }
let_it_be(:metadatum2) { create(:maven_metadatum, package: package, app_name: 'two') }
let_it_be(:metadatum3) { create(:maven_metadatum, package: package, app_name: 'three') }
subject { Packages::Maven::Metadatum.for_package_ids(package.id).pluck_app_name }
it { is_expected.to match_array([metadatum1, metadatum2, metadatum3].map(&:app_name)) }
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe ::Packages::Maven::Metadata::CreatePluginsXmlService do
let_it_be(:group_id) { 'my/test' }
let_it_be(:package) { create(:maven_package, name: group_id, version: nil) }
let(:plugins_in_database) { %w[one-maven-plugin two three-maven-plugin] }
let(:plugins_in_xml) { %w[one-maven-plugin two three-maven-plugin] }
let(:service) { described_class.new(metadata_content: metadata_xml, package: package) }
describe '#execute' do
subject { service.execute }
before do
next unless package
plugins_in_database.each do |plugin|
create(
:maven_package,
name: "#{group_id}/#{plugin}",
version: '1.0.0',
project: package.project,
maven_metadatum_attributes: {
app_group: group_id.tr('/', '.'),
app_name: plugin,
app_version: '1.0.0'
}
)
end
end
shared_examples 'returning an xml with plugins from the database' do
it 'returns an metadata versions xml with versions in the database', :aggregate_failures do
expect(subject).to be_success
expect(subject.payload[:changes_exist]).to eq(true)
expect(subject.payload[:empty_versions]).to eq(false)
expect(plugins_from(subject.payload[:metadata_content])).to match_array(plugins_in_database)
end
end
shared_examples 'returning no changes' do
it 'returns no changes', :aggregate_failures do
expect(subject).to be_success
expect(subject.payload).to eq(changes_exist: false, empty_versions: false)
end
end
context 'with same plugins on both sides' do
it_behaves_like 'returning no changes'
end
context 'with more plugins' do
let(:additional_plugins) { %w[four-maven-plugin five] }
context 'in database' do
let(:plugins_in_database) { plugins_in_xml + additional_plugins }
# we can't distinguish that the additional plugin are actually maven plugins
it_behaves_like 'returning no changes'
end
context 'in xml' do
let(:plugins_in_xml) { plugins_in_database + additional_plugins }
it_behaves_like 'returning an xml with plugins from the database'
end
end
context 'with no versions in the database' do
let(:plugins_in_database) { [] }
it 'returns a success', :aggregate_failures do
result = subject
expect(result).to be_success
expect(result.payload).to eq(changes_exist: true, empty_plugins: true)
end
end
context 'with an incomplete metadata content' do
let(:metadata_xml) { '<metadata></metadata>' }
it_behaves_like 'returning an error service response', message: 'metadata_content is invalid'
end
context 'with an invalid metadata content' do
let(:metadata_xml) { '<meta></metadata>' }
it_behaves_like 'returning an error service response', message: 'metadata_content is invalid'
end
it_behaves_like 'handling metadata content pointing to a file for the create xml service'
it_behaves_like 'handling invalid parameters for create xml service'
end
def metadata_xml
Nokogiri::XML::Builder.new do |xml|
xml.metadata do
xml.plugins do
plugins_in_xml.each do |plugin|
xml.plugin do
xml.name(plugin)
xml.prefix(prefix_from(plugin))
xml.artifactId(plugin)
end
end
end
end
end.to_xml
end
def prefix_from(artifact_id)
artifact_id.gsub(/-?maven-?/, '')
.gsub(/-?plugin-?/, '')
end
def plugins_from(xml_content)
doc = Nokogiri::XML(xml_content)
doc.xpath('//metadata/plugins/plugin/artifactId').map(&:content)
end
end
......@@ -181,59 +181,9 @@ RSpec.describe ::Packages::Maven::Metadata::CreateVersionsXmlService do
it_behaves_like 'returning an error service response', message: 'metadata_content is invalid'
end
context 'with metadata content pointing to a file' do
let(:service) { described_class.new(metadata_content: file, package: package) }
let(:file) do
Tempfile.new('metadata').tap do |file|
if file_contents
file.write(file_contents)
file.flush
file.rewind
end
end
end
after do
file.close
file.unlink
end
context 'with valid content' do
let(:file_contents) { metadata_xml }
it 'returns no changes' do
result = subject
expect(result).to be_success
expect(result.payload).to eq(changes_exist: false, empty_versions: false)
end
end
it_behaves_like 'handling metadata content pointing to a file for the create xml service'
context 'with invalid content' do
let(:file_contents) { '<meta></metadata>' }
it_behaves_like 'returning an error service response', message: 'metadata_content is invalid'
end
context 'with no content' do
let(:file_contents) { nil }
it_behaves_like 'returning an error service response', message: 'metadata_content is invalid'
end
end
context 'with no package' do
let(:metadata_xml) { '' }
let(:package) { nil }
it_behaves_like 'returning an error service response', message: 'package not set'
end
context 'with no metadata content' do
let(:metadata_xml) { nil }
it_behaves_like 'returning an error service response', message: 'metadata_content not set'
end
it_behaves_like 'handling invalid parameters for create xml service'
end
def metadata_xml
......
# frozen_string_literal: true
RSpec.shared_examples 'handling metadata content pointing to a file for the create xml service' do
context 'with metadata content pointing to a file' do
let(:service) { described_class.new(metadata_content: file, package: package) }
let(:file) do
Tempfile.new('metadata').tap do |file|
if file_contents
file.write(file_contents)
file.flush
file.rewind
end
end
end
after do
file.close
file.unlink
end
context 'with valid content' do
let(:file_contents) { metadata_xml }
it 'returns no changes' do
expect(subject).to be_success
expect(subject.payload).to eq(changes_exist: false, empty_versions: false)
end
end
context 'with invalid content' do
let(:file_contents) { '<meta></metadata>' }
it_behaves_like 'returning an error service response', message: 'metadata_content is invalid'
end
context 'with no content' do
let(:file_contents) { nil }
it_behaves_like 'returning an error service response', message: 'metadata_content is invalid'
end
end
end
RSpec.shared_examples 'handling invalid parameters for create xml service' do
context 'with no package' do
let(:metadata_xml) { '' }
let(:package) { nil }
it_behaves_like 'returning an error service response', message: 'package not set'
end
context 'with no metadata content' do
let(:metadata_xml) { nil }
it_behaves_like 'returning an error service response', message: 'metadata_content not set'
end
end
......@@ -22,18 +22,23 @@ RSpec.describe Packages::Maven::Metadata::SyncWorker, type: :worker do
subject { worker.perform(user.id, project.id, package_name) }
context 'with a valid package name' do
before do
file = CarrierWaveStringFile.new_file(file_content: versions_xml_content, filename: 'maven-metadata.xml', content_type: 'application/xml')
metadata_package_file.update!(file: file)
versions.each do |version|
create(:maven_package, name: versionless_package_for_versions.name, version: version, project: versionless_package_for_versions.project)
context 'with a jar' do
context 'with a valid package name' do
before do
metadata_package_file.update!(
file: CarrierWaveStringFile.new_file(
file_content: versions_xml_content,
filename: 'maven-metadata.xml',
content_type: 'application/xml'
)
)
versions.each do |version|
create(:maven_package, name: versionless_package_for_versions.name, version: version, project: versionless_package_for_versions.project)
end
end
end
context 'idempotent worker' do
include_examples 'an idempotent worker' do
it_behaves_like 'an idempotent worker' do
let(:job_args) { [user.id, project.id, package_name] }
it 'creates the updated metadata files', :aggregate_failures do
......@@ -45,31 +50,116 @@ RSpec.describe Packages::Maven::Metadata::SyncWorker, type: :worker do
expect(most_recent_versions.versions).to match_array(versions)
end
end
end
it 'logs the message from the service' do
expect(worker).to receive(:log_extra_metadata_on_done).with(:message, 'New metadata package file created')
it 'logs the message from the service' do
expect(worker).to receive(:log_extra_metadata_on_done).with(:message, 'New metadata package file created')
subject
end
subject
end
context 'not in the passed project' do
let(:project) { create(:project) }
context 'not in the passed project' do
let(:project) { create(:project) }
it 'does not create the updated metadata files' do
expect { subject }
.to change { ::Packages::PackageFile.count }.by(0)
.and raise_error(described_class::SyncError, 'Non existing versionless package')
it 'does not create the updated metadata files' do
expect { subject }
.to change { ::Packages::PackageFile.count }.by(0)
.and raise_error(described_class::SyncError, 'Non existing versionless package')
end
end
context 'with a user with not enough permissions' do
let(:role) { :guest }
it 'does not create the updated metadata files' do
expect { subject }
.to change { ::Packages::PackageFile.count }.by(0)
.and raise_error(described_class::SyncError, 'Not allowed')
end
end
end
end
context 'with a user with not enough permissions' do
let(:role) { :guest }
context 'with a maven plugin' do
let_it_be(:versionless_package_name_for_plugins) { versionless_package_for_versions.maven_metadatum.app_group.tr('.', '/') }
let_it_be(:versionless_package_for_versions) { create(:maven_package, name: "#{versionless_package_name_for_plugins}/one-maven-plugin", version: nil) }
let_it_be(:metadata_package_file) { create(:package_file, :xml, package: versionless_package_for_versions) }
let_it_be(:versionless_package_for_plugins) { create(:maven_package, name: versionless_package_name_for_plugins, version: nil, project: versionless_package_for_versions.project) }
let_it_be(:metadata_package_file_for_plugins) { create(:package_file, :xml, package: versionless_package_for_plugins) }
let_it_be(:addtional_maven_package_for_same_group_id) { create(:maven_package, name: "#{versionless_package_name_for_plugins}/maven-package", project: versionless_package_for_versions.project) }
let(:plugins) { %w[one-maven-plugin three-maven-plugin] }
let(:most_recent_metadata_file_for_plugins) { versionless_package_for_plugins.package_files.recent.with_file_name(Packages::Maven::Metadata.filename).first }
context 'with a valid package name' do
before do
versionless_package_for_versions.update!(name: package_name)
metadata_package_file.update!(
file: CarrierWaveStringFile.new_file(
file_content: versions_xml_content,
filename: 'maven-metadata.xml',
content_type: 'application/xml'
)
)
metadata_package_file_for_plugins.update!(
file: CarrierWaveStringFile.new_file(
file_content: plugins_xml_content,
filename: 'maven-metadata.xml',
content_type: 'application/xml'
)
)
plugins.each do |plugin|
versions.each do |version|
pkg = create(:maven_package, name: "#{versionless_package_name_for_plugins}/#{plugin}", version: version, project: versionless_package_for_versions.project)
pkg.maven_metadatum.update!(app_name: plugin)
end
end
end
it_behaves_like 'an idempotent worker' do
let(:job_args) { [user.id, project.id, package_name] }
it 'creates the updated metadata files', :aggregate_failures do
expect { subject }.to change { ::Packages::PackageFile.count }.by(5 * 2) # the two xml files are updated
most_recent_versions = versions_from(most_recent_metadata_file_for_versions.file.read)
expect(most_recent_versions.latest).to eq('3.0-SNAPSHOT')
expect(most_recent_versions.release).to eq('2.1')
expect(most_recent_versions.versions).to match_array(versions)
it 'does not create the updated metadata files' do
expect { subject }
.to change { ::Packages::PackageFile.count }.by(0)
.and raise_error(described_class::SyncError, 'Not allowed')
plugins_from_xml = plugins_from(most_recent_metadata_file_for_plugins.file.read)
expect(plugins_from_xml).to match_array(plugins)
end
end
it 'logs the message from the service' do
expect(worker).to receive(:log_extra_metadata_on_done).with(:message, 'New metadata package file created')
subject
end
context 'not in the passed project' do
let(:project) { create(:project) }
it 'does not create the updated metadata files' do
expect { subject }
.to change { ::Packages::PackageFile.count }.by(0)
.and raise_error(described_class::SyncError, 'Non existing versionless package')
end
end
context 'with a user with not enough permissions' do
let(:role) { :guest }
it 'does not create the updated metadata files' do
expect { subject }
.to change { ::Packages::PackageFile.count }.by(0)
.and raise_error(described_class::SyncError, 'Not allowed')
end
end
end
end
......@@ -112,6 +202,12 @@ RSpec.describe Packages::Maven::Metadata::SyncWorker, type: :worker do
)
end
def plugins_from(xml_content)
xml_doc = Nokogiri::XML(xml_content)
xml_doc.xpath('//metadata/plugins/plugin/name').map(&:content)
end
def versions_xml_content
Nokogiri::XML::Builder.new do |xml|
xml.metadata do
......@@ -130,4 +226,28 @@ RSpec.describe Packages::Maven::Metadata::SyncWorker, type: :worker do
end
end.to_xml
end
def plugins_xml_content
Nokogiri::XML::Builder.new do |xml|
xml.metadata do
xml.plugins do
xml.plugin do
xml.name('one-maven-plugin')
xml.prefix('one')
xml.artifactId('one-maven-plugin')
end
xml.plugin do
xml.name('two-maven-plugin')
xml.prefix('two')
xml.artifactId('two-maven-plugin')
end
xml.plugin do
xml.name('three-maven-plugin')
xml.prefix('three')
xml.artifactId('three-maven-plugin')
end
end
end
end.to_xml
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment