Commit b17998b4 authored by Mayra Cabrera's avatar Mayra Cabrera

Merge branch '29403-migrate-issue-tracker-data' into 'master'

Migrate issue trackers data

See merge request gitlab-org/gitlab!18639
parents 0008cffa 942a61d2
---
title: Migrate issue trackers data
merge_request: 18639
author:
type: other
# frozen_string_literal: true
class MigrateIssueTrackersData < ActiveRecord::Migration[5.1]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
INTERVAL = 3.minutes.to_i
BATCH_SIZE = 5_000
MIGRATION = 'MigrateIssueTrackersSensitiveData'
disable_ddl_transaction!
class Service < ActiveRecord::Base
self.table_name = 'services'
self.inheritance_column = :_type_disabled
include ::EachBatch
end
def up
relation = Service.where(category: 'issue_tracker').where("properties IS NOT NULL AND properties != '{}' AND properties != ''")
queue_background_migration_jobs_by_range_at_intervals(relation,
MIGRATION,
INTERVAL,
batch_size: BATCH_SIZE)
end
def down
# no need
end
end
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# This migration takes all issue trackers
# and move data from properties to data field tables (jira_tracker_data and issue_tracker_data)
class MigrateIssueTrackersSensitiveData
delegate :select_all, :execute, :quote_string, to: :connection
# we need to define this class and set fields encryption
class IssueTrackerData < ApplicationRecord
self.table_name = 'issue_tracker_data'
def self.encryption_options
{
key: Settings.attr_encrypted_db_key_base_32,
encode: true,
mode: :per_attribute_iv,
algorithm: 'aes-256-gcm'
}
end
attr_encrypted :project_url, encryption_options
attr_encrypted :issues_url, encryption_options
attr_encrypted :new_issue_url, encryption_options
end
# we need to define this class and set fields encryption
class JiraTrackerData < ApplicationRecord
self.table_name = 'jira_tracker_data'
def self.encryption_options
{
key: Settings.attr_encrypted_db_key_base_32,
encode: true,
mode: :per_attribute_iv,
algorithm: 'aes-256-gcm'
}
end
attr_encrypted :url, encryption_options
attr_encrypted :api_url, encryption_options
attr_encrypted :username, encryption_options
attr_encrypted :password, encryption_options
end
def perform(start_id, stop_id)
columns = 'id, properties, title, description, type'
batch_condition = "id >= #{start_id} AND id <= #{stop_id} AND category = 'issue_tracker' \
AND properties IS NOT NULL AND properties != '{}' AND properties != ''"
data_subselect = "SELECT 1 \
FROM jira_tracker_data \
WHERE jira_tracker_data.service_id = services.id \
UNION SELECT 1 \
FROM issue_tracker_data \
WHERE issue_tracker_data.service_id = services.id"
query = "SELECT #{columns} FROM services WHERE #{batch_condition} AND NOT EXISTS (#{data_subselect})"
migrated_ids = []
data_to_insert(query).each do |table, data|
service_ids = data.map { |s| s['service_id'] }
next if service_ids.empty?
migrated_ids += service_ids
Gitlab::Database.bulk_insert(table, data)
end
return if migrated_ids.empty?
move_title_description(migrated_ids)
end
private
def data_to_insert(query)
data = { 'jira_tracker_data' => [], 'issue_tracker_data' => [] }
select_all(query).each do |service|
begin
properties = JSON.parse(service['properties'])
rescue JSON::ParserError
logger.warn(
message: 'Properties data not parsed - invalid json',
service_id: service['id'],
properties: service['properties']
)
next
end
if service['type'] == 'JiraService'
row = data_row(JiraTrackerData, jira_mapping(properties), service)
key = 'jira_tracker_data'
else
row = data_row(IssueTrackerData, issue_tracker_mapping(properties), service)
key = 'issue_tracker_data'
end
data[key] << row if row
end
data
end
def data_row(klass, mapping, service)
base_params = { service_id: service['id'], created_at: Time.current, updated_at: Time.current }
klass.new(mapping).slice(*klass.column_names).compact.merge(base_params)
end
def move_title_description(service_ids)
query = "UPDATE services SET \
title = cast(properties as json)->>'title', \
description = cast(properties as json)->>'description' \
WHERE id IN (#{service_ids.join(',')}) AND title IS NULL AND description IS NULL"
execute(query)
end
def jira_mapping(properties)
{
url: properties['url'],
api_url: properties['api_url'],
username: properties['username'],
password: properties['password']
}
end
def issue_tracker_mapping(properties)
{
project_url: properties['project_url'],
issues_url: properties['issues_url'],
new_issue_url: properties['new_issue_url']
}
end
def connection
@connection ||= ActiveRecord::Base.connection
end
def logger
@logger ||= Gitlab::BackgroundMigration::Logger.build
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20190924152703_migrate_issue_trackers_data.rb')
describe MigrateIssueTrackersData, :migration do
let(:services) { table(:services) }
let(:migration_class) { Gitlab::BackgroundMigration::MigrateIssueTrackersSensitiveData }
let(:migration_name) { migration_class.to_s.demodulize }
let(:properties) do
{
'url' => 'http://example.com'
}
end
let!(:jira_service) do
services.create(id: 10, type: 'JiraService', properties: properties, category: 'issue_tracker')
end
let!(:jira_service_nil) do
services.create(id: 11, type: 'JiraService', properties: nil, category: 'issue_tracker')
end
let!(:bugzilla_service) do
services.create(id: 12, type: 'BugzillaService', properties: properties, category: 'issue_tracker')
end
let!(:youtrack_service) do
services.create(id: 13, type: 'YoutrackService', properties: properties, category: 'issue_tracker')
end
let!(:youtrack_service_empty) do
services.create(id: 14, type: 'YoutrackService', properties: '', category: 'issue_tracker')
end
let!(:gitlab_service) do
services.create(id: 15, type: 'GitlabIssueTrackerService', properties: properties, category: 'issue_tracker')
end
let!(:gitlab_service_empty) do
services.create(id: 16, type: 'GitlabIssueTrackerService', properties: {}, category: 'issue_tracker')
end
let!(:other_service) do
services.create(id: 17, type: 'OtherService', properties: properties, category: 'other_category')
end
before do
stub_const("#{described_class}::BATCH_SIZE", 2)
end
it 'schedules background migrations at correct time' do
Sidekiq::Testing.fake! do
Timecop.freeze do
migrate!
expect(migration_name).to be_scheduled_delayed_migration(3.minutes, jira_service.id, bugzilla_service.id)
expect(migration_name).to be_scheduled_delayed_migration(6.minutes, youtrack_service.id, gitlab_service.id)
expect(BackgroundMigrationWorker.jobs.size).to eq(2)
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment