Merge branch 'remove-pseudo-manifest' into 'master'

Remove pseudonymization from manifest

See merge request gitlab-org/gitlab-ee!8213

config/pseudonymizer.yml

@@ -6,10 +6,6 @@ tables:
     - user_id
     - created_at
     - updated_at
-    pseudo:
-    - id
-    - merge_request_id
-    - user_id
   approver_groups:
     whitelist:
     - id
@@ -17,27 +13,16 @@ tables:
     - group_id
     - created_at
     - updated_at
-    pseudo:
-    - id
-    - group_id
   board_assignees:
     whitelist:
     - id
     - board_id
     - assignee_id
-    pseudo:
-    - id
-    - board_id
-    - assignee_id
   board_labels:
     whitelist:
     - id
     - board_id
     - label_id
-    pseudo:
-    - id
-    - board_id
-    - label_id
   boards:
     whitelist:
     - id
@@ -47,30 +32,18 @@ tables:
     - milestone_id
     - group_id
     - weight
-    pseudo:
-    - id
-    - project_id
-    - milestone_id
-    - group_id
   epic_issues:
     whitelist:
     - id
     - epic_id
     - issue_id
     - relative_position
-    pseudo:
-    - id
-    - epic_id
-    - issue_id
   epic_metrics:
     whitelist:
     - id
     - epic_id
     - created_at
     - updated_at
-    pseudo:
-    - id
-    - epic_id
   epics:
     whitelist:
     - id
@@ -89,24 +62,10 @@ tables:
     - updated_at
     - title
     - description
-    pseudo:
-    - id
-    - milestone_id
-    - group_id
-    - author_id
-    - assignee_id
-    - iid
-    - updated_by_id
-    - last_edited_by_id
-    - title
-    - description
   issue_assignees:
     whitelist:
     - user_id
     - issue_id
-    pseudo:
-    - user_id
-    - issue_id
   issue_links:
     whitelist:
     - id
@@ -114,10 +73,6 @@ tables:
     - target_id
     - created_at
     - updated_at
-    pseudo:
-    - id
-    - source_id
-    - target_id
   issue_metrics:
     whitelist:
     - id
@@ -127,9 +82,6 @@ tables:
     - first_added_to_board_at
     - created_at
     - updated_at
-    pseudo:
-    - id
-    - issue_id
   issues:
     whitelist:
     - id
@@ -137,6 +89,7 @@ tables:
     - author_id
     - project_id
     - created_at
+    - confidential
     - updated_at
     - description
     - milestone_id
@@ -151,16 +104,6 @@ tables:
     - last_edited_by_id
     - discussion_locked
     - closed_at
-    pseudo:
-    - id
-    - title
-    - author_id
-    - project_id
-    - description
-    - milestone_id
-    - updated_by_id
-    - moved_to_id
-    - discussion_locked
   label_links:
     whitelist:
     - id
@@ -169,10 +112,6 @@ tables:
     - target_type
     - created_at
     - updated_at
-    pseudo:
-    - id
-    - label_id
-    - target_id
   label_priorities:
     whitelist:
     - id
@@ -181,10 +120,6 @@ tables:
     - priority
     - created_at
     - updated_at
-    pseudo:
-    - id
-    - project_id
-    - label_id
   labels:
     whitelist:
     - id
@@ -196,21 +131,11 @@ tables:
     - template
     - type
     - group_id
-    pseudo:
-    - id
-    - title
-    - color
-    - project_id
-    - template
-    - type
-    - group_id
   licenses:
     whitelist:
     - id
     - created_at
     - updated_at
-    pseudo:
-    - id
   merge_request_diffs:
     whitelist:
     - id
@@ -223,12 +148,6 @@ tables:
     - head_commit_sha
     - start_commit_sha
     - commits_count
-    pseudo:
-    - id
-    - merge_request_id
-    - base_commit_sha
-    - head_commit_sha
-    - start_commit_sha
   merge_request_metrics:
     whitelist:
     - id
@@ -243,12 +162,6 @@ tables:
     - merged_by_id
     - latest_closed_by_id
     - latest_closed_at
-    pseudo:
-    - id
-    - merge_request_id
-    - pipeline_id
-    - merged_by_id
-    - latest_closed_by_id
   merge_requests:
     whitelist:
     - id
@@ -278,20 +191,6 @@ tables:
     - discussion_locked
     - latest_merge_request_diff_id
     - allow_maintainer_to_push
-    pseudo:
-    - id
-    - target_branch
-    - source_branch
-    - source_project_id
-    - author_id
-    - assignee_id
-    - milestone_id
-    - target_project_id
-    - updated_by_id
-    - merge_user_id
-    - last_edited_by_id
-    - head_pipeline_id
-    - latest_merge_request_diff_id
   merge_requests_closing_issues:
     whitelist:
     - id
@@ -299,10 +198,6 @@ tables:
     - issue_id
     - created_at
     - updated_at
-    pseudo:
-    - id
-    - merge_request_id
-    - issue_id
   milestones:
     whitelist:
     - id
@@ -313,19 +208,12 @@ tables:
     - state
     - start_date
     - group_id
-    pseudo:
-    - id
-    - project_id
-    - group_id
   namespace_statistics:
     whitelist:
     - id
     - namespace_id
     - shared_runners_seconds
     - shared_runners_seconds_last_reset
-    pseudo:
-    - id
-    - namespace_id
   namespaces:
     whitelist:
     - id
@@ -353,13 +241,6 @@ tables:
     - two_factor_grace_period
     - plan_id
     - project_creation_level
-    pseudo:
-    - id
-    - name
-    - path
-    - owner_id
-    - parent_id
-    - plan_id
   members:
     whitelist:
     - id
@@ -376,12 +257,6 @@ tables:
     - expires_at
     - ldap
     - override
-    pseudo:
-    - id
-    - source_id
-    - user_id
-    - created_by_id
-    - invite_email
   notes:
     whitelist:
     - id
@@ -404,16 +279,6 @@ tables:
     - discussion_id
     - change_position
     - resolved_by_push
-    pseudo:
-    - id
-    - note
-    - author_id
-    - project_id
-    - commit_id
-    - noteable_id
-    - updated_by_id
-    - resolved_by_id
-    - discussion_id
   notification_settings:
     whitelist:
     - id
@@ -435,33 +300,11 @@ tables:
     - merge_merge_request
     - failed_pipeline
     - success_pipeline
-    pseudo:
-    - id
-    - user_id
-    - source_id
-    - source_type
-    - level
-    - new_note
-    - new_issue
-    - reopen_issue
-    - close_issue
-    - reassign_issue
-    - new_merge_request
-    - reopen_merge_request
-    - close_merge_request
-    - reassign_merge_request
-    - merge_merge_request
-    - failed_pipeline
-    - success_pipeline
   project_authorizations:
     whitelist:
     - user_id
     - project_id
     - access_level
-    pseudo:
-    - user_id
-    - project_id
-    - access_level
   project_auto_devops:
     whitelist:
     - id
@@ -470,11 +313,6 @@ tables:
     - updated_at
     - enabled
     - domain
-    pseudo:
-    - id
-    - project_id
-    - enabled
-    - domain
   project_custom_attributes:
     whitelist:
     - id
@@ -483,11 +321,6 @@ tables:
     - project_id
     - key
     - value
-    pseudo:
-    - id
-    - project_id
-    - key
-    - value
   project_features:
     whitelist:
     - id
@@ -500,15 +333,6 @@ tables:
     - created_at
     - updated_at
     - repository_access_level
-    pseudo:
-    - id
-    - project_id
-    - merge_requests_access_level
-    - issues_access_level
-    - wiki_access_level
-    - snippets_access_level
-    - builds_access_level
-    - repository_access_level
   project_group_links:
     whitelist:
     - id
@@ -518,18 +342,10 @@ tables:
     - updated_at
     - group_access
     - expires_at
-    pseudo:
-    - id
-    - project_id
-    - group_id
-    - group_access
   project_import_data:
     whitelist:
     - id
     - project_id
-    pseudo:
-    - id
-    - project_id
   project_mirror_data:
     whitelist:
     - id
@@ -538,9 +354,6 @@ tables:
     - last_update_started_at
     - last_update_scheduled_at
     - next_execution_timestamp
-    pseudo:
-    - id
-    - project_id
   project_repository_states:
     whitelist:
     - id
@@ -549,13 +362,6 @@ tables:
     - wiki_verification_checksum
     - last_repository_verification_failure
     - last_wiki_verification_failure
-    pseudo:
-    - id
-    - project_id
-    - repository_verification_checksum
-    - wiki_verification_checksum
-    - last_repository_verification_failure
-    - last_wiki_verification_failure
   project_statistics:
     whitelist:
     - id
@@ -568,17 +374,6 @@ tables:
     - build_artifacts_size
     - shared_runners_seconds
     - shared_runners_seconds_last_reset
-    pseudo:
-    - id
-    - project_id
-    - namespace_id
-    - commit_count
-    - storage_size
-    - repository_size
-    - lfs_objects_size
-    - build_artifacts_size
-    - shared_runners_seconds
-    - shared_runners_seconds_last_reset
   projects:
     whitelist:
     - id
@@ -640,62 +435,6 @@ tables:
     - pull_mirror_available_overridden
     - mirror_overwrites_diverged_branches
     - external_authorization_classification_label
-    pseudo:
-    - id
-    - name
-    - path
-    - description
-    - creator_id
-    - namespace_id
-    - import_url
-    - visibility_level
-    - archived
-    - avatar
-    - import_status
-    - merge_requests_template
-    - star_count
-    - import_type
-    - import_source
-    - approvals_before_merge
-    - reset_approvals_on_push
-    - issues_template
-    - mirror
-    - mirror_user_id
-    - ci_id
-    - shared_runners_enabled
-    - build_coverage_regex
-    - build_allow_git_fetch
-    - build_timeout
-    - mirror_trigger_builds
-    - pending_delete
-    - public_builds
-    - last_repository_check_failed
-    - only_allow_merge_if_pipeline_succeeds
-    - repository_storage
-    - repository_read_only
-    - ci_config_path
-    - only_allow_merge_if_all_discussions_are_resolved
-    - repository_size_limit
-    - auto_cancel_pending_pipelines
-    - delete_error
-    - last_repository_updated_at
-    - disable_overriding_approvers_per_merge_request
-    - storage_version
-    - resolve_outdated_diff_discussions
-    - remote_mirror_available_overridden
-    - pull_mirror_available_overridden
-    - mirror_overwrites_diverged_branches
-    - external_authorization_classification_label
-  project_mirror_data:
-    whitelist:
-      - last_update_at
-      - last_successful_update_at
-      - last_error
-      - jid
-    pseudo:
-      - last_successful_update_at
-      - last_error
-      - jid
   subscriptions:
     whitelist:
     - id
@@ -706,15 +445,9 @@ tables:
     - created_at
     - updated_at
     - project_id
-    pseudo:
-    - id
-    - user_id
-    - subscribable_id
-    - project_id
   users:
     whitelist:
     - id
-    - email
     - remember_created_at
     - sign_in_count
     - current_sign_in_at
@@ -723,16 +456,10 @@ tables:
     - last_sign_in_ip
     - created_at
     - updated_at
-    - name
     - admin
     - projects_limit
-    - skype
-    - linkedin
-    - twitter
-    - bio
     - failed_attempts
     - locked_at
-    - username
     - can_create_group
     - can_create_team
     - state
@@ -770,30 +497,4 @@ tables:
     - support_bot
     - preferred_language
     - theme_id
-    pseudo:
-    - id
-    - email
-    - current_sign_in_ip
-    - last_sign_in_ip
-    - name
-    - admin
-    - skype
-    - linkedin
-    - twitter
-    - username
-    - created_by_id
-    - avatar
-    - unconfirmed_email
-    - hide_no_ssh_key
-    - website_url
-    - notification_email
-    - location
-    - public_email
-    - consumed_timestep
-    - hide_project_limit
-    - note
-    - external
-    - organization
-    - auditor
-    - two_factor_grace_period
-    - theme_id
+

ee/spec/lib/pseudonymizer/dumper_spec.rb

@@ -76,7 +76,7 @@ describe Pseudonymizer::Dumper do
 
     it "all whitelisted attributes exist" do
       options.config['tables'].each do |table, table_def|
-        whitelisted = table_def['whitelist']
+        whitelisted = table_def.fetch('whitelist', [])
         existing_columns = ActiveRecord::Base.connection.columns(table.to_sym).map(&:name)
         diff = whitelisted - existing_columns
 
@@ -86,8 +86,8 @@ describe Pseudonymizer::Dumper do
 
     it "all pseudonymized attributes are whitelisted" do
       options.config['tables'].each do |table, table_def|
-        whitelisted = table_def['whitelist']
-        pseudonymized = table_def['pseudo']
+        whitelisted = table_def.fetch('whitelist', [])
+        pseudonymized = table_def.fetch('pseudo', [])
         diff = pseudonymized - whitelisted
 
         expect(diff).to be_empty, "#{table} should whitelist columns #{pseudonymized.inspect}: missing #{diff.inspect}"