Commit 4e51c444 authored by Stan Hu's avatar Stan Hu

Merge branch 'ashmckenzie/7516-add-qa-tests-for-geo-push-to-primary-proxy-with-lfs' into 'master'

Geo: Add LFS support to HTTP & SSH push to secondary specs

See merge request gitlab-org/gitlab-ee!7808
parents 0acf7439 adcd7520
......@@ -11,14 +11,15 @@ module QA
class Repository
include Scenario::Actable
attr_writer :password
attr_writer :password, :use_lfs
attr_accessor :env_vars
def initialize
# We set HOME to the current working directory (which is a
# temporary directory created in .perform()) so the temporarily dropped
# .netrc can be utilised
self.env_vars = [%Q{HOME="#{File.dirname(netrc_file_path)}"}]
self.env_vars = [%Q{HOME="#{tmp_home_dir}"}]
@use_lfs = false
end
def self.perform(*args)
......@@ -33,25 +34,33 @@ module QA
def username=(username)
@username = username
@uri.user = username
# Only include the user in the URI if we're using HTTP as this breaks
# SSH authentication.
@uri.user = username unless ssh_key_set?
end
def use_default_credentials
self.username, self.password = default_credentials
add_credentials_to_netrc unless ssh_key_set?
# Write out .netrc as we need it for:
#
# git & git-lfs over HTTP
# git-lfs over SSH
add_credentials_to_netrc if add_credentials?
end
def clone(opts = '')
run("git clone #{opts} #{uri} ./")
end
clone_result = run("git clone #{opts} #{uri} ./")
return clone_result.response unless clone_result.success
enable_lfs_result = enable_lfs if use_lfs?
def checkout(branch_name)
run(%Q{git checkout "#{branch_name}"})
clone_result.to_s + enable_lfs_result.to_s
end
def checkout_new_branch(branch_name)
run(%Q{git checkout -b "#{branch_name}"})
def checkout(branch_name, new_branch: false)
opts = new_branch ? '-b' : ''
run(%Q{git checkout #{opts} "#{branch_name}"}).to_s
end
def shallow_clone
......@@ -61,8 +70,6 @@ module QA
def configure_identity(name, email)
run(%Q{git config user.name #{name}})
run(%Q{git config user.email #{email}})
add_credentials_to_netrc
end
def commit_file(name, contents, message)
......@@ -73,19 +80,26 @@ module QA
def add_file(name, contents)
::File.write(name, contents)
run(%Q{git add #{name}})
if use_lfs?
git_lfs_track_result = run(%Q{git lfs track #{name} --lockable})
return git_lfs_track_result.response unless git_lfs_track_result.success
end
git_add_result = run(%Q{git add #{name}})
git_lfs_track_result.to_s + git_add_result.to_s
end
def commit(message)
run(%Q{git commit -m "#{message}"})
run(%Q{git commit -m "#{message}"}).to_s
end
def push_changes(branch = 'master')
run("git push #{uri} #{branch}")
run("git push #{uri} #{branch}").to_s
end
def commits
run('git log --oneline').split("\n")
run('git log --oneline').to_s.split("\n")
end
def use_ssh_key(key)
......@@ -97,7 +111,8 @@ module QA
keyscan_params = ['-H']
keyscan_params << "-p #{uri.port}" if uri.port
keyscan_params << uri.host
run("ssh-keyscan #{keyscan_params.join(' ')} >> #{known_hosts_file.path}")
res = run("ssh-keyscan #{keyscan_params.join(' ')} >> #{known_hosts_file.path}")
return res.response unless res.success?
self.env_vars << %Q{GIT_SSH_COMMAND="ssh -i #{private_key_file.path} -o UserKnownHostsFile=#{known_hosts_file.path}"}
end
......@@ -133,21 +148,47 @@ module QA
private
attr_reader :uri, :username, :password, :known_hosts_file, :private_key_file
attr_reader :uri, :username, :password, :known_hosts_file,
:private_key_file, :use_lfs
alias_method :use_lfs?, :use_lfs
Result = Struct.new(:success, :response) do
alias_method :success?, :success
alias_method :to_s, :response
end
def add_credentials?
return true unless ssh_key_set?
return true if ssh_key_set? && use_lfs?
false
end
def ssh_key_set?
!private_key_file.nil?
end
def enable_lfs
# git lfs install *needs* a .gitconfig defined at ${HOME}/.gitconfig
FileUtils.mkdir_p(tmp_home_dir)
touch_gitconfig_result = run("touch #{tmp_home_dir}/.gitconfig")
return touch_gitconfig_result.response unless touch_gitconfig_result.success?
git_lfs_install_result = run('git lfs install')
touch_gitconfig_result.to_s + git_lfs_install_result.to_s
end
def run(command_str, *extra_env)
command = [env_vars, *extra_env, command_str, '2>&1'].compact.join(' ')
Runtime::Logger.debug "Git: command=[#{command}]"
Runtime::Logger.debug "Git: pwd=[#{Dir.pwd}], command=[#{command}]"
output, _ = Open3.capture2(command)
output = output.chomp.gsub(/\s+$/, '')
Runtime::Logger.debug "Git: output=[#{output}]"
output, status = Open3.capture2e(command)
output.chomp!
Runtime::Logger.debug "Git: output=[#{output}], exitstatus=[#{status.exitstatus}]"
output
Result.new(status.exitstatus == 0, output)
end
def default_credentials
......@@ -158,12 +199,12 @@ module QA
end
end
def tmp_netrc_directory
@tmp_netrc_directory ||= File.join(Dir.tmpdir, "qa-netrc-credentials", $$.to_s)
def tmp_home_dir
@tmp_home_dir ||= File.join(Dir.tmpdir, "qa-netrc-credentials", $$.to_s)
end
def netrc_file_path
@netrc_file_path ||= File.join(tmp_netrc_directory, '.netrc')
@netrc_file_path ||= File.join(tmp_home_dir, '.netrc')
end
def netrc_content
......@@ -185,7 +226,7 @@ module QA
#
return if netrc_already_contains_content?
FileUtils.mkdir_p(tmp_netrc_directory)
FileUtils.mkdir_p(tmp_home_dir)
File.open(netrc_file_path, 'a') { |file| file.puts(netrc_content) }
File.chmod(0600, netrc_file_path)
end
......
......@@ -8,7 +8,7 @@ module QA
class Push < Base
attr_accessor :file_name, :file_content, :commit_message,
:branch_name, :new_branch, :output, :repository_http_uri,
:repository_ssh_uri, :ssh_key, :user
:repository_ssh_uri, :ssh_key, :user, :use_lfs
attr_writer :remote_branch
......@@ -20,6 +20,7 @@ module QA
@new_branch = true
@repository_http_uri = ""
@ssh_key = nil
@use_lfs = false
end
def remote_branch
......@@ -44,6 +45,8 @@ module QA
def fabricate!
Git::Repository.perform do |repository|
@output = ''
if ssh_key
repository.uri = repository_ssh_uri
repository.use_ssh_key(ssh_key)
......@@ -52,6 +55,8 @@ module QA
repository.use_default_credentials unless user
end
repository.use_lfs = use_lfs
username = 'GitLab QA'
email = 'root@gitlab.com'
......@@ -62,29 +67,25 @@ module QA
email = user.email
end
repository.clone
@output += repository.clone
repository.configure_identity(username, email)
if new_branch
repository.checkout_new_branch(branch_name)
else
repository.checkout(branch_name)
end
@output += repository.checkout(branch_name, new_branch: new_branch)
if @directory
@directory.each_child do |f|
repository.add_file(f.basename, f.read) if f.file?
@output += repository.add_file(f.basename, f.read) if f.file?
end
elsif @files
@files.each do |f|
repository.add_file(f[:name], f[:content])
end
else
repository.add_file(file_name, file_content)
@output += repository.add_file(file_name, file_content)
end
repository.commit(commit_message)
@output = repository.push_changes("#{branch_name}:#{remote_branch}")
@output += repository.commit(commit_message)
@output += repository.push_changes("#{branch_name}:#{remote_branch}")
repository.delete_ssh_key
end
......
......@@ -3,8 +3,10 @@
module QA
context 'Geo', :orchestrated, :geo do
describe 'GitLab HTTP push' do
let(:file_name) { 'README.md' }
context 'regular git commit' do
it 'is replicated to the secondary' do
file_name = 'README.md'
file_content = 'This is a Geo project! Commit from primary.'
Runtime::Browser.visit(:geo_primary, QA::Page::Main::Login) do
......@@ -60,5 +62,69 @@ module QA
end
end
end
context 'git-lfs commit' do
it 'is replicated to the secondary' do
file_content = 'This is a Geo project!'
lfs_file_content = 'The rendered file could not be displayed because it is stored in LFS.'
Runtime::Browser.visit(:geo_primary, QA::Page::Main::Login) do
# Visit the primary node and login
Page::Main::Login.act { sign_in_using_credentials }
# Create a new Project
project = Resource::Project.fabricate! do |project|
project.name = 'geo-project'
project.description = 'Geo test project'
end
# Perform a git push over HTTP directly to the primary
push = Resource::Repository::ProjectPush.fabricate! do |push|
push.use_lfs = true
push.project = project
push.file_name = file_name
push.file_content = "# #{file_content}"
push.commit_message = 'Add README.md'
end
project.visit!
expect(push.output).to match(/Locking support detected on remote/)
expect(push.output).to match(%r{Uploading LFS objects: 100% \(1/1\)})
# Validate git push worked and file exists with content
Page::Project::Show.perform do |show|
show.wait_for_repository_replication
expect(page).to have_content(file_name)
expect(page).to have_content(lfs_file_content)
end
Runtime::Browser.visit(:geo_secondary, QA::Page::Main::Login) do
# Visit the secondary node and login
Page::Main::OAuth.act { authorize! if needs_authorization? }
EE::Page::Main::Banner.perform do |banner|
expect(banner).to have_secondary_read_only_banner
end
Page::Main::Menu.perform { |menu| menu.go_to_projects }
Page::Dashboard::Projects.perform do |dashboard|
dashboard.wait_for_project_replication(project.name)
dashboard.go_to_project(project.name)
end
# Validate the content has been sync'd from the primary
Page::Project::Show.perform do |show|
show.wait_for_repository_replication_with(file_name)
expect(page).to have_content(file_name)
expect(page).to have_content(lfs_file_content)
end
end
end
end
end
end
end
end
......@@ -2,11 +2,13 @@
module QA
context 'Geo', :orchestrated, :geo do
describe 'GitLab HTTP push to secondary' do
describe 'GitLab Geo HTTP push secondary' do
let(:file_content_primary) { 'This is a Geo project! Commit from primary.' }
let(:file_content_secondary) { 'This is a Geo project! Commit from secondary.' }
context 'regular git commit' do
it 'is redirected to the primary and ultimately replicated to the secondary' do
file_name = 'README.md'
file_content_primary = 'This is a Geo project! Commit from primary.'
file_content_secondary = 'This is a Geo project! Commit from secondary.'
Runtime::Browser.visit(:geo_primary, QA::Page::Main::Login) do
# Visit the primary node and login
......@@ -52,7 +54,7 @@ module QA
end
# Perform a git push over HTTP at the secondary
Resource::Repository::Push.fabricate! do |push|
push = Resource::Repository::Push.fabricate! do |push|
push.new_branch = false
push.repository_http_uri = location.uri
push.file_name = file_name
......@@ -60,6 +62,15 @@ module QA
push.commit_message = "Update #{file_name}"
end
# We need to strip off the user from the URI, otherwise we won't
# get the correct output produced from the git CLI.
primary_uri = project.repository_http_location.uri
primary_uri.user = nil
# The git cli produces the 'warning: redirecting to..' output
# internally.
expect(push.output).to match(/warning: redirecting to #{primary_uri.to_s}/)
# Validate git push worked and new content is visible
Page::Project::Show.perform do |show|
show.wait_for_repository_replication_with(file_content_secondary)
......@@ -72,5 +83,87 @@ module QA
end
end
end
context 'git-lfs commit' do
it 'is redirected to the primary and ultimately replicated to the secondary' do
file_name_primary = 'README.md'
file_name_secondary = 'README_MORE.md'
Runtime::Browser.visit(:geo_primary, QA::Page::Main::Login) do
# Visit the primary node and login
Page::Main::Login.act { sign_in_using_credentials }
# Create a new Project
project = Resource::Project.fabricate! do |project|
project.name = 'geo-project'
project.description = 'Geo test project'
end
# Perform a git push over HTTP directly to the primary
#
# This push is required to ensure we have the primary credentials
# written out to the .netrc
Resource::Repository::Push.fabricate! do |push|
push.use_lfs = true
push.repository_http_uri = project.repository_http_location.uri
push.file_name = file_name_primary
push.file_content = "# #{file_content_primary}"
push.commit_message = "Add #{file_name_primary}"
end
Runtime::Browser.visit(:geo_secondary, QA::Page::Main::Login) do
# Visit the secondary node and login
Page::Main::OAuth.act { authorize! if needs_authorization? }
EE::Page::Main::Banner.perform do |banner|
expect(banner).to have_secondary_read_only_banner
end
Page::Main::Menu.perform { |menu| menu.go_to_projects }
Page::Dashboard::Projects.perform do |dashboard|
dashboard.wait_for_project_replication(project.name)
dashboard.go_to_project(project.name)
end
# Grab the HTTP URI for the secondary and store as 'location'
location = Page::Project::Show.act do
choose_repository_clone_http
repository_location
end
# Perform a git push over HTTP at the secondary
push = Resource::Repository::Push.fabricate! do |push|
push.use_lfs = true
push.new_branch = false
push.repository_http_uri = location.uri
push.file_name = file_name_secondary
push.file_content = "# #{file_content_secondary}"
push.commit_message = "Add #{file_name_secondary}"
end
# We need to strip off the user from the URI, otherwise we won't
# get the correct output produced from the git CLI.
primary_uri = project.repository_http_location.uri
primary_uri.user = nil
# The git cli produces the 'warning: redirecting to..' output
# internally.
expect(push.output).to match(/warning: redirecting to #{primary_uri.to_s}/)
expect(push.output).to match(/Locking support detected on remote "#{location.uri.to_s}"/)
expect(push.output).to match(%r{Uploading LFS objects: 100% \(2/2\)})
# Validate git push worked and new content is visible
Page::Project::Show.perform do |show|
show.wait_for_repository_replication_with(file_name_secondary)
show.refresh
expect(page).to have_content(file_name_secondary)
end
end
end
end
end
end
end
end
......@@ -3,8 +3,10 @@
module QA
context 'Geo', :orchestrated, :geo do
describe 'GitLab SSH push' do
let(:file_name) { 'README.md' }
context 'regular git commit' do
it "is replicated to the secondary" do
file_name = 'README.md'
key_title = "key for ssh tests #{Time.now.to_f}"
file_content = 'This is a Geo project! Commit from primary.'
......@@ -74,5 +76,82 @@ module QA
end
end
end
context 'git-lfs commit' do
it "is replicated to the secondary" do
key_title = "key for ssh tests #{Time.now.to_f}"
file_content = 'The rendered file could not be displayed because it is stored in LFS.'
Runtime::Browser.visit(:geo_primary, QA::Page::Main::Login) do
# Visit the primary node and login
Page::Main::Login.act { sign_in_using_credentials }
# Create a new SSH key for the user
key = Resource::SSHKey.fabricate! do |resource|
resource.title = key_title
end
# Create a new Project
project = Resource::Project.fabricate! do |project|
project.name = 'geo-project'
project.description = 'Geo test project'
end
# Perform a git push over SSH directly to the primary
push = Resource::Repository::ProjectPush.fabricate! do |push|
push.use_lfs = true
push.ssh_key = key
push.project = project
push.file_name = file_name
push.file_content = "# #{file_content}"
push.commit_message = 'Add README.md'
end
project.visit!
expect(push.output).to match(/Locking support detected on remote/)
expect(push.output).to match(%r{Uploading LFS objects: 100% \(1/1\)})
# Validate git push worked and file exists with content
Page::Project::Show.perform do |show|
show.wait_for_repository_replication
expect(page).to have_content(file_name)
expect(page).to have_content(file_content)
end
Runtime::Browser.visit(:geo_secondary, QA::Page::Main::Login) do
# Visit the secondary node and login
Page::Main::OAuth.act { authorize! if needs_authorization? }
EE::Page::Main::Banner.perform do |banner|
expect(banner).to have_secondary_read_only_banner
end
# Ensure the SSH key has replicated
Page::Main::Menu.act { go_to_profile_settings }
Page::Profile::Menu.act { click_ssh_keys }
expect(page).to have_content(key_title)
expect(page).to have_content(key.fingerprint)
# Ensure project has replicated
Page::Main::Menu.perform { |menu| menu.go_to_projects }
Page::Dashboard::Projects.perform do |dashboard|
dashboard.wait_for_project_replication(project.name)
dashboard.go_to_project(project.name)
end
# Validate the content has been sync'd from the primary
Page::Project::Show.perform do |show|
show.wait_for_repository_replication_with(file_name)
expect(page).to have_content(file_name)
expect(page).to have_content(file_content)
end
end
end
end
end
end
end
end
......@@ -3,11 +3,14 @@
module QA
context 'Geo', :orchestrated, :geo do
describe 'GitLab SSH push to secondary' do
it "is proxy'd to the primary and ultimately replicated to the secondary" do
let(:file_content_primary) { 'This is a Geo project! Commit from primary.' }
let(:file_content_secondary) { 'This is a Geo project! Commit from secondary.' }
context 'regular git commit' do
it 'is proxied to the primary and ultimately replicated to the secondary' do
file_name = 'README.md'
key_title = "key for ssh tests #{Time.now.to_f}"
file_content_primary = 'This is a Geo project! Commit from primary.'
file_content_secondary = 'This is a Geo project! Commit from secondary.'
file_content = 'This is a Geo project! Commit from secondary.'
Runtime::Browser.visit(:geo_primary, QA::Page::Main::Login) do
# Visit the primary node and login
......@@ -69,7 +72,7 @@ module QA
end
# Perform a git push over SSH at the secondary
Resource::Repository::Push.fabricate! do |push|
push = Resource::Repository::Push.fabricate! do |push|
push.new_branch = false
push.ssh_key = key
push.repository_ssh_uri = location.uri
......@@ -78,11 +81,111 @@ module QA
push.commit_message = "Update #{file_name}"
end
# Remove ssh:// from the URI to ensure we can match accurately
# as ssh:// can appear depending on how GitLab is configured.
ssh_uri = project.repository_ssh_location.git_uri.to_s.gsub(%r{ssh://}, '')
expect(push.output).to match(%r{GitLab: We'll help you by proxying this request to the primary: (?:ssh://)?#{ssh_uri}})
# Validate git push worked and new content is visible
Page::Project::Show.perform do |show|
show.wait_for_repository_replication_with(file_content_secondary)
show.wait_for_repository_replication_with(file_content)
expect(page).to have_content(file_content)
end
end
end
end
end
context 'git-lfs commit' do
it 'is proxied to the primary and ultimately replicated to the secondary' do
key_title = "key for ssh tests #{Time.now.to_f}"
file_name_primary = 'README.md'
file_name_secondary = 'README_MORE.md'
Runtime::Browser.visit(:geo_primary, QA::Page::Main::Login) do
# Visit the primary node and login
Page::Main::Login.act { sign_in_using_credentials }
# Create a new SSH key for the user
key = Resource::SSHKey.fabricate! do |resource|
resource.title = key_title
end
# Create a new Project
project = Resource::Project.fabricate! do |project|
project.name = 'geo-project'
project.description = 'Geo test project'
end
# Perform a git push over SSH directly to the primary
#
# This push is required to ensure we have the primary credentials
# written out to the .netrc
Resource::Repository::Push.fabricate! do |push|
push.use_lfs = true
push.ssh_key = key
push.repository_ssh_uri = project.repository_ssh_location.uri
push.file_name = file_name_primary
push.file_content = "# #{file_content_primary}"
push.commit_message = "Add #{file_name_primary}"
end
Runtime::Browser.visit(:geo_secondary, QA::Page::Main::Login) do
# Visit the secondary node and login
Page::Main::OAuth.act { authorize! if needs_authorization? }
expect(page).to have_content(file_content_secondary)
EE::Page::Main::Banner.perform do |banner|
expect(banner).to have_secondary_read_only_banner
end
# Ensure the SSH key has replicated
Page::Main::Menu.act { go_to_profile_settings }
Page::Profile::Menu.perform do |menu|
menu.click_ssh_keys
menu.wait_for_key_to_replicate(key_title)
end
expect(page).to have_content(key_title)
expect(page).to have_content(key.fingerprint)
# Ensure project has replicated
Page::Main::Menu.perform { |menu| menu.go_to_projects }
Page::Dashboard::Projects.perform do |dashboard|
dashboard.wait_for_project_replication(project.name)
dashboard.go_to_project(project.name)
end
# Grab the SSH URI for the secondary and store as 'location'
location = Page::Project::Show.act do
choose_repository_clone_ssh
repository_location
end
# Perform a git push over SSH at the secondary
push = Resource::Repository::Push.fabricate! do |push|
push.use_lfs = true
push.new_branch = false
push.ssh_key = key
push.repository_ssh_uri = location.uri
push.file_name = file_name_secondary
push.file_content = "# #{file_content_secondary}"
push.commit_message = "Add #{file_name_secondary}"
end
ssh_uri = project.repository_ssh_location.git_uri.to_s.gsub(%r{ssh://}, '')
expect(push.output).to match(%r{GitLab: We'll help you by proxying this request to the primary: (?:ssh://)?#{ssh_uri}})
expect(push.output).to match(/Locking support detected on remote "#{location.uri.to_s}"/)
expect(push.output).to match(%r{Uploading LFS objects: 100% \(2/2\)})
# Validate git push worked and new content is visible
Page::Project::Show.perform do |show|
show.wait_for_repository_replication_with(file_name_secondary)
show.refresh
expect(page).to have_content(file_name_secondary)
end
end
end
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment