Merge branch 'qa-backport-recent-ee-changes' into 'master'

[QA] Port recent changes from EE

See merge request gitlab-org/gitlab-ce!22391

qa/qa/factory/resource/merge_request.rb

@@ -30,6 +30,7 @@ module QA
           push.project = factory.project
           push.branch_name = factory.target_branch
           push.remote_branch = factory.source_branch
+          push.new_branch = false
           push.file_name = "added_file.txt"
           push.file_content = "File Added"
         end

qa/qa/factory/settings/hashed_storage.rb

@@ -9,7 +9,7 @@ module QA
           Page::Main::Menu.act { go_to_admin_area }
           Page::Admin::Menu.act { go_to_repository_settings }
 
-          Page::Admin::Settings::Main.perform do |setting|
+          Page::Admin::Settings::Repository.perform do |setting|
             setting.expand_repository_storage do |page|
               page.enable_hashed_storage
               page.save_settings

qa/qa/git/repository.rb

+# frozen_string_literal: true
+
 require 'cgi'
 require 'uri'
 require 'open3'
+require 'fileutils'
+require 'tmpdir'
 
 module QA
   module Git
     class Repository
       include Scenario::Actable
 
+      attr_writer :password
+      attr_accessor :env_vars
+
       def initialize
-        @ssh_cmd = ""
+        # We set HOME to the current working directory (which is a
+        # temporary directory created in .perform()) so the temporarily dropped
+        # .netrc can be utilised
+        self.env_vars = [%Q{HOME="#{File.dirname(netrc_file_path)}"}]
       end
 
       def self.perform(*args)
@@ -21,36 +31,27 @@ module QA
         @uri = URI(address)
       end
 
-      def username=(name)
-        @username = name
-        @uri.user = name
-      end
-
-      def password=(pass)
-        @password = pass
-        @uri.password = CGI.escape(pass).gsub('+', '%20')
+      def username=(username)
+        @username = username
+        @uri.user = username
       end
 
       def use_default_credentials
-        if ::QA::Runtime::User.ldap_user?
-          self.username = Runtime::User.ldap_username
-          self.password = Runtime::User.ldap_password
-        else
-          self.username = Runtime::User.username
-          self.password = Runtime::User.password
-        end
+        self.username, self.password = default_credentials
+
+        add_credentials_to_netrc unless ssh_key_set?
       end
 
       def clone(opts = '')
-        run_and_redact_credentials(build_git_command("git clone #{opts} #{@uri} ./"))
+        run("git clone #{opts} #{uri} ./")
       end
 
       def checkout(branch_name)
-        `git checkout "#{branch_name}"`
+        run(%Q{git checkout "#{branch_name}"})
       end
 
       def checkout_new_branch(branch_name)
-        `git checkout -b "#{branch_name}"`
+        run(%Q{git checkout -b "#{branch_name}"})
       end
 
       def shallow_clone
@@ -58,12 +59,10 @@ module QA
       end
 
       def configure_identity(name, email)
-        `git config user.name #{name}`
-        `git config user.email #{email}`
-      end
+        run(%Q{git config user.name #{name}})
+        run(%Q{git config user.email #{email}})
 
-      def configure_ssh_command(command)
-        @ssh_cmd = "GIT_SSH_COMMAND='#{command}'"
+        add_credentials_to_netrc
       end
 
       def commit_file(name, contents, message)
@@ -74,54 +73,103 @@ module QA
       def add_file(name, contents)
         ::File.write(name, contents)
 
-        `git add #{name}`
+        run(%Q{git add #{name}})
       end
 
       def commit(message)
-        `git commit -m "#{message}"`
+        run(%Q{git commit -m "#{message}"})
       end
 
       def push_changes(branch = 'master')
-        output, _ = run_and_redact_credentials(build_git_command("git push #{@uri} #{branch}"))
-
-        output
+        run("git push #{uri} #{branch}")
       end
 
       def commits
-        `git log --oneline`.split("\n")
+        run('git log --oneline').split("\n")
       end
 
       def use_ssh_key(key)
         @private_key_file = Tempfile.new("id_#{SecureRandom.hex(8)}")
-        File.binwrite(@private_key_file, key.private_key)
-        File.chmod(0700, @private_key_file)
+        File.binwrite(private_key_file, key.private_key)
+        File.chmod(0700, private_key_file)
 
         @known_hosts_file = Tempfile.new("known_hosts_#{SecureRandom.hex(8)}")
         keyscan_params = ['-H']
-        keyscan_params << "-p #{@uri.port}" if @uri.port
-        keyscan_params << @uri.host
-        run_and_redact_credentials("ssh-keyscan #{keyscan_params.join(' ')} >> #{@known_hosts_file.path}")
+        keyscan_params << "-p #{uri.port}" if uri.port
+        keyscan_params << uri.host
+        run("ssh-keyscan #{keyscan_params.join(' ')} >> #{known_hosts_file.path}")
 
-        configure_ssh_command("ssh -i #{@private_key_file.path} -o UserKnownHostsFile=#{@known_hosts_file.path}")
+        self.env_vars << %Q{GIT_SSH_COMMAND="ssh -i #{private_key_file.path} -o UserKnownHostsFile=#{known_hosts_file.path}"}
       end
 
       def delete_ssh_key
-        return unless @private_key_file
+        return unless ssh_key_set?
 
-        @private_key_file.close(true)
-        @known_hosts_file.close(true)
+        private_key_file.close(true)
+        known_hosts_file.close(true)
       end
 
-      def build_git_command(command_str)
-        [@ssh_cmd, command_str].compact.join(' ')
+      private
+
+      attr_reader :uri, :username, :password, :known_hosts_file, :private_key_file
+
+      def debug?
+        Runtime::Env.respond_to?(:verbose?) && Runtime::Env.verbose?
       end
 
-      private
+      def ssh_key_set?
+        !private_key_file.nil?
+      end
+
+      def run(command_str)
+        command = [env_vars, command_str, '2>&1'].compact.join(' ')
+        warn "DEBUG: command=[#{command}]" if debug?
+
+        output, _ = Open3.capture2(command)
+        output = output.chomp.gsub(/\s+$/, '')
+        warn "DEBUG: output=[#{output}]" if debug?
+
+        output
+      end
+
+      def default_credentials
+        if ::QA::Runtime::User.ldap_user?
+          [Runtime::User.ldap_username, Runtime::User.ldap_password]
+        else
+          [Runtime::User.username, Runtime::User.password]
+        end
+      end
+
+      def tmp_netrc_directory
+        @tmp_netrc_directory ||= File.join(Dir.tmpdir, "qa-netrc-credentials", $$.to_s)
+      end
+
+      def netrc_file_path
+        @netrc_file_path ||= File.join(tmp_netrc_directory, '.netrc')
+      end
+
+      def netrc_content
+        "machine #{uri.host} login #{username} password #{password}"
+      end
+
+      def netrc_already_contains_content?
+        File.exist?(netrc_file_path) &&
+          File.readlines(netrc_file_path).grep(/^#{netrc_content}$/).any?
+      end
+
+      def add_credentials_to_netrc
+        # Despite libcurl supporting a custom .netrc location through the
+        # CURLOPT_NETRC_FILE environment variable, git does not support it :(
+        # Info: https://curl.haxx.se/libcurl/c/CURLOPT_NETRC_FILE.html
+        #
+        # This will create a .netrc in the correct working directory, which is
+        # a temporary directory created in .perform()
+        #
+        return if netrc_already_contains_content?
 
-      # Since the remote URL contains the credentials, and git occasionally
-      # outputs the URL. Note that stderr is redirected to stdout.
-      def run_and_redact_credentials(command)
-        Open3.capture2("#{command} 2>&1 | sed -E 's#://[^@]+@#://****@#g'")
+        FileUtils.mkdir_p(tmp_netrc_directory)
+        File.open(netrc_file_path, 'a') { |file| file.puts(netrc_content) }
+        File.chmod(0600, netrc_file_path)
       end
     end
   end

qa/qa/page/project/menu.rb

@@ -87,6 +87,14 @@ module QA
           end
         end
 
+        def go_to_labels
+          hover_issues do
+            within_submenu do
+              click_element(:labels_link)
+            end
+          end
+        end
+
         def click_merge_requests
           within_sidebar do
             click_link('Merge Requests')
@@ -105,8 +113,10 @@ module QA
           end
         end
 
-        def go_to_labels
-          hover_issues { click_element :labels_link }
+        def click_repository
+          within_sidebar do
+            click_link('Repository')
+          end
         end
 
         private

qa/spec/git/repository_spec.rb

 describe QA::Git::Repository do
+  include Support::StubENV
+
   let(:repository) { described_class.new }
 
   before do
+    stub_env('GITLAB_USERNAME', 'root')
     cd_empty_temp_directory
     set_bad_uri
     repository.use_default_credentials
   end
 
   describe '#clone' do
-    it 'redacts credentials from the URI in output' do
-      output, _ = repository.clone
-
-      expect(output).to include("fatal: unable to access 'http://****@foo/bar.git/'")
+    it 'is unable to resolve host' do
+      expect(repository.clone).to include("fatal: unable to access 'http://root@foo/bar.git/'")
     end
   end
 
@@ -20,10 +21,8 @@ describe QA::Git::Repository do
       `git init` # need a repo to push from
     end
 
-    it 'redacts credentials from the URI in output' do
-      output, _ = repository.push_changes
-
-      expect(output).to include("error: failed to push some refs to 'http://****@foo/bar.git'")
+    it 'fails to push changes' do
+      expect(repository.push_changes).to include("error: failed to push some refs to 'http://root@foo/bar.git'")
     end
   end