EE correspondence of gitlab-org/gitlab-ce!10156

Merge branch 'test-pg-mysql' into tmp

* test-pg-mysql:
  Properly use $BUNDLE_INSTALL_FLAGS
  Always use . and add more comments to prepare_build.sh
  Remove duplicated line
  Avoid Rails generating complicated query, see:
  Move variables back to where they are
  Just set GITLAB_DATABASE in the script
  Add # host for development and test as well
  We cannot use array in yaml variables
  Set profiler only for postgresql not mysql
  Use GITLAB_DATABASE: $CI_JOB_NAME[1] so that we
  Replace on host rather than socket, feedback:
  Only upload/download knapsack reports from pg
  Switch back to original repository
  Test all on EE, only master on CE, and -mysql branches
  Note that install knapsack later than bundle install
  Test both PostgreSQL and MySQL for the win.

config/database.yml.mysql

@@ -25,6 +25,7 @@ development:
   pool: 5
   username: root
   password: "secure password"
+  # host: localhost
   # socket: /tmp/mysql.sock
 
 # Warning: The database defined as "test" will be erased and
@@ -39,4 +40,5 @@ test: &test
   pool: 5
   username: root
   password:
+  # host: localhost
   # socket: /tmp/mysql.sock

config/database.yml.postgresql

@@ -25,6 +25,7 @@ development:
   pool: 5
   username: postgres
   password:
+  # host: localhost
 
 #
 # Staging specific
@@ -36,6 +37,7 @@ staging:
   pool: 5
   username: postgres
   password:
+  # host: localhost
 
 # Warning: The database defined as "test" will be erased and
 # re-generated from your development database when you run "rake".
@@ -47,3 +49,4 @@ test: &test
   pool: 5
   username: postgres
   password:
+  # host: localhost

config/database_geo.yml.postgresql

@@ -21,6 +21,7 @@ development:
   pool: 5
   username: postgres
   password:
+  # host: localhost
 
 #
 # Staging specific
@@ -32,6 +33,7 @@ staging:
   pool: 5
   username: postgres
   password:
+  # host: localhost
 
 # Warning: The database defined as "test" will be erased and
 # re-generated from your development database when you run "rake".
@@ -43,3 +45,4 @@ test: &test
   pool: 5
   username: postgres
   password:
+  # host: localhost

config/initializers/rspec_profiling.rb

@@ -38,7 +38,7 @@ if Rails.env.test?
     end
   end
 
-  if ENV.has_key?('CI')
+  if ENV.has_key?('CI') && ENV['GITLAB_DATABASE'] == 'postgresql'
     RspecProfiling::VCS::Git.prepend(RspecProfilingExt::Git)
     RspecProfiling::Run.prepend(RspecProfilingExt::Run)
   end

scripts/prepare_build.sh

 #!/bin/sh
 
-retry() {
-    if eval "$@"; then
-        return 0
-    fi
+. scripts/utils.sh
+
+export SETUP_DB=${SETUP_DB:-true}
+export USE_BUNDLE_INSTALL=${USE_BUNDLE_INSTALL:-true}
+export BUNDLE_INSTALL_FLAGS="--without production --jobs $(nproc) --path vendor --retry 3 --quiet"
+
+# Determine the database by looking at the job name.
+# For example, we'll get pg if the job is `rspec pg 19 20`
+export GITLAB_DATABASE=$(echo $CI_JOB_NAME | cut -f2 -d' ')
+
+# This would make the default database postgresql, and we could also use
+# pg to mean postgresql.
+if [ "$GITLAB_DATABASE" != 'mysql' ]; then
+    export GITLAB_DATABASE='postgresql'
+fi
+
+cp config/database.yml.$GITLAB_DATABASE config/database.yml
+
+# EE-only
+cp config/database_geo.yml.$GITLAB_DATABASE config/database_geo.yml
+
+if [ "$GITLAB_DATABASE" = 'postgresql' ]; then
+    sed -i 's/# host:.*/host: postgres/g' config/database.yml
+
+    # EE-only
+    sed -i 's/# host:.*/host: postgres/g' config/database_geo.yml
 
-    for i in 2 1; do
-        sleep 3s
-        echo "Retrying $i..."
-        if eval "$@"; then
-            return 0
-        fi
-    done
-    return 1
-}
-
-if [ -f /.dockerenv ] || [ -f ./dockerinit ]; then
-    cp config/database.yml.mysql config/database.yml
+else # Assume it's mysql
     sed -i 's/username:.*/username: root/g' config/database.yml
     sed -i 's/password:.*/password:/g' config/database.yml
-    sed -i 's/# socket:.*/host: mysql/g' config/database.yml
+    sed -i 's/# host:.*/host: mysql/g' config/database.yml
 
-    cp config/database_geo.yml.mysql config/database_geo.yml
+    # EE-only
     sed -i 's/username:.*/username: root/g' config/database_geo.yml
     sed -i 's/password:.*/password:/g' config/database_geo.yml
-    sed -i 's/# socket:.*/host: mysql/g' config/database_geo.yml
-
-    cp config/resque.yml.example config/resque.yml
-    sed -i 's/localhost/redis/g' config/resque.yml
-
-    export FLAGS="--path vendor --retry 3 --quiet"
-else
-    rnd=$(awk 'BEGIN { srand() ; printf("%d\n",rand()*5) }')
-    export PATH="$HOME/bin:/usr/local/bin:/usr/bin:/bin"
-    cp config/database.yml.mysql config/database.yml
-    sed "s/username\:.*$/username\: runner/" -i config/database.yml
-    sed "s/password\:.*$/password\: 'password'/" -i config/database.yml
-    sed "s/gitlabhq_test/gitlabhq_test_$rnd/" -i config/database.yml
-
-    cp config/database_geo.yml.mysql config/database_geo.yml
-    sed "s/username\:.*$/username\: runner/" -i config/database_geo.yml
-    sed "s/password\:.*$/password\: 'password'/" -i config/database_geo.yml
-    sed "s/gitlabhq_test/gitlabhq_test_$rnd/" -i config/database_geo.yml
+    sed -i 's/# host:.*/host: mysql/g' config/database_geo.yml
+fi
+
+cp config/gitlab.yml.example config/gitlab.yml
+
+if [ "$USE_BUNDLE_INSTALL" != "false" ]; then
+    retry bundle install --clean $BUNDLE_INSTALL_FLAGS
+fi
+
+# Only install knapsack after bundle install! Otherwise oddly some native
+# gems could not be found under some circumstance. No idea why, hours wasted.
+retry gem install knapsack fog-aws mime-types
+
+if [ "$SETUP_DB" != "false" ]; then
+    bundle exec rake db:drop db:create db:schema:load db:migrate
+
+    if [ "$GITLAB_DATABASE" = "mysql" ]; then
+        bundle exec rake add_limits_mysql
+    fi
+
+    # EE-only
+    bundle exec rake geo:db:drop geo:db:create geo:db:schema:load geo:db:migrate
 fi

scripts/utils.sh

+retry() {
+    if eval "$@"; then
+        return 0
+    fi
+
+    for i in 2 1; do
+        sleep 3s
+        echo "Retrying $i..."
+        if eval "$@"; then
+            return 0
+        fi
+    done
+    return 1
+}