EE correspondence of gitlab-org/gitlab-ce!10156

Merge branch 'test-pg-mysql' into tmp

* test-pg-mysql:
  Properly use $BUNDLE_INSTALL_FLAGS
  Always use . and add more comments to prepare_build.sh
  Remove duplicated line
  Avoid Rails generating complicated query, see:
  Move variables back to where they are
  Just set GITLAB_DATABASE in the script
  Add # host for development and test as well
  We cannot use array in yaml variables
  Set profiler only for postgresql not mysql
  Use GITLAB_DATABASE: $CI_JOB_NAME[1] so that we
  Replace on host rather than socket, feedback:
  Only upload/download knapsack reports from pg
  Switch back to original repository
  Test all on EE, only master on CE, and -mysql branches
  Note that install knapsack later than bundle install
  Test both PostgreSQL and MySQL for the win.

.gitlab-ci.yml

-image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1"
+image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6"
 
 cache:
   key: "ruby-233"
@@ -11,8 +11,6 @@ variables:
   RAILS_ENV: "test"
   NODE_ENV: "test"
   SIMPLECOV: "true"
-  SETUP_DB: "true"
-  USE_BUNDLE_INSTALL: "true"
   GIT_DEPTH: "20"
   PHANTOMJS_VERSION: "2.1.1"
   GET_SOURCES_ATTEMPTS: "3"
@@ -22,13 +20,9 @@ variables:
   ES_JAVA_OPTS: "-Xms600m -Xmx600m"
 
 before_script:
-  - source ./scripts/prepare_build.sh
-  - cp config/gitlab.yml.example config/gitlab.yml
   - bundle --version
-  - '[ "$USE_BUNDLE_INSTALL" != "true" ] || retry bundle install --without postgres production --jobs $(nproc) --clean $FLAGS'
-  - retry gem install knapsack fog-aws mime-types
-  - '[ "$SETUP_DB" != "true" ] || bundle exec rake db:drop db:create db:schema:load db:migrate add_limits_mysql'
-  - '[ "$SETUP_DB" != "true" ] || bundle exec rake geo:db:drop geo:db:create geo:db:schema:load geo:db:migrate'
+  - . scripts/utils.sh
+  - ./scripts/prepare_build.sh
 
 stages:
 - prepare
@@ -56,21 +50,35 @@ stages:
     paths:
       - knapsack/
 
-.use-db: &use-db
+.use-pg: &use-pg
+  services:
+    - postgres:latest
+    - redis:alpine
+
+.use-mysql: &use-mysql
   services:
     - mysql:latest
     - redis:alpine
     - elasticsearch:5.1
 
+.only-master-and-ee-or-mysql: &only-master-and-ee-or-mysql
+  only:
+    - /\-(?i)mysql$/
+    - master@gitlab-org/gitlab-ce
+    - master@gitlab/gitlabhq
+    - tags@gitlab-org/gitlab-ce
+    - tags@gitlab/gitlabhq
+    - //@gitlab-org/gitlab-ee
+    - //@gitlab/gitlab-ee
+
 .rspec-knapsack: &rspec-knapsack
   stage: test
   <<: *dedicated-runner
-  <<: *use-db
   script:
     - JOB_NAME=( $CI_JOB_NAME )
-    - export CI_NODE_INDEX=${JOB_NAME[1]}
-    - export CI_NODE_TOTAL=${JOB_NAME[2]}
-    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
+    - export CI_NODE_INDEX=${JOB_NAME[-2]}
+    - export CI_NODE_TOTAL=${JOB_NAME[-1]}
+    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_${JOB_NAME[1]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
     - export KNAPSACK_GENERATE_REPORT=true
     - cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
     - knapsack rspec "--color --format documentation"
@@ -82,15 +90,23 @@ stages:
       - knapsack/
       - tmp/capybara/
 
+.rspec-knapsack-pg: &rspec-knapsack-pg
+  <<: *rspec-knapsack
+  <<: *use-pg
+
+.rspec-knapsack-mysql: &rspec-knapsack-mysql
+  <<: *rspec-knapsack
+  <<: *use-mysql
+  <<: *only-master-and-ee-or-mysql
+
 .spinach-knapsack: &spinach-knapsack
   stage: test
   <<: *dedicated-runner
-  <<: *use-db
   script:
     - JOB_NAME=( $CI_JOB_NAME )
-    - export CI_NODE_INDEX=${JOB_NAME[1]}
-    - export CI_NODE_TOTAL=${JOB_NAME[2]}
-    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
+    - export CI_NODE_INDEX=${JOB_NAME[-2]}
+    - export CI_NODE_TOTAL=${JOB_NAME[-1]}
+    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_${JOB_NAME[1]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
     - export KNAPSACK_GENERATE_REPORT=true
     - cp ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
     - knapsack spinach "-r rerun" || retry '[[ -e tmp/spinach-rerun.txt ]] && bundle exec spinach -r rerun $(cat tmp/spinach-rerun.txt)'
@@ -102,6 +118,15 @@ stages:
       - knapsack/
       - tmp/capybara/
 
+.spinach-knapsack-pg: &spinach-knapsack-pg
+  <<: *spinach-knapsack
+  <<: *use-pg
+
+.spinach-knapsack-mysql: &spinach-knapsack-mysql
+  <<: *spinach-knapsack
+  <<: *use-mysql
+  <<: *only-master-and-ee-or-mysql
+
 # Prepare and merge knapsack tests
 knapsack:
   <<: *knapsack-state
@@ -119,8 +144,8 @@ update-knapsack:
   <<: *dedicated-runner
   stage: post-test
   script:
-    - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_node_*.json
-    - scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach_node_*.json
+    - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_pg_node_*.json
+    - scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach_pg_node_*.json
     - '[[ -z ${KNAPSACK_S3_BUCKET} ]] || scripts/sync-reports put $KNAPSACK_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH'
     - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
   only:
@@ -130,7 +155,7 @@ update-knapsack:
     - master@gitlab/gitlab-ee
 
 setup-test-env:
-  <<: *use-db
+  <<: *use-pg
   <<: *dedicated-runner
   stage: prepare
   script:
@@ -145,37 +170,69 @@ setup-test-env:
       - public/assets
       - tmp/tests
 
-rspec 0 20: *rspec-knapsack
-rspec 1 20: *rspec-knapsack
-rspec 2 20: *rspec-knapsack
-rspec 3 20: *rspec-knapsack
-rspec 4 20: *rspec-knapsack
-rspec 5 20: *rspec-knapsack
-rspec 6 20: *rspec-knapsack
-rspec 7 20: *rspec-knapsack
-rspec 8 20: *rspec-knapsack
-rspec 9 20: *rspec-knapsack
-rspec 10 20: *rspec-knapsack
-rspec 11 20: *rspec-knapsack
-rspec 12 20: *rspec-knapsack
-rspec 13 20: *rspec-knapsack
-rspec 14 20: *rspec-knapsack
-rspec 15 20: *rspec-knapsack
-rspec 16 20: *rspec-knapsack
-rspec 17 20: *rspec-knapsack
-rspec 18 20: *rspec-knapsack
-rspec 19 20: *rspec-knapsack
-
-spinach 0 10: *spinach-knapsack
-spinach 1 10: *spinach-knapsack
-spinach 2 10: *spinach-knapsack
-spinach 3 10: *spinach-knapsack
-spinach 4 10: *spinach-knapsack
-spinach 5 10: *spinach-knapsack
-spinach 6 10: *spinach-knapsack
-spinach 7 10: *spinach-knapsack
-spinach 8 10: *spinach-knapsack
-spinach 9 10: *spinach-knapsack
+rspec pg 0 20: *rspec-knapsack-pg
+rspec pg 1 20: *rspec-knapsack-pg
+rspec pg 2 20: *rspec-knapsack-pg
+rspec pg 3 20: *rspec-knapsack-pg
+rspec pg 4 20: *rspec-knapsack-pg
+rspec pg 5 20: *rspec-knapsack-pg
+rspec pg 6 20: *rspec-knapsack-pg
+rspec pg 7 20: *rspec-knapsack-pg
+rspec pg 8 20: *rspec-knapsack-pg
+rspec pg 9 20: *rspec-knapsack-pg
+rspec pg 10 20: *rspec-knapsack-pg
+rspec pg 11 20: *rspec-knapsack-pg
+rspec pg 12 20: *rspec-knapsack-pg
+rspec pg 13 20: *rspec-knapsack-pg
+rspec pg 14 20: *rspec-knapsack-pg
+rspec pg 15 20: *rspec-knapsack-pg
+rspec pg 16 20: *rspec-knapsack-pg
+rspec pg 17 20: *rspec-knapsack-pg
+rspec pg 18 20: *rspec-knapsack-pg
+rspec pg 19 20: *rspec-knapsack-pg
+
+rspec mysql 0 20: *rspec-knapsack-mysql
+rspec mysql 1 20: *rspec-knapsack-mysql
+rspec mysql 2 20: *rspec-knapsack-mysql
+rspec mysql 3 20: *rspec-knapsack-mysql
+rspec mysql 4 20: *rspec-knapsack-mysql
+rspec mysql 5 20: *rspec-knapsack-mysql
+rspec mysql 6 20: *rspec-knapsack-mysql
+rspec mysql 7 20: *rspec-knapsack-mysql
+rspec mysql 8 20: *rspec-knapsack-mysql
+rspec mysql 9 20: *rspec-knapsack-mysql
+rspec mysql 10 20: *rspec-knapsack-mysql
+rspec mysql 11 20: *rspec-knapsack-mysql
+rspec mysql 12 20: *rspec-knapsack-mysql
+rspec mysql 13 20: *rspec-knapsack-mysql
+rspec mysql 14 20: *rspec-knapsack-mysql
+rspec mysql 15 20: *rspec-knapsack-mysql
+rspec mysql 16 20: *rspec-knapsack-mysql
+rspec mysql 17 20: *rspec-knapsack-mysql
+rspec mysql 18 20: *rspec-knapsack-mysql
+rspec mysql 19 20: *rspec-knapsack-mysql
+
+spinach pg 0 10: *spinach-knapsack-pg
+spinach pg 1 10: *spinach-knapsack-pg
+spinach pg 2 10: *spinach-knapsack-pg
+spinach pg 3 10: *spinach-knapsack-pg
+spinach pg 4 10: *spinach-knapsack-pg
+spinach pg 5 10: *spinach-knapsack-pg
+spinach pg 6 10: *spinach-knapsack-pg
+spinach pg 7 10: *spinach-knapsack-pg
+spinach pg 8 10: *spinach-knapsack-pg
+spinach pg 9 10: *spinach-knapsack-pg
+
+spinach mysql 0 10: *spinach-knapsack-mysql
+spinach mysql 1 10: *spinach-knapsack-mysql
+spinach mysql 2 10: *spinach-knapsack-mysql
+spinach mysql 3 10: *spinach-knapsack-mysql
+spinach mysql 4 10: *spinach-knapsack-mysql
+spinach mysql 5 10: *spinach-knapsack-mysql
+spinach mysql 6 10: *spinach-knapsack-mysql
+spinach mysql 7 10: *spinach-knapsack-mysql
+spinach mysql 8 10: *spinach-knapsack-mysql
+spinach mysql 9 10: *spinach-knapsack-mysql
 
 # Other generic tests
 .ruby-static-analysis: &ruby-static-analysis
@@ -206,24 +263,37 @@ rake flay: *exec
 license_finder: *exec
 rake downtime_check: *exec
 
-rake db:migrate:reset:
+.db-migrate-reset: &db-migrate-reset
   stage: test
-  <<: *use-db
   <<: *dedicated-runner
   script:
     - bundle exec rake db:migrate:reset
 
-rake db:rollback:
+rake pg db:migrate:reset:
+  <<: *db-migrate-reset
+  <<: *use-pg
+
+rake mysql db:migrate:reset:
+  <<: *db-migrate-reset
+  <<: *use-mysql
+
+.db-rollback: &db-rollback
   stage: test
-  <<: *use-db
   <<: *dedicated-runner
   script:
     - bundle exec rake db:rollback STEP=120
     - bundle exec rake db:migrate
 
-rake db:seed_fu:
+rake pg db:rollback:
+  <<: *db-rollback
+  <<: *use-pg
+
+rake mysql db:rollback:
+  <<: *db-rollback
+  <<: *use-mysql
+
+.db-seed_fu: &db-seed_fu
   stage: test
-  <<: *use-db
   <<: *dedicated-runner
   variables:
     SIZE: "1"
@@ -239,6 +309,14 @@ rake db:seed_fu:
     paths:
       - log/development.log
 
+rake pg db:seed_fu:
+  <<: *db-seed_fu
+  <<: *use-pg
+
+rake mysql db:seed_fu:
+  <<: *db-seed_fu
+  <<: *use-mysql
+
 rake gitlab:assets:compile:
   stage: test
   <<: *dedicated-runner
@@ -264,7 +342,7 @@ rake karma:
       - vendor/ruby
       - node_modules
   stage: test
-  <<: *use-db
+  <<: *use-pg
   <<: *dedicated-runner
   variables:
     BABEL_ENV: "coverage"
@@ -321,9 +399,8 @@ bundler:audit:
   script:
     - "bundle exec bundle-audit check --update --ignore CVE-2016-4658"
 
-migration paths:
+.migration-paths: &migration-paths
   stage: test
-  <<: *use-db
   <<: *dedicated-runner
   variables:
     SETUP_DB: "false"
@@ -335,15 +412,21 @@ migration paths:
   script:
     - git fetch origin v8.14.10
     - git checkout -f FETCH_HEAD
-    - cp config/resque.yml.example config/resque.yml
-    - sed -i 's/localhost/redis/g' config/resque.yml
-    - bundle install --without postgres production --jobs $(nproc) $FLAGS --retry=3
+    - bundle install $BUNDLE_INSTALL_FLAGS
     - bundle exec rake db:drop db:create db:schema:load db:seed_fu
     - git checkout $CI_COMMIT_SHA
-    - bundle install --without postgres production --jobs $(nproc) $FLAGS --retry=3
-    - source scripts/prepare_build.sh
+    - bundle install $BUNDLE_INSTALL_FLAGS
+    - . scripts/prepare_build.sh
     - bundle exec rake db:migrate
 
+migration pg paths:
+  <<: *migration-paths
+  <<: *use-pg
+
+migration mysql paths:
+  <<: *migration-paths
+  <<: *use-mysql
+
 coverage:
   stage: post-test
   services: []
@@ -396,7 +479,7 @@ trigger_docs:
   before_script:
     - apk update && apk add curl
   variables:
-    GIT_STRATEGY: none
+    GIT_STRATEGY: "none"
   cache: {}
   artifacts: {}
   script:

config/database.yml.mysql

@@ -25,6 +25,7 @@ development:
   pool: 5
   username: root
   password: "secure password"
+  # host: localhost
   # socket: /tmp/mysql.sock
 
 # Warning: The database defined as "test" will be erased and
@@ -39,4 +40,5 @@ test: &test
   pool: 5
   username: root
   password:
+  # host: localhost
   # socket: /tmp/mysql.sock

config/database.yml.postgresql

@@ -25,6 +25,7 @@ development:
   pool: 5
   username: postgres
   password:
+  # host: localhost
 
 #
 # Staging specific
@@ -36,6 +37,7 @@ staging:
   pool: 5
   username: postgres
   password:
+  # host: localhost
 
 # Warning: The database defined as "test" will be erased and
 # re-generated from your development database when you run "rake".
@@ -47,3 +49,4 @@ test: &test
   pool: 5
   username: postgres
   password:
+  # host: localhost

config/database_geo.yml.postgresql

@@ -21,6 +21,7 @@ development:
   pool: 5
   username: postgres
   password:
+  # host: localhost
 
 #
 # Staging specific
@@ -32,6 +33,7 @@ staging:
   pool: 5
   username: postgres
   password:
+  # host: localhost
 
 # Warning: The database defined as "test" will be erased and
 # re-generated from your development database when you run "rake".
@@ -43,3 +45,4 @@ test: &test
   pool: 5
   username: postgres
   password:
+  # host: localhost

config/initializers/rspec_profiling.rb

@@ -38,7 +38,7 @@ if Rails.env.test?
     end
   end
 
-  if ENV.has_key?('CI')
+  if ENV.has_key?('CI') && ENV['GITLAB_DATABASE'] == 'postgresql'
     RspecProfiling::VCS::Git.prepend(RspecProfilingExt::Git)
     RspecProfiling::Run.prepend(RspecProfilingExt::Run)
   end

scripts/prepare_build.sh

 #!/bin/sh
 
-retry() {
-    if eval "$@"; then
-        return 0
-    fi
+. scripts/utils.sh
 
-    for i in 2 1; do
-        sleep 3s
-        echo "Retrying $i..."
-        if eval "$@"; then
-            return 0
-        fi
-    done
-    return 1
-}
+export SETUP_DB=${SETUP_DB:-true}
+export USE_BUNDLE_INSTALL=${USE_BUNDLE_INSTALL:-true}
+export BUNDLE_INSTALL_FLAGS="--without production --jobs $(nproc) --path vendor --retry 3 --quiet"
+
+# Determine the database by looking at the job name.
+# For example, we'll get pg if the job is `rspec pg 19 20`
+export GITLAB_DATABASE=$(echo $CI_JOB_NAME | cut -f2 -d' ')
+
+# This would make the default database postgresql, and we could also use
+# pg to mean postgresql.
+if [ "$GITLAB_DATABASE" != 'mysql' ]; then
+    export GITLAB_DATABASE='postgresql'
+fi
+
+cp config/database.yml.$GITLAB_DATABASE config/database.yml
+
+# EE-only
+cp config/database_geo.yml.$GITLAB_DATABASE config/database_geo.yml
+
+if [ "$GITLAB_DATABASE" = 'postgresql' ]; then
+    sed -i 's/# host:.*/host: postgres/g' config/database.yml
+
+    # EE-only
+    sed -i 's/# host:.*/host: postgres/g' config/database_geo.yml
 
-if [ -f /.dockerenv ] || [ -f ./dockerinit ]; then
-    cp config/database.yml.mysql config/database.yml
+else # Assume it's mysql
     sed -i 's/username:.*/username: root/g' config/database.yml
     sed -i 's/password:.*/password:/g' config/database.yml
-    sed -i 's/# socket:.*/host: mysql/g' config/database.yml
+    sed -i 's/# host:.*/host: mysql/g' config/database.yml
 
-    cp config/database_geo.yml.mysql config/database_geo.yml
+    # EE-only
     sed -i 's/username:.*/username: root/g' config/database_geo.yml
     sed -i 's/password:.*/password:/g' config/database_geo.yml
-    sed -i 's/# socket:.*/host: mysql/g' config/database_geo.yml
-
-    cp config/resque.yml.example config/resque.yml
-    sed -i 's/localhost/redis/g' config/resque.yml
-
-    export FLAGS="--path vendor --retry 3 --quiet"
-else
-    rnd=$(awk 'BEGIN { srand() ; printf("%d\n",rand()*5) }')
-    export PATH="$HOME/bin:/usr/local/bin:/usr/bin:/bin"
-    cp config/database.yml.mysql config/database.yml
-    sed "s/username\:.*$/username\: runner/" -i config/database.yml
-    sed "s/password\:.*$/password\: 'password'/" -i config/database.yml
-    sed "s/gitlabhq_test/gitlabhq_test_$rnd/" -i config/database.yml
-
-    cp config/database_geo.yml.mysql config/database_geo.yml
-    sed "s/username\:.*$/username\: runner/" -i config/database_geo.yml
-    sed "s/password\:.*$/password\: 'password'/" -i config/database_geo.yml
-    sed "s/gitlabhq_test/gitlabhq_test_$rnd/" -i config/database_geo.yml
+    sed -i 's/# host:.*/host: mysql/g' config/database_geo.yml
+fi
+
+cp config/gitlab.yml.example config/gitlab.yml
+
+if [ "$USE_BUNDLE_INSTALL" != "false" ]; then
+    retry bundle install --clean $BUNDLE_INSTALL_FLAGS
+fi
+
+# Only install knapsack after bundle install! Otherwise oddly some native
+# gems could not be found under some circumstance. No idea why, hours wasted.
+retry gem install knapsack fog-aws mime-types
+
+if [ "$SETUP_DB" != "false" ]; then
+    bundle exec rake db:drop db:create db:schema:load db:migrate
+
+    if [ "$GITLAB_DATABASE" = "mysql" ]; then
+        bundle exec rake add_limits_mysql
+    fi
+
+    # EE-only
+    bundle exec rake geo:db:drop geo:db:create geo:db:schema:load geo:db:migrate
 fi

scripts/utils.sh

+retry() {
+    if eval "$@"; then
+        return 0
+    fi
+
+    for i in 2 1; do
+        sleep 3s
+        echo "Retrying $i..."
+        if eval "$@"; then
+            return 0
+        fi
+    done
+    return 1
+}