Merge branch '29772-store-knapsack-reports-in-s3' into 'master'

Save Knapsack reports in S3 instead of in shared cache

Closes #29772

See merge request !10192

.gitlab-ci.yml

@@ -14,13 +14,15 @@ variables:
   GIT_DEPTH: "20"
   PHANTOMJS_VERSION: "2.1.1"
   GET_SOURCES_ATTEMPTS: "3"
+  KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-${CI_COMMIT_REF_SLUG}.json
+  KNAPSACK_SPINACH_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/spinach_report-${CI_COMMIT_REF_SLUG}.json
 
 before_script:
   - source ./scripts/prepare_build.sh
   - cp config/gitlab.yml.example config/gitlab.yml
   - bundle --version
   - '[ "$USE_BUNDLE_INSTALL" != "true" ] || retry bundle install --without postgres production --jobs $(nproc) --clean $FLAGS'
-  - retry gem install knapsack
+  - retry gem install knapsack fog-aws mime-types
   - '[ "$SETUP_DB" != "true" ] || bundle exec rake db:drop db:create db:schema:load db:migrate add_limits_mysql'
 
 stages:
@@ -39,6 +41,7 @@ stages:
   variables:
     SETUP_DB: "false"
     USE_BUNDLE_INSTALL: "false"
+    KNAPSACK_S3_BUCKET: "gitlab-ce-cache"
   cache:
     key: "knapsack"
     paths:
@@ -61,9 +64,9 @@ stages:
     - JOB_NAME=( $CI_JOB_NAME )
     - export CI_NODE_INDEX=${JOB_NAME[1]}
     - export CI_NODE_TOTAL=${JOB_NAME[2]}
-    - export KNAPSACK_REPORT_PATH=knapsack/rspec_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
+    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
     - export KNAPSACK_GENERATE_REPORT=true
-    - cp knapsack/rspec_report.json ${KNAPSACK_REPORT_PATH}
+    - cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
     - knapsack rspec "--color --format documentation"
   artifacts:
     expire_in: 31d
@@ -81,9 +84,9 @@ stages:
     - JOB_NAME=( $CI_JOB_NAME )
     - export CI_NODE_INDEX=${JOB_NAME[1]}
     - export CI_NODE_TOTAL=${JOB_NAME[2]}
-    - export KNAPSACK_REPORT_PATH=knapsack/spinach_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
+    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
     - export KNAPSACK_GENERATE_REPORT=true
-    - cp knapsack/spinach_report.json ${KNAPSACK_REPORT_PATH}
+    - cp ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
     - knapsack spinach "-r rerun" || retry '[[ -e tmp/spinach-rerun.txt ]] && bundle exec spinach -r rerun $(cat tmp/spinach-rerun.txt)'
   artifacts:
     expire_in: 31d
@@ -94,15 +97,31 @@ stages:
       - tmp/capybara/
 
 # Prepare and merge knapsack tests
-
 knapsack:
   <<: *knapsack-state
   <<: *dedicated-runner
   stage: prepare
   script:
-    - mkdir -p knapsack/
-    - '[[ -f knapsack/rspec_report.json ]] || echo "{}" > knapsack/rspec_report.json'
-    - '[[ -f knapsack/spinach_report.json ]] || echo "{}" > knapsack/spinach_report.json'
+    - mkdir -p knapsack/${CI_PROJECT_NAME}/
+    - wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || true
+    - wget -O $KNAPSACK_SPINACH_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_SPINACH_SUITE_REPORT_PATH || true
+    - '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
+    - '[[ -f $KNAPSACK_SPINACH_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_SPINACH_SUITE_REPORT_PATH}'
+
+update-knapsack:
+  <<: *knapsack-state
+  <<: *dedicated-runner
+  stage: post-test
+  script:
+    - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_node_*.json
+    - scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach_node_*.json
+    - '[[ -z ${KNAPSACK_S3_BUCKET} ]] || scripts/sync-reports put $KNAPSACK_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH'
+    - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
+  only:
+    - master@gitlab-org/gitlab-ce
+    - master@gitlab-org/gitlab-ee
+    - master@gitlab/gitlabhq
+    - master@gitlab/gitlab-ee
 
 setup-test-env:
   <<: *use-db
@@ -122,20 +141,6 @@ setup-test-env:
       - public/assets
       - tmp/tests
 
-update-knapsack:
-  <<: *knapsack-state
-  <<: *dedicated-runner
-  stage: post-test
-  script:
-    - scripts/merge-reports knapsack/rspec_report.json knapsack/rspec_node_*.json
-    - scripts/merge-reports knapsack/spinach_report.json knapsack/spinach_node_*.json
-    - rm -f knapsack/*_node_*.json
-  only:
-    - master@gitlab-org/gitlab-ce
-    - master@gitlab-org/gitlab-ee
-    - master@gitlab/gitlabhq
-    - master@gitlab/gitlab-ee
-
 rspec 0 20: *rspec-knapsack
 rspec 1 20: *rspec-knapsack
 rspec 2 20: *rspec-knapsack

scripts/merge-reports

 #!/usr/bin/env ruby
 
 require 'json'
-require 'yaml'
 
 main_report_file = ARGV.shift
 unless main_report_file

scripts/sync-reports

+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'fog/aws'
+
+class SyncReports
+  ACTIONS = %w[get put].freeze
+
+  attr_reader :options
+
+  def initialize(options)
+    @options = options
+
+    perform_sync!
+  end
+
+  private
+
+  def perform_sync!
+    case options[:action]
+    when 'get'
+      get_reports!
+    when 'put'
+      put_reports!
+    end
+  end
+
+  def get_reports!
+    options[:report_paths].each { |report_path| get_report!(report_path) }
+  end
+
+  def put_reports!
+    options[:report_paths].each { |report_path| put_report!(report_path) }
+  end
+
+  def get_report!(report_path)
+    file = bucket.files.get(report_path)
+
+    if file.respond_to?(:body)
+      File.write(report_path, file.body)
+      puts "#{report_path} was retrieved from S3."
+    else
+      puts "#{report_path} does not seem to exist on S3."
+    end
+  end
+
+  def put_report!(report_path)
+    bucket.files.create(
+      key: report_path,
+      body: File.open(report_path),
+      public: true
+    )
+    puts "#{report_path} was uploaded to S3."
+  end
+
+  def bucket
+    @bucket ||= storage.directories.get(options[:bucket])
+  end
+
+  def storage
+    @storage ||=
+      Fog::Storage.new(
+        provider: 'AWS',
+        aws_access_key_id: ENV['AWS_ACCESS_KEY_ID'],
+        aws_secret_access_key: ENV['AWS_SECRET_ACCESS_KEY']
+      )
+  end
+end
+
+def usage!(error: 'action')
+  print "\n[ERROR]: "
+  case error
+  when 'action'
+    puts "Please specify an action as first argument: #{SyncReports::ACTIONS.join(', ')}\n\n"
+  when 'bucket'
+    puts "Please specify a bucket as second argument!\n\n"
+  when 'files'
+    puts "Please specify one or more file paths as third argument!\n\n"
+  end
+  puts "Usage: #{__FILE__} [get|put] bucket report_path ...\n\n"
+  puts "Note: the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment "\
+       "variables need to be set\n\n"
+  exit 1
+end
+
+if $0 == __FILE__
+  action = ARGV.shift
+  usage!(error: 'action') unless SyncReports::ACTIONS.include?(action)
+
+  bucket = ARGV.shift
+  usage!(error: 'bucket') unless bucket
+  usage!(error: 'files') unless ARGV.any?
+
+  SyncReports.new(action: action, bucket: bucket, report_paths: ARGV)
+end