Merge branch 'sh-workhorse-direct-s3' into 'master'

Support Workhorse directly uploading files to S3 [3/3]

See merge request gitlab-org/gitlab-workhorse!466

_support/detect-context.sh

 #!/bin/sh
 
 git grep 'context.\(Background\|TODO\)' | \
-    grep -v -e '^[^:]*_test\.go:' -e '^vendor/' -e '^_support/' -e '^cmd/[^:]*/main.go' | \
+    grep -v -e '^[^:]*_test\.go:' -v -e "lint:allow context.Background" -e '^vendor/' -e '^_support/' -e '^cmd/[^:]*/main.go' | \
     grep -e '^[^:]*\.go' | \
     awk '{
   print "Found disallowed use of context.Background or TODO"

changelogs/unreleased/sh-workhorse-direct-s3.yml

+---
+title: Support Workhorse directly uploading files to S3
+merge_request: 466
+author:
+type: added

config.toml.example

+[redis]
+URL = "unix:/home/git/gitlab/redis/redis.socket"
+
+[object_storage]
+  enabled = false
+  provider = "AWS"
+
+[object_storage.s3]
+  aws_access_key_id = "YOUR AWS ACCESS KEY"
+  aws_secret_access_key = "YOUR AWS SECRET ACCESS KEY"

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/BurntSushi/toml v0.3.1
 	github.com/FZambia/sentinel v1.0.0
 	github.com/alecthomas/chroma v0.7.3
+	github.com/aws/aws-sdk-go v1.31.7
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/getsentry/raven-go v0.1.2
 	github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721
@@ -15,17 +16,19 @@ require (
 	github.com/grpc-ecosystem/go-grpc-middleware v1.0.0
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/jfbus/httprs v0.0.0-20190827093123-b0af8319bb15
+	github.com/johannesboyne/gofakes3 v0.0.0-20200510090907-02d71f533bec
 	github.com/jpillora/backoff v0.0.0-20170918002102-8eab2debe79d
 	github.com/prometheus/client_golang v1.0.0
 	github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1
 	github.com/sebest/xff v0.0.0-20160910043805-6c115e0ffa35
+	github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 // indirect
 	github.com/sirupsen/logrus v1.3.0
 	github.com/stretchr/testify v1.5.1
 	gitlab.com/gitlab-org/gitaly v1.74.0
 	gitlab.com/gitlab-org/labkit v0.0.0-20200520155818-96e583c57891
 	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f
-	golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa
-	golang.org/x/tools v0.0.0-20200117161641-43d50277825c
+	golang.org/x/net v0.0.0-20200226121028-0de0cce0169b
+	golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375
 	google.golang.org/grpc v1.24.0
 	gopkg.in/yaml.v2 v2.2.8 // indirect
 	honnef.co/go/tools v0.0.1-2019.2.3

go.sum

@@ -43,11 +43,15 @@ github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1p
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/aws/aws-sdk-go v1.17.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aws/aws-sdk-go v1.31.7 h1:TCA+pXKvzDMA3vVqhK21cCy5GarC8pTQb/DrVOWI3iY=
+github.com/aws/aws-sdk-go v1.31.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/aymerick/raymond v2.0.2+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
 github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/certifi/gocertifi v0.0.0-20180905225744-ee1a9a0726d2 h1:MmeatFT1pTPSVb4nkPmBFN/LRZ97vPjsFKsZrU3KKTs=
 github.com/certifi/gocertifi v0.0.0-20180905225744-ee1a9a0726d2/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
@@ -101,6 +105,7 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
 github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
@@ -172,6 +177,11 @@ github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62
 github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw=
 github.com/jfbus/httprs v0.0.0-20190827093123-b0af8319bb15 h1:HPqgCwRiChGXITjjipDuTJYVPkAUpM4lp0mfo7ONpjo=
 github.com/jfbus/httprs v0.0.0-20190827093123-b0af8319bb15/go.mod h1:hve3GCzwH1IcxgpZ3UN4XKAPSKoIqJhsYF2ZifruodQ=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
+github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
+github.com/johannesboyne/gofakes3 v0.0.0-20200510090907-02d71f533bec h1:jEZFmuFe51KdrceqM4NL3dJiuog0zojzcN/VculG26o=
+github.com/johannesboyne/gofakes3 v0.0.0-20200510090907-02d71f533bec/go.mod h1:fNiSoOiEI5KlkWXn26OwKnNe58ilTIkpBlgOrt7Olu8=
 github.com/jpillora/backoff v0.0.0-20170918002102-8eab2debe79d h1:ix3WmphUvN0GDd0DO9MH0v6/5xTv+Xm1bPN+1UJn58k=
 github.com/jpillora/backoff v0.0.0-20170918002102-8eab2debe79d/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -289,10 +299,16 @@ github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1/go.mod h1:J
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8=
+github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8=
 github.com/sebest/xff v0.0.0-20160910043805-6c115e0ffa35 h1:eajwn6K3weW5cd1ZXLu2sJ4pvwlBiCWY4uDejOr73gM=
 github.com/sebest/xff v0.0.0-20160910043805-6c115e0ffa35/go.mod h1:wozgYq9WEBQBaIJe4YZ0qTSFAMxmcwBhQH0fO0R34Z0=
 github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63 h1:J6qvD6rbmOil46orKqJaRPG+zTpoGlBTUdyv8ki63L0=
+github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63/go.mod h1:n+VKSARF5y/tS9XFSP7vWDfS+GUC5vs/YT7M5XDTUEM=
+github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 h1:WnNuhiq+FOY3jNj6JXFT+eLN3CQ/oPIsDPRanvwsmbI=
+github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME=
@@ -304,6 +320,7 @@ github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:s
 github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
@@ -347,11 +364,10 @@ github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmv
 github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
 github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
 github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 gitlab.com/gitlab-org/gitaly v1.74.0 h1:yVWvO2ulWMwyzAaehgGwGjv1lBPEPvXD9p4ovFhTkbc=
 gitlab.com/gitlab-org/gitaly v1.74.0/go.mod h1:AXxSyIMuGdv25WogKreZJRGSiWGkySYvDKwBaWD4bnc=
 gitlab.com/gitlab-org/labkit v0.0.0-20190221122536-0c3fc7cdd57c/go.mod h1:rYhLgfrbEcyfinG+R3EvKu6bZSsmwQqcXzLfHWSfUKM=
-gitlab.com/gitlab-org/labkit v0.0.0-20200507062444-0149780c759d h1:Q5yZi+AelheHuvq/OK6DiaBzLU1AHrm7eWh88uE8Tsk=
-gitlab.com/gitlab-org/labkit v0.0.0-20200507062444-0149780c759d/go.mod h1:SNfxkfUwVNECgtmluVayv0GWFgEjjBs5AzgsowPQuo0=
 gitlab.com/gitlab-org/labkit v0.0.0-20200520155818-96e583c57891 h1:WiCGS5C0B0h+/dh5O7kUJoEZt34O/tbsis9QghNB3gE=
 gitlab.com/gitlab-org/labkit v0.0.0-20200520155818-96e583c57891/go.mod h1:SNfxkfUwVNECgtmluVayv0GWFgEjjBs5AzgsowPQuo0=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
@@ -394,6 +410,8 @@ golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCc
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -404,6 +422,7 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190310074541-c10a0554eabf/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -417,6 +436,10 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa h1:F+8P+gmewFQYRk6JoLQLwjBCTu3mcIURZfNkVweuRKA=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -438,6 +461,7 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190310054646-10058d7d4faa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -466,6 +490,7 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190308174544-00c44ba9c14f/go.mod h1:25r3+/G6/xytQM8iWZKq3Hn0kr0rgFKPUNVEL/dr3z4=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -479,12 +504,16 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac h1:MQEvx39qSf8vyrx3XRaOe+j
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200117161641-43d50277825c h1:2EA2K0k9bcvvEDlqD8xdlOhCOqq+O/p9Voqi4x9W1YU=
 golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375 h1:SjQ2+AKWgZLc1xej6WSzL+Dfs5Uyd5xcZH1mGC411IA=
+golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=

internal/config/config_test.go

+package config
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestLoadObjectStorageConfig(t *testing.T) {
+	config := `
+[object_storage]
+enabled = true
+provider = "AWS"
+
+[object_storage.s3]
+aws_access_key_id = "minio"
+aws_secret_access_key = "gdk-minio"
+`
+	tmpFile, err := ioutil.TempFile(os.TempDir(), "test-")
+	require.NoError(t, err)
+
+	defer os.Remove(tmpFile.Name())
+
+	_, err = tmpFile.Write([]byte(config))
+	require.NoError(t, err)
+
+	cfg, err := LoadConfig(tmpFile.Name())
+	require.NoError(t, err)
+
+	require.NotNil(t, cfg.ObjectStorageCredentials, "Expected object storage credentials")
+
+	expected := ObjectStorageCredentials{
+		Provider: "AWS",
+		S3Credentials: S3Credentials{
+			AwsAccessKeyID:     "minio",
+			AwsSecretAccessKey: "gdk-minio",
+		},
+	}
+
+	require.Equal(t, expected, *cfg.ObjectStorageCredentials)
+}

internal/filestore/file_handler.go

@@ -11,6 +11,8 @@ import (
 
 	"github.com/dgrijalva/jwt-go"
 
+	"gitlab.com/gitlab-org/labkit/log"
+
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/objectstore"
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/secret"
 )
@@ -115,7 +117,16 @@ func SaveFileFromReader(ctx context.Context, reader io.Reader, size int64, opts
 		}
 	}()
 
-	if opts.IsMultipart() {
+	useS3Client := opts.UseWorkhorseClientEnabled() && opts.ObjectStorageConfig.IsAWS() && opts.ObjectStorageConfig.IsValid()
+
+	if useS3Client {
+		remoteWriter, err = objectstore.NewS3Object(ctx, opts.RemoteTempObjectID, opts.ObjectStorageConfig.S3Credentials, opts.ObjectStorageConfig.S3Config, opts.Deadline)
+		if err != nil {
+			return nil, err
+		}
+
+		writers = append(writers, remoteWriter)
+	} else if opts.IsMultipart() {
 		remoteWriter, err = objectstore.NewMultipart(ctx, opts.PresignedParts, opts.PresignedCompleteMultipart, opts.PresignedAbortMultipart, opts.PresignedDelete, opts.PutHeaders, opts.Deadline, opts.PartSize)
 		if err != nil {
 			return nil, err
@@ -154,6 +165,24 @@ func SaveFileFromReader(ctx context.Context, reader io.Reader, size int64, opts
 		return nil, SizeError(fmt.Errorf("expected %d bytes but got only %d", size, fh.Size))
 	}
 
+	logger := log.WithContextFields(ctx, log.Fields{
+		"copied_bytes":     fh.Size,
+		"is_local":         opts.IsLocal(),
+		"is_multipart":     opts.IsMultipart(),
+		"is_remote":        opts.IsRemote(),
+		"remote_id":        opts.RemoteID,
+		"temp_file_prefix": opts.TempFilePrefix,
+		"use_s3_client":    useS3Client,
+	})
+
+	if opts.IsLocal() {
+		logger = logger.WithField("local_temp_path", opts.LocalTempPath)
+	} else if useS3Client {
+		logger = logger.WithField("remote_temp_object", opts.RemoteTempObjectID)
+	}
+
+	logger.Info("saved file")
+
 	fh.hashes = hashes.finish()
 
 	if opts.IsRemote() {

internal/filestore/file_handler_test.go

@@ -275,6 +275,32 @@ func TestSaveFile(t *testing.T) {
 	}
 }
 
+func TestSaveFileWithWorkhorseClient(t *testing.T) {
+	s3Creds, s3Config, sess, ts := test.SetupS3(t)
+	defer ts.Close()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	remoteObject := "tmp/test-file/1"
+	opts := filestore.SaveFileOpts{
+		RemoteID:           "test-file",
+		Deadline:           testDeadline(),
+		UseWorkhorseClient: true,
+		RemoteTempObjectID: remoteObject,
+		ObjectStorageConfig: filestore.ObjectStorageConfig{
+			Provider:      "AWS",
+			S3Credentials: s3Creds,
+			S3Config:      s3Config,
+		},
+	}
+
+	_, err := filestore.SaveFileFromReader(ctx, strings.NewReader(test.ObjectContent), test.ObjectSize, &opts)
+	require.NoError(t, err)
+
+	test.S3ObjectExists(t, sess, s3Config, remoteObject, test.ObjectContent)
+}
+
 func TestSaveMultipartInBodyFailure(t *testing.T) {
 	assert := assert.New(t)
 

internal/objectstore/s3_object.go

+package objectstore
+
+import (
+	"context"
+	"io"
+	"time"
+
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/config"
+
+	"github.com/aws/aws-sdk-go/aws"
+
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+
+	"gitlab.com/gitlab-org/labkit/log"
+)
+
+type S3Object struct {
+	credentials config.S3Credentials
+	config      config.S3Config
+	objectName  string
+	uploader
+}
+
+func NewS3Object(ctx context.Context, objectName string, s3Credentials config.S3Credentials, s3Config config.S3Config, deadline time.Time) (*S3Object, error) {
+	pr, pw := io.Pipe()
+	objectStorageUploadsOpen.Inc()
+	uploadCtx, cancelFn := context.WithDeadline(ctx, deadline)
+
+	o := &S3Object{
+		uploader:    newUploader(uploadCtx, pw),
+		credentials: s3Credentials,
+		config:      s3Config,
+	}
+
+	go o.trackUploadTime()
+	go o.cleanup(ctx)
+
+	go func() {
+		defer cancelFn()
+		defer objectStorageUploadsOpen.Dec()
+		defer func() {
+			// This will be returned as error to the next write operation on the pipe
+			pr.CloseWithError(o.uploadError)
+		}()
+
+		sess, err := setupS3Session(s3Credentials, s3Config)
+		if err != nil {
+			o.uploadError = err
+			log.WithError(err).Error("error creating S3 session")
+			return
+		}
+
+		o.objectName = objectName
+		uploader := s3manager.NewUploader(sess)
+
+		_, err = uploader.UploadWithContext(uploadCtx, &s3manager.UploadInput{
+			Bucket: aws.String(s3Config.Bucket),
+			Key:    aws.String(objectName),
+			Body:   pr,
+		})
+		if err != nil {
+			o.uploadError = err
+			objectStorageUploadRequestsRequestFailed.Inc()
+			log.WithError(err).Error("error uploading S3 session")
+			return
+		}
+	}()
+
+	return o, nil
+}
+
+func (o *S3Object) trackUploadTime() {
+	started := time.Now()
+	<-o.ctx.Done()
+	objectStorageUploadTime.Observe(time.Since(started).Seconds())
+}
+
+func (o *S3Object) cleanup(ctx context.Context) {
+	// wait for the upload to finish
+	<-o.ctx.Done()
+
+	if o.uploadError != nil {
+		objectStorageUploadRequestsRequestFailed.Inc()
+		o.delete()
+		return
+	}
+
+	// We have now successfully uploaded the file to object storage. Another
+	// goroutine will hand off the object to gitlab-rails.
+	<-ctx.Done()
+
+	// gitlab-rails is now done with the object so it's time to delete it.
+	o.delete()
+}
+
+func (o *S3Object) delete() {
+	if o.objectName == "" {
+		return
+	}
+
+	session, err := setupS3Session(o.credentials, o.config)
+	if err != nil {
+		log.WithError(err).Error("error setting up S3 session in delete")
+		return
+	}
+
+	svc := s3.New(session)
+	input := &s3.DeleteObjectInput{
+		Bucket: aws.String(o.config.Bucket),
+		Key:    aws.String(o.objectName),
+	}
+
+	// Note we can't use the request context because in a successful
+	// case, the original request has already completed.
+	deleteCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second) // lint:allow context.Background
+	defer cancel()
+
+	_, err = svc.DeleteObjectWithContext(deleteCtx, input)
+	if err != nil {
+		log.WithError(err).Error("error deleting S3 object", err)
+	}
+}

internal/objectstore/s3_object_test.go

+package objectstore_test
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/stretchr/testify/require"
+
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/config"
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/objectstore"
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/objectstore/test"
+)
+
+func TestS3ObjectUpload(t *testing.T) {
+	creds, config, sess, ts := test.SetupS3(t)
+	defer ts.Close()
+
+	deadline := time.Now().Add(testTimeout)
+	tmpDir, err := ioutil.TempDir("", "workhorse-test-")
+	require.NoError(t, err)
+	defer os.Remove(tmpDir)
+
+	objectName := filepath.Join(tmpDir, "s3-test-data")
+	ctx, cancel := context.WithCancel(context.Background())
+
+	object, err := objectstore.NewS3Object(ctx, objectName, creds, config, deadline)
+	require.NoError(t, err)
+
+	// copy data
+	n, err := io.Copy(object, strings.NewReader(test.ObjectContent))
+	require.NoError(t, err)
+	require.Equal(t, test.ObjectSize, n, "Uploaded file mismatch")
+
+	// close HTTP stream
+	err = object.Close()
+	require.NoError(t, err)
+
+	test.S3ObjectExists(t, sess, config, objectName, test.ObjectContent)
+
+	cancel()
+	deleted := false
+
+	retry(3, time.Second, func() error {
+		if test.S3ObjectDoesNotExist(t, sess, config, objectName) {
+			deleted = true
+			return nil
+		} else {
+			return fmt.Errorf("file is still present, retrying")
+		}
+	})
+
+	require.True(t, deleted)
+}
+
+func TestConcurrentS3ObjectUpload(t *testing.T) {
+	creds, uploadsConfig, uploadsSession, uploadServer := test.SetupS3WithBucket(t, "uploads")
+	defer uploadServer.Close()
+
+	// This will return a separate S3 endpoint
+	_, artifactsConfig, artifactsSession, artifactsServer := test.SetupS3WithBucket(t, "artifacts")
+	defer artifactsServer.Close()
+
+	deadline := time.Now().Add(testTimeout)
+	tmpDir, err := ioutil.TempDir("", "workhorse-test-")
+	require.NoError(t, err)
+	defer os.Remove(tmpDir)
+
+	var wg sync.WaitGroup
+
+	for i := 0; i < 4; i++ {
+		wg.Add(1)
+
+		go func(index int) {
+			var sess *session.Session
+			var config config.S3Config
+
+			if index%2 == 0 {
+				sess = uploadsSession
+				config = uploadsConfig
+			} else {
+				sess = artifactsSession
+				config = artifactsConfig
+			}
+
+			name := fmt.Sprintf("s3-test-data-%d", index)
+			objectName := filepath.Join(tmpDir, name)
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+
+			object, err := objectstore.NewS3Object(ctx, objectName, creds, config, deadline)
+			require.NoError(t, err)
+
+			// copy data
+			n, err := io.Copy(object, strings.NewReader(test.ObjectContent))
+			require.NoError(t, err)
+			require.Equal(t, test.ObjectSize, n, "Uploaded file mismatch")
+
+			// close HTTP stream
+			require.NoError(t, object.Close())
+
+			test.S3ObjectExists(t, sess, config, objectName, test.ObjectContent)
+			wg.Done()
+		}(i)
+	}
+
+	wg.Wait()
+}
+
+func TestS3ObjectUploadCancel(t *testing.T) {
+	creds, config, _, ts := test.SetupS3(t)
+	defer ts.Close()
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	deadline := time.Now().Add(testTimeout)
+	tmpDir, err := ioutil.TempDir("", "workhorse-test-")
+	require.NoError(t, err)
+	defer os.Remove(tmpDir)
+
+	objectName := filepath.Join(tmpDir, "s3-test-data")
+
+	object, err := objectstore.NewS3Object(ctx, objectName, creds, config, deadline)
+
+	require.NoError(t, err)
+
+	// Cancel the transfer before the data has been copied to ensure
+	// we handle this gracefully.
+	cancel()
+
+	_, err = io.Copy(object, strings.NewReader(test.ObjectContent))
+	require.Error(t, err)
+}
+
+func retry(attempts int, sleep time.Duration, fn func() error) error {
+	if err := fn(); err != nil {
+		if s, ok := err.(stop); ok {
+			// Return the original error for later checking
+			return s.error
+		}
+
+		if attempts--; attempts > 0 {
+			time.Sleep(sleep)
+			return retry(attempts, 2*sleep, fn)
+		}
+		return err
+	}
+	return nil
+}
+
+type stop struct {
+	error
+}

internal/objectstore/s3_session.go

+package objectstore
+
+import (
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/config"
+)
+
+type s3Session struct {
+	session *session.Session
+	expiry  time.Time
+}
+
+type s3SessionCache struct {
+	// An S3 session is cached by its input configuration (e.g. region,
+	// endpoint, path style, etc.), but the bucket is actually
+	// determined by the type of object to be uploaded (e.g. CI
+	// artifact, LFS, etc.) during runtime. In practice, we should only
+	// need one session per Workhorse process if we only allow one
+	// configuration for many different buckets. However, using a map
+	// indexed by the config avoids potential pitfalls in case the
+	// bucket configuration is supplied at startup or we need to support
+	// multiple S3 endpoints.
+	sessions map[config.S3Config]*s3Session
+	sync.Mutex
+}
+
+func (s *s3Session) isExpired() bool {
+	return time.Now().After(s.expiry)
+}
+
+func newS3SessionCache() *s3SessionCache {
+	cache := &s3SessionCache{sessions: make(map[config.S3Config]*s3Session)}
+
+	return cache
+}
+
+var (
+	// By default, it looks like IAM instance profiles may last 6 hours
+	// (via curl http://169.254.169.254/latest/meta-data/iam/security-credentials/<role_name>),
+	// but this may be configurable from anywhere for 15 minutes to 12
+	// hours. To be safe, refresh AWS sessions every 10 minutes.
+	sessionExpiration = time.Duration(10 * time.Minute)
+	sessionCache      = newS3SessionCache()
+)
+
+// SetupS3Session initializes a new AWS S3 session and refreshes one if
+// necessary. As recommended in https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/sessions.html,
+// sessions should be cached when possible. Sessions are safe to use
+// concurrently as long as the session isn't modified.
+func setupS3Session(s3Credentials config.S3Credentials, s3Config config.S3Config) (*session.Session, error) {
+	sessionCache.Lock()
+	defer sessionCache.Unlock()
+
+	s, ok := sessionCache.sessions[s3Config]
+
+	if !ok {
+		s = &s3Session{}
+		sessionCache.sessions[s3Config] = s
+	} else if s.session != nil && !s.isExpired() {
+		return s.session.Copy(), nil
+	}
+
+	cfg := &aws.Config{
+		Region:           aws.String(s3Config.Region),
+		S3ForcePathStyle: aws.Bool(s3Config.PathStyle),
+	}
+
+	// In case IAM profiles aren't being used, use the static credentials
+	if s3Credentials.AwsAccessKeyID != "" && s3Credentials.AwsSecretAccessKey != "" {
+		cfg.Credentials = credentials.NewStaticCredentials(s3Credentials.AwsAccessKeyID, s3Credentials.AwsSecretAccessKey, "")
+	}
+
+	if s3Config.Endpoint != "" {
+		cfg.Endpoint = aws.String(s3Config.Endpoint)
+	}
+
+	sess, err := session.NewSession(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	s.expiry = time.Now().Add(sessionExpiration)
+	s.session = sess
+
+	return sess.Copy(), nil
+}
+
+func ResetS3Session(s3Config config.S3Config) {
+	sessionCache.Lock()
+	defer sessionCache.Unlock()
+
+	s, ok := sessionCache.sessions[s3Config]
+	if ok {
+		s.session = nil
+	}
+}

internal/objectstore/s3_session_test.go

+package objectstore
+
+import (
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/stretchr/testify/require"
+
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/config"
+)
+
+func TestS3SessionSetup(t *testing.T) {
+	credentials := config.S3Credentials{}
+	cfg := config.S3Config{Region: "us-west-1", PathStyle: true}
+
+	sess, err := setupS3Session(credentials, cfg)
+	require.NoError(t, err)
+
+	require.Equal(t, aws.StringValue(sess.Config.Region), "us-west-1")
+	require.True(t, aws.BoolValue(sess.Config.S3ForcePathStyle))
+
+	require.Equal(t, len(sessionCache.sessions), 1)
+	anotherConfig := cfg
+	_, err = setupS3Session(credentials, anotherConfig)
+	require.NoError(t, err)
+	require.Equal(t, len(sessionCache.sessions), 1)
+
+	ResetS3Session(cfg)
+}
+
+func TestS3SessionExpiry(t *testing.T) {
+	credentials := config.S3Credentials{}
+	cfg := config.S3Config{Region: "us-west-1", PathStyle: true}
+
+	sess, err := setupS3Session(credentials, cfg)
+	require.NoError(t, err)
+
+	require.Equal(t, aws.StringValue(sess.Config.Region), "us-west-1")
+	require.True(t, aws.BoolValue(sess.Config.S3ForcePathStyle))
+
+	firstSession, ok := sessionCache.sessions[cfg]
+	require.True(t, ok)
+	require.False(t, firstSession.isExpired())
+
+	firstSession.expiry = time.Now().Add(-1 * time.Second)
+	require.True(t, firstSession.isExpired())
+
+	_, err = setupS3Session(credentials, cfg)
+	require.NoError(t, err)
+
+	nextSession, ok := sessionCache.sessions[cfg]
+	require.True(t, ok)
+	require.False(t, nextSession.isExpired())
+
+	ResetS3Session(cfg)
+}

internal/objectstore/test/s3_stub.go

+package test
+
+import (
+	"io/ioutil"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/stretchr/testify/require"
+
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/config"
+
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+
+	"github.com/johannesboyne/gofakes3"
+	"github.com/johannesboyne/gofakes3/backend/s3mem"
+)
+
+func SetupS3(t *testing.T) (config.S3Credentials, config.S3Config, *session.Session, *httptest.Server) {
+	return SetupS3WithBucket(t, "test-bucket")
+}
+
+func SetupS3WithBucket(t *testing.T, bucket string) (config.S3Credentials, config.S3Config, *session.Session, *httptest.Server) {
+	backend := s3mem.New()
+	faker := gofakes3.New(backend)
+	ts := httptest.NewServer(faker.Server())
+
+	creds := config.S3Credentials{
+		AwsAccessKeyID:     "YOUR-ACCESSKEYID",
+		AwsSecretAccessKey: "YOUR-SECRETACCESSKEY",
+	}
+
+	config := config.S3Config{
+		Bucket:    bucket,
+		Endpoint:  ts.URL,
+		Region:    "eu-central-1",
+		PathStyle: true,
+	}
+
+	sess, err := session.NewSession(&aws.Config{
+		Credentials:      credentials.NewStaticCredentials(creds.AwsAccessKeyID, creds.AwsSecretAccessKey, ""),
+		Endpoint:         aws.String(ts.URL),
+		Region:           aws.String(config.Region),
+		DisableSSL:       aws.Bool(true),
+		S3ForcePathStyle: aws.Bool(true),
+	})
+	require.NoError(t, err)
+
+	// Create S3 service client
+	svc := s3.New(sess)
+
+	_, err = svc.CreateBucket(&s3.CreateBucketInput{
+		Bucket: aws.String(bucket),
+	})
+	require.NoError(t, err)
+
+	return creds, config, sess, ts
+}
+
+// S3ObjectExists will fail the test if the file does not exist.
+func S3ObjectExists(t *testing.T, sess *session.Session, config config.S3Config, objectName string, expectedBytes string) {
+	downloadObject(t, sess, config, objectName, func(tmpfile *os.File, numBytes int64, err error) {
+		require.NoError(t, err)
+		require.Equal(t, int64(len(expectedBytes)), numBytes)
+
+		output, err := ioutil.ReadFile(tmpfile.Name())
+		require.NoError(t, err)
+
+		require.Equal(t, []byte(expectedBytes), output)
+	})
+}
+
+// S3ObjectDoesNotExist returns true if the object has been deleted,
+// false otherwise. The return signature is different from
+// S3ObjectExists because deletion may need to be retried since deferred
+// clean up callsinternal/objectstore/test/s3_stub.go may cause the actual deletion to happen after the
+// initial check.
+func S3ObjectDoesNotExist(t *testing.T, sess *session.Session, config config.S3Config, objectName string) bool {
+	deleted := false
+
+	downloadObject(t, sess, config, objectName, func(tmpfile *os.File, numBytes int64, err error) {
+		if err != nil && strings.Contains(err.Error(), "NoSuchKey") {
+			deleted = true
+		}
+	})
+
+	return deleted
+}
+
+func downloadObject(t *testing.T, sess *session.Session, config config.S3Config, objectName string, handler func(tmpfile *os.File, numBytes int64, err error)) {
+	tmpDir, err := ioutil.TempDir("", "workhorse-test-")
+	require.NoError(t, err)
+	defer os.Remove(tmpDir)
+
+	tmpfile, err := ioutil.TempFile(tmpDir, "s3-output")
+	require.NoError(t, err)
+	defer os.Remove(tmpfile.Name())
+
+	downloadSvc := s3manager.NewDownloader(sess)
+	numBytes, err := downloadSvc.Download(tmpfile, &s3.GetObjectInput{
+		Bucket: aws.String(config.Bucket),
+		Key:    aws.String(objectName),
+	})
+
+	handler(tmpfile, numBytes, err)
+}