Merge branch 'ac-ignore-multipart-etag' into 'master'

Ignore CompleteMultipartUpload ETag

See merge request gitlab-org/gitlab-workhorse!438

internal/objectstore/multipart.go

@@ -156,16 +156,7 @@ func (m *Multipart) complete(cmu *CompleteMultipartUpload) error {
 
 	m.extractETag(result.ETag)
 
-	return m.verifyETag(cmu)
-}
-
-func (m *Multipart) verifyETag(cmu *CompleteMultipartUpload) error {
-	expectedChecksum, err := cmu.BuildMultipartUploadETag()
-	if err != nil {
-		return err
-	}
-
-	return compareMD5(expectedChecksum, m.etag)
+	return nil
 }
 
 func (m *Multipart) readAndUploadOnePart(partURL string, putHeaders map[string]string, src io.Reader, partNumber int) (*completeMultipartUploadPart, error) {

internal/objectstore/multipart_test.go

@@ -2,7 +2,6 @@ package objectstore_test
 
 import (
 	"context"
-	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -36,12 +35,10 @@ func TestMultipartUploadWithUpcaseETags(t *testing.T) {
 
 		// POST with CompleteMultipartUpload request
 		if r.Method == "POST" {
-			expectedETag := "6e6b164c392b04bfbb82368179d9ade2-1"
-			completeBody := fmt.Sprintf(`<CompleteMultipartUploadResult>
-                                                       <Bucket>test-bucket</Bucket>
-			                               <ETag>%s</ETag>
-  	                                             </CompleteMultipartUploadResult>`,
-				strings.ToUpper(expectedETag))
+			completeBody := `<CompleteMultipartUploadResult>
+			                   <Bucket>test-bucket</Bucket>
+			                   <ETag>No Longer Checked</ETag>
+			                 </CompleteMultipartUploadResult>`
 			postCnt++
 
 			w.Write([]byte(completeBody))

internal/objectstore/s3_complete_multipart_api.go

 package objectstore
 
 import (
-	"crypto/md5"
-	"encoding/hex"
 	"encoding/xml"
 	"fmt"
 )
@@ -51,25 +49,3 @@ type compoundCompleteMultipartUploadResult struct {
 func (c *compoundCompleteMultipartUploadResult) isError() bool {
 	return c.CompleteMultipartUploadError != nil
 }
-
-// BuildMultipartUploadETag creates an S3 compatible ETag for MultipartUpload
-// Given the MD5 hash for each uploaded part of the file, concatenate
-// the hashes into a single binary string and calculate the MD5 hash of that result,
-// the append "-len(etags)"
-// http://permalink.gmane.org/gmane.comp.file-systems.s3.s3tools/583
-func (cmu *CompleteMultipartUpload) BuildMultipartUploadETag() (string, error) {
-	hasher := md5.New()
-	for _, part := range cmu.Part {
-		checksum, err := hex.DecodeString(part.ETag)
-		if err != nil {
-			return "", err
-		}
-		_, err = hasher.Write(checksum)
-		if err != nil {
-			return "", err
-		}
-	}
-
-	multipartChecksum := hasher.Sum(nil)
-	return fmt.Sprintf("%s-%d", hex.EncodeToString(multipartChecksum), len(cmu.Part)), nil
-}

internal/objectstore/s3_complete_multipart_api_test.go

-package objectstore
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestMultipartUploadETag(t *testing.T) {
-	cmu := CompleteMultipartUpload{
-		Part: []*completeMultipartUploadPart{
-			{PartNumber: 1, ETag: "550cf6b6e60f65a0e3104a26e70fea42"},
-			{PartNumber: 2, ETag: "920b914bca0a70780b40881b8f376135"},
-			{PartNumber: 3, ETag: "175719e13d23c021058bc7376696f26f"},
-		},
-	}
-	expectedETag := "1dc6ab8f946f699770f14f46a8739671-3"
-
-	etag, err := cmu.BuildMultipartUploadETag()
-	require.NoError(t, err)
-	require.Equal(t, expectedETag, etag)
-}

internal/objectstore/test/objectstore_stub.go

@@ -220,11 +220,7 @@ func (o *ObjectstoreStub) completeMultipartUpload(w http.ResponseWriter, r *http
 
 	etag, overwritten := o.overwriteMD5[objectPath]
 	if !overwritten {
-		etag, err = msg.BuildMultipartUploadETag()
-		if err != nil {
-			http.Error(w, "cannot compute ETag", 400)
-			return
-		}
+		etag = "CompleteMultipartUploadETag"
 	}
 
 	o.bucket[objectPath] = etag

internal/objectstore/test/objectstore_stub_test.go

@@ -106,7 +106,6 @@ func TestObjectStoreCompleteMultipartUpload(t *testing.T) {
 			contentMD5: "920b914bca0a70780b40881b8f376135",
 		},
 	}
-	expectedETag := "2f2f82eceacf5bd0ac5d7c3d3d388849-2"
 
 	stub.InitiateMultipartUpload(ObjectPath)
 
@@ -149,7 +148,6 @@ func TestObjectStoreCompleteMultipartUpload(t *testing.T) {
 
 	assert.Equal(len(parts), stub.PutsCnt())
 	assert.Equal(0, stub.DeletesCnt())
-	assert.Equal(expectedETag, stub.GetObjectMD5(ObjectPath))
 	assert.False(stub.IsMultipartUpload(ObjectPath), "MultipartUpload is still in progress")
 }