Feature #1282 - Support serving statically compressed .gz and .br files (#1289)

* Feature #1282 - Support pre-gzipped files

* Fix broken test cases

* Support brotli encoding as well

* Fix for #1276 - support integers and floats as metadata in markdown (#1278)

* Fix for #1276

* Use strconv.Format

* Use map[string]interface{} as variables

* One more file

* Always run all tests before commit

* Get rid of DocFlags

* Fix syntax in caddy.conf

* Update to Go 1.7.4

* Add send_timeout property to fastcgi directive.

* Convert rwc field on FCGIClient from type io.ReadWriteCloser to net.Conn.
* Return HTTP 504 to the client when a timeout occurs.
* In Handler.ServeHTTP(), close the connection before returning an HTTP
502/504.
* Refactor tests and add coverage.

* Return HTTP 504 when FastCGI connect times out.

* test: add unit test for #1283 (#1288)

* After review fixes

* Limit the number of restarts with systemd

* Prevent fd leak

* Prevent fd leak

* Refactor loops

* gofmt

caddyhttp/gzip/gzip.go

@@ -53,9 +53,6 @@ outer:
 			}
 		}
 
-		// Delete this header so gzipping is not repeated later in the chain
-		r.Header.Del("Accept-Encoding")
-
 		// gzipWriter modifies underlying writer at init,
 		// use a discard writer instead to leave ResponseWriter in
 		// original form.

caddyhttp/gzip/gzip_test.go

@@ -91,9 +91,6 @@ func nextFunc(shouldGzip bool) httpserver.Handler {
 		}
 
 		if shouldGzip {
-			if r.Header.Get("Accept-Encoding") != "" {
-				return 0, fmt.Errorf("Accept-Encoding header not expected")
-			}
 			if w.Header().Get("Content-Encoding") != "gzip" {
 				return 0, fmt.Errorf("Content-Encoding must be gzip, found %v", r.Header.Get("Content-Encoding"))
 			}

caddyhttp/gzip/responsefilter.go

@@ -25,6 +25,20 @@ func (l LengthFilter) ShouldCompress(w http.ResponseWriter) bool {
 	return l != 0 && int64(l) <= length
 }
 
+// SkipCompressedFilter is ResponseFilter that will discard already compressed responses
+type SkipCompressedFilter struct{}
+
+// ShouldCompress returns true if served file is not already compressed
+// encodings via https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding
+func (n SkipCompressedFilter) ShouldCompress(w http.ResponseWriter) bool {
+	switch w.Header().Get("Content-Encoding") {
+	case "gzip", "compress", "deflate", "br":
+		return false
+	default:
+		return true
+	}
+}
+
 // ResponseFilterWriter validates ResponseFilters. It writes
 // gzip compressed data if ResponseFilters are satisfied or
 // uncompressed data otherwise.

caddyhttp/gzip/responsefilter_test.go

@@ -87,3 +87,26 @@ func TestResponseFilterWriter(t *testing.T) {
 		}
 	}
 }
+
+func TestResponseGzippedOutput(t *testing.T) {
+	server := Gzip{Configs: []Config{
+		{ResponseFilters: []ResponseFilter{SkipCompressedFilter{}}},
+	}}
+
+	server.Next = httpserver.HandlerFunc(func(w http.ResponseWriter, r *http.Request) (int, error) {
+		w.Header().Set("Content-Encoding", "gzip")
+		w.Write([]byte("gzipped"))
+		return 200, nil
+	})
+
+	r := urlRequest("/")
+	r.Header.Set("Accept-Encoding", "gzip")
+
+	w := httptest.NewRecorder()
+	server.ServeHTTP(w, r)
+	resp := w.Body.String()
+
+	if resp != "gzipped" {
+		t.Errorf("Expected output not to be gzipped")
+	}
+}

caddyhttp/gzip/setup.go

@@ -106,6 +106,8 @@ func gzipParse(c *caddy.Controller) ([]Config, error) {
 			config.RequestFilters = append(config.RequestFilters, DefaultExtFilter())
 		}
 
+		config.ResponseFilters = append(config.ResponseFilters, SkipCompressedFilter{})
+
 		// Response Filters
 		// If min_length is specified, use it.
 		if int64(lengthFilter) != 0 {

caddyhttp/gzip/setup_test.go

@@ -99,3 +99,31 @@ func TestSetup(t *testing.T) {
 		}
 	}
 }
+
+func TestShouldAddResponseFilters(t *testing.T) {
+	configs, err := gzipParse(caddy.NewTestController("http", `gzip { min_length 654 }`))
+
+	if err != nil {
+		t.Errorf("Test expected no error but found: %v", err)
+	}
+	filters := 0
+
+	for _, config := range configs {
+		for _, filter := range config.ResponseFilters {
+			switch filter.(type) {
+			case SkipCompressedFilter:
+				filters++
+			case LengthFilter:
+				filters++
+
+				if filter != LengthFilter(654) {
+					t.Errorf("Expected LengthFilter to have length 654, got: %v", filter)
+				}
+			}
+		}
+
+		if filters != 2 {
+			t.Errorf("Expected 2 response filters to be registered, got: %v", filters)
+		}
+	}
+}

caddyhttp/staticfiles/fileserver.go

@@ -43,6 +43,9 @@ func (fs FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, err
 // serveFile writes the specified file to the HTTP response.
 // name is '/'-separated, not filepath.Separator.
 func (fs FileServer) serveFile(w http.ResponseWriter, r *http.Request, name string) (int, error) {
+
+	location := name
+
 	// Prevent absolute path access on Windows.
 	// TODO remove when stdlib http.Dir fixes this.
 	if runtime.GOOS == "windows" {
@@ -97,19 +100,29 @@ func (fs FileServer) serveFile(w http.ResponseWriter, r *http.Request, name stri
 		for _, indexPage := range IndexPages {
 			index := strings.TrimSuffix(name, "/") + "/" + indexPage
 			ff, err := fs.Root.Open(index)
-			if err == nil {
+			if err != nil {
+				continue
+			}
+
 			// this defer does not leak fds because previous iterations
 			// of the loop must have had an err, so nothing to close
 			defer ff.Close()
+
 			dd, err := ff.Stat()
-				if err == nil {
+			if err != nil {
+				ff.Close()
+				continue
+			}
+
+			// Close previous file - release fd immediately
+			f.Close()
+
 			d = dd
 			f = ff
+			location = index
 			break
 		}
 	}
-		}
-	}
 
 	// Still a directory? (we didn't find an index file)
 	// Return 404 to hide the fact that the folder exists
@@ -121,13 +134,48 @@ func (fs FileServer) serveFile(w http.ResponseWriter, r *http.Request, name stri
 		return http.StatusNotFound, nil
 	}
 
+	filename := d.Name()
+
+	for _, encoding := range staticEncodingPriority {
+		if !strings.Contains(r.Header.Get("Accept-Encoding"), encoding) {
+			continue
+		}
+
+		encodedFile, err := fs.Root.Open(location + staticEncoding[encoding])
+		if err != nil {
+			continue
+		}
+
+		encodedFileInfo, err := encodedFile.Stat()
+		if err != nil {
+			encodedFile.Close()
+			continue
+		}
+
+		// Close previous file - release fd
+		f.Close()
+
+		// Stat is needed for generating valid ETag
+		d = encodedFileInfo
+
+		// Encoded file will be served
+		f = encodedFile
+
+		w.Header().Add("Vary", "Accept-Encoding")
+		w.Header().Set("Content-Encoding", encoding)
+
+		defer f.Close()
+		break
+
+	}
+
 	// Experimental ETag header
 	e := fmt.Sprintf(`W/"%x-%x"`, d.ModTime().Unix(), d.Size())
 	w.Header().Set("ETag", e)
 
 	// Note: Errors generated by ServeContent are written immediately
 	// to the response. This usually only happens if seeking fails (rare).
-	http.ServeContent(w, r, d.Name(), d.ModTime(), f)
+	http.ServeContent(w, r, filename, d.ModTime(), f)
 
 	return http.StatusOK, nil
 }
@@ -168,3 +216,17 @@ var IndexPages = []string{
 	"default.htm",
 	"default.txt",
 }
+
+// staticEncoding is a map of content-encoding to a file extension.
+// If client accepts given encoding (via Accept-Encoding header) and compressed file with given extensions exists
+// it will be served to the client instead of original one.
+var staticEncoding = map[string]string{
+	"gzip": ".gz",
+	"br":   ".br",
+}
+
+// staticEncodingPriority is a list of preferred static encodings (most efficient compression to least one).
+var staticEncodingPriority = []string{
+	"br",
+	"gzip",
+}

caddyhttp/staticfiles/fileserver_test.go

@@ -33,6 +33,12 @@ var (
 var testFiles = map[string]string{
 	"unreachable.html":                                     "<h1>must not leak</h1>",
 	filepath.Join("webroot", "file1.html"):                 "<h1>file1.html</h1>",
+	filepath.Join("webroot", "sub", "gzipped.html"):        "<h1>gzipped.html</h1>",
+	filepath.Join("webroot", "sub", "gzipped.html.gz"):     "gzipped.html.gz",
+	filepath.Join("webroot", "sub", "gzipped.html.gz"):     "gzipped.html.gz",
+	filepath.Join("webroot", "sub", "brotli.html"):         "brotli.html",
+	filepath.Join("webroot", "sub", "brotli.html.gz"):      "brotli.html.gz",
+	filepath.Join("webroot", "sub", "brotli.html.br"):      "brotli.html.br",
 	filepath.Join("webroot", "dirwithindex", "index.html"): "<h1>dirwithindex/index.html</h1>",
 	filepath.Join("webroot", "dir", "file2.html"):          "<h1>dir/file2.html</h1>",
 	filepath.Join("webroot", "dir", "hidden.html"):         "<h1>dir/hidden.html</h1>",
@@ -72,14 +78,14 @@ func TestServeHTTP(t *testing.T) {
 		{
 			url:                 "https://foo/file1.html",
 			expectedStatus:      http.StatusOK,
-			expectedBodyContent: testFiles["file1.html"],
+			expectedBodyContent: testFiles[filepath.Join("webroot", "file1.html")],
 			expectedEtag:        `W/"1e240-13"`,
 		},
 		// Test 3 - access folder with index file with trailing slash
 		{
 			url:                 "https://foo/dirwithindex/",
 			expectedStatus:      http.StatusOK,
-			expectedBodyContent: testFiles[filepath.Join("dirwithindex", "index.html")],
+			expectedBodyContent: testFiles[filepath.Join("webroot", "dirwithindex", "index.html")],
 			expectedEtag:        `W/"1e240-20"`,
 		},
 		// Test 4 - access folder with index file without trailing slash
@@ -119,7 +125,7 @@ func TestServeHTTP(t *testing.T) {
 		{
 			url:                 "https://foo/dirwithindex/index.html",
 			expectedStatus:      http.StatusOK,
-			expectedBodyContent: testFiles[filepath.Join("dirwithindex", "index.html")],
+			expectedBodyContent: testFiles[filepath.Join("webroot", "dirwithindex", "index.html")],
 			expectedEtag:        `W/"1e240-20"`,
 		},
 		// Test 11 - send a request with query params
@@ -158,11 +164,28 @@ func TestServeHTTP(t *testing.T) {
 			url:            "https://foo/%2f..%2funreachable.html",
 			expectedStatus: http.StatusNotFound,
 		},
+		// Test 18 - try to get pre-gzipped file.
+		{
+			url:                 "https://foo/sub/gzipped.html",
+			expectedStatus:      http.StatusOK,
+			expectedBodyContent: testFiles[filepath.Join("webroot", "sub", "gzipped.html.gz")],
+			expectedEtag:        `W/"1e240-f"`,
+		},
+		// Test 19 - try to get pre-brotli encoded file.
+		{
+			url:                 "https://foo/sub/brotli.html",
+			expectedStatus:      http.StatusOK,
+			expectedBodyContent: testFiles[filepath.Join("webroot", "sub", "brotli.html.br")],
+			expectedEtag:        `W/"1e240-e"`,
+		},
 	}
 
 	for i, test := range tests {
 		responseRecorder := httptest.NewRecorder()
 		request, err := http.NewRequest("GET", test.url, nil)
+
+		request.Header.Add("Accept-Encoding", "br,gzip")
+
 		if err != nil {
 			t.Errorf("Test %d: Error making request: %v", i, err)
 		}