Added new compress post-processor, contributed by Vasiliy Tolstov

post-processor/compress/LICENSE

+The MIT License (MIT)
+
+Copyright (c) 2014 Vasiliy Tolstov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

post-processor/compress/artifact.go

@@ -5,40 +5,34 @@ import (
 	"os"
 )
 
-const BuilderId = "packer.post-processor.compress"
+const BuilderId = "vtolstov.compress"
 
 type Artifact struct {
-	Path     string
-	Provider string
+	builderId string
+	dir       string
+	f         []string
 }
 
-func NewArtifact(provider, path string) *Artifact {
-	return &Artifact{
-		Path:     path,
-		Provider: provider,
-	}
-}
-
-func (*Artifact) BuilderId() string {
+func (a *Artifact) BuilderId() string {
 	return BuilderId
 }
 
-func (self *Artifact) Id() string {
-	return ""
+func (a *Artifact) Files() []string {
+	return a.f
 }
 
-func (self *Artifact) Files() []string {
-	return []string{self.Path}
+func (*Artifact) Id() string {
+	return "COMPRESS"
 }
 
-func (self *Artifact) String() string {
-	return fmt.Sprintf("'%s' compressing: %s", self.Provider, self.Path)
+func (a *Artifact) String() string {
+	return fmt.Sprintf("VM compressed files in directory: %s", a.dir)
 }
 
 func (*Artifact) State(name string) interface{} {
 	return nil
 }
 
-func (self *Artifact) Destroy() error {
-	return os.Remove(self.Path)
+func (a *Artifact) Destroy() error {
+	return os.RemoveAll(a.dir)
 }

post-processor/compress/benchmark.go

+// +build ignore
+
+package main
+
+import (
+	"compress/flate"
+	gzip "compress/gzip"
+	"io"
+	"io/ioutil"
+	"fmt"
+	"os"
+	"runtime"
+	"testing"
+
+	bgzf "github.com/biogo/hts/bgzf"
+	pgzip "github.com/klauspost/pgzip"
+	lz4 "github.com/pierrec/lz4"
+)
+
+type Compressor struct {
+	r  *os.File
+	w  *os.File
+	sr int64
+	sw int64
+}
+
+func (c *Compressor) Close() error {
+	var err error
+
+	fi, _ := c.w.Stat()
+	c.sw = fi.Size()
+	if err = c.w.Close(); err != nil {
+		return err
+	}
+
+	fi, _ = c.r.Stat()
+	c.sr = fi.Size()
+	if err = c.r.Close(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func NewCompressor(src, dst string) (*Compressor, error) {
+	r, err := os.Open(src)
+	if err != nil {
+		return nil, err
+	}
+
+	w, err := os.Create(dst)
+	if err != nil {
+		r.Close()
+		return nil, err
+	}
+
+	c := &Compressor{r: r, w: w}
+	return c, nil
+}
+
+func main() {
+
+        runtime.GOMAXPROCS(runtime.NumCPU())
+
+	var resw testing.BenchmarkResult
+	var resr testing.BenchmarkResult
+
+	c, err := NewCompressor("/tmp/image.r", "/tmp/image.w")
+	if err != nil {
+		panic(err)
+	}
+	resw = testing.Benchmark(c.BenchmarkGZIPWriter)
+	c.w.Seek(0, 0)
+	resr = testing.Benchmark(c.BenchmarkGZIPReader)
+	c.Close()
+	fmt.Printf("gzip:\twriter %s\treader %s\tsize %d\n", resw.T.String(), resr.T.String(), c.sw)
+
+	c, err = NewCompressor("/tmp/image.r", "/tmp/image.w")
+	if err != nil {
+		panic(err)
+	}
+	resw = testing.Benchmark(c.BenchmarkBGZFWriter)
+	c.w.Seek(0, 0)
+	resr = testing.Benchmark(c.BenchmarkBGZFReader)
+	c.Close()
+	fmt.Printf("bgzf:\twriter %s\treader %s\tsize %d\n", resw.T.String(), resr.T.String(), c.sw)
+
+	c, err = NewCompressor("/tmp/image.r", "/tmp/image.w")
+	if err != nil {
+		panic(err)
+	}
+	resw = testing.Benchmark(c.BenchmarkPGZIPWriter)
+	c.w.Seek(0, 0)
+	resr = testing.Benchmark(c.BenchmarkPGZIPReader)
+	c.Close()
+	fmt.Printf("pgzip:\twriter %s\treader %s\tsize %d\n", resw.T.String(), resr.T.String(), c.sw)
+
+	c, err = NewCompressor("/tmp/image.r", "/tmp/image.w")
+	if err != nil {
+		panic(err)
+	}
+	resw = testing.Benchmark(c.BenchmarkLZ4Writer)
+	c.w.Seek(0, 0)
+	resr = testing.Benchmark(c.BenchmarkLZ4Reader)
+	c.Close()
+	fmt.Printf("lz4:\twriter %s\treader %s\tsize %d\n", resw.T.String(), resr.T.String(), c.sw)
+
+}
+
+func (c *Compressor) BenchmarkGZIPWriter(b *testing.B) {
+	cw, _ := gzip.NewWriterLevel(c.w, flate.BestSpeed)
+	b.ResetTimer()
+
+	_, err := io.Copy(cw, c.r)
+	if err != nil {
+		b.Fatal(err)
+	}
+	cw.Close()
+	c.w.Sync()
+}
+
+func (c *Compressor) BenchmarkGZIPReader(b *testing.B) {
+	cr, _ := gzip.NewReader(c.w)
+	b.ResetTimer()
+
+	_, err := io.Copy(ioutil.Discard, cr)
+	if err != nil {
+		b.Fatal(err)
+	}
+}
+
+func (c *Compressor) BenchmarkBGZFWriter(b *testing.B) {
+	cw, _ := bgzf.NewWriterLevel(c.w, flate.BestSpeed, runtime.NumCPU())
+	b.ResetTimer()
+
+	_, err := io.Copy(cw, c.r)
+	if err != nil {
+		b.Fatal(err)
+	}
+	c.w.Sync()
+}
+
+func (c *Compressor) BenchmarkBGZFReader(b *testing.B) {
+	cr, _ := bgzf.NewReader(c.w, 0)
+	b.ResetTimer()
+
+	_, err := io.Copy(ioutil.Discard, cr)
+	if err != nil {
+		b.Fatal(err)
+	}
+}
+
+func (c *Compressor) BenchmarkPGZIPWriter(b *testing.B) {
+	cw, _ := pgzip.NewWriterLevel(c.w, flate.BestSpeed)
+	b.ResetTimer()
+
+	_, err := io.Copy(cw, c.r)
+	if err != nil {
+		b.Fatal(err)
+	}
+	cw.Close()
+	c.w.Sync()
+}
+
+func (c *Compressor) BenchmarkPGZIPReader(b *testing.B) {
+	cr, _ := pgzip.NewReader(c.w)
+	b.ResetTimer()
+
+	_, err := io.Copy(ioutil.Discard, cr)
+	if err != nil {
+		b.Fatal(err)
+	}
+}
+
+func (c *Compressor) BenchmarkLZ4Writer(b *testing.B) {
+	cw := lz4.NewWriter(c.w)
+//	cw.Header.HighCompression = true
+	cw.Header.NoChecksum = true
+	b.ResetTimer()
+
+	_, err := io.Copy(cw, c.r)
+	if err != nil {
+		b.Fatal(err)
+	}
+	cw.Close()
+	c.w.Sync()
+}
+
+func (c *Compressor) BenchmarkLZ4Reader(b *testing.B) {
+	cr := lz4.NewReader(c.w)
+	b.ResetTimer()
+
+	_, err := io.Copy(ioutil.Discard, cr)
+	if err != nil {
+		b.Fatal(err)
+	}
+}

post-processor/compress/post-processor.go

 package compress
 
 import (
-	"archive/tar"
-	"compress/gzip"
+	tar "archive/tar"
+	zip "archive/zip"
+	"compress/flate"
+	gzip "compress/gzip"
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
 
+	bgzf "github.com/biogo/hts/bgzf"
+	pgzip "github.com/klauspost/pgzip"
 	"github.com/mitchellh/packer/common"
-	"github.com/mitchellh/packer/helper/config"
 	"github.com/mitchellh/packer/packer"
-	"github.com/mitchellh/packer/template/interpolate"
+	lz4 "github.com/pierrec/lz4"
+	"gopkg.in/yaml.v2"
 )
 
+type Metadata map[string]Metaitem
+
+type Metaitem struct {
+	CompSize int64  `yaml:"compsize"`
+	OrigSize int64  `yaml:"origsize"`
+	CompType string `yaml:"comptype"`
+	CompDate string `yaml:"compdate"`
+}
+
 type Config struct {
 	common.PackerConfig `mapstructure:",squash"`
 
-	OutputPath string `mapstructure:"output"`
-
-	ctx interpolate.Context
+	OutputPath        string `mapstructure:"output"`
+	OutputFile        string `mapstructure:"file"`
+	Compression       int    `mapstructure:"compression"`
+	Metadata          bool   `mapstructure:"metadata"`
+	NumCPU            int    `mapstructure:"numcpu"`
+	Format            string `mapstructure:"format"`
+	KeepInputArtifact bool   `mapstructure:"keep_input_artifact"`
+	tpl               *packer.ConfigTemplate
 }
 
-type PostProcessor struct {
-	config Config
+type CompressPostProcessor struct {
+	cfg Config
 }
 
-func (self *PostProcessor) Configure(raws ...interface{}) error {
-	err := config.Decode(&self.config, &config.DecodeOpts{
-		Interpolate: true,
-		InterpolateFilter: &interpolate.RenderFilter{
-			Exclude: []string{},
-		},
-	}, raws...)
+func (p *CompressPostProcessor) Configure(raws ...interface{}) error {
+	p.cfg.Compression = -1
+	_, err := common.DecodeConfig(&p.cfg, raws...)
 	if err != nil {
 		return err
 	}
 
+	errs := new(packer.MultiError)
+
+	p.cfg.tpl, err = packer.NewConfigTemplate()
+	if err != nil {
+		return err
+	}
+	p.cfg.tpl.UserVars = p.cfg.PackerUserVars
+
+	if p.cfg.OutputPath == "" {
+		p.cfg.OutputPath = "packer_{{.BuildName}}_{{.Provider}}"
+	}
+
+	if err = p.cfg.tpl.Validate(p.cfg.OutputPath); err != nil {
+		errs = packer.MultiErrorAppend(
+			errs, fmt.Errorf("Error parsing target template: %s", err))
+	}
+
+	templates := map[string]*string{
+		"output": &p.cfg.OutputPath,
+	}
+
+	if p.cfg.Compression > flate.BestCompression {
+		p.cfg.Compression = flate.BestCompression
+	}
+	if p.cfg.Compression == -1 {
+		p.cfg.Compression = flate.DefaultCompression
+	}
+
+	if p.cfg.NumCPU < 1 {
+		p.cfg.NumCPU = runtime.NumCPU()
+	}
+
+	runtime.GOMAXPROCS(p.cfg.NumCPU)
+
+	for key, ptr := range templates {
+		if *ptr == "" {
+			errs = packer.MultiErrorAppend(
+				errs, fmt.Errorf("%s must be set", key))
+		}
+
+		*ptr, err = p.cfg.tpl.Process(*ptr, nil)
+		if err != nil {
+			errs = packer.MultiErrorAppend(
+				errs, fmt.Errorf("Error processing %s: %s", key, err))
+		}
+	}
+
+	if len(errs.Errors) > 0 {
+		return errs
+	}
+
 	return nil
 
 }
 
-func (self *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) {
-	ui.Say(fmt.Sprintf("Creating archive for '%s'", artifact.BuilderId()))
+func (p *CompressPostProcessor) fillMetadata(metadata Metadata, files []string) Metadata {
+	// layout shows by example how the reference time should be represented.
+	const layout = "2006-01-02_15-04-05"
+	t := time.Now()
 
-	// Create the compressed archive file at the appropriate OutputPath.
-	fw, err := os.Create(self.config.OutputPath)
+	if !p.cfg.Metadata {
+		return metadata
+	}
+	for _, f := range files {
+		if fi, err := os.Stat(f); err != nil {
+			continue
+		} else {
+			if i, ok := metadata[filepath.Base(f)]; !ok {
+				metadata[filepath.Base(f)] = Metaitem{CompType: p.cfg.Format, OrigSize: fi.Size(), CompDate: t.Format(layout)}
+			} else {
+				i.CompSize = fi.Size()
+				i.CompDate = t.Format(layout)
+				metadata[filepath.Base(f)] = i
+			}
+		}
+	}
+	return metadata
+}
+
+func (p *CompressPostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) {
+	newartifact := &Artifact{builderId: artifact.BuilderId(), dir: p.cfg.OutputPath}
+	var metafile string = filepath.Join(p.cfg.OutputPath, "metadata")
+
+	_, err := os.Stat(newartifact.dir)
+	if err == nil {
+		return nil, false, fmt.Errorf("output dir must not exists: %s", err)
+	}
+	err = os.MkdirAll(newartifact.dir, 0755)
 	if err != nil {
-		return nil, false, fmt.Errorf(
-			"Failed creating file for compressed archive: %s", self.config.OutputPath)
+		return nil, false, fmt.Errorf("failed to create output: %s", err)
+	}
+
+	formats := strings.Split(p.cfg.Format, ".")
+	files := artifact.Files()
+
+	metadata := make(Metadata, 0)
+	metadata = p.fillMetadata(metadata, files)
+
+	for _, compress := range formats {
+		switch compress {
+		case "tar":
+			files, err = p.cmpTAR(files, filepath.Join(p.cfg.OutputPath, p.cfg.OutputFile))
+			metadata = p.fillMetadata(metadata, files)
+		case "zip":
+			files, err = p.cmpZIP(files, filepath.Join(p.cfg.OutputPath, p.cfg.OutputFile))
+			metadata = p.fillMetadata(metadata, files)
+		case "pgzip":
+			files, err = p.cmpPGZIP(files, p.cfg.OutputPath)
+			metadata = p.fillMetadata(metadata, files)
+		case "gzip":
+			files, err = p.cmpGZIP(files, p.cfg.OutputPath)
+			metadata = p.fillMetadata(metadata, files)
+		case "bgzf":
+			files, err = p.cmpBGZF(files, p.cfg.OutputPath)
+			metadata = p.fillMetadata(metadata, files)
+		case "lz4":
+			files, err = p.cmpLZ4(files, p.cfg.OutputPath)
+			metadata = p.fillMetadata(metadata, files)
+		case "e2fs":
+			files, err = p.cmpE2FS(files, filepath.Join(p.cfg.OutputPath, p.cfg.OutputFile))
+			metadata = p.fillMetadata(metadata, files)
+		}
+		if err != nil {
+			return nil, false, fmt.Errorf("Failed to compress: %s", err)
+		}
+	}
+
+	if p.cfg.Metadata {
+		fp, err := os.Create(metafile)
+		if err != nil {
+			return nil, false, err
+		}
+		if buf, err := yaml.Marshal(metadata); err != nil {
+			fp.Close()
+			return nil, false, err
+		} else {
+			if _, err = fp.Write(buf); err != nil {
+				fp.Close()
+				return nil, false, err
+			}
+			fp.Close()
+		}
+	}
+
+	newartifact.f = append(newartifact.f, files...)
+	if p.cfg.Metadata {
+		newartifact.f = append(newartifact.f, metafile)
+	}
+
+	return newartifact, p.cfg.KeepInputArtifact, nil
+}
+
+func (p *CompressPostProcessor) cmpTAR(src []string, dst string) ([]string, error) {
+	fw, err := os.Create(dst)
+	if err != nil {
+		return nil, fmt.Errorf("tar error: %s", err)
 	}
 	defer fw.Close()
 
-	gw := gzip.NewWriter(fw)
-	defer gw.Close()
+	tw := tar.NewWriter(fw)
+	defer tw.Close()
 
-	// Iterate through all of the artifact's files and put them into the
-	// compressed archive using the tar/gzip writers.
-	for _, path := range artifact.Files() {
-		fi, err := os.Stat(path)
+	for _, name := range src {
+		fi, err := os.Stat(name)
 		if err != nil {
-			return nil, false, fmt.Errorf(
-				"Failed stating file: %s", path)
+			return nil, fmt.Errorf("tar error: %s", err)
 		}
 
-		target, _ := os.Readlink(path)
+		target, _ := os.Readlink(name)
 		header, err := tar.FileInfoHeader(fi, target)
 		if err != nil {
-			return nil, false, fmt.Errorf(
-				"Failed creating archive header: %s", path)
+			return nil, fmt.Errorf("tar erorr: %s", err)
 		}
 
-		tw := tar.NewWriter(gw)
-		defer tw.Close()
-
-		// Write the header first to the archive. This takes partial data
-		// from the FileInfo that is grabbed by running the stat command.
-		if err := tw.WriteHeader(header); err != nil {
-			return nil, false, fmt.Errorf(
-				"Failed writing archive header: %s", path)
+		if err = tw.WriteHeader(header); err != nil {
+			return nil, fmt.Errorf("tar error: %s", err)
 		}
 
-		// Open the target file for archiving and compressing.
-		fr, err := os.Open(path)
+		fr, err := os.Open(name)
 		if err != nil {
-			return nil, false, fmt.Errorf(
-				"Failed opening file '%s' to write compressed archive.", path)
+			return nil, fmt.Errorf("tar error: %s", err)
 		}
-		defer fr.Close()
 
 		if _, err = io.Copy(tw, fr); err != nil {
-			return nil, false, fmt.Errorf(
-				"Failed copying file to archive: %s", path)
+			fr.Close()
+			return nil, fmt.Errorf("tar error: %s", err)
+		}
+		fr.Close()
+	}
+	return []string{dst}, nil
+}
+
+func (p *CompressPostProcessor) cmpGZIP(src []string, dst string) ([]string, error) {
+	var res []string
+	for _, name := range src {
+		filename := filepath.Join(dst, filepath.Base(name))
+		fw, err := os.Create(filename)
+		if err != nil {
+			return nil, fmt.Errorf("gzip error: %s", err)
+		}
+		cw, err := gzip.NewWriterLevel(fw, p.cfg.Compression)
+		if err != nil {
+			fw.Close()
+			return nil, fmt.Errorf("gzip error: %s", err)
+		}
+		fr, err := os.Open(name)
+		if err != nil {
+			cw.Close()
+			fw.Close()
+			return nil, fmt.Errorf("gzip error: %s", err)
+		}
+		if _, err = io.Copy(cw, fr); err != nil {
+			cw.Close()
+			fr.Close()
+			fw.Close()
+			return nil, fmt.Errorf("gzip error: %s", err)
+		}
+		cw.Close()
+		fr.Close()
+		fw.Close()
+		res = append(res, filename)
+	}
+	return res, nil
+}
+
+func (p *CompressPostProcessor) cmpPGZIP(src []string, dst string) ([]string, error) {
+	var res []string
+	for _, name := range src {
+		filename := filepath.Join(dst, filepath.Base(name))
+		fw, err := os.Create(filename)
+		if err != nil {
+			return nil, fmt.Errorf("pgzip error: %s", err)
+		}
+		cw, err := pgzip.NewWriterLevel(fw, p.cfg.Compression)
+		if err != nil {
+			fw.Close()
+			return nil, fmt.Errorf("pgzip error: %s", err)
+		}
+		fr, err := os.Open(name)
+		if err != nil {
+			cw.Close()
+			fw.Close()
+			return nil, fmt.Errorf("pgzip error: %s", err)
+		}
+		if _, err = io.Copy(cw, fr); err != nil {
+			cw.Close()
+			fr.Close()
+			fw.Close()
+			return nil, fmt.Errorf("pgzip error: %s", err)
+		}
+		cw.Close()
+		fr.Close()
+		fw.Close()
+		res = append(res, filename)
+	}
+	return res, nil
+}
+
+func (p *CompressPostProcessor) cmpLZ4(src []string, dst string) ([]string, error) {
+	var res []string
+	for _, name := range src {
+		filename := filepath.Join(dst, filepath.Base(name))
+		fw, err := os.Create(filename)
+		if err != nil {
+			return nil, fmt.Errorf("lz4 error: %s", err)
+		}
+		cw := lz4.NewWriter(fw)
+		if err != nil {
+			fw.Close()
+			return nil, fmt.Errorf("lz4 error: %s", err)
+		}
+		if p.cfg.Compression > flate.DefaultCompression {
+			cw.Header.HighCompression = true
+		}
+		fr, err := os.Open(name)
+		if err != nil {
+			cw.Close()
+			fw.Close()
+			return nil, fmt.Errorf("lz4 error: %s", err)
+		}
+		if _, err = io.Copy(cw, fr); err != nil {
+			cw.Close()
+			fr.Close()
+			fw.Close()
+			return nil, fmt.Errorf("lz4 error: %s", err)
+		}
+		cw.Close()
+		fr.Close()
+		fw.Close()
+		res = append(res, filename)
+	}
+	return res, nil
+}
+
+func (p *CompressPostProcessor) cmpBGZF(src []string, dst string) ([]string, error) {
+	var res []string
+	for _, name := range src {
+		filename := filepath.Join(dst, filepath.Base(name))
+		fw, err := os.Create(filename)
+		if err != nil {
+			return nil, fmt.Errorf("bgzf error: %s", err)
+		}
+
+		cw, err := bgzf.NewWriterLevel(fw, p.cfg.Compression, runtime.NumCPU())
+		if err != nil {
+			return nil, fmt.Errorf("bgzf error: %s", err)
+		}
+		fr, err := os.Open(name)
+		if err != nil {
+			cw.Close()
+			fw.Close()
+			return nil, fmt.Errorf("bgzf error: %s", err)
+		}
+		if _, err = io.Copy(cw, fr); err != nil {
+			cw.Close()
+			fr.Close()
+			fw.Close()
+			return nil, fmt.Errorf("bgzf error: %s", err)
+		}
+		cw.Close()
+		fr.Close()
+		fw.Close()
+		res = append(res, filename)
+	}
+	return res, nil
+}
+
+func (p *CompressPostProcessor) cmpE2FS(src []string, dst string) ([]string, error) {
+	panic("not implemented")
+}
+
+func (p *CompressPostProcessor) cmpZIP(src []string, dst string) ([]string, error) {
+	fw, err := os.Create(dst)
+	if err != nil {
+		return nil, fmt.Errorf("zip error: %s", err)
+	}
+	defer fw.Close()
+
+	zw := zip.NewWriter(fw)
+	defer zw.Close()
+
+	for _, name := range src {
+		header, err := zw.Create(name)
+		if err != nil {
+			return nil, fmt.Errorf("zip erorr: %s", err)
+		}
+
+		fr, err := os.Open(name)
+		if err != nil {
+			return nil, fmt.Errorf("zip error: %s", err)
+		}
+
+		if _, err = io.Copy(header, fr); err != nil {
+			fr.Close()
+			return nil, fmt.Errorf("zip error: %s", err)
 		}
+		fr.Close()
 	}
+	return []string{dst}, nil
 
-	return NewArtifact(artifact.BuilderId(), self.config.OutputPath), false, nil
 }